1//===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// SI Implementation of TargetInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIInstrInfo.h"
15#include "AMDGPU.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPULaneMaskUtils.h"
18#include "GCNHazardRecognizer.h"
19#include "GCNSubtarget.h"
20#include "SIMachineFunctionInfo.h"
21#include "Utils/AMDGPUBaseInfo.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
25#include "llvm/CodeGen/LiveIntervals.h"
26#include "llvm/CodeGen/LiveVariables.h"
27#include "llvm/CodeGen/MachineCycleAnalysis.h"
28#include "llvm/CodeGen/MachineDominators.h"
29#include "llvm/CodeGen/MachineFrameInfo.h"
30#include "llvm/CodeGen/MachineScheduler.h"
31#include "llvm/CodeGen/RegisterScavenging.h"
32#include "llvm/CodeGen/ScheduleDAG.h"
33#include "llvm/IR/DiagnosticInfo.h"
34#include "llvm/IR/IntrinsicsAMDGPU.h"
35#include "llvm/MC/MCContext.h"
36#include "llvm/Support/CommandLine.h"
37#include "llvm/Target/TargetMachine.h"
38#include <tuple>
39
40using namespace llvm;
41
42#define DEBUG_TYPE "si-instr-info"
43
44#define GET_INSTRINFO_CTOR_DTOR
45#include "AMDGPUGenInstrInfo.inc"
46
47namespace llvm::AMDGPU {
48#define GET_ImageDimIntrinsicTable_IMPL
49#define GET_RsrcIntrinsics_IMPL
50#define GET_GFX1250BlockingCyclesTable_DECL
51#define GET_GFX1250BlockingCyclesTable_IMPL
52
53struct AMDGPUBlockingCyclesInfo {
54 uint16_t Opcode;
55 uint8_t GFX1250BlockingCycles;
56};
57
58#include "AMDGPUGenSearchableTables.inc"
59} // namespace llvm::AMDGPU
60
61// Must be at least 4 to be able to branch over minimum unconditional branch
62// code. This is only for making it possible to write reasonably small tests for
63// long branches.
64static cl::opt<unsigned>
65BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(Val: 16),
66 cl::desc("Restrict range of branch instructions (DEBUG)"));
67
68static cl::opt<bool> Fix16BitCopies(
69 "amdgpu-fix-16-bit-physreg-copies",
70 cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"),
71 cl::init(Val: true),
72 cl::ReallyHidden);
73
74SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
75 : AMDGPUGenInstrInfo(ST, RI, AMDGPU::ADJCALLSTACKUP,
76 AMDGPU::ADJCALLSTACKDOWN),
77 RI(ST), ST(ST) {
78 SchedModel.init(TSInfo: &ST);
79}
80
81//===----------------------------------------------------------------------===//
82// TargetInstrInfo callbacks
83//===----------------------------------------------------------------------===//
84
85static unsigned getNumOperandsNoGlue(SDNode *Node) {
86 unsigned N = Node->getNumOperands();
87 while (N && Node->getOperand(Num: N - 1).getValueType() == MVT::Glue)
88 --N;
89 return N;
90}
91
92/// Returns true if both nodes have the same value for the given
93/// operand \p Op, or if both nodes do not have this operand.
94static bool nodesHaveSameOperandValue(SDNode *N0, SDNode *N1,
95 AMDGPU::OpName OpName) {
96 unsigned Opc0 = N0->getMachineOpcode();
97 unsigned Opc1 = N1->getMachineOpcode();
98
99 int Op0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: OpName);
100 int Op1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: OpName);
101
102 if (Op0Idx == -1 && Op1Idx == -1)
103 return true;
104
105
106 if ((Op0Idx == -1 && Op1Idx != -1) ||
107 (Op1Idx == -1 && Op0Idx != -1))
108 return false;
109
110 // getNamedOperandIdx returns the index for the MachineInstr's operands,
111 // which includes the result as the first operand. We are indexing into the
112 // MachineSDNode's operands, so we need to skip the result operand to get
113 // the real index.
114 --Op0Idx;
115 --Op1Idx;
116
117 return N0->getOperand(Num: Op0Idx) == N1->getOperand(Num: Op1Idx);
118}
119
120static bool canRemat(const MachineInstr &MI) {
121
122 if (SIInstrInfo::isVOP1(MI) || SIInstrInfo::isVOP2(MI) ||
123 SIInstrInfo::isVOP3(MI) || SIInstrInfo::isSDWA(MI) ||
124 SIInstrInfo::isSALU(MI))
125 return true;
126
127 if (SIInstrInfo::isSMRD(MI)) {
128 return !MI.memoperands_empty() &&
129 llvm::all_of(Range: MI.memoperands(), P: [](const MachineMemOperand *MMO) {
130 return MMO->isLoad() && MMO->isInvariant();
131 });
132 }
133
134 return false;
135}
136
137// Split relocation flags for 64-bit global-address materialization into a
138// common base and the hi/lo relocation variants.
139static std::tuple<unsigned, unsigned, unsigned>
140splitGlobalAddressRelocFlags(const GCNSubtarget &ST,
141 const MachineOperand &SrcOp) {
142 const unsigned BaseFlags = SrcOp.getTargetFlags() & ~SIInstrInfo::MO_MASK;
143 const unsigned Reloc = SrcOp.getTargetFlags() & SIInstrInfo::MO_MASK;
144
145 // Infer the relocation type from the existing flags on the global operand.
146 // The relocation type should have been determined earlier in the pipeline.
147 unsigned LoReloc, HiReloc;
148 switch (Reloc) {
149 case SIInstrInfo::MO_REL32_LO:
150 case SIInstrInfo::MO_REL32_HI:
151 case SIInstrInfo::MO_REL64:
152 LoReloc = SIInstrInfo::MO_REL32_LO;
153 HiReloc = SIInstrInfo::MO_REL32_HI;
154 break;
155 case SIInstrInfo::MO_GOTPCREL32_LO:
156 case SIInstrInfo::MO_GOTPCREL32_HI:
157 LoReloc = SIInstrInfo::MO_GOTPCREL32_LO;
158 HiReloc = SIInstrInfo::MO_GOTPCREL32_HI;
159 break;
160 case SIInstrInfo::MO_GOTPCREL:
161 case SIInstrInfo::MO_GOTPCREL64:
162 // For 64-bit GOT-relative, use the 64-bit relocation.
163 LoReloc = SIInstrInfo::MO_GOTPCREL64;
164 HiReloc = SIInstrInfo::MO_GOTPCREL64;
165 break;
166 case SIInstrInfo::MO_ABS32_LO:
167 case SIInstrInfo::MO_ABS32_HI:
168 case SIInstrInfo::MO_ABS64:
169 LoReloc = SIInstrInfo::MO_ABS32_LO;
170 HiReloc = SIInstrInfo::MO_ABS32_HI;
171 break;
172 default:
173 llvm_unreachable("unknown relocation type for global address");
174 break;
175 }
176
177 return {BaseFlags, LoReloc, HiReloc};
178}
179
180bool SIInstrInfo::isReMaterializableImpl(
181 const MachineInstr &MI) const {
182
183 if (canRemat(MI)) {
184 // Normally VALU use of exec would block the rematerialization, but that
185 // is OK in this case to have an implicit exec read as all VALU do.
186 // We really want all of the generic logic for this except for this.
187
188 // Another potential implicit use is mode register. The core logic of
189 // the RA will not attempt rematerialization if mode is set anywhere
190 // in the function, otherwise it is safe since mode is not changed.
191
192 // There is difference to generic method which does not allow
193 // rematerialization if there are virtual register uses. We allow this,
194 // therefore this method includes SOP instructions as well.
195 if (!MI.hasImplicitDef() &&
196 MI.getNumImplicitOperands() == MI.getDesc().implicit_uses().size() &&
197 !MI.mayRaiseFPException())
198 return true;
199 }
200
201 // Everything below copied from TargetInstrInfo::isReMaterializableImpl. The
202 // only difference is that we allow operations that perform read-modify-write
203 // on sub-registers.
204
205 // Remat clients assume operand 0 is the defined register.
206 if (!MI.getNumOperands() || !MI.getOperand(i: 0).isReg())
207 return false;
208 Register DefReg = MI.getOperand(i: 0).getReg();
209
210 const MachineFunction &MF = *MI.getMF();
211
212 // A load from a fixed stack slot can be rematerialized. This may be
213 // redundant with subsequent checks, but it's target-independent,
214 // simple, and a common case.
215 int FrameIdx = 0;
216 if (isLoadFromStackSlot(MI, FrameIndex&: FrameIdx) &&
217 MF.getFrameInfo().isImmutableObjectIndex(ObjectIdx: FrameIdx))
218 return true;
219
220 // Avoid instructions obviously unsafe for remat.
221 if (MI.isNotDuplicable() || MI.mayStore() || MI.mayRaiseFPException() ||
222 MI.hasUnmodeledSideEffects())
223 return false;
224
225 // Don't remat inline asm. We have no idea how expensive it is
226 // even if it's side effect free.
227 if (MI.isInlineAsm())
228 return false;
229
230 // Avoid instructions which load from potentially varying memory.
231 if (MI.mayLoad() && !MI.isDereferenceableInvariantLoad())
232 return false;
233
234 const MachineRegisterInfo &MRI = MF.getRegInfo();
235
236 // If any of the registers accessed are non-constant, conservatively assume
237 // the instruction is not rematerializable.
238 for (const MachineOperand &MO : MI.operands()) {
239 if (!MO.isReg())
240 continue;
241 Register Reg = MO.getReg();
242 if (Reg == 0)
243 continue;
244
245 // Check for a well-behaved physical register.
246 if (Reg.isPhysical()) {
247 if (MO.isUse()) {
248 // If the physreg has no defs anywhere, it's just an ambient register
249 // and we can freely move its uses. Alternatively, if it's allocatable,
250 // it could get allocated to something with a def during allocation.
251 if (!MRI.isConstantPhysReg(PhysReg: Reg))
252 return false;
253 } else {
254 // A physreg def. We can't remat it.
255 return false;
256 }
257 continue;
258 }
259
260 // Only allow one virtual-register def. There may be multiple defs of the
261 // same virtual register, though.
262 if (MO.isDef() && Reg != DefReg)
263 return false;
264 }
265
266 return true;
267}
268
269bool SIInstrInfo::isSrc1DPPRevOpcode(const GCNSubtarget &ST, uint32_t Opcode) {
270 switch (Opcode) {
271 // v_subrev_u16 (gfx9)
272 case AMDGPU::V_SUBREV_U16_e32:
273 case AMDGPU::V_SUBREV_U16_e64:
274 // v_subrev_u32 (gfx9) / v_subrev_nc_u32 (gfx10+)
275 case AMDGPU::V_SUBREV_U32_e32:
276 case AMDGPU::V_SUBREV_U32_e64:
277 // v_subrev_co_u32
278 case AMDGPU::V_SUBREV_CO_U32_e32:
279 case AMDGPU::V_SUBREV_CO_U32_e64:
280 // v_subbrev_u32 (gfx9) / v_subrev_co_ci_u32 (gfx10+)
281 case AMDGPU::V_SUBBREV_U32_e32:
282 case AMDGPU::V_SUBBREV_U32_e64:
283 return true;
284 // REV shift opcodes worked this way before GFX11, verified on hardware
285 case AMDGPU::V_ASHRREV_I16_e32:
286 case AMDGPU::V_ASHRREV_I16_e64:
287 case AMDGPU::V_ASHRREV_I32_e32:
288 case AMDGPU::V_ASHRREV_I32_e64:
289 case AMDGPU::V_ASHRREV_I64_e64:
290 case AMDGPU::V_LSHLREV_B16_e32:
291 case AMDGPU::V_LSHLREV_B16_e64:
292 case AMDGPU::V_LSHLREV_B32_e32:
293 case AMDGPU::V_LSHLREV_B32_e64:
294 case AMDGPU::V_LSHLREV_B64_e64:
295 case AMDGPU::V_LSHRREV_B16_e32:
296 case AMDGPU::V_LSHRREV_B16_e64:
297 case AMDGPU::V_LSHRREV_B32_e32:
298 case AMDGPU::V_LSHRREV_B32_e64:
299 case AMDGPU::V_LSHRREV_B64_e64:
300 return !ST.hasGFX11Insts();
301 default:
302 return false;
303 }
304}
305
306// Returns true if the result of a VALU instruction depends on exec.
307bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {
308 assert(isVALU(MI, /*AllowLDSDMA=*/true));
309
310 // If it is convergent it depends on EXEC.
311 if (MI.isConvergent())
312 return true;
313
314 // If it defines an SGPR it depends on EXEC, unless it's dead.
315 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
316 for (const MachineOperand &Def : MI.defs()) {
317 if (Def.isDead())
318 continue;
319
320 Register Reg = Def.getReg();
321 if (Reg && RI.isSGPRReg(MRI, Reg))
322 return true;
323 }
324
325 return false;
326}
327
328bool SIInstrInfo::isIgnorableUse(const MachineInstr &MI, unsigned OpIdx) const {
329 const MachineOperand &MO = MI.getOperand(i: OpIdx);
330 // Any implicit use of exec by VALU is not a real register read.
331 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&
332 isVALU(MI, /*AllowLDSDMA=*/true) && !resultDependsOnExec(MI);
333}
334
335bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
336 MachineBasicBlock *SuccToSinkTo,
337 MachineCycleInfo *CI) const {
338 // Allow sinking if MI edits lane mask (divergent i1 in sgpr).
339 if (MI.getOpcode() == AMDGPU::SI_IF_BREAK)
340 return true;
341
342 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
343 // Check if sinking of MI would create temporal divergent use.
344 for (auto Op : MI.uses()) {
345 if (Op.isReg() && Op.getReg().isVirtual() &&
346 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Op.getReg()))) {
347 MachineInstr *SgprDef = MRI.getVRegDef(Reg: Op.getReg());
348 if (!SgprDef)
349 continue;
350
351 // SgprDef defined inside cycle
352 CycleRef FromCycle = CI->getCycle(Block: SgprDef->getParent());
353 if (!FromCycle)
354 continue;
355
356 CycleRef ToCycle = CI->getCycle(Block: SuccToSinkTo);
357 // Check if there is a FromCycle that contains SgprDef's basic block but
358 // does not contain SuccToSinkTo and also has divergent exit condition.
359 while (FromCycle && !(ToCycle && CI->contains(Outer: FromCycle, Inner: ToCycle))) {
360 SmallVector<MachineBasicBlock *, 1> ExitingBlocks;
361 CI->getExitingBlocks(C: FromCycle, TmpStorage&: ExitingBlocks);
362
363 // FromCycle has divergent exit condition.
364 for (MachineBasicBlock *ExitingBlock : ExitingBlocks) {
365 if (hasDivergentBranch(MBB: ExitingBlock))
366 return false;
367 }
368
369 FromCycle = CI->getParentCycle(C: FromCycle);
370 }
371 }
372 }
373
374 return true;
375}
376
377bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,
378 int64_t &Offset0,
379 int64_t &Offset1) const {
380 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
381 return false;
382
383 unsigned Opc0 = Load0->getMachineOpcode();
384 unsigned Opc1 = Load1->getMachineOpcode();
385
386 // Make sure both are actually loads.
387 if (!get(Opcode: Opc0).mayLoad() || !get(Opcode: Opc1).mayLoad())
388 return false;
389
390 // A mayLoad instruction without a def is not a load. Likely a prefetch.
391 if (!get(Opcode: Opc0).getNumDefs() || !get(Opcode: Opc1).getNumDefs())
392 return false;
393
394 if (isDS(Opcode: Opc0) && isDS(Opcode: Opc1)) {
395
396 // FIXME: Handle this case:
397 if (getNumOperandsNoGlue(Node: Load0) != getNumOperandsNoGlue(Node: Load1))
398 return false;
399
400 // Check base reg.
401 if (Load0->getOperand(Num: 0) != Load1->getOperand(Num: 0))
402 return false;
403
404 // Skip read2 / write2 variants for simplicity.
405 // TODO: We should report true if the used offsets are adjacent (excluded
406 // st64 versions).
407 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: AMDGPU::OpName::offset);
408 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: AMDGPU::OpName::offset);
409 if (Offset0Idx == -1 || Offset1Idx == -1)
410 return false;
411
412 // XXX - be careful of dataless loads
413 // getNamedOperandIdx returns the index for MachineInstrs. Since they
414 // include the output in the operand list, but SDNodes don't, we need to
415 // subtract the index by one.
416 Offset0Idx -= get(Opcode: Opc0).NumDefs;
417 Offset1Idx -= get(Opcode: Opc1).NumDefs;
418 Offset0 = Load0->getConstantOperandVal(Num: Offset0Idx);
419 Offset1 = Load1->getConstantOperandVal(Num: Offset1Idx);
420 return true;
421 }
422
423 if (isSMRD(Opcode: Opc0) && isSMRD(Opcode: Opc1)) {
424 // Skip time and cache invalidation instructions.
425 if (!AMDGPU::hasNamedOperand(Opcode: Opc0, NamedIdx: AMDGPU::OpName::sbase) ||
426 !AMDGPU::hasNamedOperand(Opcode: Opc1, NamedIdx: AMDGPU::OpName::sbase))
427 return false;
428
429 unsigned NumOps = getNumOperandsNoGlue(Node: Load0);
430 if (NumOps != getNumOperandsNoGlue(Node: Load1))
431 return false;
432
433 // Check base reg.
434 if (Load0->getOperand(Num: 0) != Load1->getOperand(Num: 0))
435 return false;
436
437 // Match register offsets, if both register and immediate offsets present.
438 assert(NumOps == 4 || NumOps == 5);
439 if (NumOps == 5 && Load0->getOperand(Num: 1) != Load1->getOperand(Num: 1))
440 return false;
441
442 const ConstantSDNode *Load0Offset =
443 dyn_cast<ConstantSDNode>(Val: Load0->getOperand(Num: NumOps - 3));
444 const ConstantSDNode *Load1Offset =
445 dyn_cast<ConstantSDNode>(Val: Load1->getOperand(Num: NumOps - 3));
446
447 if (!Load0Offset || !Load1Offset)
448 return false;
449
450 Offset0 = Load0Offset->getZExtValue();
451 Offset1 = Load1Offset->getZExtValue();
452 return true;
453 }
454
455 // MUBUF and MTBUF can access the same addresses.
456 if ((isMUBUF(Opcode: Opc0) || isMTBUF(Opcode: Opc0)) && (isMUBUF(Opcode: Opc1) || isMTBUF(Opcode: Opc1))) {
457
458 // MUBUF and MTBUF have vaddr at different indices.
459 if (!nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::soffset) ||
460 !nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::vaddr) ||
461 !nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::srsrc))
462 return false;
463
464 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: AMDGPU::OpName::offset);
465 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: AMDGPU::OpName::offset);
466
467 if (OffIdx0 == -1 || OffIdx1 == -1)
468 return false;
469
470 // getNamedOperandIdx returns the index for MachineInstrs. Since they
471 // include the output in the operand list, but SDNodes don't, we need to
472 // subtract the index by one.
473 OffIdx0 -= get(Opcode: Opc0).NumDefs;
474 OffIdx1 -= get(Opcode: Opc1).NumDefs;
475
476 SDValue Off0 = Load0->getOperand(Num: OffIdx0);
477 SDValue Off1 = Load1->getOperand(Num: OffIdx1);
478
479 // The offset might be a FrameIndexSDNode.
480 if (!isa<ConstantSDNode>(Val: Off0) || !isa<ConstantSDNode>(Val: Off1))
481 return false;
482
483 Offset0 = Off0->getAsZExtVal();
484 Offset1 = Off1->getAsZExtVal();
485 return true;
486 }
487
488 return false;
489}
490
491static bool isStride64(unsigned Opc) {
492 switch (Opc) {
493 case AMDGPU::DS_READ2ST64_B32:
494 case AMDGPU::DS_READ2ST64_B64:
495 case AMDGPU::DS_WRITE2ST64_B32:
496 case AMDGPU::DS_WRITE2ST64_B64:
497 return true;
498 default:
499 return false;
500 }
501}
502
503bool SIInstrInfo::getMemOperandsWithOffsetWidth(
504 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
505 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
506 const TargetRegisterInfo *TRI) const {
507 if (!LdSt.mayLoadOrStore())
508 return false;
509
510 unsigned Opc = LdSt.getOpcode();
511 OffsetIsScalable = false;
512 const MachineOperand *BaseOp, *OffsetOp;
513 int DataOpIdx;
514
515 if (isDS(MI: LdSt)) {
516 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::addr);
517 OffsetOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
518 if (OffsetOp) {
519 // Normal, single offset LDS instruction.
520 if (!BaseOp) {
521 // DS_CONSUME/DS_APPEND use M0 for the base address.
522 // TODO: find the implicit use operand for M0 and use that as BaseOp?
523 return false;
524 }
525 BaseOps.push_back(Elt: BaseOp);
526 Offset = OffsetOp->getImm();
527 // Get appropriate operand, and compute width accordingly.
528 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
529 if (DataOpIdx == -1)
530 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
531 if (Opc == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
532 Width = LocationSize::precise(Value: 64);
533 else
534 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
535 } else {
536 // The 2 offset instructions use offset0 and offset1 instead. We can treat
537 // these as a load with a single offset if the 2 offsets are consecutive.
538 // We will use this for some partially aligned loads.
539 const MachineOperand *Offset0Op =
540 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset0);
541 const MachineOperand *Offset1Op =
542 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset1);
543
544 unsigned Offset0 = Offset0Op->getImm() & 0xff;
545 unsigned Offset1 = Offset1Op->getImm() & 0xff;
546 if (Offset0 + 1 != Offset1)
547 return false;
548
549 // Each of these offsets is in element sized units, so we need to convert
550 // to bytes of the individual reads.
551
552 unsigned EltSize;
553 if (LdSt.mayLoad())
554 EltSize = TRI->getRegSizeInBits(RC: *getOpRegClass(MI: LdSt, OpNo: 0)) / 16;
555 else {
556 assert(LdSt.mayStore());
557 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
558 EltSize = TRI->getRegSizeInBits(RC: *getOpRegClass(MI: LdSt, OpNo: Data0Idx)) / 8;
559 }
560
561 if (isStride64(Opc))
562 EltSize *= 64;
563
564 BaseOps.push_back(Elt: BaseOp);
565 Offset = EltSize * Offset0;
566 // Get appropriate operand(s), and compute width accordingly.
567 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
568 if (DataOpIdx == -1) {
569 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
570 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
571 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data1);
572 Width = LocationSize::precise(
573 Value: Width.getValue() + TypeSize::getFixed(ExactSize: getOpSize(MI: LdSt, OpNo: DataOpIdx)));
574 } else {
575 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
576 }
577 }
578 return true;
579 }
580
581 if (isMUBUF(MI: LdSt) || isMTBUF(MI: LdSt)) {
582 const MachineOperand *RSrc = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::srsrc);
583 if (!RSrc) // e.g. BUFFER_WBINVL1_VOL
584 return false;
585 BaseOps.push_back(Elt: RSrc);
586 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr);
587 if (BaseOp && !BaseOp->isFI())
588 BaseOps.push_back(Elt: BaseOp);
589 const MachineOperand *OffsetImm =
590 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
591 Offset = OffsetImm->getImm();
592 const MachineOperand *SOffset =
593 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::soffset);
594 if (SOffset) {
595 if (SOffset->isReg())
596 BaseOps.push_back(Elt: SOffset);
597 else
598 Offset += SOffset->getImm();
599 }
600 // Get appropriate operand, and compute width accordingly.
601 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
602 if (DataOpIdx == -1)
603 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
604 if (DataOpIdx == -1) // LDS DMA
605 return false;
606 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
607 return true;
608 }
609
610 if (isImage(MI: LdSt)) {
611 auto RsrcOpName =
612 isMIMG(MI: LdSt) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
613 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: RsrcOpName);
614 BaseOps.push_back(Elt: &LdSt.getOperand(i: SRsrcIdx));
615 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0);
616 if (VAddr0Idx >= 0) {
617 // GFX10 possible NSA encoding.
618 for (int I = VAddr0Idx; I < SRsrcIdx; ++I)
619 BaseOps.push_back(Elt: &LdSt.getOperand(i: I));
620 } else {
621 BaseOps.push_back(Elt: getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr));
622 }
623 Offset = 0;
624 // Get appropriate operand, and compute width accordingly.
625 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
626 if (DataOpIdx == -1)
627 return false; // no return sampler
628 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
629 return true;
630 }
631
632 if (isSMRD(MI: LdSt)) {
633 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::sbase);
634 if (!BaseOp) // e.g. S_MEMTIME
635 return false;
636 BaseOps.push_back(Elt: BaseOp);
637 OffsetOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
638 Offset = OffsetOp ? OffsetOp->getImm() : 0;
639 // Get appropriate operand, and compute width accordingly.
640 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::sdst);
641 if (DataOpIdx == -1)
642 return false;
643 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
644 return true;
645 }
646
647 if (isFLAT(MI: LdSt)) {
648 // Instructions have either vaddr or saddr or both or none.
649 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr);
650 if (BaseOp)
651 BaseOps.push_back(Elt: BaseOp);
652 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::saddr);
653 if (BaseOp)
654 BaseOps.push_back(Elt: BaseOp);
655 Offset = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset)->getImm();
656 // Get appropriate operand, and compute width accordingly.
657 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
658 if (DataOpIdx == -1)
659 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
660 if (DataOpIdx == -1) // LDS DMA
661 return false;
662 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
663 return true;
664 }
665
666 return false;
667}
668
669static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
670 ArrayRef<const MachineOperand *> BaseOps1,
671 const MachineInstr &MI2,
672 ArrayRef<const MachineOperand *> BaseOps2) {
673 // Only examine the first "base" operand of each instruction, on the
674 // assumption that it represents the real base address of the memory access.
675 // Other operands are typically offsets or indices from this base address.
676 if (BaseOps1.front()->isIdenticalTo(Other: *BaseOps2.front()))
677 return true;
678
679 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
680 return false;
681
682 auto *MO1 = *MI1.memoperands_begin();
683 auto *MO2 = *MI2.memoperands_begin();
684 if (MO1->getAddrSpace() != MO2->getAddrSpace())
685 return false;
686
687 const auto *Base1 = MO1->getValue();
688 const auto *Base2 = MO2->getValue();
689 if (!Base1 || !Base2)
690 return false;
691 Base1 = getUnderlyingObject(V: Base1);
692 Base2 = getUnderlyingObject(V: Base2);
693
694 if (isa<UndefValue>(Val: Base1) || isa<UndefValue>(Val: Base2))
695 return false;
696
697 return Base1 == Base2;
698}
699
700bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
701 int64_t Offset1, bool OffsetIsScalable1,
702 ArrayRef<const MachineOperand *> BaseOps2,
703 int64_t Offset2, bool OffsetIsScalable2,
704 unsigned ClusterSize,
705 unsigned NumBytes) const {
706 // If the mem ops (to be clustered) do not have the same base ptr, then they
707 // should not be clustered
708 unsigned MaxMemoryClusterDWords = DefaultMemoryClusterDWordsLimit;
709 if (!BaseOps1.empty() && !BaseOps2.empty()) {
710 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
711 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
712 if (!memOpsHaveSameBasePtr(MI1: FirstLdSt, BaseOps1, MI2: SecondLdSt, BaseOps2))
713 return false;
714
715 const SIMachineFunctionInfo *MFI =
716 FirstLdSt.getMF()->getInfo<SIMachineFunctionInfo>();
717 MaxMemoryClusterDWords = MFI->getMaxMemoryClusterDWords();
718 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
719 // If only one base op is empty, they do not have the same base ptr
720 return false;
721 }
722
723 // In order to avoid register pressure, on an average, the number of DWORDS
724 // loaded together by all clustered mem ops should not exceed
725 // MaxMemoryClusterDWords. This is an empirical value based on certain
726 // observations and performance related experiments.
727 // The good thing about this heuristic is - it avoids clustering of too many
728 // sub-word loads, and also avoids clustering of wide loads. Below is the
729 // brief summary of how the heuristic behaves for various `LoadSize` when
730 // MaxMemoryClusterDWords is 8.
731 //
732 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops
733 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops
734 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops
735 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops
736 // (5) LoadSize >= 17: do not cluster
737 const unsigned LoadSize = NumBytes / ClusterSize;
738 const unsigned NumDWords = ((LoadSize + 3) / 4) * ClusterSize;
739 return NumDWords <= MaxMemoryClusterDWords;
740}
741
742// FIXME: This behaves strangely. If, for example, you have 32 load + stores,
743// the first 16 loads will be interleaved with the stores, and the next 16 will
744// be clustered as expected. It should really split into 2 16 store batches.
745//
746// Loads are clustered until this returns false, rather than trying to schedule
747// groups of stores. This also means we have to deal with saying different
748// address space loads should be clustered, and ones which might cause bank
749// conflicts.
750//
751// This might be deprecated so it might not be worth that much effort to fix.
752bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1,
753 int64_t Offset0, int64_t Offset1,
754 unsigned NumLoads) const {
755 assert(Offset1 > Offset0 &&
756 "Second offset should be larger than first offset!");
757 // If we have less than 16 loads in a row, and the offsets are within 64
758 // bytes, then schedule together.
759
760 // A cacheline is 64 bytes (for global memory).
761 return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
762}
763
764static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB,
765 MachineBasicBlock::iterator MI,
766 const DebugLoc &DL, MCRegister DestReg,
767 MCRegister SrcReg, bool KillSrc,
768 const char *Msg = "illegal VGPR to SGPR copy") {
769 MachineFunction *MF = MBB.getParent();
770
771 LLVMContext &C = MF->getFunction().getContext();
772 C.diagnose(DI: DiagnosticInfoUnsupported(MF->getFunction(), Msg, DL, DS_Error));
773
774 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_ILLEGAL_COPY), DestReg)
775 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
776}
777
778/// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not
779/// possible to have a direct copy in these cases on GFX908, so an intermediate
780/// VGPR copy is required.
781static void indirectCopyToAGPR(const SIInstrInfo &TII, MachineBasicBlock &MBB,
782 MachineBasicBlock::iterator MI,
783 const DebugLoc &DL, MCRegister DestReg,
784 MCRegister SrcReg, bool KillSrc,
785 RegScavenger &RS, bool RegsOverlap,
786 Register ImpUseSuperReg = Register()) {
787 assert((TII.getSubtarget().hasMAIInsts() &&
788 !TII.getSubtarget().hasGFX90AInsts()) &&
789 "Expected GFX908 subtarget.");
790
791 assert((AMDGPU::SReg_32RegClass.contains(SrcReg) ||
792 AMDGPU::AGPR_32RegClass.contains(SrcReg)) &&
793 "Source register of the copy should be either an SGPR or an AGPR.");
794
795 assert(AMDGPU::AGPR_32RegClass.contains(DestReg) &&
796 "Destination register of the copy should be an AGPR.");
797
798 const SIRegisterInfo &RI = TII.getRegisterInfo();
799
800 // First try to find defining accvgpr_write to avoid temporary registers.
801 // In the case of copies of overlapping AGPRs, we conservatively do not
802 // reuse previous accvgpr_writes. Otherwise, we may incorrectly pick up
803 // an accvgpr_write used for this same copy due to implicit-defs
804 if (!RegsOverlap) {
805 for (auto Def = MI, E = MBB.begin(); Def != E; ) {
806 --Def;
807
808 if (!Def->modifiesRegister(Reg: SrcReg, TRI: &RI))
809 continue;
810
811 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
812 Def->getOperand(i: 0).getReg() != SrcReg)
813 break;
814
815 MachineOperand &DefOp = Def->getOperand(i: 1);
816 assert(DefOp.isReg() || DefOp.isImm());
817
818 if (DefOp.isReg()) {
819 bool SafeToPropagate = true;
820 // Check that register source operand is not clobbered before MI.
821 // Immediate operands are always safe to propagate.
822 for (auto I = Def; I != MI && SafeToPropagate; ++I)
823 if (I->modifiesRegister(Reg: DefOp.getReg(), TRI: &RI))
824 SafeToPropagate = false;
825
826 if (!SafeToPropagate)
827 break;
828
829 for (auto I = Def; I != MI; ++I)
830 I->clearRegisterKills(Reg: DefOp.getReg(), RegInfo: &RI);
831 }
832
833 MachineInstrBuilder Builder =
834 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64),
835 DestReg)
836 .add(MO: DefOp);
837
838 if (ImpUseSuperReg) {
839 Builder.addReg(RegNo: ImpUseSuperReg,
840 Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
841 }
842
843 return;
844 }
845 }
846
847 RS.enterBasicBlockEnd(MBB);
848 RS.backward(I: std::next(x: MI));
849
850 // Ideally we want to have three registers for a long reg_sequence copy
851 // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
852 unsigned MaxVGPRs = RI.getRegPressureLimit(RC: &AMDGPU::VGPR_32RegClass,
853 MF&: *MBB.getParent());
854
855 // Registers in the sequence are allocated contiguously so we can just
856 // use register number to pick one of three round-robin temps.
857 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3;
858 Register Tmp =
859 MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getVGPRForAGPRCopy();
860 assert(MBB.getParent()->getRegInfo().isReserved(Tmp) &&
861 "VGPR used for an intermediate copy should have been reserved.");
862
863 // Only loop through if there are any free registers left. We don't want to
864 // spill.
865 while (RegNo--) {
866 Register Tmp2 = RS.scavengeRegisterBackwards(RC: AMDGPU::VGPR_32RegClass, To: MI,
867 /* RestoreAfter */ false, SPAdj: 0,
868 /* AllowSpill */ false);
869 if (!Tmp2 || RI.getHWRegIndex(Reg: Tmp2) >= MaxVGPRs)
870 break;
871 Tmp = Tmp2;
872 RS.setRegUsed(Reg: Tmp);
873 }
874
875 // Insert copy to temporary VGPR.
876 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;
877 if (AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg)) {
878 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64;
879 } else {
880 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
881 }
882
883 MachineInstrBuilder UseBuilder = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: TmpCopyOp), DestReg: Tmp)
884 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
885 if (ImpUseSuperReg) {
886 UseBuilder.addReg(RegNo: ImpUseSuperReg,
887 Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
888 }
889
890 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
891 .addReg(RegNo: Tmp, Flags: RegState::Kill);
892}
893
894static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB,
895 MachineBasicBlock::iterator MI, const DebugLoc &DL,
896 MCRegister DestReg, MCRegister SrcReg, bool KillSrc,
897 const TargetRegisterClass *RC, bool Forward) {
898 const SIRegisterInfo &RI = TII.getRegisterInfo();
899 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, EltSize: 4);
900 MachineBasicBlock::iterator I = MI;
901 MachineInstr *FirstMI = nullptr, *LastMI = nullptr;
902
903 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) {
904 int16_t SubIdx = BaseIndices[Idx];
905 Register DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
906 Register SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
907 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
908 unsigned Opcode = AMDGPU::S_MOV_B32;
909
910 // Is SGPR aligned? If so try to combine with next.
911 bool AlignedDest = ((DestSubReg - AMDGPU::SGPR0) % 2) == 0;
912 bool AlignedSrc = ((SrcSubReg - AMDGPU::SGPR0) % 2) == 0;
913 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) {
914 // Can use SGPR64 copy
915 unsigned Channel = RI.getChannelFromSubReg(SubReg: SubIdx);
916 SubIdx = RI.getSubRegFromChannel(Channel, NumRegs: 2);
917 DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
918 SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
919 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
920 Opcode = AMDGPU::S_MOV_B64;
921 Idx++;
922 }
923
924 LastMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode), DestReg: DestSubReg)
925 .addReg(RegNo: SrcSubReg)
926 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
927
928 if (!FirstMI)
929 FirstMI = LastMI;
930
931 if (!Forward)
932 I--;
933 }
934
935 assert(FirstMI && LastMI);
936 if (!Forward)
937 std::swap(a&: FirstMI, b&: LastMI);
938
939 if (KillSrc)
940 LastMI->addRegisterKilled(IncomingReg: SrcReg, RegInfo: &RI);
941}
942
943void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
944 MachineBasicBlock::iterator MI,
945 const DebugLoc &DL, Register DestReg,
946 Register SrcReg, bool KillSrc, bool RenamableDest,
947 bool RenamableSrc) const {
948 const TargetRegisterClass *RC = RI.getPhysRegBaseClass(Reg: DestReg);
949 unsigned Size = RI.getRegSizeInBits(RC: *RC);
950 const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(Reg: SrcReg);
951 unsigned SrcSize = RI.getRegSizeInBits(RC: *SrcRC);
952
953 // The rest of copyPhysReg assumes Src and Dst size are the same size.
954 // TODO-GFX11_16BIT If all true 16 bit instruction patterns are completed can
955 // we remove Fix16BitCopies and this code block?
956 if (Fix16BitCopies) {
957 if (((Size == 16) != (SrcSize == 16))) {
958 // Non-VGPR Src and Dst will later be expanded back to 32 bits.
959 assert(ST.useRealTrue16Insts());
960 Register &RegToFix = (Size == 32) ? DestReg : SrcReg;
961 MCRegister SubReg = RI.getSubReg(Reg: RegToFix, Idx: AMDGPU::lo16);
962 RegToFix = SubReg;
963
964 if (DestReg == SrcReg) {
965 // Identity copy. Insert empty bundle since ExpandPostRA expects an
966 // instruction here.
967 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::BUNDLE));
968 return;
969 }
970 RC = RI.getPhysRegBaseClass(Reg: DestReg);
971 Size = RI.getRegSizeInBits(RC: *RC);
972 SrcRC = RI.getPhysRegBaseClass(Reg: SrcReg);
973 SrcSize = RI.getRegSizeInBits(RC: *SrcRC);
974 }
975 }
976
977 if (RC == &AMDGPU::VGPR_32RegClass) {
978 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
979 AMDGPU::SReg_32RegClass.contains(SrcReg) ||
980 AMDGPU::AGPR_32RegClass.contains(SrcReg));
981 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg) ?
982 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;
983 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
984 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
985 return;
986 }
987
988 if (RC == &AMDGPU::SReg_32_XM0RegClass ||
989 RC == &AMDGPU::SReg_32RegClass) {
990 if (SrcReg == AMDGPU::SCC) {
991 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B32), DestReg)
992 .addImm(Val: 1)
993 .addImm(Val: 0);
994 return;
995 }
996
997 if (!AMDGPU::SReg_32RegClass.contains(Reg: SrcReg)) {
998 if (DestReg == AMDGPU::VCC_LO) {
999 // FIXME: Hack until VReg_1 removed.
1000 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
1001 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CMP_NE_U32_e32))
1002 .addImm(Val: 0)
1003 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1004 return;
1005 }
1006
1007 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1008 return;
1009 }
1010
1011 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg)
1012 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1013 return;
1014 }
1015
1016 if (RC == &AMDGPU::SReg_64RegClass) {
1017 if (SrcReg == AMDGPU::SCC) {
1018 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B64), DestReg)
1019 .addImm(Val: 1)
1020 .addImm(Val: 0);
1021 return;
1022 }
1023
1024 if (!AMDGPU::SReg_64_EncodableRegClass.contains(Reg: SrcReg)) {
1025 if (DestReg == AMDGPU::VCC) {
1026 // FIXME: Hack until VReg_1 removed.
1027 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
1028 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CMP_NE_U32_e32))
1029 .addImm(Val: 0)
1030 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1031 return;
1032 }
1033
1034 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1035 return;
1036 }
1037
1038 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B64), DestReg)
1039 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1040 return;
1041 }
1042
1043 if (DestReg == AMDGPU::SCC) {
1044 // Copying 64-bit or 32-bit sources to SCC barely makes sense,
1045 // but SelectionDAG emits such copies for i1 sources.
1046 if (AMDGPU::SReg_64RegClass.contains(Reg: SrcReg)) {
1047 // This copy can only be produced by patterns
1048 // with explicit SCC, which are known to be enabled
1049 // only for subtargets with S_CMP_LG_U64 present.
1050 assert(ST.hasScalarCompareEq64());
1051 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U64))
1052 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
1053 .addImm(Val: 0);
1054 } else {
1055 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
1056 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U32))
1057 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
1058 .addImm(Val: 0);
1059 }
1060
1061 return;
1062 }
1063
1064 if (RC == &AMDGPU::AGPR_32RegClass) {
1065 if (AMDGPU::VGPR_32RegClass.contains(Reg: SrcReg) ||
1066 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(Reg: SrcReg))) {
1067 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
1068 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1069 return;
1070 }
1071
1072 if (AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg) && ST.hasGFX90AInsts()) {
1073 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_MOV_B32), DestReg)
1074 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1075 return;
1076 }
1077
1078 // FIXME: Pass should maintain scavenger to avoid scan through the block on
1079 // every AGPR spill.
1080 RegScavenger RS;
1081 const bool Overlap = RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1082 indirectCopyToAGPR(TII: *this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS, RegsOverlap: Overlap);
1083 return;
1084 }
1085
1086 if (Size == 16) {
1087 assert(AMDGPU::VGPR_16RegClass.contains(SrcReg) ||
1088 AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
1089 AMDGPU::AGPR_LO16RegClass.contains(SrcReg));
1090
1091 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(Reg: DestReg);
1092 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(Reg: SrcReg);
1093 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(Reg: DestReg);
1094 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(Reg: SrcReg);
1095 bool DstLow = !AMDGPU::isHi16Reg(Reg: DestReg, MRI: RI);
1096 bool SrcLow = !AMDGPU::isHi16Reg(Reg: SrcReg, MRI: RI);
1097 MCRegister NewDestReg = RI.get32BitRegister(Reg: DestReg);
1098 MCRegister NewSrcReg = RI.get32BitRegister(Reg: SrcReg);
1099
1100 if (IsSGPRDst) {
1101 if (!IsSGPRSrc) {
1102 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1103 return;
1104 }
1105
1106 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: NewDestReg)
1107 .addReg(RegNo: NewSrcReg, Flags: getKillRegState(B: KillSrc));
1108 return;
1109 }
1110
1111 if (IsAGPRDst || IsAGPRSrc) {
1112 if (!DstLow || !SrcLow) {
1113 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
1114 Msg: "Cannot use hi16 subreg with an AGPR!");
1115 }
1116
1117 copyPhysReg(MBB, MI, DL, DestReg: NewDestReg, SrcReg: NewSrcReg, KillSrc);
1118 return;
1119 }
1120
1121 if (ST.useRealTrue16Insts()) {
1122 if (IsSGPRSrc) {
1123 assert(SrcLow);
1124 SrcReg = NewSrcReg;
1125 }
1126 // Use the smaller instruction encoding if possible.
1127 if (AMDGPU::VGPR_16_Lo128RegClass.contains(Reg: DestReg) &&
1128 (IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.contains(Reg: SrcReg))) {
1129 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B16_t16_e32), DestReg)
1130 .addReg(RegNo: SrcReg);
1131 } else {
1132 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B16_t16_e64), DestReg)
1133 .addImm(Val: 0) // src0_modifiers
1134 .addReg(RegNo: SrcReg)
1135 .addImm(Val: 0); // op_sel
1136 }
1137 return;
1138 }
1139
1140 if (IsSGPRSrc && !ST.hasSDWAScalar()) {
1141 if (!DstLow || !SrcLow) {
1142 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
1143 Msg: "Cannot use hi16 subreg on VI!");
1144 }
1145
1146 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: NewDestReg)
1147 .addReg(RegNo: NewSrcReg, Flags: getKillRegState(B: KillSrc));
1148 return;
1149 }
1150
1151 auto MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_sdwa), DestReg: NewDestReg)
1152 .addImm(Val: 0) // src0_modifiers
1153 .addReg(RegNo: NewSrcReg)
1154 .addImm(Val: 0) // clamp
1155 .addImm(Val: DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1156 : AMDGPU::SDWA::SdwaSel::WORD_1)
1157 .addImm(Val: AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE)
1158 .addImm(Val: SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1159 : AMDGPU::SDWA::SdwaSel::WORD_1)
1160 .addReg(RegNo: NewDestReg, Flags: RegState::Implicit | RegState::Undef);
1161 // First implicit operand is $exec.
1162 MIB->tieOperands(DefIdx: 0, UseIdx: MIB->getNumOperands() - 1);
1163 return;
1164 }
1165
1166 // Returns true if Dst and Src are in Opc's HwMode-resolved destination and
1167 // source operand classes.
1168 auto CanCopyWith = [&](unsigned Opc, MCRegister Dst, MCRegister Src,
1169 unsigned SrcOp = 1) {
1170 const MCInstrDesc &Desc = get(Opcode: Opc);
1171 const TargetRegisterClass *DstOpRC = getRegClass(MCID: Desc, OpNum: 0);
1172 const TargetRegisterClass *SrcOpRC = getRegClass(MCID: Desc, OpNum: SrcOp);
1173 return DstOpRC && SrcOpRC && DstOpRC->contains(Reg: Dst) &&
1174 SrcOpRC->contains(Reg: Src);
1175 };
1176
1177 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(RC: SrcRC))) {
1178 if (ST.hasVMovB64Inst() &&
1179 CanCopyWith(AMDGPU::V_MOV_B64_e32, DestReg, SrcReg)) {
1180 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B64_e32), DestReg)
1181 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1182 return;
1183 }
1184 if (ST.hasPkMovB32() &&
1185 CanCopyWith(AMDGPU::V_PK_MOV_B32, DestReg, SrcReg, /*SrcOp=*/2)) {
1186 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg)
1187 .addImm(Val: SISrcMods::OP_SEL_1)
1188 .addReg(RegNo: SrcReg)
1189 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
1190 .addReg(RegNo: SrcReg)
1191 .addImm(Val: 0) // op_sel_lo
1192 .addImm(Val: 0) // op_sel_hi
1193 .addImm(Val: 0) // neg_lo
1194 .addImm(Val: 0) // neg_hi
1195 .addImm(Val: 0) // clamp
1196 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
1197 return;
1198 }
1199 }
1200
1201 const bool Forward = RI.getHWRegIndex(Reg: DestReg) <= RI.getHWRegIndex(Reg: SrcReg);
1202 if (RI.isSGPRClass(RC)) {
1203 if (!RI.isSGPRClass(RC: SrcRC)) {
1204 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1205 return;
1206 }
1207 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1208 expandSGPRCopy(TII: *this, MBB, MI, DL, DestReg, SrcReg, KillSrc: CanKillSuperReg, RC,
1209 Forward);
1210 return;
1211 }
1212
1213 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1214 unsigned WideOpcode = AMDGPU::INSTRUCTION_LIST_END;
1215 if (RI.isAGPRClass(RC)) {
1216 if (ST.hasGFX90AInsts() && RI.isAGPRClass(RC: SrcRC))
1217 Opcode = AMDGPU::V_ACCVGPR_MOV_B32;
1218 else if (RI.hasVGPRs(RC: SrcRC) ||
1219 (ST.hasGFX90AInsts() && RI.isSGPRClass(RC: SrcRC)))
1220 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
1221 else
1222 Opcode = AMDGPU::INSTRUCTION_LIST_END;
1223 } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(RC: SrcRC)) {
1224 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;
1225 } else if (RI.isVGPRClass(RC)) {
1226 if (ST.hasVMovB64Inst())
1227 WideOpcode = AMDGPU::V_MOV_B64_e32;
1228 else if (ST.hasPkMovB32())
1229 WideOpcode = AMDGPU::V_PK_MOV_B32;
1230 }
1231
1232 const TargetRegisterClass *WideDstRC{}, *WideSrcRC{};
1233 if (WideOpcode != AMDGPU::INSTRUCTION_LIST_END) {
1234 const MCInstrDesc &Desc = get(Opcode: WideOpcode);
1235 unsigned SrcOp = WideOpcode == AMDGPU::V_PK_MOV_B32 ? 2 : 1;
1236 WideDstRC = getRegClass(MCID: Desc, OpNum: 0);
1237 WideSrcRC = getRegClass(MCID: Desc, OpNum: SrcOp);
1238 }
1239
1240 // If there is an overlap, we can't kill the super-register on the last
1241 // instruction, since it will also kill the components made live by this def.
1242 const bool Overlap = RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1243 const bool CanKillSuperReg = KillSrc && !Overlap;
1244
1245 // For the cases where we need an intermediate instruction/temporary register
1246 // (destination is an AGPR), we need a scavenger.
1247 //
1248 // FIXME: The pass should maintain this for us so we don't have to re-scan the
1249 // whole block for every handled copy.
1250 std::unique_ptr<RegScavenger> RS;
1251 if (Opcode == AMDGPU::INSTRUCTION_LIST_END)
1252 RS = std::make_unique<RegScavenger>();
1253
1254 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize: 4);
1255
1256 for (unsigned Idx{}; Idx < SubIndices.size();) {
1257 unsigned NumRegs = 1;
1258 unsigned ThisOpcode = Opcode;
1259 unsigned SubIdx =
1260 Forward ? SubIndices[Idx] : SubIndices[SubIndices.size() - Idx - 1];
1261
1262 if (WideDstRC && WideSrcRC && Idx + 1 < SubIndices.size()) {
1263 unsigned Channel = RI.getChannelFromSubReg(SubReg: SubIdx);
1264 if (!Forward)
1265 --Channel;
1266
1267 unsigned WideSubIdx = RI.getSubRegFromChannel(Channel, NumRegs: 2);
1268 Register WideDst = RI.getSubReg(Reg: DestReg, Idx: WideSubIdx);
1269 Register WideSrc = RI.getSubReg(Reg: SrcReg, Idx: WideSubIdx);
1270
1271 if (WideDst && WideSrc && WideDstRC->contains(Reg: WideDst) &&
1272 WideSrcRC->contains(Reg: WideSrc)) {
1273 SubIdx = WideSubIdx;
1274 NumRegs = 2;
1275 ThisOpcode = WideOpcode;
1276 }
1277 }
1278
1279 Register DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
1280 Register SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
1281 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
1282
1283 Idx += NumRegs;
1284 bool UseKill = CanKillSuperReg && Idx == SubIndices.size();
1285
1286 if (ThisOpcode == AMDGPU::INSTRUCTION_LIST_END) {
1287 Register ImpUseSuper = SrcReg;
1288 indirectCopyToAGPR(TII: *this, MBB, MI, DL, DestReg: DestSubReg, SrcReg: SrcSubReg, KillSrc: UseKill,
1289 RS&: *RS, RegsOverlap: Overlap, ImpUseSuperReg: ImpUseSuper);
1290 } else if (ThisOpcode == AMDGPU::V_PK_MOV_B32) {
1291 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg: DestSubReg)
1292 .addImm(Val: SISrcMods::OP_SEL_1)
1293 .addReg(RegNo: SrcSubReg)
1294 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
1295 .addReg(RegNo: SrcSubReg)
1296 .addImm(Val: 0) // op_sel_lo
1297 .addImm(Val: 0) // op_sel_hi
1298 .addImm(Val: 0) // neg_lo
1299 .addImm(Val: 0) // neg_hi
1300 .addImm(Val: 0) // clamp
1301 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: UseKill) | RegState::Implicit);
1302 } else {
1303 MachineInstrBuilder Builder =
1304 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: ThisOpcode), DestReg: DestSubReg).addReg(RegNo: SrcSubReg);
1305
1306 Builder.addReg(RegNo: SrcReg, Flags: getKillRegState(B: UseKill) | RegState::Implicit);
1307 }
1308 }
1309}
1310
1311int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
1312 int32_t NewOpc;
1313
1314 // Try to map original to commuted opcode
1315 NewOpc = AMDGPU::getCommuteRev(Opcode);
1316 if (NewOpc != -1)
1317 // Check if the commuted (REV) opcode exists on the target.
1318 return pseudoToMCOpcode(Opcode: NewOpc) != -1 ? NewOpc : -1;
1319
1320 // Try to map commuted to original opcode
1321 NewOpc = AMDGPU::getCommuteOrig(Opcode);
1322 if (NewOpc != -1)
1323 // Check if the original (non-REV) opcode exists on the target.
1324 return pseudoToMCOpcode(Opcode: NewOpc) != -1 ? NewOpc : -1;
1325
1326 return Opcode;
1327}
1328
1329bool SIInstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
1330 const Register Reg,
1331 int64_t &ImmVal) const {
1332 switch (MI.getOpcode()) {
1333 case AMDGPU::V_MOV_B32_e32:
1334 case AMDGPU::S_MOV_B32:
1335 case AMDGPU::S_MOVK_I32:
1336 case AMDGPU::S_MOV_B64:
1337 case AMDGPU::V_MOV_B64_e32:
1338 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
1339 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
1340 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
1341 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
1342 case AMDGPU::V_MOV_B64_PSEUDO:
1343 case AMDGPU::V_MOV_B16_t16_e32: {
1344 const MachineOperand &Src0 = MI.getOperand(i: 1);
1345 if (Src0.isImm()) {
1346 ImmVal = Src0.getImm();
1347 return MI.getOperand(i: 0).getReg() == Reg;
1348 }
1349
1350 return false;
1351 }
1352 case AMDGPU::V_MOV_B16_t16_e64: {
1353 const MachineOperand &Src0 = MI.getOperand(i: 2);
1354 if (Src0.isImm() && !MI.getOperand(i: 1).getImm()) {
1355 ImmVal = Src0.getImm();
1356 return MI.getOperand(i: 0).getReg() == Reg;
1357 }
1358
1359 return false;
1360 }
1361 case AMDGPU::S_BREV_B32:
1362 case AMDGPU::V_BFREV_B32_e32:
1363 case AMDGPU::V_BFREV_B32_e64: {
1364 const MachineOperand &Src0 = MI.getOperand(i: 1);
1365 if (Src0.isImm()) {
1366 ImmVal = static_cast<int64_t>(reverseBits<int32_t>(Val: Src0.getImm()));
1367 return MI.getOperand(i: 0).getReg() == Reg;
1368 }
1369
1370 return false;
1371 }
1372 case AMDGPU::S_NOT_B32:
1373 case AMDGPU::V_NOT_B32_e32:
1374 case AMDGPU::V_NOT_B32_e64: {
1375 const MachineOperand &Src0 = MI.getOperand(i: 1);
1376 if (Src0.isImm()) {
1377 ImmVal = static_cast<int64_t>(~static_cast<int32_t>(Src0.getImm()));
1378 return MI.getOperand(i: 0).getReg() == Reg;
1379 }
1380
1381 return false;
1382 }
1383 default:
1384 return false;
1385 }
1386}
1387
1388std::optional<int64_t>
1389SIInstrInfo::getImmOrMaterializedImm(const MachineRegisterInfo &MRI,
1390 const MachineOperand &Op,
1391 MachineInstr **DefMI) const {
1392 if (DefMI)
1393 *DefMI = nullptr;
1394
1395 if (Op.isImm())
1396 return Op.getImm();
1397
1398 if (!Op.isReg() || !Op.getReg().isVirtual())
1399 return std::nullopt;
1400 MachineInstr *Def = MRI.getUniqueVRegDef(Reg: Op.getReg());
1401 if (Def && Def->isMoveImmediate()) {
1402 const MachineOperand &ImmSrc = Def->getOperand(i: 1);
1403 if (ImmSrc.isImm()) {
1404 if (DefMI)
1405 *DefMI = Def;
1406 return extractSubregFromImm(ImmVal: ImmSrc.getImm(), SubRegIndex: Op.getSubReg());
1407 }
1408 }
1409
1410 return std::nullopt;
1411}
1412
1413std::optional<int64_t>
1414SIInstrInfo::getImmOrMaterializedImm(const MachineRegisterInfo &MRI,
1415 Register Reg, MachineInstr **DefMI) const {
1416 MachineOperand Op = MachineOperand::CreateReg(Reg, isDef: false);
1417 return getImmOrMaterializedImm(MRI, Op, DefMI);
1418}
1419
1420unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
1421
1422 if (RI.isAGPRClass(RC: DstRC))
1423 return AMDGPU::COPY;
1424 if (RI.getRegSizeInBits(RC: *DstRC) == 16) {
1425 // Assume hi bits are unneeded. Only _e64 true16 instructions are legal
1426 // before RA.
1427 return RI.isSGPRClass(RC: DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64;
1428 }
1429 if (RI.getRegSizeInBits(RC: *DstRC) == 32)
1430 return RI.isSGPRClass(RC: DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1431 if (RI.getRegSizeInBits(RC: *DstRC) == 64 && RI.isSGPRClass(RC: DstRC))
1432 return AMDGPU::S_MOV_B64;
1433 if (RI.getRegSizeInBits(RC: *DstRC) == 64 && !RI.isSGPRClass(RC: DstRC))
1434 return AMDGPU::V_MOV_B64_PSEUDO;
1435 return AMDGPU::COPY;
1436}
1437
1438const MCInstrDesc &
1439SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize,
1440 bool IsIndirectSrc) const {
1441 if (IsIndirectSrc) {
1442 if (VecSize <= 32) // 4 bytes
1443 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);
1444 if (VecSize <= 64) // 8 bytes
1445 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);
1446 if (VecSize <= 96) // 12 bytes
1447 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);
1448 if (VecSize <= 128) // 16 bytes
1449 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);
1450 if (VecSize <= 160) // 20 bytes
1451 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);
1452 if (VecSize <= 192) // 24 bytes
1453 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V6);
1454 if (VecSize <= 224) // 28 bytes
1455 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V7);
1456 if (VecSize <= 256) // 32 bytes
1457 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);
1458 if (VecSize <= 288) // 36 bytes
1459 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9);
1460 if (VecSize <= 320) // 40 bytes
1461 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10);
1462 if (VecSize <= 352) // 44 bytes
1463 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11);
1464 if (VecSize <= 384) // 48 bytes
1465 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12);
1466 if (VecSize <= 512) // 64 bytes
1467 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);
1468 if (VecSize <= 1024) // 128 bytes
1469 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);
1470
1471 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos");
1472 }
1473
1474 if (VecSize <= 32) // 4 bytes
1475 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);
1476 if (VecSize <= 64) // 8 bytes
1477 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);
1478 if (VecSize <= 96) // 12 bytes
1479 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);
1480 if (VecSize <= 128) // 16 bytes
1481 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);
1482 if (VecSize <= 160) // 20 bytes
1483 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);
1484 if (VecSize <= 192) // 24 bytes
1485 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V6);
1486 if (VecSize <= 224) // 28 bytes
1487 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V7);
1488 if (VecSize <= 256) // 32 bytes
1489 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);
1490 if (VecSize <= 288) // 36 bytes
1491 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9);
1492 if (VecSize <= 320) // 40 bytes
1493 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10);
1494 if (VecSize <= 352) // 44 bytes
1495 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11);
1496 if (VecSize <= 384) // 48 bytes
1497 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12);
1498 if (VecSize <= 512) // 64 bytes
1499 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);
1500 if (VecSize <= 1024) // 128 bytes
1501 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);
1502
1503 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos");
1504}
1505
1506static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) {
1507 if (VecSize <= 32) // 4 bytes
1508 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1509 if (VecSize <= 64) // 8 bytes
1510 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1511 if (VecSize <= 96) // 12 bytes
1512 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1513 if (VecSize <= 128) // 16 bytes
1514 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1515 if (VecSize <= 160) // 20 bytes
1516 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1517 if (VecSize <= 192) // 24 bytes
1518 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V6;
1519 if (VecSize <= 224) // 28 bytes
1520 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V7;
1521 if (VecSize <= 256) // 32 bytes
1522 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1523 if (VecSize <= 288) // 36 bytes
1524 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1525 if (VecSize <= 320) // 40 bytes
1526 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1527 if (VecSize <= 352) // 44 bytes
1528 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1529 if (VecSize <= 384) // 48 bytes
1530 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1531 if (VecSize <= 512) // 64 bytes
1532 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1533 if (VecSize <= 1024) // 128 bytes
1534 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1535
1536 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1537}
1538
1539static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) {
1540 if (VecSize <= 32) // 4 bytes
1541 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1542 if (VecSize <= 64) // 8 bytes
1543 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1544 if (VecSize <= 96) // 12 bytes
1545 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1546 if (VecSize <= 128) // 16 bytes
1547 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1548 if (VecSize <= 160) // 20 bytes
1549 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1550 if (VecSize <= 192) // 24 bytes
1551 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V6;
1552 if (VecSize <= 224) // 28 bytes
1553 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V7;
1554 if (VecSize <= 256) // 32 bytes
1555 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1556 if (VecSize <= 288) // 36 bytes
1557 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1558 if (VecSize <= 320) // 40 bytes
1559 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1560 if (VecSize <= 352) // 44 bytes
1561 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1562 if (VecSize <= 384) // 48 bytes
1563 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1564 if (VecSize <= 512) // 64 bytes
1565 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1566 if (VecSize <= 1024) // 128 bytes
1567 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1568
1569 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1570}
1571
1572static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) {
1573 if (VecSize <= 64) // 8 bytes
1574 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;
1575 if (VecSize <= 128) // 16 bytes
1576 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;
1577 if (VecSize <= 256) // 32 bytes
1578 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;
1579 if (VecSize <= 512) // 64 bytes
1580 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;
1581 if (VecSize <= 1024) // 128 bytes
1582 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;
1583
1584 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1585}
1586
1587const MCInstrDesc &
1588SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize,
1589 bool IsSGPR) const {
1590 if (IsSGPR) {
1591 switch (EltSize) {
1592 case 32:
1593 return get(Opcode: getIndirectSGPRWriteMovRelPseudo32(VecSize));
1594 case 64:
1595 return get(Opcode: getIndirectSGPRWriteMovRelPseudo64(VecSize));
1596 default:
1597 llvm_unreachable("invalid reg indexing elt size");
1598 }
1599 }
1600
1601 assert(EltSize == 32 && "invalid reg indexing elt size");
1602 return get(Opcode: getIndirectVGPRWriteMovRelPseudoOpc(VecSize));
1603}
1604
1605static unsigned getSGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1606 switch (Size) {
1607 case 4:
1608 return NeedsCFI ? AMDGPU::SI_SPILL_S32_CFI_SAVE : AMDGPU::SI_SPILL_S32_SAVE;
1609 case 8:
1610 return NeedsCFI ? AMDGPU::SI_SPILL_S64_CFI_SAVE : AMDGPU::SI_SPILL_S64_SAVE;
1611 case 12:
1612 return NeedsCFI ? AMDGPU::SI_SPILL_S96_CFI_SAVE : AMDGPU::SI_SPILL_S96_SAVE;
1613 case 16:
1614 return NeedsCFI ? AMDGPU::SI_SPILL_S128_CFI_SAVE
1615 : AMDGPU::SI_SPILL_S128_SAVE;
1616 case 20:
1617 return NeedsCFI ? AMDGPU::SI_SPILL_S160_CFI_SAVE
1618 : AMDGPU::SI_SPILL_S160_SAVE;
1619 case 24:
1620 return NeedsCFI ? AMDGPU::SI_SPILL_S192_CFI_SAVE
1621 : AMDGPU::SI_SPILL_S192_SAVE;
1622 case 28:
1623 return NeedsCFI ? AMDGPU::SI_SPILL_S224_CFI_SAVE
1624 : AMDGPU::SI_SPILL_S224_SAVE;
1625 case 32:
1626 return AMDGPU::SI_SPILL_S256_SAVE;
1627 case 36:
1628 return AMDGPU::SI_SPILL_S288_SAVE;
1629 case 40:
1630 return AMDGPU::SI_SPILL_S320_SAVE;
1631 case 44:
1632 return AMDGPU::SI_SPILL_S352_SAVE;
1633 case 48:
1634 return AMDGPU::SI_SPILL_S384_SAVE;
1635 case 64:
1636 return NeedsCFI ? AMDGPU::SI_SPILL_S512_CFI_SAVE
1637 : AMDGPU::SI_SPILL_S512_SAVE;
1638 case 128:
1639 return NeedsCFI ? AMDGPU::SI_SPILL_S1024_CFI_SAVE
1640 : AMDGPU::SI_SPILL_S1024_SAVE;
1641 default:
1642 llvm_unreachable("unknown register size");
1643 }
1644}
1645
1646static unsigned getVGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1647 switch (Size) {
1648 case 2:
1649 return AMDGPU::SI_SPILL_V16_SAVE;
1650 case 4:
1651 return NeedsCFI ? AMDGPU::SI_SPILL_V32_CFI_SAVE : AMDGPU::SI_SPILL_V32_SAVE;
1652 case 8:
1653 return NeedsCFI ? AMDGPU::SI_SPILL_V64_CFI_SAVE : AMDGPU::SI_SPILL_V64_SAVE;
1654 case 12:
1655 return NeedsCFI ? AMDGPU::SI_SPILL_V96_CFI_SAVE : AMDGPU::SI_SPILL_V96_SAVE;
1656 case 16:
1657 return NeedsCFI ? AMDGPU::SI_SPILL_V128_CFI_SAVE
1658 : AMDGPU::SI_SPILL_V128_SAVE;
1659 case 20:
1660 return NeedsCFI ? AMDGPU::SI_SPILL_V160_CFI_SAVE
1661 : AMDGPU::SI_SPILL_V160_SAVE;
1662 case 24:
1663 return NeedsCFI ? AMDGPU::SI_SPILL_V192_CFI_SAVE
1664 : AMDGPU::SI_SPILL_V192_SAVE;
1665 case 28:
1666 return NeedsCFI ? AMDGPU::SI_SPILL_V224_CFI_SAVE
1667 : AMDGPU::SI_SPILL_V224_SAVE;
1668 case 32:
1669 return NeedsCFI ? AMDGPU::SI_SPILL_V256_CFI_SAVE
1670 : AMDGPU::SI_SPILL_V256_SAVE;
1671 case 36:
1672 return NeedsCFI ? AMDGPU::SI_SPILL_V288_CFI_SAVE
1673 : AMDGPU::SI_SPILL_V288_SAVE;
1674 case 40:
1675 return NeedsCFI ? AMDGPU::SI_SPILL_V320_CFI_SAVE
1676 : AMDGPU::SI_SPILL_V320_SAVE;
1677 case 44:
1678 return NeedsCFI ? AMDGPU::SI_SPILL_V352_CFI_SAVE
1679 : AMDGPU::SI_SPILL_V352_SAVE;
1680 case 48:
1681 return NeedsCFI ? AMDGPU::SI_SPILL_V384_CFI_SAVE
1682 : AMDGPU::SI_SPILL_V384_SAVE;
1683 case 64:
1684 return NeedsCFI ? AMDGPU::SI_SPILL_V512_CFI_SAVE
1685 : AMDGPU::SI_SPILL_V512_SAVE;
1686 case 128:
1687 return NeedsCFI ? AMDGPU::SI_SPILL_V1024_CFI_SAVE
1688 : AMDGPU::SI_SPILL_V1024_SAVE;
1689 default:
1690 llvm_unreachable("unknown register size");
1691 }
1692}
1693
1694static unsigned getAVSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1695 switch (Size) {
1696 case 4:
1697 return NeedsCFI ? AMDGPU::SI_SPILL_AV32_CFI_SAVE
1698 : AMDGPU::SI_SPILL_AV32_SAVE;
1699 case 8:
1700 return NeedsCFI ? AMDGPU::SI_SPILL_AV64_CFI_SAVE
1701 : AMDGPU::SI_SPILL_AV64_SAVE;
1702 case 12:
1703 return NeedsCFI ? AMDGPU::SI_SPILL_AV96_CFI_SAVE
1704 : AMDGPU::SI_SPILL_AV96_SAVE;
1705 case 16:
1706 return NeedsCFI ? AMDGPU::SI_SPILL_AV128_CFI_SAVE
1707 : AMDGPU::SI_SPILL_AV128_SAVE;
1708 case 20:
1709 return NeedsCFI ? AMDGPU::SI_SPILL_AV160_CFI_SAVE
1710 : AMDGPU::SI_SPILL_AV160_SAVE;
1711 case 24:
1712 return NeedsCFI ? AMDGPU::SI_SPILL_AV192_CFI_SAVE
1713 : AMDGPU::SI_SPILL_AV192_SAVE;
1714 case 28:
1715 return NeedsCFI ? AMDGPU::SI_SPILL_AV224_CFI_SAVE
1716 : AMDGPU::SI_SPILL_AV224_SAVE;
1717 case 32:
1718 return NeedsCFI ? AMDGPU::SI_SPILL_AV256_CFI_SAVE
1719 : AMDGPU::SI_SPILL_AV256_SAVE;
1720 case 36:
1721 return AMDGPU::SI_SPILL_AV288_SAVE;
1722 case 40:
1723 return AMDGPU::SI_SPILL_AV320_SAVE;
1724 case 44:
1725 return AMDGPU::SI_SPILL_AV352_SAVE;
1726 case 48:
1727 return AMDGPU::SI_SPILL_AV384_SAVE;
1728 case 64:
1729 return NeedsCFI ? AMDGPU::SI_SPILL_AV512_CFI_SAVE
1730 : AMDGPU::SI_SPILL_AV512_SAVE;
1731 case 128:
1732 return NeedsCFI ? AMDGPU::SI_SPILL_AV1024_CFI_SAVE
1733 : AMDGPU::SI_SPILL_AV1024_SAVE;
1734 default:
1735 llvm_unreachable("unknown register size");
1736 }
1737}
1738
1739static unsigned getWWMRegSpillSaveOpcode(unsigned Size,
1740 bool IsVectorSuperClass) {
1741 // Currently, there is only 32-bit WWM register spills needed.
1742 if (Size != 4)
1743 llvm_unreachable("unknown wwm register spill size");
1744
1745 if (IsVectorSuperClass)
1746 return AMDGPU::SI_SPILL_WWM_AV32_SAVE;
1747
1748 return AMDGPU::SI_SPILL_WWM_V32_SAVE;
1749}
1750
1751unsigned SIInstrInfo::getVectorRegSpillSaveOpcode(
1752 Register Reg, const TargetRegisterClass *RC, unsigned Size,
1753 const SIMachineFunctionInfo &MFI, bool NeedsCFI) const {
1754 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
1755
1756 // Choose the right opcode if spilling a WWM register.
1757 if (MFI.checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
1758 return getWWMRegSpillSaveOpcode(Size, IsVectorSuperClass);
1759
1760 // TODO: Check if AGPRs are available
1761 if (ST.hasMAIInsts())
1762 return getAVSpillSaveOpcode(Size, NeedsCFI);
1763
1764 return getVGPRSpillSaveOpcode(Size, NeedsCFI);
1765}
1766
1767void SIInstrInfo::storeRegToStackSlotImpl(
1768 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
1769 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
1770 MachineInstr::MIFlag Flags, bool NeedsCFI) const {
1771 MachineFunction *MF = MBB.getParent();
1772 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1773 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1774 const DebugLoc &DL = MBB.findDebugLoc(MBBI: MI);
1775
1776 MachinePointerInfo PtrInfo
1777 = MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
1778 MachineMemOperand *MMO = MF->getMachineMemOperand(
1779 PtrInfo, F: MachineMemOperand::MOStore, Size: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
1780 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
1781 unsigned SpillSize = RI.getSpillSize(RC: *RC);
1782
1783 MachineRegisterInfo &MRI = MF->getRegInfo();
1784 if (RI.isSGPRClass(RC)) {
1785 if (FrameInfo.getStackID(ObjectIdx: FrameIndex) == TargetStackID::SGPRSpill)
1786 MFI->setHasSpilledSGPRs();
1787 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled");
1788 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI &&
1789 SrcReg != AMDGPU::EXEC && "exec should not be spilled");
1790
1791 // We are only allowed to create one new instruction when spilling
1792 // registers, so we need to use pseudo instruction for spilling SGPRs.
1793 const MCInstrDesc &OpDesc =
1794 get(Opcode: getSGPRSpillSaveOpcode(Size: SpillSize, NeedsCFI));
1795
1796 // The SGPR spill/restore instructions only work on number sgprs, so we need
1797 // to make sure we are using the correct register class.
1798 if (SrcReg.isVirtual() && SpillSize == 4) {
1799 MRI.constrainRegClass(Reg: SrcReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
1800 }
1801
1802 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: OpDesc)
1803 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill)) // data
1804 .addFrameIndex(Idx: FrameIndex) // addr
1805 .addMemOperand(MMO)
1806 .addReg(RegNo: MFI->getStackPtrOffsetReg(), Flags: RegState::Implicit);
1807
1808 return;
1809 }
1810
1811 unsigned Opcode = getVectorRegSpillSaveOpcode(Reg: VReg ? VReg : SrcReg, RC,
1812 Size: SpillSize, MFI: *MFI, NeedsCFI);
1813 MFI->setHasSpilledVGPRs();
1814
1815 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode))
1816 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill)) // data
1817 .addFrameIndex(Idx: FrameIndex) // addr
1818 .addReg(RegNo: MFI->getStackPtrOffsetReg()) // scratch_offset
1819 .addImm(Val: 0) // offset
1820 .addMemOperand(MMO);
1821}
1822
1823void SIInstrInfo::storeRegToStackSlot(
1824 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
1825 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
1826 MachineInstr::MIFlag Flags) const {
1827 storeRegToStackSlotImpl(MBB, MI, SrcReg, isKill, FrameIndex, RC, VReg, Flags,
1828 NeedsCFI: false);
1829}
1830
1831void SIInstrInfo::storeRegToStackSlotCFI(MachineBasicBlock &MBB,
1832 MachineBasicBlock::iterator MI,
1833 Register SrcReg, bool isKill,
1834 int FrameIndex,
1835 const TargetRegisterClass *RC) const {
1836 storeRegToStackSlotImpl(MBB, MI, SrcReg, isKill, FrameIndex, RC, VReg: Register(),
1837 Flags: MachineInstr::NoFlags, NeedsCFI: true);
1838}
1839
1840static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1841 switch (Size) {
1842 case 4:
1843 return AMDGPU::SI_SPILL_S32_RESTORE;
1844 case 8:
1845 return AMDGPU::SI_SPILL_S64_RESTORE;
1846 case 12:
1847 return AMDGPU::SI_SPILL_S96_RESTORE;
1848 case 16:
1849 return AMDGPU::SI_SPILL_S128_RESTORE;
1850 case 20:
1851 return AMDGPU::SI_SPILL_S160_RESTORE;
1852 case 24:
1853 return AMDGPU::SI_SPILL_S192_RESTORE;
1854 case 28:
1855 return AMDGPU::SI_SPILL_S224_RESTORE;
1856 case 32:
1857 return AMDGPU::SI_SPILL_S256_RESTORE;
1858 case 36:
1859 return AMDGPU::SI_SPILL_S288_RESTORE;
1860 case 40:
1861 return AMDGPU::SI_SPILL_S320_RESTORE;
1862 case 44:
1863 return AMDGPU::SI_SPILL_S352_RESTORE;
1864 case 48:
1865 return AMDGPU::SI_SPILL_S384_RESTORE;
1866 case 64:
1867 return AMDGPU::SI_SPILL_S512_RESTORE;
1868 case 128:
1869 return AMDGPU::SI_SPILL_S1024_RESTORE;
1870 default:
1871 llvm_unreachable("unknown register size");
1872 }
1873}
1874
1875static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1876 switch (Size) {
1877 case 2:
1878 return AMDGPU::SI_SPILL_V16_RESTORE;
1879 case 4:
1880 return AMDGPU::SI_SPILL_V32_RESTORE;
1881 case 8:
1882 return AMDGPU::SI_SPILL_V64_RESTORE;
1883 case 12:
1884 return AMDGPU::SI_SPILL_V96_RESTORE;
1885 case 16:
1886 return AMDGPU::SI_SPILL_V128_RESTORE;
1887 case 20:
1888 return AMDGPU::SI_SPILL_V160_RESTORE;
1889 case 24:
1890 return AMDGPU::SI_SPILL_V192_RESTORE;
1891 case 28:
1892 return AMDGPU::SI_SPILL_V224_RESTORE;
1893 case 32:
1894 return AMDGPU::SI_SPILL_V256_RESTORE;
1895 case 36:
1896 return AMDGPU::SI_SPILL_V288_RESTORE;
1897 case 40:
1898 return AMDGPU::SI_SPILL_V320_RESTORE;
1899 case 44:
1900 return AMDGPU::SI_SPILL_V352_RESTORE;
1901 case 48:
1902 return AMDGPU::SI_SPILL_V384_RESTORE;
1903 case 64:
1904 return AMDGPU::SI_SPILL_V512_RESTORE;
1905 case 128:
1906 return AMDGPU::SI_SPILL_V1024_RESTORE;
1907 default:
1908 llvm_unreachable("unknown register size");
1909 }
1910}
1911
1912static unsigned getAVSpillRestoreOpcode(unsigned Size) {
1913 switch (Size) {
1914 case 4:
1915 return AMDGPU::SI_SPILL_AV32_RESTORE;
1916 case 8:
1917 return AMDGPU::SI_SPILL_AV64_RESTORE;
1918 case 12:
1919 return AMDGPU::SI_SPILL_AV96_RESTORE;
1920 case 16:
1921 return AMDGPU::SI_SPILL_AV128_RESTORE;
1922 case 20:
1923 return AMDGPU::SI_SPILL_AV160_RESTORE;
1924 case 24:
1925 return AMDGPU::SI_SPILL_AV192_RESTORE;
1926 case 28:
1927 return AMDGPU::SI_SPILL_AV224_RESTORE;
1928 case 32:
1929 return AMDGPU::SI_SPILL_AV256_RESTORE;
1930 case 36:
1931 return AMDGPU::SI_SPILL_AV288_RESTORE;
1932 case 40:
1933 return AMDGPU::SI_SPILL_AV320_RESTORE;
1934 case 44:
1935 return AMDGPU::SI_SPILL_AV352_RESTORE;
1936 case 48:
1937 return AMDGPU::SI_SPILL_AV384_RESTORE;
1938 case 64:
1939 return AMDGPU::SI_SPILL_AV512_RESTORE;
1940 case 128:
1941 return AMDGPU::SI_SPILL_AV1024_RESTORE;
1942 default:
1943 llvm_unreachable("unknown register size");
1944 }
1945}
1946
1947static unsigned getWWMRegSpillRestoreOpcode(unsigned Size,
1948 bool IsVectorSuperClass) {
1949 // Currently, there is only 32-bit WWM register spills needed.
1950 if (Size != 4)
1951 llvm_unreachable("unknown wwm register spill size");
1952
1953 if (IsVectorSuperClass) // TODO: Always use this if there are AGPRs
1954 return AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
1955
1956 return AMDGPU::SI_SPILL_WWM_V32_RESTORE;
1957}
1958
1959unsigned SIInstrInfo::getVectorRegSpillRestoreOpcode(
1960 Register Reg, const TargetRegisterClass *RC, unsigned Size,
1961 const SIMachineFunctionInfo &MFI) const {
1962 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
1963
1964 // Choose the right opcode if restoring a WWM register.
1965 if (MFI.checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
1966 return getWWMRegSpillRestoreOpcode(Size, IsVectorSuperClass);
1967
1968 // TODO: Check if AGPRs are available
1969 if (ST.hasMAIInsts())
1970 return getAVSpillRestoreOpcode(Size);
1971
1972 assert(!RI.isAGPRClass(RC));
1973 return getVGPRSpillRestoreOpcode(Size);
1974}
1975
1976void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1977 MachineBasicBlock::iterator MI,
1978 Register DestReg, int FrameIndex,
1979 const TargetRegisterClass *RC,
1980 Register VReg, unsigned SubReg,
1981 MachineInstr::MIFlag Flags) const {
1982 MachineFunction *MF = MBB.getParent();
1983 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1984 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1985 const DebugLoc &DL = MBB.findDebugLoc(MBBI: MI);
1986 unsigned SpillSize = RI.getSpillSize(RC: *RC);
1987
1988 MachinePointerInfo PtrInfo
1989 = MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
1990
1991 MachineMemOperand *MMO = MF->getMachineMemOperand(
1992 PtrInfo, F: MachineMemOperand::MOLoad, Size: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
1993 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
1994
1995 if (RI.isSGPRClass(RC)) {
1996 if (FrameInfo.getStackID(ObjectIdx: FrameIndex) == TargetStackID::SGPRSpill)
1997 MFI->setHasSpilledSGPRs();
1998 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into");
1999 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI &&
2000 DestReg != AMDGPU::EXEC && "exec should not be spilled");
2001
2002 // FIXME: Maybe this should not include a memoperand because it will be
2003 // lowered to non-memory instructions.
2004 const MCInstrDesc &OpDesc = get(Opcode: getSGPRSpillRestoreOpcode(Size: SpillSize));
2005 if (DestReg.isVirtual() && SpillSize == 4) {
2006 MachineRegisterInfo &MRI = MF->getRegInfo();
2007 MRI.constrainRegClass(Reg: DestReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
2008 }
2009
2010 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: OpDesc, DestReg)
2011 .addFrameIndex(Idx: FrameIndex) // addr
2012 .addMemOperand(MMO)
2013 .addReg(RegNo: MFI->getStackPtrOffsetReg(), Flags: RegState::Implicit);
2014
2015 return;
2016 }
2017
2018 unsigned Opcode = getVectorRegSpillRestoreOpcode(Reg: VReg ? VReg : DestReg, RC,
2019 Size: SpillSize, MFI: *MFI);
2020 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode), DestReg)
2021 .addFrameIndex(Idx: FrameIndex) // vaddr
2022 .addReg(RegNo: MFI->getStackPtrOffsetReg()) // scratch_offset
2023 .addImm(Val: 0) // offset
2024 .addMemOperand(MMO);
2025}
2026
2027void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
2028 MachineBasicBlock::iterator MI) const {
2029 insertNoops(MBB, MI, Quantity: 1);
2030}
2031
2032void SIInstrInfo::insertNoops(MachineBasicBlock &MBB,
2033 MachineBasicBlock::iterator MI,
2034 unsigned Quantity) const {
2035 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2036 unsigned MaxSNopCount = 1u << ST.getSNopBits();
2037 while (Quantity > 0) {
2038 unsigned Arg = std::min(a: Quantity, b: MaxSNopCount);
2039 Quantity -= Arg;
2040 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOP)).addImm(Val: Arg - 1);
2041 }
2042}
2043
2044MachineBasicBlock *SIInstrInfo::insertSimulatedTrap(MachineRegisterInfo &MRI,
2045 MachineBasicBlock &MBB,
2046 MachineInstr &MI,
2047 const DebugLoc &DL) const {
2048 MachineFunction *MF = MBB.getParent();
2049 constexpr unsigned DoorbellIDMask = 0x3ff;
2050 constexpr unsigned ECQueueWaveAbort = 0x400;
2051
2052 MachineBasicBlock *TrapBB = &MBB;
2053 MachineBasicBlock *HaltLoopBB = MF->CreateMachineBasicBlock();
2054
2055 if (!MBB.succ_empty() || std::next(x: MI.getIterator()) != MBB.end()) {
2056 MBB.splitAt(SplitInst&: MI, /*UpdateLiveIns=*/false);
2057 TrapBB = MF->CreateMachineBasicBlock();
2058 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CBRANCH_EXECNZ)).addMBB(MBB: TrapBB);
2059 MF->push_back(MBB: TrapBB);
2060 MBB.addSuccessor(Succ: TrapBB);
2061 }
2062 // Start with a `s_trap 2`, if we're in PRIV=1 and we need the workaround this
2063 // will be a nop.
2064 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_TRAP))
2065 .addImm(Val: static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));
2066 Register DoorbellReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2067 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SENDMSG_RTN_B32),
2068 DestReg: DoorbellReg)
2069 .addImm(Val: AMDGPU::SendMsg::ID_RTN_GET_DOORBELL);
2070 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::TTMP2)
2071 .addUse(RegNo: AMDGPU::M0);
2072 Register DoorbellRegMasked =
2073 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2074 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_AND_B32), DestReg: DoorbellRegMasked)
2075 .addUse(RegNo: DoorbellReg)
2076 .addImm(Val: DoorbellIDMask);
2077 Register SetWaveAbortBit =
2078 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2079 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_OR_B32), DestReg: SetWaveAbortBit)
2080 .addUse(RegNo: DoorbellRegMasked)
2081 .addImm(Val: ECQueueWaveAbort);
2082 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
2083 .addUse(RegNo: SetWaveAbortBit);
2084 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SENDMSG))
2085 .addImm(Val: AMDGPU::SendMsg::ID_INTERRUPT);
2086 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
2087 .addUse(RegNo: AMDGPU::TTMP2);
2088 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH)).addMBB(MBB: HaltLoopBB);
2089 TrapBB->addSuccessor(Succ: HaltLoopBB);
2090
2091 BuildMI(BB&: *HaltLoopBB, I: HaltLoopBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETHALT)).addImm(Val: 5);
2092 BuildMI(BB&: *HaltLoopBB, I: HaltLoopBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
2093 .addMBB(MBB: HaltLoopBB);
2094 MF->push_back(MBB: HaltLoopBB);
2095 HaltLoopBB->addSuccessor(Succ: HaltLoopBB);
2096
2097 return MBB.getNextNode();
2098}
2099
2100unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) {
2101 switch (MI.getOpcode()) {
2102 default:
2103 if (MI.isMetaInstruction())
2104 return 0;
2105 return 1; // FIXME: Do wait states equal cycles?
2106
2107 case AMDGPU::S_NOP:
2108 return MI.getOperand(i: 0).getImm() + 1;
2109 // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The
2110 // hazard, even if one exist, won't really be visible. Should we handle it?
2111 }
2112}
2113
2114bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2115 MachineBasicBlock &MBB = *MI.getParent();
2116 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2117 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
2118
2119 switch (MI.getOpcode()) {
2120 default: return TargetInstrInfo::expandPostRAPseudo(MI);
2121 case AMDGPU::S_MOV_B64_term:
2122 // This is only a terminator to get the correct spill code placement during
2123 // register allocation.
2124 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2125 break;
2126
2127 case AMDGPU::S_MOV_B32_term:
2128 // This is only a terminator to get the correct spill code placement during
2129 // register allocation.
2130 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B32));
2131 break;
2132
2133 case AMDGPU::S_XOR_B64_term:
2134 // This is only a terminator to get the correct spill code placement during
2135 // register allocation.
2136 MI.setDesc(get(Opcode: AMDGPU::S_XOR_B64));
2137 break;
2138
2139 case AMDGPU::S_XOR_B32_term:
2140 // This is only a terminator to get the correct spill code placement during
2141 // register allocation.
2142 MI.setDesc(get(Opcode: AMDGPU::S_XOR_B32));
2143 break;
2144 case AMDGPU::S_OR_B64_term:
2145 // This is only a terminator to get the correct spill code placement during
2146 // register allocation.
2147 MI.setDesc(get(Opcode: AMDGPU::S_OR_B64));
2148 break;
2149 case AMDGPU::S_OR_B32_term:
2150 // This is only a terminator to get the correct spill code placement during
2151 // register allocation.
2152 MI.setDesc(get(Opcode: AMDGPU::S_OR_B32));
2153 break;
2154
2155 case AMDGPU::S_ANDN2_B64_term:
2156 // This is only a terminator to get the correct spill code placement during
2157 // register allocation.
2158 MI.setDesc(get(Opcode: AMDGPU::S_ANDN2_B64));
2159 break;
2160
2161 case AMDGPU::S_ANDN2_B32_term:
2162 // This is only a terminator to get the correct spill code placement during
2163 // register allocation.
2164 MI.setDesc(get(Opcode: AMDGPU::S_ANDN2_B32));
2165 break;
2166
2167 case AMDGPU::S_AND_B64_term:
2168 // This is only a terminator to get the correct spill code placement during
2169 // register allocation.
2170 MI.setDesc(get(Opcode: AMDGPU::S_AND_B64));
2171 break;
2172
2173 case AMDGPU::S_AND_B32_term:
2174 // This is only a terminator to get the correct spill code placement during
2175 // register allocation.
2176 MI.setDesc(get(Opcode: AMDGPU::S_AND_B32));
2177 break;
2178
2179 case AMDGPU::S_AND_SAVEEXEC_B64_term:
2180 // This is only a terminator to get the correct spill code placement during
2181 // register allocation.
2182 MI.setDesc(get(Opcode: AMDGPU::S_AND_SAVEEXEC_B64));
2183 break;
2184
2185 case AMDGPU::S_AND_SAVEEXEC_B32_term:
2186 // This is only a terminator to get the correct spill code placement during
2187 // register allocation.
2188 MI.setDesc(get(Opcode: AMDGPU::S_AND_SAVEEXEC_B32));
2189 break;
2190
2191 case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
2192 MI.setDesc(get(Opcode: AMDGPU::V_CMPX_EQ_U32_nosdst_e32));
2193 break;
2194 case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
2195 MI.setDesc(get(Opcode: AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
2196 break;
2197
2198 case AMDGPU::SI_SPILL_S32_TO_VGPR:
2199 MI.setDesc(get(Opcode: AMDGPU::V_WRITELANE_B32));
2200 break;
2201
2202 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
2203 MI.setDesc(get(Opcode: AMDGPU::V_READLANE_B32));
2204 break;
2205 case AMDGPU::AV_MOV_B32_IMM_PSEUDO: {
2206 Register Dst = MI.getOperand(i: 0).getReg();
2207 bool IsAGPR = SIRegisterInfo::isAGPRClass(RC: RI.getPhysRegBaseClass(Reg: Dst));
2208 MI.setDesc(
2209 get(Opcode: IsAGPR ? AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::V_MOV_B32_e32));
2210 break;
2211 }
2212 case AMDGPU::AV_MOV_B64_IMM_PSEUDO: {
2213 Register Dst = MI.getOperand(i: 0).getReg();
2214 if (SIRegisterInfo::isAGPRClass(RC: RI.getPhysRegBaseClass(Reg: Dst))) {
2215 int64_t Imm = MI.getOperand(i: 1).getImm();
2216
2217 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2218 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2219 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: DstLo)
2220 .addImm(Val: SignExtend64<32>(x: Imm));
2221 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: DstHi)
2222 .addImm(Val: SignExtend64<32>(x: Imm >> 32));
2223 MI.eraseFromParent();
2224 break;
2225 }
2226
2227 [[fallthrough]];
2228 }
2229 case AMDGPU::V_MOV_B64_PSEUDO: {
2230 Register Dst = MI.getOperand(i: 0).getReg();
2231 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2232 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2233
2234 const MCInstrDesc &Mov64Desc = get(Opcode: AMDGPU::V_MOV_B64_e32);
2235 const TargetRegisterClass *Mov64RC = getRegClass(MCID: Mov64Desc, /*OpNum=*/0);
2236
2237 const MachineOperand &SrcOp = MI.getOperand(i: 1);
2238 // FIXME: Will this work for 64-bit floating point immediates?
2239 assert(!SrcOp.isFPImm());
2240 if (ST.hasVMovB64Inst() && Mov64RC->contains(Reg: Dst)) {
2241 MI.setDesc(Mov64Desc);
2242 if (SrcOp.isReg() || isInlineConstant(MI, OpIdx: 1) ||
2243 (SrcOp.isImm() &&
2244 (isUInt<32>(x: SrcOp.getImm()) || ST.has64BitLiterals())) ||
2245 (SrcOp.isGlobal() && ST.has64BitLiterals()))
2246 break;
2247 }
2248 if (SrcOp.isGlobal()) {
2249 // The address is unknown until link time, so the PK_MOV inline-constant
2250 // shortcut cannot apply.
2251 const GlobalValue *GV = SrcOp.getGlobal();
2252 int64_t Offset = SrcOp.getOffset();
2253 unsigned BaseFlags, LoReloc, HiReloc;
2254 std::tie(args&: BaseFlags, args&: LoReloc, args&: HiReloc) =
2255 splitGlobalAddressRelocFlags(ST, SrcOp);
2256
2257 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2258 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | LoReloc);
2259 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2260 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | HiReloc);
2261 } else if (SrcOp.isImm()) {
2262 APInt Imm(64, SrcOp.getImm());
2263 APInt Lo(32, Imm.getLoBits(numBits: 32).getZExtValue());
2264 APInt Hi(32, Imm.getHiBits(numBits: 32).getZExtValue());
2265 const MCInstrDesc &PkMovDesc = get(Opcode: AMDGPU::V_PK_MOV_B32);
2266 const TargetRegisterClass *PkMovRC = getRegClass(MCID: PkMovDesc, /*OpNum=*/0);
2267
2268 if (ST.hasPkMovB32() && Lo == Hi && isInlineConstant(Imm: Lo) &&
2269 PkMovRC->contains(Reg: Dst)) {
2270 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: PkMovDesc, DestReg: Dst)
2271 .addImm(Val: SISrcMods::OP_SEL_1)
2272 .addImm(Val: Lo.getSExtValue())
2273 .addImm(Val: SISrcMods::OP_SEL_1)
2274 .addImm(Val: Lo.getSExtValue())
2275 .addImm(Val: 0) // op_sel_lo
2276 .addImm(Val: 0) // op_sel_hi
2277 .addImm(Val: 0) // neg_lo
2278 .addImm(Val: 0) // neg_hi
2279 .addImm(Val: 0); // clamp
2280 } else {
2281 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2282 .addImm(Val: Lo.getSExtValue());
2283 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2284 .addImm(Val: Hi.getSExtValue());
2285 }
2286 } else {
2287 assert(SrcOp.isReg());
2288 if (ST.hasPkMovB32() &&
2289 !RI.isAGPR(MRI: MBB.getParent()->getRegInfo(), Reg: SrcOp.getReg())) {
2290 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg: Dst)
2291 .addImm(Val: SISrcMods::OP_SEL_1) // src0_mod
2292 .addReg(RegNo: SrcOp.getReg())
2293 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) // src1_mod
2294 .addReg(RegNo: SrcOp.getReg())
2295 .addImm(Val: 0) // op_sel_lo
2296 .addImm(Val: 0) // op_sel_hi
2297 .addImm(Val: 0) // neg_lo
2298 .addImm(Val: 0) // neg_hi
2299 .addImm(Val: 0); // clamp
2300 } else {
2301 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2302 .addReg(RegNo: RI.getSubReg(Reg: SrcOp.getReg(), Idx: AMDGPU::sub0));
2303 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2304 .addReg(RegNo: RI.getSubReg(Reg: SrcOp.getReg(), Idx: AMDGPU::sub1));
2305 }
2306 }
2307 MI.eraseFromParent();
2308 break;
2309 }
2310 case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
2311 expandMovDPP64(MI);
2312 break;
2313 }
2314 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2315 const MachineOperand &SrcOp = MI.getOperand(i: 1);
2316 assert(!SrcOp.isFPImm());
2317
2318 if (ST.has64BitLiterals()) {
2319 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2320 break;
2321 }
2322
2323 if (SrcOp.isGlobal()) {
2324 Register Dst = MI.getOperand(i: 0).getReg();
2325 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2326 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2327 const GlobalValue *GV = SrcOp.getGlobal();
2328 int64_t Offset = SrcOp.getOffset();
2329 unsigned BaseFlags, LoReloc, HiReloc;
2330 std::tie(args&: BaseFlags, args&: LoReloc, args&: HiReloc) =
2331 splitGlobalAddressRelocFlags(ST, SrcOp);
2332
2333 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstLo)
2334 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | LoReloc);
2335 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi)
2336 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | HiReloc);
2337 MI.eraseFromParent();
2338 break;
2339 }
2340
2341 // SrcOp is immediate
2342 APInt Imm(64, SrcOp.getImm());
2343 if (Imm.isIntN(N: 32) || isInlineConstant(Imm)) {
2344 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2345 break;
2346 }
2347
2348 Register Dst = MI.getOperand(i: 0).getReg();
2349 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2350 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2351
2352 APInt Lo(32, Imm.getLoBits(numBits: 32).getZExtValue());
2353 APInt Hi(32, Imm.getHiBits(numBits: 32).getZExtValue());
2354 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstLo)
2355 .addImm(Val: Lo.getSExtValue());
2356 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi)
2357 .addImm(Val: Hi.getSExtValue());
2358 MI.eraseFromParent();
2359 break;
2360 }
2361 case AMDGPU::V_SET_INACTIVE_B32: {
2362 // Lower V_SET_INACTIVE_B32 to V_CNDMASK_B32.
2363 Register DstReg = MI.getOperand(i: 0).getReg();
2364 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
2365 .add(MO: MI.getOperand(i: 3))
2366 .add(MO: MI.getOperand(i: 4))
2367 .add(MO: MI.getOperand(i: 1))
2368 .add(MO: MI.getOperand(i: 2))
2369 .add(MO: MI.getOperand(i: 5));
2370 MI.eraseFromParent();
2371 break;
2372 }
2373 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2374 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2375 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2376 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2377 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2378 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V6:
2379 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V7:
2380 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2381 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2382 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2383 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2384 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2385 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2386 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2387 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2388 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2389 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2390 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2391 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2392 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V6:
2393 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V7:
2394 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2395 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2396 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2397 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2398 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2399 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2400 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2401 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:
2402 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:
2403 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:
2404 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:
2405 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {
2406 const TargetRegisterClass *EltRC = getOpRegClass(MI, OpNo: 2);
2407
2408 unsigned Opc;
2409 if (RI.hasVGPRs(RC: EltRC)) {
2410 Opc = AMDGPU::V_MOVRELD_B32_e32;
2411 } else {
2412 Opc = RI.getRegSizeInBits(RC: *EltRC) == 64 ? AMDGPU::S_MOVRELD_B64
2413 : AMDGPU::S_MOVRELD_B32;
2414 }
2415
2416 const MCInstrDesc &OpDesc = get(Opcode: Opc);
2417 Register VecReg = MI.getOperand(i: 0).getReg();
2418 bool IsUndef = MI.getOperand(i: 1).isUndef();
2419 unsigned SubReg = MI.getOperand(i: 3).getImm();
2420 assert(VecReg == MI.getOperand(1).getReg());
2421
2422 MachineInstrBuilder MIB =
2423 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: OpDesc)
2424 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2425 .add(MO: MI.getOperand(i: 2))
2426 .addReg(RegNo: VecReg, Flags: RegState::ImplicitDefine)
2427 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2428
2429 const int ImpDefIdx =
2430 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2431 const int ImpUseIdx = ImpDefIdx + 1;
2432 MIB->tieOperands(DefIdx: ImpDefIdx, UseIdx: ImpUseIdx);
2433 MI.eraseFromParent();
2434 break;
2435 }
2436 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:
2437 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:
2438 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:
2439 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:
2440 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:
2441 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V6:
2442 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V7:
2443 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:
2444 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9:
2445 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10:
2446 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11:
2447 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12:
2448 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:
2449 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {
2450 assert(ST.useVGPRIndexMode());
2451 Register VecReg = MI.getOperand(i: 0).getReg();
2452 bool IsUndef = MI.getOperand(i: 1).isUndef();
2453 MachineOperand &Idx = MI.getOperand(i: 3);
2454 Register SubReg = MI.getOperand(i: 4).getImm();
2455
2456 MachineInstr *SetOn = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_ON))
2457 .add(MO: Idx)
2458 .addImm(Val: AMDGPU::VGPRIndexMode::DST_ENABLE);
2459 SetOn->getOperand(i: 3).setIsUndef();
2460
2461 const MCInstrDesc &OpDesc = get(Opcode: AMDGPU::V_MOV_B32_indirect_write);
2462 MachineInstrBuilder MIB =
2463 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: OpDesc)
2464 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2465 .add(MO: MI.getOperand(i: 2))
2466 .addReg(RegNo: VecReg, Flags: RegState::ImplicitDefine)
2467 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2468
2469 const int ImpDefIdx =
2470 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2471 const int ImpUseIdx = ImpDefIdx + 1;
2472 MIB->tieOperands(DefIdx: ImpDefIdx, UseIdx: ImpUseIdx);
2473
2474 MachineInstr *SetOff = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_OFF));
2475
2476 finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2477
2478 MI.eraseFromParent();
2479 break;
2480 }
2481 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:
2482 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:
2483 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:
2484 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:
2485 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:
2486 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V6:
2487 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V7:
2488 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:
2489 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9:
2490 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10:
2491 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11:
2492 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12:
2493 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:
2494 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {
2495 assert(ST.useVGPRIndexMode());
2496 Register Dst = MI.getOperand(i: 0).getReg();
2497 Register VecReg = MI.getOperand(i: 1).getReg();
2498 bool IsUndef = MI.getOperand(i: 1).isUndef();
2499 Register SubReg = MI.getOperand(i: 3).getImm();
2500
2501 MachineInstr *SetOn = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_ON))
2502 .add(MO: MI.getOperand(i: 2))
2503 .addImm(Val: AMDGPU::VGPRIndexMode::SRC0_ENABLE);
2504 SetOn->getOperand(i: 3).setIsUndef();
2505
2506 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_indirect_read))
2507 .addDef(RegNo: Dst)
2508 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2509 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2510
2511 MachineInstr *SetOff = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_OFF));
2512
2513 finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2514
2515 MI.eraseFromParent();
2516 break;
2517 }
2518 case AMDGPU::SI_PC_ADD_REL_OFFSET: {
2519 MachineFunction &MF = *MBB.getParent();
2520 Register Reg = MI.getOperand(i: 0).getReg();
2521 Register RegLo = RI.getSubReg(Reg, Idx: AMDGPU::sub0);
2522 Register RegHi = RI.getSubReg(Reg, Idx: AMDGPU::sub1);
2523 MachineOperand OpLo = MI.getOperand(i: 1);
2524 MachineOperand OpHi = MI.getOperand(i: 2);
2525
2526 // Create a bundle so these instructions won't be re-ordered by the
2527 // post-RA scheduler.
2528 MIBundleBuilder Bundler(MBB, MI);
2529 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: Reg));
2530
2531 // What we want here is an offset from the value returned by s_getpc (which
2532 // is the address of the s_add_u32 instruction) to the global variable, but
2533 // since the encoding of $symbol starts 4 bytes after the start of the
2534 // s_add_u32 instruction, we end up with an offset that is 4 bytes too
2535 // small. This requires us to add 4 to the global variable offset in order
2536 // to compute the correct address. Similarly for the s_addc_u32 instruction,
2537 // the encoding of $symbol starts 12 bytes after the start of the s_add_u32
2538 // instruction.
2539
2540 int64_t Adjust = 0;
2541 if (ST.hasGetPCZeroExtension()) {
2542 // Fix up hardware that does not sign-extend the 48-bit PC value by
2543 // inserting: s_sext_i32_i16 reghi, reghi
2544 Bundler.append(
2545 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SEXT_I32_I16), DestReg: RegHi).addReg(RegNo: RegHi));
2546 Adjust += 4;
2547 }
2548
2549 if (OpLo.isGlobal())
2550 OpLo.setOffset(OpLo.getOffset() + Adjust + 4);
2551 Bundler.append(
2552 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U32), DestReg: RegLo).addReg(RegNo: RegLo).add(MO: OpLo));
2553
2554 if (OpHi.isGlobal())
2555 OpHi.setOffset(OpHi.getOffset() + Adjust + 12);
2556 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADDC_U32), DestReg: RegHi)
2557 .addReg(RegNo: RegHi)
2558 .add(MO: OpHi));
2559
2560 finalizeBundle(MBB, FirstMI: Bundler.begin());
2561
2562 MI.eraseFromParent();
2563 break;
2564 }
2565 case AMDGPU::SI_PC_ADD_REL_OFFSET64: {
2566 MachineFunction &MF = *MBB.getParent();
2567 Register Reg = MI.getOperand(i: 0).getReg();
2568 MachineOperand Op = MI.getOperand(i: 1);
2569
2570 // Create a bundle so these instructions won't be re-ordered by the
2571 // post-RA scheduler.
2572 MIBundleBuilder Bundler(MBB, MI);
2573 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: Reg));
2574 if (Op.isGlobal())
2575 Op.setOffset(Op.getOffset() + 4);
2576 Bundler.append(
2577 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U64), DestReg: Reg).addReg(RegNo: Reg).add(MO: Op));
2578
2579 finalizeBundle(MBB, FirstMI: Bundler.begin());
2580
2581 MI.eraseFromParent();
2582 break;
2583 }
2584 case AMDGPU::ENTER_STRICT_WWM: {
2585 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2586 // Whole Wave Mode is entered.
2587 MI.setDesc(get(Opcode: LMC.OrSaveExecOpc));
2588 break;
2589 }
2590 case AMDGPU::ENTER_STRICT_WQM: {
2591 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2592 // STRICT_WQM is entered.
2593 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: LMC.MovOpc), DestReg: MI.getOperand(i: 0).getReg())
2594 .addReg(RegNo: LMC.ExecReg);
2595 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: LMC.WQMOpc), DestReg: LMC.ExecReg).addReg(RegNo: LMC.ExecReg);
2596
2597 MI.eraseFromParent();
2598 break;
2599 }
2600 case AMDGPU::EXIT_STRICT_WWM:
2601 case AMDGPU::EXIT_STRICT_WQM: {
2602 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2603 // WWM/STICT_WQM is exited.
2604 MI.setDesc(get(Opcode: LMC.MovOpc));
2605 break;
2606 }
2607 case AMDGPU::SI_RETURN: {
2608 const MachineFunction *MF = MBB.getParent();
2609 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2610 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2611 // Hiding the return address use with SI_RETURN may lead to extra kills in
2612 // the function and missing live-ins. We are fine in practice because callee
2613 // saved register handling ensures the register value is restored before
2614 // RET, but we need the undef flag here to appease the MachineVerifier
2615 // liveness checks.
2616 MachineInstrBuilder MIB =
2617 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETPC_B64_return))
2618 .addReg(RegNo: TRI->getReturnAddressReg(MF: *MF), Flags: RegState::Undef);
2619
2620 MIB.copyImplicitOps(OtherMI: MI);
2621 MI.eraseFromParent();
2622 break;
2623 }
2624
2625 case AMDGPU::S_MUL_U64_U32_PSEUDO:
2626 case AMDGPU::S_MUL_I64_I32_PSEUDO:
2627 MI.setDesc(get(Opcode: AMDGPU::S_MUL_U64));
2628 break;
2629
2630 case AMDGPU::S_GETPC_B64_pseudo:
2631 MI.setDesc(get(Opcode: AMDGPU::S_GETPC_B64));
2632 if (ST.hasGetPCZeroExtension()) {
2633 Register Dst = MI.getOperand(i: 0).getReg();
2634 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2635 // Fix up hardware that does not sign-extend the 48-bit PC value by
2636 // inserting: s_sext_i32_i16 dsthi, dsthi
2637 BuildMI(BB&: MBB, I: std::next(x: MI.getIterator()), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SEXT_I32_I16),
2638 DestReg: DstHi)
2639 .addReg(RegNo: DstHi);
2640 }
2641 break;
2642
2643 case AMDGPU::V_MAX_BF16_PSEUDO_e64: {
2644 assert(ST.hasBF16PackedInsts());
2645 MI.setDesc(get(Opcode: AMDGPU::V_PK_MAX_NUM_BF16));
2646 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // op_sel
2647 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // neg_lo
2648 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // neg_hi
2649 auto Op0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
2650 Op0->setImm(Op0->getImm() | SISrcMods::OP_SEL_1);
2651 auto Op1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
2652 Op1->setImm(Op1->getImm() | SISrcMods::OP_SEL_1);
2653 break;
2654 }
2655
2656 case AMDGPU::GET_STACK_BASE:
2657 // The stack starts at offset 0 unless we need to reserve some space at the
2658 // bottom.
2659 if (ST.getFrameLowering()->mayReserveScratchForCWSR(MF: *MBB.getParent())) {
2660 // When CWSR is used in dynamic VGPR mode, the trap handler needs to save
2661 // some of the VGPRs. The size of the required scratch space has already
2662 // been computed by prolog epilog insertion.
2663 const SIMachineFunctionInfo *MFI =
2664 MBB.getParent()->getInfo<SIMachineFunctionInfo>();
2665 unsigned VGPRSize = MFI->getScratchReservedForDynamicVGPRs();
2666 Register DestReg = MI.getOperand(i: 0).getReg();
2667 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETREG_B32), DestReg)
2668 .addImm(Val: AMDGPU::Hwreg::HwregEncoding::encode(
2669 Values: AMDGPU::Hwreg::ID_HW_ID2, Values: AMDGPU::Hwreg::OFFSET_ME_ID, Values: 2));
2670 // The MicroEngine ID is 0 for the graphics queue, and 1 or 2 for compute
2671 // (3 is unused, so we ignore it). Unfortunately, S_GETREG doesn't set
2672 // SCC, so we need to check for 0 manually.
2673 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U32)).addImm(Val: 0).addReg(RegNo: DestReg);
2674 // Change the implicif-def of SCC to an explicit use (but first remove
2675 // the dead flag if present).
2676 MI.getOperand(i: MI.getNumExplicitOperands()).setIsDead(false);
2677 MI.getOperand(i: MI.getNumExplicitOperands()).setIsUse();
2678 MI.setDesc(get(Opcode: AMDGPU::S_CMOVK_I32));
2679 MI.addOperand(Op: MachineOperand::CreateImm(Val: VGPRSize));
2680 } else {
2681 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B32));
2682 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2683 MI.removeOperand(
2684 OpNo: MI.getNumExplicitOperands()); // Drop implicit def of SCC.
2685 }
2686 break;
2687 }
2688
2689 return true;
2690}
2691
2692void SIInstrInfo::reMaterialize(MachineBasicBlock &MBB,
2693 MachineBasicBlock::iterator I, Register DestReg,
2694 unsigned SubIdx, const MachineInstr &Orig,
2695 LaneBitmask UsedLanes) const {
2696
2697 // Try shrinking the instruction to remat only the part needed for current
2698 // context.
2699 // TODO: Handle more cases.
2700 unsigned Opcode = Orig.getOpcode();
2701 switch (Opcode) {
2702 case AMDGPU::S_MOV_B64:
2703 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2704 if (SubIdx != 0)
2705 break;
2706
2707 if (!Orig.getOperand(i: 1).isImm())
2708 break;
2709
2710 // Shrink S_MOV_B64 to S_MOV_B32 when UsedLanes indicates only a single
2711 // 32-bit lane of the 64-bit value is live at the rematerialization point.
2712 if (UsedLanes.all())
2713 break;
2714
2715 // Determine which half of the 64-bit immediate corresponds to the use.
2716 unsigned OrigSubReg = Orig.getOperand(i: 0).getSubReg();
2717 unsigned LoSubReg = RI.composeSubRegIndices(a: OrigSubReg, b: AMDGPU::sub0);
2718 unsigned HiSubReg = RI.composeSubRegIndices(a: OrigSubReg, b: AMDGPU::sub1);
2719
2720 bool NeedLo = (UsedLanes & RI.getSubRegIndexLaneMask(SubIdx: LoSubReg)).any();
2721 bool NeedHi = (UsedLanes & RI.getSubRegIndexLaneMask(SubIdx: HiSubReg)).any();
2722
2723 if (NeedLo && NeedHi)
2724 break;
2725
2726 int64_t Imm64 = Orig.getOperand(i: 1).getImm();
2727 int32_t Imm32 = NeedLo ? Lo_32(Value: Imm64) : Hi_32(Value: Imm64);
2728
2729 unsigned UseSubReg = NeedLo ? LoSubReg : HiSubReg;
2730
2731 // Emit S_MOV_B32 defining just the needed 32-bit subreg of DestReg.
2732 BuildMI(BB&: MBB, I, MIMD: Orig.getDebugLoc(), MCID: get(Opcode: AMDGPU::S_MOV_B32))
2733 .addReg(RegNo: DestReg, Flags: RegState::Define | RegState::Undef, SubReg: UseSubReg)
2734 .addImm(Val: Imm32);
2735 return;
2736 }
2737
2738 case AMDGPU::S_LOAD_DWORDX16_IMM:
2739 case AMDGPU::S_LOAD_DWORDX8_IMM: {
2740 if (SubIdx != 0)
2741 break;
2742
2743 if (I == MBB.end())
2744 break;
2745
2746 if (I->isBundled())
2747 break;
2748
2749 // Look for a single use of the register that is also a subreg.
2750 Register RegToFind = Orig.getOperand(i: 0).getReg();
2751 MachineOperand *UseMO = nullptr;
2752 for (auto &CandMO : I->operands()) {
2753 if (!CandMO.isReg() || CandMO.getReg() != RegToFind || CandMO.isDef())
2754 continue;
2755 if (UseMO) {
2756 UseMO = nullptr;
2757 break;
2758 }
2759 UseMO = &CandMO;
2760 }
2761 if (!UseMO || UseMO->getSubReg() == AMDGPU::NoSubRegister)
2762 break;
2763
2764 unsigned Offset = RI.getSubRegIdxOffset(Idx: UseMO->getSubReg());
2765 unsigned SubregSize = RI.getSubRegIdxSize(Idx: UseMO->getSubReg());
2766
2767 MachineFunction *MF = MBB.getParent();
2768 MachineRegisterInfo &MRI = MF->getRegInfo();
2769 assert(MRI.use_nodbg_empty(DestReg) && "DestReg should have no users yet.");
2770
2771 unsigned NewOpcode = -1;
2772 if (SubregSize == 256)
2773 NewOpcode = AMDGPU::S_LOAD_DWORDX8_IMM;
2774 else if (SubregSize == 128)
2775 NewOpcode = AMDGPU::S_LOAD_DWORDX4_IMM;
2776 else
2777 break;
2778
2779 const MCInstrDesc &TID = get(Opcode: NewOpcode);
2780 const TargetRegisterClass *NewRC =
2781 RI.getAllocatableClass(RC: getRegClass(MCID: TID, OpNum: 0));
2782 MRI.setRegClass(Reg: DestReg, RC: NewRC);
2783
2784 UseMO->setReg(DestReg);
2785 UseMO->setSubReg(AMDGPU::NoSubRegister);
2786
2787 // Use a smaller load with the desired size, possibly with updated offset.
2788 MachineInstr *MI = MF->CloneMachineInstr(Orig: &Orig);
2789 MI->setDesc(TID);
2790 MI->getOperand(i: 0).setReg(DestReg);
2791 MI->getOperand(i: 0).setSubReg(AMDGPU::NoSubRegister);
2792 if (Offset) {
2793 MachineOperand *OffsetMO = getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::offset);
2794 int64_t FinalOffset = OffsetMO->getImm() + Offset / 8;
2795 OffsetMO->setImm(FinalOffset);
2796 }
2797 SmallVector<MachineMemOperand *> NewMMOs;
2798 for (const MachineMemOperand *MemOp : Orig.memoperands())
2799 NewMMOs.push_back(Elt: MF->getMachineMemOperand(MMO: MemOp, PtrInfo: MemOp->getPointerInfo(),
2800 Size: SubregSize / 8));
2801 MI->setMemRefs(MF&: *MF, MemRefs: NewMMOs);
2802
2803 MBB.insert(I, MI);
2804 return;
2805 }
2806
2807 default:
2808 break;
2809 }
2810
2811 TargetInstrInfo::reMaterialize(MBB, MI: I, DestReg, SubIdx, Orig, UsedLanes);
2812}
2813
2814std::pair<MachineInstr*, MachineInstr*>
2815SIInstrInfo::expandMovDPP64(MachineInstr &MI) const {
2816 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
2817
2818 if (ST.hasVMovB64Inst() && ST.hasFeature(Feature: AMDGPU::FeatureDPALU_DPP) &&
2819 AMDGPU::isLegalDPALU_DPPControl(
2820 ST, DC: getNamedOperand(MI, OperandName: AMDGPU::OpName::dpp_ctrl)->getImm())) {
2821 MI.setDesc(get(Opcode: AMDGPU::V_MOV_B64_dpp));
2822 return std::pair(&MI, nullptr);
2823 }
2824
2825 MachineBasicBlock &MBB = *MI.getParent();
2826 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2827 MachineFunction *MF = MBB.getParent();
2828 MachineRegisterInfo &MRI = MF->getRegInfo();
2829 Register Dst = MI.getOperand(i: 0).getReg();
2830 unsigned Part = 0;
2831 MachineInstr *Split[2];
2832
2833 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
2834 auto MovDPP = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_dpp));
2835 if (Dst.isPhysical()) {
2836 MovDPP.addDef(RegNo: RI.getSubReg(Reg: Dst, Idx: Sub));
2837 } else {
2838 assert(MRI.isSSA());
2839 auto Tmp = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2840 MovDPP.addDef(RegNo: Tmp);
2841 }
2842
2843 for (unsigned I = 1; I <= 2; ++I) { // old and src operands.
2844 const MachineOperand &SrcOp = MI.getOperand(i: I);
2845 assert(!SrcOp.isFPImm());
2846 if (SrcOp.isImm()) {
2847 APInt Imm(64, SrcOp.getImm());
2848 Imm.ashrInPlace(ShiftAmt: Part * 32);
2849 MovDPP.addImm(Val: Imm.getLoBits(numBits: 32).getZExtValue());
2850 } else {
2851 assert(SrcOp.isReg());
2852 Register Src = SrcOp.getReg();
2853 if (Src.isPhysical())
2854 MovDPP.addReg(RegNo: RI.getSubReg(Reg: Src, Idx: Sub));
2855 else
2856 MovDPP.addReg(RegNo: Src, Flags: getUndefRegState(B: SrcOp.isUndef()), SubReg: Sub);
2857 }
2858 }
2859
2860 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.explicit_operands(), N: 3))
2861 MovDPP.addImm(Val: MO.getImm());
2862
2863 Split[Part] = MovDPP;
2864 ++Part;
2865 }
2866
2867 if (Dst.isVirtual())
2868 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
2869 .addReg(RegNo: Split[0]->getOperand(i: 0).getReg())
2870 .addImm(Val: AMDGPU::sub0)
2871 .addReg(RegNo: Split[1]->getOperand(i: 0).getReg())
2872 .addImm(Val: AMDGPU::sub1);
2873
2874 MI.eraseFromParent();
2875 return std::pair(Split[0], Split[1]);
2876}
2877
2878std::optional<DestSourcePair>
2879SIInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
2880 if (MI.getOpcode() == AMDGPU::WWM_COPY)
2881 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2882
2883 return std::nullopt;
2884}
2885
2886bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,
2887 AMDGPU::OpName Src0OpName,
2888 MachineOperand &Src1,
2889 AMDGPU::OpName Src1OpName) const {
2890 MachineOperand *Src0Mods = getNamedOperand(MI, OperandName: Src0OpName);
2891 if (!Src0Mods)
2892 return false;
2893
2894 MachineOperand *Src1Mods = getNamedOperand(MI, OperandName: Src1OpName);
2895 assert(Src1Mods &&
2896 "All commutable instructions have both src0 and src1 modifiers");
2897
2898 int Src0ModsVal = Src0Mods->getImm();
2899 int Src1ModsVal = Src1Mods->getImm();
2900
2901 Src1Mods->setImm(Src0ModsVal);
2902 Src0Mods->setImm(Src1ModsVal);
2903 return true;
2904}
2905
2906static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI,
2907 MachineOperand &RegOp,
2908 MachineOperand &NonRegOp) {
2909 Register Reg = RegOp.getReg();
2910 unsigned SubReg = RegOp.getSubReg();
2911 bool IsKill = RegOp.isKill();
2912 bool IsDead = RegOp.isDead();
2913 bool IsUndef = RegOp.isUndef();
2914 bool IsDebug = RegOp.isDebug();
2915
2916 if (NonRegOp.isImm())
2917 RegOp.ChangeToImmediate(ImmVal: NonRegOp.getImm());
2918 else if (NonRegOp.isFI())
2919 RegOp.ChangeToFrameIndex(Idx: NonRegOp.getIndex());
2920 else if (NonRegOp.isGlobal()) {
2921 RegOp.ChangeToGA(GV: NonRegOp.getGlobal(), Offset: NonRegOp.getOffset(),
2922 TargetFlags: NonRegOp.getTargetFlags());
2923 } else
2924 return nullptr;
2925
2926 // Make sure we don't reinterpret a subreg index in the target flags.
2927 RegOp.setTargetFlags(NonRegOp.getTargetFlags());
2928
2929 NonRegOp.ChangeToRegister(Reg, isDef: false, isImp: false, isKill: IsKill, isDead: IsDead, isUndef: IsUndef, isDebug: IsDebug);
2930 NonRegOp.setSubReg(SubReg);
2931
2932 return &MI;
2933}
2934
2935static MachineInstr *swapImmOperands(MachineInstr &MI,
2936 MachineOperand &NonRegOp1,
2937 MachineOperand &NonRegOp2) {
2938 unsigned TargetFlags = NonRegOp1.getTargetFlags();
2939 int64_t NonRegVal = NonRegOp1.getImm();
2940
2941 NonRegOp1.setImm(NonRegOp2.getImm());
2942 NonRegOp2.setImm(NonRegVal);
2943 NonRegOp1.setTargetFlags(NonRegOp2.getTargetFlags());
2944 NonRegOp2.setTargetFlags(TargetFlags);
2945 return &MI;
2946}
2947
2948bool SIInstrInfo::isLegalToSwap(const MachineInstr &MI, unsigned OpIdx0,
2949 unsigned OpIdx1) const {
2950 const MCInstrDesc &InstDesc = MI.getDesc();
2951 const MCOperandInfo &OpInfo0 = InstDesc.operands()[OpIdx0];
2952 const MCOperandInfo &OpInfo1 = InstDesc.operands()[OpIdx1];
2953
2954 unsigned Opc = MI.getOpcode();
2955 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
2956
2957 const MachineOperand &MO0 = MI.getOperand(i: OpIdx0);
2958 const MachineOperand &MO1 = MI.getOperand(i: OpIdx1);
2959
2960 // Swap doesn't breach constant bus or literal limits
2961 // It may move literal to position other than src0, this is not allowed
2962 // pre-gfx10 However, most test cases need literals in Src0 for VOP
2963 // FIXME: After gfx9, literal can be in place other than Src0
2964 if (isVALU(MI, /*AllowLDSDMA=*/false)) {
2965 if ((int)OpIdx0 == Src0Idx && !MO0.isReg() &&
2966 !isInlineConstant(MO: MO0, OpInfo: OpInfo1))
2967 return false;
2968 if ((int)OpIdx1 == Src0Idx && !MO1.isReg() &&
2969 !isInlineConstant(MO: MO1, OpInfo: OpInfo0))
2970 return false;
2971 }
2972
2973 if ((int)OpIdx1 != Src0Idx && MO0.isReg()) {
2974 if (OpInfo1.RegClass == -1)
2975 return OpInfo1.OperandType == MCOI::OPERAND_UNKNOWN;
2976 return isLegalRegOperand(MI, OpIdx: OpIdx1, MO: MO0) &&
2977 (!MO1.isReg() || isLegalRegOperand(MI, OpIdx: OpIdx0, MO: MO1));
2978 }
2979 if ((int)OpIdx0 != Src0Idx && MO1.isReg()) {
2980 if (OpInfo0.RegClass == -1)
2981 return OpInfo0.OperandType == MCOI::OPERAND_UNKNOWN;
2982 return (!MO0.isReg() || isLegalRegOperand(MI, OpIdx: OpIdx1, MO: MO0)) &&
2983 isLegalRegOperand(MI, OpIdx: OpIdx0, MO: MO1);
2984 }
2985
2986 // No need to check 64-bit literals since swapping does not bring new
2987 // 64-bit literals into current instruction to fold to 32-bit
2988
2989 return isImmOperandLegal(MI, OpNo: OpIdx1, MO: MO0);
2990}
2991
2992bool SIInstrInfo::isNonCommutableDPP(const MachineInstr &MI) const {
2993 if (!isDPP(MI))
2994 return false;
2995 const MachineOperand *DppCtrl = getNamedOperand(MI, OperandName: AMDGPU::OpName::dpp_ctrl);
2996 return !DppCtrl || DppCtrl->getImm() != AMDGPU::DPP::QUAD_PERM_ID;
2997}
2998
2999MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
3000 unsigned Src0Idx,
3001 unsigned Src1Idx) const {
3002 assert(!NewMI && "this should never be used");
3003
3004 if (isNonCommutableDPP(MI))
3005 return nullptr;
3006
3007 unsigned Opc = MI.getOpcode();
3008 int CommutedOpcode = commuteOpcode(Opcode: Opc);
3009 if (CommutedOpcode == -1)
3010 return nullptr;
3011
3012 if (Src0Idx > Src1Idx)
3013 std::swap(a&: Src0Idx, b&: Src1Idx);
3014
3015 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
3016 static_cast<int>(Src0Idx) &&
3017 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
3018 static_cast<int>(Src1Idx) &&
3019 "inconsistency with findCommutedOpIndices");
3020
3021 if (!isLegalToSwap(MI, OpIdx0: Src0Idx, OpIdx1: Src1Idx))
3022 return nullptr;
3023
3024 MachineInstr *CommutedMI = nullptr;
3025 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
3026 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
3027 if (Src0.isReg() && Src1.isReg()) {
3028 // Be sure to copy the source modifiers to the right place.
3029 CommutedMI =
3030 TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1: Src0Idx, OpIdx2: Src1Idx);
3031 } else if (Src0.isReg() && !Src1.isReg()) {
3032 CommutedMI = swapRegAndNonRegOperand(MI, RegOp&: Src0, NonRegOp&: Src1);
3033 } else if (!Src0.isReg() && Src1.isReg()) {
3034 CommutedMI = swapRegAndNonRegOperand(MI, RegOp&: Src1, NonRegOp&: Src0);
3035 } else if (Src0.isImm() && Src1.isImm()) {
3036 CommutedMI = swapImmOperands(MI, NonRegOp1&: Src0, NonRegOp2&: Src1);
3037 } else {
3038 // FIXME: Found two non registers to commute. This does happen.
3039 return nullptr;
3040 }
3041
3042 if (CommutedMI) {
3043 swapSourceModifiers(MI, Src0, Src0OpName: AMDGPU::OpName::src0_modifiers,
3044 Src1, Src1OpName: AMDGPU::OpName::src1_modifiers);
3045
3046 swapSourceModifiers(MI, Src0, Src0OpName: AMDGPU::OpName::src0_sel, Src1,
3047 Src1OpName: AMDGPU::OpName::src1_sel);
3048
3049 CommutedMI->setDesc(get(Opcode: CommutedOpcode));
3050 }
3051
3052 return CommutedMI;
3053}
3054
3055// This needs to be implemented because the source modifiers may be inserted
3056// between the true commutable operands, and the base
3057// TargetInstrInfo::commuteInstruction uses it.
3058bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
3059 unsigned &SrcOpIdx0,
3060 unsigned &SrcOpIdx1) const {
3061 if (isNonCommutableDPP(MI))
3062 return false;
3063
3064 return findCommutedOpIndices(Desc: MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
3065}
3066
3067bool SIInstrInfo::findCommutedOpIndices(const MCInstrDesc &Desc,
3068 unsigned &SrcOpIdx0,
3069 unsigned &SrcOpIdx1) const {
3070 if (!Desc.isCommutable())
3071 return false;
3072
3073 unsigned Opc = Desc.getOpcode();
3074 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
3075 if (Src0Idx == -1)
3076 return false;
3077
3078 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
3079 if (Src1Idx == -1)
3080 return false;
3081
3082 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx0, ResultIdx2&: SrcOpIdx1, CommutableOpIdx1: Src0Idx, CommutableOpIdx2: Src1Idx);
3083}
3084
3085bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
3086 int64_t BrOffset) const {
3087 // BranchRelaxation should never have to check s_setpc_b64 or s_add_pc_i64
3088 // because its dest block is unanalyzable.
3089 assert(isSOPP(BranchOp) || isSOPK(BranchOp));
3090
3091 // Convert to dwords.
3092 BrOffset /= 4;
3093
3094 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
3095 // from the next instruction.
3096 BrOffset -= 1;
3097
3098 return isIntN(N: BranchOffsetBits, x: BrOffset);
3099}
3100
3101MachineBasicBlock *
3102SIInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
3103 return MI.getOperand(i: 0).getMBB();
3104}
3105
3106bool SIInstrInfo::hasDivergentBranch(const MachineBasicBlock *MBB) const {
3107 for (const MachineInstr &MI : MBB->terminators()) {
3108 if (MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE ||
3109 MI.getOpcode() == AMDGPU::SI_LOOP ||
3110 MI.getOpcode() == AMDGPU::SI_WATERFALL_LOOP)
3111 return true;
3112 }
3113 return false;
3114}
3115
3116void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
3117 MachineBasicBlock &DestBB,
3118 MachineBasicBlock &RestoreBB,
3119 const DebugLoc &DL, int64_t BrOffset,
3120 RegScavenger *RS) const {
3121 assert(MBB.empty() &&
3122 "new block should be inserted for expanding unconditional branch");
3123 assert(MBB.pred_size() == 1);
3124 assert(RestoreBB.empty() &&
3125 "restore block should be inserted for restoring clobbered registers");
3126
3127 MachineFunction *MF = MBB.getParent();
3128 MachineRegisterInfo &MRI = MF->getRegInfo();
3129 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
3130 auto I = MBB.end();
3131 auto &MCCtx = MF->getContext();
3132
3133 if (ST.useAddPC64Inst()) {
3134 MCSymbol *Offset =
3135 MCCtx.createTempSymbol(Name: "offset", /*AlwaysAddSuffix=*/true);
3136 auto AddPC = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_PC_I64))
3137 .addSym(Sym: Offset, TargetFlags: MO_FAR_BRANCH_OFFSET);
3138 MCSymbol *PostAddPCLabel =
3139 MCCtx.createTempSymbol(Name: "post_addpc", /*AlwaysAddSuffix=*/true);
3140 AddPC->setPostInstrSymbol(MF&: *MF, Symbol: PostAddPCLabel);
3141 auto *OffsetExpr = MCBinaryExpr::createSub(
3142 LHS: MCSymbolRefExpr::create(Symbol: DestBB.getSymbol(), Ctx&: MCCtx),
3143 RHS: MCSymbolRefExpr::create(Symbol: PostAddPCLabel, Ctx&: MCCtx), Ctx&: MCCtx);
3144 Offset->setVariableValue(OffsetExpr);
3145 return;
3146 }
3147
3148 assert(RS && "RegScavenger required for long branching");
3149
3150 // FIXME: Virtual register workaround for RegScavenger not working with empty
3151 // blocks.
3152 Register PCReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
3153
3154 // Note: as this is used after hazard recognizer we need to apply some hazard
3155 // workarounds directly.
3156 const bool FlushSGPRWrites = (ST.isWave64() && ST.hasVALUMaskWriteHazard()) ||
3157 ST.hasVALUReadSGPRHazard();
3158 auto ApplyHazardWorkarounds = [this, &MBB, &I, &DL, FlushSGPRWrites]() {
3159 if (FlushSGPRWrites)
3160 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
3161 .addImm(Val: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST));
3162 };
3163
3164 // We need to compute the offset relative to the instruction immediately after
3165 // s_getpc_b64. Insert pc arithmetic code before last terminator.
3166 MachineInstr *GetPC = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: PCReg);
3167 ApplyHazardWorkarounds();
3168
3169 MCSymbol *PostGetPCLabel =
3170 MCCtx.createTempSymbol(Name: "post_getpc", /*AlwaysAddSuffix=*/true);
3171 GetPC->setPostInstrSymbol(MF&: *MF, Symbol: PostGetPCLabel);
3172
3173 MCSymbol *OffsetLo =
3174 MCCtx.createTempSymbol(Name: "offset_lo", /*AlwaysAddSuffix=*/true);
3175 MCSymbol *OffsetHi =
3176 MCCtx.createTempSymbol(Name: "offset_hi", /*AlwaysAddSuffix=*/true);
3177 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U32))
3178 .addReg(RegNo: PCReg, Flags: RegState::Define, SubReg: AMDGPU::sub0)
3179 .addReg(RegNo: PCReg, Flags: {}, SubReg: AMDGPU::sub0)
3180 .addSym(Sym: OffsetLo, TargetFlags: MO_FAR_BRANCH_OFFSET);
3181 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADDC_U32))
3182 .addReg(RegNo: PCReg, Flags: RegState::Define, SubReg: AMDGPU::sub1)
3183 .addReg(RegNo: PCReg, Flags: {}, SubReg: AMDGPU::sub1)
3184 .addSym(Sym: OffsetHi, TargetFlags: MO_FAR_BRANCH_OFFSET);
3185 ApplyHazardWorkarounds();
3186
3187 // Insert the indirect branch after the other terminator.
3188 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETPC_B64))
3189 .addReg(RegNo: PCReg);
3190
3191 // If a spill is needed for the pc register pair, we need to insert a spill
3192 // restore block right before the destination block, and insert a short branch
3193 // into the old destination block's fallthrough predecessor.
3194 // e.g.:
3195 //
3196 // s_cbranch_scc0 skip_long_branch:
3197 //
3198 // long_branch_bb:
3199 // spill s[8:9]
3200 // s_getpc_b64 s[8:9]
3201 // s_add_u32 s8, s8, restore_bb
3202 // s_addc_u32 s9, s9, 0
3203 // s_setpc_b64 s[8:9]
3204 //
3205 // skip_long_branch:
3206 // foo;
3207 //
3208 // .....
3209 //
3210 // dest_bb_fallthrough_predecessor:
3211 // bar;
3212 // s_branch dest_bb
3213 //
3214 // restore_bb:
3215 // restore s[8:9]
3216 // fallthrough dest_bb
3217 ///
3218 // dest_bb:
3219 // buzz;
3220
3221 Register LongBranchReservedReg = MFI->getLongBranchReservedReg();
3222 Register Scav;
3223
3224 // If we've previously reserved a register for long branches
3225 // avoid running the scavenger and just use those registers
3226 if (LongBranchReservedReg) {
3227 RS->enterBasicBlock(MBB);
3228 Scav = LongBranchReservedReg;
3229 } else {
3230 RS->enterBasicBlockEnd(MBB);
3231 Scav = RS->scavengeRegisterBackwards(
3232 RC: AMDGPU::SReg_64RegClass, To: MachineBasicBlock::iterator(GetPC),
3233 /* RestoreAfter */ false, SPAdj: 0, /* AllowSpill */ false);
3234 }
3235 if (Scav) {
3236 RS->setRegUsed(Reg: Scav);
3237 MRI.replaceRegWith(FromReg: PCReg, ToReg: Scav);
3238 MRI.clearVirtRegs();
3239 } else {
3240 // As SGPR needs VGPR to be spilled, we reuse the slot of temporary VGPR for
3241 // SGPR spill.
3242 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3243 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3244 TRI->spillEmergencySGPR(MI: GetPC, RestoreMBB&: RestoreBB, SGPR: AMDGPU::SGPR0_SGPR1, RS);
3245 MRI.replaceRegWith(FromReg: PCReg, ToReg: AMDGPU::SGPR0_SGPR1);
3246 MRI.clearVirtRegs();
3247 }
3248
3249 MCSymbol *DestLabel = Scav ? DestBB.getSymbol() : RestoreBB.getSymbol();
3250 // Now, the distance could be defined.
3251 auto *Offset = MCBinaryExpr::createSub(
3252 LHS: MCSymbolRefExpr::create(Symbol: DestLabel, Ctx&: MCCtx),
3253 RHS: MCSymbolRefExpr::create(Symbol: PostGetPCLabel, Ctx&: MCCtx), Ctx&: MCCtx);
3254 // Add offset assignments.
3255 auto *Mask = MCConstantExpr::create(Value: 0xFFFFFFFFULL, Ctx&: MCCtx);
3256 OffsetLo->setVariableValue(MCBinaryExpr::createAnd(LHS: Offset, RHS: Mask, Ctx&: MCCtx));
3257 auto *ShAmt = MCConstantExpr::create(Value: 32, Ctx&: MCCtx);
3258 OffsetHi->setVariableValue(MCBinaryExpr::createAShr(LHS: Offset, RHS: ShAmt, Ctx&: MCCtx));
3259}
3260
3261unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
3262 switch (Cond) {
3263 case SIInstrInfo::SCC_TRUE:
3264 return AMDGPU::S_CBRANCH_SCC1;
3265 case SIInstrInfo::SCC_FALSE:
3266 return AMDGPU::S_CBRANCH_SCC0;
3267 case SIInstrInfo::VCCNZ:
3268 return AMDGPU::S_CBRANCH_VCCNZ;
3269 case SIInstrInfo::VCCZ:
3270 return AMDGPU::S_CBRANCH_VCCZ;
3271 case SIInstrInfo::EXECNZ:
3272 return AMDGPU::S_CBRANCH_EXECNZ;
3273 case SIInstrInfo::EXECZ:
3274 return AMDGPU::S_CBRANCH_EXECZ;
3275 default:
3276 llvm_unreachable("invalid branch predicate");
3277 }
3278}
3279
3280SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
3281 switch (Opcode) {
3282 case AMDGPU::S_CBRANCH_SCC0:
3283 return SCC_FALSE;
3284 case AMDGPU::S_CBRANCH_SCC1:
3285 return SCC_TRUE;
3286 case AMDGPU::S_CBRANCH_VCCNZ:
3287 return VCCNZ;
3288 case AMDGPU::S_CBRANCH_VCCZ:
3289 return VCCZ;
3290 case AMDGPU::S_CBRANCH_EXECNZ:
3291 return EXECNZ;
3292 case AMDGPU::S_CBRANCH_EXECZ:
3293 return EXECZ;
3294 default:
3295 return INVALID_BR;
3296 }
3297}
3298
3299bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB,
3300 MachineBasicBlock::iterator I,
3301 MachineBasicBlock *&TBB,
3302 MachineBasicBlock *&FBB,
3303 SmallVectorImpl<MachineOperand> &Cond,
3304 bool AllowModify) const {
3305 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3306 // Unconditional Branch
3307 TBB = I->getOperand(i: 0).getMBB();
3308 return false;
3309 }
3310
3311 BranchPredicate Pred = getBranchPredicate(Opcode: I->getOpcode());
3312 if (Pred == INVALID_BR)
3313 return true;
3314
3315 MachineBasicBlock *CondBB = I->getOperand(i: 0).getMBB();
3316 Cond.push_back(Elt: MachineOperand::CreateImm(Val: Pred));
3317 Cond.push_back(Elt: I->getOperand(i: 1)); // Save the branch register.
3318
3319 ++I;
3320
3321 if (I == MBB.end()) {
3322 // Conditional branch followed by fall-through.
3323 TBB = CondBB;
3324 return false;
3325 }
3326
3327 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3328 TBB = CondBB;
3329 FBB = I->getOperand(i: 0).getMBB();
3330 return false;
3331 }
3332
3333 return true;
3334}
3335
3336bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
3337 MachineBasicBlock *&FBB,
3338 SmallVectorImpl<MachineOperand> &Cond,
3339 bool AllowModify) const {
3340 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
3341 auto E = MBB.end();
3342 if (I == E)
3343 return false;
3344
3345 // Skip over the instructions that are artificially terminators for special
3346 // exec management.
3347 while (I != E && !I->isBranch() && !I->isReturn()) {
3348 switch (I->getOpcode()) {
3349 case AMDGPU::S_MOV_B64_term:
3350 case AMDGPU::S_XOR_B64_term:
3351 case AMDGPU::S_OR_B64_term:
3352 case AMDGPU::S_ANDN2_B64_term:
3353 case AMDGPU::S_AND_B64_term:
3354 case AMDGPU::S_AND_SAVEEXEC_B64_term:
3355 case AMDGPU::S_MOV_B32_term:
3356 case AMDGPU::S_XOR_B32_term:
3357 case AMDGPU::S_OR_B32_term:
3358 case AMDGPU::S_ANDN2_B32_term:
3359 case AMDGPU::S_AND_B32_term:
3360 case AMDGPU::S_AND_SAVEEXEC_B32_term:
3361 case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
3362 case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
3363 break;
3364 case AMDGPU::SI_IF:
3365 case AMDGPU::SI_ELSE:
3366 case AMDGPU::SI_KILL_I1_TERMINATOR:
3367 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
3368 // FIXME: It's messy that these need to be considered here at all.
3369 return true;
3370 default:
3371 llvm_unreachable("unexpected non-branch terminator inst");
3372 }
3373
3374 ++I;
3375 }
3376
3377 if (I == E)
3378 return false;
3379
3380 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
3381}
3382
3383unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB,
3384 int *BytesRemoved) const {
3385 unsigned Count = 0;
3386 unsigned RemovedSize = 0;
3387 for (MachineInstr &MI : llvm::make_early_inc_range(Range: MBB.terminators())) {
3388 // Skip over artificial terminators when removing instructions.
3389 if (MI.isBranch() || MI.isReturn()) {
3390 RemovedSize += getInstSizeInBytes(MI);
3391 MI.eraseFromParent();
3392 ++Count;
3393 }
3394 }
3395
3396 if (BytesRemoved)
3397 *BytesRemoved = RemovedSize;
3398
3399 return Count;
3400}
3401
3402// Copy the flags onto the implicit condition register operand.
3403static void preserveCondRegFlags(MachineOperand &CondReg,
3404 const MachineOperand &OrigCond) {
3405 CondReg.setIsUndef(OrigCond.isUndef());
3406 CondReg.setIsKill(OrigCond.isKill());
3407}
3408
3409unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,
3410 MachineBasicBlock *TBB,
3411 MachineBasicBlock *FBB,
3412 ArrayRef<MachineOperand> Cond,
3413 const DebugLoc &DL,
3414 int *BytesAdded) const {
3415 if (!FBB && Cond.empty()) {
3416 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
3417 .addMBB(MBB: TBB);
3418 if (BytesAdded)
3419 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3420 return 1;
3421 }
3422
3423 assert(TBB && Cond[0].isImm());
3424
3425 unsigned Opcode
3426 = getBranchOpcode(Cond: static_cast<BranchPredicate>(Cond[0].getImm()));
3427
3428 if (!FBB) {
3429 MachineInstr *CondBr =
3430 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode))
3431 .addMBB(MBB: TBB);
3432
3433 // Copy the flags onto the implicit condition register operand.
3434 preserveCondRegFlags(CondReg&: CondBr->getOperand(i: 1), OrigCond: Cond[1]);
3435 fixImplicitOperands(MI&: *CondBr);
3436
3437 if (BytesAdded)
3438 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3439 return 1;
3440 }
3441
3442 assert(TBB && FBB);
3443
3444 MachineInstr *CondBr =
3445 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode))
3446 .addMBB(MBB: TBB);
3447 fixImplicitOperands(MI&: *CondBr);
3448 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
3449 .addMBB(MBB: FBB);
3450
3451 MachineOperand &CondReg = CondBr->getOperand(i: 1);
3452 CondReg.setIsUndef(Cond[1].isUndef());
3453 CondReg.setIsKill(Cond[1].isKill());
3454
3455 if (BytesAdded)
3456 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8;
3457
3458 return 2;
3459}
3460
3461bool SIInstrInfo::reverseBranchCondition(
3462 SmallVectorImpl<MachineOperand> &Cond) const {
3463 if (Cond.size() != 2) {
3464 return true;
3465 }
3466
3467 if (Cond[0].isImm()) {
3468 Cond[0].setImm(-Cond[0].getImm());
3469 return false;
3470 }
3471
3472 return true;
3473}
3474
3475namespace {
3476class AMDGPUPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
3477private:
3478 /// The compare instruction for loop control
3479 const MachineInstr *CmpInst = nullptr;
3480 /// The normalized condition used by createTripCountGreaterCondition()
3481 SmallVector<MachineOperand, 2> Cond;
3482
3483public:
3484 AMDGPUPipelinerLoopInfo(MachineInstr *CmpInst,
3485 const SmallVectorImpl<MachineOperand> &Cond)
3486 : CmpInst(CmpInst), Cond(Cond.begin(), Cond.end()) {}
3487
3488 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
3489 return CmpInst && MI == CmpInst;
3490 }
3491
3492 std::optional<bool> createTripCountGreaterCondition(
3493 int TC, MachineBasicBlock &MBB,
3494 SmallVectorImpl<MachineOperand> &CondParam) override {
3495 CondParam = this->Cond;
3496 return {};
3497 }
3498
3499 void adjustTripCount(int TripCountAdjust) override {}
3500
3501 void setPreheader(MachineBasicBlock *NewPreheader) override {}
3502};
3503} // namespace
3504
3505std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
3506SIInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
3507 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
3508 SmallVector<MachineOperand, 2> Cond;
3509 // Unanalyzable terminator.
3510 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))
3511 return nullptr;
3512
3513 // Infinite loops are not supported.
3514 if (TBB == LoopBB && FBB == LoopBB)
3515 return nullptr;
3516
3517 // Must be conditional branch.
3518 if (FBB == nullptr)
3519 return nullptr;
3520
3521 assert((TBB == LoopBB || FBB == LoopBB) &&
3522 "The Loop must be a single-basic-block loop");
3523
3524 // Divergent (VCC/EXEC) back-edge is not supported.
3525 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
3526 if (Pred != SCC_TRUE && Pred != SCC_FALSE)
3527 return nullptr;
3528
3529 // Calls and inline assembly are not supported.
3530 for (const MachineInstr &MI : *LoopBB)
3531 if (MI.isCall() || MI.isInlineAsm())
3532 return nullptr;
3533
3534 // Normalization for createTripCountGreaterCondition(): make Cond mean
3535 // "exit the loop" so the expander emits correct prolog guard branches.
3536 if (TBB == LoopBB)
3537 reverseBranchCondition(Cond);
3538
3539 auto Instructions = make_range(
3540 x: MachineBasicBlock::reverse_iterator(LoopBB->getFirstTerminator()),
3541 y: LoopBB->rend());
3542 auto CmpI = llvm::find_if(Range&: Instructions, P: [&](const MachineInstr &MI) {
3543 return MI.modifiesRegister(Reg: Cond[1].getReg(), TRI: &RI);
3544 });
3545
3546 if (CmpI == Instructions.end() || CmpI->isPHI())
3547 return nullptr;
3548 MachineInstr *CmpInst = &*CmpI;
3549
3550 return std::make_unique<AMDGPUPipelinerLoopInfo>(args&: CmpInst, args&: Cond);
3551}
3552
3553bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
3554 ArrayRef<MachineOperand> Cond,
3555 Register DstReg, Register TrueReg,
3556 Register FalseReg, int &CondCycles,
3557 int &TrueCycles, int &FalseCycles) const {
3558 switch (Cond[0].getImm()) {
3559 case VCCNZ:
3560 case VCCZ: {
3561 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3562 const TargetRegisterClass *RC = MRI.getRegClass(Reg: TrueReg);
3563 if (MRI.getRegClass(Reg: FalseReg) != RC)
3564 return false;
3565
3566 int NumInsts = AMDGPU::getRegBitWidth(RC: *RC) / 32;
3567 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
3568
3569 // Limit to equal cost for branch vs. N v_cndmask_b32s.
3570 return RI.hasVGPRs(RC) && NumInsts <= 6;
3571 }
3572 case SCC_TRUE:
3573 case SCC_FALSE: {
3574 // FIXME: We could insert for VGPRs if we could replace the original compare
3575 // with a vector one.
3576 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3577 const TargetRegisterClass *RC = MRI.getRegClass(Reg: TrueReg);
3578 if (MRI.getRegClass(Reg: FalseReg) != RC)
3579 return false;
3580
3581 int NumInsts = AMDGPU::getRegBitWidth(RC: *RC) / 32;
3582
3583 // Multiples of 8 can do s_cselect_b64
3584 if (NumInsts % 2 == 0)
3585 NumInsts /= 2;
3586
3587 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
3588 return RI.isSGPRClass(RC);
3589 }
3590 default:
3591 return false;
3592 }
3593}
3594
3595void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,
3596 MachineBasicBlock::iterator I, const DebugLoc &DL,
3597 Register DstReg, ArrayRef<MachineOperand> Cond,
3598 Register TrueReg, Register FalseReg) const {
3599 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
3600 if (Pred == VCCZ || Pred == SCC_FALSE) {
3601 Pred = static_cast<BranchPredicate>(-Pred);
3602 std::swap(a&: TrueReg, b&: FalseReg);
3603 }
3604
3605 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3606 const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: DstReg);
3607 unsigned DstSize = RI.getRegSizeInBits(RC: *DstRC);
3608
3609 if (DstSize == 32) {
3610 MachineInstr *Select;
3611 if (Pred == SCC_TRUE) {
3612 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: DstReg)
3613 .addReg(RegNo: TrueReg)
3614 .addReg(RegNo: FalseReg);
3615 } else {
3616 // Instruction's operands are backwards from what is expected.
3617 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e32), DestReg: DstReg)
3618 .addReg(RegNo: FalseReg)
3619 .addReg(RegNo: TrueReg);
3620 }
3621
3622 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3623 return;
3624 }
3625
3626 if (DstSize == 64 && Pred == SCC_TRUE) {
3627 MachineInstr *Select =
3628 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B64), DestReg: DstReg)
3629 .addReg(RegNo: TrueReg)
3630 .addReg(RegNo: FalseReg);
3631
3632 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3633 return;
3634 }
3635
3636 static const int16_t Sub0_15[] = {
3637 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
3638 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
3639 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
3640 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
3641 };
3642
3643 static const int16_t Sub0_15_64[] = {
3644 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
3645 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
3646 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
3647 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
3648 };
3649
3650 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
3651 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
3652 const int16_t *SubIndices = Sub0_15;
3653 int NElts = DstSize / 32;
3654
3655 // 64-bit select is only available for SALU.
3656 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
3657 if (Pred == SCC_TRUE) {
3658 if (NElts % 2) {
3659 SelOp = AMDGPU::S_CSELECT_B32;
3660 EltRC = &AMDGPU::SGPR_32RegClass;
3661 } else {
3662 SelOp = AMDGPU::S_CSELECT_B64;
3663 EltRC = &AMDGPU::SGPR_64RegClass;
3664 SubIndices = Sub0_15_64;
3665 NElts /= 2;
3666 }
3667 }
3668
3669 MachineInstrBuilder MIB = BuildMI(
3670 BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
3671
3672 I = MIB->getIterator();
3673
3674 SmallVector<Register, 8> Regs;
3675 for (int Idx = 0; Idx != NElts; ++Idx) {
3676 Register DstElt = MRI.createVirtualRegister(RegClass: EltRC);
3677 Regs.push_back(Elt: DstElt);
3678
3679 unsigned SubIdx = SubIndices[Idx];
3680
3681 MachineInstr *Select;
3682 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) {
3683 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SelOp), DestReg: DstElt)
3684 .addReg(RegNo: FalseReg, Flags: {}, SubReg: SubIdx)
3685 .addReg(RegNo: TrueReg, Flags: {}, SubReg: SubIdx);
3686 } else {
3687 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SelOp), DestReg: DstElt)
3688 .addReg(RegNo: TrueReg, Flags: {}, SubReg: SubIdx)
3689 .addReg(RegNo: FalseReg, Flags: {}, SubReg: SubIdx);
3690 }
3691
3692 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3693 fixImplicitOperands(MI&: *Select);
3694
3695 MIB.addReg(RegNo: DstElt)
3696 .addImm(Val: SubIdx);
3697 }
3698}
3699
3700bool SIInstrInfo::isXcntDrain(const MachineInstr &MI) {
3701
3702 if (MI.isBranch() || MI.isCall() || MI.isReturn() || MI.isIndirectBranch())
3703 return true;
3704
3705 switch (MI.getOpcode()) {
3706 case AMDGPU::S_ENDPGM:
3707 case AMDGPU::S_ENDPGM_SAVED:
3708 case AMDGPU::S_TRAP:
3709 case AMDGPU::S_GETREG_B32:
3710 case AMDGPU::S_SETREG_B32:
3711 case AMDGPU::S_SETREG_B32_mode:
3712 case AMDGPU::S_SETREG_IMM32_B32:
3713 case AMDGPU::S_SETREG_IMM32_B32_mode:
3714 case AMDGPU::S_SENDMSG:
3715 case AMDGPU::S_SENDMSGHALT:
3716 case AMDGPU::S_SENDMSG_RTN_B32:
3717 case AMDGPU::S_SENDMSG_RTN_B64:
3718 case AMDGPU::S_BARRIER_WAIT:
3719 case AMDGPU::S_BARRIER_SIGNAL_M0:
3720 case AMDGPU::S_BARRIER_SIGNAL_IMM:
3721 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0:
3722 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM:
3723 return true;
3724 default:
3725 return false;
3726 }
3727}
3728
3729bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) {
3730 switch (MI.getOpcode()) {
3731 case AMDGPU::V_MOV_B16_t16_e32:
3732 case AMDGPU::V_MOV_B16_t16_e64:
3733 case AMDGPU::V_MOV_B32_e32:
3734 case AMDGPU::V_MOV_B32_e64:
3735 case AMDGPU::V_MOV_B64_PSEUDO:
3736 case AMDGPU::V_MOV_B64_e32:
3737 case AMDGPU::V_MOV_B64_e64:
3738 case AMDGPU::S_MOV_B32:
3739 case AMDGPU::S_MOV_B64:
3740 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3741 case AMDGPU::COPY:
3742 case AMDGPU::WWM_COPY:
3743 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3744 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3745 case AMDGPU::V_ACCVGPR_MOV_B32:
3746 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
3747 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
3748 return true;
3749 default:
3750 return false;
3751 }
3752}
3753
3754unsigned SIInstrInfo::getFoldableCopySrcIdx(const MachineInstr &MI) {
3755 switch (MI.getOpcode()) {
3756 case AMDGPU::V_MOV_B16_t16_e32:
3757 case AMDGPU::V_MOV_B16_t16_e64:
3758 return 2;
3759 case AMDGPU::V_MOV_B32_e32:
3760 case AMDGPU::V_MOV_B32_e64:
3761 case AMDGPU::V_MOV_B64_PSEUDO:
3762 case AMDGPU::V_MOV_B64_e32:
3763 case AMDGPU::V_MOV_B64_e64:
3764 case AMDGPU::S_MOV_B32:
3765 case AMDGPU::S_MOV_B64:
3766 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3767 case AMDGPU::COPY:
3768 case AMDGPU::WWM_COPY:
3769 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3770 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3771 case AMDGPU::V_ACCVGPR_MOV_B32:
3772 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
3773 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
3774 return 1;
3775 default:
3776 llvm_unreachable("MI is not a foldable copy");
3777 }
3778}
3779
3780static constexpr AMDGPU::OpName ModifierOpNames[] = {
3781 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
3782 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
3783 AMDGPU::OpName::omod, AMDGPU::OpName::op_sel};
3784
3785void SIInstrInfo::removeModOperands(MachineInstr &MI) const {
3786 unsigned Opc = MI.getOpcode();
3787 for (AMDGPU::OpName Name : reverse(C: ModifierOpNames)) {
3788 int Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name);
3789 if (Idx >= 0)
3790 MI.removeOperand(OpNo: Idx);
3791 }
3792}
3793
3794void SIInstrInfo::mutateAndCleanupImplicit(MachineInstr &MI,
3795 const MCInstrDesc &NewDesc) const {
3796 MI.setDesc(NewDesc);
3797
3798 // Remove any leftover implicit operands from mutating the instruction. e.g.
3799 // if we replace an s_and_b32 with a copy, we don't need the implicit scc def
3800 // anymore.
3801 const MCInstrDesc &Desc = MI.getDesc();
3802 unsigned NumOps = Desc.getNumOperands() + Desc.implicit_uses().size() +
3803 Desc.implicit_defs().size();
3804
3805 for (unsigned I = MI.getNumOperands() - 1; I >= NumOps; --I)
3806 MI.removeOperand(OpNo: I);
3807}
3808
3809std::optional<int64_t> SIInstrInfo::extractSubregFromImm(int64_t Imm,
3810 unsigned SubRegIndex) {
3811 switch (SubRegIndex) {
3812 case AMDGPU::NoSubRegister:
3813 return Imm;
3814 case AMDGPU::sub0:
3815 return SignExtend64<32>(x: Imm);
3816 case AMDGPU::sub1:
3817 return SignExtend64<32>(x: Imm >> 32);
3818 case AMDGPU::lo16:
3819 return SignExtend64<16>(x: Imm);
3820 case AMDGPU::hi16:
3821 return SignExtend64<16>(x: Imm >> 16);
3822 case AMDGPU::sub1_lo16:
3823 return SignExtend64<16>(x: Imm >> 32);
3824 case AMDGPU::sub1_hi16:
3825 return SignExtend64<16>(x: Imm >> 48);
3826 default:
3827 return std::nullopt;
3828 }
3829
3830 llvm_unreachable("covered subregister switch");
3831}
3832
3833static unsigned getNewFMAAKInst(const GCNSubtarget &ST, unsigned Opc) {
3834 switch (Opc) {
3835 case AMDGPU::V_MAC_F16_e32:
3836 case AMDGPU::V_MAC_F16_e64:
3837 case AMDGPU::V_MAD_F16_e64:
3838 return AMDGPU::V_MADAK_F16;
3839 case AMDGPU::V_MAC_F32_e32:
3840 case AMDGPU::V_MAC_F32_e64:
3841 case AMDGPU::V_MAD_F32_e64:
3842 return AMDGPU::V_MADAK_F32;
3843 case AMDGPU::V_FMAC_F32_e32:
3844 case AMDGPU::V_FMAC_F32_e64:
3845 case AMDGPU::V_FMA_F32_e64:
3846 return AMDGPU::V_FMAAK_F32;
3847 case AMDGPU::V_FMAC_F16_e32:
3848 case AMDGPU::V_FMAC_F16_e64:
3849 case AMDGPU::V_FMAC_F16_t16_e64:
3850 case AMDGPU::V_FMAC_F16_fake16_e64:
3851 case AMDGPU::V_FMAC_F16_t16_e32:
3852 case AMDGPU::V_FMAC_F16_fake16_e32:
3853 case AMDGPU::V_FMA_F16_e64:
3854 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
3855 ? AMDGPU::V_FMAAK_F16_t16
3856 : AMDGPU::V_FMAAK_F16_fake16
3857 : AMDGPU::V_FMAAK_F16;
3858 case AMDGPU::V_FMAC_F64_e32:
3859 case AMDGPU::V_FMAC_F64_e64:
3860 case AMDGPU::V_FMA_F64_e64:
3861 return AMDGPU::V_FMAAK_F64;
3862 default:
3863 llvm_unreachable("invalid instruction");
3864 }
3865}
3866
3867static unsigned getNewFMAMKInst(const GCNSubtarget &ST, unsigned Opc) {
3868 switch (Opc) {
3869 case AMDGPU::V_MAC_F16_e32:
3870 case AMDGPU::V_MAC_F16_e64:
3871 case AMDGPU::V_MAD_F16_e64:
3872 return AMDGPU::V_MADMK_F16;
3873 case AMDGPU::V_MAC_F32_e32:
3874 case AMDGPU::V_MAC_F32_e64:
3875 case AMDGPU::V_MAD_F32_e64:
3876 return AMDGPU::V_MADMK_F32;
3877 case AMDGPU::V_FMAC_F32_e32:
3878 case AMDGPU::V_FMAC_F32_e64:
3879 case AMDGPU::V_FMA_F32_e64:
3880 return AMDGPU::V_FMAMK_F32;
3881 case AMDGPU::V_FMAC_F16_e32:
3882 case AMDGPU::V_FMAC_F16_e64:
3883 case AMDGPU::V_FMAC_F16_t16_e64:
3884 case AMDGPU::V_FMAC_F16_fake16_e64:
3885 case AMDGPU::V_FMAC_F16_t16_e32:
3886 case AMDGPU::V_FMAC_F16_fake16_e32:
3887 case AMDGPU::V_FMA_F16_e64:
3888 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
3889 ? AMDGPU::V_FMAMK_F16_t16
3890 : AMDGPU::V_FMAMK_F16_fake16
3891 : AMDGPU::V_FMAMK_F16;
3892 case AMDGPU::V_FMAC_F64_e32:
3893 case AMDGPU::V_FMAC_F64_e64:
3894 case AMDGPU::V_FMA_F64_e64:
3895 return AMDGPU::V_FMAMK_F64;
3896 default:
3897 llvm_unreachable("invalid instruction");
3898 }
3899}
3900
3901bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
3902 Register Reg, MachineRegisterInfo *MRI) const {
3903 int64_t Imm;
3904 if (!getConstValDefinedInReg(MI: DefMI, Reg, ImmVal&: Imm))
3905 return false;
3906
3907 const bool HasMultipleUses = !MRI->hasOneNonDBGUse(RegNo: Reg);
3908
3909 assert(!DefMI.getOperand(0).getSubReg() && "Expected SSA form");
3910
3911 unsigned Opc = UseMI.getOpcode();
3912 if (Opc == AMDGPU::COPY) {
3913 assert(!UseMI.getOperand(0).getSubReg() && "Expected SSA form");
3914
3915 Register DstReg = UseMI.getOperand(i: 0).getReg();
3916 Register UseSubReg = UseMI.getOperand(i: 1).getSubReg();
3917
3918 const TargetRegisterClass *DstRC = RI.getRegClassForReg(MRI: *MRI, Reg: DstReg);
3919
3920 if (HasMultipleUses) {
3921 // TODO: This should fold in more cases with multiple use, but we need to
3922 // more carefully consider what those uses are.
3923 unsigned ImmDefSize = RI.getRegSizeInBits(RC: *MRI->getRegClass(Reg));
3924
3925 // Avoid breaking up a 64-bit inline immediate into a subregister extract.
3926 if (UseSubReg != AMDGPU::NoSubRegister && ImmDefSize == 64)
3927 return false;
3928
3929 // Most of the time folding a 32-bit inline constant is free (though this
3930 // might not be true if we can't later fold it into a real user).
3931 //
3932 // FIXME: This isInlineConstant check is imprecise if
3933 // getConstValDefinedInReg handled the tricky non-mov cases.
3934 if (ImmDefSize == 32 &&
3935 !isInlineConstant(ImmVal: Imm, OperandType: AMDGPU::OPERAND_REG_IMM_INT32))
3936 return false;
3937 }
3938
3939 bool Is16Bit = UseSubReg != AMDGPU::NoSubRegister &&
3940 RI.getSubRegIdxSize(Idx: UseSubReg) == 16;
3941
3942 if (Is16Bit) {
3943 if (RI.hasVGPRs(RC: DstRC))
3944 return false; // Do not clobber vgpr_hi16
3945
3946 if (DstReg.isVirtual() && UseSubReg != AMDGPU::lo16)
3947 return false;
3948 }
3949
3950 MachineFunction *MF = UseMI.getMF();
3951
3952 unsigned NewOpc = AMDGPU::INSTRUCTION_LIST_END;
3953 MCRegister MovDstPhysReg =
3954 DstReg.isPhysical() ? DstReg.asMCReg() : MCRegister();
3955
3956 std::optional<int64_t> SubRegImm = extractSubregFromImm(Imm, SubRegIndex: UseSubReg);
3957
3958 // TODO: Try to fold with AMDGPU::V_MOV_B16_t16_e64
3959 for (unsigned MovOp :
3960 {AMDGPU::S_MOV_B32, AMDGPU::V_MOV_B32_e32, AMDGPU::S_MOV_B64,
3961 AMDGPU::V_MOV_B64_PSEUDO, AMDGPU::V_ACCVGPR_WRITE_B32_e64}) {
3962 const MCInstrDesc &MovDesc = get(Opcode: MovOp);
3963
3964 const TargetRegisterClass *MovDstRC = getRegClass(MCID: MovDesc, OpNum: 0);
3965 if (Is16Bit) {
3966 // We just need to find a correctly sized register class, so the
3967 // subregister index compatibility doesn't matter since we're statically
3968 // extracting the immediate value.
3969 MovDstRC = RI.getMatchingSuperRegClass(A: MovDstRC, B: DstRC, Idx: AMDGPU::lo16);
3970 if (!MovDstRC)
3971 continue;
3972
3973 if (MovDstPhysReg) {
3974 // FIXME: We probably should not do this. If there is a live value in
3975 // the high half of the register, it will be corrupted.
3976 MovDstPhysReg =
3977 RI.getMatchingSuperReg(Reg: MovDstPhysReg, SubIdx: AMDGPU::lo16, RC: MovDstRC);
3978 if (!MovDstPhysReg)
3979 continue;
3980 }
3981 }
3982
3983 // Result class isn't the right size, try the next instruction.
3984 if (MovDstPhysReg) {
3985 if (!MovDstRC->contains(Reg: MovDstPhysReg))
3986 return false;
3987 } else if (!MRI->constrainRegClass(Reg: DstReg, RC: MovDstRC)) {
3988 // TODO: This will be overly conservative in the case of 16-bit virtual
3989 // SGPRs. We could hack up the virtual register uses to use a compatible
3990 // 32-bit class.
3991 continue;
3992 }
3993
3994 const MCOperandInfo &OpInfo = MovDesc.operands()[1];
3995
3996 // Ensure the interpreted immediate value is a valid operand in the new
3997 // mov.
3998 //
3999 // FIXME: isImmOperandLegal should have form that doesn't require existing
4000 // MachineInstr or MachineOperand
4001 if (!RI.opCanUseLiteralConstant(OpType: OpInfo.OperandType) &&
4002 !isInlineConstant(ImmVal: *SubRegImm, OperandType: OpInfo.OperandType))
4003 break;
4004
4005 NewOpc = MovOp;
4006 break;
4007 }
4008
4009 if (NewOpc == AMDGPU::INSTRUCTION_LIST_END)
4010 return false;
4011
4012 if (Is16Bit) {
4013 UseMI.getOperand(i: 0).setSubReg(AMDGPU::NoSubRegister);
4014 if (MovDstPhysReg)
4015 UseMI.getOperand(i: 0).setReg(MovDstPhysReg);
4016 assert(UseMI.getOperand(1).getReg().isVirtual());
4017 }
4018
4019 const MCInstrDesc &NewMCID = get(Opcode: NewOpc);
4020 UseMI.setDesc(NewMCID);
4021 UseMI.getOperand(i: 1).ChangeToImmediate(ImmVal: *SubRegImm);
4022 UseMI.addImplicitDefUseOperands(MF&: *MF);
4023 return true;
4024 }
4025
4026 if (HasMultipleUses)
4027 return false;
4028
4029 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
4030 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
4031 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
4032 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
4033 Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4034 Opc == AMDGPU::V_FMAC_F16_fake16_e64 || Opc == AMDGPU::V_FMA_F64_e64 ||
4035 Opc == AMDGPU::V_FMAC_F64_e64) {
4036 // Don't fold if we are using source or output modifiers. The new VOP2
4037 // instructions don't have them.
4038 if (hasAnyModifiersSet(MI: UseMI))
4039 return false;
4040
4041 // If this is a free constant, there's no reason to do this.
4042 // TODO: We could fold this here instead of letting SIFoldOperands do it
4043 // later.
4044 int Src0Idx = getNamedOperandIdx(Opcode: UseMI.getOpcode(), Name: AMDGPU::OpName::src0);
4045
4046 // Any src operand can be used for the legality check.
4047 if (isInlineConstant(MI: UseMI, OpIdx: Src0Idx, ImmVal: Imm))
4048 return false;
4049
4050 MachineOperand *Src0 = &UseMI.getOperand(i: Src0Idx);
4051
4052 MachineOperand *Src1 = getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src1);
4053 MachineOperand *Src2 = getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src2);
4054
4055 auto CopyRegOperandToNarrowerRC =
4056 [MRI, this](MachineInstr &MI, unsigned OpNo,
4057 const TargetRegisterClass *NewRC) -> void {
4058 if (!MI.getOperand(i: OpNo).isReg())
4059 return;
4060 Register Reg = MI.getOperand(i: OpNo).getReg();
4061 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI: *MRI, Reg);
4062 if (RI.getCommonSubClass(A: RC, B: NewRC) != NewRC)
4063 return;
4064 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
4065 BuildMI(BB&: *MI.getParent(), I: MI.getIterator(), MIMD: MI.getDebugLoc(),
4066 MCID: get(Opcode: AMDGPU::COPY), DestReg: Tmp)
4067 .addReg(RegNo: Reg);
4068 MI.getOperand(i: OpNo).setReg(Tmp);
4069 MI.getOperand(i: OpNo).setIsKill();
4070 };
4071
4072 // Multiplied part is the constant: Use v_madmk_{f16, f32}.
4073 if ((Src0->isReg() && Src0->getReg() == Reg) ||
4074 (Src1->isReg() && Src1->getReg() == Reg)) {
4075 MachineOperand *RegSrc =
4076 Src1->isReg() && Src1->getReg() == Reg ? Src0 : Src1;
4077 if (!RegSrc->isReg())
4078 return false;
4079 if (RI.isSGPRClass(RC: MRI->getRegClass(Reg: RegSrc->getReg())) &&
4080 ST.getConstantBusLimit(Opcode: Opc) < 2)
4081 return false;
4082
4083 if (!Src2->isReg() || RI.isSGPRClass(RC: MRI->getRegClass(Reg: Src2->getReg())))
4084 return false;
4085
4086 // If src2 is also a literal constant then we have to choose which one to
4087 // fold. In general it is better to choose madak so that the other literal
4088 // can be materialized in an sgpr instead of a vgpr:
4089 // s_mov_b32 s0, literal
4090 // v_madak_f32 v0, s0, v0, literal
4091 // Instead of:
4092 // v_mov_b32 v1, literal
4093 // v_madmk_f32 v0, v0, literal, v1
4094 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src2->getReg());
4095 if (Def && Def->isMoveImmediate() &&
4096 !isInlineConstant(MO: Def->getOperand(i: 1)))
4097 return false;
4098
4099 unsigned NewOpc = getNewFMAMKInst(ST, Opc);
4100 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
4101 return false;
4102
4103 const std::optional<int64_t> SubRegImm = extractSubregFromImm(
4104 Imm, SubRegIndex: RegSrc == Src1 ? Src0->getSubReg() : Src1->getSubReg());
4105
4106 // FIXME: This would be a lot easier if we could return a new instruction
4107 // instead of having to modify in place.
4108
4109 Register SrcReg = RegSrc->getReg();
4110 unsigned SrcSubReg = RegSrc->getSubReg();
4111 Src0->setReg(SrcReg);
4112 Src0->setSubReg(SrcSubReg);
4113 Src0->setIsKill(RegSrc->isKill());
4114
4115 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
4116 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4117 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
4118 Opc == AMDGPU::V_FMAC_F16_e64 || Opc == AMDGPU::V_FMAC_F64_e64)
4119 UseMI.untieRegOperand(
4120 OpIdx: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2));
4121
4122 Src1->ChangeToImmediate(ImmVal: *SubRegImm);
4123
4124 removeModOperands(MI&: UseMI);
4125 UseMI.setDesc(get(Opcode: NewOpc));
4126
4127 if (NewOpc == AMDGPU::V_FMAMK_F16_t16 ||
4128 NewOpc == AMDGPU::V_FMAMK_F16_fake16) {
4129 const TargetRegisterClass *NewRC = getRegClass(MCID: get(Opcode: NewOpc), OpNum: 0);
4130 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
4131 BuildMI(BB&: *UseMI.getParent(), I: std::next(x: UseMI.getIterator()),
4132 MIMD: UseMI.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
4133 DestReg: UseMI.getOperand(i: 0).getReg())
4134 .addReg(RegNo: Tmp, Flags: RegState::Kill);
4135 UseMI.getOperand(i: 0).setReg(Tmp);
4136 CopyRegOperandToNarrowerRC(UseMI, 1, NewRC);
4137 CopyRegOperandToNarrowerRC(UseMI, 3, NewRC);
4138 }
4139
4140 bool DeleteDef = MRI->use_nodbg_empty(RegNo: Reg);
4141 if (DeleteDef)
4142 DefMI.eraseFromParent();
4143
4144 return true;
4145 }
4146
4147 // Added part is the constant: Use v_madak_{f16, f32}.
4148 if (Src2->isReg() && Src2->getReg() == Reg) {
4149 if (ST.getConstantBusLimit(Opcode: Opc) < 2) {
4150 // Not allowed to use constant bus for another operand.
4151 // We can however allow an inline immediate as src0.
4152 bool Src0Inlined = false;
4153 if (Src0->isReg()) {
4154 // Try to inline constant if possible.
4155 // If the Def moves immediate and the use is single
4156 // We are saving VGPR here.
4157 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src0->getReg());
4158 if (Def && Def->isMoveImmediate() &&
4159 isInlineConstant(MO: Def->getOperand(i: 1)) &&
4160 MRI->hasOneNonDBGUse(RegNo: Src0->getReg())) {
4161 Src0->ChangeToImmediate(ImmVal: Def->getOperand(i: 1).getImm());
4162 Src0Inlined = true;
4163 } else if (ST.getConstantBusLimit(Opcode: Opc) <= 1 &&
4164 RI.isSGPRReg(MRI: *MRI, Reg: Src0->getReg())) {
4165 return false;
4166 }
4167 // VGPR is okay as Src0 - fallthrough
4168 }
4169
4170 if (Src1->isReg() && !Src0Inlined) {
4171 // We have one slot for inlinable constant so far - try to fill it
4172 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src1->getReg());
4173 if (Def && Def->isMoveImmediate() &&
4174 isInlineConstant(MO: Def->getOperand(i: 1)) &&
4175 MRI->hasOneNonDBGUse(RegNo: Src1->getReg()) && commuteInstruction(MI&: UseMI))
4176 Src0->ChangeToImmediate(ImmVal: Def->getOperand(i: 1).getImm());
4177 else if (RI.isSGPRReg(MRI: *MRI, Reg: Src1->getReg()))
4178 return false;
4179 // VGPR is okay as Src1 - fallthrough
4180 }
4181 }
4182
4183 unsigned NewOpc = getNewFMAAKInst(ST, Opc);
4184 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
4185 return false;
4186
4187 // FIXME: This would be a lot easier if we could return a new instruction
4188 // instead of having to modify in place.
4189
4190 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
4191 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4192 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
4193 Opc == AMDGPU::V_FMAC_F16_e64 || Opc == AMDGPU::V_FMAC_F64_e64)
4194 UseMI.untieRegOperand(
4195 OpIdx: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2));
4196
4197 const std::optional<int64_t> SubRegImm =
4198 extractSubregFromImm(Imm, SubRegIndex: Src2->getSubReg());
4199
4200 // ChangingToImmediate adds Src2 back to the instruction.
4201 Src2->ChangeToImmediate(ImmVal: *SubRegImm);
4202
4203 // These come before src2.
4204 removeModOperands(MI&: UseMI);
4205 UseMI.setDesc(get(Opcode: NewOpc));
4206
4207 if (NewOpc == AMDGPU::V_FMAAK_F16_t16 ||
4208 NewOpc == AMDGPU::V_FMAAK_F16_fake16) {
4209 const TargetRegisterClass *NewRC = getRegClass(MCID: get(Opcode: NewOpc), OpNum: 0);
4210 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
4211 BuildMI(BB&: *UseMI.getParent(), I: std::next(x: UseMI.getIterator()),
4212 MIMD: UseMI.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
4213 DestReg: UseMI.getOperand(i: 0).getReg())
4214 .addReg(RegNo: Tmp, Flags: RegState::Kill);
4215 UseMI.getOperand(i: 0).setReg(Tmp);
4216 CopyRegOperandToNarrowerRC(UseMI, 1, NewRC);
4217 CopyRegOperandToNarrowerRC(UseMI, 2, NewRC);
4218 }
4219
4220 // It might happen that UseMI was commuted
4221 // and we now have SGPR as SRC1. If so 2 inlined
4222 // constant and SGPR are illegal.
4223 legalizeOperands(MI&: UseMI);
4224
4225 int NewSrc0Idx =
4226 AMDGPU::getNamedOperandIdx(Opcode: UseMI.getOpcode(), Name: AMDGPU::OpName::src0);
4227 if (!isOperandLegal(MI: UseMI, OpIdx: NewSrc0Idx))
4228 legalizeOpWithMove(MI&: UseMI, OpIdx: NewSrc0Idx);
4229
4230 bool DeleteDef = MRI->use_nodbg_empty(RegNo: Reg);
4231 if (DeleteDef)
4232 DefMI.eraseFromParent();
4233
4234 return true;
4235 }
4236 }
4237
4238 return false;
4239}
4240
4241static bool
4242memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1,
4243 ArrayRef<const MachineOperand *> BaseOps2) {
4244 if (BaseOps1.size() != BaseOps2.size())
4245 return false;
4246 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) {
4247 if (!BaseOps1[I]->isIdenticalTo(Other: *BaseOps2[I]))
4248 return false;
4249 }
4250 return true;
4251}
4252
4253static bool offsetsDoNotOverlap(LocationSize WidthA, int OffsetA,
4254 LocationSize WidthB, int OffsetB) {
4255 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
4256 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
4257 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
4258 return LowWidth.hasValue() &&
4259 LowOffset + (int)LowWidth.getValue() <= HighOffset;
4260}
4261
4262bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
4263 const MachineInstr &MIb) const {
4264 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;
4265 int64_t Offset0, Offset1;
4266 LocationSize Dummy0 = LocationSize::precise(Value: 0);
4267 LocationSize Dummy1 = LocationSize::precise(Value: 0);
4268 bool Offset0IsScalable, Offset1IsScalable;
4269 if (!getMemOperandsWithOffsetWidth(LdSt: MIa, BaseOps&: BaseOps0, Offset&: Offset0, OffsetIsScalable&: Offset0IsScalable,
4270 Width&: Dummy0, TRI: &RI) ||
4271 !getMemOperandsWithOffsetWidth(LdSt: MIb, BaseOps&: BaseOps1, Offset&: Offset1, OffsetIsScalable&: Offset1IsScalable,
4272 Width&: Dummy1, TRI: &RI))
4273 return false;
4274
4275 if (!memOpsHaveSameBaseOperands(BaseOps1: BaseOps0, BaseOps2: BaseOps1))
4276 return false;
4277
4278 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
4279 // FIXME: Handle ds_read2 / ds_write2.
4280 return false;
4281 }
4282 LocationSize Width0 = MIa.memoperands().front()->getSize();
4283 LocationSize Width1 = MIb.memoperands().front()->getSize();
4284 return offsetsDoNotOverlap(WidthA: Width0, OffsetA: Offset0, WidthB: Width1, OffsetB: Offset1);
4285}
4286
4287bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
4288 const MachineInstr &MIb) const {
4289 assert(MIa.mayLoadOrStore() &&
4290 "MIa must load from or modify a memory location");
4291 assert(MIb.mayLoadOrStore() &&
4292 "MIb must load from or modify a memory location");
4293
4294 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects())
4295 return false;
4296
4297 // XXX - Can we relax this between address spaces?
4298 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
4299 return false;
4300
4301 if (isLDSDMA(MI: MIa) || isLDSDMA(MI: MIb))
4302 return false;
4303
4304 if (MIa.isBundle() || MIb.isBundle())
4305 return false;
4306
4307 // TODO: Should we check the address space from the MachineMemOperand? That
4308 // would allow us to distinguish objects we know don't alias based on the
4309 // underlying address space, even if it was lowered to a different one,
4310 // e.g. private accesses lowered to use MUBUF instructions on a scratch
4311 // buffer.
4312 if (isDS(MI: MIa)) {
4313 if (isDS(MI: MIb))
4314 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4315
4316 return !isFLAT(MI: MIb) || isSegmentSpecificFLAT(MI: MIb);
4317 }
4318
4319 if (isMUBUF(MI: MIa) || isMTBUF(MI: MIa)) {
4320 if (isMUBUF(MI: MIb) || isMTBUF(MI: MIb))
4321 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4322
4323 if (isFLAT(MI: MIb))
4324 return isFLATScratch(MI: MIb);
4325
4326 return !isSMRD(MI: MIb);
4327 }
4328
4329 if (isSMRD(MI: MIa)) {
4330 if (isSMRD(MI: MIb))
4331 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4332
4333 if (isFLAT(MI: MIb))
4334 return isFLATScratch(MI: MIb);
4335
4336 return !isMUBUF(MI: MIb) && !isMTBUF(MI: MIb);
4337 }
4338
4339 if (isFLAT(MI: MIa)) {
4340 if (isFLAT(MI: MIb)) {
4341 if ((isFLATScratch(MI: MIa) && isFLATGlobal(MI: MIb)) ||
4342 (isFLATGlobal(MI: MIa) && isFLATScratch(MI: MIb)))
4343 return true;
4344
4345 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4346 }
4347
4348 return false;
4349 }
4350
4351 return false;
4352}
4353
4354static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI,
4355 MachineInstr &NewMI) {
4356 if (LV) {
4357 unsigned NumOps = MI.getNumOperands();
4358 for (unsigned I = 1; I < NumOps; ++I) {
4359 MachineOperand &Op = MI.getOperand(i: I);
4360 if (Op.isReg() && Op.isKill())
4361 LV->replaceKillInstruction(Reg: Op.getReg(), OldMI&: MI, NewMI);
4362 }
4363 }
4364}
4365
4366static unsigned getNewFMAInst(const GCNSubtarget &ST, unsigned Opc) {
4367 switch (Opc) {
4368 case AMDGPU::V_MAC_F16_e32:
4369 case AMDGPU::V_MAC_F16_e64:
4370 return AMDGPU::V_MAD_F16_e64;
4371 case AMDGPU::V_MAC_F32_e32:
4372 case AMDGPU::V_MAC_F32_e64:
4373 return AMDGPU::V_MAD_F32_e64;
4374 case AMDGPU::V_MAC_LEGACY_F32_e32:
4375 case AMDGPU::V_MAC_LEGACY_F32_e64:
4376 return AMDGPU::V_MAD_LEGACY_F32_e64;
4377 case AMDGPU::V_FMAC_LEGACY_F32_e32:
4378 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4379 return AMDGPU::V_FMA_LEGACY_F32_e64;
4380 case AMDGPU::V_FMAC_F16_e32:
4381 case AMDGPU::V_FMAC_F16_e64:
4382 case AMDGPU::V_FMAC_F16_t16_e64:
4383 case AMDGPU::V_FMAC_F16_fake16_e64:
4384 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
4385 ? AMDGPU::V_FMA_F16_gfx9_t16_e64
4386 : AMDGPU::V_FMA_F16_gfx9_fake16_e64
4387 : AMDGPU::V_FMA_F16_gfx9_e64;
4388 case AMDGPU::V_FMAC_F32_e32:
4389 case AMDGPU::V_FMAC_F32_e64:
4390 return AMDGPU::V_FMA_F32_e64;
4391 case AMDGPU::V_FMAC_F64_e32:
4392 case AMDGPU::V_FMAC_F64_e64:
4393 return AMDGPU::V_FMA_F64_e64;
4394 default:
4395 llvm_unreachable("invalid instruction");
4396 }
4397}
4398
4399/// Helper struct for the implementation of 3-address conversion to communicate
4400/// updates made to instruction operands.
4401struct SIInstrInfo::ThreeAddressUpdates {
4402 /// Other instruction whose def is no longer used by the converted
4403 /// instruction.
4404 MachineInstr *RemoveMIUse = nullptr;
4405};
4406
4407MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI,
4408 LiveVariables *LV,
4409 LiveIntervals *LIS) const {
4410 MachineBasicBlock &MBB = *MI.getParent();
4411 MachineInstr *CandidateMI = &MI;
4412
4413 if (MI.isBundle()) {
4414 // This is a temporary placeholder for bundle handling that enables us to
4415 // exercise the relevant code paths in the two-address instruction pass.
4416 if (MI.getBundleSize() != 1)
4417 return nullptr;
4418 CandidateMI = MI.getNextNode();
4419 }
4420
4421 ThreeAddressUpdates U;
4422 MachineInstr *NewMI = convertToThreeAddressImpl(MI&: *CandidateMI, Updates&: U);
4423 if (!NewMI)
4424 return nullptr;
4425
4426 if (MI.isBundle()) {
4427 CandidateMI->eraseFromBundle();
4428
4429 for (MachineOperand &MO : MI.all_defs()) {
4430 if (MO.isTied())
4431 MI.untieRegOperand(OpIdx: MO.getOperandNo());
4432 }
4433 } else {
4434 updateLiveVariables(LV, MI, NewMI&: *NewMI);
4435 if (LIS) {
4436 LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *NewMI);
4437 // SlotIndex of defs needs to be updated when converting to early-clobber
4438 MachineOperand &Def = NewMI->getOperand(i: 0);
4439 if (Def.isEarlyClobber() && Def.isReg() &&
4440 LIS->hasInterval(Reg: Def.getReg())) {
4441 SlotIndex OldIndex = LIS->getInstructionIndex(Instr: *NewMI).getRegSlot(EC: false);
4442 SlotIndex NewIndex = LIS->getInstructionIndex(Instr: *NewMI).getRegSlot(EC: true);
4443 auto &LI = LIS->getInterval(Reg: Def.getReg());
4444 auto UpdateDefIndex = [&](LiveRange &LR) {
4445 auto *S = LR.find(Pos: OldIndex);
4446 if (S != LR.end() && S->start == OldIndex) {
4447 assert(S->valno && S->valno->def == OldIndex);
4448 S->start = NewIndex;
4449 S->valno->def = NewIndex;
4450 }
4451 };
4452 UpdateDefIndex(LI);
4453 for (auto &SR : LI.subranges())
4454 UpdateDefIndex(SR);
4455 }
4456 }
4457 }
4458
4459 if (U.RemoveMIUse) {
4460 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4461 // The only user is the instruction which will be killed.
4462 Register DefReg = U.RemoveMIUse->getOperand(i: 0).getReg();
4463
4464 if (MRI.hasOneNonDBGUse(RegNo: DefReg)) {
4465 // We cannot just remove the DefMI here, calling pass will crash.
4466 U.RemoveMIUse->setDesc(get(Opcode: AMDGPU::IMPLICIT_DEF));
4467 U.RemoveMIUse->getOperand(i: 0).setIsDead(true);
4468 for (unsigned I = U.RemoveMIUse->getNumOperands() - 1; I != 0; --I)
4469 U.RemoveMIUse->removeOperand(OpNo: I);
4470 if (LV)
4471 LV->getVarInfo(Reg: DefReg).AliveBlocks.clear();
4472 }
4473
4474 if (MI.isBundle()) {
4475 VirtRegInfo VRI = AnalyzeVirtRegInBundle(MI, Reg: DefReg);
4476 if (!VRI.Reads && !VRI.Writes) {
4477 for (MachineOperand &MO : MI.all_uses()) {
4478 if (MO.isReg() && MO.getReg() == DefReg) {
4479 assert(MO.getSubReg() == 0 &&
4480 "tied sub-registers in bundles currently not supported");
4481 MI.removeOperand(OpNo: MO.getOperandNo());
4482 break;
4483 }
4484 }
4485
4486 if (LIS)
4487 LIS->shrinkToUses(li: &LIS->getInterval(Reg: DefReg));
4488 }
4489 } else if (LIS) {
4490 LiveInterval &DefLI = LIS->getInterval(Reg: DefReg);
4491
4492 // We cannot delete the original instruction here, so hack out the use
4493 // in the original instruction with a dummy register so we can use
4494 // shrinkToUses to deal with any multi-use edge cases. Other targets do
4495 // not have the complexity of deleting a use to consider here.
4496 Register DummyReg = MRI.cloneVirtualRegister(VReg: DefReg);
4497 for (MachineOperand &MIOp : MI.uses()) {
4498 if (MIOp.isReg() && MIOp.getReg() == DefReg) {
4499 MIOp.setIsUndef(true);
4500 MIOp.setReg(DummyReg);
4501 }
4502 }
4503
4504 if (MI.isBundle()) {
4505 VirtRegInfo VRI = AnalyzeVirtRegInBundle(MI, Reg: DefReg);
4506 if (!VRI.Reads && !VRI.Writes) {
4507 for (MachineOperand &MIOp : MI.uses()) {
4508 if (MIOp.isReg() && MIOp.getReg() == DefReg) {
4509 MIOp.setIsUndef(true);
4510 MIOp.setReg(DummyReg);
4511 }
4512 }
4513 }
4514
4515 MI.addOperand(Op: MachineOperand::CreateReg(Reg: DummyReg, isDef: false, isImp: false, isKill: false,
4516 isDead: false, /*isUndef=*/true));
4517 }
4518
4519 LIS->shrinkToUses(li: &DefLI);
4520 }
4521 }
4522
4523 return MI.isBundle() ? &MI : NewMI;
4524}
4525
4526MachineInstr *
4527SIInstrInfo::convertToThreeAddressImpl(MachineInstr &MI,
4528 ThreeAddressUpdates &U) const {
4529 MachineBasicBlock &MBB = *MI.getParent();
4530 unsigned Opc = MI.getOpcode();
4531
4532 // Handle MFMA.
4533 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opcode: Opc);
4534 if (NewMFMAOpc != -1) {
4535 MachineInstrBuilder MIB =
4536 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewMFMAOpc));
4537 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I)
4538 MIB.add(MO: MI.getOperand(i: I));
4539 return MIB;
4540 }
4541
4542 if (SIInstrInfo::isWMMA(MI)) {
4543 unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(Opc: MI.getOpcode());
4544 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4545 .setMIFlags(MI.getFlags());
4546 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I)
4547 MIB->addOperand(Op: MI.getOperand(i: I));
4548 return MIB;
4549 }
4550
4551 assert(Opc != AMDGPU::V_FMAC_F16_t16_e32 &&
4552 Opc != AMDGPU::V_FMAC_F16_fake16_e32 &&
4553 "V_FMAC_F16_t16/fake16_e32 is not supported and not expected to be "
4554 "present pre-RA");
4555
4556 // Handle MAC/FMAC.
4557 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
4558 bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 ||
4559 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 ||
4560 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
4561 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64;
4562 bool Src0Literal = false;
4563
4564 switch (Opc) {
4565 default:
4566 return nullptr;
4567 case AMDGPU::V_MAC_F16_e64:
4568 case AMDGPU::V_FMAC_F16_e64:
4569 case AMDGPU::V_FMAC_F16_t16_e64:
4570 case AMDGPU::V_FMAC_F16_fake16_e64:
4571 case AMDGPU::V_MAC_F32_e64:
4572 case AMDGPU::V_MAC_LEGACY_F32_e64:
4573 case AMDGPU::V_FMAC_F32_e64:
4574 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4575 case AMDGPU::V_FMAC_F64_e64:
4576 break;
4577 case AMDGPU::V_MAC_F16_e32:
4578 case AMDGPU::V_FMAC_F16_e32:
4579 case AMDGPU::V_MAC_F32_e32:
4580 case AMDGPU::V_MAC_LEGACY_F32_e32:
4581 case AMDGPU::V_FMAC_F32_e32:
4582 case AMDGPU::V_FMAC_LEGACY_F32_e32:
4583 case AMDGPU::V_FMAC_F64_e32: {
4584 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
4585 Name: AMDGPU::OpName::src0);
4586 const MachineOperand *Src0 = &MI.getOperand(i: Src0Idx);
4587 if (!Src0->isReg() && !Src0->isImm())
4588 return nullptr;
4589
4590 if (Src0->isImm() && !isInlineConstant(MI, OpIdx: Src0Idx, MO: *Src0))
4591 Src0Literal = true;
4592
4593 break;
4594 }
4595 }
4596
4597 MachineInstrBuilder MIB;
4598 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
4599 const MachineOperand *Src0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
4600 const MachineOperand *Src0Mods =
4601 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
4602 const MachineOperand *Src1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
4603 const MachineOperand *Src1Mods =
4604 getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
4605 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
4606 const MachineOperand *Src2Mods =
4607 getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers);
4608 const MachineOperand *Clamp = getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
4609 const MachineOperand *Omod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
4610 const MachineOperand *OpSel = getNamedOperand(MI, OperandName: AMDGPU::OpName::op_sel);
4611
4612 if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsLegacy &&
4613 (!IsF64 || ST.hasFmaakFmamkF64Insts()) &&
4614 // If we have an SGPR input, we will violate the constant bus restriction.
4615 (ST.getConstantBusLimit(Opcode: Opc) > 1 || !Src0->isReg() ||
4616 !RI.isSGPRReg(MRI: MBB.getParent()->getRegInfo(), Reg: Src0->getReg()))) {
4617 MachineInstr *DefMI = nullptr;
4618 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4619 std::optional<int64_t> ImmOpt;
4620 int64_t Imm;
4621
4622 if (!Src0Literal &&
4623 (ImmOpt = getImmOrMaterializedImm(MRI, Op: *Src2, DefMI: &DefMI))) {
4624 unsigned NewOpc = getNewFMAAKInst(ST, Opc);
4625 if (pseudoToMCOpcode(Opcode: NewOpc) != -1) {
4626 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4627 .add(MO: *Dst)
4628 .add(MO: *Src0)
4629 .add(MO: *Src1)
4630 .addImm(Val: *ImmOpt)
4631 .setMIFlags(MI.getFlags());
4632 U.RemoveMIUse = DefMI;
4633 return MIB;
4634 }
4635 }
4636 unsigned NewOpc = getNewFMAMKInst(ST, Opc);
4637 if (!Src0Literal &&
4638 (ImmOpt = getImmOrMaterializedImm(MRI, Op: *Src1, DefMI: &DefMI))) {
4639 if (pseudoToMCOpcode(Opcode: NewOpc) != -1) {
4640 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4641 .add(MO: *Dst)
4642 .add(MO: *Src0)
4643 .addImm(Val: *ImmOpt)
4644 .add(MO: *Src2)
4645 .setMIFlags(MI.getFlags());
4646 U.RemoveMIUse = DefMI;
4647 return MIB;
4648 }
4649 }
4650 if ((ImmOpt = getImmOrMaterializedImm(MRI, Op: *Src0, DefMI: &DefMI))) {
4651 Imm = *ImmOpt;
4652 if (pseudoToMCOpcode(Opcode: NewOpc) != -1 &&
4653 isOperandLegal(
4654 MI, OpIdx: AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::src0),
4655 MO: Src1)) {
4656 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4657 .add(MO: *Dst)
4658 .add(MO: *Src1)
4659 .addImm(Val: Imm)
4660 .add(MO: *Src2)
4661 .setMIFlags(MI.getFlags());
4662 U.RemoveMIUse = DefMI;
4663 return MIB;
4664 }
4665 }
4666 }
4667
4668 // VOP2 mac/fmac with a literal operand cannot be converted to VOP3 mad/fma
4669 // if VOP3 does not allow a literal operand.
4670 if (Src0Literal && !ST.hasVOP3Literal())
4671 return nullptr;
4672
4673 unsigned NewOpc = getNewFMAInst(ST, Opc);
4674
4675 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
4676 return nullptr;
4677
4678 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4679 .add(MO: *Dst)
4680 .addImm(Val: Src0Mods ? Src0Mods->getImm() : 0)
4681 .add(MO: *Src0)
4682 .addImm(Val: Src1Mods ? Src1Mods->getImm() : 0)
4683 .add(MO: *Src1)
4684 .addImm(Val: Src2Mods ? Src2Mods->getImm() : 0)
4685 .add(MO: *Src2)
4686 .addImm(Val: Clamp ? Clamp->getImm() : 0)
4687 .addImm(Val: Omod ? Omod->getImm() : 0)
4688 .setMIFlags(MI.getFlags());
4689 if (AMDGPU::hasNamedOperand(Opcode: NewOpc, NamedIdx: AMDGPU::OpName::op_sel))
4690 MIB.addImm(Val: OpSel ? OpSel->getImm() : 0);
4691 return MIB;
4692}
4693
4694// It's not generally safe to move VALU instructions across these since it will
4695// start using the register as a base index rather than directly.
4696// XXX - Why isn't hasSideEffects sufficient for these?
4697static bool changesVGPRIndexingMode(const MachineInstr &MI) {
4698 switch (MI.getOpcode()) {
4699 case AMDGPU::S_SET_GPR_IDX_ON:
4700 case AMDGPU::S_SET_GPR_IDX_MODE:
4701 case AMDGPU::S_SET_GPR_IDX_OFF:
4702 return true;
4703 default:
4704 return false;
4705 }
4706}
4707
4708bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
4709 const MachineBasicBlock *MBB,
4710 const MachineFunction &MF) const {
4711 // Skipping the check for SP writes in the base implementation. The reason it
4712 // was added was apparently due to compile time concerns.
4713 //
4714 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops
4715 // but is probably avoidable.
4716
4717 // Copied from base implementation.
4718 // Terminators and labels can't be scheduled around.
4719 if (MI.isTerminator() || MI.isPosition())
4720 return true;
4721
4722 // INLINEASM_BR can jump to another block
4723 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
4724 return true;
4725
4726 if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(i: 0).getImm() == 0)
4727 return true;
4728
4729 // Target-independent instructions do not have an implicit-use of EXEC, even
4730 // when they operate on VGPRs. Treating EXEC modifications as scheduling
4731 // boundaries prevents incorrect movements of such instructions.
4732 return MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &RI) ||
4733 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
4734 MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
4735 MI.getOpcode() == AMDGPU::S_SETPRIO ||
4736 MI.getOpcode() == AMDGPU::S_SETPRIO_INC_WG ||
4737 changesVGPRIndexingMode(MI);
4738}
4739
4740bool SIInstrInfo::isAlwaysGDS(uint32_t Opcode) const {
4741 return Opcode == AMDGPU::DS_ORDERED_COUNT ||
4742 Opcode == AMDGPU::DS_ADD_GS_REG_RTN ||
4743 Opcode == AMDGPU::DS_SUB_GS_REG_RTN || isGWS(Opcode);
4744}
4745
4746bool SIInstrInfo::mayAccessScratch(const MachineInstr &MI) const {
4747 // Instructions that access scratch use FLAT encoding or BUF encodings.
4748 if ((!isFLAT(MI) || isFLATGlobal(MI)) && !isBUF(MI))
4749 return false;
4750
4751 // SCRATCH instructions always access scratch.
4752 if (isFLATScratch(MI))
4753 return true;
4754
4755 // If FLAT_SCRATCH registers are not initialized, we can never access scratch
4756 // via the aperture.
4757 if (MI.getMF()->getFunction().hasFnAttribute(Kind: "amdgpu-no-flat-scratch-init"))
4758 return false;
4759
4760 // If there are no memory operands then conservatively assume the flat
4761 // operation may access scratch.
4762 if (MI.memoperands_empty())
4763 return true;
4764
4765 // See if any memory operand specifies an address space that involves scratch.
4766 return any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *Memop) {
4767 unsigned AS = Memop->getAddrSpace();
4768 if (AS == AMDGPUAS::FLAT_ADDRESS) {
4769 const MDNode *MD = Memop->getAAInfo().NoAliasAddrSpace;
4770 return !MD || !AMDGPU::hasValueInRangeLikeMetadata(
4771 MD: *MD, Val: AMDGPUAS::PRIVATE_ADDRESS);
4772 }
4773 return AS == AMDGPUAS::PRIVATE_ADDRESS;
4774 });
4775}
4776
4777bool SIInstrInfo::mayAccessVMEMThroughFlat(const MachineInstr &MI) const {
4778 assert(isFLAT(MI));
4779
4780 // All flat instructions use the VMEM counter except prefetch.
4781 if (!usesVM_CNT(MI))
4782 return false;
4783
4784 // If there are no memory operands then conservatively assume the flat
4785 // operation may access VMEM.
4786 if (MI.memoperands_empty())
4787 return true;
4788
4789 // See if any memory operand specifies an address space that involves VMEM.
4790 // Flat operations only supported FLAT, LOCAL (LDS), or address spaces
4791 // involving VMEM such as GLOBAL, CONSTANT, PRIVATE (SCRATCH), etc. The REGION
4792 // (GDS) address space is not supported by flat operations. Therefore, simply
4793 // return true unless only the LDS address space is found.
4794 for (const MachineMemOperand *Memop : MI.memoperands()) {
4795 unsigned AS = Memop->getAddrSpace();
4796 assert(AS != AMDGPUAS::REGION_ADDRESS);
4797 if (AS != AMDGPUAS::LOCAL_ADDRESS)
4798 return true;
4799 }
4800
4801 return false;
4802}
4803
4804bool SIInstrInfo::mayAccessLDSThroughFlat(const MachineInstr &MI,
4805 bool TgSplit) const {
4806 assert(isFLAT(MI));
4807
4808 // Flat instruction such as SCRATCH and GLOBAL do not use the lgkm counter.
4809 if (!usesLGKM_CNT(MI))
4810 return false;
4811
4812 // If in tgsplit mode then there can be no use of LDS.
4813 if (TgSplit)
4814 return false;
4815
4816 // If there are no memory operands then conservatively assume the flat
4817 // operation may access LDS.
4818 if (MI.memoperands_empty())
4819 return true;
4820
4821 // See if any memory operand specifies an address space that involves LDS.
4822 for (const MachineMemOperand *Memop : MI.memoperands()) {
4823 unsigned AS = Memop->getAddrSpace();
4824 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)
4825 return true;
4826 }
4827
4828 return false;
4829}
4830
4831bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) {
4832 // Skip the full operand and register alias search modifiesRegister
4833 // does. There's only a handful of instructions that touch this, it's only an
4834 // implicit def, and doesn't alias any other registers.
4835 return is_contained(Range: MI.getDesc().implicit_defs(), Element: AMDGPU::MODE);
4836}
4837
4838bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const {
4839 unsigned Opcode = MI.getOpcode();
4840
4841 if (MI.mayStore() && isSMRD(MI))
4842 return true; // scalar store or atomic
4843
4844 // This will terminate the function when other lanes may need to continue.
4845 if (MI.isReturn())
4846 return true;
4847
4848 // These instructions cause shader I/O that may cause hardware lockups
4849 // when executed with an empty EXEC mask.
4850 //
4851 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
4852 // EXEC = 0, but checking for that case here seems not worth it
4853 // given the typical code patterns.
4854 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
4855 isEXP(Opcode) || Opcode == AMDGPU::DS_ORDERED_COUNT ||
4856 Opcode == AMDGPU::S_TRAP || Opcode == AMDGPU::S_WAIT_EVENT ||
4857 Opcode == AMDGPU::S_SETHALT)
4858 return true;
4859
4860 if (MI.isCall() || MI.isInlineAsm())
4861 return true; // conservative assumption
4862
4863 // Assume that barrier interactions are only intended with active lanes.
4864 if (isBarrier(Opcode))
4865 return true;
4866
4867 // A mode change is a scalar operation that influences vector instructions.
4868 if (modifiesModeRegister(MI))
4869 return true;
4870
4871 // These are like SALU instructions in terms of effects, so it's questionable
4872 // whether we should return true for those.
4873 //
4874 // However, executing them with EXEC = 0 causes them to operate on undefined
4875 // data, which we avoid by returning true here.
4876 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 ||
4877 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32 ||
4878 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
4879 Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR)
4880 return true;
4881
4882 return false;
4883}
4884
4885bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI,
4886 const MachineInstr &MI) const {
4887 if (MI.isMetaInstruction())
4888 return false;
4889
4890 // This won't read exec if this is an SGPR->SGPR copy.
4891 if (MI.isCopyLike()) {
4892 if (!RI.isSGPRReg(MRI, Reg: MI.getOperand(i: 0).getReg()))
4893 return true;
4894
4895 // Make sure this isn't copying exec as a normal operand
4896 return MI.readsRegister(Reg: AMDGPU::EXEC, TRI: &RI);
4897 }
4898
4899 // Make a conservative assumption about the callee.
4900 if (MI.isCall())
4901 return true;
4902
4903 // Be conservative with any unhandled generic opcodes.
4904 if (!isTargetSpecificOpcode(Opcode: MI.getOpcode()))
4905 return true;
4906
4907 return !isSALU(MI) || MI.readsRegister(Reg: AMDGPU::EXEC, TRI: &RI);
4908}
4909
4910bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
4911 switch (Imm.getBitWidth()) {
4912 case 1: // This likely will be a condition code mask.
4913 return true;
4914
4915 case 32:
4916 return AMDGPU::isInlinableLiteral32(Literal: Imm.getSExtValue(),
4917 HasInv2Pi: ST.hasInv2PiInlineImm());
4918 case 64:
4919 return AMDGPU::isInlinableLiteral64(Literal: Imm.getSExtValue(),
4920 HasInv2Pi: ST.hasInv2PiInlineImm());
4921 case 16:
4922 return ST.has16BitInsts() &&
4923 AMDGPU::isInlinableLiteralI16(Literal: Imm.getSExtValue(),
4924 HasInv2Pi: ST.hasInv2PiInlineImm());
4925 default:
4926 llvm_unreachable("invalid bitwidth");
4927 }
4928}
4929
4930bool SIInstrInfo::isInlineConstant(const APFloat &Imm) const {
4931 APInt IntImm = Imm.bitcastToAPInt();
4932 int64_t IntImmVal = IntImm.getSExtValue();
4933 bool HasInv2Pi = ST.hasInv2PiInlineImm();
4934 switch (APFloat::SemanticsToEnum(Sem: Imm.getSemantics())) {
4935 default:
4936 llvm_unreachable("invalid fltSemantics");
4937 case APFloatBase::S_IEEEsingle:
4938 case APFloatBase::S_IEEEdouble:
4939 return isInlineConstant(Imm: IntImm);
4940 case APFloatBase::S_BFloat:
4941 return ST.has16BitInsts() &&
4942 AMDGPU::isInlinableLiteralBF16(Literal: IntImmVal, HasInv2Pi);
4943 case APFloatBase::S_IEEEhalf:
4944 return ST.has16BitInsts() &&
4945 AMDGPU::isInlinableLiteralFP16(Literal: IntImmVal, HasInv2Pi);
4946 }
4947}
4948
4949bool SIInstrInfo::isInlineConstant(int64_t Imm, uint8_t OperandType) const {
4950 // MachineOperand provides no way to tell the true operand size, since it only
4951 // records a 64-bit value. We need to know the size to determine if a 32-bit
4952 // floating point immediate bit pattern is legal for an integer immediate. It
4953 // would be for any 32-bit integer operand, but would not be for a 64-bit one.
4954 switch (OperandType) {
4955 case AMDGPU::OPERAND_REG_IMM_INT32:
4956 case AMDGPU::OPERAND_REG_IMM_FP32:
4957 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
4958 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
4959 case AMDGPU::OPERAND_REG_IMM_V2FP32:
4960 case AMDGPU::OPERAND_REG_IMM_V2INT32:
4961 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
4962 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
4963 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: {
4964 int32_t Trunc = static_cast<int32_t>(Imm);
4965 return AMDGPU::isInlinableLiteral32(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
4966 }
4967 case AMDGPU::OPERAND_REG_IMM_INT64:
4968 case AMDGPU::OPERAND_REG_IMM_FP64:
4969 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
4970 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
4971 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
4972 case AMDGPU::OPERAND_REG_IMM_V2FP64:
4973 case AMDGPU::OPERAND_REG_IMM_V2INT64:
4974 return AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: ST.hasInv2PiInlineImm());
4975 case AMDGPU::OPERAND_REG_IMM_INT16:
4976 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
4977 // We would expect inline immediates to not be concerned with an integer/fp
4978 // distinction. However, in the case of 16-bit integer operations, the
4979 // "floating point" values appear to not work. It seems read the low 16-bits
4980 // of 32-bit immediates, which happens to always work for the integer
4981 // values.
4982 //
4983 // See llvm bugzilla 46302.
4984 //
4985 // TODO: Theoretically we could use op-sel to use the high bits of the
4986 // 32-bit FP values.
4987 return AMDGPU::isInlinableIntLiteral(Literal: Imm);
4988 case AMDGPU::OPERAND_REG_IMM_V2INT16:
4989 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
4990 return AMDGPU::isInlinableLiteralV2I16(Literal: Imm);
4991 case AMDGPU::OPERAND_REG_IMM_V2FP16:
4992 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
4993 return AMDGPU::isInlinableLiteralV2F16(Literal: Imm);
4994 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
4995 return AMDGPU::isPKFMACF16InlineConstant(Literal: Imm, IsGFX11Plus: ST.isGFX11Plus());
4996 case AMDGPU::OPERAND_REG_IMM_V2BF16:
4997 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
4998 return AMDGPU::isInlinableLiteralV2BF16(Literal: Imm);
4999 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
5000 return false;
5001 case AMDGPU::OPERAND_REG_IMM_FP16:
5002 case AMDGPU::OPERAND_REG_INLINE_C_FP16: {
5003 if (isInt<16>(x: Imm) || isUInt<16>(x: Imm)) {
5004 // A few special case instructions have 16-bit operands on subtargets
5005 // where 16-bit instructions are not legal.
5006 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
5007 // constants in these cases
5008 int16_t Trunc = static_cast<int16_t>(Imm);
5009 return ST.has16BitInsts() &&
5010 AMDGPU::isInlinableLiteralFP16(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
5011 }
5012
5013 return false;
5014 }
5015 case AMDGPU::OPERAND_REG_IMM_BF16:
5016 case AMDGPU::OPERAND_REG_INLINE_C_BF16: {
5017 if (isInt<16>(x: Imm) || isUInt<16>(x: Imm)) {
5018 int16_t Trunc = static_cast<int16_t>(Imm);
5019 return ST.has16BitInsts() &&
5020 AMDGPU::isInlinableLiteralBF16(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
5021 }
5022 return false;
5023 }
5024 case AMDGPU::OPERAND_KIMM32:
5025 case AMDGPU::OPERAND_KIMM16:
5026 case AMDGPU::OPERAND_KIMM64:
5027 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
5028 return false;
5029 case AMDGPU::OPERAND_INLINE_C_AV64_PSEUDO:
5030 return isLegalAV64PseudoImm(Imm);
5031 case AMDGPU::OPERAND_INPUT_MODS:
5032 case MCOI::OPERAND_IMMEDIATE:
5033 // Always embedded in the instruction for free.
5034 return true;
5035 case MCOI::OPERAND_UNKNOWN:
5036 case MCOI::OPERAND_REGISTER:
5037 case MCOI::OPERAND_PCREL:
5038 case MCOI::OPERAND_GENERIC_0:
5039 case MCOI::OPERAND_GENERIC_1:
5040 case MCOI::OPERAND_GENERIC_2:
5041 case MCOI::OPERAND_GENERIC_3:
5042 case MCOI::OPERAND_GENERIC_4:
5043 case MCOI::OPERAND_GENERIC_5:
5044 // Just ignore anything else.
5045 return false;
5046 default:
5047 llvm_unreachable("invalid operand type");
5048 }
5049}
5050
5051static bool compareMachineOp(const MachineOperand &Op0,
5052 const MachineOperand &Op1) {
5053 if (Op0.getType() != Op1.getType())
5054 return false;
5055
5056 switch (Op0.getType()) {
5057 case MachineOperand::MO_Register:
5058 return Op0.getReg() == Op1.getReg();
5059 case MachineOperand::MO_Immediate:
5060 return Op0.getImm() == Op1.getImm();
5061 default:
5062 llvm_unreachable("Didn't expect to be comparing these operand types");
5063 }
5064}
5065
5066bool SIInstrInfo::isLiteralOperandLegal(const MCInstrDesc &InstDesc,
5067 const MCOperandInfo &OpInfo) const {
5068 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
5069 return true;
5070
5071 if (!RI.opCanUseLiteralConstant(OpType: OpInfo.OperandType))
5072 return false;
5073
5074 if (!isVOP3(Desc: InstDesc) || !AMDGPU::isSISrcOperand(OpInfo))
5075 return true;
5076
5077 return ST.hasVOP3Literal();
5078}
5079
5080bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
5081 int64_t ImmVal) const {
5082 const unsigned Opc = InstDesc.getOpcode();
5083 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
5084 if (Src1Idx != -1 && isDPP(Opcode: Opc) && !ST.hasDPPSrc1SGPR() &&
5085 OpNo == static_cast<unsigned>(Src1Idx))
5086 return false;
5087
5088 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
5089 if (isInlineConstant(Imm: ImmVal, OperandType: OpInfo.OperandType)) {
5090 if (isMAI(Desc: InstDesc) && ST.hasMFMAInlineLiteralBug() &&
5091 OpNo == (unsigned)AMDGPU::getNamedOperandIdx(Opcode: InstDesc.getOpcode(),
5092 Name: AMDGPU::OpName::src2))
5093 return false;
5094
5095 if (ST.hasBF16InlineConstFromUpperFP32() && isVOP1(Opcode: Opc)) {
5096 if ((OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_BF16 ||
5097 OpInfo.OperandType == AMDGPU::OPERAND_REG_INLINE_C_BF16) &&
5098 isInlineConstant(Imm: ImmVal, OperandType: OpInfo.OperandType))
5099 return false;
5100 }
5101
5102 return RI.opCanUseInlineConstant(OpType: OpInfo.OperandType);
5103 }
5104
5105 return isLiteralOperandLegal(InstDesc, OpInfo);
5106}
5107
5108bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
5109 const MachineOperand &MO) const {
5110 if (MO.isImm())
5111 return isImmOperandLegal(InstDesc, OpNo, ImmVal: MO.getImm());
5112
5113 assert((MO.isTargetIndex() || MO.isFI() || MO.isGlobal()) &&
5114 "unexpected imm-like operand kind");
5115 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
5116 return isLiteralOperandLegal(InstDesc, OpInfo);
5117}
5118
5119bool SIInstrInfo::isLegalAV64PseudoImm(uint64_t Imm) const {
5120 // 2 32-bit inline constants packed into one.
5121 return AMDGPU::isInlinableLiteral32(Literal: Lo_32(Value: Imm), HasInv2Pi: ST.hasInv2PiInlineImm()) &&
5122 AMDGPU::isInlinableLiteral32(Literal: Hi_32(Value: Imm), HasInv2Pi: ST.hasInv2PiInlineImm());
5123}
5124
5125bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
5126 // GFX90A does not have V_MUL_LEGACY_F32_e32.
5127 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())
5128 return false;
5129
5130 int Op32 = AMDGPU::getVOPe32(Opcode);
5131 if (Op32 == -1)
5132 return false;
5133
5134 return pseudoToMCOpcode(Opcode: Op32) != -1;
5135}
5136
5137bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
5138 // The src0_modifier operand is present on all instructions
5139 // that have modifiers.
5140
5141 return AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0_modifiers);
5142}
5143
5144bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI,
5145 AMDGPU::OpName OpName) const {
5146 const MachineOperand *Mods = getNamedOperand(MI, OperandName: OpName);
5147 return Mods && Mods->getImm();
5148}
5149
5150bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const {
5151 return any_of(Range: ModifierOpNames,
5152 P: [&](AMDGPU::OpName Name) { return hasModifiersSet(MI, OpName: Name); });
5153}
5154
5155bool SIInstrInfo::canShrink(const MachineInstr &MI,
5156 const MachineRegisterInfo &MRI) const {
5157 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
5158 // Can't shrink instruction with three operands.
5159 if (Src2) {
5160 switch (MI.getOpcode()) {
5161 default: return false;
5162
5163 case AMDGPU::V_ADDC_U32_e64:
5164 case AMDGPU::V_SUBB_U32_e64:
5165 case AMDGPU::V_SUBBREV_U32_e64: {
5166 const MachineOperand *Src1
5167 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
5168 if (!Src1->isReg() || !RI.isVGPR(MRI, Reg: Src1->getReg()))
5169 return false;
5170 // Additional verification is needed for sdst/src2.
5171 return true;
5172 }
5173 case AMDGPU::V_MAC_F16_e64:
5174 case AMDGPU::V_MAC_F32_e64:
5175 case AMDGPU::V_MAC_LEGACY_F32_e64:
5176 case AMDGPU::V_FMAC_F16_e64:
5177 case AMDGPU::V_FMAC_F16_t16_e64:
5178 case AMDGPU::V_FMAC_F16_fake16_e64:
5179 case AMDGPU::V_FMAC_F32_e64:
5180 case AMDGPU::V_FMAC_F64_e64:
5181 case AMDGPU::V_FMAC_LEGACY_F32_e64:
5182 if (!Src2->isReg() || !RI.isVGPR(MRI, Reg: Src2->getReg()) ||
5183 hasModifiersSet(MI, OpName: AMDGPU::OpName::src2_modifiers))
5184 return false;
5185 break;
5186
5187 case AMDGPU::V_CNDMASK_B32_e64:
5188 break;
5189 }
5190 }
5191
5192 const MachineOperand *Src1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
5193 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Reg: Src1->getReg()) ||
5194 hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers)))
5195 return false;
5196
5197 // Make sure src0 isn't using any modifiers.
5198 if (hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers))
5199 return false;
5200
5201 // Can it be shrunk to a valid 32 bit opcode?
5202 if (!hasVALU32BitEncoding(Opcode: MI.getOpcode()))
5203 return false;
5204
5205 const MachineOperand *Src0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
5206 if (Src0 && Src0->isImm()) {
5207 unsigned Op32 = AMDGPU::getVOPe32(Opcode: MI.getOpcode());
5208 if (!isImmOperandLegal(
5209 InstDesc: get(Opcode: Op32), OpNo: AMDGPU::getNamedOperandIdx(Opcode: Op32, Name: AMDGPU::OpName::src0),
5210 MO: *Src0))
5211 return false;
5212 }
5213
5214 // Check output modifiers
5215 return !hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) &&
5216 !hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) &&
5217 !hasModifiersSet(MI, OpName: AMDGPU::OpName::byte_sel) &&
5218 // TODO: Can we avoid checking bound_ctrl/fi here?
5219 // They are only used by permlane*_swap special case.
5220 !hasModifiersSet(MI, OpName: AMDGPU::OpName::bound_ctrl) &&
5221 !hasModifiersSet(MI, OpName: AMDGPU::OpName::fi);
5222}
5223
5224// Set VCC operand with all flags from \p Orig, except for setting it as
5225// implicit.
5226static void copyFlagsToImplicitVCC(MachineInstr &MI,
5227 const MachineOperand &Orig) {
5228
5229 for (MachineOperand &Use : MI.implicit_operands()) {
5230 if (Use.isUse() &&
5231 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) {
5232 Use.setIsUndef(Orig.isUndef());
5233 Use.setIsKill(Orig.isKill());
5234 return;
5235 }
5236 }
5237}
5238
5239MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI,
5240 unsigned Op32) const {
5241 MachineBasicBlock *MBB = MI.getParent();
5242
5243 const MCInstrDesc &Op32Desc = get(Opcode: Op32);
5244 MachineInstrBuilder Inst32 =
5245 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: Op32Desc)
5246 .setMIFlags(MI.getFlags());
5247
5248 // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
5249 // For VOPC instructions, this is replaced by an implicit def of vcc.
5250
5251 // We assume the defs of the shrunk opcode are in the same order, and the
5252 // shrunk opcode loses the last def (SGPR def, in the VOP3->VOPC case).
5253 for (int I = 0, E = Op32Desc.getNumDefs(); I != E; ++I)
5254 Inst32.add(MO: MI.getOperand(i: I));
5255
5256 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
5257
5258 int Idx = MI.getNumExplicitDefs();
5259 for (const MachineOperand &Use : MI.explicit_uses()) {
5260 int OpTy = MI.getDesc().operands()[Idx++].OperandType;
5261 if (OpTy == AMDGPU::OPERAND_INPUT_MODS || OpTy == MCOI::OPERAND_IMMEDIATE)
5262 continue;
5263
5264 if (&Use == Src2) {
5265 if (AMDGPU::getNamedOperandIdx(Opcode: Op32, Name: AMDGPU::OpName::src2) == -1) {
5266 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
5267 // replaced with an implicit read of vcc or vcc_lo. The implicit read
5268 // of vcc was already added during the initial BuildMI, but we
5269 // 1) may need to change vcc to vcc_lo to preserve the original register
5270 // 2) have to preserve the original flags.
5271 copyFlagsToImplicitVCC(MI&: *Inst32, Orig: *Src2);
5272 continue;
5273 }
5274 }
5275
5276 Inst32.add(MO: Use);
5277 }
5278
5279 // FIXME: Losing implicit operands
5280 fixImplicitOperands(MI&: *Inst32);
5281 return Inst32;
5282}
5283
5284bool SIInstrInfo::physRegUsesConstantBus(const MachineOperand &RegOp) const {
5285 // Null is free
5286 Register Reg = RegOp.getReg();
5287 if (Reg == AMDGPU::SGPR_NULL || Reg == AMDGPU::SGPR_NULL64)
5288 return false;
5289
5290 // SGPRs use the constant bus
5291
5292 // FIXME: implicit registers that are not part of the MCInstrDesc's implicit
5293 // physical register operands should also count, except for exec.
5294 if (RegOp.isImplicit())
5295 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::M0;
5296
5297 // SGPRs use the constant bus
5298 return AMDGPU::SReg_32RegClass.contains(Reg) ||
5299 AMDGPU::SReg_64RegClass.contains(Reg);
5300}
5301
5302bool SIInstrInfo::regUsesConstantBus(const MachineOperand &RegOp,
5303 const MachineRegisterInfo &MRI) const {
5304 Register Reg = RegOp.getReg();
5305 return Reg.isVirtual() ? RI.isSGPRClass(RC: MRI.getRegClass(Reg))
5306 : physRegUsesConstantBus(RegOp);
5307}
5308
5309bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,
5310 const MachineOperand &MO,
5311 const MCOperandInfo &OpInfo) const {
5312 // Literal constants use the constant bus.
5313 if (!MO.isReg())
5314 return !isInlineConstant(MO, OpInfo);
5315
5316 Register Reg = MO.getReg();
5317 return Reg.isVirtual() ? RI.isSGPRClass(RC: MRI.getRegClass(Reg))
5318 : physRegUsesConstantBus(RegOp: MO);
5319}
5320
5321static Register findImplicitSGPRRead(const MachineInstr &MI) {
5322 for (const MachineOperand &MO : MI.implicit_operands()) {
5323 // We only care about reads.
5324 if (MO.isDef())
5325 continue;
5326
5327 switch (MO.getReg()) {
5328 case AMDGPU::VCC:
5329 case AMDGPU::VCC_LO:
5330 case AMDGPU::VCC_HI:
5331 case AMDGPU::M0:
5332 case AMDGPU::FLAT_SCR:
5333 return MO.getReg();
5334
5335 default:
5336 break;
5337 }
5338 }
5339
5340 return Register();
5341}
5342
5343static bool shouldReadExec(const MachineInstr &MI) {
5344 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) {
5345 switch (MI.getOpcode()) {
5346 case AMDGPU::V_READLANE_B32:
5347 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
5348 case AMDGPU::V_WRITELANE_B32:
5349 case AMDGPU::SI_SPILL_S32_TO_VGPR:
5350 return false;
5351 }
5352
5353 return true;
5354 }
5355
5356 if (MI.isPreISelOpcode() ||
5357 SIInstrInfo::isGenericOpcode(Opc: MI.getOpcode()) ||
5358 SIInstrInfo::isSALU(MI) ||
5359 SIInstrInfo::isSMRD(MI))
5360 return false;
5361
5362 return true;
5363}
5364
5365static bool isRegOrFI(const MachineOperand &MO) {
5366 return MO.isReg() || MO.isFI();
5367}
5368
5369static bool isSubRegOf(const SIRegisterInfo &TRI,
5370 const MachineOperand &SuperVec,
5371 const MachineOperand &SubReg) {
5372 if (SubReg.getReg().isPhysical())
5373 return TRI.isSubRegister(RegA: SuperVec.getReg(), RegB: SubReg.getReg());
5374
5375 return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
5376 SubReg.getReg() == SuperVec.getReg();
5377}
5378
5379// Verify the illegal copy from vector register to SGPR for generic opcode COPY
5380bool SIInstrInfo::verifyCopy(const MachineInstr &MI,
5381 const MachineRegisterInfo &MRI,
5382 StringRef &ErrInfo) const {
5383 Register DstReg = MI.getOperand(i: 0).getReg();
5384 Register SrcReg = MI.getOperand(i: 1).getReg();
5385 // This is a check for copy from vector register to SGPR
5386 if (RI.isVectorRegister(MRI, Reg: SrcReg) && RI.isSGPRReg(MRI, Reg: DstReg)) {
5387 ErrInfo = "illegal copy from vector register to SGPR";
5388 return false;
5389 }
5390 return true;
5391}
5392
5393bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
5394 StringRef &ErrInfo) const {
5395 uint32_t Opcode = MI.getOpcode();
5396 const MachineFunction *MF = MI.getMF();
5397 const MachineRegisterInfo &MRI = MF->getRegInfo();
5398
5399 // FIXME: At this point the COPY verify is done only for non-ssa forms.
5400 // Find a better property to recognize the point where instruction selection
5401 // is just done.
5402 // We can only enforce this check after SIFixSGPRCopies pass so that the
5403 // illegal copies are legalized and thereafter we don't expect a pass
5404 // inserting similar copies.
5405 if (!MRI.isSSA() && MI.isCopy())
5406 return verifyCopy(MI, MRI, ErrInfo);
5407
5408 if (SIInstrInfo::isGenericOpcode(Opc: Opcode))
5409 return true;
5410
5411 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0);
5412 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src1);
5413 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src2);
5414 int Src3Idx = -1;
5415 if (Src0Idx == -1) {
5416 // VOPD V_DUAL_* instructions use different operand names.
5417 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0X);
5418 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vsrc1X);
5419 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0Y);
5420 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vsrc1Y);
5421 }
5422
5423 // Make sure the number of operands is correct.
5424 const MCInstrDesc &Desc = get(Opcode);
5425 if (!Desc.isVariadic() &&
5426 Desc.getNumOperands() != MI.getNumExplicitOperands()) {
5427 ErrInfo = "Instruction has wrong number of operands.";
5428 return false;
5429 }
5430
5431 if (MI.isInlineAsm()) {
5432 // Verify register classes for inlineasm constraints.
5433 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
5434 I != E; ++I) {
5435 const TargetRegisterClass *RC = MI.getRegClassConstraint(OpIdx: I, TII: this, TRI: &RI);
5436 if (!RC)
5437 continue;
5438
5439 const MachineOperand &Op = MI.getOperand(i: I);
5440 if (!Op.isReg())
5441 continue;
5442
5443 Register Reg = Op.getReg();
5444 if (!Reg.isVirtual() && !RC->contains(Reg)) {
5445 ErrInfo = "inlineasm operand has incorrect register class.";
5446 return false;
5447 }
5448 }
5449
5450 return true;
5451 }
5452
5453 if (isImage(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) {
5454 ErrInfo = "missing memory operand from image instruction.";
5455 return false;
5456 }
5457
5458 // Make sure the register classes are correct.
5459 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
5460 const MachineOperand &MO = MI.getOperand(i);
5461 if (MO.isFPImm()) {
5462 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
5463 "all fp values to integers.";
5464 return false;
5465 }
5466
5467 const MCOperandInfo &OpInfo = Desc.operands()[i];
5468
5469 switch (OpInfo.OperandType) {
5470 case MCOI::OPERAND_REGISTER:
5471 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
5472 ErrInfo = "Illegal immediate value for operand.";
5473 return false;
5474 }
5475 break;
5476 case AMDGPU::OPERAND_REG_IMM_INT32:
5477 case AMDGPU::OPERAND_REG_IMM_INT16:
5478 case AMDGPU::OPERAND_REG_IMM_FP32:
5479 case AMDGPU::OPERAND_REG_IMM_BF16:
5480 case AMDGPU::OPERAND_REG_IMM_FP16:
5481 case AMDGPU::OPERAND_REG_IMM_V2FP16:
5482 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
5483 case AMDGPU::OPERAND_REG_IMM_V2INT16:
5484 case AMDGPU::OPERAND_REG_IMM_V2BF16:
5485 case AMDGPU::OPERAND_REG_IMM_V2FP64:
5486 case AMDGPU::OPERAND_REG_IMM_V2INT64:
5487 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
5488 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
5489 break;
5490 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
5491 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
5492 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
5493 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
5494 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
5495 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
5496 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
5497 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
5498 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
5499 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
5500 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
5501 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
5502 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: {
5503 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, OpIdx: i))) {
5504 ErrInfo = "Illegal immediate value for operand.";
5505 return false;
5506 }
5507 break;
5508 }
5509 case AMDGPU::OPERAND_REG_IMM_FP64:
5510 case AMDGPU::OPERAND_REG_IMM_INT64:
5511 case AMDGPU::OPERAND_REG_IMM_V2INT32:
5512 case AMDGPU::OPERAND_REG_IMM_V2FP32:
5513 if (ST.has64BitLiterals() && Desc.getSize() != 4 && MO.isImm() &&
5514 !isInlineConstant(MI, OpIdx: i) &&
5515 !AMDGPU::isValid32BitLiteral(Val: MO.getImm(),
5516 IsFP64: OpInfo.OperandType ==
5517 AMDGPU::OPERAND_REG_IMM_FP64)) {
5518 ErrInfo = "illegal 64-bit immediate value for operand.";
5519 return false;
5520 }
5521 break;
5522 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
5523 case AMDGPU::OPERAND_INPUT_MODS:
5524 if (!MI.getOperand(i).isImm() || !isInlineConstant(MI, OpIdx: i)) {
5525 ErrInfo = "Expected inline constant for operand.";
5526 return false;
5527 }
5528 break;
5529 case AMDGPU::OPERAND_SDWA_VOPC_DST:
5530 case AMDGPU::OPERAND_KIMM16:
5531 break;
5532 case MCOI::OPERAND_IMMEDIATE:
5533 case AMDGPU::OPERAND_KIMM32:
5534 case AMDGPU::OPERAND_KIMM64:
5535 case AMDGPU::OPERAND_INLINE_C_AV64_PSEUDO:
5536 // Check if this operand is an immediate.
5537 // FrameIndex operands will be replaced by immediates, so they are
5538 // allowed.
5539 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
5540 ErrInfo = "Expected immediate, but got non-immediate";
5541 return false;
5542 }
5543 break;
5544 case MCOI::OPERAND_UNKNOWN:
5545 case MCOI::OPERAND_MEMORY:
5546 case MCOI::OPERAND_PCREL:
5547 break;
5548 default:
5549 if (OpInfo.isGenericType())
5550 continue;
5551 break;
5552 }
5553 }
5554
5555 // Verify SDWA
5556 if (isSDWA(MI)) {
5557 if (!ST.hasSDWA()) {
5558 ErrInfo = "SDWA is not supported on this target";
5559 return false;
5560 }
5561
5562 for (auto Op : {AMDGPU::OpName::src0_sel, AMDGPU::OpName::src1_sel,
5563 AMDGPU::OpName::dst_sel}) {
5564 const MachineOperand *MO = getNamedOperand(MI, OperandName: Op);
5565 if (!MO)
5566 continue;
5567 int64_t Imm = MO->getImm();
5568 if (Imm < 0 || Imm > AMDGPU::SDWA::SdwaSel::DWORD) {
5569 ErrInfo = "Invalid SDWA selection";
5570 return false;
5571 }
5572 }
5573
5574 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdst);
5575
5576 for (int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) {
5577 if (OpIdx == -1)
5578 continue;
5579 const MachineOperand &MO = MI.getOperand(i: OpIdx);
5580
5581 if (!ST.hasSDWAScalar()) {
5582 // Only VGPRS on VI
5583 if (!MO.isReg() || !RI.hasVGPRs(RC: RI.getRegClassForReg(MRI, Reg: MO.getReg()))) {
5584 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
5585 return false;
5586 }
5587 } else {
5588 // No immediates on GFX9
5589 if (!MO.isReg()) {
5590 ErrInfo =
5591 "Only reg allowed as operands in SDWA instructions on GFX9+";
5592 return false;
5593 }
5594 }
5595 }
5596
5597 if (!ST.hasSDWAOmod()) {
5598 // No omod allowed on VI
5599 const MachineOperand *OMod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
5600 if (OMod != nullptr &&
5601 (!OMod->isImm() || OMod->getImm() != 0)) {
5602 ErrInfo = "OMod not allowed in SDWA instructions on VI";
5603 return false;
5604 }
5605 }
5606
5607 if (Opcode == AMDGPU::V_CVT_F32_FP8_sdwa ||
5608 Opcode == AMDGPU::V_CVT_F32_BF8_sdwa ||
5609 Opcode == AMDGPU::V_CVT_PK_F32_FP8_sdwa ||
5610 Opcode == AMDGPU::V_CVT_PK_F32_BF8_sdwa) {
5611 const MachineOperand *Src0ModsMO =
5612 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
5613 unsigned Mods = Src0ModsMO->getImm();
5614 if (Mods & SISrcMods::ABS || Mods & SISrcMods::NEG ||
5615 Mods & SISrcMods::SEXT) {
5616 ErrInfo = "sext, abs and neg are not allowed on this instruction";
5617 return false;
5618 }
5619 }
5620
5621 uint32_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
5622 if (isVOPC(Opcode: BasicOpcode)) {
5623 if (!ST.hasSDWASdst() && DstIdx != -1) {
5624 // Only vcc allowed as dst on VI for VOPC
5625 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5626 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
5627 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
5628 return false;
5629 }
5630 } else if (!ST.hasSDWAOutModsVOPC()) {
5631 // No clamp allowed on GFX9 for VOPC
5632 const MachineOperand *Clamp = getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
5633 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
5634 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
5635 return false;
5636 }
5637
5638 // No omod allowed on GFX9 for VOPC
5639 const MachineOperand *OMod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
5640 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
5641 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
5642 return false;
5643 }
5644 }
5645 }
5646
5647 const MachineOperand *DstUnused = getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_unused);
5648 if (DstUnused && DstUnused->isImm() &&
5649 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
5650 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5651 if (!Dst.isReg() || !Dst.isTied()) {
5652 ErrInfo = "Dst register should have tied register";
5653 return false;
5654 }
5655
5656 const MachineOperand &TiedMO =
5657 MI.getOperand(i: MI.findTiedOperandIdx(OpIdx: DstIdx));
5658 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
5659 ErrInfo =
5660 "Dst register should be tied to implicit use of preserved register";
5661 return false;
5662 }
5663 if (TiedMO.getReg().isPhysical() && Dst.getReg() != TiedMO.getReg()) {
5664 ErrInfo = "Dst register should use same physical register as preserved";
5665 return false;
5666 }
5667 }
5668 }
5669
5670 if (isDPP(MI) && !ST.hasDPPSrc1SGPR() && Src1Idx != -1) {
5671 const MachineOperand &Src1MO = MI.getOperand(i: Src1Idx);
5672 if (Src1MO.isReg() && RI.isSGPRReg(MRI, Reg: Src1MO.getReg())) {
5673 ErrInfo = "DPP src1 cannot be SGPR on this subtarget";
5674 return false;
5675 }
5676 if (Src1MO.isImm()) {
5677 ErrInfo = "DPP src1 cannot be an immediate on this subtarget";
5678 return false;
5679 }
5680 }
5681
5682 // Verify MIMG / VIMAGE / VSAMPLE
5683 if (isImage(Opcode) && !MI.mayStore()) {
5684 // Ensure that the return type used is large enough for all the options
5685 // being used TFE/LWE require an extra result register.
5686 const MachineOperand *DMask = getNamedOperand(MI, OperandName: AMDGPU::OpName::dmask);
5687 if (DMask) {
5688 uint64_t DMaskImm = DMask->getImm();
5689 uint32_t RegCount = isGather4(Opcode) ? 4 : llvm::popcount(Value: DMaskImm);
5690 const MachineOperand *TFE = getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe);
5691 const MachineOperand *LWE = getNamedOperand(MI, OperandName: AMDGPU::OpName::lwe);
5692 const MachineOperand *D16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::d16);
5693
5694 // Adjust for packed 16 bit values
5695 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
5696 RegCount = divideCeil(Numerator: RegCount, Denominator: 2);
5697
5698 // Adjust if using LWE or TFE
5699 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
5700 RegCount += 1;
5701
5702 const uint32_t DstIdx =
5703 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdata);
5704 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5705 if (Dst.isReg()) {
5706 const TargetRegisterClass *DstRC = getOpRegClass(MI, OpNo: DstIdx);
5707 uint32_t DstSize = RI.getRegSizeInBits(RC: *DstRC) / 32;
5708 if (RegCount > DstSize) {
5709 ErrInfo = "Image instruction returns too many registers for dst "
5710 "register class";
5711 return false;
5712 }
5713 }
5714 }
5715 }
5716
5717 // Verify VOP*. Ignore multiple sgpr operands on writelane.
5718 if (isVALU(MI, /*AllowLDSDMA=*/false) &&
5719 Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
5720 unsigned ConstantBusCount = 0;
5721 bool UsesLiteral = false;
5722 const MachineOperand *LiteralVal = nullptr;
5723
5724 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::imm);
5725 if (ImmIdx != -1) {
5726 ++ConstantBusCount;
5727 UsesLiteral = true;
5728 LiteralVal = &MI.getOperand(i: ImmIdx);
5729 }
5730
5731 SmallVector<Register, 2> SGPRsUsed;
5732 Register SGPRUsed;
5733
5734 // Only look at the true operands. Only a real operand can use the constant
5735 // bus, and we don't want to check pseudo-operands like the source modifier
5736 // flags.
5737 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) {
5738 if (OpIdx == -1)
5739 continue;
5740 const MachineOperand &MO = MI.getOperand(i: OpIdx);
5741 if (usesConstantBus(MRI, MO, OpInfo: MI.getDesc().operands()[OpIdx])) {
5742 if (MO.isReg()) {
5743 SGPRUsed = MO.getReg();
5744 if (!llvm::is_contained(Range&: SGPRsUsed, Element: SGPRUsed)) {
5745 ++ConstantBusCount;
5746 SGPRsUsed.push_back(Elt: SGPRUsed);
5747 }
5748 } else if (!MO.isFI()) { // Treat FI like a register.
5749 if (!UsesLiteral) {
5750 ++ConstantBusCount;
5751 UsesLiteral = true;
5752 LiteralVal = &MO;
5753 } else if (!MO.isIdenticalTo(Other: *LiteralVal)) {
5754 assert(isVOP2(MI) || isVOP3(MI));
5755 ErrInfo = "VOP2/VOP3 instruction uses more than one literal";
5756 return false;
5757 }
5758 }
5759 }
5760 }
5761
5762 SGPRUsed = findImplicitSGPRRead(MI);
5763 if (SGPRUsed) {
5764 // Implicit uses may safely overlap true operands
5765 if (llvm::all_of(Range&: SGPRsUsed, P: [this, SGPRUsed](unsigned SGPR) {
5766 return !RI.regsOverlap(RegA: SGPRUsed, RegB: SGPR);
5767 })) {
5768 ++ConstantBusCount;
5769 SGPRsUsed.push_back(Elt: SGPRUsed);
5770 }
5771 }
5772
5773 // v_writelane_b32 is an exception from constant bus restriction:
5774 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
5775 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
5776 Opcode != AMDGPU::V_WRITELANE_B32) {
5777 ErrInfo = "VOP* instruction violates constant bus restriction";
5778 return false;
5779 }
5780
5781 if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) {
5782 ErrInfo = "VOP3 instruction uses literal";
5783 return false;
5784 }
5785 }
5786
5787 // Special case for writelane - this can break the multiple constant bus rule,
5788 // but still can't use more than one SGPR register
5789 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
5790 unsigned SGPRCount = 0;
5791 Register SGPRUsed;
5792
5793 for (int OpIdx : {Src0Idx, Src1Idx}) {
5794 if (OpIdx == -1)
5795 break;
5796
5797 const MachineOperand &MO = MI.getOperand(i: OpIdx);
5798
5799 if (usesConstantBus(MRI, MO, OpInfo: MI.getDesc().operands()[OpIdx])) {
5800 if (MO.isReg() && MO.getReg() != AMDGPU::M0) {
5801 if (MO.getReg() != SGPRUsed)
5802 ++SGPRCount;
5803 SGPRUsed = MO.getReg();
5804 }
5805 }
5806 if (SGPRCount > ST.getConstantBusLimit(Opcode)) {
5807 ErrInfo = "WRITELANE instruction violates constant bus restriction";
5808 return false;
5809 }
5810 }
5811 }
5812
5813 // Verify misc. restrictions on specific instructions.
5814 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
5815 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
5816 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5817 const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
5818 const MachineOperand &Src2 = MI.getOperand(i: Src2Idx);
5819 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
5820 if (!compareMachineOp(Op0: Src0, Op1: Src1) &&
5821 !compareMachineOp(Op0: Src0, Op1: Src2)) {
5822 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
5823 return false;
5824 }
5825 }
5826 if ((getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers)->getImm() &
5827 SISrcMods::ABS) ||
5828 (getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers)->getImm() &
5829 SISrcMods::ABS) ||
5830 (getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers)->getImm() &
5831 SISrcMods::ABS)) {
5832 ErrInfo = "ABS not allowed in VOP3B instructions";
5833 return false;
5834 }
5835 }
5836
5837 if (isSOP2(MI) || isSOPC(MI)) {
5838 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5839 const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
5840
5841 if (!isRegOrFI(MO: Src0) && !isRegOrFI(MO: Src1) &&
5842 !isInlineConstant(MO: Src0, OpInfo: Desc.operands()[Src0Idx]) &&
5843 !isInlineConstant(MO: Src1, OpInfo: Desc.operands()[Src1Idx]) &&
5844 !Src0.isIdenticalTo(Other: Src1)) {
5845 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
5846 return false;
5847 }
5848 }
5849
5850 if (isSOPK(MI)) {
5851 const auto *Op = getNamedOperand(MI, OperandName: AMDGPU::OpName::simm16);
5852 if (Desc.isBranch()) {
5853 if (!Op->isMBB()) {
5854 ErrInfo = "invalid branch target for SOPK instruction";
5855 return false;
5856 }
5857 } else {
5858 uint64_t Imm = Op->getImm();
5859 if (sopkIsZext(Opcode)) {
5860 if (!isUInt<16>(x: Imm)) {
5861 ErrInfo = "invalid immediate for SOPK instruction";
5862 return false;
5863 }
5864 } else {
5865 if (!isInt<16>(x: Imm)) {
5866 ErrInfo = "invalid immediate for SOPK instruction";
5867 return false;
5868 }
5869 }
5870 }
5871 }
5872
5873 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
5874 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
5875 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
5876 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
5877 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
5878 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
5879
5880 const unsigned StaticNumOps =
5881 Desc.getNumOperands() + Desc.implicit_uses().size();
5882 const unsigned NumImplicitOps = IsDst ? 2 : 1;
5883
5884 // Require additional implicit operands. This allows a fixup done by the
5885 // post RA scheduler where the main implicit operand is killed and
5886 // implicit-defs are added for sub-registers that remain live after this
5887 // instruction.
5888 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
5889 ErrInfo = "missing implicit register operands";
5890 return false;
5891 }
5892
5893 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
5894 if (IsDst) {
5895 if (!Dst->isUse()) {
5896 ErrInfo = "v_movreld_b32 vdst should be a use operand";
5897 return false;
5898 }
5899
5900 unsigned UseOpIdx;
5901 if (!MI.isRegTiedToUseOperand(DefOpIdx: StaticNumOps, UseOpIdx: &UseOpIdx) ||
5902 UseOpIdx != StaticNumOps + 1) {
5903 ErrInfo = "movrel implicit operands should be tied";
5904 return false;
5905 }
5906 }
5907
5908 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5909 const MachineOperand &ImpUse
5910 = MI.getOperand(i: StaticNumOps + NumImplicitOps - 1);
5911 if (!ImpUse.isReg() || !ImpUse.isUse() ||
5912 !isSubRegOf(TRI: RI, SuperVec: ImpUse, SubReg: IsDst ? *Dst : Src0)) {
5913 ErrInfo = "src0 should be subreg of implicit vector use";
5914 return false;
5915 }
5916 }
5917
5918 // Make sure we aren't losing exec uses in the td files. This mostly requires
5919 // being careful when using let Uses to try to add other use registers.
5920 if (shouldReadExec(MI)) {
5921 if (!MI.hasRegisterImplicitUseOperand(Reg: AMDGPU::EXEC)) {
5922 ErrInfo = "VALU instruction does not implicitly read exec mask";
5923 return false;
5924 }
5925 }
5926
5927 if (isSMRD(MI)) {
5928 if (MI.mayStore() &&
5929 ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) {
5930 // The register offset form of scalar stores may only use m0 as the
5931 // soffset register.
5932 const MachineOperand *Soff = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
5933 if (Soff && Soff->getReg() != AMDGPU::M0) {
5934 ErrInfo = "scalar stores must use m0 as offset register";
5935 return false;
5936 }
5937 }
5938 }
5939
5940 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) {
5941 const MachineOperand *Offset = getNamedOperand(MI, OperandName: AMDGPU::OpName::offset);
5942 if (Offset->getImm() != 0) {
5943 ErrInfo = "subtarget does not support offsets in flat instructions";
5944 return false;
5945 }
5946 }
5947
5948 if (isDS(MI) && !ST.hasGDS()) {
5949 const MachineOperand *GDSOp = getNamedOperand(MI, OperandName: AMDGPU::OpName::gds);
5950 if (GDSOp && GDSOp->getImm() != 0) {
5951 ErrInfo = "GDS is not supported on this subtarget";
5952 return false;
5953 }
5954 }
5955
5956 if (isImage(MI)) {
5957 const MachineOperand *DimOp = getNamedOperand(MI, OperandName: AMDGPU::OpName::dim);
5958 if (DimOp) {
5959 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
5960 Name: AMDGPU::OpName::vaddr0);
5961 AMDGPU::OpName RSrcOpName =
5962 isMIMG(MI) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
5963 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: RSrcOpName);
5964 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: Opcode);
5965 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
5966 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
5967 const AMDGPU::MIMGDimInfo *Dim =
5968 AMDGPU::getMIMGDimInfoByEncoding(DimEnc: DimOp->getImm());
5969
5970 if (!Dim) {
5971 ErrInfo = "dim is out of range";
5972 return false;
5973 }
5974
5975 bool IsA16 = false;
5976 if (ST.hasR128A16()) {
5977 const MachineOperand *R128A16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::r128);
5978 IsA16 = R128A16->getImm() != 0;
5979 } else if (ST.hasA16()) {
5980 const MachineOperand *A16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::a16);
5981 IsA16 = A16->getImm() != 0;
5982 }
5983
5984 bool IsNSA = RsrcIdx - VAddr0Idx > 1;
5985
5986 unsigned AddrWords =
5987 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, IsG16Supported: ST.hasG16());
5988
5989 unsigned VAddrWords;
5990 if (IsNSA) {
5991 VAddrWords = RsrcIdx - VAddr0Idx;
5992 if (ST.hasPartialNSAEncoding() &&
5993 AddrWords > ST.getNSAMaxSize(HasSampler: isVSAMPLE(MI))) {
5994 unsigned LastVAddrIdx = RsrcIdx - 1;
5995 VAddrWords += getOpSize(MI, OpNo: LastVAddrIdx) / 4 - 1;
5996 }
5997 } else {
5998 VAddrWords = getOpSize(MI, OpNo: VAddr0Idx) / 4;
5999 if (AddrWords > 12)
6000 AddrWords = 16;
6001 }
6002
6003 if (VAddrWords != AddrWords) {
6004 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords
6005 << " but got " << VAddrWords << "\n");
6006 ErrInfo = "bad vaddr size";
6007 return false;
6008 }
6009 }
6010 }
6011
6012 const MachineOperand *DppCt = getNamedOperand(MI, OperandName: AMDGPU::OpName::dpp_ctrl);
6013 if (DppCt) {
6014 using namespace AMDGPU::DPP;
6015
6016 unsigned DC = DppCt->getImm();
6017 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
6018 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
6019 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) ||
6020 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) ||
6021 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) ||
6022 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) ||
6023 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) {
6024 ErrInfo = "Invalid dpp_ctrl value";
6025 return false;
6026 }
6027 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
6028 !ST.hasDPPWavefrontShifts()) {
6029 ErrInfo = "Invalid dpp_ctrl value: "
6030 "wavefront shifts are not supported on GFX10+";
6031 return false;
6032 }
6033 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
6034 !ST.hasDPPBroadcasts()) {
6035 ErrInfo = "Invalid dpp_ctrl value: "
6036 "broadcasts are not supported on GFX10+";
6037 return false;
6038 }
6039 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST &&
6040 ST.getGeneration() < AMDGPUSubtarget::GFX10) {
6041 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST &&
6042 DC <= DppCtrl::ROW_NEWBCAST_LAST &&
6043 !ST.hasGFX90AInsts()) {
6044 ErrInfo = "Invalid dpp_ctrl value: "
6045 "row_newbroadcast/row_share is not supported before "
6046 "GFX90A/GFX10";
6047 return false;
6048 }
6049 if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) {
6050 ErrInfo = "Invalid dpp_ctrl value: "
6051 "row_share and row_xmask are not supported before GFX10";
6052 return false;
6053 }
6054 }
6055
6056 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&
6057 !AMDGPU::isLegalDPALU_DPPControl(ST, DC) &&
6058 AMDGPU::isDPALU_DPP(OpDesc: Desc, MII: *this, ST)) {
6059 ErrInfo = "Invalid dpp_ctrl value: "
6060 "DP ALU dpp only support row_newbcast";
6061 return false;
6062 }
6063 }
6064
6065 if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) {
6066 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
6067 AMDGPU::OpName DataName =
6068 isDS(Opcode) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata;
6069 const MachineOperand *Data = getNamedOperand(MI, OperandName: DataName);
6070 const MachineOperand *Data2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::data1);
6071 if (Data && !Data->isReg())
6072 Data = nullptr;
6073
6074 if (!ST.hasGFX90AInsts()) {
6075 if ((Dst && RI.isAGPR(MRI, Reg: Dst->getReg())) ||
6076 (Data && RI.isAGPR(MRI, Reg: Data->getReg())) ||
6077 (Data2 && RI.isAGPR(MRI, Reg: Data2->getReg()))) {
6078 ErrInfo = "Invalid register class: "
6079 "agpr loads and stores not supported on this GPU";
6080 return false;
6081 }
6082 }
6083 }
6084
6085 if (ST.needsAlignedVGPRs()) {
6086 const auto isAlignedReg = [&MI, &MRI, this](AMDGPU::OpName OpName) -> bool {
6087 const MachineOperand *Op = getNamedOperand(MI, OperandName: OpName);
6088 if (!Op)
6089 return true;
6090 Register Reg = Op->getReg();
6091 if (Reg.isPhysical())
6092 return !(RI.getHWRegIndex(Reg) & 1);
6093 const TargetRegisterClass &RC = *MRI.getRegClass(Reg);
6094 return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
6095 !(RI.getChannelFromSubReg(SubReg: Op->getSubReg()) & 1);
6096 };
6097
6098 if (isMIMG(MI)) {
6099 if (!isAlignedReg(AMDGPU::OpName::vaddr)) {
6100 ErrInfo = "Subtarget requires even aligned vector registers "
6101 "for vaddr operand of image instructions";
6102 return false;
6103 }
6104 }
6105 }
6106
6107 if (Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts()) {
6108 const MachineOperand *Src = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
6109 if (Src->isReg() && RI.isSGPRReg(MRI, Reg: Src->getReg())) {
6110 ErrInfo = "Invalid register class: "
6111 "v_accvgpr_write with an SGPR is not supported on this GPU";
6112 return false;
6113 }
6114 }
6115
6116 if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) {
6117 const MachineOperand &SrcOp = MI.getOperand(i: 1);
6118 if (!SrcOp.isReg() || SrcOp.getReg().isVirtual()) {
6119 ErrInfo = "pseudo expects only physical SGPRs";
6120 return false;
6121 }
6122 }
6123
6124 if (const MachineOperand *CPol = getNamedOperand(MI, OperandName: AMDGPU::OpName::cpol)) {
6125 if (CPol->getImm() & AMDGPU::CPol::SCAL) {
6126 if (!ST.hasScaleOffset()) {
6127 ErrInfo = "Subtarget does not support offset scaling";
6128 return false;
6129 }
6130 if (!AMDGPU::supportsScaleOffset(MII: *this, Opcode: MI.getOpcode())) {
6131 ErrInfo = "Instruction does not support offset scaling";
6132 return false;
6133 }
6134 }
6135 }
6136
6137 // See SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
6138 if (AMDGPU::isSingleSGPRReadInst(Opc: Opcode)) {
6139 for (unsigned I = 0; I < 3; ++I) {
6140 if (!isLegalSingleSGPRReadInstOperand(MRI, MI, SrcN: I))
6141 return false;
6142 }
6143 }
6144
6145 if (ST.hasFlatScratchHiInB64InstHazard() && isSALU(MI) &&
6146 MI.readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, TRI: nullptr)) {
6147 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
6148 if ((Dst && RI.getRegClassForReg(MRI, Reg: Dst->getReg()) ==
6149 &AMDGPU::SReg_64RegClass) ||
6150 Opcode == AMDGPU::S_BITCMP0_B64 || Opcode == AMDGPU::S_BITCMP1_B64) {
6151 ErrInfo = "Instruction cannot read flat_scratch_base_hi";
6152 return false;
6153 }
6154 }
6155
6156 return true;
6157}
6158
6159unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
6160 if (MI.getOpcode() == AMDGPU::S_MOV_B32) {
6161 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6162 return MI.getOperand(i: 1).isReg() || RI.isAGPR(MRI, Reg: MI.getOperand(i: 0).getReg())
6163 ? AMDGPU::COPY
6164 : AMDGPU::V_MOV_B32_e32;
6165 }
6166 return getVALUOp(Opc: MI.getOpcode());
6167}
6168
6169// It is more readable to list mapped opcodes on the same line.
6170// clang-format off
6171
6172unsigned SIInstrInfo::getVALUOp(unsigned Opc) const {
6173 switch (Opc) {
6174 default: return AMDGPU::INSTRUCTION_LIST_END;
6175 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
6176 case AMDGPU::COPY: return AMDGPU::COPY;
6177 case AMDGPU::PHI: return AMDGPU::PHI;
6178 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
6179 case AMDGPU::WQM: return AMDGPU::WQM;
6180 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
6181 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM;
6182 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM;
6183 case AMDGPU::S_ADD_I32:
6184 return ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
6185 case AMDGPU::S_ADDC_U32:
6186 return AMDGPU::V_ADDC_U32_e32;
6187 case AMDGPU::S_SUB_I32:
6188 return ST.hasAddNoCarryInsts() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32;
6189 // FIXME: These are not consistently handled, and selected when the carry is
6190 // used.
6191 case AMDGPU::S_ADD_U32:
6192 return AMDGPU::V_ADD_CO_U32_e32;
6193 case AMDGPU::S_SUB_U32:
6194 return AMDGPU::V_SUB_CO_U32_e32;
6195 case AMDGPU::S_ADD_U64_PSEUDO:
6196 return AMDGPU::V_ADD_U64_PSEUDO;
6197 case AMDGPU::S_SUB_U64_PSEUDO:
6198 return AMDGPU::V_SUB_U64_PSEUDO;
6199 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
6200 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64;
6201 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64;
6202 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64;
6203 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
6204 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
6205 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
6206 case AMDGPU::S_XNOR_B32:
6207 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
6208 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
6209 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
6210 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
6211 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
6212 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
6213 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64;
6214 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
6215 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64;
6216 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
6217 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64;
6218 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64;
6219 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64;
6220 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64;
6221 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64;
6222 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
6223 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
6224 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
6225 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
6226 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64;
6227 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64;
6228 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64;
6229 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64;
6230 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64;
6231 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64;
6232 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64;
6233 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64;
6234 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64;
6235 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64;
6236 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64;
6237 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64;
6238 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64;
6239 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64;
6240 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
6241 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
6242 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
6243 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
6244 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
6245 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
6246 case AMDGPU::S_CVT_F32_I32: return AMDGPU::V_CVT_F32_I32_e64;
6247 case AMDGPU::S_CVT_F32_U32: return AMDGPU::V_CVT_F32_U32_e64;
6248 case AMDGPU::S_CVT_I32_F32: return AMDGPU::V_CVT_I32_F32_e64;
6249 case AMDGPU::S_CVT_U32_F32: return AMDGPU::V_CVT_U32_F32_e64;
6250 case AMDGPU::S_CVT_F32_F16:
6251 case AMDGPU::S_CVT_HI_F32_F16:
6252 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F32_F16_t16_e64
6253 : AMDGPU::V_CVT_F32_F16_fake16_e64;
6254 case AMDGPU::S_CVT_F16_F32:
6255 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F16_F32_t16_e64
6256 : AMDGPU::V_CVT_F16_F32_fake16_e64;
6257 case AMDGPU::S_CEIL_F32: return AMDGPU::V_CEIL_F32_e64;
6258 case AMDGPU::S_FLOOR_F32: return AMDGPU::V_FLOOR_F32_e64;
6259 case AMDGPU::S_TRUNC_F32: return AMDGPU::V_TRUNC_F32_e64;
6260 case AMDGPU::S_RNDNE_F32: return AMDGPU::V_RNDNE_F32_e64;
6261 case AMDGPU::S_CEIL_F16:
6262 return ST.useRealTrue16Insts() ? AMDGPU::V_CEIL_F16_t16_e64
6263 : AMDGPU::V_CEIL_F16_fake16_e64;
6264 case AMDGPU::S_FLOOR_F16:
6265 return ST.useRealTrue16Insts() ? AMDGPU::V_FLOOR_F16_t16_e64
6266 : AMDGPU::V_FLOOR_F16_fake16_e64;
6267 case AMDGPU::S_TRUNC_F16:
6268 return ST.useRealTrue16Insts() ? AMDGPU::V_TRUNC_F16_t16_e64
6269 : AMDGPU::V_TRUNC_F16_fake16_e64;
6270 case AMDGPU::S_RNDNE_F16:
6271 return ST.useRealTrue16Insts() ? AMDGPU::V_RNDNE_F16_t16_e64
6272 : AMDGPU::V_RNDNE_F16_fake16_e64;
6273 case AMDGPU::S_ADD_F32: return AMDGPU::V_ADD_F32_e64;
6274 case AMDGPU::S_SUB_F32: return AMDGPU::V_SUB_F32_e64;
6275 case AMDGPU::S_MIN_F32: return AMDGPU::V_MIN_F32_e64;
6276 case AMDGPU::S_MAX_F32: return AMDGPU::V_MAX_F32_e64;
6277 case AMDGPU::S_MINIMUM_F32: return AMDGPU::V_MINIMUM_F32_e64;
6278 case AMDGPU::S_MAXIMUM_F32: return AMDGPU::V_MAXIMUM_F32_e64;
6279 case AMDGPU::S_MUL_F32: return AMDGPU::V_MUL_F32_e64;
6280 case AMDGPU::S_ADD_F16:
6281 return ST.useRealTrue16Insts() ? AMDGPU::V_ADD_F16_t16_e64
6282 : AMDGPU::V_ADD_F16_fake16_e64;
6283 case AMDGPU::S_SUB_F16:
6284 return ST.useRealTrue16Insts() ? AMDGPU::V_SUB_F16_t16_e64
6285 : AMDGPU::V_SUB_F16_fake16_e64;
6286 case AMDGPU::S_MIN_F16:
6287 return ST.useRealTrue16Insts() ? AMDGPU::V_MIN_F16_t16_e64
6288 : AMDGPU::V_MIN_F16_fake16_e64;
6289 case AMDGPU::S_MAX_F16:
6290 return ST.useRealTrue16Insts() ? AMDGPU::V_MAX_F16_t16_e64
6291 : AMDGPU::V_MAX_F16_fake16_e64;
6292 case AMDGPU::S_MINIMUM_F16:
6293 return ST.useRealTrue16Insts() ? AMDGPU::V_MINIMUM_F16_t16_e64
6294 : AMDGPU::V_MINIMUM_F16_fake16_e64;
6295 case AMDGPU::S_MAXIMUM_F16:
6296 return ST.useRealTrue16Insts() ? AMDGPU::V_MAXIMUM_F16_t16_e64
6297 : AMDGPU::V_MAXIMUM_F16_fake16_e64;
6298 case AMDGPU::S_MUL_F16:
6299 return ST.useRealTrue16Insts() ? AMDGPU::V_MUL_F16_t16_e64
6300 : AMDGPU::V_MUL_F16_fake16_e64;
6301 case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64;
6302 case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64;
6303 case AMDGPU::S_FMAC_F16:
6304 return ST.useRealTrue16Insts() ? AMDGPU::V_FMAC_F16_t16_e64
6305 : AMDGPU::V_FMAC_F16_fake16_e64;
6306 case AMDGPU::S_FMAMK_F32: return AMDGPU::V_FMAMK_F32;
6307 case AMDGPU::S_FMAAK_F32: return AMDGPU::V_FMAAK_F32;
6308 case AMDGPU::S_CMP_LT_F32: return AMDGPU::V_CMP_LT_F32_e64;
6309 case AMDGPU::S_CMP_EQ_F32: return AMDGPU::V_CMP_EQ_F32_e64;
6310 case AMDGPU::S_CMP_LE_F32: return AMDGPU::V_CMP_LE_F32_e64;
6311 case AMDGPU::S_CMP_GT_F32: return AMDGPU::V_CMP_GT_F32_e64;
6312 case AMDGPU::S_CMP_LG_F32: return AMDGPU::V_CMP_LG_F32_e64;
6313 case AMDGPU::S_CMP_GE_F32: return AMDGPU::V_CMP_GE_F32_e64;
6314 case AMDGPU::S_CMP_O_F32: return AMDGPU::V_CMP_O_F32_e64;
6315 case AMDGPU::S_CMP_U_F32: return AMDGPU::V_CMP_U_F32_e64;
6316 case AMDGPU::S_CMP_NGE_F32: return AMDGPU::V_CMP_NGE_F32_e64;
6317 case AMDGPU::S_CMP_NLG_F32: return AMDGPU::V_CMP_NLG_F32_e64;
6318 case AMDGPU::S_CMP_NGT_F32: return AMDGPU::V_CMP_NGT_F32_e64;
6319 case AMDGPU::S_CMP_NLE_F32: return AMDGPU::V_CMP_NLE_F32_e64;
6320 case AMDGPU::S_CMP_NEQ_F32: return AMDGPU::V_CMP_NEQ_F32_e64;
6321 case AMDGPU::S_CMP_NLT_F32: return AMDGPU::V_CMP_NLT_F32_e64;
6322 case AMDGPU::S_CMP_LT_F16:
6323 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LT_F16_t16_e64
6324 : AMDGPU::V_CMP_LT_F16_fake16_e64;
6325 case AMDGPU::S_CMP_EQ_F16:
6326 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_EQ_F16_t16_e64
6327 : AMDGPU::V_CMP_EQ_F16_fake16_e64;
6328 case AMDGPU::S_CMP_LE_F16:
6329 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LE_F16_t16_e64
6330 : AMDGPU::V_CMP_LE_F16_fake16_e64;
6331 case AMDGPU::S_CMP_GT_F16:
6332 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GT_F16_t16_e64
6333 : AMDGPU::V_CMP_GT_F16_fake16_e64;
6334 case AMDGPU::S_CMP_LG_F16:
6335 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LG_F16_t16_e64
6336 : AMDGPU::V_CMP_LG_F16_fake16_e64;
6337 case AMDGPU::S_CMP_GE_F16:
6338 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GE_F16_t16_e64
6339 : AMDGPU::V_CMP_GE_F16_fake16_e64;
6340 case AMDGPU::S_CMP_O_F16:
6341 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_O_F16_t16_e64
6342 : AMDGPU::V_CMP_O_F16_fake16_e64;
6343 case AMDGPU::S_CMP_U_F16:
6344 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_U_F16_t16_e64
6345 : AMDGPU::V_CMP_U_F16_fake16_e64;
6346 case AMDGPU::S_CMP_NGE_F16:
6347 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGE_F16_t16_e64
6348 : AMDGPU::V_CMP_NGE_F16_fake16_e64;
6349 case AMDGPU::S_CMP_NLG_F16:
6350 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLG_F16_t16_e64
6351 : AMDGPU::V_CMP_NLG_F16_fake16_e64;
6352 case AMDGPU::S_CMP_NGT_F16:
6353 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGT_F16_t16_e64
6354 : AMDGPU::V_CMP_NGT_F16_fake16_e64;
6355 case AMDGPU::S_CMP_NLE_F16:
6356 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLE_F16_t16_e64
6357 : AMDGPU::V_CMP_NLE_F16_fake16_e64;
6358 case AMDGPU::S_CMP_NEQ_F16:
6359 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NEQ_F16_t16_e64
6360 : AMDGPU::V_CMP_NEQ_F16_fake16_e64;
6361 case AMDGPU::S_CMP_NLT_F16:
6362 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLT_F16_t16_e64
6363 : AMDGPU::V_CMP_NLT_F16_fake16_e64;
6364 case AMDGPU::V_S_EXP_F32_e64: return AMDGPU::V_EXP_F32_e64;
6365 case AMDGPU::V_S_EXP_F16_e64:
6366 return ST.useRealTrue16Insts() ? AMDGPU::V_EXP_F16_t16_e64
6367 : AMDGPU::V_EXP_F16_fake16_e64;
6368 case AMDGPU::V_S_LOG_F32_e64: return AMDGPU::V_LOG_F32_e64;
6369 case AMDGPU::V_S_LOG_F16_e64:
6370 return ST.useRealTrue16Insts() ? AMDGPU::V_LOG_F16_t16_e64
6371 : AMDGPU::V_LOG_F16_fake16_e64;
6372 case AMDGPU::V_S_RCP_F32_e64: return AMDGPU::V_RCP_F32_e64;
6373 case AMDGPU::V_S_RCP_F16_e64:
6374 return ST.useRealTrue16Insts() ? AMDGPU::V_RCP_F16_t16_e64
6375 : AMDGPU::V_RCP_F16_fake16_e64;
6376 case AMDGPU::V_S_RSQ_F32_e64: return AMDGPU::V_RSQ_F32_e64;
6377 case AMDGPU::V_S_RSQ_F16_e64:
6378 return ST.useRealTrue16Insts() ? AMDGPU::V_RSQ_F16_t16_e64
6379 : AMDGPU::V_RSQ_F16_fake16_e64;
6380 case AMDGPU::V_S_SQRT_F32_e64: return AMDGPU::V_SQRT_F32_e64;
6381 case AMDGPU::V_S_SQRT_F16_e64:
6382 return ST.useRealTrue16Insts() ? AMDGPU::V_SQRT_F16_t16_e64
6383 : AMDGPU::V_SQRT_F16_fake16_e64;
6384 }
6385 llvm_unreachable(
6386 "Unexpected scalar opcode without corresponding vector one!");
6387}
6388
6389// clang-format on
6390
6391void SIInstrInfo::insertScratchExecCopy(MachineFunction &MF,
6392 MachineBasicBlock &MBB,
6393 MachineBasicBlock::iterator MBBI,
6394 const DebugLoc &DL, Register Reg,
6395 bool IsSCCLive,
6396 SlotIndexes *Indexes) const {
6397 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
6398 const SIInstrInfo *TII = ST.getInstrInfo();
6399 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
6400 if (IsSCCLive) {
6401 // Insert two move instructions, one to save the original value of EXEC and
6402 // the other to turn on all bits in EXEC. This is required as we can't use
6403 // the single instruction S_OR_SAVEEXEC that clobbers SCC.
6404 auto StoreExecMI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: Reg)
6405 .addReg(RegNo: LMC.ExecReg, Flags: RegState::Kill);
6406 auto FlipExecMI =
6407 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg).addImm(Val: -1);
6408 if (Indexes) {
6409 Indexes->insertMachineInstrInMaps(MI&: *StoreExecMI);
6410 Indexes->insertMachineInstrInMaps(MI&: *FlipExecMI);
6411 }
6412 } else {
6413 auto SaveExec =
6414 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.OrSaveExecOpc), DestReg: Reg).addImm(Val: -1);
6415 SaveExec->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
6416 if (Indexes)
6417 Indexes->insertMachineInstrInMaps(MI&: *SaveExec);
6418 }
6419}
6420
6421void SIInstrInfo::restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
6422 MachineBasicBlock::iterator MBBI,
6423 const DebugLoc &DL, Register Reg,
6424 SlotIndexes *Indexes) const {
6425 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
6426 auto ExecRestoreMI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
6427 .addReg(RegNo: Reg, Flags: RegState::Kill);
6428 if (Indexes)
6429 Indexes->insertMachineInstrInMaps(MI&: *ExecRestoreMI);
6430}
6431
6432MachineInstr *
6433SIInstrInfo::getWholeWaveFunctionSetup(MachineFunction &MF) const {
6434 assert(MF.getInfo<SIMachineFunctionInfo>()->isWholeWaveFunction() &&
6435 "Not a whole wave func");
6436 MachineBasicBlock &MBB = *MF.begin();
6437 for (MachineInstr &MI : MBB)
6438 if (MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_SETUP ||
6439 MI.getOpcode() == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
6440 return &MI;
6441
6442 llvm_unreachable("Couldn't find SI_SETUP_WHOLE_WAVE_FUNC instruction");
6443}
6444
6445const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
6446 unsigned OpNo) const {
6447 const MCInstrDesc &Desc = get(Opcode: MI.getOpcode());
6448 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
6449 Desc.operands()[OpNo].RegClass == -1) {
6450 Register Reg = MI.getOperand(i: OpNo).getReg();
6451
6452 if (Reg.isVirtual()) {
6453 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6454 return MRI.getRegClass(Reg);
6455 }
6456 return RI.getPhysRegBaseClass(Reg);
6457 }
6458
6459 int16_t RegClass = getOpRegClassID(OpInfo: Desc.operands()[OpNo]);
6460 return RegClass < 0 ? nullptr : RI.getRegClass(i: RegClass);
6461}
6462
6463// Convert VOP3 operand index to source number.
6464static unsigned VOP3OpIdxToSrcN(const MachineInstr &MI, unsigned OpIdx) {
6465 constexpr AMDGPU::OpName OpNames[] = {
6466 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};
6467
6468 for (auto [I, OpName] : enumerate(First: OpNames)) {
6469 int SrcIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[I]);
6470 if (static_cast<unsigned>(SrcIdx) == OpIdx)
6471 return I;
6472 }
6473
6474 return UINT_MAX;
6475}
6476
6477void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
6478 MachineBasicBlock::iterator I = MI;
6479 MachineBasicBlock *MBB = MI.getParent();
6480 MachineOperand &MO = MI.getOperand(i: OpIdx);
6481 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
6482 unsigned RCID = getOpRegClassID(OpInfo: get(Opcode: MI.getOpcode()).operands()[OpIdx]);
6483 const TargetRegisterClass *RC = RI.getRegClass(i: RCID);
6484 unsigned Size = RI.getRegSizeInBits(RC: *RC);
6485 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO
6486 : Size == 16 ? AMDGPU::V_MOV_B16_t16_e64
6487 : AMDGPU::V_MOV_B32_e32;
6488 if (MO.isReg())
6489 Opcode = AMDGPU::COPY;
6490 else if (RI.isSGPRClass(RC))
6491 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
6492
6493 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(SRC: RC);
6494 Register Reg = MRI.createVirtualRegister(RegClass: VRC);
6495 DebugLoc DL = MBB->findDebugLoc(MBBI: I);
6496
6497 if (Size == 128 && AMDGPU::isPackedSingleSGPR64BitInst(Opc: MI.getOpcode()) &&
6498 isLegalSingleSGPRReadInstOperand(MRI, MI, SrcN: VOP3OpIdxToSrcN(MI, OpIdx))) {
6499 // Special case for V_PK_*64 instructions: these do not have OPSEL but SGPR
6500 // sources behave like OPSEL is set replicating low 64-bits into high. VGPR
6501 // sources in turn read actual 4 registers. To move operand from an SGPR to
6502 // a VGPR we need to replicate low half.
6503 // We also do not select immediates for these instructions so it always has
6504 // to be an SGPR register here.
6505 // Operands which are not legal as per isLegalSingleSGPRReadInstOperand()
6506 // sent here specifically to fix a non-splat SGPR and shall perform a full
6507 // copy.
6508
6509 const TargetRegisterClass *VRC64 = RI.getVGPRClassForBitWidth(BitWidth: 64);
6510 Register Low64 = MRI.createVirtualRegister(RegClass: VRC64);
6511 assert(MO.isReg() && RI.isSGPRReg(MRI, MO.getReg()));
6512 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Low64)
6513 .addReg(RegNo: MO.getReg(), Flags: {}, SubReg: AMDGPU::sub0_sub1);
6514 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Reg)
6515 .addReg(RegNo: Low64)
6516 .addImm(Val: AMDGPU::sub0_sub1)
6517 .addReg(RegNo: Low64, Flags: RegState::Kill)
6518 .addImm(Val: AMDGPU::sub2_sub3);
6519 } else if (Opcode == AMDGPU::V_MOV_B16_t16_e64) {
6520 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: Reg)
6521 .addImm(Val: 0) // src0_modifiers
6522 .add(MO)
6523 .addImm(Val: 0); // op_sel
6524 } else {
6525 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: Reg).add(MO);
6526 }
6527
6528 MO.ChangeToRegister(Reg, isDef: false);
6529}
6530
6531unsigned SIInstrInfo::buildExtractSubReg(
6532 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
6533 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
6534 unsigned SubIdx, const TargetRegisterClass *SubRC) const {
6535 if (!SuperReg.getReg().isVirtual())
6536 return RI.getSubReg(Reg: SuperReg.getReg(), Idx: SubIdx);
6537
6538 MachineBasicBlock *MBB = MI->getParent();
6539 const DebugLoc &DL = MI->getDebugLoc();
6540 Register SubReg = MRI.createVirtualRegister(RegClass: SubRC);
6541
6542 unsigned NewSubIdx = RI.composeSubRegIndices(a: SuperReg.getSubReg(), b: SubIdx);
6543 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: SubReg)
6544 .addReg(RegNo: SuperReg.getReg(), Flags: {}, SubReg: NewSubIdx);
6545 return SubReg;
6546}
6547
6548MachineOperand SIInstrInfo::buildExtractSubRegOrImm(
6549 MachineBasicBlock::iterator MII, MachineRegisterInfo &MRI,
6550 const MachineOperand &Op, const TargetRegisterClass *SuperRC,
6551 unsigned SubIdx, const TargetRegisterClass *SubRC) const {
6552 if (Op.isImm()) {
6553 if (SubIdx == AMDGPU::sub0)
6554 return MachineOperand::CreateImm(Val: static_cast<int32_t>(Op.getImm()));
6555 if (SubIdx == AMDGPU::sub1)
6556 return MachineOperand::CreateImm(Val: static_cast<int32_t>(Op.getImm() >> 32));
6557
6558 llvm_unreachable("Unhandled register index for immediate");
6559 }
6560
6561 unsigned SubReg = buildExtractSubReg(MI: MII, MRI, SuperReg: Op, SuperRC,
6562 SubIdx, SubRC);
6563 return MachineOperand::CreateReg(Reg: SubReg, isDef: false);
6564}
6565
6566// Change the order of operands from (0, 1, 2) to (0, 2, 1)
6567void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
6568 assert(Inst.getNumExplicitOperands() == 3);
6569 MachineOperand Op1 = Inst.getOperand(i: 1);
6570 Inst.removeOperand(OpNo: 1);
6571 Inst.addOperand(Op: Op1);
6572}
6573
6574bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
6575 const MCOperandInfo &OpInfo,
6576 const MachineOperand &MO) const {
6577 if (!MO.isReg())
6578 return false;
6579
6580 Register Reg = MO.getReg();
6581
6582 const TargetRegisterClass *DRC = RI.getRegClass(i: getOpRegClassID(OpInfo));
6583 if (Reg.isPhysical())
6584 return DRC->contains(Reg);
6585
6586 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
6587
6588 if (MO.getSubReg()) {
6589 const TargetRegisterClass *SuperRC =
6590 RI.getLargestLegalSuperClass(RC, MF: MRI.getMF());
6591 if (!SuperRC)
6592 return false;
6593 return RI.getMatchingSuperRegClass(A: SuperRC, B: DRC, Idx: MO.getSubReg()) != nullptr;
6594 }
6595
6596 return RI.getCommonSubClass(A: DRC, B: RC) != nullptr;
6597}
6598
6599bool SIInstrInfo::isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
6600 const MachineOperand &MO) const {
6601 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6602 const MCOperandInfo OpInfo = MI.getDesc().operands()[OpIdx];
6603 unsigned Opc = MI.getOpcode();
6604
6605 // See SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
6606 if (MO.isReg() && RI.isSGPRReg(MRI, Reg: MO.getReg()) &&
6607 AMDGPU::isSingleSGPRReadInst(Opc: MI.getOpcode()) &&
6608 !isLegalSingleSGPRReadInstOperand(MRI, MI, SrcN: VOP3OpIdxToSrcN(MI, OpIdx),
6609 MO: &MO))
6610 return false;
6611
6612 if (!isLegalRegOperand(MRI, OpInfo, MO))
6613 return false;
6614
6615 // check Accumulate GPR operand
6616 bool IsAGPR = RI.isAGPR(MRI, Reg: MO.getReg());
6617 if (IsAGPR && !ST.hasMAIInsts())
6618 return false;
6619 if (IsAGPR && (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
6620 (MI.mayLoad() || MI.mayStore() || isDS(Opcode: Opc) || isMIMG(Opcode: Opc)))
6621 return false;
6622 // Atomics should have both vdst and vdata either vgpr or agpr.
6623 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
6624 const int DataIdx = AMDGPU::getNamedOperandIdx(
6625 Opcode: Opc, Name: isDS(Opcode: Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata);
6626 if ((int)OpIdx == VDstIdx && DataIdx != -1 &&
6627 MI.getOperand(i: DataIdx).isReg() &&
6628 RI.isAGPR(MRI, Reg: MI.getOperand(i: DataIdx).getReg()) != IsAGPR)
6629 return false;
6630 if ((int)OpIdx == DataIdx) {
6631 if (VDstIdx != -1 &&
6632 RI.isAGPR(MRI, Reg: MI.getOperand(i: VDstIdx).getReg()) != IsAGPR)
6633 return false;
6634 // DS instructions with 2 src operands also must have tied RC.
6635 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data1);
6636 if (Data1Idx != -1 && MI.getOperand(i: Data1Idx).isReg() &&
6637 RI.isAGPR(MRI, Reg: MI.getOperand(i: Data1Idx).getReg()) != IsAGPR)
6638 return false;
6639 }
6640
6641 // Check V_ACCVGPR_WRITE_B32_e64
6642 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() &&
6643 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0) &&
6644 RI.isSGPRReg(MRI, Reg: MO.getReg()))
6645 return false;
6646
6647 if (ST.hasFlatScratchHiInB64InstHazard() &&
6648 MO.getReg() == AMDGPU::SRC_FLAT_SCRATCH_BASE_HI && isSALU(MI)) {
6649 if (const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst)) {
6650 if (AMDGPU::getRegBitWidth(RC: *RI.getRegClassForReg(MRI, Reg: Dst->getReg())) ==
6651 64)
6652 return false;
6653 }
6654 if (Opc == AMDGPU::S_BITCMP0_B64 || Opc == AMDGPU::S_BITCMP1_B64)
6655 return false;
6656 }
6657 if (!ST.hasDPPSrc1SGPR() && isDPP(MI) && RI.isSGPRReg(MRI, Reg: MO.getReg()) &&
6658 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1))
6659 return false;
6660
6661 return true;
6662}
6663
6664bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI,
6665 const MCOperandInfo &OpInfo,
6666 const MachineOperand &MO) const {
6667 if (MO.isReg())
6668 return isLegalRegOperand(MRI, OpInfo, MO);
6669
6670 // Handle non-register types that are treated like immediates.
6671 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
6672 return true;
6673}
6674
6675bool SIInstrInfo::isLegalSingleSGPRReadInstOperand(
6676 const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,
6677 const MachineOperand *MO) const {
6678 constexpr unsigned NumOps = 3;
6679 constexpr AMDGPU::OpName OpNames[NumOps * 2] = {
6680 AMDGPU::OpName::src0, AMDGPU::OpName::src1,
6681 AMDGPU::OpName::src2, AMDGPU::OpName::src0_modifiers,
6682 AMDGPU::OpName::src1_modifiers, AMDGPU::OpName::src2_modifiers};
6683
6684 assert(SrcN < NumOps);
6685
6686 if (!MO) {
6687 int SrcIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[SrcN]);
6688 if (SrcIdx == -1)
6689 return true;
6690 MO = &MI.getOperand(i: SrcIdx);
6691 }
6692
6693 if (!MO->isReg() || !RI.isSGPRReg(MRI, Reg: MO->getReg()))
6694 return true;
6695
6696 int ModsIdx =
6697 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[NumOps + SrcN]);
6698 if (ModsIdx == -1)
6699 return false;
6700
6701 unsigned Mods = MI.getOperand(i: ModsIdx).getImm();
6702 bool OpSel = Mods & SISrcMods::OP_SEL_0;
6703 bool OpSelHi = Mods & SISrcMods::OP_SEL_1;
6704
6705 return !OpSel && !OpSelHi;
6706}
6707
6708bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
6709 const MachineOperand *MO) const {
6710 const MachineFunction &MF = *MI.getMF();
6711 const MachineRegisterInfo &MRI = MF.getRegInfo();
6712 const MCInstrDesc &InstDesc = MI.getDesc();
6713 const MCOperandInfo &OpInfo = InstDesc.operands()[OpIdx];
6714 int64_t RegClass = getOpRegClassID(OpInfo);
6715 const TargetRegisterClass *DefinedRC =
6716 RegClass != -1 ? RI.getRegClass(i: RegClass) : nullptr;
6717 if (!MO)
6718 MO = &MI.getOperand(i: OpIdx);
6719
6720 const bool IsInlineConst = !MO->isReg() && isInlineConstant(MO: *MO, OpInfo);
6721
6722 if (isVALU(MI, /*AllowLDSDMA=*/false) && !IsInlineConst &&
6723 usesConstantBus(MRI, MO: *MO, OpInfo)) {
6724 const MachineOperand *UsedLiteral = nullptr;
6725
6726 int ConstantBusLimit = ST.getConstantBusLimit(Opcode: MI.getOpcode());
6727 int LiteralLimit = !isVOP3(MI) || ST.hasVOP3Literal() ? 1 : 0;
6728
6729 // TODO: Be more permissive with frame indexes.
6730 if (!MO->isReg() && !isInlineConstant(MO: *MO, OpInfo)) {
6731 if (!LiteralLimit--)
6732 return false;
6733
6734 UsedLiteral = MO;
6735 }
6736
6737 SmallDenseSet<RegSubRegPair> SGPRsUsed;
6738 if (MO->isReg())
6739 SGPRsUsed.insert(V: RegSubRegPair(MO->getReg(), MO->getSubReg()));
6740
6741 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
6742 if (i == OpIdx)
6743 continue;
6744 const MachineOperand &Op = MI.getOperand(i);
6745 if (Op.isReg()) {
6746 if (Op.isUse()) {
6747 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
6748 if (regUsesConstantBus(RegOp: Op, MRI) && SGPRsUsed.insert(V: SGPR).second) {
6749 if (--ConstantBusLimit <= 0)
6750 return false;
6751 }
6752 }
6753 } else if (AMDGPU::isSISrcOperand(OpInfo: InstDesc.operands()[i]) &&
6754 !isInlineConstant(MO: Op, OpInfo: InstDesc.operands()[i])) {
6755 // The same literal may be used multiple times.
6756 if (!UsedLiteral)
6757 UsedLiteral = &Op;
6758 else if (UsedLiteral->isIdenticalTo(Other: Op))
6759 continue;
6760
6761 if (!LiteralLimit--)
6762 return false;
6763 if (--ConstantBusLimit <= 0)
6764 return false;
6765 }
6766 }
6767 } else if (!IsInlineConst && !MO->isReg() && isSALU(MI)) {
6768 // There can be at most one literal operand, but it can be repeated.
6769 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
6770 if (i == OpIdx)
6771 continue;
6772 const MachineOperand &Op = MI.getOperand(i);
6773 if (!Op.isReg() && !Op.isFI() && !Op.isRegMask() &&
6774 !isInlineConstant(MO: Op, OpInfo: InstDesc.operands()[i]) &&
6775 !Op.isIdenticalTo(Other: *MO))
6776 return false;
6777
6778 // Do not fold a non-inlineable and non-register operand into an
6779 // instruction that already has a frame index. The frame index handling
6780 // code could not handle well when a frame index co-exists with another
6781 // non-register operand, unless that operand is an inlineable immediate.
6782 if (Op.isFI())
6783 return false;
6784 }
6785 }
6786
6787 if (MO->isReg()) {
6788 if (!DefinedRC)
6789 return OpInfo.OperandType == MCOI::OPERAND_UNKNOWN;
6790 return isLegalRegOperand(MI, OpIdx, MO: *MO);
6791 }
6792
6793 if (MO->isImm()) {
6794 uint64_t Imm = MO->getImm();
6795 bool Is64BitFPOp = OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_FP64 ||
6796 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP64;
6797 bool Is64BitOp = Is64BitFPOp ||
6798 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64 ||
6799 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT32 ||
6800 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32 ||
6801 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT64;
6802 if (Is64BitOp &&
6803 !AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: ST.hasInv2PiInlineImm())) {
6804 if (!AMDGPU::isValid32BitLiteral(Val: Imm, IsFP64: Is64BitFPOp) &&
6805 (!ST.has64BitLiterals() || InstDesc.getSize() != 4))
6806 return false;
6807
6808 // FIXME: We can use sign extended 64-bit literals, but only for signed
6809 // operands. At the moment we do not know if an operand is signed.
6810 // Such operand will be encoded as its low 32 bits and then either
6811 // correctly sign extended or incorrectly zero extended by HW.
6812 // If 64-bit literals are supported and the literal will be encoded
6813 // as full 64 bit we still can use it.
6814 if (!Is64BitFPOp && (int32_t)Imm < 0 &&
6815 (!ST.has64BitLiterals() || AMDGPU::isValid32BitLiteral(Val: Imm, IsFP64: false)))
6816 return false;
6817 }
6818 }
6819
6820 // Handle non-register types that are treated like immediates.
6821 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
6822
6823 if (!DefinedRC) {
6824 // This operand expects an immediate.
6825 return true;
6826 }
6827
6828 return isImmOperandLegal(MI, OpNo: OpIdx, MO: *MO);
6829}
6830
6831bool SIInstrInfo::isNeverCoissue(MachineInstr &MI) const {
6832 bool IsGFX950Only = ST.hasGFX950Insts();
6833 bool IsGFX940Only = ST.hasGFX940Insts();
6834
6835 if (!IsGFX950Only && !IsGFX940Only)
6836 return false;
6837
6838 if (!isVALU(MI, /*AllowLDSDMA=*/false))
6839 return false;
6840
6841 // V_COS, V_EXP, V_RCP, etc.
6842 if (isTRANS(MI))
6843 return true;
6844
6845 // DOT2, DOT2C, DOT4, etc.
6846 if (isDOT(MI))
6847 return true;
6848
6849 // MFMA, SMFMA
6850 if (isMFMA(MI))
6851 return true;
6852
6853 unsigned Opcode = MI.getOpcode();
6854 switch (Opcode) {
6855 case AMDGPU::V_CVT_PK_BF8_F32_e64:
6856 case AMDGPU::V_CVT_PK_FP8_F32_e64:
6857 case AMDGPU::V_MQSAD_PK_U16_U8_e64:
6858 case AMDGPU::V_MQSAD_U32_U8_e64:
6859 case AMDGPU::V_PK_ADD_F16:
6860 case AMDGPU::V_PK_ADD_F32:
6861 case AMDGPU::V_PK_ADD_I16:
6862 case AMDGPU::V_PK_ADD_U16:
6863 case AMDGPU::V_PK_ASHRREV_I16:
6864 case AMDGPU::V_PK_FMA_F16:
6865 case AMDGPU::V_PK_FMA_F32:
6866 case AMDGPU::V_PK_FMAC_F16_e32:
6867 case AMDGPU::V_PK_FMAC_F16_e64:
6868 case AMDGPU::V_PK_LSHLREV_B16:
6869 case AMDGPU::V_PK_LSHRREV_B16:
6870 case AMDGPU::V_PK_MAD_I16:
6871 case AMDGPU::V_PK_MAD_U16:
6872 case AMDGPU::V_PK_MAX_F16:
6873 case AMDGPU::V_PK_MAX_I16:
6874 case AMDGPU::V_PK_MAX_U16:
6875 case AMDGPU::V_PK_MIN_F16:
6876 case AMDGPU::V_PK_MIN_I16:
6877 case AMDGPU::V_PK_MIN_U16:
6878 case AMDGPU::V_PK_MOV_B32:
6879 case AMDGPU::V_PK_MUL_F16:
6880 case AMDGPU::V_PK_MUL_F32:
6881 case AMDGPU::V_PK_MUL_LO_U16:
6882 case AMDGPU::V_PK_SUB_I16:
6883 case AMDGPU::V_PK_SUB_U16:
6884 case AMDGPU::V_QSAD_PK_U16_U8_e64:
6885 return true;
6886 default:
6887 return false;
6888 }
6889}
6890
6891void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
6892 MachineInstr &MI) const {
6893 unsigned Opc = MI.getOpcode();
6894 const MCInstrDesc &InstrDesc = get(Opcode: Opc);
6895
6896 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
6897 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
6898
6899 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
6900 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
6901
6902 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
6903 // we need to only have one constant bus use before GFX10.
6904 bool HasImplicitSGPR = findImplicitSGPRRead(MI);
6905 if (HasImplicitSGPR && ST.getConstantBusLimit(Opcode: Opc) <= 1 && Src0.isReg() &&
6906 RI.isSGPRReg(MRI, Reg: Src0.getReg()))
6907 legalizeOpWithMove(MI, OpIdx: Src0Idx);
6908
6909 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
6910 // both the value to write (src0) and lane select (src1). Fix up non-SGPR
6911 // src0/src1 with V_READFIRSTLANE.
6912 if (Opc == AMDGPU::V_WRITELANE_B32) {
6913 const DebugLoc &DL = MI.getDebugLoc();
6914 if (Src0.isReg() && RI.isVGPR(MRI, Reg: Src0.getReg())) {
6915 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6916 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
6917 .add(MO: Src0);
6918 Src0.ChangeToRegister(Reg, isDef: false);
6919 }
6920 if (Src1.isReg() && RI.isVGPR(MRI, Reg: Src1.getReg())) {
6921 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6922 const DebugLoc &DL = MI.getDebugLoc();
6923 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
6924 .add(MO: Src1);
6925 Src1.ChangeToRegister(Reg, isDef: false);
6926 }
6927 return;
6928 }
6929
6930 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2.
6931 if (Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F16_e32) {
6932 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
6933 if (!RI.isVGPR(MRI, Reg: MI.getOperand(i: Src2Idx).getReg()))
6934 legalizeOpWithMove(MI, OpIdx: Src2Idx);
6935 }
6936
6937 // VOP2 src0 instructions support all operand types, so we don't need to check
6938 // their legality. If src1 is already legal, we don't need to do anything.
6939 if (isLegalRegOperand(MRI, OpInfo: InstrDesc.operands()[Src1Idx], MO: Src1))
6940 return;
6941
6942 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
6943 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
6944 // select is uniform.
6945 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
6946 RI.isVGPR(MRI, Reg: Src1.getReg())) {
6947 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6948 const DebugLoc &DL = MI.getDebugLoc();
6949 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
6950 .add(MO: Src1);
6951 Src1.ChangeToRegister(Reg, isDef: false);
6952 return;
6953 }
6954
6955 // We do not use commuteInstruction here because it is too aggressive and will
6956 // commute if it is possible. We only want to commute here if it improves
6957 // legality. This can be called a fairly large number of times so don't waste
6958 // compile time pointlessly swapping and checking legality again.
6959 if (HasImplicitSGPR || !MI.isCommutable()) {
6960 legalizeOpWithMove(MI, OpIdx: Src1Idx);
6961 return;
6962 }
6963
6964 // If src0 can be used as src1, commuting will make the operands legal.
6965 // Otherwise we have to give up and insert a move.
6966 //
6967 // TODO: Other immediate-like operand kinds could be commuted if there was a
6968 // MachineOperand::ChangeTo* for them.
6969 if ((!Src1.isImm() && !Src1.isReg()) ||
6970 !isLegalRegOperand(MRI, OpInfo: InstrDesc.operands()[Src1Idx], MO: Src0)) {
6971 legalizeOpWithMove(MI, OpIdx: Src1Idx);
6972 return;
6973 }
6974
6975 int CommutedOpc = commuteOpcode(MI);
6976 if (CommutedOpc == -1) {
6977 legalizeOpWithMove(MI, OpIdx: Src1Idx);
6978 return;
6979 }
6980
6981 MI.setDesc(get(Opcode: CommutedOpc));
6982
6983 Register Src0Reg = Src0.getReg();
6984 unsigned Src0SubReg = Src0.getSubReg();
6985 bool Src0Kill = Src0.isKill();
6986
6987 if (Src1.isImm())
6988 Src0.ChangeToImmediate(ImmVal: Src1.getImm());
6989 else if (Src1.isReg()) {
6990 Src0.ChangeToRegister(Reg: Src1.getReg(), isDef: false, isImp: false, isKill: Src1.isKill());
6991 Src0.setSubReg(Src1.getSubReg());
6992 } else
6993 llvm_unreachable("Should only have register or immediate operands");
6994
6995 Src1.ChangeToRegister(Reg: Src0Reg, isDef: false, isImp: false, isKill: Src0Kill);
6996 Src1.setSubReg(Src0SubReg);
6997 fixImplicitOperands(MI);
6998}
6999
7000// Legalize VOP3 operands. All operand types are supported for any operand
7001// but only one literal constant and only starting from GFX10.
7002void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
7003 MachineInstr &MI) const {
7004 unsigned Opc = MI.getOpcode();
7005
7006 int VOP3Idx[3] = {
7007 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0),
7008 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1),
7009 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2)
7010 };
7011
7012 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 ||
7013 Opc == AMDGPU::V_PERMLANEX16_B32_e64 ||
7014 Opc == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
7015 Opc == AMDGPU::V_PERMLANE_UP_B32_e64 ||
7016 Opc == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
7017 Opc == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
7018 Opc == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64) {
7019 // src1 and src2 must be scalar
7020 MachineOperand &Src1 = MI.getOperand(i: VOP3Idx[1]);
7021 const DebugLoc &DL = MI.getDebugLoc();
7022 if (Src1.isReg() && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src1.getReg()))) {
7023 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7024 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7025 .add(MO: Src1);
7026 Src1.ChangeToRegister(Reg, isDef: false);
7027 }
7028 if (VOP3Idx[2] != -1) {
7029 MachineOperand &Src2 = MI.getOperand(i: VOP3Idx[2]);
7030 if (Src2.isReg() && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src2.getReg()))) {
7031 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7032 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7033 .add(MO: Src2);
7034 Src2.ChangeToRegister(Reg, isDef: false);
7035 }
7036 }
7037 }
7038
7039 // Find the one SGPR operand we are allowed to use.
7040 int ConstantBusLimit = ST.getConstantBusLimit(Opcode: Opc);
7041 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
7042 SmallDenseSet<unsigned> SGPRsUsed;
7043 Register SGPRReg = findUsedSGPR(MI, OpIndices: VOP3Idx);
7044 if (SGPRReg) {
7045 SGPRsUsed.insert(V: SGPRReg);
7046 --ConstantBusLimit;
7047 }
7048
7049 for (int Idx : VOP3Idx) {
7050 if (Idx == -1)
7051 break;
7052 MachineOperand &MO = MI.getOperand(i: Idx);
7053
7054 if (!MO.isReg()) {
7055 if (isInlineConstant(MO, OpInfo: get(Opcode: Opc).operands()[Idx]))
7056 continue;
7057
7058 if (LiteralLimit > 0 && ConstantBusLimit > 0) {
7059 --LiteralLimit;
7060 --ConstantBusLimit;
7061 continue;
7062 }
7063
7064 --LiteralLimit;
7065 --ConstantBusLimit;
7066 legalizeOpWithMove(MI, OpIdx: Idx);
7067 continue;
7068 }
7069
7070 if (!RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg: MO.getReg())))
7071 continue; // VGPRs are legal
7072
7073 // We can use one SGPR in each VOP3 instruction prior to GFX10
7074 // and two starting from GFX10.
7075 if (SGPRsUsed.count(V: MO.getReg()))
7076 continue;
7077 if (ConstantBusLimit > 0) {
7078 SGPRsUsed.insert(V: MO.getReg());
7079 --ConstantBusLimit;
7080 continue;
7081 }
7082
7083 // If we make it this far, then the operand is not legal and we must
7084 // legalize it.
7085 legalizeOpWithMove(MI, OpIdx: Idx);
7086 }
7087
7088 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2 tied to vdst.
7089 if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64) &&
7090 !RI.isVGPR(MRI, Reg: MI.getOperand(i: VOP3Idx[2]).getReg()))
7091 legalizeOpWithMove(MI, OpIdx: VOP3Idx[2]);
7092
7093 // Fix the register class of single-sgpr-read instructions on gfx12+. See
7094 // SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
7095 if (AMDGPU::isSingleSGPRReadInst(Opc)) {
7096 for (unsigned I = 0; I < 3; ++I) {
7097 if (!isLegalSingleSGPRReadInstOperand(MRI, MI, /*SrcN=*/I))
7098 legalizeOpWithMove(MI, OpIdx: VOP3Idx[I]);
7099 }
7100 }
7101}
7102
7103Register SIInstrInfo::readlaneVGPRToSGPR(
7104 Register SrcReg, MachineInstr &UseMI, MachineRegisterInfo &MRI,
7105 const TargetRegisterClass *DstRC /*=nullptr*/) const {
7106 const TargetRegisterClass *VRC = MRI.getRegClass(Reg: SrcReg);
7107 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
7108 if (DstRC)
7109 SRC = RI.getCommonSubClass(A: SRC, B: DstRC);
7110
7111 Register DstReg = MRI.createVirtualRegister(RegClass: SRC);
7112 unsigned SubRegs = RI.getRegSizeInBits(RC: *VRC) / 32;
7113
7114 if (RI.hasAGPRs(RC: VRC)) {
7115 VRC = RI.getEquivalentVGPRClass(SRC: VRC);
7116 Register NewSrcReg = MRI.createVirtualRegister(RegClass: VRC);
7117 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7118 MCID: get(Opcode: TargetOpcode::COPY), DestReg: NewSrcReg)
7119 .addReg(RegNo: SrcReg);
7120 SrcReg = NewSrcReg;
7121 }
7122
7123 if (SubRegs == 1) {
7124 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7125 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
7126 .addReg(RegNo: SrcReg);
7127 return DstReg;
7128 }
7129
7130 SmallVector<Register, 8> SRegs;
7131 for (unsigned i = 0; i < SubRegs; ++i) {
7132 Register SGPR = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
7133 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7134 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: SGPR)
7135 .addReg(RegNo: SrcReg, Flags: {}, SubReg: RI.getSubRegFromChannel(Channel: i));
7136 SRegs.push_back(Elt: SGPR);
7137 }
7138
7139 MachineInstrBuilder MIB =
7140 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7141 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
7142 for (unsigned i = 0; i < SubRegs; ++i) {
7143 MIB.addReg(RegNo: SRegs[i]);
7144 MIB.addImm(Val: RI.getSubRegFromChannel(Channel: i));
7145 }
7146 return DstReg;
7147}
7148
7149void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,
7150 MachineInstr &MI) const {
7151
7152 // If the pointer is store in VGPRs, then we need to move them to
7153 // SGPRs using v_readfirstlane. This is safe because we only select
7154 // loads with uniform pointers to SMRD instruction so we know the
7155 // pointer value is uniform.
7156 MachineOperand *SBase = getNamedOperand(MI, OperandName: AMDGPU::OpName::sbase);
7157 if (SBase && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SBase->getReg()))) {
7158 Register SGPR = readlaneVGPRToSGPR(SrcReg: SBase->getReg(), UseMI&: MI, MRI);
7159 SBase->setReg(SGPR);
7160 }
7161 MachineOperand *SOff = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
7162 if (SOff && !RI.isSGPRReg(MRI, Reg: SOff->getReg())) {
7163 Register SGPR = readlaneVGPRToSGPR(SrcReg: SOff->getReg(), UseMI&: MI, MRI);
7164 SOff->setReg(SGPR);
7165 }
7166}
7167
7168bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const {
7169 unsigned Opc = Inst.getOpcode();
7170 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::saddr);
7171 if (OldSAddrIdx < 0)
7172 return false;
7173
7174 assert(isSegmentSpecificFLAT(Inst) || (isFLAT(Inst) && ST.hasFlatGVSMode()));
7175
7176 int NewOpc = AMDGPU::getGlobalVaddrOp(Opcode: Opc);
7177 if (NewOpc < 0)
7178 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opcode: Opc);
7179 if (NewOpc < 0)
7180 return false;
7181
7182 MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
7183 MachineOperand &SAddr = Inst.getOperand(i: OldSAddrIdx);
7184 if (RI.isSGPRReg(MRI, Reg: SAddr.getReg()))
7185 return false;
7186
7187 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::vaddr);
7188 if (NewVAddrIdx < 0)
7189 return false;
7190
7191 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
7192
7193 // Check vaddr, it shall be zero or absent.
7194 MachineInstr *VAddrDef = nullptr;
7195 if (OldVAddrIdx >= 0) {
7196 MachineOperand &VAddr = Inst.getOperand(i: OldVAddrIdx);
7197 VAddrDef = MRI.getUniqueVRegDef(Reg: VAddr.getReg());
7198 if (!VAddrDef || !VAddrDef->isMoveImmediate() ||
7199 !VAddrDef->getOperand(i: 1).isImm() ||
7200 VAddrDef->getOperand(i: 1).getImm() != 0)
7201 return false;
7202 }
7203
7204 const MCInstrDesc &NewDesc = get(Opcode: NewOpc);
7205 Inst.setDesc(NewDesc);
7206
7207 // Callers expect iterator to be valid after this call, so modify the
7208 // instruction in place.
7209 if (OldVAddrIdx == NewVAddrIdx) {
7210 MachineOperand &NewVAddr = Inst.getOperand(i: NewVAddrIdx);
7211 // Clear use list from the old vaddr holding a zero register.
7212 MRI.removeRegOperandFromUseList(MO: &NewVAddr);
7213 MRI.moveOperands(Dst: &NewVAddr, Src: &SAddr, NumOps: 1);
7214 Inst.removeOperand(OpNo: OldSAddrIdx);
7215 // Update the use list with the pointer we have just moved from vaddr to
7216 // saddr position. Otherwise new vaddr will be missing from the use list.
7217 MRI.removeRegOperandFromUseList(MO: &NewVAddr);
7218 MRI.addRegOperandToUseList(MO: &NewVAddr);
7219 } else {
7220 assert(OldSAddrIdx == NewVAddrIdx);
7221
7222 if (OldVAddrIdx >= 0) {
7223 int NewVDstIn = AMDGPU::getNamedOperandIdx(Opcode: NewOpc,
7224 Name: AMDGPU::OpName::vdst_in);
7225
7226 // removeOperand doesn't try to fixup tied operand indexes at it goes, so
7227 // it asserts. Untie the operands for now and retie them afterwards.
7228 if (NewVDstIn != -1) {
7229 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst_in);
7230 Inst.untieRegOperand(OpIdx: OldVDstIn);
7231 }
7232
7233 Inst.removeOperand(OpNo: OldVAddrIdx);
7234
7235 if (NewVDstIn != -1) {
7236 int NewVDst = AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::vdst);
7237 Inst.tieOperands(DefIdx: NewVDst, UseIdx: NewVDstIn);
7238 }
7239 }
7240 }
7241
7242 if (VAddrDef && MRI.use_nodbg_empty(RegNo: VAddrDef->getOperand(i: 0).getReg()))
7243 VAddrDef->eraseFromParent();
7244
7245 return true;
7246}
7247
7248// FIXME: Remove this when SelectionDAG is obsoleted.
7249void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI,
7250 MachineInstr &MI) const {
7251 if (!isSegmentSpecificFLAT(MI) && !ST.hasFlatGVSMode())
7252 return;
7253
7254 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence
7255 // thinks they are uniform, so a readfirstlane should be valid.
7256 MachineOperand *SAddr = getNamedOperand(MI, OperandName: AMDGPU::OpName::saddr);
7257 if (!SAddr || RI.isSGPRClass(RC: MRI.getRegClass(Reg: SAddr->getReg())))
7258 return;
7259
7260 if (moveFlatAddrToVGPR(Inst&: MI))
7261 return;
7262
7263 const TargetRegisterClass *DeclaredRC =
7264 getRegClass(MCID: MI.getDesc(), OpNum: SAddr->getOperandNo());
7265
7266 Register ToSGPR = readlaneVGPRToSGPR(SrcReg: SAddr->getReg(), UseMI&: MI, MRI, DstRC: DeclaredRC);
7267 SAddr->setReg(ToSGPR);
7268}
7269
7270void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
7271 MachineBasicBlock::iterator I,
7272 const TargetRegisterClass *DstRC,
7273 MachineOperand &Op,
7274 MachineRegisterInfo &MRI,
7275 const DebugLoc &DL) const {
7276 Register OpReg = Op.getReg();
7277 unsigned OpSubReg = Op.getSubReg();
7278
7279 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
7280 RI.getRegClassForReg(MRI, Reg: OpReg), OpSubReg);
7281
7282 // Check if operand is already the correct register class.
7283 if (DstRC == OpRC)
7284 return;
7285
7286 Register DstReg = MRI.createVirtualRegister(RegClass: DstRC);
7287 auto Copy = BuildMI(BB&: InsertMBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: DstReg)
7288 .addReg(RegNo: OpReg, Flags: {}, SubReg: OpSubReg);
7289 Op.setReg(DstReg);
7290 Op.setSubReg(AMDGPU::NoSubRegister);
7291
7292 MachineInstr *Def = MRI.getVRegDef(Reg: OpReg);
7293 if (!Def)
7294 return;
7295
7296 // Try to eliminate the copy if it is copying an immediate value.
7297 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass)
7298 foldImmediate(UseMI&: *Copy, DefMI&: *Def, Reg: OpReg, MRI: &MRI);
7299
7300 bool ImpDef = Def->isImplicitDef();
7301 while (!ImpDef && Def && Def->isCopy()) {
7302 if (Def->getOperand(i: 1).getReg().isPhysical())
7303 break;
7304 Def = MRI.getUniqueVRegDef(Reg: Def->getOperand(i: 1).getReg());
7305 ImpDef = Def && Def->isImplicitDef();
7306 }
7307 if (!RI.isSGPRClass(RC: DstRC) && !Copy->readsRegister(Reg: AMDGPU::EXEC, TRI: &RI) &&
7308 !ImpDef)
7309 Copy.addReg(RegNo: AMDGPU::EXEC, Flags: RegState::Implicit);
7310}
7311
7312// Emit the actual waterfall loop, executing the wrapped instruction for each
7313// unique value of \p ScalarOps across all lanes. In the best case we execute 1
7314// iteration, in the worst case we execute 64 (once per lane).
7315static void emitLoadScalarOpsFromVGPRLoop(
7316 const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &PredBB,
7317 MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
7318 ArrayRef<MachineOperand *> ScalarOps, ArrayRef<Register> PhySGPRs = {}) {
7319 MachineFunction &MF = *LoopBB.getParent();
7320 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
7321 const SIRegisterInfo *TRI = ST.getRegisterInfo();
7322 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
7323 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
7324
7325 // Emit v_cmpx_eq and s_andn2_wrexec when both instructions are
7326 // available. Otherwise, use the previous pattern of v_cmp_eq,
7327 // s_and_saveexec, and s_xor.
7328 bool UseNewExecInstructions =
7329 ST.hasNoSdstCMPX() && TII.pseudoToMCOpcode(Opcode: LMC.AndN2WrExecOpc) != -1;
7330
7331 MachineBasicBlock::iterator I = LoopBB.begin();
7332 Register CondReg;
7333
7334 Register PhiExec;
7335 Register NewExec;
7336
7337 if (UseNewExecInstructions) {
7338 PhiExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7339 NewExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7340 Register InitExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7341 BuildMI(BB&: PredBB, I: PredBB.end(), MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: InitExec)
7342 .addReg(RegNo: LMC.ExecReg);
7343
7344 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::PHI), DestReg: PhiExec)
7345 .addReg(RegNo: InitExec)
7346 .addMBB(MBB: &PredBB)
7347 .addReg(RegNo: NewExec)
7348 .addMBB(MBB: &BodyBB);
7349 }
7350
7351 // Placement of v_cmpx instructions (when index is longer than 64 bit)
7352 // involves a trade-off between register pressure and latency:
7353 // (a) Defering all v_cmpx after all v_readfirstlane may increase
7354 // register pressure because arguments and results of all
7355 // v_readfirstlane instructions must stay live until deferred v_cmpx use them.
7356 // (b) Interleaving v_cmpx with v_readfirstlanes may reduce live ranges and
7357 // increase latency by placing v_readfirstlane instructions
7358 // immediately before v_cmpx instruction that directly depend on it.
7359 ///
7360 // Emitting interleaved v_cmpx and v_readfirstlane requires
7361 // block splitting because v_cmpx changes EXEC mask and therefore for safety
7362 // v_cmpx needs to be treated as terminator until after register allocation
7363 // (spill placement) and instruction reordering.
7364 //
7365 // Current implementation defers v_cmpx and leaves other instruction
7366 // scheduling decisions to later passes, where register pressure is known or
7367 // easier to approximate.
7368 // Non-terminators (V_READFIRSTLANE and REG_SEQUENCE) are inserted before I;
7369 // v_cmpx instructions are inserted at the end of LoopBB.
7370 // After the first v_cmpx is emitted, I is updated to point to it
7371 // so subsequent non-terminators are inserted before all v_cmpx instructions.
7372 for (auto [Idx, ScalarOp] : enumerate(First&: ScalarOps)) {
7373 unsigned RegSize = TRI->getRegSizeInBits(Reg: ScalarOp->getReg(), MRI);
7374 unsigned NumSubRegs = RegSize / 32;
7375 Register VScalarOp = ScalarOp->getReg();
7376
7377 const TargetRegisterClass *RFLSrcRC =
7378 TII.getRegClass(MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), OpNum: 1);
7379
7380 if (NumSubRegs == 1) {
7381 const TargetRegisterClass *VScalarOpRC = MRI.getRegClass(Reg: VScalarOp);
7382 if (const TargetRegisterClass *Common =
7383 TRI->getCommonSubClass(A: VScalarOpRC, B: RFLSrcRC);
7384 Common != VScalarOpRC) {
7385 Register VRReg = MRI.createVirtualRegister(RegClass: Common);
7386 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VRReg).addReg(RegNo: VScalarOp);
7387 VScalarOp = VRReg;
7388 }
7389 Register CurReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7390
7391 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurReg)
7392 .addReg(RegNo: VScalarOp);
7393
7394 if (UseNewExecInstructions) {
7395 auto CmpxMI = BuildMI(BB&: LoopBB, I: LoopBB.end(), MIMD: DL,
7396 MCID: TII.get(Opcode: AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term))
7397 .addReg(RegNo: CurReg)
7398 .addReg(RegNo: VScalarOp);
7399 if (I == LoopBB.end())
7400 I = CmpxMI.getInstr()->getIterator();
7401 } else {
7402 Register NewCondReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7403
7404 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: NewCondReg)
7405 .addReg(RegNo: CurReg)
7406 .addReg(RegNo: VScalarOp);
7407
7408 // Combine the comparison results with AND.
7409 if (!CondReg) { // First.
7410 CondReg = NewCondReg;
7411 } else { // If not the first, we create an AND.
7412 Register AndReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7413 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndOpc), DestReg: AndReg)
7414 .addReg(RegNo: CondReg)
7415 .addReg(RegNo: NewCondReg);
7416 CondReg = AndReg;
7417 }
7418 }
7419
7420 // Update ScalarOp operand to use the SGPR ScalarOp.
7421 if (PhySGPRs.empty() || !PhySGPRs[Idx].isValid())
7422 ScalarOp->setReg(CurReg);
7423 else {
7424 // Insert into the same block of use
7425 BuildMI(BB&: *ScalarOp->getParent()->getParent(), I: ScalarOp->getParent(), MIMD: DL,
7426 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: PhySGPRs[Idx])
7427 .addReg(RegNo: CurReg);
7428 ScalarOp->setReg(PhySGPRs[Idx]);
7429 }
7430 ScalarOp->setIsKill();
7431 } else {
7432 SmallVector<Register, 8> ReadlanePieces;
7433 RegState VScalarOpUndef = getUndefRegState(B: ScalarOp->isUndef());
7434 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 &&
7435 "Unhandled register size");
7436
7437 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) {
7438 Register CurRegLo =
7439 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7440 Register CurRegHi =
7441 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7442
7443 // Read the next variant <- also loop target.
7444 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurRegLo)
7445 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg: TRI->getSubRegFromChannel(Channel: Idx));
7446
7447 // Read the next variant <- also loop target.
7448 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurRegHi)
7449 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef,
7450 SubReg: TRI->getSubRegFromChannel(Channel: Idx + 1));
7451
7452 ReadlanePieces.push_back(Elt: CurRegLo);
7453 ReadlanePieces.push_back(Elt: CurRegHi);
7454
7455 // Comparison is to be done as 64-bit.
7456 Register CurReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_64RegClass);
7457 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: CurReg)
7458 .addReg(RegNo: CurRegLo)
7459 .addImm(Val: AMDGPU::sub0)
7460 .addReg(RegNo: CurRegHi)
7461 .addImm(Val: AMDGPU::sub1);
7462
7463 unsigned SubReg =
7464 NumSubRegs <= 2 ? 0 : TRI->getSubRegFromChannel(Channel: Idx, NumRegs: 2);
7465
7466 if (UseNewExecInstructions) {
7467 auto CmpxMI = BuildMI(BB&: LoopBB, I: LoopBB.end(), MIMD: DL,
7468 MCID: TII.get(Opcode: AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term))
7469 .addReg(RegNo: CurReg)
7470 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg);
7471 if (I == LoopBB.end())
7472 I = CmpxMI.getInstr()->getIterator();
7473 } else {
7474 Register NewCondReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7475 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U64_e64), DestReg: NewCondReg)
7476 .addReg(RegNo: CurReg)
7477 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg);
7478
7479 // Combine the comparison results with AND.
7480 if (!CondReg) { // First.
7481 CondReg = NewCondReg;
7482 } else { // If not the first, we create an AND.
7483 Register AndReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7484 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndOpc), DestReg: AndReg)
7485 .addReg(RegNo: CondReg)
7486 .addReg(RegNo: NewCondReg);
7487 CondReg = AndReg;
7488 }
7489 }
7490 } // End for loop.
7491
7492 const auto *SScalarOpRC =
7493 TRI->getEquivalentSGPRClass(VRC: MRI.getRegClass(Reg: VScalarOp));
7494 Register SScalarOp = MRI.createVirtualRegister(RegClass: SScalarOpRC);
7495
7496 // Build scalar ScalarOp.
7497 auto Merge =
7498 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: SScalarOp);
7499 unsigned Channel = 0;
7500 for (Register Piece : ReadlanePieces) {
7501 Merge.addReg(RegNo: Piece).addImm(Val: TRI->getSubRegFromChannel(Channel: Channel++));
7502 }
7503
7504 // Update ScalarOp operand to use the SGPR ScalarOp.
7505 if (PhySGPRs.empty() || !PhySGPRs[Idx].isValid())
7506 ScalarOp->setReg(SScalarOp);
7507 else {
7508 BuildMI(BB&: *ScalarOp->getParent()->getParent(), I: ScalarOp->getParent(), MIMD: DL,
7509 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: PhySGPRs[Idx])
7510 .addReg(RegNo: SScalarOp);
7511 ScalarOp->setReg(PhySGPRs[Idx]);
7512 }
7513 ScalarOp->setIsKill();
7514 }
7515 }
7516
7517 // AndSaveExecOpc modifies EXEC but can't be isTerminator=1: terminators
7518 // that define virtual registers aren't supported.
7519 Register SaveExec;
7520 if (!UseNewExecInstructions) {
7521 SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7522 MRI.setSimpleHint(VReg: SaveExec, PrefReg: CondReg);
7523
7524 // Update EXEC to matching lanes, saving original to SaveExec.
7525 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndSaveExecOpc), DestReg: SaveExec)
7526 .addReg(RegNo: CondReg, Flags: RegState::Kill);
7527 }
7528
7529 // The original instruction is here; we insert the terminators after it.
7530 I = BodyBB.end();
7531
7532 if (UseNewExecInstructions) {
7533 // Compute the remaining lanes into a plain virtual register and write EXEC
7534 // from a terminator, so spill code for NewExec is placed before EXEC
7535 // changes. SIOptimizeExecMasking opportunistically folds the pair back
7536 // into S_ANDN2_WREXEC after register allocation.
7537 MRI.setSimpleHint(VReg: NewExec, PrefReg: PhiExec);
7538 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndN2Opc), DestReg: NewExec)
7539 .addReg(RegNo: PhiExec)
7540 .addReg(RegNo: LMC.ExecReg);
7541 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.MovTermOpc), DestReg: LMC.ExecReg)
7542 .addReg(RegNo: NewExec);
7543 } else {
7544 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
7545 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.XorTermOpc), DestReg: LMC.ExecReg)
7546 .addReg(RegNo: LMC.ExecReg)
7547 .addReg(RegNo: SaveExec);
7548 }
7549
7550 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::SI_WATERFALL_LOOP)).addMBB(MBB: &LoopBB);
7551}
7552
7553// Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register
7554// with SGPRs by iterating over all unique values across all lanes.
7555// Returns the loop basic block that now contains \p MI.
7556static MachineBasicBlock *
7557generateWaterFallLoop(const SIInstrInfo &TII, MachineInstr &MI,
7558 ArrayRef<MachineOperand *> ScalarOps,
7559 MachineDominatorTree *MDT,
7560 MachineBasicBlock::iterator Begin = nullptr,
7561 MachineBasicBlock::iterator End = nullptr,
7562 ArrayRef<Register> PhySGPRs = {}) {
7563 assert((PhySGPRs.empty() || PhySGPRs.size() == ScalarOps.size()) &&
7564 "Physical SGPRs must be empty or match the number of scalar operands");
7565 MachineBasicBlock &MBB = *MI.getParent();
7566 MachineFunction &MF = *MBB.getParent();
7567 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
7568 const SIRegisterInfo *TRI = ST.getRegisterInfo();
7569 MachineRegisterInfo &MRI = MF.getRegInfo();
7570 if (!Begin.isValid())
7571 Begin = &MI;
7572 if (!End.isValid()) {
7573 End = &MI;
7574 ++End;
7575 }
7576 const DebugLoc &DL = MI.getDebugLoc();
7577 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
7578 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
7579
7580 // Save SCC. Waterfall Loop may overwrite SCC.
7581 Register SaveSCCReg;
7582
7583 // FIXME: We should maintain SCC liveness while doing the FixSGPRCopies walk
7584 // rather than unlimited scan everywhere
7585 bool SCCNotDead =
7586 MBB.computeRegisterLiveness(TRI, Reg: AMDGPU::SCC, Before: MI,
7587 Neighborhood: std::numeric_limits<unsigned>::max()) !=
7588 MachineBasicBlock::LQR_Dead;
7589 if (SCCNotDead) {
7590 SaveSCCReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7591 BuildMI(BB&: MBB, I: Begin, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: SaveSCCReg)
7592 .addImm(Val: 1)
7593 .addImm(Val: 0);
7594 }
7595
7596 Register SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7597
7598 // Save the EXEC mask
7599 BuildMI(BB&: MBB, I: Begin, MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: SaveExec).addReg(RegNo: LMC.ExecReg);
7600
7601 // Killed uses in the instruction we are waterfalling around will be
7602 // incorrect due to the added control-flow.
7603 MachineBasicBlock::iterator AfterMI = MI;
7604 ++AfterMI;
7605 for (auto I = Begin; I != AfterMI; I++) {
7606 for (auto &MO : I->all_uses())
7607 MRI.clearKillFlags(Reg: MO.getReg());
7608 }
7609
7610 // To insert the loop we need to split the block. Move everything after this
7611 // point to a new block, and insert a new empty block between the two.
7612 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
7613 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock();
7614 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
7615 MachineFunction::iterator MBBI(MBB);
7616 ++MBBI;
7617
7618 MF.insert(MBBI, MBB: LoopBB);
7619 MF.insert(MBBI, MBB: BodyBB);
7620 MF.insert(MBBI, MBB: RemainderBB);
7621
7622 LoopBB->addSuccessor(Succ: BodyBB);
7623 BodyBB->addSuccessor(Succ: LoopBB);
7624 BodyBB->addSuccessor(Succ: RemainderBB);
7625
7626 // Move Begin to MI to the BodyBB, and the remainder of the block to
7627 // RemainderBB.
7628 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
7629 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: End, To: MBB.end());
7630 BodyBB->splice(Where: BodyBB->begin(), Other: &MBB, From: Begin, To: MBB.end());
7631
7632 MBB.addSuccessor(Succ: LoopBB);
7633
7634 // Update dominators. We know that MBB immediately dominates LoopBB, that
7635 // LoopBB immediately dominates BodyBB, and BodyBB immediately dominates
7636 // RemainderBB. RemainderBB immediately dominates all of the successors
7637 // transferred to it from MBB that MBB used to properly dominate.
7638 if (MDT) {
7639 MDT->addNewBlock(BB: LoopBB, DomBB: &MBB);
7640 MDT->addNewBlock(BB: BodyBB, DomBB: LoopBB);
7641 MDT->addNewBlock(BB: RemainderBB, DomBB: BodyBB);
7642 for (auto &Succ : RemainderBB->successors()) {
7643 if (MDT->properlyDominates(A: &MBB, B: Succ)) {
7644 MDT->changeImmediateDominator(BB: Succ, NewBB: RemainderBB);
7645 }
7646 }
7647 }
7648
7649 emitLoadScalarOpsFromVGPRLoop(TII, MRI, PredBB&: MBB, LoopBB&: *LoopBB, BodyBB&: *BodyBB, DL, ScalarOps,
7650 PhySGPRs);
7651
7652 MachineBasicBlock::iterator First = RemainderBB->begin();
7653 // Restore SCC
7654 if (SCCNotDead) {
7655 BuildMI(BB&: *RemainderBB, I: First, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CMP_LG_U32))
7656 .addReg(RegNo: SaveSCCReg, Flags: RegState::Kill)
7657 .addImm(Val: 0);
7658 }
7659
7660 // Restore the EXEC mask
7661 BuildMI(BB&: *RemainderBB, I: First, MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
7662 .addReg(RegNo: SaveExec);
7663 return BodyBB;
7664}
7665
7666// Extract pointer from Rsrc and return a zero-value Rsrc replacement.
7667static std::tuple<unsigned, unsigned>
7668extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) {
7669 MachineBasicBlock &MBB = *MI.getParent();
7670 MachineFunction &MF = *MBB.getParent();
7671 MachineRegisterInfo &MRI = MF.getRegInfo();
7672
7673 // Extract the ptr from the resource descriptor.
7674 unsigned RsrcPtr =
7675 TII.buildExtractSubReg(MI, MRI, SuperReg: Rsrc, SuperRC: &AMDGPU::VReg_128RegClass,
7676 SubIdx: AMDGPU::sub0_sub1, SubRC: &AMDGPU::VReg_64RegClass);
7677
7678 // Create an empty resource descriptor
7679 Register Zero64 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
7680 Register SRsrcFormatLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
7681 Register SRsrcFormatHi = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
7682 Register NewSRsrc = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_128RegClass);
7683 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
7684
7685 // Zero64 = 0
7686 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B64), DestReg: Zero64)
7687 .addImm(Val: 0);
7688
7689 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
7690 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: SRsrcFormatLo)
7691 .addImm(Val: Lo_32(Value: RsrcDataFormat));
7692
7693 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
7694 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: SRsrcFormatHi)
7695 .addImm(Val: Hi_32(Value: RsrcDataFormat));
7696
7697 // NewSRsrc = {Zero64, SRsrcFormat}
7698 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewSRsrc)
7699 .addReg(RegNo: Zero64)
7700 .addImm(Val: AMDGPU::sub0_sub1)
7701 .addReg(RegNo: SRsrcFormatLo)
7702 .addImm(Val: AMDGPU::sub2)
7703 .addReg(RegNo: SRsrcFormatHi)
7704 .addImm(Val: AMDGPU::sub3);
7705
7706 return std::tuple(RsrcPtr, NewSRsrc);
7707}
7708
7709MachineBasicBlock *
7710SIInstrInfo::legalizeOperands(MachineInstr &MI,
7711 MachineDominatorTree *MDT) const {
7712 MachineFunction &MF = *MI.getMF();
7713 MachineRegisterInfo &MRI = MF.getRegInfo();
7714 MachineBasicBlock *CreatedBB = nullptr;
7715
7716 // Legalize True16
7717 if (ST.useRealTrue16Insts())
7718 legalizeOperandsVALUt16(Inst&: MI, MRI);
7719
7720 // Legalize VOP2
7721 if (isVOP2(MI) || isVOPC(MI)) {
7722 legalizeOperandsVOP2(MRI, MI);
7723 return CreatedBB;
7724 }
7725
7726 // Legalize VOP3
7727 if (isVOP3(MI)) {
7728 legalizeOperandsVOP3(MRI, MI);
7729 return CreatedBB;
7730 }
7731
7732 // Legalize SMRD
7733 if (isSMRD(MI)) {
7734 legalizeOperandsSMRD(MRI, MI);
7735 return CreatedBB;
7736 }
7737
7738 // Legalize FLAT
7739 if (isFLAT(MI)) {
7740 legalizeOperandsFLAT(MRI, MI);
7741 return CreatedBB;
7742 }
7743
7744 // Legalize PHI
7745 // The register class of the operands must be the same type as the register
7746 // class of the output.
7747 if (MI.getOpcode() == AMDGPU::PHI) {
7748 const TargetRegisterClass *VRC = getOpRegClass(MI, OpNo: 0);
7749 assert(!RI.isSGPRClass(VRC));
7750
7751 // Update all the operands so they have the same type.
7752 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
7753 MachineOperand &Op = MI.getOperand(i: I);
7754 if (!Op.isReg() || !Op.getReg().isVirtual())
7755 continue;
7756
7757 // MI is a PHI instruction.
7758 MachineBasicBlock *InsertBB = MI.getOperand(i: I + 1).getMBB();
7759 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
7760
7761 // Avoid creating no-op copies with the same src and dst reg class. These
7762 // confuse some of the machine passes.
7763 legalizeGenericOperand(InsertMBB&: *InsertBB, I: Insert, DstRC: VRC, Op, MRI, DL: MI.getDebugLoc());
7764 }
7765 }
7766
7767 // REG_SEQUENCE doesn't really require operand legalization, but if one has a
7768 // VGPR dest type and SGPR sources, insert copies so all operands are
7769 // VGPRs. This seems to help operand folding / the register coalescer.
7770 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
7771 MachineBasicBlock *MBB = MI.getParent();
7772 const TargetRegisterClass *DstRC = getOpRegClass(MI, OpNo: 0);
7773 if (RI.hasVGPRs(RC: DstRC)) {
7774 // Update all the operands so they are VGPR register classes. These may
7775 // not be the same register class because REG_SEQUENCE supports mixing
7776 // subregister index types e.g. sub0_sub1 + sub2 + sub3
7777 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
7778 MachineOperand &Op = MI.getOperand(i: I);
7779 if (!Op.isReg() || !Op.getReg().isVirtual())
7780 continue;
7781
7782 const TargetRegisterClass *OpRC = MRI.getRegClass(Reg: Op.getReg());
7783 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(SRC: OpRC);
7784 if (VRC == OpRC)
7785 continue;
7786
7787 legalizeGenericOperand(InsertMBB&: *MBB, I: MI, DstRC: VRC, Op, MRI, DL: MI.getDebugLoc());
7788 Op.setIsKill();
7789 }
7790 }
7791
7792 return CreatedBB;
7793 }
7794
7795 // Legalize INSERT_SUBREG
7796 // src0 must have the same register class as dst
7797 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
7798 Register Dst = MI.getOperand(i: 0).getReg();
7799 Register Src0 = MI.getOperand(i: 1).getReg();
7800 const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: Dst);
7801 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0);
7802 if (DstRC != Src0RC) {
7803 MachineBasicBlock *MBB = MI.getParent();
7804 MachineOperand &Op = MI.getOperand(i: 1);
7805 legalizeGenericOperand(InsertMBB&: *MBB, I: MI, DstRC, Op, MRI, DL: MI.getDebugLoc());
7806 }
7807 return CreatedBB;
7808 }
7809
7810 // Legalize SI_INIT_M0
7811 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
7812 MachineOperand &Src = MI.getOperand(i: 0);
7813 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
7814 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
7815 return CreatedBB;
7816 }
7817
7818 // Legalize S_BITREPLICATE, S_QUADMASK and S_WQM
7819 if (MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 ||
7820 MI.getOpcode() == AMDGPU::S_QUADMASK_B32 ||
7821 MI.getOpcode() == AMDGPU::S_QUADMASK_B64 ||
7822 MI.getOpcode() == AMDGPU::S_WQM_B32 ||
7823 MI.getOpcode() == AMDGPU::S_WQM_B64 ||
7824 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U32 ||
7825 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U64) {
7826 MachineOperand &Src = MI.getOperand(i: 1);
7827 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
7828 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
7829 return CreatedBB;
7830 }
7831
7832 // Legalize MIMG/VIMAGE/VSAMPLE and MUBUF/MTBUF for shaders.
7833 //
7834 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
7835 // scratch memory access. In both cases, the legalization never involves
7836 // conversion to the addr64 form.
7837 if (isImage(MI) || (AMDGPU::isGraphics(CC: MF.getFunction().getCallingConv()) &&
7838 (isMUBUF(MI) || isMTBUF(MI)))) {
7839 AMDGPU::OpName RSrcOpName = (isVIMAGE(MI) || isVSAMPLE(MI))
7840 ? AMDGPU::OpName::rsrc
7841 : AMDGPU::OpName::srsrc;
7842 MachineOperand *SRsrc = getNamedOperand(MI, OperandName: RSrcOpName);
7843 if (SRsrc && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SRsrc->getReg())))
7844 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {SRsrc}, MDT);
7845
7846 AMDGPU::OpName SampOpName =
7847 isMIMG(MI) ? AMDGPU::OpName::ssamp : AMDGPU::OpName::samp;
7848 MachineOperand *SSamp = getNamedOperand(MI, OperandName: SampOpName);
7849 if (SSamp && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SSamp->getReg())))
7850 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {SSamp}, MDT);
7851
7852 return CreatedBB;
7853 }
7854
7855 // Legalize SI_CALL
7856 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) {
7857 MachineOperand *Dest = &MI.getOperand(i: 0);
7858 if (!RI.isSGPRClass(RC: MRI.getRegClass(Reg: Dest->getReg()))) {
7859 createWaterFallForSiCall(MI: &MI, MDT, ScalarOps: {Dest});
7860 }
7861 }
7862
7863 // Legalize s_sleep_var.
7864 if (MI.getOpcode() == AMDGPU::S_SLEEP_VAR) {
7865 const DebugLoc &DL = MI.getDebugLoc();
7866 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7867 int Src0Idx =
7868 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
7869 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
7870 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7871 .add(MO: Src0);
7872 Src0.ChangeToRegister(Reg, isDef: false);
7873 return nullptr;
7874 }
7875
7876 // Legalize TENSOR_LOAD_TO_LDS_d2/_d4, TENSOR_STORE_FROM_LDS_d2/_d4. All their
7877 // operands are scalar.
7878 if (MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_d2 ||
7879 MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_d4 ||
7880 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_d2 ||
7881 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_d4) {
7882 for (MachineOperand &Src : MI.explicit_operands()) {
7883 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
7884 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
7885 }
7886 return CreatedBB;
7887 }
7888
7889 // Legalize MUBUF instructions.
7890 bool isSoffsetLegal = true;
7891 int SoffsetIdx =
7892 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::soffset);
7893 if (SoffsetIdx != -1) {
7894 MachineOperand *Soffset = &MI.getOperand(i: SoffsetIdx);
7895 if (Soffset->isReg() && Soffset->getReg().isVirtual() &&
7896 !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Soffset->getReg()))) {
7897 isSoffsetLegal = false;
7898 }
7899 }
7900
7901 bool isRsrcLegal = true;
7902 int RsrcIdx =
7903 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::srsrc);
7904 if (RsrcIdx != -1) {
7905 MachineOperand *Rsrc = &MI.getOperand(i: RsrcIdx);
7906 if (Rsrc->isReg() && !RI.isSGPRReg(MRI, Reg: Rsrc->getReg()))
7907 isRsrcLegal = false;
7908 }
7909
7910 // The operands are legal.
7911 if (isRsrcLegal && isSoffsetLegal)
7912 return CreatedBB;
7913
7914 if (!isRsrcLegal) {
7915 // Legalize a VGPR Rsrc
7916 //
7917 // If the instruction is _ADDR64, we can avoid a waterfall by extracting
7918 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
7919 // a zero-value SRsrc.
7920 //
7921 // If the instruction is _OFFSET (both idxen and offen disabled), and we
7922 // support ADDR64 instructions, we can convert to ADDR64 and do the same as
7923 // above.
7924 //
7925 // Otherwise we are on non-ADDR64 hardware, and/or we have
7926 // idxen/offen/bothen and we fall back to a waterfall loop.
7927
7928 MachineOperand *Rsrc = &MI.getOperand(i: RsrcIdx);
7929 MachineBasicBlock &MBB = *MI.getParent();
7930
7931 MachineOperand *VAddr = getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
7932 if (VAddr && AMDGPU::getIfAddr64Inst(Opcode: MI.getOpcode()) != -1) {
7933 // This is already an ADDR64 instruction so we need to add the pointer
7934 // extracted from the resource descriptor to the current value of VAddr.
7935 Register NewVAddrLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7936 Register NewVAddrHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7937 Register NewVAddr = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
7938
7939 const auto *BoolXExecRC = RI.getWaveMaskRegClass();
7940 Register CondReg0 = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7941 Register CondReg1 = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7942
7943 unsigned RsrcPtr, NewSRsrc;
7944 std::tie(args&: RsrcPtr, args&: NewSRsrc) = extractRsrcPtr(TII: *this, MI, Rsrc&: *Rsrc);
7945
7946 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
7947 const DebugLoc &DL = MI.getDebugLoc();
7948 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg: NewVAddrLo)
7949 .addDef(RegNo: CondReg0)
7950 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub0)
7951 .addReg(RegNo: VAddr->getReg(), Flags: {}, SubReg: AMDGPU::sub0)
7952 .addImm(Val: 0);
7953
7954 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
7955 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADDC_U32_e64), DestReg: NewVAddrHi)
7956 .addDef(RegNo: CondReg1, Flags: RegState::Dead)
7957 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub1)
7958 .addReg(RegNo: VAddr->getReg(), Flags: {}, SubReg: AMDGPU::sub1)
7959 .addReg(RegNo: CondReg0, Flags: RegState::Kill)
7960 .addImm(Val: 0);
7961
7962 // NewVaddr = {NewVaddrHi, NewVaddrLo}
7963 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewVAddr)
7964 .addReg(RegNo: NewVAddrLo)
7965 .addImm(Val: AMDGPU::sub0)
7966 .addReg(RegNo: NewVAddrHi)
7967 .addImm(Val: AMDGPU::sub1);
7968
7969 VAddr->setReg(NewVAddr);
7970 Rsrc->setReg(NewSRsrc);
7971 } else if (!VAddr && ST.hasAddr64()) {
7972 // This instructions is the _OFFSET variant, so we need to convert it to
7973 // ADDR64.
7974 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS &&
7975 "FIXME: Need to emit flat atomics here");
7976
7977 unsigned RsrcPtr, NewSRsrc;
7978 std::tie(args&: RsrcPtr, args&: NewSRsrc) = extractRsrcPtr(TII: *this, MI, Rsrc&: *Rsrc);
7979
7980 Register NewVAddr = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
7981 MachineOperand *VData = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata);
7982 MachineOperand *Offset = getNamedOperand(MI, OperandName: AMDGPU::OpName::offset);
7983 MachineOperand *SOffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
7984 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(Opcode: MI.getOpcode());
7985
7986 // Atomics with return have an additional tied operand and are
7987 // missing some of the special bits.
7988 MachineOperand *VDataIn = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata_in);
7989 MachineInstr *Addr64;
7990
7991 if (!VDataIn) {
7992 // Regular buffer load / store.
7993 MachineInstrBuilder MIB =
7994 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Addr64Opcode))
7995 .add(MO: *VData)
7996 .addReg(RegNo: NewVAddr)
7997 .addReg(RegNo: NewSRsrc)
7998 .add(MO: *SOffset)
7999 .add(MO: *Offset);
8000
8001 if (const MachineOperand *CPol =
8002 getNamedOperand(MI, OperandName: AMDGPU::OpName::cpol)) {
8003 MIB.addImm(Val: CPol->getImm());
8004 }
8005
8006 if (const MachineOperand *TFE =
8007 getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe)) {
8008 MIB.addImm(Val: TFE->getImm());
8009 }
8010
8011 MIB.addImm(Val: getNamedImmOperand(MI, OperandName: AMDGPU::OpName::swz));
8012
8013 MIB.cloneMemRefs(OtherMI: MI);
8014 Addr64 = MIB;
8015 } else {
8016 // Atomics with return.
8017 Addr64 = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Addr64Opcode))
8018 .add(MO: *VData)
8019 .add(MO: *VDataIn)
8020 .addReg(RegNo: NewVAddr)
8021 .addReg(RegNo: NewSRsrc)
8022 .add(MO: *SOffset)
8023 .add(MO: *Offset)
8024 .addImm(Val: getNamedImmOperand(MI, OperandName: AMDGPU::OpName::cpol))
8025 .cloneMemRefs(OtherMI: MI);
8026 }
8027
8028 MI.removeFromParent();
8029
8030 // NewVaddr = {NewVaddrHi, NewVaddrLo}
8031 BuildMI(BB&: MBB, I: Addr64, MIMD: Addr64->getDebugLoc(), MCID: get(Opcode: AMDGPU::REG_SEQUENCE),
8032 DestReg: NewVAddr)
8033 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub0)
8034 .addImm(Val: AMDGPU::sub0)
8035 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub1)
8036 .addImm(Val: AMDGPU::sub1);
8037 } else {
8038 // Legalize a VGPR Rsrc and soffset together.
8039 if (!isSoffsetLegal) {
8040 MachineOperand *Soffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
8041 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Rsrc, Soffset}, MDT);
8042 return CreatedBB;
8043 }
8044 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Rsrc}, MDT);
8045 return CreatedBB;
8046 }
8047 }
8048
8049 // Legalize a VGPR soffset.
8050 if (!isSoffsetLegal) {
8051 MachineOperand *Soffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
8052 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Soffset}, MDT);
8053 return CreatedBB;
8054 }
8055 return CreatedBB;
8056}
8057
8058void SIInstrWorklist::insert(MachineInstr *MI) {
8059 if (InSet.insert(Ptr: MI).second)
8060 InstrList.push_back(Elt: MI);
8061 // Add MBUF instructiosn to deferred list.
8062 int RsrcIdx =
8063 AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(), Name: AMDGPU::OpName::srsrc);
8064 if (RsrcIdx != -1) {
8065 DeferredList.insert(X: MI);
8066 }
8067}
8068
8069bool SIInstrWorklist::isDeferred(MachineInstr *MI) {
8070 return DeferredList.contains(key: MI);
8071}
8072
8073// Legalize size mismatches between 16bit and 32bit registers in v2s copy
8074// lowering (change sgpr to vgpr).
8075// This is mainly caused by 16bit SALU and 16bit VALU using reg with different
8076// size. Need to legalize the size of the operands during the vgpr lowering
8077// chain. This can be removed after we have sgpr16 in place
8078void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &MI, unsigned OpIdx,
8079 MachineRegisterInfo &MRI) const {
8080 if (!ST.useRealTrue16Insts())
8081 return;
8082
8083 unsigned Opcode = MI.getOpcode();
8084 MachineBasicBlock *MBB = MI.getParent();
8085 // Legalize operands and check for size mismatch
8086 if (OpIdx >= MI.getNumExplicitOperands() ||
8087 OpIdx >= get(Opcode).getNumOperands() ||
8088 get(Opcode).operands()[OpIdx].RegClass == -1)
8089 return;
8090
8091 MachineOperand &Op = MI.getOperand(i: OpIdx);
8092 if (!Op.isReg() || !Op.getReg().isVirtual() || Op.isDef())
8093 return;
8094
8095 const TargetRegisterClass *CurrRC = MRI.getRegClass(Reg: Op.getReg());
8096 if (!RI.isVGPRClass(RC: CurrRC))
8097 return;
8098
8099 int16_t RCID = getOpRegClassID(OpInfo: get(Opcode).operands()[OpIdx]);
8100 const TargetRegisterClass *ExpectedRC = RI.getRegClass(i: RCID);
8101 if (RI.getMatchingSuperRegClass(A: CurrRC, B: ExpectedRC, Idx: AMDGPU::lo16)) {
8102 // Default to the lo16 only if the subregister is not specified.
8103 if (Op.getSubReg() == AMDGPU::NoSubRegister)
8104 Op.setSubReg(AMDGPU::lo16);
8105 return;
8106 }
8107
8108 const TargetRegisterClass *CurrSRC =
8109 RI.getSubRegisterClass(CurrRC, Op.getSubReg());
8110 if (RI.getMatchingSuperRegClass(A: ExpectedRC, B: CurrSRC, Idx: AMDGPU::lo16)) {
8111 const DebugLoc &DL = MI.getDebugLoc();
8112 Register NewDstReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8113 Register Undef = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
8114 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
8115 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewDstReg)
8116 .addReg(RegNo: Op.getReg(), Flags: {}, SubReg: Op.getSubReg())
8117 .addImm(Val: AMDGPU::lo16)
8118 .addReg(RegNo: Undef)
8119 .addImm(Val: AMDGPU::hi16);
8120 Op.setReg(NewDstReg);
8121 Op.setSubReg(AMDGPU::NoSubRegister);
8122 }
8123}
8124void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &MI,
8125 MachineRegisterInfo &MRI) const {
8126 for (unsigned OpIdx = 0; OpIdx < MI.getNumExplicitOperands(); OpIdx++)
8127 legalizeOperandsVALUt16(MI, OpIdx, MRI);
8128}
8129
8130void SIInstrInfo::createWaterFallForSiCall(MachineInstr *MI,
8131 MachineDominatorTree *MDT,
8132 ArrayRef<MachineOperand *> ScalarOps,
8133 ArrayRef<Register> PhySGPRs) const {
8134 assert(MI->getOpcode() == AMDGPU::SI_CALL_ISEL &&
8135 "This only handle waterfall for SI_CALL_ISEL");
8136 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and
8137 // following copies, we also need to move copies from and to physical
8138 // registers into the loop block.
8139 // Also move the copies to physical registers into the loop block
8140 MachineBasicBlock &MBB = *MI->getParent();
8141 MachineBasicBlock::iterator Start(MI);
8142 while (Start->getOpcode() != AMDGPU::ADJCALLSTACKUP)
8143 --Start;
8144 MachineBasicBlock::iterator End(MI);
8145 while (End->getOpcode() != AMDGPU::ADJCALLSTACKDOWN)
8146 ++End;
8147
8148 // Also include following copies of the return value
8149 ++End;
8150 while (End != MBB.end() && End->isCopy() &&
8151 MI->definesRegister(Reg: End->getOperand(i: 1).getReg(), TRI: &RI))
8152 ++End;
8153
8154 generateWaterFallLoop(TII: *this, MI&: *MI, ScalarOps, MDT, Begin: Start, End, PhySGPRs);
8155}
8156
8157void SIInstrInfo::moveToVALU(SIInstrWorklist &Worklist,
8158 MachineDominatorTree *MDT) const {
8159 DenseMap<MachineInstr *, V2PhysSCopyInfo> WaterFalls;
8160 DenseMap<MachineInstr *, bool> V2SPhyCopiesToErase;
8161 while (!Worklist.empty()) {
8162 MachineInstr &Inst = *Worklist.top();
8163 Worklist.erase_top();
8164 // Skip MachineInstr in the deferred list.
8165 if (Worklist.isDeferred(MI: &Inst))
8166 continue;
8167 moveToVALUImpl(Worklist, MDT, Inst, WaterFalls, V2SPhyCopiesToErase);
8168 }
8169
8170 // Deferred list of instructions will be processed once
8171 // all the MachineInstr in the worklist are done.
8172 for (MachineInstr *Inst : Worklist.getDeferredList()) {
8173 moveToVALUImpl(Worklist, MDT, Inst&: *Inst, WaterFalls, V2SPhyCopiesToErase);
8174 assert(Worklist.empty() &&
8175 "Deferred MachineInstr are not supposed to re-populate worklist");
8176 }
8177
8178 for (auto &Entry : WaterFalls) {
8179 if (Entry.first->getOpcode() == AMDGPU::SI_CALL_ISEL)
8180 createWaterFallForSiCall(MI: Entry.first, MDT, ScalarOps: Entry.second.MOs,
8181 PhySGPRs: Entry.second.SGPRs);
8182 }
8183
8184 for (std::pair<MachineInstr *, bool> Entry : V2SPhyCopiesToErase)
8185 if (Entry.second)
8186 Entry.first->eraseFromParent();
8187}
8188void SIInstrInfo::createReadFirstLaneFromCopyToPhysReg(
8189 MachineRegisterInfo &MRI, Register DstReg, MachineInstr &Inst) const {
8190 // If it's a copy of a VGPR to a physical SGPR, insert a V_READFIRSTLANE and
8191 // hope for the best.
8192 const TargetRegisterClass *DstRC = RI.getRegClassForReg(MRI, Reg: DstReg);
8193 ArrayRef<int16_t> SubRegIndices = RI.getRegSplitParts(RC: DstRC, EltSize: 4);
8194 if (SubRegIndices.size() <= 1) {
8195 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
8196 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8197 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: NewDst)
8198 .add(MO: Inst.getOperand(i: 1));
8199 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
8200 DestReg: DstReg)
8201 .addReg(RegNo: NewDst);
8202 } else {
8203 SmallVector<Register, 8> DstRegs;
8204 for (int16_t Indice : SubRegIndices) {
8205 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
8206 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8207 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: NewDst)
8208 .addReg(RegNo: Inst.getOperand(i: 1).getReg(), Flags: {}, SubReg: Indice);
8209
8210 DstRegs.push_back(Elt: NewDst);
8211 }
8212 MachineInstrBuilder MIB =
8213 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8214 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
8215 for (unsigned i = 0; i < SubRegIndices.size(); ++i) {
8216 MIB.addReg(RegNo: DstRegs[i]);
8217 MIB.addImm(Val: RI.getSubRegFromChannel(Channel: i));
8218 }
8219 }
8220}
8221
8222void SIInstrInfo::handleCopyToPhysHelper(
8223 SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst,
8224 MachineRegisterInfo &MRI,
8225 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
8226 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const {
8227 if (DstReg == AMDGPU::M0) {
8228 createReadFirstLaneFromCopyToPhysReg(MRI, DstReg, Inst);
8229 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
8230 return;
8231 }
8232 Register SrcReg = Inst.getOperand(i: 1).getReg();
8233 MachineBasicBlock::iterator I = Inst.getIterator();
8234 MachineBasicBlock::iterator E = Inst.getParent()->end();
8235 // Only search current block since phyreg's def & use cannot cross
8236 // blocks when MF.NoPhi = false.
8237 while (++I != E) {
8238 // For SI_CALL_ISEL users, replace the phys SGPR with the VGPR source
8239 // and record the operand for later waterfall loop generation.
8240 if (I->getOpcode() == AMDGPU::SI_CALL_ISEL) {
8241 MachineInstr *UseMI = &*I;
8242 for (unsigned i = 0; i < UseMI->getNumOperands(); ++i) {
8243 if (UseMI->getOperand(i).isReg() &&
8244 UseMI->getOperand(i).getReg() == DstReg) {
8245 MachineOperand *MO = &UseMI->getOperand(i);
8246 MO->setReg(SrcReg);
8247 V2PhysSCopyInfo &V2SCopyInfo = WaterFalls[UseMI];
8248 V2SCopyInfo.MOs.push_back(Elt: MO);
8249 V2SCopyInfo.SGPRs.push_back(Elt: DstReg);
8250 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
8251 }
8252 }
8253 } else if (I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG &&
8254 I->getOperand(i: 0).isReg() &&
8255 I->getOperand(i: 0).getReg() == DstReg) {
8256 createReadFirstLaneFromCopyToPhysReg(MRI, DstReg, Inst);
8257 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
8258 } else if (I->readsRegister(Reg: DstReg, TRI: &RI)) {
8259 // COPY cannot be erased if other type of inst uses it.
8260 V2SPhyCopiesToErase[&Inst] = false;
8261 }
8262 if (I->findRegisterDefOperand(Reg: DstReg, TRI: &RI))
8263 break;
8264 }
8265}
8266
8267void SIInstrInfo::moveToVALUImpl(
8268 SIInstrWorklist &Worklist, MachineDominatorTree *MDT, MachineInstr &Inst,
8269 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
8270 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const {
8271
8272 MachineBasicBlock *MBB = Inst.getParent();
8273 if (!MBB)
8274 return;
8275 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
8276 unsigned Opcode = Inst.getOpcode();
8277 unsigned NewOpcode = getVALUOp(MI: Inst);
8278 const DebugLoc &DL = Inst.getDebugLoc();
8279
8280 // Handle some special cases
8281 switch (Opcode) {
8282 default:
8283 break;
8284 case AMDGPU::S_ADD_I32:
8285 case AMDGPU::S_SUB_I32: {
8286 // FIXME: The u32 versions currently selected use the carry.
8287 bool Changed;
8288 MachineBasicBlock *CreatedBBTmp = nullptr;
8289 std::tie(args&: Changed, args&: CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT);
8290 if (Changed)
8291 return;
8292
8293 // Default handling
8294 break;
8295 }
8296
8297 case AMDGPU::S_MUL_U64:
8298 if (ST.useVMulU64Inst()) {
8299 NewOpcode = AMDGPU::V_MUL_U64_e64;
8300 break;
8301 }
8302 // Split s_mul_u64 in 32-bit vector multiplications.
8303 splitScalarSMulU64(Worklist, Inst, MDT);
8304 Inst.eraseFromParent();
8305 return;
8306
8307 case AMDGPU::S_MUL_U64_U32_PSEUDO:
8308 case AMDGPU::S_MUL_I64_I32_PSEUDO:
8309 // This is a special case of s_mul_u64 where all the operands are either
8310 // zero extended or sign extended.
8311 splitScalarSMulPseudo(Worklist, Inst, MDT);
8312 Inst.eraseFromParent();
8313 return;
8314
8315 case AMDGPU::S_AND_B64:
8316 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_AND_B32, MDT);
8317 Inst.eraseFromParent();
8318 return;
8319
8320 case AMDGPU::S_OR_B64:
8321 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_OR_B32, MDT);
8322 Inst.eraseFromParent();
8323 return;
8324
8325 case AMDGPU::S_XOR_B64:
8326 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_XOR_B32, MDT);
8327 Inst.eraseFromParent();
8328 return;
8329
8330 case AMDGPU::S_NAND_B64:
8331 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_NAND_B32, MDT);
8332 Inst.eraseFromParent();
8333 return;
8334
8335 case AMDGPU::S_NOR_B64:
8336 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_NOR_B32, MDT);
8337 Inst.eraseFromParent();
8338 return;
8339
8340 case AMDGPU::S_XNOR_B64:
8341 if (ST.hasDLInsts())
8342 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_XNOR_B32, MDT);
8343 else
8344 splitScalar64BitXnor(Worklist, Inst, MDT);
8345 Inst.eraseFromParent();
8346 return;
8347
8348 case AMDGPU::S_ANDN2_B64:
8349 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_ANDN2_B32, MDT);
8350 Inst.eraseFromParent();
8351 return;
8352
8353 case AMDGPU::S_ORN2_B64:
8354 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_ORN2_B32, MDT);
8355 Inst.eraseFromParent();
8356 return;
8357
8358 case AMDGPU::S_BREV_B64:
8359 splitScalar64BitUnaryOp(Worklist, Inst, Opcode: AMDGPU::S_BREV_B32, Swap: true);
8360 Inst.eraseFromParent();
8361 return;
8362
8363 case AMDGPU::S_NOT_B64:
8364 splitScalar64BitUnaryOp(Worklist, Inst, Opcode: AMDGPU::S_NOT_B32);
8365 Inst.eraseFromParent();
8366 return;
8367
8368 case AMDGPU::S_BCNT1_I32_B64:
8369 splitScalar64BitBCNT(Worklist, Inst);
8370 Inst.eraseFromParent();
8371 return;
8372
8373 case AMDGPU::S_BFE_I64:
8374 splitScalar64BitBFE(Worklist, Inst);
8375 Inst.eraseFromParent();
8376 return;
8377
8378 case AMDGPU::S_FLBIT_I32_B64:
8379 splitScalar64BitCountOp(Worklist, Inst, Opcode: AMDGPU::V_FFBH_U32_e32);
8380 Inst.eraseFromParent();
8381 return;
8382 case AMDGPU::S_FF1_I32_B64:
8383 splitScalar64BitCountOp(Worklist, Inst, Opcode: AMDGPU::V_FFBL_B32_e32);
8384 Inst.eraseFromParent();
8385 return;
8386
8387 case AMDGPU::S_LSHL_B32:
8388 if (ST.hasOnlyRevVALUShifts()) {
8389 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
8390 swapOperands(Inst);
8391 }
8392 break;
8393 case AMDGPU::S_ASHR_I32:
8394 if (ST.hasOnlyRevVALUShifts()) {
8395 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
8396 swapOperands(Inst);
8397 }
8398 break;
8399 case AMDGPU::S_LSHR_B32:
8400 if (ST.hasOnlyRevVALUShifts()) {
8401 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
8402 swapOperands(Inst);
8403 }
8404 break;
8405 case AMDGPU::S_LSHL_B64:
8406 if (ST.hasOnlyRevVALUShifts()) {
8407 NewOpcode = ST.getGeneration() >= AMDGPUSubtarget::GFX12
8408 ? AMDGPU::V_LSHLREV_B64_pseudo_e64
8409 : AMDGPU::V_LSHLREV_B64_e64;
8410 swapOperands(Inst);
8411 }
8412 break;
8413 case AMDGPU::S_ASHR_I64:
8414 if (ST.hasOnlyRevVALUShifts()) {
8415 NewOpcode = AMDGPU::V_ASHRREV_I64_e64;
8416 swapOperands(Inst);
8417 }
8418 break;
8419 case AMDGPU::S_LSHR_B64:
8420 if (ST.hasOnlyRevVALUShifts()) {
8421 NewOpcode = AMDGPU::V_LSHRREV_B64_e64;
8422 swapOperands(Inst);
8423 }
8424 break;
8425
8426 case AMDGPU::S_ABS_I32:
8427 lowerScalarAbs(Worklist, Inst);
8428 Inst.eraseFromParent();
8429 return;
8430
8431 case AMDGPU::S_ABSDIFF_I32:
8432 lowerScalarAbsDiff(Worklist, Inst);
8433 Inst.eraseFromParent();
8434 return;
8435
8436 case AMDGPU::S_CBRANCH_SCC0:
8437 case AMDGPU::S_CBRANCH_SCC1: {
8438 // Clear unused bits of vcc
8439 Register CondReg = Inst.getOperand(i: 1).getReg();
8440 bool IsSCC = CondReg == AMDGPU::SCC;
8441 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
8442 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: LMC.AndOpc), DestReg: LMC.VccReg)
8443 .addReg(RegNo: LMC.ExecReg)
8444 .addReg(RegNo: IsSCC ? LMC.VccReg : CondReg);
8445 Inst.removeOperand(OpNo: 1);
8446 } break;
8447
8448 case AMDGPU::S_BFE_U64:
8449 case AMDGPU::S_BFM_B64:
8450 llvm_unreachable("Moving this op to VALU not implemented");
8451
8452 case AMDGPU::S_PACK_LL_B32_B16:
8453 case AMDGPU::S_PACK_LH_B32_B16:
8454 case AMDGPU::S_PACK_HL_B32_B16:
8455 case AMDGPU::S_PACK_HH_B32_B16:
8456 movePackToVALU(Worklist, MRI, Inst);
8457 Inst.eraseFromParent();
8458 return;
8459
8460 case AMDGPU::S_XNOR_B32:
8461 lowerScalarXnor(Worklist, Inst);
8462 Inst.eraseFromParent();
8463 return;
8464
8465 case AMDGPU::S_NAND_B32:
8466 splitScalarNotBinop(Worklist, Inst, Opcode: AMDGPU::S_AND_B32);
8467 Inst.eraseFromParent();
8468 return;
8469
8470 case AMDGPU::S_NOR_B32:
8471 splitScalarNotBinop(Worklist, Inst, Opcode: AMDGPU::S_OR_B32);
8472 Inst.eraseFromParent();
8473 return;
8474
8475 case AMDGPU::S_ANDN2_B32:
8476 splitScalarBinOpN2(Worklist, Inst, Opcode: AMDGPU::S_AND_B32);
8477 Inst.eraseFromParent();
8478 return;
8479
8480 case AMDGPU::S_ORN2_B32:
8481 splitScalarBinOpN2(Worklist, Inst, Opcode: AMDGPU::S_OR_B32);
8482 Inst.eraseFromParent();
8483 return;
8484
8485 // TODO: remove as soon as everything is ready
8486 // to replace VGPR to SGPR copy with V_READFIRSTLANEs.
8487 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO
8488 // can only be selected from the uniform SDNode.
8489 case AMDGPU::S_ADD_CO_PSEUDO:
8490 case AMDGPU::S_SUB_CO_PSEUDO: {
8491 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
8492 ? AMDGPU::V_ADDC_U32_e64
8493 : AMDGPU::V_SUBB_U32_e64;
8494 const auto *CarryRC = RI.getWaveMaskRegClass();
8495
8496 Register CarryInReg = Inst.getOperand(i: 4).getReg();
8497 if (!MRI.constrainRegClass(Reg: CarryInReg, RC: CarryRC)) {
8498 Register NewCarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
8499 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY), DestReg: NewCarryReg)
8500 .addReg(RegNo: CarryInReg);
8501 }
8502
8503 Register CarryOutReg = Inst.getOperand(i: 1).getReg();
8504
8505 Register DestReg = MRI.createVirtualRegister(RegClass: RI.getEquivalentVGPRClass(
8506 SRC: MRI.getRegClass(Reg: Inst.getOperand(i: 0).getReg())));
8507 MachineInstr *CarryOp =
8508 BuildMI(BB&: *MBB, I: &Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: Opc), DestReg)
8509 .addReg(RegNo: CarryOutReg, Flags: RegState::Define)
8510 .add(MO: Inst.getOperand(i: 2))
8511 .add(MO: Inst.getOperand(i: 3))
8512 .addReg(RegNo: CarryInReg)
8513 .addImm(Val: 0);
8514 legalizeOperands(MI&: *CarryOp);
8515 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: DestReg);
8516 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8517 Inst.eraseFromParent();
8518 }
8519 return;
8520 case AMDGPU::S_UADDO_PSEUDO:
8521 case AMDGPU::S_USUBO_PSEUDO: {
8522 MachineOperand &Dest0 = Inst.getOperand(i: 0);
8523 MachineOperand &Dest1 = Inst.getOperand(i: 1);
8524 MachineOperand &Src0 = Inst.getOperand(i: 2);
8525 MachineOperand &Src1 = Inst.getOperand(i: 3);
8526
8527 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
8528 ? AMDGPU::V_ADD_CO_U32_e64
8529 : AMDGPU::V_SUB_CO_U32_e64;
8530 const TargetRegisterClass *NewRC =
8531 RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest0.getReg()));
8532 Register DestReg = MRI.createVirtualRegister(RegClass: NewRC);
8533 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I: &Inst, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
8534 .addReg(RegNo: Dest1.getReg(), Flags: RegState::Define)
8535 .add(MO: Src0)
8536 .add(MO: Src1)
8537 .addImm(Val: 0); // clamp bit
8538
8539 legalizeOperands(MI&: *NewInstr, MDT);
8540 MRI.replaceRegWith(FromReg: Dest0.getReg(), ToReg: DestReg);
8541 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8542 Inst.eraseFromParent();
8543 }
8544 return;
8545 case AMDGPU::S_LSHL1_ADD_U32:
8546 case AMDGPU::S_LSHL2_ADD_U32:
8547 case AMDGPU::S_LSHL3_ADD_U32:
8548 case AMDGPU::S_LSHL4_ADD_U32: {
8549 MachineOperand &Dest = Inst.getOperand(i: 0);
8550 MachineOperand &Src0 = Inst.getOperand(i: 1);
8551 MachineOperand &Src1 = Inst.getOperand(i: 2);
8552 unsigned ShiftAmt = (Opcode == AMDGPU::S_LSHL1_ADD_U32 ? 1
8553 : Opcode == AMDGPU::S_LSHL2_ADD_U32 ? 2
8554 : Opcode == AMDGPU::S_LSHL3_ADD_U32 ? 3
8555 : 4);
8556
8557 const TargetRegisterClass *NewRC =
8558 RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest.getReg()));
8559 Register DestReg = MRI.createVirtualRegister(RegClass: NewRC);
8560 MachineInstr *NewInstr =
8561 BuildMI(BB&: *MBB, I: &Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_ADD_U32_e64), DestReg)
8562 .add(MO: Src0)
8563 .addImm(Val: ShiftAmt)
8564 .add(MO: Src1);
8565
8566 legalizeOperands(MI&: *NewInstr, MDT);
8567 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: DestReg);
8568 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8569 Inst.eraseFromParent();
8570 }
8571 return;
8572 case AMDGPU::S_CSELECT_B32:
8573 case AMDGPU::S_CSELECT_B64:
8574 lowerSelect(Worklist, Inst, MDT);
8575 Inst.eraseFromParent();
8576 return;
8577 case AMDGPU::S_CMP_EQ_I32:
8578 case AMDGPU::S_CMP_LG_I32:
8579 case AMDGPU::S_CMP_GT_I32:
8580 case AMDGPU::S_CMP_GE_I32:
8581 case AMDGPU::S_CMP_LT_I32:
8582 case AMDGPU::S_CMP_LE_I32:
8583 case AMDGPU::S_CMP_EQ_U32:
8584 case AMDGPU::S_CMP_LG_U32:
8585 case AMDGPU::S_CMP_GT_U32:
8586 case AMDGPU::S_CMP_GE_U32:
8587 case AMDGPU::S_CMP_LT_U32:
8588 case AMDGPU::S_CMP_LE_U32:
8589 case AMDGPU::S_CMP_EQ_U64:
8590 case AMDGPU::S_CMP_LG_U64:
8591 case AMDGPU::S_CMP_LT_F32:
8592 case AMDGPU::S_CMP_EQ_F32:
8593 case AMDGPU::S_CMP_LE_F32:
8594 case AMDGPU::S_CMP_GT_F32:
8595 case AMDGPU::S_CMP_LG_F32:
8596 case AMDGPU::S_CMP_GE_F32:
8597 case AMDGPU::S_CMP_O_F32:
8598 case AMDGPU::S_CMP_U_F32:
8599 case AMDGPU::S_CMP_NGE_F32:
8600 case AMDGPU::S_CMP_NLG_F32:
8601 case AMDGPU::S_CMP_NGT_F32:
8602 case AMDGPU::S_CMP_NLE_F32:
8603 case AMDGPU::S_CMP_NEQ_F32:
8604 case AMDGPU::S_CMP_NLT_F32: {
8605 Register CondReg = MRI.createVirtualRegister(RegClass: RI.getWaveMaskRegClass());
8606 auto NewInstr =
8607 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode), DestReg: CondReg)
8608 .setMIFlags(Inst.getFlags());
8609 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src0_modifiers) >=
8610 0) {
8611 NewInstr
8612 .addImm(Val: 0) // src0_modifiers
8613 .add(MO: Inst.getOperand(i: 0)) // src0
8614 .addImm(Val: 0) // src1_modifiers
8615 .add(MO: Inst.getOperand(i: 1)) // src1
8616 .addImm(Val: 0); // clamp
8617 } else {
8618 NewInstr.add(MO: Inst.getOperand(i: 0)).add(MO: Inst.getOperand(i: 1));
8619 }
8620 legalizeOperands(MI&: *NewInstr, MDT);
8621 int SCCIdx = Inst.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
8622 const MachineOperand &SCCOp = Inst.getOperand(i: SCCIdx);
8623 addSCCDefUsersToVALUWorklist(Op: SCCOp, SCCDefInst&: Inst, Worklist, NewCond: CondReg);
8624 Inst.eraseFromParent();
8625 return;
8626 }
8627 case AMDGPU::S_CMP_LT_F16:
8628 case AMDGPU::S_CMP_EQ_F16:
8629 case AMDGPU::S_CMP_LE_F16:
8630 case AMDGPU::S_CMP_GT_F16:
8631 case AMDGPU::S_CMP_LG_F16:
8632 case AMDGPU::S_CMP_GE_F16:
8633 case AMDGPU::S_CMP_O_F16:
8634 case AMDGPU::S_CMP_U_F16:
8635 case AMDGPU::S_CMP_NGE_F16:
8636 case AMDGPU::S_CMP_NLG_F16:
8637 case AMDGPU::S_CMP_NGT_F16:
8638 case AMDGPU::S_CMP_NLE_F16:
8639 case AMDGPU::S_CMP_NEQ_F16:
8640 case AMDGPU::S_CMP_NLT_F16: {
8641 Register CondReg = MRI.createVirtualRegister(RegClass: RI.getWaveMaskRegClass());
8642 auto NewInstr =
8643 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode), DestReg: CondReg)
8644 .setMIFlags(Inst.getFlags());
8645 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::src0_modifiers)) {
8646 NewInstr
8647 .addImm(Val: 0) // src0_modifiers
8648 .add(MO: Inst.getOperand(i: 0)) // src0
8649 .addImm(Val: 0) // src1_modifiers
8650 .add(MO: Inst.getOperand(i: 1)) // src1
8651 .addImm(Val: 0); // clamp
8652 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::op_sel))
8653 NewInstr.addImm(Val: 0); // op_sel0
8654 } else {
8655 NewInstr
8656 .add(MO: Inst.getOperand(i: 0))
8657 .add(MO: Inst.getOperand(i: 1));
8658 }
8659 legalizeOperands(MI&: *NewInstr, MDT);
8660 int SCCIdx = Inst.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
8661 const MachineOperand &SCCOp = Inst.getOperand(i: SCCIdx);
8662 addSCCDefUsersToVALUWorklist(Op: SCCOp, SCCDefInst&: Inst, Worklist, NewCond: CondReg);
8663 Inst.eraseFromParent();
8664 return;
8665 }
8666 case AMDGPU::S_CVT_HI_F32_F16: {
8667 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8668 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8669 if (ST.useRealTrue16Insts()) {
8670 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: TmpReg)
8671 .add(MO: Inst.getOperand(i: 1));
8672 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8673 .addImm(Val: 0) // src0_modifiers
8674 .addReg(RegNo: TmpReg, Flags: {}, SubReg: AMDGPU::hi16)
8675 .addImm(Val: 0) // clamp
8676 .addImm(Val: 0) // omod
8677 .addImm(Val: 0); // op_sel0
8678 } else {
8679 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
8680 .addImm(Val: 16)
8681 .add(MO: Inst.getOperand(i: 1));
8682 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8683 .addImm(Val: 0) // src0_modifiers
8684 .addReg(RegNo: TmpReg)
8685 .addImm(Val: 0) // clamp
8686 .addImm(Val: 0); // omod
8687 }
8688
8689 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8690 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8691 Inst.eraseFromParent();
8692 return;
8693 }
8694 case AMDGPU::S_MINIMUM_F32:
8695 case AMDGPU::S_MAXIMUM_F32: {
8696 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8697 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8698 .addImm(Val: 0) // src0_modifiers
8699 .add(MO: Inst.getOperand(i: 1))
8700 .addImm(Val: 0) // src1_modifiers
8701 .add(MO: Inst.getOperand(i: 2))
8702 .addImm(Val: 0) // clamp
8703 .addImm(Val: 0); // omod
8704 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8705
8706 legalizeOperands(MI&: *NewInstr, MDT);
8707 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8708 Inst.eraseFromParent();
8709 return;
8710 }
8711 case AMDGPU::S_MINIMUM_F16:
8712 case AMDGPU::S_MAXIMUM_F16: {
8713 Register NewDst = MRI.createVirtualRegister(RegClass: ST.useRealTrue16Insts()
8714 ? &AMDGPU::VGPR_16RegClass
8715 : &AMDGPU::VGPR_32RegClass);
8716 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8717 .addImm(Val: 0) // src0_modifiers
8718 .add(MO: Inst.getOperand(i: 1))
8719 .addImm(Val: 0) // src1_modifiers
8720 .add(MO: Inst.getOperand(i: 2))
8721 .addImm(Val: 0) // clamp
8722 .addImm(Val: 0) // omod
8723 .addImm(Val: 0); // opsel0
8724 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8725 legalizeOperands(MI&: *NewInstr, MDT);
8726 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8727 Inst.eraseFromParent();
8728 return;
8729 }
8730 case AMDGPU::V_S_EXP_F16_e64:
8731 case AMDGPU::V_S_LOG_F16_e64:
8732 case AMDGPU::V_S_RCP_F16_e64:
8733 case AMDGPU::V_S_RSQ_F16_e64:
8734 case AMDGPU::V_S_SQRT_F16_e64: {
8735 Register NewDst = MRI.createVirtualRegister(RegClass: ST.useRealTrue16Insts()
8736 ? &AMDGPU::VGPR_16RegClass
8737 : &AMDGPU::VGPR_32RegClass);
8738 auto NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8739 .add(MO: Inst.getOperand(i: 1)) // src0_modifiers
8740 .add(MO: Inst.getOperand(i: 2))
8741 .add(MO: Inst.getOperand(i: 3)) // clamp
8742 .add(MO: Inst.getOperand(i: 4)) // omod
8743 .setMIFlags(Inst.getFlags());
8744 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::op_sel))
8745 NewInstr.addImm(Val: 0); // opsel0
8746 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8747 legalizeOperands(MI&: *NewInstr, MDT);
8748 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8749 Inst.eraseFromParent();
8750 return;
8751 }
8752 }
8753
8754 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
8755 // We cannot move this instruction to the VALU, so we should try to
8756 // legalize its operands instead.
8757 legalizeOperands(MI&: Inst, MDT);
8758 return;
8759 }
8760 // Handle converting generic instructions like COPY-to-SGPR into
8761 // COPY-to-VGPR.
8762 if (NewOpcode == Opcode) {
8763 Register DstReg = Inst.getOperand(i: 0).getReg();
8764 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
8765
8766 if (Inst.isCopy() && DstReg.isPhysical() &&
8767 Inst.getOperand(i: 1).getReg().isVirtual()) {
8768 handleCopyToPhysHelper(Worklist, DstReg, Inst, MRI, WaterFalls,
8769 V2SPhyCopiesToErase);
8770 return;
8771 }
8772
8773 if (Inst.isCopy() && Inst.getOperand(i: 1).getReg().isVirtual()) {
8774 Register NewDstReg = Inst.getOperand(i: 1).getReg();
8775 const TargetRegisterClass *SrcRC = RI.getRegClassForReg(MRI, Reg: NewDstReg);
8776 if (const TargetRegisterClass *CommonRC =
8777 RI.getCommonSubClass(A: NewDstRC, B: SrcRC)) {
8778 // Instead of creating a copy where src and dst are the same register
8779 // class, we just replace all uses of dst with src. These kinds of
8780 // copies interfere with the heuristics MachineSink uses to decide
8781 // whether or not to split a critical edge. Since the pass assumes
8782 // that copies will end up as machine instructions and not be
8783 // eliminated.
8784 addUsersToMoveToVALUWorklist(Reg: DstReg, MRI, Worklist);
8785 unsigned SrcSubReg = Inst.getOperand(i: 1).getSubReg();
8786 bool IsUndef = Inst.getOperand(i: 1).isUndef();
8787 for (MachineOperand &UseMO :
8788 make_early_inc_range(Range: MRI.use_operands(Reg: DstReg))) {
8789 UseMO.setSubReg(
8790 RI.composeSubRegIndices(a: SrcSubReg, b: UseMO.getSubReg()));
8791 UseMO.setReg(NewDstReg);
8792 if (IsUndef)
8793 UseMO.setIsUndef();
8794 }
8795 MRI.clearKillFlags(Reg: NewDstReg);
8796
8797 if (!MRI.constrainRegClass(Reg: NewDstReg, RC: CommonRC))
8798 llvm_unreachable("failed to constrain register");
8799
8800 Inst.eraseFromParent();
8801
8802 for (MachineOperand &UseMO :
8803 make_early_inc_range(Range: MRI.use_operands(Reg: NewDstReg))) {
8804 MachineInstr &UseMI = *UseMO.getParent();
8805
8806 // Legalize t16 operands since replaceReg is called after
8807 // addUsersToVALU.
8808 legalizeOperandsVALUt16(MI&: UseMI, MRI);
8809
8810 unsigned OpIdx = UseMI.getOperandNo(I: &UseMO);
8811 if (const TargetRegisterClass *OpRC =
8812 getRegClass(MCID: UseMI.getDesc(), OpNum: OpIdx))
8813 MRI.constrainRegClass(Reg: NewDstReg, RC: OpRC);
8814 }
8815
8816 return;
8817 }
8818 }
8819
8820 // If this is a v2s copy between 16bit and 32bit reg,
8821 // replace vgpr copy to reg_sequence/extract_subreg
8822 // This can be remove after we have sgpr16 in place
8823 if (ST.useRealTrue16Insts() && Inst.isCopy() &&
8824 Inst.getOperand(i: 1).getReg().isVirtual() &&
8825 RI.isVGPR(MRI, Reg: Inst.getOperand(i: 1).getReg())) {
8826 const TargetRegisterClass *SrcRegRC = getOpRegClass(MI: Inst, OpNo: 1);
8827 if (RI.getMatchingSuperRegClass(A: NewDstRC, B: SrcRegRC, Idx: AMDGPU::lo16)) {
8828 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
8829 Register Undef = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
8830 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8831 MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
8832 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8833 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewDstReg)
8834 .addReg(RegNo: Inst.getOperand(i: 1).getReg())
8835 .addImm(Val: AMDGPU::lo16)
8836 .addReg(RegNo: Undef)
8837 .addImm(Val: AMDGPU::hi16);
8838 Inst.eraseFromParent();
8839 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
8840 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
8841 return;
8842 } else if (RI.getMatchingSuperRegClass(A: SrcRegRC, B: NewDstRC,
8843 Idx: AMDGPU::lo16)) {
8844 Inst.getOperand(i: 1).setSubReg(AMDGPU::lo16);
8845 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
8846 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
8847 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
8848 return;
8849 }
8850 }
8851
8852 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
8853 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
8854 legalizeOperands(MI&: Inst, MDT);
8855 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
8856 return;
8857 }
8858
8859 // Use the new VALU Opcode.
8860 auto NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode))
8861 .setMIFlags(Inst.getFlags());
8862 if (isVOP3(Opcode: NewOpcode) && !isVOP3(Opcode)) {
8863 // Intersperse VOP3 modifiers among the SALU operands.
8864 NewInstr->addOperand(Op: Inst.getOperand(i: 0));
8865 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
8866 Name: AMDGPU::OpName::src0_modifiers) >= 0)
8867 NewInstr.addImm(Val: 0);
8868 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::src0)) {
8869 const MachineOperand &Src = Inst.getOperand(i: 1);
8870 NewInstr->addOperand(Op: Src);
8871 }
8872
8873 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
8874 // We are converting these to a BFE, so we need to add the missing
8875 // operands for the size and offset.
8876 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
8877 NewInstr.addImm(Val: 0);
8878 NewInstr.addImm(Val: Size);
8879 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
8880 // The VALU version adds the second operand to the result, so insert an
8881 // extra 0 operand.
8882 NewInstr.addImm(Val: 0);
8883 } else if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
8884 const MachineOperand &OffsetWidthOp = Inst.getOperand(i: 2);
8885 // If we need to move this to VGPRs, we need to unpack the second
8886 // operand back into the 2 separate ones for bit offset and width.
8887 assert(OffsetWidthOp.isImm() &&
8888 "Scalar BFE is only implemented for constant width and offset");
8889 uint32_t Imm = OffsetWidthOp.getImm();
8890
8891 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
8892 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
8893 NewInstr.addImm(Val: Offset);
8894 NewInstr.addImm(Val: BitWidth);
8895 } else {
8896 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
8897 Name: AMDGPU::OpName::src1_modifiers) >= 0)
8898 NewInstr.addImm(Val: 0);
8899 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src1) >= 0)
8900 NewInstr->addOperand(Op: Inst.getOperand(i: 2));
8901 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
8902 Name: AMDGPU::OpName::src2_modifiers) >= 0)
8903 NewInstr.addImm(Val: 0);
8904 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src2) >= 0)
8905 NewInstr->addOperand(Op: Inst.getOperand(i: 3));
8906 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::clamp) >= 0)
8907 NewInstr.addImm(Val: 0);
8908 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::omod) >= 0)
8909 NewInstr.addImm(Val: 0);
8910 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::op_sel) >= 0)
8911 NewInstr.addImm(Val: 0);
8912 }
8913 } else {
8914 // Just copy the SALU operands.
8915 for (const MachineOperand &Op : Inst.explicit_operands())
8916 NewInstr->addOperand(Op);
8917 }
8918
8919 // Remove any references to SCC. Vector instructions can't read from it, and
8920 // We're just about to add the implicit use / defs of VCC, and we don't want
8921 // both.
8922 for (MachineOperand &Op : Inst.implicit_operands()) {
8923 if (Op.getReg() == AMDGPU::SCC) {
8924 // Only propagate through live-def of SCC.
8925 if (Op.isDef() && !Op.isDead())
8926 addSCCDefUsersToVALUWorklist(Op, SCCDefInst&: Inst, Worklist);
8927 if (Op.isUse())
8928 addSCCDefsToVALUWorklist(SCCUseInst: NewInstr, Worklist);
8929 }
8930 }
8931 Inst.eraseFromParent();
8932 Register NewDstReg;
8933 if (NewInstr->getOperand(i: 0).isReg() && NewInstr->getOperand(i: 0).isDef()) {
8934 Register DstReg = NewInstr->getOperand(i: 0).getReg();
8935 assert(DstReg.isVirtual());
8936 // Update the destination register class.
8937 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst: *NewInstr);
8938 assert(NewDstRC);
8939 NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
8940 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
8941 }
8942 fixImplicitOperands(MI&: *NewInstr);
8943
8944 // Legalize the operands
8945 legalizeOperands(MI&: *NewInstr, MDT);
8946 if (NewDstReg)
8947 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
8948}
8949
8950// Add/sub require special handling to deal with carry outs.
8951std::pair<bool, MachineBasicBlock *>
8952SIInstrInfo::moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
8953 MachineDominatorTree *MDT) const {
8954 if (ST.hasAddNoCarryInsts()) {
8955 // Assume there is no user of scc since we don't select this in that case.
8956 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
8957 // is used.
8958
8959 MachineBasicBlock &MBB = *Inst.getParent();
8960 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8961
8962 Register OldDstReg = Inst.getOperand(i: 0).getReg();
8963 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8964
8965 unsigned Opc = Inst.getOpcode();
8966 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
8967
8968 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
8969 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
8970
8971 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
8972 Inst.removeOperand(OpNo: 3);
8973
8974 Inst.setDesc(get(Opcode: NewOpc));
8975 Inst.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // clamp bit
8976 Inst.addImplicitDefUseOperands(MF&: *MBB.getParent());
8977 MRI.replaceRegWith(FromReg: OldDstReg, ToReg: ResultReg);
8978 MachineBasicBlock *NewBB = legalizeOperands(MI&: Inst, MDT);
8979
8980 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
8981 return std::pair(true, NewBB);
8982 }
8983
8984 return std::pair(false, nullptr);
8985}
8986
8987void SIInstrInfo::lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
8988 MachineDominatorTree *MDT) const {
8989
8990 MachineBasicBlock &MBB = *Inst.getParent();
8991 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8992 MachineBasicBlock::iterator MII = Inst;
8993 const DebugLoc &DL = Inst.getDebugLoc();
8994
8995 MachineOperand &Dest = Inst.getOperand(i: 0);
8996 MachineOperand &Src0 = Inst.getOperand(i: 1);
8997 MachineOperand &Src1 = Inst.getOperand(i: 2);
8998 MachineOperand &Cond = Inst.getOperand(i: 3);
8999
9000 Register CondReg = Cond.getReg();
9001 bool IsSCC = (CondReg == AMDGPU::SCC);
9002
9003 // Remove S_CSELECT instructions that we previously inserted to feed the SCC
9004 // condition output from S_CMP into the SGPR condition input of V_CNDMASK. If
9005 // the S_CMP has been promoted to V_CMP then we can feed its SGPR condition
9006 // output directly into the V_CNDMASK.
9007 if (!IsSCC && Src0.isImm() && (Src0.getImm() == -1) && Src1.isImm() &&
9008 (Src1.getImm() == 0)) {
9009 for (MachineOperand &UseMO :
9010 make_early_inc_range(Range: MRI.use_nodbg_operands(Reg: Dest.getReg()))) {
9011 MachineInstr &UseMI = *UseMO.getParent();
9012 switch (UseMI.getOpcode()) {
9013 case AMDGPU::V_CNDMASK_B16_fake16_e32:
9014 case AMDGPU::V_CNDMASK_B16_fake16_e64:
9015 case AMDGPU::V_CNDMASK_B16_t16_e32:
9016 case AMDGPU::V_CNDMASK_B16_t16_e64:
9017 case AMDGPU::V_CNDMASK_B32_e32:
9018 case AMDGPU::V_CNDMASK_B32_e64:
9019 case AMDGPU::V_CNDMASK_B64_PSEUDO:
9020 if (UseMO.isImplicit() ||
9021 &UseMO == getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src2))
9022 UseMO.setReg(CondReg);
9023 }
9024 }
9025 if (MRI.use_nodbg_empty(RegNo: Dest.getReg()))
9026 return;
9027 }
9028
9029 Register NewCondReg = CondReg;
9030 if (IsSCC) {
9031 const TargetRegisterClass *TC = RI.getWaveMaskRegClass();
9032 NewCondReg = MRI.createVirtualRegister(RegClass: TC);
9033
9034 // Now look for the closest SCC def if it is a copy
9035 // replacing the CondReg with the COPY source register
9036 bool CopyFound = false;
9037 for (MachineInstr &CandI :
9038 make_range(x: std::next(x: MachineBasicBlock::reverse_iterator(Inst)),
9039 y: Inst.getParent()->rend())) {
9040 if (CandI.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isDead: false, Overlap: false) !=
9041 -1) {
9042 if (CandI.isCopy() && CandI.getOperand(i: 0).getReg() == AMDGPU::SCC) {
9043 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: NewCondReg)
9044 .addReg(RegNo: CandI.getOperand(i: 1).getReg());
9045 CopyFound = true;
9046 }
9047 break;
9048 }
9049 }
9050 if (!CopyFound) {
9051 // SCC def is not a copy
9052 // Insert a trivial select instead of creating a copy, because a copy from
9053 // SCC would semantically mean just copying a single bit, but we may need
9054 // the result to be a vector condition mask that needs preserving.
9055 unsigned Opcode =
9056 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
9057 auto NewSelect =
9058 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: NewCondReg).addImm(Val: -1).addImm(Val: 0);
9059 NewSelect->getOperand(i: 3).setIsUndef(Cond.isUndef());
9060 }
9061 }
9062
9063 Register NewDestReg = MRI.createVirtualRegister(
9064 RegClass: RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest.getReg())));
9065 MachineInstr *NewInst;
9066 if (Inst.getOpcode() == AMDGPU::S_CSELECT_B32) {
9067 NewInst = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: NewDestReg)
9068 .addImm(Val: 0)
9069 .add(MO: Src1) // False
9070 .addImm(Val: 0)
9071 .add(MO: Src0) // True
9072 .addReg(RegNo: NewCondReg);
9073 } else {
9074 NewInst =
9075 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B64_PSEUDO), DestReg: NewDestReg)
9076 .add(MO: Src1) // False
9077 .add(MO: Src0) // True
9078 .addReg(RegNo: NewCondReg);
9079 }
9080 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDestReg);
9081 legalizeOperands(MI&: *NewInst, MDT);
9082 addUsersToMoveToVALUWorklist(Reg: NewDestReg, MRI, Worklist);
9083}
9084
9085void SIInstrInfo::lowerScalarAbs(SIInstrWorklist &Worklist,
9086 MachineInstr &Inst) const {
9087 MachineBasicBlock &MBB = *Inst.getParent();
9088 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9089 MachineBasicBlock::iterator MII = Inst;
9090 const DebugLoc &DL = Inst.getDebugLoc();
9091
9092 MachineOperand &Dest = Inst.getOperand(i: 0);
9093 MachineOperand &Src = Inst.getOperand(i: 1);
9094 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9095 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9096
9097 unsigned SubOp = ST.hasAddNoCarryInsts() ? AMDGPU::V_SUB_U32_e32
9098 : AMDGPU::V_SUB_CO_U32_e32;
9099
9100 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: TmpReg)
9101 .addImm(Val: 0)
9102 .addReg(RegNo: Src.getReg());
9103
9104 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MAX_I32_e64), DestReg: ResultReg)
9105 .addReg(RegNo: Src.getReg())
9106 .addReg(RegNo: TmpReg);
9107
9108 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9109 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9110}
9111
9112void SIInstrInfo::lowerScalarAbsDiff(SIInstrWorklist &Worklist,
9113 MachineInstr &Inst) const {
9114 MachineBasicBlock &MBB = *Inst.getParent();
9115 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9116 MachineBasicBlock::iterator MII = Inst;
9117 const DebugLoc &DL = Inst.getDebugLoc();
9118
9119 MachineOperand &Dest = Inst.getOperand(i: 0);
9120 MachineOperand &Src1 = Inst.getOperand(i: 1);
9121 MachineOperand &Src2 = Inst.getOperand(i: 2);
9122 Register SubResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9123 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9124 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9125
9126 unsigned SubOp = ST.hasAddNoCarryInsts() ? AMDGPU::V_SUB_U32_e32
9127 : AMDGPU::V_SUB_CO_U32_e32;
9128
9129 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: SubResultReg)
9130 .addReg(RegNo: Src1.getReg())
9131 .addReg(RegNo: Src2.getReg());
9132
9133 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: TmpReg).addImm(Val: 0).addReg(RegNo: SubResultReg);
9134
9135 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MAX_I32_e64), DestReg: ResultReg)
9136 .addReg(RegNo: SubResultReg)
9137 .addReg(RegNo: TmpReg);
9138
9139 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9140 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9141}
9142
9143void SIInstrInfo::lowerScalarXnor(SIInstrWorklist &Worklist,
9144 MachineInstr &Inst) const {
9145 MachineBasicBlock &MBB = *Inst.getParent();
9146 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9147 MachineBasicBlock::iterator MII = Inst;
9148 const DebugLoc &DL = Inst.getDebugLoc();
9149
9150 MachineOperand &Dest = Inst.getOperand(i: 0);
9151 MachineOperand &Src0 = Inst.getOperand(i: 1);
9152 MachineOperand &Src1 = Inst.getOperand(i: 2);
9153
9154 if (ST.hasDLInsts()) {
9155 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9156 legalizeGenericOperand(InsertMBB&: MBB, I: MII, DstRC: &AMDGPU::VGPR_32RegClass, Op&: Src0, MRI, DL);
9157 legalizeGenericOperand(InsertMBB&: MBB, I: MII, DstRC: &AMDGPU::VGPR_32RegClass, Op&: Src1, MRI, DL);
9158
9159 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_XNOR_B32_e64), DestReg: NewDest)
9160 .add(MO: Src0)
9161 .add(MO: Src1);
9162
9163 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9164 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9165 } else {
9166 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
9167 // invert either source and then perform the XOR. If either source is a
9168 // scalar register, then we can leave the inversion on the scalar unit to
9169 // achieve a better distribution of scalar and vector instructions.
9170 bool Src0IsSGPR = Src0.isReg() &&
9171 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src0.getReg()));
9172 bool Src1IsSGPR = Src1.isReg() &&
9173 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src1.getReg()));
9174 MachineInstr *Xor;
9175 Register Temp = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9176 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9177
9178 // Build a pair of scalar instructions and add them to the work list.
9179 // The next iteration over the work list will lower these to the vector
9180 // unit as necessary.
9181 if (Src0IsSGPR) {
9182 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Temp).add(MO: Src0);
9183 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: NewDest)
9184 .addReg(RegNo: Temp)
9185 .add(MO: Src1);
9186 } else if (Src1IsSGPR) {
9187 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Temp).add(MO: Src1);
9188 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: NewDest)
9189 .add(MO: Src0)
9190 .addReg(RegNo: Temp);
9191 } else {
9192 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: Temp)
9193 .add(MO: Src0)
9194 .add(MO: Src1);
9195 MachineInstr *Not =
9196 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: NewDest).addReg(RegNo: Temp);
9197 Worklist.insert(MI: Not);
9198 }
9199
9200 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9201
9202 Worklist.insert(MI: Xor);
9203
9204 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9205 }
9206}
9207
9208void SIInstrInfo::splitScalarNotBinop(SIInstrWorklist &Worklist,
9209 MachineInstr &Inst,
9210 unsigned Opcode) const {
9211 MachineBasicBlock &MBB = *Inst.getParent();
9212 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9213 MachineBasicBlock::iterator MII = Inst;
9214 const DebugLoc &DL = Inst.getDebugLoc();
9215
9216 MachineOperand &Dest = Inst.getOperand(i: 0);
9217 MachineOperand &Src0 = Inst.getOperand(i: 1);
9218 MachineOperand &Src1 = Inst.getOperand(i: 2);
9219
9220 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9221 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9222
9223 MachineInstr &Op = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: Interm)
9224 .add(MO: Src0)
9225 .add(MO: Src1);
9226
9227 MachineInstr &Not = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: NewDest)
9228 .addReg(RegNo: Interm);
9229
9230 Worklist.insert(MI: &Op);
9231 Worklist.insert(MI: &Not);
9232
9233 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9234 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9235}
9236
9237void SIInstrInfo::splitScalarBinOpN2(SIInstrWorklist &Worklist,
9238 MachineInstr &Inst,
9239 unsigned Opcode) const {
9240 MachineBasicBlock &MBB = *Inst.getParent();
9241 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9242 MachineBasicBlock::iterator MII = Inst;
9243 const DebugLoc &DL = Inst.getDebugLoc();
9244
9245 MachineOperand &Dest = Inst.getOperand(i: 0);
9246 MachineOperand &Src0 = Inst.getOperand(i: 1);
9247 MachineOperand &Src1 = Inst.getOperand(i: 2);
9248
9249 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
9250 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
9251
9252 MachineInstr &Not = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Interm)
9253 .add(MO: Src1);
9254
9255 MachineInstr &Op = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: NewDest)
9256 .add(MO: Src0)
9257 .addReg(RegNo: Interm);
9258
9259 Worklist.insert(MI: &Not);
9260 Worklist.insert(MI: &Op);
9261
9262 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9263 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9264}
9265
9266void SIInstrInfo::splitScalar64BitUnaryOp(SIInstrWorklist &Worklist,
9267 MachineInstr &Inst, unsigned Opcode,
9268 bool Swap) const {
9269 MachineBasicBlock &MBB = *Inst.getParent();
9270 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9271
9272 MachineOperand &Dest = Inst.getOperand(i: 0);
9273 MachineOperand &Src0 = Inst.getOperand(i: 1);
9274 const DebugLoc &DL = Inst.getDebugLoc();
9275
9276 MachineBasicBlock::iterator MII = Inst;
9277
9278 const MCInstrDesc &InstDesc = get(Opcode);
9279 const TargetRegisterClass *Src0RC = Src0.isReg() ?
9280 MRI.getRegClass(Reg: Src0.getReg()) :
9281 &AMDGPU::SGPR_32RegClass;
9282
9283 const TargetRegisterClass *Src0SubRC =
9284 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9285
9286 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9287 SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9288
9289 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9290 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(SRC: DestRC);
9291 const TargetRegisterClass *NewDestSubRC =
9292 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
9293
9294 Register DestSub0 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9295 MachineInstr &LoHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub0).add(MO: SrcReg0Sub0);
9296
9297 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9298 SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9299
9300 Register DestSub1 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9301 MachineInstr &HiHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub1).add(MO: SrcReg0Sub1);
9302
9303 if (Swap)
9304 std::swap(a&: DestSub0, b&: DestSub1);
9305
9306 Register FullDestReg = MRI.createVirtualRegister(RegClass: NewDestRC);
9307 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9308 .addReg(RegNo: DestSub0)
9309 .addImm(Val: AMDGPU::sub0)
9310 .addReg(RegNo: DestSub1)
9311 .addImm(Val: AMDGPU::sub1);
9312
9313 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9314
9315 Worklist.insert(MI: &LoHalf);
9316 Worklist.insert(MI: &HiHalf);
9317
9318 // We don't need to legalizeOperands here because for a single operand, src0
9319 // will support any kind of input.
9320
9321 // Move all users of this moved value.
9322 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9323}
9324
9325// There is not a vector equivalent of s_mul_u64. For this reason, we need to
9326// split the s_mul_u64 in 32-bit vector multiplications.
9327void SIInstrInfo::splitScalarSMulU64(SIInstrWorklist &Worklist,
9328 MachineInstr &Inst,
9329 MachineDominatorTree *MDT) const {
9330 MachineBasicBlock &MBB = *Inst.getParent();
9331 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9332
9333 Register FullDestReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9334 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9335 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9336
9337 MachineOperand &Dest = Inst.getOperand(i: 0);
9338 MachineOperand &Src0 = Inst.getOperand(i: 1);
9339 MachineOperand &Src1 = Inst.getOperand(i: 2);
9340 const DebugLoc &DL = Inst.getDebugLoc();
9341 MachineBasicBlock::iterator MII = Inst;
9342
9343 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0.getReg());
9344 const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: Src1.getReg());
9345 const TargetRegisterClass *Src0SubRC =
9346 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9347 if (RI.isSGPRClass(RC: Src0SubRC))
9348 Src0SubRC = RI.getEquivalentVGPRClass(SRC: Src0SubRC);
9349 const TargetRegisterClass *Src1SubRC =
9350 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9351 if (RI.isSGPRClass(RC: Src1SubRC))
9352 Src1SubRC = RI.getEquivalentVGPRClass(SRC: Src1SubRC);
9353
9354 // First, we extract the low 32-bit and high 32-bit values from each of the
9355 // operands.
9356 MachineOperand Op0L =
9357 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9358 MachineOperand Op1L =
9359 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9360 MachineOperand Op0H =
9361 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9362 MachineOperand Op1H =
9363 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
9364
9365 // The multilication is done as follows:
9366 //
9367 // Op1H Op1L
9368 // * Op0H Op0L
9369 // --------------------
9370 // Op1H*Op0L Op1L*Op0L
9371 // + Op1H*Op0H Op1L*Op0H
9372 // -----------------------------------------
9373 // (Op1H*Op0L + Op1L*Op0H + carry) Op1L*Op0L
9374 //
9375 // We drop Op1H*Op0H because the result of the multiplication is a 64-bit
9376 // value and that would overflow.
9377 // The low 32-bit value is Op1L*Op0L.
9378 // The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from Op1L*Op0L).
9379
9380 Register Op1L_Op0H_Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9381 MachineInstr *Op1L_Op0H =
9382 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: Op1L_Op0H_Reg)
9383 .add(MO: Op1L)
9384 .add(MO: Op0H);
9385
9386 Register Op1H_Op0L_Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9387 MachineInstr *Op1H_Op0L =
9388 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: Op1H_Op0L_Reg)
9389 .add(MO: Op1H)
9390 .add(MO: Op0L);
9391
9392 Register CarryReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9393 MachineInstr *Carry =
9394 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_HI_U32_e64), DestReg: CarryReg)
9395 .add(MO: Op1L)
9396 .add(MO: Op0L);
9397
9398 MachineInstr *LoHalf =
9399 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: DestSub0)
9400 .add(MO: Op1L)
9401 .add(MO: Op0L);
9402
9403 Register AddReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9404 MachineInstr *Add = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg: AddReg)
9405 .addReg(RegNo: Op1L_Op0H_Reg)
9406 .addReg(RegNo: Op1H_Op0L_Reg);
9407
9408 MachineInstr *HiHalf =
9409 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg: DestSub1)
9410 .addReg(RegNo: AddReg)
9411 .addReg(RegNo: CarryReg);
9412
9413 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9414 .addReg(RegNo: DestSub0)
9415 .addImm(Val: AMDGPU::sub0)
9416 .addReg(RegNo: DestSub1)
9417 .addImm(Val: AMDGPU::sub1);
9418
9419 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9420
9421 // Try to legalize the operands in case we need to swap the order to keep it
9422 // valid.
9423 legalizeOperands(MI&: *Op1L_Op0H, MDT);
9424 legalizeOperands(MI&: *Op1H_Op0L, MDT);
9425 legalizeOperands(MI&: *Carry, MDT);
9426 legalizeOperands(MI&: *LoHalf, MDT);
9427 legalizeOperands(MI&: *Add, MDT);
9428 legalizeOperands(MI&: *HiHalf, MDT);
9429
9430 // Move all users of this moved value.
9431 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9432}
9433
9434// Lower S_MUL_U64_U32_PSEUDO/S_MUL_I64_I32_PSEUDO in two 32-bit vector
9435// multiplications.
9436void SIInstrInfo::splitScalarSMulPseudo(SIInstrWorklist &Worklist,
9437 MachineInstr &Inst,
9438 MachineDominatorTree *MDT) const {
9439 MachineBasicBlock &MBB = *Inst.getParent();
9440 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9441
9442 Register FullDestReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9443 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9444 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9445
9446 MachineOperand &Dest = Inst.getOperand(i: 0);
9447 MachineOperand &Src0 = Inst.getOperand(i: 1);
9448 MachineOperand &Src1 = Inst.getOperand(i: 2);
9449 const DebugLoc &DL = Inst.getDebugLoc();
9450 MachineBasicBlock::iterator MII = Inst;
9451
9452 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0.getReg());
9453 const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: Src1.getReg());
9454 const TargetRegisterClass *Src0SubRC =
9455 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9456 if (RI.isSGPRClass(RC: Src0SubRC))
9457 Src0SubRC = RI.getEquivalentVGPRClass(SRC: Src0SubRC);
9458 const TargetRegisterClass *Src1SubRC =
9459 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9460 if (RI.isSGPRClass(RC: Src1SubRC))
9461 Src1SubRC = RI.getEquivalentVGPRClass(SRC: Src1SubRC);
9462
9463 // First, we extract the low 32-bit and high 32-bit values from each of the
9464 // operands.
9465 MachineOperand Op0L =
9466 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9467 MachineOperand Op1L =
9468 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9469
9470 unsigned Opc = Inst.getOpcode();
9471 unsigned NewOpc = Opc == AMDGPU::S_MUL_U64_U32_PSEUDO
9472 ? AMDGPU::V_MUL_HI_U32_e64
9473 : AMDGPU::V_MUL_HI_I32_e64;
9474 MachineInstr *HiHalf =
9475 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: NewOpc), DestReg: DestSub1).add(MO: Op1L).add(MO: Op0L);
9476
9477 MachineInstr *LoHalf =
9478 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: DestSub0)
9479 .add(MO: Op1L)
9480 .add(MO: Op0L);
9481
9482 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9483 .addReg(RegNo: DestSub0)
9484 .addImm(Val: AMDGPU::sub0)
9485 .addReg(RegNo: DestSub1)
9486 .addImm(Val: AMDGPU::sub1);
9487
9488 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9489
9490 // Try to legalize the operands in case we need to swap the order to keep it
9491 // valid.
9492 legalizeOperands(MI&: *HiHalf, MDT);
9493 legalizeOperands(MI&: *LoHalf, MDT);
9494
9495 // Move all users of this moved value.
9496 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9497}
9498
9499void SIInstrInfo::splitScalar64BitBinaryOp(SIInstrWorklist &Worklist,
9500 MachineInstr &Inst, unsigned Opcode,
9501 MachineDominatorTree *MDT) const {
9502 MachineBasicBlock &MBB = *Inst.getParent();
9503 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9504
9505 MachineOperand &Dest = Inst.getOperand(i: 0);
9506 MachineOperand &Src0 = Inst.getOperand(i: 1);
9507 MachineOperand &Src1 = Inst.getOperand(i: 2);
9508 const DebugLoc &DL = Inst.getDebugLoc();
9509
9510 MachineBasicBlock::iterator MII = Inst;
9511
9512 const MCInstrDesc &InstDesc = get(Opcode);
9513 const TargetRegisterClass *Src0RC = Src0.isReg() ?
9514 MRI.getRegClass(Reg: Src0.getReg()) :
9515 &AMDGPU::SGPR_32RegClass;
9516
9517 const TargetRegisterClass *Src0SubRC =
9518 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9519 const TargetRegisterClass *Src1RC = Src1.isReg() ?
9520 MRI.getRegClass(Reg: Src1.getReg()) :
9521 &AMDGPU::SGPR_32RegClass;
9522
9523 const TargetRegisterClass *Src1SubRC =
9524 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9525
9526 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9527 SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9528 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC,
9529 SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9530 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9531 SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9532 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC,
9533 SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
9534
9535 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9536 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(SRC: DestRC);
9537 const TargetRegisterClass *NewDestSubRC =
9538 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
9539
9540 Register DestSub0 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9541 MachineInstr &LoHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub0)
9542 .add(MO: SrcReg0Sub0)
9543 .add(MO: SrcReg1Sub0);
9544
9545 Register DestSub1 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9546 MachineInstr &HiHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub1)
9547 .add(MO: SrcReg0Sub1)
9548 .add(MO: SrcReg1Sub1);
9549
9550 Register FullDestReg = MRI.createVirtualRegister(RegClass: NewDestRC);
9551 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9552 .addReg(RegNo: DestSub0)
9553 .addImm(Val: AMDGPU::sub0)
9554 .addReg(RegNo: DestSub1)
9555 .addImm(Val: AMDGPU::sub1);
9556
9557 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9558
9559 Worklist.insert(MI: &LoHalf);
9560 Worklist.insert(MI: &HiHalf);
9561
9562 // Move all users of this moved value.
9563 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9564}
9565
9566void SIInstrInfo::splitScalar64BitXnor(SIInstrWorklist &Worklist,
9567 MachineInstr &Inst,
9568 MachineDominatorTree *MDT) const {
9569 MachineBasicBlock &MBB = *Inst.getParent();
9570 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9571
9572 MachineOperand &Dest = Inst.getOperand(i: 0);
9573 MachineOperand &Src0 = Inst.getOperand(i: 1);
9574 MachineOperand &Src1 = Inst.getOperand(i: 2);
9575 const DebugLoc &DL = Inst.getDebugLoc();
9576
9577 MachineBasicBlock::iterator MII = Inst;
9578
9579 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9580
9581 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
9582
9583 MachineOperand* Op0;
9584 MachineOperand* Op1;
9585
9586 if (Src0.isReg() && RI.isSGPRReg(MRI, Reg: Src0.getReg())) {
9587 Op0 = &Src0;
9588 Op1 = &Src1;
9589 } else {
9590 Op0 = &Src1;
9591 Op1 = &Src0;
9592 }
9593
9594 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B64), DestReg: Interm)
9595 .add(MO: *Op0);
9596
9597 Register NewDest = MRI.createVirtualRegister(RegClass: DestRC);
9598
9599 MachineInstr &Xor = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B64), DestReg: NewDest)
9600 .addReg(RegNo: Interm)
9601 .add(MO: *Op1);
9602
9603 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9604
9605 Worklist.insert(MI: &Xor);
9606}
9607
9608void SIInstrInfo::splitScalar64BitBCNT(SIInstrWorklist &Worklist,
9609 MachineInstr &Inst) const {
9610 MachineBasicBlock &MBB = *Inst.getParent();
9611 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9612
9613 MachineBasicBlock::iterator MII = Inst;
9614 const DebugLoc &DL = Inst.getDebugLoc();
9615
9616 MachineOperand &Dest = Inst.getOperand(i: 0);
9617 MachineOperand &Src = Inst.getOperand(i: 1);
9618
9619 const MCInstrDesc &InstDesc = get(Opcode: AMDGPU::V_BCNT_U32_B32_e64);
9620 const TargetRegisterClass *SrcRC = Src.isReg() ?
9621 MRI.getRegClass(Reg: Src.getReg()) :
9622 &AMDGPU::SGPR_32RegClass;
9623
9624 Register MidReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9625 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9626
9627 const TargetRegisterClass *SrcSubRC =
9628 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
9629
9630 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC,
9631 SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
9632 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC,
9633 SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
9634
9635 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg).add(MO: SrcRegSub0).addImm(Val: 0);
9636
9637 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: ResultReg).add(MO: SrcRegSub1).addReg(RegNo: MidReg);
9638
9639 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9640
9641 // We don't need to legalize operands here. src0 for either instruction can be
9642 // an SGPR, and the second input is unused or determined here.
9643 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9644}
9645
9646void SIInstrInfo::splitScalar64BitBFE(SIInstrWorklist &Worklist,
9647 MachineInstr &Inst) const {
9648 MachineBasicBlock &MBB = *Inst.getParent();
9649 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9650 MachineBasicBlock::iterator MII = Inst;
9651 const DebugLoc &DL = Inst.getDebugLoc();
9652
9653 MachineOperand &Dest = Inst.getOperand(i: 0);
9654 uint32_t Imm = Inst.getOperand(i: 2).getImm();
9655 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
9656 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
9657
9658 (void) Offset;
9659
9660 // Only sext_inreg cases handled.
9661 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
9662 Offset == 0 && "Not implemented");
9663
9664 if (BitWidth < 32) {
9665 Register MidRegLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9666 Register MidRegHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9667 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9668
9669 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_BFE_I32_e64), DestReg: MidRegLo)
9670 .addReg(RegNo: Inst.getOperand(i: 1).getReg(), Flags: {}, SubReg: AMDGPU::sub0)
9671 .addImm(Val: 0)
9672 .addImm(Val: BitWidth);
9673
9674 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ASHRREV_I32_e32), DestReg: MidRegHi)
9675 .addImm(Val: 31)
9676 .addReg(RegNo: MidRegLo);
9677
9678 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: ResultReg)
9679 .addReg(RegNo: MidRegLo)
9680 .addImm(Val: AMDGPU::sub0)
9681 .addReg(RegNo: MidRegHi)
9682 .addImm(Val: AMDGPU::sub1);
9683
9684 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9685 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9686 return;
9687 }
9688
9689 MachineOperand &Src = Inst.getOperand(i: 1);
9690 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9691 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9692
9693 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ASHRREV_I32_e64), DestReg: TmpReg)
9694 .addImm(Val: 31)
9695 .addReg(RegNo: Src.getReg(), Flags: {}, SubReg: AMDGPU::sub0);
9696
9697 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: ResultReg)
9698 .addReg(RegNo: Src.getReg(), Flags: {}, SubReg: AMDGPU::sub0)
9699 .addImm(Val: AMDGPU::sub0)
9700 .addReg(RegNo: TmpReg)
9701 .addImm(Val: AMDGPU::sub1);
9702
9703 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9704 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9705}
9706
9707void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist,
9708 MachineInstr &Inst, unsigned Opcode,
9709 MachineDominatorTree *MDT) const {
9710 // (S_FLBIT_I32_B64 hi:lo) ->
9711 // -> (umin (V_FFBH_U32_e32 hi), (or (V_FFBH_U32_e32 lo), 32))
9712 // (S_FF1_I32_B64 hi:lo) ->
9713 // ->(umin (or (V_FFBL_B32_e32 hi), 32) (V_FFBL_B32_e32 lo))
9714
9715 MachineBasicBlock &MBB = *Inst.getParent();
9716 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9717 MachineBasicBlock::iterator MII = Inst;
9718 const DebugLoc &DL = Inst.getDebugLoc();
9719
9720 MachineOperand &Dest = Inst.getOperand(i: 0);
9721 MachineOperand &Src = Inst.getOperand(i: 1);
9722
9723 const MCInstrDesc &InstDesc = get(Opcode);
9724
9725 bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32;
9726
9727 const TargetRegisterClass *SrcRC =
9728 Src.isReg() ? MRI.getRegClass(Reg: Src.getReg()) : &AMDGPU::SGPR_32RegClass;
9729 const TargetRegisterClass *SrcSubRC =
9730 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
9731
9732 MachineOperand SrcRegSub0 =
9733 buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC, SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
9734 MachineOperand SrcRegSub1 =
9735 buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC, SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
9736
9737 Register MidReg1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9738 Register MidReg2 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9739 Register MidReg3 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9740 Register MidReg4 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9741
9742 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg1).add(MO: SrcRegSub0);
9743
9744 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg2).add(MO: SrcRegSub1);
9745
9746 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_OR_B32_e32), DestReg: MidReg3)
9747 .addImm(Val: 32)
9748 .addReg(RegNo: IsCtlz ? MidReg1 : MidReg2);
9749
9750 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MIN_U32_e64), DestReg: MidReg4)
9751 .addReg(RegNo: MidReg3)
9752 .addReg(RegNo: IsCtlz ? MidReg2 : MidReg1);
9753
9754 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: MidReg4);
9755
9756 addUsersToMoveToVALUWorklist(Reg: MidReg4, MRI, Worklist);
9757}
9758
9759void SIInstrInfo::addUsersToMoveToVALUWorklist(
9760 Register DstReg, MachineRegisterInfo &MRI,
9761 SIInstrWorklist &Worklist) const {
9762 for (MachineOperand &MO : make_early_inc_range(Range: MRI.use_operands(Reg: DstReg))) {
9763 MachineInstr &UseMI = *MO.getParent();
9764
9765 unsigned OpNo = 0;
9766
9767 switch (UseMI.getOpcode()) {
9768 case AMDGPU::COPY:
9769 case AMDGPU::WQM:
9770 case AMDGPU::SOFT_WQM:
9771 case AMDGPU::STRICT_WWM:
9772 case AMDGPU::STRICT_WQM:
9773 case AMDGPU::REG_SEQUENCE:
9774 case AMDGPU::PHI:
9775 case AMDGPU::INSERT_SUBREG:
9776 break;
9777 default:
9778 OpNo = MO.getOperandNo();
9779 break;
9780 }
9781
9782 const TargetRegisterClass *OpRC = getOpRegClass(MI: UseMI, OpNo);
9783 MRI.constrainRegClass(Reg: DstReg, RC: OpRC);
9784
9785 if (!RI.hasVectorRegisters(RC: OpRC))
9786 Worklist.insert(MI: &UseMI);
9787 else
9788 // Legalization could change user list.
9789 legalizeOperandsVALUt16(MI&: UseMI, OpIdx: OpNo, MRI);
9790 }
9791}
9792
9793void SIInstrInfo::movePackToVALU(SIInstrWorklist &Worklist,
9794 MachineRegisterInfo &MRI,
9795 MachineInstr &Inst) const {
9796 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9797 MachineBasicBlock *MBB = Inst.getParent();
9798 MachineOperand &Src0 = Inst.getOperand(i: 1);
9799 MachineOperand &Src1 = Inst.getOperand(i: 2);
9800 const DebugLoc &DL = Inst.getDebugLoc();
9801
9802 if (ST.useRealTrue16Insts()) {
9803 Register SrcReg0, SrcReg1;
9804 if (!Src0.isReg() || !RI.isVGPR(MRI, Reg: Src0.getReg())) {
9805 SrcReg0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9806 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL,
9807 MCID: get(Opcode: Src0.isImm() ? AMDGPU::V_MOV_B32_e32 : AMDGPU::COPY), DestReg: SrcReg0)
9808 .add(MO: Src0);
9809 } else {
9810 SrcReg0 = Src0.getReg();
9811 }
9812
9813 if (!Src1.isReg() || !RI.isVGPR(MRI, Reg: Src1.getReg())) {
9814 SrcReg1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9815 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL,
9816 MCID: get(Opcode: Src1.isImm() ? AMDGPU::V_MOV_B32_e32 : AMDGPU::COPY), DestReg: SrcReg1)
9817 .add(MO: Src1);
9818 } else {
9819 SrcReg1 = Src1.getReg();
9820 }
9821
9822 bool isSrc0Reg16 = MRI.constrainRegClass(Reg: SrcReg0, RC: &AMDGPU::VGPR_16RegClass);
9823 bool isSrc1Reg16 = MRI.constrainRegClass(Reg: SrcReg1, RC: &AMDGPU::VGPR_16RegClass);
9824
9825 auto NewMI = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: ResultReg);
9826 switch (Inst.getOpcode()) {
9827 case AMDGPU::S_PACK_LL_B32_B16:
9828 NewMI
9829 .addReg(RegNo: SrcReg0, Flags: {},
9830 SubReg: isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
9831 .addImm(Val: AMDGPU::lo16)
9832 .addReg(RegNo: SrcReg1, Flags: {},
9833 SubReg: isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
9834 .addImm(Val: AMDGPU::hi16);
9835 break;
9836 case AMDGPU::S_PACK_LH_B32_B16:
9837 NewMI
9838 .addReg(RegNo: SrcReg0, Flags: {},
9839 SubReg: isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
9840 .addImm(Val: AMDGPU::lo16)
9841 .addReg(RegNo: SrcReg1, Flags: {}, SubReg: AMDGPU::hi16)
9842 .addImm(Val: AMDGPU::hi16);
9843 break;
9844 case AMDGPU::S_PACK_HL_B32_B16:
9845 NewMI.addReg(RegNo: SrcReg0, Flags: {}, SubReg: AMDGPU::hi16)
9846 .addImm(Val: AMDGPU::lo16)
9847 .addReg(RegNo: SrcReg1, Flags: {},
9848 SubReg: isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
9849 .addImm(Val: AMDGPU::hi16);
9850 break;
9851 case AMDGPU::S_PACK_HH_B32_B16:
9852 NewMI.addReg(RegNo: SrcReg0, Flags: {}, SubReg: AMDGPU::hi16)
9853 .addImm(Val: AMDGPU::lo16)
9854 .addReg(RegNo: SrcReg1, Flags: {}, SubReg: AMDGPU::hi16)
9855 .addImm(Val: AMDGPU::hi16);
9856 break;
9857 default:
9858 llvm_unreachable("unhandled s_pack_* instruction");
9859 }
9860
9861 MachineOperand &Dest = Inst.getOperand(i: 0);
9862 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9863 addUsersToMoveToVALUWorklist(DstReg: ResultReg, MRI, Worklist);
9864 return;
9865 }
9866
9867 switch (Inst.getOpcode()) {
9868 case AMDGPU::S_PACK_LL_B32_B16: {
9869 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9870 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9871
9872 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are
9873 // 0.
9874 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
9875 .addImm(Val: 0xffff);
9876
9877 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_AND_B32_e64), DestReg: TmpReg)
9878 .addReg(RegNo: ImmReg, Flags: RegState::Kill)
9879 .add(MO: Src0);
9880
9881 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: ResultReg)
9882 .add(MO: Src1)
9883 .addImm(Val: 16)
9884 .addReg(RegNo: TmpReg, Flags: RegState::Kill);
9885 break;
9886 }
9887 case AMDGPU::S_PACK_LH_B32_B16: {
9888 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9889 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
9890 .addImm(Val: 0xffff);
9891 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_BFI_B32_e64), DestReg: ResultReg)
9892 .addReg(RegNo: ImmReg, Flags: RegState::Kill)
9893 .add(MO: Src0)
9894 .add(MO: Src1);
9895 break;
9896 }
9897 case AMDGPU::S_PACK_HL_B32_B16: {
9898 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9899 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
9900 .addImm(Val: 16)
9901 .add(MO: Src0);
9902 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: ResultReg)
9903 .add(MO: Src1)
9904 .addImm(Val: 16)
9905 .addReg(RegNo: TmpReg, Flags: RegState::Kill);
9906 break;
9907 }
9908 case AMDGPU::S_PACK_HH_B32_B16: {
9909 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9910 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9911 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
9912 .addImm(Val: 16)
9913 .add(MO: Src0);
9914 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
9915 .addImm(Val: 0xffff0000);
9916 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_AND_OR_B32_e64), DestReg: ResultReg)
9917 .add(MO: Src1)
9918 .addReg(RegNo: ImmReg, Flags: RegState::Kill)
9919 .addReg(RegNo: TmpReg, Flags: RegState::Kill);
9920 break;
9921 }
9922 default:
9923 llvm_unreachable("unhandled s_pack_* instruction");
9924 }
9925
9926 MachineOperand &Dest = Inst.getOperand(i: 0);
9927 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9928 addUsersToMoveToVALUWorklist(DstReg: ResultReg, MRI, Worklist);
9929}
9930
9931void SIInstrInfo::addSCCDefUsersToVALUWorklist(const MachineOperand &Op,
9932 MachineInstr &SCCDefInst,
9933 SIInstrWorklist &Worklist,
9934 Register NewCond) const {
9935
9936 // Ensure that def inst defines SCC, which is still live.
9937 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() &&
9938 !Op.isDead() && Op.getParent() == &SCCDefInst);
9939 SmallVector<MachineInstr *, 4> CopyToDelete;
9940 // This assumes that all the users of SCC are in the same block
9941 // as the SCC def.
9942 for (MachineInstr &MI : // Skip the def inst itself.
9943 make_range(x: std::next(x: MachineBasicBlock::iterator(SCCDefInst)),
9944 y: SCCDefInst.getParent()->end())) {
9945 // Check if SCC is used first.
9946 int SCCIdx = MI.findRegisterUseOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isKill: false);
9947 if (SCCIdx != -1) {
9948 if (MI.isCopy()) {
9949 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
9950 Register DestReg = MI.getOperand(i: 0).getReg();
9951
9952 MRI.replaceRegWith(FromReg: DestReg, ToReg: NewCond);
9953 CopyToDelete.push_back(Elt: &MI);
9954 } else {
9955
9956 if (NewCond.isValid())
9957 MI.getOperand(i: SCCIdx).setReg(NewCond);
9958
9959 Worklist.insert(MI: &MI);
9960 }
9961 }
9962 // Exit if we find another SCC def.
9963 if (MI.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isDead: false, Overlap: false) != -1)
9964 break;
9965 }
9966 for (auto &Copy : CopyToDelete)
9967 Copy->eraseFromParent();
9968}
9969
9970// Instructions that use SCC may be converted to VALU instructions. When that
9971// happens, the SCC register is changed to VCC_LO. The instruction that defines
9972// SCC must be changed to an instruction that defines VCC. This function makes
9973// sure that the instruction that defines SCC is added to the moveToVALU
9974// worklist.
9975void SIInstrInfo::addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
9976 SIInstrWorklist &Worklist) const {
9977 // Look for a preceding instruction that either defines VCC or SCC. If VCC
9978 // then there is nothing to do because the defining instruction has been
9979 // converted to a VALU already. If SCC then that instruction needs to be
9980 // converted to a VALU.
9981 for (MachineInstr &MI :
9982 make_range(x: std::next(x: MachineBasicBlock::reverse_iterator(SCCUseInst)),
9983 y: SCCUseInst->getParent()->rend())) {
9984 if (MI.modifiesRegister(Reg: AMDGPU::VCC, TRI: &RI))
9985 break;
9986 if (MI.definesRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
9987 Worklist.insert(MI: &MI);
9988 break;
9989 }
9990 }
9991}
9992
9993const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
9994 const MachineInstr &Inst) const {
9995 const TargetRegisterClass *NewDstRC = getOpRegClass(MI: Inst, OpNo: 0);
9996
9997 switch (Inst.getOpcode()) {
9998 // For target instructions, getOpRegClass just returns the virtual register
9999 // class associated with the operand, so we need to find an equivalent VGPR
10000 // register class in order to move the instruction to the VALU.
10001 case AMDGPU::COPY:
10002 case AMDGPU::PHI:
10003 case AMDGPU::REG_SEQUENCE:
10004 case AMDGPU::INSERT_SUBREG:
10005 case AMDGPU::WQM:
10006 case AMDGPU::SOFT_WQM:
10007 case AMDGPU::STRICT_WWM:
10008 case AMDGPU::STRICT_WQM: {
10009 const TargetRegisterClass *SrcRC = getOpRegClass(MI: Inst, OpNo: 1);
10010 if (RI.isAGPRClass(RC: SrcRC)) {
10011 if (RI.isAGPRClass(RC: NewDstRC))
10012 return nullptr;
10013
10014 switch (Inst.getOpcode()) {
10015 case AMDGPU::PHI:
10016 case AMDGPU::REG_SEQUENCE:
10017 case AMDGPU::INSERT_SUBREG:
10018 NewDstRC = RI.getEquivalentAGPRClass(SRC: NewDstRC);
10019 break;
10020 default:
10021 NewDstRC = RI.getEquivalentVGPRClass(SRC: NewDstRC);
10022 }
10023
10024 if (!NewDstRC)
10025 return nullptr;
10026 } else {
10027 if (!RI.isSGPRClass(RC: NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass)
10028 return nullptr;
10029
10030 NewDstRC = RI.getEquivalentVGPRClass(SRC: NewDstRC);
10031 if (!NewDstRC)
10032 return nullptr;
10033 }
10034
10035 return NewDstRC;
10036 }
10037 default:
10038 return NewDstRC;
10039 }
10040}
10041
10042// Find the one SGPR operand we are allowed to use.
10043Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
10044 int OpIndices[3]) const {
10045 const MCInstrDesc &Desc = MI.getDesc();
10046
10047 // Find the one SGPR operand we are allowed to use.
10048 //
10049 // First we need to consider the instruction's operand requirements before
10050 // legalizing. Some operands are required to be SGPRs, such as implicit uses
10051 // of VCC, but we are still bound by the constant bus requirement to only use
10052 // one.
10053 //
10054 // If the operand's class is an SGPR, we can never move it.
10055
10056 Register SGPRReg = findImplicitSGPRRead(MI);
10057 if (SGPRReg)
10058 return SGPRReg;
10059
10060 Register UsedSGPRs[3] = {Register()};
10061 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
10062
10063 for (unsigned i = 0; i < 3; ++i) {
10064 int Idx = OpIndices[i];
10065 if (Idx == -1)
10066 break;
10067
10068 const MachineOperand &MO = MI.getOperand(i: Idx);
10069 if (!MO.isReg())
10070 continue;
10071
10072 // Is this operand statically required to be an SGPR based on the operand
10073 // constraints?
10074 const TargetRegisterClass *OpRC =
10075 RI.getRegClass(i: getOpRegClassID(OpInfo: Desc.operands()[Idx]));
10076 bool IsRequiredSGPR = RI.isSGPRClass(RC: OpRC);
10077 if (IsRequiredSGPR)
10078 return MO.getReg();
10079
10080 // If this could be a VGPR or an SGPR, Check the dynamic register class.
10081 Register Reg = MO.getReg();
10082 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
10083 if (RI.isSGPRClass(RC: RegRC))
10084 UsedSGPRs[i] = Reg;
10085 }
10086
10087 // We don't have a required SGPR operand, so we have a bit more freedom in
10088 // selecting operands to move.
10089
10090 // Try to select the most used SGPR. If an SGPR is equal to one of the
10091 // others, we choose that.
10092 //
10093 // e.g.
10094 // V_FMA_F32 v0, s0, s0, s0 -> No moves
10095 // V_FMA_F32 v0, s0, s1, s0 -> Move s1
10096
10097 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
10098 // prefer those.
10099
10100 if (UsedSGPRs[0]) {
10101 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
10102 SGPRReg = UsedSGPRs[0];
10103 }
10104
10105 if (!SGPRReg && UsedSGPRs[1]) {
10106 if (UsedSGPRs[1] == UsedSGPRs[2])
10107 SGPRReg = UsedSGPRs[1];
10108 }
10109
10110 return SGPRReg;
10111}
10112
10113MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI,
10114 AMDGPU::OpName OperandName) const {
10115 if (OperandName == AMDGPU::OpName::NUM_OPERAND_NAMES)
10116 return nullptr;
10117
10118 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OperandName);
10119 if (Idx == -1)
10120 return nullptr;
10121
10122 return &MI.getOperand(i: Idx);
10123}
10124
10125uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {
10126 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
10127 int64_t Format = ST.getGeneration() >= AMDGPUSubtarget::GFX11
10128 ? (int64_t)AMDGPU::UfmtGFX11::UFMT_32_FLOAT
10129 : (int64_t)AMDGPU::UfmtGFX10::UFMT_32_FLOAT;
10130 return (Format << 44) |
10131 (1ULL << 56) | // RESOURCE_LEVEL = 1
10132 (3ULL << 60); // OOB_SELECT = 3
10133 }
10134
10135 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
10136 if (ST.isAmdHsaOS()) {
10137 // Set ATC = 1. GFX9 doesn't have this bit.
10138 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10139 RsrcDataFormat |= (1ULL << 56);
10140
10141 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
10142 // BTW, it disables TC L2 and therefore decreases performance.
10143 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)
10144 RsrcDataFormat |= (2ULL << 59);
10145 }
10146
10147 return RsrcDataFormat;
10148}
10149
10150uint64_t SIInstrInfo::getScratchRsrcWords23() const {
10151 uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
10152 AMDGPU::RSRC_TID_ENABLE |
10153 0xffffffff; // Size;
10154
10155 // GFX9 doesn't have ELEMENT_SIZE.
10156 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
10157 uint64_t EltSizeValue = Log2_32(Value: ST.getMaxPrivateElementSize(ForBufferRSrc: true)) - 1;
10158 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
10159 }
10160
10161 // IndexStride = 64 / 32.
10162 uint64_t IndexStride = ST.isWave64() ? 3 : 2;
10163 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
10164
10165 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
10166 // Clear them unless we want a huge stride.
10167 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
10168 ST.getGeneration() <= AMDGPUSubtarget::GFX9)
10169 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
10170
10171 return Rsrc23;
10172}
10173
10174bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const {
10175 unsigned Opc = MI.getOpcode();
10176
10177 return isSMRD(Opcode: Opc);
10178}
10179
10180bool SIInstrInfo::isHighLatencyDef(int Opc) const {
10181 return get(Opcode: Opc).mayLoad() &&
10182 (isMUBUF(Opcode: Opc) || isMTBUF(Opcode: Opc) || isMIMG(Opcode: Opc) || isFLAT(Opcode: Opc));
10183}
10184
10185Register SIInstrInfo::isStackAccess(const MachineInstr &MI, int &FrameIndex,
10186 TypeSize &MemBytes) const {
10187 const MachineOperand *Addr = getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
10188 if (!Addr || !Addr->isFI())
10189 return Register();
10190
10191 assert(!MI.memoperands_empty() &&
10192 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS);
10193
10194 FrameIndex = Addr->getIndex();
10195
10196 int VDataIdx =
10197 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdata);
10198 MemBytes = TypeSize::getFixed(ExactSize: getOpSize(Opcode: MI.getOpcode(), OpNo: VDataIdx));
10199 return MI.getOperand(i: VDataIdx).getReg();
10200}
10201
10202Register SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex,
10203 TypeSize &MemBytes) const {
10204 const MachineOperand *Addr = getNamedOperand(MI, OperandName: AMDGPU::OpName::addr);
10205 assert(Addr && Addr->isFI());
10206 FrameIndex = Addr->getIndex();
10207
10208 int DataIdx =
10209 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::data);
10210 MemBytes = TypeSize::getFixed(ExactSize: getOpSize(Opcode: MI.getOpcode(), OpNo: DataIdx));
10211 return MI.getOperand(i: DataIdx).getReg();
10212}
10213
10214Register SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
10215 int &FrameIndex,
10216 TypeSize &MemBytes) const {
10217 if (!MI.mayLoad())
10218 return Register();
10219
10220 if (isMUBUF(MI) || isVGPRSpill(MI))
10221 return isStackAccess(MI, FrameIndex, MemBytes);
10222
10223 if (isSGPRSpill(MI))
10224 return isSGPRStackAccess(MI, FrameIndex, MemBytes);
10225
10226 return Register();
10227}
10228
10229Register SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
10230 int &FrameIndex,
10231 TypeSize &MemBytes) const {
10232 if (!MI.mayStore())
10233 return Register();
10234
10235 if (isMUBUF(MI) || isVGPRSpill(MI))
10236 return isStackAccess(MI, FrameIndex, MemBytes);
10237
10238 if (isSGPRSpill(MI))
10239 return isSGPRStackAccess(MI, FrameIndex, MemBytes);
10240
10241 return Register();
10242}
10243
10244unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
10245 unsigned Opc = MI.getOpcode();
10246 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: Opc);
10247 unsigned DescSize = Desc.getSize();
10248
10249 // If we have a definitive size, we can use it. Otherwise we need to inspect
10250 // the operands to know the size.
10251 if (isFixedSize(MI)) {
10252 unsigned Size = DescSize;
10253
10254 // If we hit the buggy offset, an extra nop will be inserted in MC so
10255 // estimate the worst case.
10256 if (MI.isBranch() && ST.hasOffset3fBug())
10257 Size += 4;
10258
10259 return Size;
10260 }
10261
10262 // Instructions may have a 32-bit literal encoded after them. Check
10263 // operands that could ever be literals.
10264 if (isVALU(MI, /*AllowLDSDMA=*/false) || isSALU(MI)) {
10265 if (isDPP(MI))
10266 return DescSize;
10267 bool HasLiteral = false;
10268 unsigned LiteralSize = 4;
10269 for (int I = 0, E = MI.getNumExplicitOperands(); I != E; ++I) {
10270 const MachineOperand &Op = MI.getOperand(i: I);
10271 const MCOperandInfo &OpInfo = Desc.operands()[I];
10272 if (!Op.isReg() && !isInlineConstant(MO: Op, OpInfo)) {
10273 HasLiteral = true;
10274 if (ST.has64BitLiterals()) {
10275 switch (OpInfo.OperandType) {
10276 default:
10277 break;
10278 case AMDGPU::OPERAND_REG_IMM_FP64:
10279 case AMDGPU::OPERAND_REG_IMM_V2FP64:
10280 if (!AMDGPU::isValid32BitLiteral(Val: Op.getImm(), IsFP64: true))
10281 LiteralSize = 8;
10282 break;
10283 case AMDGPU::OPERAND_REG_IMM_INT64:
10284 case AMDGPU::OPERAND_REG_IMM_V2INT64:
10285 // A 32-bit literal is only valid when the value fits in BOTH signed
10286 // and unsigned 32-bit ranges [0, 2^31-1], matching the MC code
10287 // emitter's getLit64Encoding logic. This is because of the lack of
10288 // abilility to tell signedness of the literal, therefore we need to
10289 // be conservative and assume values outside this range require a
10290 // 64-bit literal encoding (8 bytes).
10291 if (!Op.isImm() || !isInt<32>(x: Op.getImm()) ||
10292 !isUInt<32>(x: Op.getImm()))
10293 LiteralSize = 8;
10294 break;
10295 }
10296 }
10297 break;
10298 }
10299 }
10300 return HasLiteral ? DescSize + LiteralSize : DescSize;
10301 }
10302
10303 // Check whether we have extra NSA words.
10304 if (isMIMG(MI)) {
10305 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0);
10306 if (VAddr0Idx < 0)
10307 return 8;
10308
10309 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::srsrc);
10310 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4);
10311 }
10312
10313 switch (Opc) {
10314 case TargetOpcode::BUNDLE:
10315 return getInstBundleSize(MI);
10316 case TargetOpcode::INLINEASM:
10317 case TargetOpcode::INLINEASM_BR: {
10318 const MachineFunction *MF = MI.getMF();
10319 const char *AsmStr = MI.getOperand(i: 0).getSymbolName();
10320 return getInlineAsmLength(Str: AsmStr, MAI: MF->getTarget().getMCAsmInfo(), STI: &ST);
10321 }
10322 default:
10323 if (MI.isMetaInstruction())
10324 return 0;
10325
10326 // If D16 Pseudo inst, get correct MC code size
10327 const auto *D16Info = AMDGPU::getT16D16Helper(T16Op: Opc);
10328 if (D16Info) {
10329 // Assume d16_lo/hi inst are always in same size
10330 unsigned LoInstOpcode = D16Info->LoOp;
10331 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: LoInstOpcode);
10332 DescSize = Desc.getSize();
10333 }
10334
10335 // If FMA Pseudo inst, get correct MC code size
10336 if (Opc == AMDGPU::V_FMA_MIX_F16_t16 || Opc == AMDGPU::V_FMA_MIX_BF16_t16) {
10337 // All potential lowerings are the same size; arbitrarily pick one.
10338 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: AMDGPU::V_FMA_MIXLO_F16);
10339 DescSize = Desc.getSize();
10340 }
10341
10342 return DescSize;
10343 }
10344}
10345
10346TargetInstrInfo::InstSizeVerifyMode
10347SIInstrInfo::getInstSizeVerifyMode(const MachineInstr &MI) const {
10348 if (MI.isBranch() && ST.hasOffset3fBug())
10349 return InstSizeVerifyMode::NoVerify;
10350 return InstSizeVerifyMode::ExactSize;
10351}
10352
10353bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const {
10354 if (!isFLAT(MI))
10355 return false;
10356
10357 if (MI.memoperands_empty())
10358 return true;
10359
10360 for (const MachineMemOperand *MMO : MI.memoperands()) {
10361 if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)
10362 return true;
10363 }
10364 return false;
10365}
10366
10367ArrayRef<std::pair<int, const char *>>
10368SIInstrInfo::getSerializableTargetIndices() const {
10369 static const std::pair<int, const char *> TargetIndices[] = {
10370 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
10371 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
10372 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
10373 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
10374 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
10375 return ArrayRef(TargetIndices);
10376}
10377
10378/// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The
10379/// post-RA version of misched uses CreateTargetMIHazardRecognizer.
10380ScheduleHazardRecognizer *
10381SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
10382 const ScheduleDAG *DAG) const {
10383 return new GCNHazardRecognizer(DAG->MF);
10384}
10385
10386/// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer
10387/// pass.
10388ScheduleHazardRecognizer *
10389SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF,
10390 MachineLoopInfo *MLI) const {
10391 return new GCNHazardRecognizer(
10392 MF, GCNHazardRecognizer::OperatingMode::HazardRecognizerMode, MLI);
10393}
10394
10395// Called during:
10396// - pre-RA scheduling and post-RA scheduling
10397ScheduleHazardRecognizer *
10398SIInstrInfo::CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
10399 const ScheduleDAGMI *DAG) const {
10400 // Borrowed from Arm Target
10401 // We would like to restrict this hazard recognizer to only
10402 // post-RA scheduling; we can tell that we're post-RA because we don't
10403 // track VRegLiveness.
10404 if (!DAG->hasVRegLiveness())
10405 return new GCNHazardRecognizer(DAG->MF);
10406 return TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG);
10407}
10408
10409std::pair<unsigned, unsigned>
10410SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
10411 return std::pair(TF & MO_MASK, TF & ~MO_MASK);
10412}
10413
10414ArrayRef<std::pair<unsigned, const char *>>
10415SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
10416 static const std::pair<unsigned, const char *> TargetFlags[] = {
10417 {MO_GOTPCREL, "amdgpu-gotprel"},
10418 {MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo"},
10419 {MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi"},
10420 {MO_GOTPCREL64, "amdgpu-gotprel64"},
10421 {MO_REL32_LO, "amdgpu-rel32-lo"},
10422 {MO_REL32_HI, "amdgpu-rel32-hi"},
10423 {MO_REL64, "amdgpu-rel64"},
10424 {MO_ABS32_LO, "amdgpu-abs32-lo"},
10425 {MO_ABS32_HI, "amdgpu-abs32-hi"},
10426 {MO_ABS64, "amdgpu-abs64"},
10427 };
10428
10429 return ArrayRef(TargetFlags);
10430}
10431
10432ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
10433SIInstrInfo::getSerializableMachineMemOperandTargetFlags() const {
10434 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
10435 {
10436 {MONoClobber, "amdgpu-noclobber"},
10437 {MOLastUse, "amdgpu-last-use"},
10438 {MOCooperative, "amdgpu-cooperative"},
10439 {MOThreadPrivate, "amdgpu-thread-private"},
10440 };
10441
10442 return ArrayRef(TargetFlags);
10443}
10444
10445unsigned SIInstrInfo::getLiveRangeSplitOpcode(Register SrcReg,
10446 const MachineFunction &MF) const {
10447 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
10448 assert(SrcReg.isVirtual());
10449 if (MFI->checkFlag(Reg: SrcReg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
10450 return AMDGPU::WWM_COPY;
10451
10452 return AMDGPU::COPY;
10453}
10454
10455bool SIInstrInfo::canAddToBBProlog(const MachineInstr &MI) const {
10456 uint32_t Opcode = MI.getOpcode();
10457 // Check if it is SGPR spill or wwm-register spill Opcode.
10458 if (isSGPRSpill(Opcode) || isWWMRegSpillOpcode(Opcode))
10459 return true;
10460
10461 const MachineFunction *MF = MI.getMF();
10462 const MachineRegisterInfo &MRI = MF->getRegInfo();
10463 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
10464
10465 // See if this is Liverange split instruction inserted for SGPR or
10466 // wwm-register. The implicit def inserted for wwm-registers should also be
10467 // included as they can appear at the bb begin.
10468 bool IsLRSplitInst = MI.getFlag(Flag: MachineInstr::LRSplit);
10469 if (!IsLRSplitInst && Opcode != AMDGPU::IMPLICIT_DEF)
10470 return false;
10471
10472 Register Reg = MI.getOperand(i: 0).getReg();
10473 if (RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg)))
10474 return IsLRSplitInst;
10475
10476 return MFI->isWWMReg(Reg);
10477}
10478
10479bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI,
10480 Register Reg) const {
10481 // We need to handle instructions which may be inserted during register
10482 // allocation to handle the prolog. The initial prolog instruction may have
10483 // been separated from the start of the block by spills and copies inserted
10484 // needed by the prolog. However, the insertions for scalar registers can
10485 // always be placed at the BB top as they are independent of the exec mask
10486 // value.
10487 bool IsNullOrVectorRegister = true;
10488 if (Reg) {
10489 const MachineFunction *MF = MI.getMF();
10490 const MachineRegisterInfo &MRI = MF->getRegInfo();
10491 IsNullOrVectorRegister = !RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg));
10492 }
10493
10494 return IsNullOrVectorRegister &&
10495 (canAddToBBProlog(MI) ||
10496 (!MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY &&
10497 MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &RI)));
10498}
10499
10500MachineInstrBuilder
10501SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
10502 MachineBasicBlock::iterator I,
10503 const DebugLoc &DL,
10504 Register DestReg) const {
10505 if (ST.hasAddNoCarryInsts())
10506 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e64), DestReg);
10507
10508 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
10509 Register UnusedCarry = MRI.createVirtualRegister(RegClass: RI.getBoolRC());
10510 MRI.setRegAllocationHint(VReg: UnusedCarry, Type: 0, PrefReg: RI.getVCC());
10511
10512 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg)
10513 .addReg(RegNo: UnusedCarry, Flags: RegState::Define | RegState::Dead);
10514}
10515
10516MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
10517 MachineBasicBlock::iterator I,
10518 const DebugLoc &DL,
10519 Register DestReg,
10520 RegScavenger &RS) const {
10521 if (ST.hasAddNoCarryInsts())
10522 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg);
10523
10524 // If available, prefer to use vcc.
10525 Register UnusedCarry = !RS.isRegUsed(Reg: AMDGPU::VCC)
10526 ? Register(RI.getVCC())
10527 : RS.scavengeRegisterBackwards(
10528 RC: *RI.getBoolRC(), To: I, /* RestoreAfter */ false,
10529 SPAdj: 0, /* AllowSpill */ false);
10530
10531 // TODO: Users need to deal with this.
10532 if (!UnusedCarry.isValid())
10533 return MachineInstrBuilder();
10534
10535 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg)
10536 .addReg(RegNo: UnusedCarry, Flags: RegState::Define | RegState::Dead);
10537}
10538
10539bool SIInstrInfo::isKillTerminator(unsigned Opcode) {
10540 switch (Opcode) {
10541 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
10542 case AMDGPU::SI_KILL_I1_TERMINATOR:
10543 return true;
10544 default:
10545 return false;
10546 }
10547}
10548
10549const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const {
10550 switch (Opcode) {
10551 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
10552 return get(Opcode: AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);
10553 case AMDGPU::SI_KILL_I1_PSEUDO:
10554 return get(Opcode: AMDGPU::SI_KILL_I1_TERMINATOR);
10555 default:
10556 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO");
10557 }
10558}
10559
10560bool SIInstrInfo::isLegalMUBUFImmOffset(unsigned Imm) const {
10561 return Imm <= getMaxMUBUFImmOffset(ST);
10562}
10563
10564unsigned SIInstrInfo::getMaxMUBUFImmOffset(const GCNSubtarget &ST) {
10565 // GFX12 field is non-negative 24-bit signed byte offset.
10566 const unsigned OffsetBits =
10567 ST.getGeneration() >= AMDGPUSubtarget::GFX12 ? 23 : 12;
10568 return (1 << OffsetBits) - 1;
10569}
10570
10571void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const {
10572 if (!ST.isWave32())
10573 return;
10574
10575 if (MI.isInlineAsm())
10576 return;
10577
10578 if (MI.getNumOperands() < MI.getDesc().getNumOperands())
10579 return;
10580
10581 for (auto &Op : MI.implicit_operands()) {
10582 if (Op.isReg() && Op.getReg() == AMDGPU::VCC)
10583 Op.setReg(AMDGPU::VCC_LO);
10584 }
10585}
10586
10587bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const {
10588 if (!isSMRD(MI))
10589 return false;
10590
10591 // Check that it is using a buffer resource.
10592 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::sbase);
10593 if (Idx == -1) // e.g. s_memtime
10594 return false;
10595
10596 const int16_t RCID = getOpRegClassID(OpInfo: MI.getDesc().operands()[Idx]);
10597 return RI.getRegClass(i: RCID)->hasSubClassEq(RC: &AMDGPU::SGPR_128RegClass);
10598}
10599
10600// Given Imm, split it into the values to put into the SOffset and ImmOffset
10601// fields in an MUBUF instruction. Return false if it is not possible (due to a
10602// hardware bug needing a workaround).
10603//
10604// The required alignment ensures that individual address components remain
10605// aligned if they are aligned to begin with. It also ensures that additional
10606// offsets within the given alignment can be added to the resulting ImmOffset.
10607bool SIInstrInfo::splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset,
10608 uint32_t &ImmOffset, Align Alignment) const {
10609 const uint64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST);
10610 const uint32_t MaxImm = alignDown(Value: MaxOffset, Align: Alignment.value());
10611 uint32_t Overflow = 0;
10612
10613 if (Imm > MaxImm) {
10614 if (Imm <= MaxImm + 64) {
10615 // Use an SOffset inline constant for 4..64
10616 Overflow = Imm - MaxImm;
10617 Imm = MaxImm;
10618 } else {
10619 // Try to keep the same value in SOffset for adjacent loads, so that
10620 // the corresponding register contents can be re-used.
10621 //
10622 // Load values with all low-bits (except for alignment bits) set into
10623 // SOffset, so that a larger range of values can be covered using
10624 // s_movk_i32.
10625 //
10626 // Atomic operations fail to work correctly when individual address
10627 // components are unaligned, even if their sum is aligned.
10628 uint32_t High = (Imm + Alignment.value()) & ~MaxOffset;
10629 uint32_t Low = (Imm + Alignment.value()) & MaxOffset;
10630 Imm = Low;
10631 Overflow = High - Alignment.value();
10632 }
10633 }
10634
10635 if (Overflow > 0) {
10636 // There is a hardware bug in SI and CI which prevents address clamping in
10637 // MUBUF instructions from working correctly with SOffsets. The immediate
10638 // offset is unaffected.
10639 if (ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
10640 return false;
10641
10642 // It is not possible to set immediate in SOffset field on some targets.
10643 if (ST.hasRestrictedSOffset())
10644 return false;
10645 }
10646
10647 ImmOffset = Imm;
10648 SOffset = Overflow;
10649 return true;
10650}
10651
10652// Depending on the used address space and instructions, some immediate offsets
10653// are allowed and some are not.
10654// Pre-GFX12, flat instruction offsets can only be non-negative, global and
10655// scratch instruction offsets can also be negative. On GFX12, offsets can be
10656// negative for all variants.
10657//
10658// There are several bugs related to these offsets:
10659// On gfx10.1, flat instructions that go into the global address space cannot
10660// use an offset.
10661//
10662// For scratch instructions, the address can be either an SGPR or a VGPR.
10663// The following offsets can be used, depending on the architecture (x means
10664// cannot be used):
10665// +----------------------------+------+------+
10666// | Address-Mode | SGPR | VGPR |
10667// +----------------------------+------+------+
10668// | gfx9 | | |
10669// | negative, 4-aligned offset | x | ok |
10670// | negative, unaligned offset | x | ok |
10671// +----------------------------+------+------+
10672// | gfx10 | | |
10673// | negative, 4-aligned offset | ok | ok |
10674// | negative, unaligned offset | ok | x |
10675// +----------------------------+------+------+
10676// | gfx10.3 | | |
10677// | negative, 4-aligned offset | ok | ok |
10678// | negative, unaligned offset | ok | ok |
10679// +----------------------------+------+------+
10680//
10681// This function ignores the addressing mode, so if an offset cannot be used in
10682// one addressing mode, it is considered illegal.
10683bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
10684 AMDGPU::FlatAddrSpace FlatVariant) const {
10685 // TODO: Should 0 be special cased?
10686 if (!ST.hasFlatInstOffsets())
10687 return false;
10688
10689 using AMDGPU::FlatAddrSpace;
10690 if (ST.hasFlatSegmentOffsetBug() && FlatVariant == FlatAddrSpace::FLAT &&
10691 (AddrSpace == AMDGPUAS::FLAT_ADDRESS ||
10692 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS))
10693 return false;
10694
10695 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
10696 FlatVariant == FlatAddrSpace::FlatScratch && Offset < 0 &&
10697 (Offset % 4) != 0) {
10698 return false;
10699 }
10700
10701 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
10702 unsigned N = AMDGPU::getNumFlatOffsetBits(ST);
10703 return isIntN(N, x: Offset) && (AllowNegative || Offset >= 0);
10704}
10705
10706// See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not.
10707std::pair<int64_t, int64_t>
10708SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,
10709 AMDGPU::FlatAddrSpace FlatVariant) const {
10710 int64_t RemainderOffset = COffsetVal;
10711 int64_t ImmField = 0;
10712
10713 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
10714 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST) - 1;
10715
10716 if (AllowNegative) {
10717 // Use signed division by a power of two to truncate towards 0.
10718 int64_t D = 1LL << NumBits;
10719 RemainderOffset = (COffsetVal / D) * D;
10720 ImmField = COffsetVal - RemainderOffset;
10721
10722 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
10723 FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch && ImmField < 0 &&
10724 (ImmField % 4) != 0) {
10725 // Make ImmField a multiple of 4
10726 RemainderOffset += ImmField % 4;
10727 ImmField -= ImmField % 4;
10728 }
10729 } else if (COffsetVal >= 0) {
10730 ImmField = COffsetVal & maskTrailingOnes<uint64_t>(N: NumBits);
10731 RemainderOffset = COffsetVal - ImmField;
10732 }
10733
10734 assert(isLegalFLATOffset(ImmField, AddrSpace, FlatVariant));
10735 assert(RemainderOffset + ImmField == COffsetVal);
10736 return {ImmField, RemainderOffset};
10737}
10738
10739bool SIInstrInfo::allowNegativeFlatOffset(
10740 AMDGPU::FlatAddrSpace FlatVariant) const {
10741 if (ST.hasNegativeScratchOffsetBug() &&
10742 FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch)
10743 return false;
10744
10745 return FlatVariant != AMDGPU::FlatAddrSpace::FLAT || AMDGPU::isGFX12Plus(STI: ST);
10746}
10747
10748static unsigned subtargetEncodingFamily(const GCNSubtarget &ST) {
10749 switch (ST.getGeneration()) {
10750 default:
10751 break;
10752 case AMDGPUSubtarget::SOUTHERN_ISLANDS:
10753 case AMDGPUSubtarget::SEA_ISLANDS:
10754 return SIEncodingFamily::SI;
10755 case AMDGPUSubtarget::VOLCANIC_ISLANDS:
10756 // The GFX80 encoding family only contains buffer instructions with unpacked
10757 // D16 data; pseudoToMCOpcode falls back on VI for everything else.
10758 // TODO: remove this when we discard GFX80 encoding.
10759 return ST.hasUnpackedD16VMem() ? SIEncodingFamily::GFX80
10760 : SIEncodingFamily::VI;
10761 case AMDGPUSubtarget::GFX9:
10762 return SIEncodingFamily::VI;
10763 case AMDGPUSubtarget::GFX10:
10764 return SIEncodingFamily::GFX10;
10765 case AMDGPUSubtarget::GFX11:
10766 return ST.hasGFX11_7Insts() ? SIEncodingFamily::GFX1170
10767 : SIEncodingFamily::GFX11;
10768 case AMDGPUSubtarget::GFX12:
10769 return ST.hasGFX1250Insts() ? SIEncodingFamily::GFX1250
10770 : SIEncodingFamily::GFX12;
10771 case AMDGPUSubtarget::GFX13:
10772 return SIEncodingFamily::GFX13;
10773 }
10774 llvm_unreachable("Unknown subtarget generation!");
10775}
10776
10777bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const {
10778 switch(MCOp) {
10779 // These opcodes use indirect register addressing so
10780 // they need special handling by codegen (currently missing).
10781 // Therefore it is too risky to allow these opcodes
10782 // to be selected by dpp combiner or sdwa peepholer.
10783 case AMDGPU::V_MOVRELS_B32_dpp_gfx10:
10784 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
10785 case AMDGPU::V_MOVRELD_B32_dpp_gfx10:
10786 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10:
10787 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10:
10788 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
10789 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10:
10790 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
10791 return true;
10792 default:
10793 return false;
10794 }
10795}
10796
10797#define GENERATE_RENAMED_GFX9_CASES(OPCODE) \
10798 case OPCODE##_dpp: \
10799 case OPCODE##_e32: \
10800 case OPCODE##_e64: \
10801 case OPCODE##_e64_dpp: \
10802 case OPCODE##_sdwa:
10803
10804static bool isRenamedInGFX9(int Opcode) {
10805 switch (Opcode) {
10806 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADDC_U32)
10807 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_CO_U32)
10808 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_U32)
10809 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBBREV_U32)
10810 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBB_U32)
10811 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_CO_U32)
10812 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_U32)
10813 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_CO_U32)
10814 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_U32)
10815 //
10816 case AMDGPU::V_DIV_FIXUP_F16_gfx9_e64:
10817 case AMDGPU::V_DIV_FIXUP_F16_gfx9_fake16_e64:
10818 case AMDGPU::V_FMA_F16_gfx9_e64:
10819 case AMDGPU::V_FMA_F16_gfx9_fake16_e64:
10820 case AMDGPU::V_INTERP_P2_F16:
10821 case AMDGPU::V_MAD_F16_e64:
10822 case AMDGPU::V_MAD_U16_e64:
10823 case AMDGPU::V_MAD_I16_e64:
10824 return true;
10825 default:
10826 return false;
10827 }
10828}
10829
10830int SIInstrInfo::pseudoToMCOpcode(int Opcode) const {
10831 assert(Opcode == (int)SIInstrInfo::getNonSoftWaitcntOpcode(Opcode) &&
10832 "SIInsertWaitcnts should have promoted soft waitcnt instructions!");
10833
10834 unsigned Gen = subtargetEncodingFamily(ST);
10835
10836 if (ST.getGeneration() == AMDGPUSubtarget::GFX9 && isRenamedInGFX9(Opcode))
10837 Gen = SIEncodingFamily::GFX9;
10838
10839 if (SIInstrFlags::isSDWA(O: get(Opcode))) {
10840 switch (ST.getGeneration()) {
10841 default:
10842 Gen = SIEncodingFamily::SDWA;
10843 break;
10844 case AMDGPUSubtarget::GFX9:
10845 Gen = SIEncodingFamily::SDWA9;
10846 break;
10847 case AMDGPUSubtarget::GFX10:
10848 Gen = SIEncodingFamily::SDWA10;
10849 break;
10850 }
10851 }
10852
10853 if (isMAI(Opcode)) {
10854 int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode);
10855 if (MFMAOp != -1)
10856 Opcode = MFMAOp;
10857 }
10858
10859 int32_t MCOp = AMDGPU::getMCOpcode(Opcode, Gen);
10860
10861 // Only buffer instructions with unpacked D16 data have a GFX80 encoding.
10862 // Anything else on such a subtarget uses the plain VI encoding.
10863 // TODO: remove this when we discard GFX80 encoding.
10864 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && Gen == SIEncodingFamily::GFX80)
10865 MCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::VI);
10866
10867 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && ST.hasGFX11_7Insts())
10868 MCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX11);
10869
10870 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && ST.hasGFX1250Insts())
10871 MCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX12);
10872
10873 // -1 means that Opcode is already a native instruction.
10874 if (MCOp == -1)
10875 return Opcode;
10876
10877 if (ST.hasGFX90AInsts()) {
10878 uint32_t NMCOp = AMDGPU::INSTRUCTION_LIST_END;
10879 if (ST.hasGFX940Insts())
10880 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX940);
10881 if (NMCOp == AMDGPU::INSTRUCTION_LIST_END)
10882 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX90A);
10883 if (NMCOp == AMDGPU::INSTRUCTION_LIST_END)
10884 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX9);
10885 if (NMCOp != AMDGPU::INSTRUCTION_LIST_END)
10886 MCOp = NMCOp;
10887 }
10888
10889 // INSTRUCTION_LIST_END means that Opcode is a pseudo instruction that has no
10890 // encoding in the given subtarget generation.
10891 if (MCOp == AMDGPU::INSTRUCTION_LIST_END)
10892 return -1;
10893
10894 if (isAsmOnlyOpcode(MCOp))
10895 return -1;
10896
10897 return MCOp;
10898}
10899
10900static
10901TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) {
10902 assert(RegOpnd.isReg());
10903 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() :
10904 getRegSubRegPair(O: RegOpnd);
10905}
10906
10907TargetInstrInfo::RegSubRegPair
10908llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) {
10909 assert(MI.isRegSequence());
10910 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I)
10911 if (MI.getOperand(i: 1 + 2 * I + 1).getImm() == SubReg) {
10912 auto &RegOp = MI.getOperand(i: 1 + 2 * I);
10913 return getRegOrUndef(RegOpnd: RegOp);
10914 }
10915 return TargetInstrInfo::RegSubRegPair();
10916}
10917
10918// Try to find the definition of reg:subreg in subreg-manipulation pseudos
10919// Following a subreg of reg:subreg isn't supported
10920static bool followSubRegDef(MachineInstr &MI,
10921 TargetInstrInfo::RegSubRegPair &RSR) {
10922 if (!RSR.SubReg)
10923 return false;
10924 switch (MI.getOpcode()) {
10925 default: break;
10926 case AMDGPU::REG_SEQUENCE:
10927 RSR = getRegSequenceSubReg(MI, SubReg: RSR.SubReg);
10928 return true;
10929 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg
10930 case AMDGPU::INSERT_SUBREG:
10931 if (RSR.SubReg == (unsigned)MI.getOperand(i: 3).getImm())
10932 // inserted the subreg we're looking for
10933 RSR = getRegOrUndef(RegOpnd: MI.getOperand(i: 2));
10934 else { // the subreg in the rest of the reg
10935 auto R1 = getRegOrUndef(RegOpnd: MI.getOperand(i: 1));
10936 if (R1.SubReg) // subreg of subreg isn't supported
10937 return false;
10938 RSR.Reg = R1.Reg;
10939 }
10940 return true;
10941 }
10942 return false;
10943}
10944
10945MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
10946 const MachineRegisterInfo &MRI) {
10947 assert(MRI.isSSA());
10948 if (!P.Reg.isVirtual())
10949 return nullptr;
10950
10951 auto RSR = P;
10952 auto *DefInst = MRI.getVRegDef(Reg: RSR.Reg);
10953 while (auto *MI = DefInst) {
10954 DefInst = nullptr;
10955 switch (MI->getOpcode()) {
10956 case AMDGPU::COPY:
10957 case AMDGPU::V_MOV_B32_e32: {
10958 auto &Op1 = MI->getOperand(i: 1);
10959 if (Op1.isReg() && Op1.getReg().isVirtual()) {
10960 if (Op1.isUndef())
10961 return nullptr;
10962 RSR = getRegSubRegPair(O: Op1);
10963 DefInst = MRI.getVRegDef(Reg: RSR.Reg);
10964 }
10965 break;
10966 }
10967 default:
10968 if (followSubRegDef(MI&: *MI, RSR)) {
10969 if (!RSR.Reg)
10970 return nullptr;
10971 DefInst = MRI.getVRegDef(Reg: RSR.Reg);
10972 }
10973 }
10974 if (!DefInst)
10975 return MI;
10976 }
10977 return nullptr;
10978}
10979
10980bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
10981 Register VReg,
10982 const MachineInstr &DefMI,
10983 const MachineInstr &UseMI) {
10984 assert(MRI.isSSA() && "Must be run on SSA");
10985
10986 auto *TRI = MRI.getTargetRegisterInfo();
10987 auto *DefBB = DefMI.getParent();
10988
10989 // Don't bother searching between blocks, although it is possible this block
10990 // doesn't modify exec.
10991 if (UseMI.getParent() != DefBB)
10992 return true;
10993
10994 const int MaxInstScan = 20;
10995 int NumInst = 0;
10996
10997 // Stop scan at the use.
10998 auto E = UseMI.getIterator();
10999 for (auto I = std::next(x: DefMI.getIterator()); I != E; ++I) {
11000 if (I->isDebugInstr())
11001 continue;
11002
11003 if (++NumInst > MaxInstScan)
11004 return true;
11005
11006 if (I->modifiesRegister(Reg: AMDGPU::EXEC, TRI))
11007 return true;
11008 }
11009
11010 return false;
11011}
11012
11013bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
11014 Register VReg,
11015 const MachineInstr &DefMI) {
11016 assert(MRI.isSSA() && "Must be run on SSA");
11017
11018 auto *TRI = MRI.getTargetRegisterInfo();
11019 auto *DefBB = DefMI.getParent();
11020
11021 const int MaxUseScan = 10;
11022 int NumUse = 0;
11023
11024 for (auto &Use : MRI.use_nodbg_operands(Reg: VReg)) {
11025 auto &UseInst = *Use.getParent();
11026 // Don't bother searching between blocks, although it is possible this block
11027 // doesn't modify exec.
11028 if (UseInst.getParent() != DefBB || UseInst.isPHI())
11029 return true;
11030
11031 if (++NumUse > MaxUseScan)
11032 return true;
11033 }
11034
11035 if (NumUse == 0)
11036 return false;
11037
11038 const int MaxInstScan = 20;
11039 int NumInst = 0;
11040
11041 // Stop scan when we have seen all the uses.
11042 for (auto I = std::next(x: DefMI.getIterator()); ; ++I) {
11043 assert(I != DefBB->end());
11044
11045 if (I->isDebugInstr())
11046 continue;
11047
11048 if (++NumInst > MaxInstScan)
11049 return true;
11050
11051 for (const MachineOperand &Op : I->operands()) {
11052 // We don't check reg masks here as they're used only on calls:
11053 // 1. EXEC is only considered const within one BB
11054 // 2. Call should be a terminator instruction if present in a BB
11055
11056 if (!Op.isReg())
11057 continue;
11058
11059 Register Reg = Op.getReg();
11060 if (Op.isUse()) {
11061 if (Reg == VReg && --NumUse == 0)
11062 return false;
11063 } else if (TRI->regsOverlap(RegA: Reg, RegB: AMDGPU::EXEC))
11064 return true;
11065 }
11066 }
11067}
11068
11069MachineInstr *SIInstrInfo::createPHIDestinationCopy(
11070 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt,
11071 const DebugLoc &DL, Register Src, Register Dst) const {
11072 auto Cur = MBB.begin();
11073 if (Cur != MBB.end())
11074 do {
11075 if (!Cur->isPHI() && Cur->readsRegister(Reg: Dst, /*TRI=*/nullptr))
11076 return BuildMI(BB&: MBB, I: Cur, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Dst).addReg(RegNo: Src);
11077 ++Cur;
11078 } while (Cur != MBB.end() && Cur != LastPHIIt);
11079
11080 return TargetInstrInfo::createPHIDestinationCopy(MBB, InsPt: LastPHIIt, DL, Src,
11081 Dst);
11082}
11083
11084MachineInstr *SIInstrInfo::createPHISourceCopy(
11085 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt,
11086 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const {
11087 if (InsPt != MBB.end() &&
11088 (InsPt->getOpcode() == AMDGPU::SI_IF ||
11089 InsPt->getOpcode() == AMDGPU::SI_ELSE ||
11090 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) &&
11091 InsPt->definesRegister(Reg: Src, /*TRI=*/nullptr)) {
11092 InsPt++;
11093 return BuildMI(BB&: MBB, I: InsPt, MIMD: DL,
11094 MCID: get(Opcode: AMDGPU::LaneMaskConstants::get(ST).MovTermOpc), DestReg: Dst)
11095 .addReg(RegNo: Src, Flags: {}, SubReg: SrcSubReg)
11096 .addReg(RegNo: AMDGPU::EXEC, Flags: RegState::Implicit);
11097 }
11098 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg,
11099 Dst);
11100}
11101
11102bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); }
11103
11104bool SIInstrInfo::hasRAWDependency(const MachineInstr &FirstMI,
11105 const MachineInstr &SecondMI) const {
11106 for (const auto &Use : SecondMI.all_uses()) {
11107 if (Use.isReg() && FirstMI.modifiesRegister(Reg: Use.getReg(), TRI: &RI))
11108 return true;
11109 }
11110 return false;
11111}
11112
11113/// If OpX is multicycle, anti-dependencies are not allowed.
11114/// isDPMACCInstruction was not designed for VOPD, but it is fit for the
11115/// purpose.
11116bool llvm::SIInstrInfo::isVOPDAntidependencyAllowed(
11117 const MachineInstr &OpX) const {
11118 return !AMDGPU::isDPMACCInstruction(Opc: OpX.getOpcode());
11119}
11120
11121MachineInstr *
11122SIInstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
11123 ArrayRef<unsigned> Ops, int FrameIndex,
11124 MachineInstr *&CopyMI, LiveIntervals *LIS,
11125 VirtRegMap *VRM) const {
11126 // This is a bit of a hack (copied from AArch64). Consider this instruction:
11127 //
11128 // %0:sreg_32 = COPY $m0
11129 //
11130 // We explicitly chose SReg_32 for the virtual register so such a copy might
11131 // be eliminated by RegisterCoalescer. However, that may not be possible, and
11132 // %0 may even spill. We can't spill $m0 normally (it would require copying to
11133 // a numbered SGPR anyway), and since it is in the SReg_32 register class,
11134 // TargetInstrInfo::foldMemoryOperand() is going to try.
11135 // A similar issue also exists with spilling and reloading $exec registers.
11136 //
11137 // To prevent that, constrain the %0 register class here.
11138 if (isFullCopyInstr(MI)) {
11139 Register DstReg = MI.getOperand(i: 0).getReg();
11140 Register SrcReg = MI.getOperand(i: 1).getReg();
11141 if ((DstReg.isVirtual() || SrcReg.isVirtual()) &&
11142 (DstReg.isVirtual() != SrcReg.isVirtual())) {
11143 MachineRegisterInfo &MRI = MF.getRegInfo();
11144 Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg;
11145 const TargetRegisterClass *RC = MRI.getRegClass(Reg: VirtReg);
11146 if (RC->hasSuperClassEq(RC: &AMDGPU::SReg_32RegClass)) {
11147 MRI.constrainRegClass(Reg: VirtReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
11148 return nullptr;
11149 }
11150 if (RC->hasSuperClassEq(RC: &AMDGPU::SReg_64RegClass)) {
11151 MRI.constrainRegClass(Reg: VirtReg, RC: &AMDGPU::SReg_64_XEXECRegClass);
11152 return nullptr;
11153 }
11154 }
11155 }
11156
11157 return nullptr;
11158}
11159
11160unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
11161 const MachineInstr &MI,
11162 unsigned *PredCost) const {
11163 if (MI.isBundle()) {
11164 MachineBasicBlock::const_instr_iterator I(MI.getIterator());
11165 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end());
11166 unsigned Lat = 0, Count = 0;
11167 for (++I; I != E && I->isBundledWithPred(); ++I) {
11168 ++Count;
11169 Lat = std::max(a: Lat, b: SchedModel.computeInstrLatency(MI: &*I));
11170 }
11171 return Lat + Count - 1;
11172 }
11173
11174 return SchedModel.computeInstrLatency(MI: &MI);
11175}
11176
11177unsigned SIInstrInfo::getBlockingCycles(const MachineInstr &MI) const {
11178 if (!ST.hasGFX1250VALUBlockingCycles())
11179 return 0;
11180
11181 // Use processor-specific lookup table
11182 if (const auto *Entry = AMDGPU::getGFX1250BlockingCyclesInfo(Opcode: MI.getOpcode()))
11183 return Entry->GFX1250BlockingCycles;
11184
11185 return 0;
11186}
11187
11188const MachineOperand &
11189SIInstrInfo::getCalleeOperand(const MachineInstr &MI) const {
11190 if (const MachineOperand *CallAddrOp =
11191 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0))
11192 return *CallAddrOp;
11193 return TargetInstrInfo::getCalleeOperand(MI);
11194}
11195
11196ValueUniformity
11197SIInstrInfo::getGenericValueUniformity(const MachineInstr &MI) const {
11198 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
11199 unsigned Opcode = MI.getOpcode();
11200
11201 auto HandleAddrSpaceCast = [this, &MRI](const MachineInstr &MI) {
11202 Register Dst = MI.getOperand(i: 0).getReg();
11203 Register Src = MI.getOperand(i: 1).getReg();
11204 LLT DstTy = MRI.getType(Reg: Dst);
11205 LLT SrcTy = MRI.getType(Reg: Src);
11206 unsigned DstAS = DstTy.getAddressSpace();
11207 unsigned SrcAS = SrcTy.getAddressSpace();
11208 return SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
11209 DstAS == AMDGPUAS::FLAT_ADDRESS &&
11210 ST.hasGloballyAddressableScratch()
11211 ? ValueUniformity::NeverUniform
11212 : ValueUniformity::Default;
11213 };
11214
11215 // If the target supports globally addressable scratch, the mapping from
11216 // scratch memory to the flat aperture changes therefore an address space cast
11217 // is no longer uniform.
11218 if (Opcode == TargetOpcode::G_ADDRSPACE_CAST)
11219 return HandleAddrSpaceCast(MI);
11220
11221 if (auto *GI = dyn_cast<GIntrinsic>(Val: &MI)) {
11222 auto IID = GI->getIntrinsicID();
11223 if (AMDGPU::isIntrinsicSourceOfDivergence(IntrID: IID))
11224 return ValueUniformity::NeverUniform;
11225 if (AMDGPU::isIntrinsicAlwaysUniform(IntrID: IID))
11226 return ValueUniformity::AlwaysUniform;
11227
11228 switch (IID) {
11229 case Intrinsic::amdgcn_if:
11230 case Intrinsic::amdgcn_else:
11231 // FIXME: Uniform if second result
11232 break;
11233 }
11234
11235 return ValueUniformity::Default;
11236 }
11237
11238 // Loads from the private and flat address spaces are divergent, because
11239 // threads can execute the load instruction with the same inputs and get
11240 // different results.
11241 //
11242 // All other loads are not divergent, because if threads issue loads with the
11243 // same arguments, they will always get the same result.
11244 if (Opcode == AMDGPU::G_LOAD || Opcode == AMDGPU::G_ZEXTLOAD ||
11245 Opcode == AMDGPU::G_SEXTLOAD) {
11246 if (MI.memoperands_empty())
11247 return ValueUniformity::NeverUniform; // conservative assumption
11248
11249 if (llvm::any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *mmo) {
11250 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
11251 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
11252 })) {
11253 // At least one MMO in a non-global address space.
11254 return ValueUniformity::NeverUniform;
11255 }
11256 return ValueUniformity::Default;
11257 }
11258
11259 if (SIInstrInfo::isGenericAtomicRMWOpcode(Opc: Opcode) ||
11260 Opcode == AMDGPU::G_ATOMIC_CMPXCHG ||
11261 Opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS ||
11262 AMDGPU::isGenericAtomic(Opc: Opcode)) {
11263 return ValueUniformity::NeverUniform;
11264 }
11265
11266 // Result is computed from uniform SP and uniform wave-wide max size.
11267 if (Opcode == TargetOpcode::G_DYN_STACKALLOC)
11268 return ValueUniformity::AlwaysUniform;
11269
11270 if (Opcode == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
11271 return ValueUniformity::NeverUniform;
11272
11273 return ValueUniformity::Default;
11274}
11275
11276const MIRFormatter *SIInstrInfo::getMIRFormatter() const {
11277 if (!Formatter)
11278 Formatter = std::make_unique<AMDGPUMIRFormatter>(args: ST);
11279 return Formatter.get();
11280}
11281
11282ValueUniformity SIInstrInfo::getValueUniformity(const MachineInstr &MI) const {
11283
11284 if (isNeverUniform(MI))
11285 return ValueUniformity::NeverUniform;
11286
11287 unsigned opcode = MI.getOpcode();
11288 if (opcode == AMDGPU::V_READLANE_B32 ||
11289 opcode == AMDGPU::V_READFIRSTLANE_B32 ||
11290 opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR)
11291 return ValueUniformity::AlwaysUniform;
11292
11293 // If any of defs is divergent, report as NeverUniform. isUniformReg will
11294 // calculate in more detail for each def from its reg class, if available.
11295 if (MI.isInlineAsm()) {
11296 for (const MachineOperand &MO : MI.operands()) {
11297 if (!MO.isReg() || !MO.isDef())
11298 continue;
11299 const TargetRegisterClass *RC =
11300 MI.getRegClassConstraint(OpIdx: MO.getOperandNo(), TII: this, TRI: &RI);
11301 if (!RC || !RI.isSGPRClass(RC))
11302 return ValueUniformity::NeverUniform;
11303 }
11304 }
11305
11306 if (isCopyInstr(MI)) {
11307 const MachineOperand &srcOp = MI.getOperand(i: 1);
11308 if (srcOp.isReg() && srcOp.getReg().isPhysical()) {
11309 const TargetRegisterClass *regClass =
11310 RI.getPhysRegBaseClass(Reg: srcOp.getReg());
11311 return RI.isSGPRClass(RC: regClass) ? ValueUniformity::AlwaysUniform
11312 : ValueUniformity::NeverUniform;
11313 }
11314 return ValueUniformity::Default;
11315 }
11316
11317 // GMIR handling
11318 if (MI.isPreISelOpcode())
11319 return SIInstrInfo::getGenericValueUniformity(MI);
11320
11321 // Atomics are divergent because they are executed sequentially: when an
11322 // atomic operation refers to the same address in each thread, then each
11323 // thread after the first sees the value written by the previous thread as
11324 // original value.
11325
11326 if (isAtomic(MI))
11327 return ValueUniformity::NeverUniform;
11328
11329 // Loads from the private and flat address spaces are divergent, because
11330 // threads can execute the load instruction with the same inputs and get
11331 // different results.
11332 if (isFLAT(MI) && MI.mayLoad()) {
11333 if (MI.memoperands_empty())
11334 return ValueUniformity::NeverUniform; // conservative assumption
11335
11336 if (llvm::any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *mmo) {
11337 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
11338 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
11339 })) {
11340 // At least one MMO in a non-global address space.
11341 return ValueUniformity::NeverUniform;
11342 }
11343
11344 return ValueUniformity::Default;
11345 }
11346
11347 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
11348 const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo();
11349
11350 // FIXME: It's conceptually broken to report this for an instruction, and not
11351 // a specific def operand. For inline asm in particular, there could be mixed
11352 // uniform and divergent results.
11353 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
11354 const MachineOperand &SrcOp = MI.getOperand(i: I);
11355 if (!SrcOp.isReg())
11356 continue;
11357
11358 Register Reg = SrcOp.getReg();
11359 if (!Reg || !SrcOp.readsReg())
11360 continue;
11361
11362 // If RegBank is null, this is unassigned or an unallocatable special
11363 // register, which are all scalars.
11364 const RegisterBank *RegBank = RBI->getRegBank(Reg, MRI, TRI: RI);
11365 if (RegBank && RegBank->getID() != AMDGPU::SGPRRegBankID)
11366 return ValueUniformity::NeverUniform;
11367 }
11368
11369 // TODO: Uniformity check condtions above can be rearranged for more
11370 // redability
11371
11372 // TODO: amdgcn.{ballot, [if]cmp} should be AlwaysUniform, but they are
11373 // currently turned into no-op COPYs by SelectionDAG ISel and are
11374 // therefore no longer recognizable.
11375
11376 return ValueUniformity::Default;
11377}
11378
11379unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) {
11380 switch (MF.getFunction().getCallingConv()) {
11381 case CallingConv::AMDGPU_PS:
11382 return 1;
11383 case CallingConv::AMDGPU_VS:
11384 return 2;
11385 case CallingConv::AMDGPU_GS:
11386 return 3;
11387 case CallingConv::AMDGPU_HS:
11388 case CallingConv::AMDGPU_LS:
11389 case CallingConv::AMDGPU_ES: {
11390 const Function &F = MF.getFunction();
11391 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
11392 F, "ds_ordered_count unsupported for this calling conv"));
11393 [[fallthrough]];
11394 }
11395 case CallingConv::AMDGPU_CS:
11396 case CallingConv::AMDGPU_KERNEL:
11397 case CallingConv::C:
11398 case CallingConv::Fast:
11399 default:
11400 // Assume other calling conventions are various compute callable functions
11401 return 0;
11402 }
11403}
11404
11405bool SIInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
11406 Register &SrcReg2, int64_t &CmpMask,
11407 int64_t &CmpValue) const {
11408 if (!MI.getOperand(i: 0).isReg() || MI.getOperand(i: 0).getSubReg())
11409 return false;
11410
11411 switch (MI.getOpcode()) {
11412 default:
11413 break;
11414 case AMDGPU::S_CMP_EQ_U32:
11415 case AMDGPU::S_CMP_EQ_I32:
11416 case AMDGPU::S_CMP_LG_U32:
11417 case AMDGPU::S_CMP_LG_I32:
11418 case AMDGPU::S_CMP_LT_U32:
11419 case AMDGPU::S_CMP_LT_I32:
11420 case AMDGPU::S_CMP_GT_U32:
11421 case AMDGPU::S_CMP_GT_I32:
11422 case AMDGPU::S_CMP_LE_U32:
11423 case AMDGPU::S_CMP_LE_I32:
11424 case AMDGPU::S_CMP_GE_U32:
11425 case AMDGPU::S_CMP_GE_I32:
11426 case AMDGPU::S_CMP_EQ_U64:
11427 case AMDGPU::S_CMP_LG_U64:
11428 SrcReg = MI.getOperand(i: 0).getReg();
11429 if (MI.getOperand(i: 1).isReg()) {
11430 if (MI.getOperand(i: 1).getSubReg())
11431 return false;
11432 SrcReg2 = MI.getOperand(i: 1).getReg();
11433 CmpValue = 0;
11434 } else if (MI.getOperand(i: 1).isImm()) {
11435 SrcReg2 = Register();
11436 CmpValue = MI.getOperand(i: 1).getImm();
11437 } else {
11438 return false;
11439 }
11440 CmpMask = ~0;
11441 return true;
11442 case AMDGPU::S_CMPK_EQ_U32:
11443 case AMDGPU::S_CMPK_EQ_I32:
11444 case AMDGPU::S_CMPK_LG_U32:
11445 case AMDGPU::S_CMPK_LG_I32:
11446 case AMDGPU::S_CMPK_LT_U32:
11447 case AMDGPU::S_CMPK_LT_I32:
11448 case AMDGPU::S_CMPK_GT_U32:
11449 case AMDGPU::S_CMPK_GT_I32:
11450 case AMDGPU::S_CMPK_LE_U32:
11451 case AMDGPU::S_CMPK_LE_I32:
11452 case AMDGPU::S_CMPK_GE_U32:
11453 case AMDGPU::S_CMPK_GE_I32:
11454 SrcReg = MI.getOperand(i: 0).getReg();
11455 SrcReg2 = Register();
11456 CmpValue = MI.getOperand(i: 1).getImm();
11457 CmpMask = ~0;
11458 return true;
11459 }
11460
11461 return false;
11462}
11463
11464static bool isSCCDeadOnExit(MachineBasicBlock *MBB) {
11465 for (MachineBasicBlock *S : MBB->successors()) {
11466 if (S->isLiveIn(Reg: AMDGPU::SCC))
11467 return false;
11468 }
11469 return true;
11470}
11471
11472// Invert all uses of SCC following SCCDef because SCCDef may be deleted and
11473// (incoming SCC) = !(SCC defined by SCCDef).
11474// Return true if all uses can be re-written, false otherwise.
11475bool SIInstrInfo::invertSCCUse(MachineInstr *SCCDef) const {
11476 MachineBasicBlock *MBB = SCCDef->getParent();
11477 SmallVector<MachineInstr *> InvertInstr;
11478 bool SCCIsDead = false;
11479
11480 // Scan instructions for SCC uses that need to be inverted until SCC is dead.
11481 constexpr unsigned ScanLimit = 12;
11482 unsigned Count = 0;
11483 for (MachineInstr &MI :
11484 make_range(x: std::next(x: MachineBasicBlock::iterator(SCCDef)), y: MBB->end())) {
11485 if (++Count > ScanLimit)
11486 return false;
11487 if (MI.readsRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
11488 if (MI.getOpcode() == AMDGPU::S_CSELECT_B32 ||
11489 MI.getOpcode() == AMDGPU::S_CSELECT_B64 ||
11490 MI.getOpcode() == AMDGPU::S_CBRANCH_SCC0 ||
11491 MI.getOpcode() == AMDGPU::S_CBRANCH_SCC1)
11492 InvertInstr.push_back(Elt: &MI);
11493 else
11494 return false;
11495 }
11496 if (MI.definesRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
11497 SCCIsDead = true;
11498 break;
11499 }
11500 }
11501 if (!SCCIsDead && isSCCDeadOnExit(MBB))
11502 SCCIsDead = true;
11503
11504 // SCC may have more uses. Can't invert all of them.
11505 if (!SCCIsDead)
11506 return false;
11507
11508 // Invert uses
11509 for (MachineInstr *MI : InvertInstr) {
11510 if (MI->getOpcode() == AMDGPU::S_CSELECT_B32 ||
11511 MI->getOpcode() == AMDGPU::S_CSELECT_B64) {
11512 swapOperands(Inst&: *MI);
11513 } else if (MI->getOpcode() == AMDGPU::S_CBRANCH_SCC0 ||
11514 MI->getOpcode() == AMDGPU::S_CBRANCH_SCC1) {
11515 MI->setDesc(get(Opcode: MI->getOpcode() == AMDGPU::S_CBRANCH_SCC0
11516 ? AMDGPU::S_CBRANCH_SCC1
11517 : AMDGPU::S_CBRANCH_SCC0));
11518 } else {
11519 llvm_unreachable("SCC used but no inversion handling");
11520 }
11521 }
11522 return true;
11523}
11524
11525// SCC is already valid after SCCValid.
11526// SCCRedefine will redefine SCC to the same value already available after
11527// SCCValid. If there are no intervening SCC conflicts delete SCCRedefine and
11528// update kill/dead flags if necessary.
11529bool SIInstrInfo::optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,
11530 bool NeedInversion) const {
11531 MachineInstr *KillsSCC = nullptr;
11532 if (SCCValid->getParent() != SCCRedefine->getParent())
11533 return false;
11534 for (MachineInstr &MI : make_range(x: std::next(x: SCCValid->getIterator()),
11535 y: SCCRedefine->getIterator())) {
11536 if (MI.modifiesRegister(Reg: AMDGPU::SCC, TRI: &RI))
11537 return false;
11538 if (MI.killsRegister(Reg: AMDGPU::SCC, TRI: &RI))
11539 KillsSCC = &MI;
11540 }
11541 if (NeedInversion && !invertSCCUse(SCCDef: SCCRedefine))
11542 return false;
11543 if (MachineOperand *SccDef =
11544 SCCValid->findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr))
11545 SccDef->setIsDead(false);
11546 if (KillsSCC)
11547 KillsSCC->clearRegisterKills(Reg: AMDGPU::SCC, /*TRI=*/RegInfo: nullptr);
11548 SCCRedefine->eraseFromParent();
11549 return true;
11550}
11551
11552static bool foldableSelect(const MachineInstr &Def) {
11553 if (Def.getOpcode() != AMDGPU::S_CSELECT_B32 &&
11554 Def.getOpcode() != AMDGPU::S_CSELECT_B64)
11555 return false;
11556 bool Op1IsNonZeroImm =
11557 Def.getOperand(i: 1).isImm() && Def.getOperand(i: 1).getImm() != 0;
11558 bool Op2IsZeroImm =
11559 Def.getOperand(i: 2).isImm() && Def.getOperand(i: 2).getImm() == 0;
11560 return Op1IsNonZeroImm && Op2IsZeroImm;
11561}
11562
11563static bool setsSCCIfResultIsZero(const MachineInstr &Def, bool &NeedInversion,
11564 unsigned &NewDefOpc) {
11565 // S_ADD_U32 X, 1 sets SCC on carryout which can only happen if result==0.
11566 // S_ADD_I32 X, 1 can be converted to S_ADD_U32 X, 1 if SCC is dead.
11567 if (Def.getOpcode() != AMDGPU::S_ADD_I32 &&
11568 Def.getOpcode() != AMDGPU::S_ADD_U32)
11569 return false;
11570 const MachineOperand &AddSrc1 = Def.getOperand(i: 1);
11571 const MachineOperand &AddSrc2 = Def.getOperand(i: 2);
11572 const MachineRegisterInfo &MRI = Def.getMF()->getRegInfo();
11573 const SIInstrInfo *TII = static_cast<const SIInstrInfo *>(
11574 Def.getMF()->getSubtarget().getInstrInfo());
11575
11576 auto Imm1 = TII->getImmOrMaterializedImm(MRI, Op: AddSrc1);
11577 auto Imm2 = TII->getImmOrMaterializedImm(MRI, Op: AddSrc2);
11578 if ((!Imm1 || *Imm1 != 1) && (!Imm2 || *Imm2 != 1))
11579 return false;
11580
11581 if (Def.getOpcode() == AMDGPU::S_ADD_I32) {
11582 const MachineOperand *SccDef =
11583 Def.findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
11584 if (!SccDef->isDead())
11585 return false;
11586 NewDefOpc = AMDGPU::S_ADD_U32;
11587 }
11588 NeedInversion = !NeedInversion;
11589 return true;
11590}
11591
11592bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
11593 Register SrcReg2, int64_t CmpMask,
11594 int64_t CmpValue,
11595 const MachineRegisterInfo *MRI) const {
11596 if (!SrcReg || SrcReg.isPhysical())
11597 return false;
11598
11599 if (SrcReg2) {
11600 auto ImmOpt = getImmOrMaterializedImm(MRI: *MRI, Reg: SrcReg2);
11601 if (!ImmOpt)
11602 return false;
11603 CmpValue = *ImmOpt;
11604 }
11605
11606 const auto optimizeCmpSelect = [&CmpInstr, SrcReg, CmpValue, MRI,
11607 this](bool NeedInversion) -> bool {
11608 if (CmpValue != 0)
11609 return false;
11610
11611 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
11612 if (!Def)
11613 return false;
11614
11615 // For S_OP that set SCC = DST!=0, do the transformation
11616 //
11617 // s_cmp_[lg|eq]_* (S_OP ...), 0 => (S_OP ...)
11618 //
11619 // For (S_OP ...) that set SCC = DST==0, invert NeedInversion and
11620 // do the transformation:
11621 //
11622 // s_cmp_[lg|eq]_* (S_OP ...), 0 => (S_OP ...)
11623 //
11624 // If foldableSelect, s_cmp_lg_* is redundant because the SCC input value
11625 // for S_CSELECT* already has the same value that will be calculated by
11626 // s_cmp_lg_*
11627 //
11628 // s_cmp_[lg|eq]_* (S_CSELECT* (non-zero imm), 0), 0 => (S_CSELECT*
11629 // (non-zero imm), 0)
11630
11631 unsigned NewDefOpc = Def->getOpcode();
11632 if (!setsSCCIfResultIsNonZero(*Def) &&
11633 !setsSCCIfResultIsZero(Def: *Def, NeedInversion, NewDefOpc) &&
11634 !foldableSelect(Def: *Def))
11635 return false;
11636
11637 if (!optimizeSCC(SCCValid: Def, SCCRedefine: &CmpInstr, NeedInversion))
11638 return false;
11639
11640 if (NewDefOpc != Def->getOpcode())
11641 Def->setDesc(get(Opcode: NewDefOpc));
11642
11643 // If s_or_b32 result, sY, is unused (i.e. it is effectively a 64-bit
11644 // s_cmp_lg of a register pair) and the inputs are the hi and lo-halves of a
11645 // 64-bit foldableSelect then delete s_or_b32 in the sequence:
11646 // sX = s_cselect_b64 (non-zero imm), 0
11647 // sLo = copy sX.sub0
11648 // sHi = copy sX.sub1
11649 // sY = s_or_b32 sLo, sHi
11650 if (Def->getOpcode() == AMDGPU::S_OR_B32 &&
11651 MRI->use_nodbg_empty(RegNo: Def->getOperand(i: 0).getReg())) {
11652 const MachineOperand &OrOpnd1 = Def->getOperand(i: 1);
11653 const MachineOperand &OrOpnd2 = Def->getOperand(i: 2);
11654 if (OrOpnd1.isReg() && OrOpnd2.isReg()) {
11655 MachineInstr *Def1 = MRI->getVRegDef(Reg: OrOpnd1.getReg());
11656 MachineInstr *Def2 = MRI->getVRegDef(Reg: OrOpnd2.getReg());
11657 if (Def1 && Def1->getOpcode() == AMDGPU::COPY && Def2 &&
11658 Def2->getOpcode() == AMDGPU::COPY && Def1->getOperand(i: 1).isReg() &&
11659 Def2->getOperand(i: 1).isReg() &&
11660 Def1->getOperand(i: 1).getSubReg() == AMDGPU::sub0 &&
11661 Def2->getOperand(i: 1).getSubReg() == AMDGPU::sub1 &&
11662 Def1->getOperand(i: 1).getReg() == Def2->getOperand(i: 1).getReg()) {
11663 MachineInstr *Select = MRI->getVRegDef(Reg: Def1->getOperand(i: 1).getReg());
11664 if (Select && foldableSelect(Def: *Select))
11665 optimizeSCC(SCCValid: Select, SCCRedefine: Def, /*NeedInversion=*/false);
11666 }
11667 }
11668 }
11669 return true;
11670 };
11671
11672 const auto optimizeCmpAnd = [&CmpInstr, SrcReg, CmpValue, MRI,
11673 this](int64_t ExpectedValue, unsigned SrcSize,
11674 bool IsReversible, bool IsSigned) -> bool {
11675 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11676 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11677 // s_cmp_ge_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11678 // s_cmp_ge_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11679 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 1 << n => s_and_b64 $src, 1 << n
11680 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11681 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11682 // s_cmp_gt_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11683 // s_cmp_gt_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11684 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 0 => s_and_b64 $src, 1 << n
11685 //
11686 // Signed ge/gt are not used for the sign bit.
11687 //
11688 // If result of the AND is unused except in the compare:
11689 // s_and_b(32|64) $src, 1 << n => s_bitcmp1_b(32|64) $src, n
11690 //
11691 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
11692 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
11693 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 0 => s_bitcmp0_b64 $src, n
11694 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
11695 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
11696 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 1 << n => s_bitcmp0_b64 $src, n
11697
11698 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
11699 if (!Def)
11700 return false;
11701
11702 if (Def->getOpcode() != AMDGPU::S_AND_B32 &&
11703 Def->getOpcode() != AMDGPU::S_AND_B64)
11704 return false;
11705
11706 int64_t Mask;
11707 const auto isMask = [&Mask, SrcSize, MRI,
11708 this](const MachineOperand *MO) -> bool {
11709 auto ImmOpt = this->getImmOrMaterializedImm(MRI: *MRI, Op: *MO);
11710 if (!ImmOpt)
11711 return false;
11712 Mask = *ImmOpt;
11713 Mask &= maxUIntN(N: SrcSize);
11714 return isPowerOf2_64(Value: Mask);
11715 };
11716
11717 MachineOperand *SrcOp = &Def->getOperand(i: 1);
11718 if (isMask(SrcOp))
11719 SrcOp = &Def->getOperand(i: 2);
11720 else if (isMask(&Def->getOperand(i: 2)))
11721 SrcOp = &Def->getOperand(i: 1);
11722 else
11723 return false;
11724
11725 // A valid Mask is required to have a single bit set, hence a non-zero and
11726 // power-of-two value. This verifies that we will not do 64-bit shift below.
11727 assert(llvm::has_single_bit<uint64_t>(Mask) && "Invalid mask.");
11728 unsigned BitNo = llvm::countr_zero(Val: (uint64_t)Mask);
11729 if (IsSigned && BitNo == SrcSize - 1)
11730 return false;
11731
11732 ExpectedValue <<= BitNo;
11733
11734 bool IsReversedCC = false;
11735 if (CmpValue != ExpectedValue) {
11736 if (!IsReversible)
11737 return false;
11738 IsReversedCC = CmpValue == (ExpectedValue ^ Mask);
11739 if (!IsReversedCC)
11740 return false;
11741 }
11742
11743 Register DefReg = Def->getOperand(i: 0).getReg();
11744 if (IsReversedCC && !MRI->hasOneNonDBGUse(RegNo: DefReg))
11745 return false;
11746
11747 if (!optimizeSCC(SCCValid: Def, SCCRedefine: &CmpInstr, /*NeedInversion=*/false))
11748 return false;
11749
11750 if (!MRI->use_nodbg_empty(RegNo: DefReg)) {
11751 assert(!IsReversedCC);
11752 return true;
11753 }
11754
11755 // Replace AND with unused result with a S_BITCMP.
11756 MachineBasicBlock *MBB = Def->getParent();
11757
11758 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32
11759 : AMDGPU::S_BITCMP1_B32
11760 : IsReversedCC ? AMDGPU::S_BITCMP0_B64
11761 : AMDGPU::S_BITCMP1_B64;
11762
11763 BuildMI(BB&: *MBB, I: Def, MIMD: Def->getDebugLoc(), MCID: get(Opcode: NewOpc))
11764 .add(MO: *SrcOp)
11765 .addImm(Val: BitNo);
11766 Def->eraseFromParent();
11767
11768 return true;
11769 };
11770
11771 switch (CmpInstr.getOpcode()) {
11772 default:
11773 break;
11774 case AMDGPU::S_CMP_EQ_U32:
11775 case AMDGPU::S_CMP_EQ_I32:
11776 case AMDGPU::S_CMPK_EQ_U32:
11777 case AMDGPU::S_CMPK_EQ_I32:
11778 return optimizeCmpAnd(1, 32, true, false) ||
11779 optimizeCmpSelect(/*NeedInversion=*/true);
11780 case AMDGPU::S_CMP_GE_U32:
11781 case AMDGPU::S_CMPK_GE_U32:
11782 return optimizeCmpAnd(1, 32, false, false);
11783 case AMDGPU::S_CMP_GE_I32:
11784 case AMDGPU::S_CMPK_GE_I32:
11785 return optimizeCmpAnd(1, 32, false, true);
11786 case AMDGPU::S_CMP_EQ_U64:
11787 return optimizeCmpAnd(1, 64, true, false);
11788 case AMDGPU::S_CMP_LG_U32:
11789 case AMDGPU::S_CMP_LG_I32:
11790 case AMDGPU::S_CMPK_LG_U32:
11791 case AMDGPU::S_CMPK_LG_I32:
11792 return optimizeCmpAnd(0, 32, true, false) ||
11793 optimizeCmpSelect(/*NeedInversion=*/false);
11794 case AMDGPU::S_CMP_GT_U32:
11795 case AMDGPU::S_CMPK_GT_U32:
11796 return optimizeCmpAnd(0, 32, false, false);
11797 case AMDGPU::S_CMP_GT_I32:
11798 case AMDGPU::S_CMPK_GT_I32:
11799 return optimizeCmpAnd(0, 32, false, true);
11800 case AMDGPU::S_CMP_LG_U64:
11801 return optimizeCmpAnd(0, 64, true, false) ||
11802 optimizeCmpSelect(/*NeedInversion=*/false);
11803 }
11804
11805 return false;
11806}
11807
11808void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI,
11809 AMDGPU::OpName OpName) const {
11810 if (!ST.needsAlignedVGPRs())
11811 return;
11812
11813 int OpNo = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpName);
11814 if (OpNo < 0)
11815 return;
11816 MachineOperand &Op = MI.getOperand(i: OpNo);
11817 if (getOpSize(MI, OpNo) > 4)
11818 return;
11819
11820 // Add implicit aligned super-reg to force alignment on the data operand.
11821 const DebugLoc &DL = MI.getDebugLoc();
11822 MachineBasicBlock *BB = MI.getParent();
11823 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
11824 Register DataReg = Op.getReg();
11825 bool IsAGPR = RI.isAGPR(MRI, Reg: DataReg);
11826 Register Undef = MRI.createVirtualRegister(
11827 RegClass: IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);
11828 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
11829 Register NewVR =
11830 MRI.createVirtualRegister(RegClass: IsAGPR ? &AMDGPU::AReg_64_Align2RegClass
11831 : &AMDGPU::VReg_64_Align2RegClass);
11832 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewVR)
11833 .addReg(RegNo: DataReg, Flags: {}, SubReg: Op.getSubReg())
11834 .addImm(Val: AMDGPU::sub0)
11835 .addReg(RegNo: Undef)
11836 .addImm(Val: AMDGPU::sub1);
11837 Op.setReg(NewVR);
11838 Op.setSubReg(AMDGPU::sub0);
11839 MI.addOperand(Op: MachineOperand::CreateReg(Reg: NewVR, isDef: false, isImp: true));
11840}
11841
11842unsigned SIInstrInfo::getRepeatRate(const MachineInstr &MI) const {
11843 if (!SchedModel.hasInstrSchedModel())
11844 return 0;
11845
11846 // The repeat rate is the throughput-limiting resource occupancy: the largest
11847 // number of cycles any written processor resource is held.
11848 const MCSchedClassDesc *SCDesc = SchedModel.resolveSchedClass(MI: &MI);
11849 unsigned RepeatRate = 0;
11850 for (TargetSchedModel::ProcResIter
11851 PI = SchedModel.getWriteProcResBegin(SC: SCDesc),
11852 PE = SchedModel.getWriteProcResEnd(SC: SCDesc);
11853 PI != PE; ++PI) {
11854 RepeatRate = std::max(a: RepeatRate, b: (unsigned)PI->ReleaseAtCycle);
11855 }
11856
11857 return RepeatRate;
11858}
11859
11860bool SIInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const {
11861 if (isIGLP(MI: *MI))
11862 return false;
11863
11864 return TargetInstrInfo::isGlobalMemoryObject(MI);
11865}
11866
11867bool SIInstrInfo::isXDLWMMA(const MachineInstr &MI) const {
11868 if (!isWMMA(MI) && !isSWMMAC(MI))
11869 return false;
11870
11871 if (ST.hasGFX1250Insts())
11872 return AMDGPU::getWMMAIsXDL(Opc: MI.getOpcode());
11873
11874 return true;
11875}
11876
11877bool SIInstrInfo::isXDL(const MachineInstr &MI) const {
11878 unsigned Opcode = MI.getOpcode();
11879
11880 if (AMDGPU::isGFX12Plus(STI: ST))
11881 return isDOT(MI) || isXDLWMMA(MI);
11882
11883 if (!isMAI(MI) || isDGEMM(Opcode) ||
11884 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
11885 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
11886 return false;
11887
11888 if (!ST.hasGFX940Insts())
11889 return true;
11890
11891 return AMDGPU::getMAIIsGFX940XDL(Opc: Opcode);
11892}
11893