1//===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// SI Implementation of TargetInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIInstrInfo.h"
15#include "AMDGPU.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPULaneMaskUtils.h"
18#include "GCNHazardRecognizer.h"
19#include "GCNSubtarget.h"
20#include "SIMachineFunctionInfo.h"
21#include "Utils/AMDGPUBaseInfo.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
25#include "llvm/CodeGen/LiveIntervals.h"
26#include "llvm/CodeGen/LiveVariables.h"
27#include "llvm/CodeGen/MachineCycleAnalysis.h"
28#include "llvm/CodeGen/MachineDominators.h"
29#include "llvm/CodeGen/MachineFrameInfo.h"
30#include "llvm/CodeGen/MachineScheduler.h"
31#include "llvm/CodeGen/RegisterScavenging.h"
32#include "llvm/CodeGen/ScheduleDAG.h"
33#include "llvm/IR/DiagnosticInfo.h"
34#include "llvm/IR/IntrinsicsAMDGPU.h"
35#include "llvm/MC/MCContext.h"
36#include "llvm/Support/CommandLine.h"
37#include "llvm/Target/TargetMachine.h"
38#include <tuple>
39
40using namespace llvm;
41
42#define DEBUG_TYPE "si-instr-info"
43
44#define GET_INSTRINFO_CTOR_DTOR
45#include "AMDGPUGenInstrInfo.inc"
46
47namespace llvm::AMDGPU {
48#define GET_D16ImageDimIntrinsics_IMPL
49#define GET_ImageDimIntrinsicTable_IMPL
50#define GET_RsrcIntrinsics_IMPL
51#include "AMDGPUGenSearchableTables.inc"
52} // namespace llvm::AMDGPU
53
54// Must be at least 4 to be able to branch over minimum unconditional branch
55// code. This is only for making it possible to write reasonably small tests for
56// long branches.
57static cl::opt<unsigned>
58BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(Val: 16),
59 cl::desc("Restrict range of branch instructions (DEBUG)"));
60
61static cl::opt<bool> Fix16BitCopies(
62 "amdgpu-fix-16-bit-physreg-copies",
63 cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"),
64 cl::init(Val: true),
65 cl::ReallyHidden);
66
67SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
68 : AMDGPUGenInstrInfo(ST, RI, AMDGPU::ADJCALLSTACKUP,
69 AMDGPU::ADJCALLSTACKDOWN),
70 RI(ST), ST(ST) {
71 SchedModel.init(TSInfo: &ST);
72}
73
74//===----------------------------------------------------------------------===//
75// TargetInstrInfo callbacks
76//===----------------------------------------------------------------------===//
77
78static unsigned getNumOperandsNoGlue(SDNode *Node) {
79 unsigned N = Node->getNumOperands();
80 while (N && Node->getOperand(Num: N - 1).getValueType() == MVT::Glue)
81 --N;
82 return N;
83}
84
85/// Returns true if both nodes have the same value for the given
86/// operand \p Op, or if both nodes do not have this operand.
87static bool nodesHaveSameOperandValue(SDNode *N0, SDNode *N1,
88 AMDGPU::OpName OpName) {
89 unsigned Opc0 = N0->getMachineOpcode();
90 unsigned Opc1 = N1->getMachineOpcode();
91
92 int Op0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: OpName);
93 int Op1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: OpName);
94
95 if (Op0Idx == -1 && Op1Idx == -1)
96 return true;
97
98
99 if ((Op0Idx == -1 && Op1Idx != -1) ||
100 (Op1Idx == -1 && Op0Idx != -1))
101 return false;
102
103 // getNamedOperandIdx returns the index for the MachineInstr's operands,
104 // which includes the result as the first operand. We are indexing into the
105 // MachineSDNode's operands, so we need to skip the result operand to get
106 // the real index.
107 --Op0Idx;
108 --Op1Idx;
109
110 return N0->getOperand(Num: Op0Idx) == N1->getOperand(Num: Op1Idx);
111}
112
113static bool canRemat(const MachineInstr &MI) {
114
115 if (SIInstrInfo::isVOP1(MI) || SIInstrInfo::isVOP2(MI) ||
116 SIInstrInfo::isVOP3(MI) || SIInstrInfo::isSDWA(MI) ||
117 SIInstrInfo::isSALU(MI))
118 return true;
119
120 if (SIInstrInfo::isSMRD(MI)) {
121 return !MI.memoperands_empty() &&
122 llvm::all_of(Range: MI.memoperands(), P: [](const MachineMemOperand *MMO) {
123 return MMO->isLoad() && MMO->isInvariant();
124 });
125 }
126
127 return false;
128}
129
130// Split relocation flags for 64-bit global-address materialization into a
131// common base and the hi/lo relocation variants.
132static std::tuple<unsigned, unsigned, unsigned>
133splitGlobalAddressRelocFlags(const GCNSubtarget &ST,
134 const MachineOperand &SrcOp) {
135 const unsigned BaseFlags = SrcOp.getTargetFlags() & ~SIInstrInfo::MO_MASK;
136 const unsigned Reloc = SrcOp.getTargetFlags() & SIInstrInfo::MO_MASK;
137
138 // Infer the relocation type from the existing flags on the global operand.
139 // The relocation type should have been determined earlier in the pipeline.
140 unsigned LoReloc, HiReloc;
141 switch (Reloc) {
142 case SIInstrInfo::MO_REL32_LO:
143 case SIInstrInfo::MO_REL32_HI:
144 case SIInstrInfo::MO_REL64:
145 LoReloc = SIInstrInfo::MO_REL32_LO;
146 HiReloc = SIInstrInfo::MO_REL32_HI;
147 break;
148 case SIInstrInfo::MO_GOTPCREL32_LO:
149 case SIInstrInfo::MO_GOTPCREL32_HI:
150 LoReloc = SIInstrInfo::MO_GOTPCREL32_LO;
151 HiReloc = SIInstrInfo::MO_GOTPCREL32_HI;
152 break;
153 case SIInstrInfo::MO_GOTPCREL:
154 case SIInstrInfo::MO_GOTPCREL64:
155 // For 64-bit GOT-relative, use the 64-bit relocation.
156 LoReloc = SIInstrInfo::MO_GOTPCREL64;
157 HiReloc = SIInstrInfo::MO_GOTPCREL64;
158 break;
159 case SIInstrInfo::MO_ABS32_LO:
160 case SIInstrInfo::MO_ABS32_HI:
161 case SIInstrInfo::MO_ABS64:
162 LoReloc = SIInstrInfo::MO_ABS32_LO;
163 HiReloc = SIInstrInfo::MO_ABS32_HI;
164 break;
165 default:
166 llvm_unreachable("unknown relocation type for global address");
167 break;
168 }
169
170 return {BaseFlags, LoReloc, HiReloc};
171}
172
173bool SIInstrInfo::isReMaterializableImpl(
174 const MachineInstr &MI) const {
175
176 if (canRemat(MI)) {
177 // Normally VALU use of exec would block the rematerialization, but that
178 // is OK in this case to have an implicit exec read as all VALU do.
179 // We really want all of the generic logic for this except for this.
180
181 // Another potential implicit use is mode register. The core logic of
182 // the RA will not attempt rematerialization if mode is set anywhere
183 // in the function, otherwise it is safe since mode is not changed.
184
185 // There is difference to generic method which does not allow
186 // rematerialization if there are virtual register uses. We allow this,
187 // therefore this method includes SOP instructions as well.
188 if (!MI.hasImplicitDef() &&
189 MI.getNumImplicitOperands() == MI.getDesc().implicit_uses().size() &&
190 !MI.mayRaiseFPException())
191 return true;
192 }
193
194 // Everything below copied from TargetInstrInfo::isReMaterializableImpl. The
195 // only difference is that we allow operations that perform read-modify-write
196 // on sub-registers.
197
198 // Remat clients assume operand 0 is the defined register.
199 if (!MI.getNumOperands() || !MI.getOperand(i: 0).isReg())
200 return false;
201 Register DefReg = MI.getOperand(i: 0).getReg();
202
203 const MachineFunction &MF = *MI.getMF();
204
205 // A load from a fixed stack slot can be rematerialized. This may be
206 // redundant with subsequent checks, but it's target-independent,
207 // simple, and a common case.
208 int FrameIdx = 0;
209 if (isLoadFromStackSlot(MI, FrameIndex&: FrameIdx) &&
210 MF.getFrameInfo().isImmutableObjectIndex(ObjectIdx: FrameIdx))
211 return true;
212
213 // Avoid instructions obviously unsafe for remat.
214 if (MI.isNotDuplicable() || MI.mayStore() || MI.mayRaiseFPException() ||
215 MI.hasUnmodeledSideEffects())
216 return false;
217
218 // Don't remat inline asm. We have no idea how expensive it is
219 // even if it's side effect free.
220 if (MI.isInlineAsm())
221 return false;
222
223 // Avoid instructions which load from potentially varying memory.
224 if (MI.mayLoad() && !MI.isDereferenceableInvariantLoad())
225 return false;
226
227 const MachineRegisterInfo &MRI = MF.getRegInfo();
228
229 // If any of the registers accessed are non-constant, conservatively assume
230 // the instruction is not rematerializable.
231 for (const MachineOperand &MO : MI.operands()) {
232 if (!MO.isReg())
233 continue;
234 Register Reg = MO.getReg();
235 if (Reg == 0)
236 continue;
237
238 // Check for a well-behaved physical register.
239 if (Reg.isPhysical()) {
240 if (MO.isUse()) {
241 // If the physreg has no defs anywhere, it's just an ambient register
242 // and we can freely move its uses. Alternatively, if it's allocatable,
243 // it could get allocated to something with a def during allocation.
244 if (!MRI.isConstantPhysReg(PhysReg: Reg))
245 return false;
246 } else {
247 // A physreg def. We can't remat it.
248 return false;
249 }
250 continue;
251 }
252
253 // Only allow one virtual-register def. There may be multiple defs of the
254 // same virtual register, though.
255 if (MO.isDef() && Reg != DefReg)
256 return false;
257 }
258
259 return true;
260}
261
262// Returns true if the result of a VALU instruction depends on exec.
263bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {
264 assert(isVALU(MI, /*AllowLDSDMA=*/true));
265
266 // If it is convergent it depends on EXEC.
267 if (MI.isConvergent())
268 return true;
269
270 // If it defines an SGPR it depends on EXEC, unless it's dead.
271 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
272 for (const MachineOperand &Def : MI.defs()) {
273 if (Def.isDead())
274 continue;
275
276 Register Reg = Def.getReg();
277 if (Reg && RI.isSGPRReg(MRI, Reg))
278 return true;
279 }
280
281 return false;
282}
283
284bool SIInstrInfo::isIgnorableUse(const MachineOperand &MO) const {
285 // Any implicit use of exec by VALU is not a real register read.
286 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&
287 isVALU(MI: *MO.getParent(), /*AllowLDSDMA=*/true) &&
288 !resultDependsOnExec(MI: *MO.getParent());
289}
290
291bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
292 MachineBasicBlock *SuccToSinkTo,
293 MachineCycleInfo *CI) const {
294 // Allow sinking if MI edits lane mask (divergent i1 in sgpr).
295 if (MI.getOpcode() == AMDGPU::SI_IF_BREAK)
296 return true;
297
298 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
299 // Check if sinking of MI would create temporal divergent use.
300 for (auto Op : MI.uses()) {
301 if (Op.isReg() && Op.getReg().isVirtual() &&
302 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Op.getReg()))) {
303 MachineInstr *SgprDef = MRI.getVRegDef(Reg: Op.getReg());
304 if (!SgprDef)
305 continue;
306
307 // SgprDef defined inside cycle
308 CycleRef FromCycle = CI->getCycle(Block: SgprDef->getParent());
309 if (!FromCycle)
310 continue;
311
312 CycleRef ToCycle = CI->getCycle(Block: SuccToSinkTo);
313 // Check if there is a FromCycle that contains SgprDef's basic block but
314 // does not contain SuccToSinkTo and also has divergent exit condition.
315 while (FromCycle && !(ToCycle && CI->contains(Outer: FromCycle, Inner: ToCycle))) {
316 SmallVector<MachineBasicBlock *, 1> ExitingBlocks;
317 CI->getExitingBlocks(C: FromCycle, TmpStorage&: ExitingBlocks);
318
319 // FromCycle has divergent exit condition.
320 for (MachineBasicBlock *ExitingBlock : ExitingBlocks) {
321 if (hasDivergentBranch(MBB: ExitingBlock))
322 return false;
323 }
324
325 FromCycle = CI->getParentCycle(C: FromCycle);
326 }
327 }
328 }
329
330 return true;
331}
332
333bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,
334 int64_t &Offset0,
335 int64_t &Offset1) const {
336 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
337 return false;
338
339 unsigned Opc0 = Load0->getMachineOpcode();
340 unsigned Opc1 = Load1->getMachineOpcode();
341
342 // Make sure both are actually loads.
343 if (!get(Opcode: Opc0).mayLoad() || !get(Opcode: Opc1).mayLoad())
344 return false;
345
346 // A mayLoad instruction without a def is not a load. Likely a prefetch.
347 if (!get(Opcode: Opc0).getNumDefs() || !get(Opcode: Opc1).getNumDefs())
348 return false;
349
350 if (isDS(Opcode: Opc0) && isDS(Opcode: Opc1)) {
351
352 // FIXME: Handle this case:
353 if (getNumOperandsNoGlue(Node: Load0) != getNumOperandsNoGlue(Node: Load1))
354 return false;
355
356 // Check base reg.
357 if (Load0->getOperand(Num: 0) != Load1->getOperand(Num: 0))
358 return false;
359
360 // Skip read2 / write2 variants for simplicity.
361 // TODO: We should report true if the used offsets are adjacent (excluded
362 // st64 versions).
363 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: AMDGPU::OpName::offset);
364 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: AMDGPU::OpName::offset);
365 if (Offset0Idx == -1 || Offset1Idx == -1)
366 return false;
367
368 // XXX - be careful of dataless loads
369 // getNamedOperandIdx returns the index for MachineInstrs. Since they
370 // include the output in the operand list, but SDNodes don't, we need to
371 // subtract the index by one.
372 Offset0Idx -= get(Opcode: Opc0).NumDefs;
373 Offset1Idx -= get(Opcode: Opc1).NumDefs;
374 Offset0 = Load0->getConstantOperandVal(Num: Offset0Idx);
375 Offset1 = Load1->getConstantOperandVal(Num: Offset1Idx);
376 return true;
377 }
378
379 if (isSMRD(Opcode: Opc0) && isSMRD(Opcode: Opc1)) {
380 // Skip time and cache invalidation instructions.
381 if (!AMDGPU::hasNamedOperand(Opcode: Opc0, NamedIdx: AMDGPU::OpName::sbase) ||
382 !AMDGPU::hasNamedOperand(Opcode: Opc1, NamedIdx: AMDGPU::OpName::sbase))
383 return false;
384
385 unsigned NumOps = getNumOperandsNoGlue(Node: Load0);
386 if (NumOps != getNumOperandsNoGlue(Node: Load1))
387 return false;
388
389 // Check base reg.
390 if (Load0->getOperand(Num: 0) != Load1->getOperand(Num: 0))
391 return false;
392
393 // Match register offsets, if both register and immediate offsets present.
394 assert(NumOps == 4 || NumOps == 5);
395 if (NumOps == 5 && Load0->getOperand(Num: 1) != Load1->getOperand(Num: 1))
396 return false;
397
398 const ConstantSDNode *Load0Offset =
399 dyn_cast<ConstantSDNode>(Val: Load0->getOperand(Num: NumOps - 3));
400 const ConstantSDNode *Load1Offset =
401 dyn_cast<ConstantSDNode>(Val: Load1->getOperand(Num: NumOps - 3));
402
403 if (!Load0Offset || !Load1Offset)
404 return false;
405
406 Offset0 = Load0Offset->getZExtValue();
407 Offset1 = Load1Offset->getZExtValue();
408 return true;
409 }
410
411 // MUBUF and MTBUF can access the same addresses.
412 if ((isMUBUF(Opcode: Opc0) || isMTBUF(Opcode: Opc0)) && (isMUBUF(Opcode: Opc1) || isMTBUF(Opcode: Opc1))) {
413
414 // MUBUF and MTBUF have vaddr at different indices.
415 if (!nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::soffset) ||
416 !nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::vaddr) ||
417 !nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::srsrc))
418 return false;
419
420 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: AMDGPU::OpName::offset);
421 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: AMDGPU::OpName::offset);
422
423 if (OffIdx0 == -1 || OffIdx1 == -1)
424 return false;
425
426 // getNamedOperandIdx returns the index for MachineInstrs. Since they
427 // include the output in the operand list, but SDNodes don't, we need to
428 // subtract the index by one.
429 OffIdx0 -= get(Opcode: Opc0).NumDefs;
430 OffIdx1 -= get(Opcode: Opc1).NumDefs;
431
432 SDValue Off0 = Load0->getOperand(Num: OffIdx0);
433 SDValue Off1 = Load1->getOperand(Num: OffIdx1);
434
435 // The offset might be a FrameIndexSDNode.
436 if (!isa<ConstantSDNode>(Val: Off0) || !isa<ConstantSDNode>(Val: Off1))
437 return false;
438
439 Offset0 = Off0->getAsZExtVal();
440 Offset1 = Off1->getAsZExtVal();
441 return true;
442 }
443
444 return false;
445}
446
447static bool isStride64(unsigned Opc) {
448 switch (Opc) {
449 case AMDGPU::DS_READ2ST64_B32:
450 case AMDGPU::DS_READ2ST64_B64:
451 case AMDGPU::DS_WRITE2ST64_B32:
452 case AMDGPU::DS_WRITE2ST64_B64:
453 return true;
454 default:
455 return false;
456 }
457}
458
459bool SIInstrInfo::getMemOperandsWithOffsetWidth(
460 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
461 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
462 const TargetRegisterInfo *TRI) const {
463 if (!LdSt.mayLoadOrStore())
464 return false;
465
466 unsigned Opc = LdSt.getOpcode();
467 OffsetIsScalable = false;
468 const MachineOperand *BaseOp, *OffsetOp;
469 int DataOpIdx;
470
471 if (isDS(MI: LdSt)) {
472 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::addr);
473 OffsetOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
474 if (OffsetOp) {
475 // Normal, single offset LDS instruction.
476 if (!BaseOp) {
477 // DS_CONSUME/DS_APPEND use M0 for the base address.
478 // TODO: find the implicit use operand for M0 and use that as BaseOp?
479 return false;
480 }
481 BaseOps.push_back(Elt: BaseOp);
482 Offset = OffsetOp->getImm();
483 // Get appropriate operand, and compute width accordingly.
484 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
485 if (DataOpIdx == -1)
486 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
487 if (Opc == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
488 Width = LocationSize::precise(Value: 64);
489 else
490 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
491 } else {
492 // The 2 offset instructions use offset0 and offset1 instead. We can treat
493 // these as a load with a single offset if the 2 offsets are consecutive.
494 // We will use this for some partially aligned loads.
495 const MachineOperand *Offset0Op =
496 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset0);
497 const MachineOperand *Offset1Op =
498 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset1);
499
500 unsigned Offset0 = Offset0Op->getImm() & 0xff;
501 unsigned Offset1 = Offset1Op->getImm() & 0xff;
502 if (Offset0 + 1 != Offset1)
503 return false;
504
505 // Each of these offsets is in element sized units, so we need to convert
506 // to bytes of the individual reads.
507
508 unsigned EltSize;
509 if (LdSt.mayLoad())
510 EltSize = TRI->getRegSizeInBits(RC: *getOpRegClass(MI: LdSt, OpNo: 0)) / 16;
511 else {
512 assert(LdSt.mayStore());
513 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
514 EltSize = TRI->getRegSizeInBits(RC: *getOpRegClass(MI: LdSt, OpNo: Data0Idx)) / 8;
515 }
516
517 if (isStride64(Opc))
518 EltSize *= 64;
519
520 BaseOps.push_back(Elt: BaseOp);
521 Offset = EltSize * Offset0;
522 // Get appropriate operand(s), and compute width accordingly.
523 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
524 if (DataOpIdx == -1) {
525 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
526 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
527 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data1);
528 Width = LocationSize::precise(
529 Value: Width.getValue() + TypeSize::getFixed(ExactSize: getOpSize(MI: LdSt, OpNo: DataOpIdx)));
530 } else {
531 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
532 }
533 }
534 return true;
535 }
536
537 if (isMUBUF(MI: LdSt) || isMTBUF(MI: LdSt)) {
538 const MachineOperand *RSrc = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::srsrc);
539 if (!RSrc) // e.g. BUFFER_WBINVL1_VOL
540 return false;
541 BaseOps.push_back(Elt: RSrc);
542 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr);
543 if (BaseOp && !BaseOp->isFI())
544 BaseOps.push_back(Elt: BaseOp);
545 const MachineOperand *OffsetImm =
546 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
547 Offset = OffsetImm->getImm();
548 const MachineOperand *SOffset =
549 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::soffset);
550 if (SOffset) {
551 if (SOffset->isReg())
552 BaseOps.push_back(Elt: SOffset);
553 else
554 Offset += SOffset->getImm();
555 }
556 // Get appropriate operand, and compute width accordingly.
557 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
558 if (DataOpIdx == -1)
559 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
560 if (DataOpIdx == -1) // LDS DMA
561 return false;
562 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
563 return true;
564 }
565
566 if (isImage(MI: LdSt)) {
567 auto RsrcOpName =
568 isMIMG(MI: LdSt) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
569 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: RsrcOpName);
570 BaseOps.push_back(Elt: &LdSt.getOperand(i: SRsrcIdx));
571 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0);
572 if (VAddr0Idx >= 0) {
573 // GFX10 possible NSA encoding.
574 for (int I = VAddr0Idx; I < SRsrcIdx; ++I)
575 BaseOps.push_back(Elt: &LdSt.getOperand(i: I));
576 } else {
577 BaseOps.push_back(Elt: getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr));
578 }
579 Offset = 0;
580 // Get appropriate operand, and compute width accordingly.
581 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
582 if (DataOpIdx == -1)
583 return false; // no return sampler
584 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
585 return true;
586 }
587
588 if (isSMRD(MI: LdSt)) {
589 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::sbase);
590 if (!BaseOp) // e.g. S_MEMTIME
591 return false;
592 BaseOps.push_back(Elt: BaseOp);
593 OffsetOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
594 Offset = OffsetOp ? OffsetOp->getImm() : 0;
595 // Get appropriate operand, and compute width accordingly.
596 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::sdst);
597 if (DataOpIdx == -1)
598 return false;
599 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
600 return true;
601 }
602
603 if (isFLAT(MI: LdSt)) {
604 // Instructions have either vaddr or saddr or both or none.
605 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr);
606 if (BaseOp)
607 BaseOps.push_back(Elt: BaseOp);
608 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::saddr);
609 if (BaseOp)
610 BaseOps.push_back(Elt: BaseOp);
611 Offset = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset)->getImm();
612 // Get appropriate operand, and compute width accordingly.
613 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
614 if (DataOpIdx == -1)
615 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
616 if (DataOpIdx == -1) // LDS DMA
617 return false;
618 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
619 return true;
620 }
621
622 return false;
623}
624
625static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
626 ArrayRef<const MachineOperand *> BaseOps1,
627 const MachineInstr &MI2,
628 ArrayRef<const MachineOperand *> BaseOps2) {
629 // Only examine the first "base" operand of each instruction, on the
630 // assumption that it represents the real base address of the memory access.
631 // Other operands are typically offsets or indices from this base address.
632 if (BaseOps1.front()->isIdenticalTo(Other: *BaseOps2.front()))
633 return true;
634
635 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
636 return false;
637
638 auto *MO1 = *MI1.memoperands_begin();
639 auto *MO2 = *MI2.memoperands_begin();
640 if (MO1->getAddrSpace() != MO2->getAddrSpace())
641 return false;
642
643 const auto *Base1 = MO1->getValue();
644 const auto *Base2 = MO2->getValue();
645 if (!Base1 || !Base2)
646 return false;
647 Base1 = getUnderlyingObject(V: Base1);
648 Base2 = getUnderlyingObject(V: Base2);
649
650 if (isa<UndefValue>(Val: Base1) || isa<UndefValue>(Val: Base2))
651 return false;
652
653 return Base1 == Base2;
654}
655
656bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
657 int64_t Offset1, bool OffsetIsScalable1,
658 ArrayRef<const MachineOperand *> BaseOps2,
659 int64_t Offset2, bool OffsetIsScalable2,
660 unsigned ClusterSize,
661 unsigned NumBytes) const {
662 // If the mem ops (to be clustered) do not have the same base ptr, then they
663 // should not be clustered
664 unsigned MaxMemoryClusterDWords = DefaultMemoryClusterDWordsLimit;
665 if (!BaseOps1.empty() && !BaseOps2.empty()) {
666 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
667 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
668 if (!memOpsHaveSameBasePtr(MI1: FirstLdSt, BaseOps1, MI2: SecondLdSt, BaseOps2))
669 return false;
670
671 const SIMachineFunctionInfo *MFI =
672 FirstLdSt.getMF()->getInfo<SIMachineFunctionInfo>();
673 MaxMemoryClusterDWords = MFI->getMaxMemoryClusterDWords();
674 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
675 // If only one base op is empty, they do not have the same base ptr
676 return false;
677 }
678
679 // In order to avoid register pressure, on an average, the number of DWORDS
680 // loaded together by all clustered mem ops should not exceed
681 // MaxMemoryClusterDWords. This is an empirical value based on certain
682 // observations and performance related experiments.
683 // The good thing about this heuristic is - it avoids clustering of too many
684 // sub-word loads, and also avoids clustering of wide loads. Below is the
685 // brief summary of how the heuristic behaves for various `LoadSize` when
686 // MaxMemoryClusterDWords is 8.
687 //
688 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops
689 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops
690 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops
691 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops
692 // (5) LoadSize >= 17: do not cluster
693 const unsigned LoadSize = NumBytes / ClusterSize;
694 const unsigned NumDWords = ((LoadSize + 3) / 4) * ClusterSize;
695 return NumDWords <= MaxMemoryClusterDWords;
696}
697
698// FIXME: This behaves strangely. If, for example, you have 32 load + stores,
699// the first 16 loads will be interleaved with the stores, and the next 16 will
700// be clustered as expected. It should really split into 2 16 store batches.
701//
702// Loads are clustered until this returns false, rather than trying to schedule
703// groups of stores. This also means we have to deal with saying different
704// address space loads should be clustered, and ones which might cause bank
705// conflicts.
706//
707// This might be deprecated so it might not be worth that much effort to fix.
708bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1,
709 int64_t Offset0, int64_t Offset1,
710 unsigned NumLoads) const {
711 assert(Offset1 > Offset0 &&
712 "Second offset should be larger than first offset!");
713 // If we have less than 16 loads in a row, and the offsets are within 64
714 // bytes, then schedule together.
715
716 // A cacheline is 64 bytes (for global memory).
717 return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
718}
719
720static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB,
721 MachineBasicBlock::iterator MI,
722 const DebugLoc &DL, MCRegister DestReg,
723 MCRegister SrcReg, bool KillSrc,
724 const char *Msg = "illegal VGPR to SGPR copy") {
725 MachineFunction *MF = MBB.getParent();
726
727 LLVMContext &C = MF->getFunction().getContext();
728 C.diagnose(DI: DiagnosticInfoUnsupported(MF->getFunction(), Msg, DL, DS_Error));
729
730 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_ILLEGAL_COPY), DestReg)
731 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
732}
733
734/// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not
735/// possible to have a direct copy in these cases on GFX908, so an intermediate
736/// VGPR copy is required.
737static void indirectCopyToAGPR(const SIInstrInfo &TII, MachineBasicBlock &MBB,
738 MachineBasicBlock::iterator MI,
739 const DebugLoc &DL, MCRegister DestReg,
740 MCRegister SrcReg, bool KillSrc,
741 RegScavenger &RS, bool RegsOverlap,
742 Register ImpUseSuperReg = Register()) {
743 assert((TII.getSubtarget().hasMAIInsts() &&
744 !TII.getSubtarget().hasGFX90AInsts()) &&
745 "Expected GFX908 subtarget.");
746
747 assert((AMDGPU::SReg_32RegClass.contains(SrcReg) ||
748 AMDGPU::AGPR_32RegClass.contains(SrcReg)) &&
749 "Source register of the copy should be either an SGPR or an AGPR.");
750
751 assert(AMDGPU::AGPR_32RegClass.contains(DestReg) &&
752 "Destination register of the copy should be an AGPR.");
753
754 const SIRegisterInfo &RI = TII.getRegisterInfo();
755
756 // First try to find defining accvgpr_write to avoid temporary registers.
757 // In the case of copies of overlapping AGPRs, we conservatively do not
758 // reuse previous accvgpr_writes. Otherwise, we may incorrectly pick up
759 // an accvgpr_write used for this same copy due to implicit-defs
760 if (!RegsOverlap) {
761 for (auto Def = MI, E = MBB.begin(); Def != E; ) {
762 --Def;
763
764 if (!Def->modifiesRegister(Reg: SrcReg, TRI: &RI))
765 continue;
766
767 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
768 Def->getOperand(i: 0).getReg() != SrcReg)
769 break;
770
771 MachineOperand &DefOp = Def->getOperand(i: 1);
772 assert(DefOp.isReg() || DefOp.isImm());
773
774 if (DefOp.isReg()) {
775 bool SafeToPropagate = true;
776 // Check that register source operand is not clobbered before MI.
777 // Immediate operands are always safe to propagate.
778 for (auto I = Def; I != MI && SafeToPropagate; ++I)
779 if (I->modifiesRegister(Reg: DefOp.getReg(), TRI: &RI))
780 SafeToPropagate = false;
781
782 if (!SafeToPropagate)
783 break;
784
785 for (auto I = Def; I != MI; ++I)
786 I->clearRegisterKills(Reg: DefOp.getReg(), RegInfo: &RI);
787 }
788
789 MachineInstrBuilder Builder =
790 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64),
791 DestReg)
792 .add(MO: DefOp);
793
794 if (ImpUseSuperReg) {
795 Builder.addReg(RegNo: ImpUseSuperReg,
796 Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
797 }
798
799 return;
800 }
801 }
802
803 RS.enterBasicBlockEnd(MBB);
804 RS.backward(I: std::next(x: MI));
805
806 // Ideally we want to have three registers for a long reg_sequence copy
807 // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
808 unsigned MaxVGPRs = RI.getRegPressureLimit(RC: &AMDGPU::VGPR_32RegClass,
809 MF&: *MBB.getParent());
810
811 // Registers in the sequence are allocated contiguously so we can just
812 // use register number to pick one of three round-robin temps.
813 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3;
814 Register Tmp =
815 MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getVGPRForAGPRCopy();
816 assert(MBB.getParent()->getRegInfo().isReserved(Tmp) &&
817 "VGPR used for an intermediate copy should have been reserved.");
818
819 // Only loop through if there are any free registers left. We don't want to
820 // spill.
821 while (RegNo--) {
822 Register Tmp2 = RS.scavengeRegisterBackwards(RC: AMDGPU::VGPR_32RegClass, To: MI,
823 /* RestoreAfter */ false, SPAdj: 0,
824 /* AllowSpill */ false);
825 if (!Tmp2 || RI.getHWRegIndex(Reg: Tmp2) >= MaxVGPRs)
826 break;
827 Tmp = Tmp2;
828 RS.setRegUsed(Reg: Tmp);
829 }
830
831 // Insert copy to temporary VGPR.
832 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;
833 if (AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg)) {
834 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64;
835 } else {
836 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
837 }
838
839 MachineInstrBuilder UseBuilder = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: TmpCopyOp), DestReg: Tmp)
840 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
841 if (ImpUseSuperReg) {
842 UseBuilder.addReg(RegNo: ImpUseSuperReg,
843 Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
844 }
845
846 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
847 .addReg(RegNo: Tmp, Flags: RegState::Kill);
848}
849
850static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB,
851 MachineBasicBlock::iterator MI, const DebugLoc &DL,
852 MCRegister DestReg, MCRegister SrcReg, bool KillSrc,
853 const TargetRegisterClass *RC, bool Forward) {
854 const SIRegisterInfo &RI = TII.getRegisterInfo();
855 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, EltSize: 4);
856 MachineBasicBlock::iterator I = MI;
857 MachineInstr *FirstMI = nullptr, *LastMI = nullptr;
858
859 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) {
860 int16_t SubIdx = BaseIndices[Idx];
861 Register DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
862 Register SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
863 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
864 unsigned Opcode = AMDGPU::S_MOV_B32;
865
866 // Is SGPR aligned? If so try to combine with next.
867 bool AlignedDest = ((DestSubReg - AMDGPU::SGPR0) % 2) == 0;
868 bool AlignedSrc = ((SrcSubReg - AMDGPU::SGPR0) % 2) == 0;
869 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) {
870 // Can use SGPR64 copy
871 unsigned Channel = RI.getChannelFromSubReg(SubReg: SubIdx);
872 SubIdx = RI.getSubRegFromChannel(Channel, NumRegs: 2);
873 DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
874 SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
875 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
876 Opcode = AMDGPU::S_MOV_B64;
877 Idx++;
878 }
879
880 LastMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode), DestReg: DestSubReg)
881 .addReg(RegNo: SrcSubReg)
882 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
883
884 if (!FirstMI)
885 FirstMI = LastMI;
886
887 if (!Forward)
888 I--;
889 }
890
891 assert(FirstMI && LastMI);
892 if (!Forward)
893 std::swap(a&: FirstMI, b&: LastMI);
894
895 if (KillSrc)
896 LastMI->addRegisterKilled(IncomingReg: SrcReg, RegInfo: &RI);
897}
898
899void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
900 MachineBasicBlock::iterator MI,
901 const DebugLoc &DL, Register DestReg,
902 Register SrcReg, bool KillSrc, bool RenamableDest,
903 bool RenamableSrc) const {
904 const TargetRegisterClass *RC = RI.getPhysRegBaseClass(Reg: DestReg);
905 unsigned Size = RI.getRegSizeInBits(RC: *RC);
906 const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(Reg: SrcReg);
907 unsigned SrcSize = RI.getRegSizeInBits(RC: *SrcRC);
908
909 // The rest of copyPhysReg assumes Src and Dst size are the same size.
910 // TODO-GFX11_16BIT If all true 16 bit instruction patterns are completed can
911 // we remove Fix16BitCopies and this code block?
912 if (Fix16BitCopies) {
913 if (((Size == 16) != (SrcSize == 16))) {
914 // Non-VGPR Src and Dst will later be expanded back to 32 bits.
915 assert(ST.useRealTrue16Insts());
916 Register &RegToFix = (Size == 32) ? DestReg : SrcReg;
917 MCRegister SubReg = RI.getSubReg(Reg: RegToFix, Idx: AMDGPU::lo16);
918 RegToFix = SubReg;
919
920 if (DestReg == SrcReg) {
921 // Identity copy. Insert empty bundle since ExpandPostRA expects an
922 // instruction here.
923 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::BUNDLE));
924 return;
925 }
926 RC = RI.getPhysRegBaseClass(Reg: DestReg);
927 Size = RI.getRegSizeInBits(RC: *RC);
928 SrcRC = RI.getPhysRegBaseClass(Reg: SrcReg);
929 SrcSize = RI.getRegSizeInBits(RC: *SrcRC);
930 }
931 }
932
933 if (RC == &AMDGPU::VGPR_32RegClass) {
934 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
935 AMDGPU::SReg_32RegClass.contains(SrcReg) ||
936 AMDGPU::AGPR_32RegClass.contains(SrcReg));
937 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg) ?
938 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;
939 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
940 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
941 return;
942 }
943
944 if (RC == &AMDGPU::SReg_32_XM0RegClass ||
945 RC == &AMDGPU::SReg_32RegClass) {
946 if (SrcReg == AMDGPU::SCC) {
947 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B32), DestReg)
948 .addImm(Val: 1)
949 .addImm(Val: 0);
950 return;
951 }
952
953 if (!AMDGPU::SReg_32RegClass.contains(Reg: SrcReg)) {
954 if (DestReg == AMDGPU::VCC_LO) {
955 // FIXME: Hack until VReg_1 removed.
956 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
957 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CMP_NE_U32_e32))
958 .addImm(Val: 0)
959 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
960 return;
961 }
962
963 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
964 return;
965 }
966
967 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg)
968 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
969 return;
970 }
971
972 if (RC == &AMDGPU::SReg_64RegClass) {
973 if (SrcReg == AMDGPU::SCC) {
974 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B64), DestReg)
975 .addImm(Val: 1)
976 .addImm(Val: 0);
977 return;
978 }
979
980 if (!AMDGPU::SReg_64_EncodableRegClass.contains(Reg: SrcReg)) {
981 if (DestReg == AMDGPU::VCC) {
982 // FIXME: Hack until VReg_1 removed.
983 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
984 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CMP_NE_U32_e32))
985 .addImm(Val: 0)
986 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
987 return;
988 }
989
990 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
991 return;
992 }
993
994 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B64), DestReg)
995 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
996 return;
997 }
998
999 if (DestReg == AMDGPU::SCC) {
1000 // Copying 64-bit or 32-bit sources to SCC barely makes sense,
1001 // but SelectionDAG emits such copies for i1 sources.
1002 if (AMDGPU::SReg_64RegClass.contains(Reg: SrcReg)) {
1003 // This copy can only be produced by patterns
1004 // with explicit SCC, which are known to be enabled
1005 // only for subtargets with S_CMP_LG_U64 present.
1006 assert(ST.hasScalarCompareEq64());
1007 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U64))
1008 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
1009 .addImm(Val: 0);
1010 } else {
1011 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
1012 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U32))
1013 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
1014 .addImm(Val: 0);
1015 }
1016
1017 return;
1018 }
1019
1020 if (RC == &AMDGPU::AGPR_32RegClass) {
1021 if (AMDGPU::VGPR_32RegClass.contains(Reg: SrcReg) ||
1022 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(Reg: SrcReg))) {
1023 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
1024 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1025 return;
1026 }
1027
1028 if (AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg) && ST.hasGFX90AInsts()) {
1029 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_MOV_B32), DestReg)
1030 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1031 return;
1032 }
1033
1034 // FIXME: Pass should maintain scavenger to avoid scan through the block on
1035 // every AGPR spill.
1036 RegScavenger RS;
1037 const bool Overlap = RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1038 indirectCopyToAGPR(TII: *this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS, RegsOverlap: Overlap);
1039 return;
1040 }
1041
1042 if (Size == 16) {
1043 assert(AMDGPU::VGPR_16RegClass.contains(SrcReg) ||
1044 AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
1045 AMDGPU::AGPR_LO16RegClass.contains(SrcReg));
1046
1047 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(Reg: DestReg);
1048 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(Reg: SrcReg);
1049 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(Reg: DestReg);
1050 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(Reg: SrcReg);
1051 bool DstLow = !AMDGPU::isHi16Reg(Reg: DestReg, MRI: RI);
1052 bool SrcLow = !AMDGPU::isHi16Reg(Reg: SrcReg, MRI: RI);
1053 MCRegister NewDestReg = RI.get32BitRegister(Reg: DestReg);
1054 MCRegister NewSrcReg = RI.get32BitRegister(Reg: SrcReg);
1055
1056 if (IsSGPRDst) {
1057 if (!IsSGPRSrc) {
1058 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1059 return;
1060 }
1061
1062 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: NewDestReg)
1063 .addReg(RegNo: NewSrcReg, Flags: getKillRegState(B: KillSrc));
1064 return;
1065 }
1066
1067 if (IsAGPRDst || IsAGPRSrc) {
1068 if (!DstLow || !SrcLow) {
1069 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
1070 Msg: "Cannot use hi16 subreg with an AGPR!");
1071 }
1072
1073 copyPhysReg(MBB, MI, DL, DestReg: NewDestReg, SrcReg: NewSrcReg, KillSrc);
1074 return;
1075 }
1076
1077 if (ST.useRealTrue16Insts()) {
1078 if (IsSGPRSrc) {
1079 assert(SrcLow);
1080 SrcReg = NewSrcReg;
1081 }
1082 // Use the smaller instruction encoding if possible.
1083 if (AMDGPU::VGPR_16_Lo128RegClass.contains(Reg: DestReg) &&
1084 (IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.contains(Reg: SrcReg))) {
1085 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B16_t16_e32), DestReg)
1086 .addReg(RegNo: SrcReg);
1087 } else {
1088 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B16_t16_e64), DestReg)
1089 .addImm(Val: 0) // src0_modifiers
1090 .addReg(RegNo: SrcReg)
1091 .addImm(Val: 0); // op_sel
1092 }
1093 return;
1094 }
1095
1096 if (IsSGPRSrc && !ST.hasSDWAScalar()) {
1097 if (!DstLow || !SrcLow) {
1098 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
1099 Msg: "Cannot use hi16 subreg on VI!");
1100 }
1101
1102 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: NewDestReg)
1103 .addReg(RegNo: NewSrcReg, Flags: getKillRegState(B: KillSrc));
1104 return;
1105 }
1106
1107 auto MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_sdwa), DestReg: NewDestReg)
1108 .addImm(Val: 0) // src0_modifiers
1109 .addReg(RegNo: NewSrcReg)
1110 .addImm(Val: 0) // clamp
1111 .addImm(Val: DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1112 : AMDGPU::SDWA::SdwaSel::WORD_1)
1113 .addImm(Val: AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE)
1114 .addImm(Val: SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1115 : AMDGPU::SDWA::SdwaSel::WORD_1)
1116 .addReg(RegNo: NewDestReg, Flags: RegState::Implicit | RegState::Undef);
1117 // First implicit operand is $exec.
1118 MIB->tieOperands(DefIdx: 0, UseIdx: MIB->getNumOperands() - 1);
1119 return;
1120 }
1121
1122 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(RC: SrcRC))) {
1123 if (ST.hasVMovB64Inst()) {
1124 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B64_e32), DestReg)
1125 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1126 return;
1127 }
1128 if (ST.hasPkMovB32()) {
1129 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg)
1130 .addImm(Val: SISrcMods::OP_SEL_1)
1131 .addReg(RegNo: SrcReg)
1132 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
1133 .addReg(RegNo: SrcReg)
1134 .addImm(Val: 0) // op_sel_lo
1135 .addImm(Val: 0) // op_sel_hi
1136 .addImm(Val: 0) // neg_lo
1137 .addImm(Val: 0) // neg_hi
1138 .addImm(Val: 0) // clamp
1139 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
1140 return;
1141 }
1142 }
1143
1144 const bool Forward = RI.getHWRegIndex(Reg: DestReg) <= RI.getHWRegIndex(Reg: SrcReg);
1145 if (RI.isSGPRClass(RC)) {
1146 if (!RI.isSGPRClass(RC: SrcRC)) {
1147 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1148 return;
1149 }
1150 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1151 expandSGPRCopy(TII: *this, MBB, MI, DL, DestReg, SrcReg, KillSrc: CanKillSuperReg, RC,
1152 Forward);
1153 return;
1154 }
1155
1156 unsigned EltSize = 4;
1157 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1158 if (RI.isAGPRClass(RC)) {
1159 if (ST.hasGFX90AInsts() && RI.isAGPRClass(RC: SrcRC))
1160 Opcode = AMDGPU::V_ACCVGPR_MOV_B32;
1161 else if (RI.hasVGPRs(RC: SrcRC) ||
1162 (ST.hasGFX90AInsts() && RI.isSGPRClass(RC: SrcRC)))
1163 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
1164 else
1165 Opcode = AMDGPU::INSTRUCTION_LIST_END;
1166 } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(RC: SrcRC)) {
1167 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;
1168 } else if ((Size % 64 == 0) && RI.hasVGPRs(RC) &&
1169 (RI.isProperlyAlignedRC(RC: *RC) &&
1170 (SrcRC == RC || RI.isSGPRClass(RC: SrcRC)))) {
1171 // TODO: In 96-bit case, could do a 64-bit mov and then a 32-bit mov.
1172 if (ST.hasVMovB64Inst()) {
1173 Opcode = AMDGPU::V_MOV_B64_e32;
1174 EltSize = 8;
1175 } else if (ST.hasPkMovB32()) {
1176 Opcode = AMDGPU::V_PK_MOV_B32;
1177 EltSize = 8;
1178 }
1179 }
1180
1181 // For the cases where we need an intermediate instruction/temporary register
1182 // (destination is an AGPR), we need a scavenger.
1183 //
1184 // FIXME: The pass should maintain this for us so we don't have to re-scan the
1185 // whole block for every handled copy.
1186 std::unique_ptr<RegScavenger> RS;
1187 if (Opcode == AMDGPU::INSTRUCTION_LIST_END)
1188 RS = std::make_unique<RegScavenger>();
1189
1190 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
1191
1192 // If there is an overlap, we can't kill the super-register on the last
1193 // instruction, since it will also kill the components made live by this def.
1194 const bool Overlap = RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1195 const bool CanKillSuperReg = KillSrc && !Overlap;
1196
1197 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
1198 unsigned SubIdx;
1199 if (Forward)
1200 SubIdx = SubIndices[Idx];
1201 else
1202 SubIdx = SubIndices[SubIndices.size() - Idx - 1];
1203 Register DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
1204 Register SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
1205 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
1206
1207 bool UseKill = CanKillSuperReg && Idx == SubIndices.size() - 1;
1208
1209 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) {
1210 Register ImpUseSuper = SrcReg;
1211 indirectCopyToAGPR(TII: *this, MBB, MI, DL, DestReg: DestSubReg, SrcReg: SrcSubReg, KillSrc: UseKill,
1212 RS&: *RS, RegsOverlap: Overlap, ImpUseSuperReg: ImpUseSuper);
1213 } else if (Opcode == AMDGPU::V_PK_MOV_B32) {
1214 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg: DestSubReg)
1215 .addImm(Val: SISrcMods::OP_SEL_1)
1216 .addReg(RegNo: SrcSubReg)
1217 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
1218 .addReg(RegNo: SrcSubReg)
1219 .addImm(Val: 0) // op_sel_lo
1220 .addImm(Val: 0) // op_sel_hi
1221 .addImm(Val: 0) // neg_lo
1222 .addImm(Val: 0) // neg_hi
1223 .addImm(Val: 0) // clamp
1224 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: UseKill) | RegState::Implicit);
1225 } else {
1226 MachineInstrBuilder Builder =
1227 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode), DestReg: DestSubReg).addReg(RegNo: SrcSubReg);
1228
1229 Builder.addReg(RegNo: SrcReg, Flags: getKillRegState(B: UseKill) | RegState::Implicit);
1230 }
1231 }
1232}
1233
1234int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
1235 int32_t NewOpc;
1236
1237 // Try to map original to commuted opcode
1238 NewOpc = AMDGPU::getCommuteRev(Opcode);
1239 if (NewOpc != -1)
1240 // Check if the commuted (REV) opcode exists on the target.
1241 return pseudoToMCOpcode(Opcode: NewOpc) != -1 ? NewOpc : -1;
1242
1243 // Try to map commuted to original opcode
1244 NewOpc = AMDGPU::getCommuteOrig(Opcode);
1245 if (NewOpc != -1)
1246 // Check if the original (non-REV) opcode exists on the target.
1247 return pseudoToMCOpcode(Opcode: NewOpc) != -1 ? NewOpc : -1;
1248
1249 return Opcode;
1250}
1251
1252bool SIInstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
1253 const Register Reg,
1254 int64_t &ImmVal) const {
1255 switch (MI.getOpcode()) {
1256 case AMDGPU::V_MOV_B32_e32:
1257 case AMDGPU::S_MOV_B32:
1258 case AMDGPU::S_MOVK_I32:
1259 case AMDGPU::S_MOV_B64:
1260 case AMDGPU::V_MOV_B64_e32:
1261 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
1262 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
1263 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
1264 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
1265 case AMDGPU::V_MOV_B64_PSEUDO:
1266 case AMDGPU::V_MOV_B16_t16_e32: {
1267 const MachineOperand &Src0 = MI.getOperand(i: 1);
1268 if (Src0.isImm()) {
1269 ImmVal = Src0.getImm();
1270 return MI.getOperand(i: 0).getReg() == Reg;
1271 }
1272
1273 return false;
1274 }
1275 case AMDGPU::V_MOV_B16_t16_e64: {
1276 const MachineOperand &Src0 = MI.getOperand(i: 2);
1277 if (Src0.isImm() && !MI.getOperand(i: 1).getImm()) {
1278 ImmVal = Src0.getImm();
1279 return MI.getOperand(i: 0).getReg() == Reg;
1280 }
1281
1282 return false;
1283 }
1284 case AMDGPU::S_BREV_B32:
1285 case AMDGPU::V_BFREV_B32_e32:
1286 case AMDGPU::V_BFREV_B32_e64: {
1287 const MachineOperand &Src0 = MI.getOperand(i: 1);
1288 if (Src0.isImm()) {
1289 ImmVal = static_cast<int64_t>(reverseBits<int32_t>(Val: Src0.getImm()));
1290 return MI.getOperand(i: 0).getReg() == Reg;
1291 }
1292
1293 return false;
1294 }
1295 case AMDGPU::S_NOT_B32:
1296 case AMDGPU::V_NOT_B32_e32:
1297 case AMDGPU::V_NOT_B32_e64: {
1298 const MachineOperand &Src0 = MI.getOperand(i: 1);
1299 if (Src0.isImm()) {
1300 ImmVal = static_cast<int64_t>(~static_cast<int32_t>(Src0.getImm()));
1301 return MI.getOperand(i: 0).getReg() == Reg;
1302 }
1303
1304 return false;
1305 }
1306 default:
1307 return false;
1308 }
1309}
1310
1311std::optional<int64_t>
1312SIInstrInfo::getImmOrMaterializedImm(const MachineRegisterInfo &MRI,
1313 const MachineOperand &Op,
1314 MachineInstr **DefMI) const {
1315 if (DefMI)
1316 *DefMI = nullptr;
1317
1318 if (Op.isImm())
1319 return Op.getImm();
1320
1321 if (!Op.isReg() || !Op.getReg().isVirtual())
1322 return std::nullopt;
1323 MachineInstr *Def = MRI.getUniqueVRegDef(Reg: Op.getReg());
1324 if (Def && Def->isMoveImmediate()) {
1325 const MachineOperand &ImmSrc = Def->getOperand(i: 1);
1326 if (ImmSrc.isImm()) {
1327 if (DefMI)
1328 *DefMI = Def;
1329 return extractSubregFromImm(ImmVal: ImmSrc.getImm(), SubRegIndex: Op.getSubReg());
1330 }
1331 }
1332
1333 return std::nullopt;
1334}
1335
1336std::optional<int64_t>
1337SIInstrInfo::getImmOrMaterializedImm(const MachineRegisterInfo &MRI,
1338 Register Reg, MachineInstr **DefMI) const {
1339 MachineOperand Op = MachineOperand::CreateReg(Reg, isDef: false);
1340 return getImmOrMaterializedImm(MRI, Op, DefMI);
1341}
1342
1343unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
1344
1345 if (RI.isAGPRClass(RC: DstRC))
1346 return AMDGPU::COPY;
1347 if (RI.getRegSizeInBits(RC: *DstRC) == 16) {
1348 // Assume hi bits are unneeded. Only _e64 true16 instructions are legal
1349 // before RA.
1350 return RI.isSGPRClass(RC: DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64;
1351 }
1352 if (RI.getRegSizeInBits(RC: *DstRC) == 32)
1353 return RI.isSGPRClass(RC: DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1354 if (RI.getRegSizeInBits(RC: *DstRC) == 64 && RI.isSGPRClass(RC: DstRC))
1355 return AMDGPU::S_MOV_B64;
1356 if (RI.getRegSizeInBits(RC: *DstRC) == 64 && !RI.isSGPRClass(RC: DstRC))
1357 return AMDGPU::V_MOV_B64_PSEUDO;
1358 return AMDGPU::COPY;
1359}
1360
1361const MCInstrDesc &
1362SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize,
1363 bool IsIndirectSrc) const {
1364 if (IsIndirectSrc) {
1365 if (VecSize <= 32) // 4 bytes
1366 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);
1367 if (VecSize <= 64) // 8 bytes
1368 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);
1369 if (VecSize <= 96) // 12 bytes
1370 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);
1371 if (VecSize <= 128) // 16 bytes
1372 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);
1373 if (VecSize <= 160) // 20 bytes
1374 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);
1375 if (VecSize <= 192) // 24 bytes
1376 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V6);
1377 if (VecSize <= 224) // 28 bytes
1378 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V7);
1379 if (VecSize <= 256) // 32 bytes
1380 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);
1381 if (VecSize <= 288) // 36 bytes
1382 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9);
1383 if (VecSize <= 320) // 40 bytes
1384 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10);
1385 if (VecSize <= 352) // 44 bytes
1386 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11);
1387 if (VecSize <= 384) // 48 bytes
1388 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12);
1389 if (VecSize <= 512) // 64 bytes
1390 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);
1391 if (VecSize <= 1024) // 128 bytes
1392 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);
1393
1394 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos");
1395 }
1396
1397 if (VecSize <= 32) // 4 bytes
1398 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);
1399 if (VecSize <= 64) // 8 bytes
1400 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);
1401 if (VecSize <= 96) // 12 bytes
1402 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);
1403 if (VecSize <= 128) // 16 bytes
1404 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);
1405 if (VecSize <= 160) // 20 bytes
1406 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);
1407 if (VecSize <= 192) // 24 bytes
1408 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V6);
1409 if (VecSize <= 224) // 28 bytes
1410 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V7);
1411 if (VecSize <= 256) // 32 bytes
1412 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);
1413 if (VecSize <= 288) // 36 bytes
1414 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9);
1415 if (VecSize <= 320) // 40 bytes
1416 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10);
1417 if (VecSize <= 352) // 44 bytes
1418 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11);
1419 if (VecSize <= 384) // 48 bytes
1420 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12);
1421 if (VecSize <= 512) // 64 bytes
1422 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);
1423 if (VecSize <= 1024) // 128 bytes
1424 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);
1425
1426 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos");
1427}
1428
1429static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) {
1430 if (VecSize <= 32) // 4 bytes
1431 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1432 if (VecSize <= 64) // 8 bytes
1433 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1434 if (VecSize <= 96) // 12 bytes
1435 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1436 if (VecSize <= 128) // 16 bytes
1437 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1438 if (VecSize <= 160) // 20 bytes
1439 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1440 if (VecSize <= 192) // 24 bytes
1441 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V6;
1442 if (VecSize <= 224) // 28 bytes
1443 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V7;
1444 if (VecSize <= 256) // 32 bytes
1445 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1446 if (VecSize <= 288) // 36 bytes
1447 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1448 if (VecSize <= 320) // 40 bytes
1449 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1450 if (VecSize <= 352) // 44 bytes
1451 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1452 if (VecSize <= 384) // 48 bytes
1453 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1454 if (VecSize <= 512) // 64 bytes
1455 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1456 if (VecSize <= 1024) // 128 bytes
1457 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1458
1459 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1460}
1461
1462static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) {
1463 if (VecSize <= 32) // 4 bytes
1464 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1465 if (VecSize <= 64) // 8 bytes
1466 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1467 if (VecSize <= 96) // 12 bytes
1468 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1469 if (VecSize <= 128) // 16 bytes
1470 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1471 if (VecSize <= 160) // 20 bytes
1472 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1473 if (VecSize <= 192) // 24 bytes
1474 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V6;
1475 if (VecSize <= 224) // 28 bytes
1476 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V7;
1477 if (VecSize <= 256) // 32 bytes
1478 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1479 if (VecSize <= 288) // 36 bytes
1480 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1481 if (VecSize <= 320) // 40 bytes
1482 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1483 if (VecSize <= 352) // 44 bytes
1484 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1485 if (VecSize <= 384) // 48 bytes
1486 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1487 if (VecSize <= 512) // 64 bytes
1488 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1489 if (VecSize <= 1024) // 128 bytes
1490 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1491
1492 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1493}
1494
1495static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) {
1496 if (VecSize <= 64) // 8 bytes
1497 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;
1498 if (VecSize <= 128) // 16 bytes
1499 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;
1500 if (VecSize <= 256) // 32 bytes
1501 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;
1502 if (VecSize <= 512) // 64 bytes
1503 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;
1504 if (VecSize <= 1024) // 128 bytes
1505 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;
1506
1507 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1508}
1509
1510const MCInstrDesc &
1511SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize,
1512 bool IsSGPR) const {
1513 if (IsSGPR) {
1514 switch (EltSize) {
1515 case 32:
1516 return get(Opcode: getIndirectSGPRWriteMovRelPseudo32(VecSize));
1517 case 64:
1518 return get(Opcode: getIndirectSGPRWriteMovRelPseudo64(VecSize));
1519 default:
1520 llvm_unreachable("invalid reg indexing elt size");
1521 }
1522 }
1523
1524 assert(EltSize == 32 && "invalid reg indexing elt size");
1525 return get(Opcode: getIndirectVGPRWriteMovRelPseudoOpc(VecSize));
1526}
1527
1528static unsigned getSGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1529 switch (Size) {
1530 case 4:
1531 return NeedsCFI ? AMDGPU::SI_SPILL_S32_CFI_SAVE : AMDGPU::SI_SPILL_S32_SAVE;
1532 case 8:
1533 return NeedsCFI ? AMDGPU::SI_SPILL_S64_CFI_SAVE : AMDGPU::SI_SPILL_S64_SAVE;
1534 case 12:
1535 return NeedsCFI ? AMDGPU::SI_SPILL_S96_CFI_SAVE : AMDGPU::SI_SPILL_S96_SAVE;
1536 case 16:
1537 return NeedsCFI ? AMDGPU::SI_SPILL_S128_CFI_SAVE
1538 : AMDGPU::SI_SPILL_S128_SAVE;
1539 case 20:
1540 return NeedsCFI ? AMDGPU::SI_SPILL_S160_CFI_SAVE
1541 : AMDGPU::SI_SPILL_S160_SAVE;
1542 case 24:
1543 return NeedsCFI ? AMDGPU::SI_SPILL_S192_CFI_SAVE
1544 : AMDGPU::SI_SPILL_S192_SAVE;
1545 case 28:
1546 return NeedsCFI ? AMDGPU::SI_SPILL_S224_CFI_SAVE
1547 : AMDGPU::SI_SPILL_S224_SAVE;
1548 case 32:
1549 return AMDGPU::SI_SPILL_S256_SAVE;
1550 case 36:
1551 return AMDGPU::SI_SPILL_S288_SAVE;
1552 case 40:
1553 return AMDGPU::SI_SPILL_S320_SAVE;
1554 case 44:
1555 return AMDGPU::SI_SPILL_S352_SAVE;
1556 case 48:
1557 return AMDGPU::SI_SPILL_S384_SAVE;
1558 case 64:
1559 return NeedsCFI ? AMDGPU::SI_SPILL_S512_CFI_SAVE
1560 : AMDGPU::SI_SPILL_S512_SAVE;
1561 case 128:
1562 return NeedsCFI ? AMDGPU::SI_SPILL_S1024_CFI_SAVE
1563 : AMDGPU::SI_SPILL_S1024_SAVE;
1564 default:
1565 llvm_unreachable("unknown register size");
1566 }
1567}
1568
1569static unsigned getVGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1570 switch (Size) {
1571 case 2:
1572 return AMDGPU::SI_SPILL_V16_SAVE;
1573 case 4:
1574 return NeedsCFI ? AMDGPU::SI_SPILL_V32_CFI_SAVE : AMDGPU::SI_SPILL_V32_SAVE;
1575 case 8:
1576 return NeedsCFI ? AMDGPU::SI_SPILL_V64_CFI_SAVE : AMDGPU::SI_SPILL_V64_SAVE;
1577 case 12:
1578 return NeedsCFI ? AMDGPU::SI_SPILL_V96_CFI_SAVE : AMDGPU::SI_SPILL_V96_SAVE;
1579 case 16:
1580 return NeedsCFI ? AMDGPU::SI_SPILL_V128_CFI_SAVE
1581 : AMDGPU::SI_SPILL_V128_SAVE;
1582 case 20:
1583 return NeedsCFI ? AMDGPU::SI_SPILL_V160_CFI_SAVE
1584 : AMDGPU::SI_SPILL_V160_SAVE;
1585 case 24:
1586 return NeedsCFI ? AMDGPU::SI_SPILL_V192_CFI_SAVE
1587 : AMDGPU::SI_SPILL_V192_SAVE;
1588 case 28:
1589 return NeedsCFI ? AMDGPU::SI_SPILL_V224_CFI_SAVE
1590 : AMDGPU::SI_SPILL_V224_SAVE;
1591 case 32:
1592 return NeedsCFI ? AMDGPU::SI_SPILL_V256_CFI_SAVE
1593 : AMDGPU::SI_SPILL_V256_SAVE;
1594 case 36:
1595 return NeedsCFI ? AMDGPU::SI_SPILL_V288_CFI_SAVE
1596 : AMDGPU::SI_SPILL_V288_SAVE;
1597 case 40:
1598 return NeedsCFI ? AMDGPU::SI_SPILL_V320_CFI_SAVE
1599 : AMDGPU::SI_SPILL_V320_SAVE;
1600 case 44:
1601 return NeedsCFI ? AMDGPU::SI_SPILL_V352_CFI_SAVE
1602 : AMDGPU::SI_SPILL_V352_SAVE;
1603 case 48:
1604 return NeedsCFI ? AMDGPU::SI_SPILL_V384_CFI_SAVE
1605 : AMDGPU::SI_SPILL_V384_SAVE;
1606 case 64:
1607 return NeedsCFI ? AMDGPU::SI_SPILL_V512_CFI_SAVE
1608 : AMDGPU::SI_SPILL_V512_SAVE;
1609 case 128:
1610 return NeedsCFI ? AMDGPU::SI_SPILL_V1024_CFI_SAVE
1611 : AMDGPU::SI_SPILL_V1024_SAVE;
1612 default:
1613 llvm_unreachable("unknown register size");
1614 }
1615}
1616
1617static unsigned getAVSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1618 switch (Size) {
1619 case 4:
1620 return NeedsCFI ? AMDGPU::SI_SPILL_AV32_CFI_SAVE
1621 : AMDGPU::SI_SPILL_AV32_SAVE;
1622 case 8:
1623 return NeedsCFI ? AMDGPU::SI_SPILL_AV64_CFI_SAVE
1624 : AMDGPU::SI_SPILL_AV64_SAVE;
1625 case 12:
1626 return NeedsCFI ? AMDGPU::SI_SPILL_AV96_CFI_SAVE
1627 : AMDGPU::SI_SPILL_AV96_SAVE;
1628 case 16:
1629 return NeedsCFI ? AMDGPU::SI_SPILL_AV128_CFI_SAVE
1630 : AMDGPU::SI_SPILL_AV128_SAVE;
1631 case 20:
1632 return NeedsCFI ? AMDGPU::SI_SPILL_AV160_CFI_SAVE
1633 : AMDGPU::SI_SPILL_AV160_SAVE;
1634 case 24:
1635 return NeedsCFI ? AMDGPU::SI_SPILL_AV192_CFI_SAVE
1636 : AMDGPU::SI_SPILL_AV192_SAVE;
1637 case 28:
1638 return NeedsCFI ? AMDGPU::SI_SPILL_AV224_CFI_SAVE
1639 : AMDGPU::SI_SPILL_AV224_SAVE;
1640 case 32:
1641 return NeedsCFI ? AMDGPU::SI_SPILL_AV256_CFI_SAVE
1642 : AMDGPU::SI_SPILL_AV256_SAVE;
1643 case 36:
1644 return AMDGPU::SI_SPILL_AV288_SAVE;
1645 case 40:
1646 return AMDGPU::SI_SPILL_AV320_SAVE;
1647 case 44:
1648 return AMDGPU::SI_SPILL_AV352_SAVE;
1649 case 48:
1650 return AMDGPU::SI_SPILL_AV384_SAVE;
1651 case 64:
1652 return NeedsCFI ? AMDGPU::SI_SPILL_AV512_CFI_SAVE
1653 : AMDGPU::SI_SPILL_AV512_SAVE;
1654 case 128:
1655 return NeedsCFI ? AMDGPU::SI_SPILL_AV1024_CFI_SAVE
1656 : AMDGPU::SI_SPILL_AV1024_SAVE;
1657 default:
1658 llvm_unreachable("unknown register size");
1659 }
1660}
1661
1662static unsigned getWWMRegSpillSaveOpcode(unsigned Size,
1663 bool IsVectorSuperClass) {
1664 // Currently, there is only 32-bit WWM register spills needed.
1665 if (Size != 4)
1666 llvm_unreachable("unknown wwm register spill size");
1667
1668 if (IsVectorSuperClass)
1669 return AMDGPU::SI_SPILL_WWM_AV32_SAVE;
1670
1671 return AMDGPU::SI_SPILL_WWM_V32_SAVE;
1672}
1673
1674unsigned SIInstrInfo::getVectorRegSpillSaveOpcode(
1675 Register Reg, const TargetRegisterClass *RC, unsigned Size,
1676 const SIMachineFunctionInfo &MFI, bool NeedsCFI) const {
1677 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
1678
1679 // Choose the right opcode if spilling a WWM register.
1680 if (MFI.checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
1681 return getWWMRegSpillSaveOpcode(Size, IsVectorSuperClass);
1682
1683 // TODO: Check if AGPRs are available
1684 if (ST.hasMAIInsts())
1685 return getAVSpillSaveOpcode(Size, NeedsCFI);
1686
1687 return getVGPRSpillSaveOpcode(Size, NeedsCFI);
1688}
1689
1690void SIInstrInfo::storeRegToStackSlotImpl(
1691 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
1692 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
1693 MachineInstr::MIFlag Flags, bool NeedsCFI) const {
1694 MachineFunction *MF = MBB.getParent();
1695 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1696 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1697 const DebugLoc &DL = MBB.findDebugLoc(MBBI: MI);
1698
1699 MachinePointerInfo PtrInfo
1700 = MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
1701 MachineMemOperand *MMO = MF->getMachineMemOperand(
1702 PtrInfo, F: MachineMemOperand::MOStore, Size: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
1703 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
1704 unsigned SpillSize = RI.getSpillSize(RC: *RC);
1705
1706 MachineRegisterInfo &MRI = MF->getRegInfo();
1707 if (RI.isSGPRClass(RC)) {
1708 if (FrameInfo.getStackID(ObjectIdx: FrameIndex) == TargetStackID::SGPRSpill)
1709 MFI->setHasSpilledSGPRs();
1710 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled");
1711 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI &&
1712 SrcReg != AMDGPU::EXEC && "exec should not be spilled");
1713
1714 // We are only allowed to create one new instruction when spilling
1715 // registers, so we need to use pseudo instruction for spilling SGPRs.
1716 const MCInstrDesc &OpDesc =
1717 get(Opcode: getSGPRSpillSaveOpcode(Size: SpillSize, NeedsCFI));
1718
1719 // The SGPR spill/restore instructions only work on number sgprs, so we need
1720 // to make sure we are using the correct register class.
1721 if (SrcReg.isVirtual() && SpillSize == 4) {
1722 MRI.constrainRegClass(Reg: SrcReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
1723 }
1724
1725 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: OpDesc)
1726 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill)) // data
1727 .addFrameIndex(Idx: FrameIndex) // addr
1728 .addMemOperand(MMO)
1729 .addReg(RegNo: MFI->getStackPtrOffsetReg(), Flags: RegState::Implicit);
1730
1731 return;
1732 }
1733
1734 unsigned Opcode = getVectorRegSpillSaveOpcode(Reg: VReg ? VReg : SrcReg, RC,
1735 Size: SpillSize, MFI: *MFI, NeedsCFI);
1736 MFI->setHasSpilledVGPRs();
1737
1738 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode))
1739 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill)) // data
1740 .addFrameIndex(Idx: FrameIndex) // addr
1741 .addReg(RegNo: MFI->getStackPtrOffsetReg()) // scratch_offset
1742 .addImm(Val: 0) // offset
1743 .addMemOperand(MMO);
1744}
1745
1746void SIInstrInfo::storeRegToStackSlot(
1747 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
1748 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
1749 MachineInstr::MIFlag Flags) const {
1750 storeRegToStackSlotImpl(MBB, MI, SrcReg, isKill, FrameIndex, RC, VReg, Flags,
1751 NeedsCFI: false);
1752}
1753
1754void SIInstrInfo::storeRegToStackSlotCFI(MachineBasicBlock &MBB,
1755 MachineBasicBlock::iterator MI,
1756 Register SrcReg, bool isKill,
1757 int FrameIndex,
1758 const TargetRegisterClass *RC) const {
1759 storeRegToStackSlotImpl(MBB, MI, SrcReg, isKill, FrameIndex, RC, VReg: Register(),
1760 Flags: MachineInstr::NoFlags, NeedsCFI: true);
1761}
1762
1763static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1764 switch (Size) {
1765 case 4:
1766 return AMDGPU::SI_SPILL_S32_RESTORE;
1767 case 8:
1768 return AMDGPU::SI_SPILL_S64_RESTORE;
1769 case 12:
1770 return AMDGPU::SI_SPILL_S96_RESTORE;
1771 case 16:
1772 return AMDGPU::SI_SPILL_S128_RESTORE;
1773 case 20:
1774 return AMDGPU::SI_SPILL_S160_RESTORE;
1775 case 24:
1776 return AMDGPU::SI_SPILL_S192_RESTORE;
1777 case 28:
1778 return AMDGPU::SI_SPILL_S224_RESTORE;
1779 case 32:
1780 return AMDGPU::SI_SPILL_S256_RESTORE;
1781 case 36:
1782 return AMDGPU::SI_SPILL_S288_RESTORE;
1783 case 40:
1784 return AMDGPU::SI_SPILL_S320_RESTORE;
1785 case 44:
1786 return AMDGPU::SI_SPILL_S352_RESTORE;
1787 case 48:
1788 return AMDGPU::SI_SPILL_S384_RESTORE;
1789 case 64:
1790 return AMDGPU::SI_SPILL_S512_RESTORE;
1791 case 128:
1792 return AMDGPU::SI_SPILL_S1024_RESTORE;
1793 default:
1794 llvm_unreachable("unknown register size");
1795 }
1796}
1797
1798static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1799 switch (Size) {
1800 case 2:
1801 return AMDGPU::SI_SPILL_V16_RESTORE;
1802 case 4:
1803 return AMDGPU::SI_SPILL_V32_RESTORE;
1804 case 8:
1805 return AMDGPU::SI_SPILL_V64_RESTORE;
1806 case 12:
1807 return AMDGPU::SI_SPILL_V96_RESTORE;
1808 case 16:
1809 return AMDGPU::SI_SPILL_V128_RESTORE;
1810 case 20:
1811 return AMDGPU::SI_SPILL_V160_RESTORE;
1812 case 24:
1813 return AMDGPU::SI_SPILL_V192_RESTORE;
1814 case 28:
1815 return AMDGPU::SI_SPILL_V224_RESTORE;
1816 case 32:
1817 return AMDGPU::SI_SPILL_V256_RESTORE;
1818 case 36:
1819 return AMDGPU::SI_SPILL_V288_RESTORE;
1820 case 40:
1821 return AMDGPU::SI_SPILL_V320_RESTORE;
1822 case 44:
1823 return AMDGPU::SI_SPILL_V352_RESTORE;
1824 case 48:
1825 return AMDGPU::SI_SPILL_V384_RESTORE;
1826 case 64:
1827 return AMDGPU::SI_SPILL_V512_RESTORE;
1828 case 128:
1829 return AMDGPU::SI_SPILL_V1024_RESTORE;
1830 default:
1831 llvm_unreachable("unknown register size");
1832 }
1833}
1834
1835static unsigned getAVSpillRestoreOpcode(unsigned Size) {
1836 switch (Size) {
1837 case 4:
1838 return AMDGPU::SI_SPILL_AV32_RESTORE;
1839 case 8:
1840 return AMDGPU::SI_SPILL_AV64_RESTORE;
1841 case 12:
1842 return AMDGPU::SI_SPILL_AV96_RESTORE;
1843 case 16:
1844 return AMDGPU::SI_SPILL_AV128_RESTORE;
1845 case 20:
1846 return AMDGPU::SI_SPILL_AV160_RESTORE;
1847 case 24:
1848 return AMDGPU::SI_SPILL_AV192_RESTORE;
1849 case 28:
1850 return AMDGPU::SI_SPILL_AV224_RESTORE;
1851 case 32:
1852 return AMDGPU::SI_SPILL_AV256_RESTORE;
1853 case 36:
1854 return AMDGPU::SI_SPILL_AV288_RESTORE;
1855 case 40:
1856 return AMDGPU::SI_SPILL_AV320_RESTORE;
1857 case 44:
1858 return AMDGPU::SI_SPILL_AV352_RESTORE;
1859 case 48:
1860 return AMDGPU::SI_SPILL_AV384_RESTORE;
1861 case 64:
1862 return AMDGPU::SI_SPILL_AV512_RESTORE;
1863 case 128:
1864 return AMDGPU::SI_SPILL_AV1024_RESTORE;
1865 default:
1866 llvm_unreachable("unknown register size");
1867 }
1868}
1869
1870static unsigned getWWMRegSpillRestoreOpcode(unsigned Size,
1871 bool IsVectorSuperClass) {
1872 // Currently, there is only 32-bit WWM register spills needed.
1873 if (Size != 4)
1874 llvm_unreachable("unknown wwm register spill size");
1875
1876 if (IsVectorSuperClass) // TODO: Always use this if there are AGPRs
1877 return AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
1878
1879 return AMDGPU::SI_SPILL_WWM_V32_RESTORE;
1880}
1881
1882unsigned SIInstrInfo::getVectorRegSpillRestoreOpcode(
1883 Register Reg, const TargetRegisterClass *RC, unsigned Size,
1884 const SIMachineFunctionInfo &MFI) const {
1885 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
1886
1887 // Choose the right opcode if restoring a WWM register.
1888 if (MFI.checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
1889 return getWWMRegSpillRestoreOpcode(Size, IsVectorSuperClass);
1890
1891 // TODO: Check if AGPRs are available
1892 if (ST.hasMAIInsts())
1893 return getAVSpillRestoreOpcode(Size);
1894
1895 assert(!RI.isAGPRClass(RC));
1896 return getVGPRSpillRestoreOpcode(Size);
1897}
1898
1899void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1900 MachineBasicBlock::iterator MI,
1901 Register DestReg, int FrameIndex,
1902 const TargetRegisterClass *RC,
1903 Register VReg, unsigned SubReg,
1904 MachineInstr::MIFlag Flags) const {
1905 MachineFunction *MF = MBB.getParent();
1906 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1907 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1908 const DebugLoc &DL = MBB.findDebugLoc(MBBI: MI);
1909 unsigned SpillSize = RI.getSpillSize(RC: *RC);
1910
1911 MachinePointerInfo PtrInfo
1912 = MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
1913
1914 MachineMemOperand *MMO = MF->getMachineMemOperand(
1915 PtrInfo, F: MachineMemOperand::MOLoad, Size: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
1916 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
1917
1918 if (RI.isSGPRClass(RC)) {
1919 if (FrameInfo.getStackID(ObjectIdx: FrameIndex) == TargetStackID::SGPRSpill)
1920 MFI->setHasSpilledSGPRs();
1921 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into");
1922 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI &&
1923 DestReg != AMDGPU::EXEC && "exec should not be spilled");
1924
1925 // FIXME: Maybe this should not include a memoperand because it will be
1926 // lowered to non-memory instructions.
1927 const MCInstrDesc &OpDesc = get(Opcode: getSGPRSpillRestoreOpcode(Size: SpillSize));
1928 if (DestReg.isVirtual() && SpillSize == 4) {
1929 MachineRegisterInfo &MRI = MF->getRegInfo();
1930 MRI.constrainRegClass(Reg: DestReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
1931 }
1932
1933 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: OpDesc, DestReg)
1934 .addFrameIndex(Idx: FrameIndex) // addr
1935 .addMemOperand(MMO)
1936 .addReg(RegNo: MFI->getStackPtrOffsetReg(), Flags: RegState::Implicit);
1937
1938 return;
1939 }
1940
1941 unsigned Opcode = getVectorRegSpillRestoreOpcode(Reg: VReg ? VReg : DestReg, RC,
1942 Size: SpillSize, MFI: *MFI);
1943 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode), DestReg)
1944 .addFrameIndex(Idx: FrameIndex) // vaddr
1945 .addReg(RegNo: MFI->getStackPtrOffsetReg()) // scratch_offset
1946 .addImm(Val: 0) // offset
1947 .addMemOperand(MMO);
1948}
1949
1950void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
1951 MachineBasicBlock::iterator MI) const {
1952 insertNoops(MBB, MI, Quantity: 1);
1953}
1954
1955void SIInstrInfo::insertNoops(MachineBasicBlock &MBB,
1956 MachineBasicBlock::iterator MI,
1957 unsigned Quantity) const {
1958 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
1959 unsigned MaxSNopCount = 1u << ST.getSNopBits();
1960 while (Quantity > 0) {
1961 unsigned Arg = std::min(a: Quantity, b: MaxSNopCount);
1962 Quantity -= Arg;
1963 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOP)).addImm(Val: Arg - 1);
1964 }
1965}
1966
1967MachineBasicBlock *SIInstrInfo::insertSimulatedTrap(MachineRegisterInfo &MRI,
1968 MachineBasicBlock &MBB,
1969 MachineInstr &MI,
1970 const DebugLoc &DL) const {
1971 MachineFunction *MF = MBB.getParent();
1972 constexpr unsigned DoorbellIDMask = 0x3ff;
1973 constexpr unsigned ECQueueWaveAbort = 0x400;
1974
1975 MachineBasicBlock *TrapBB = &MBB;
1976 MachineBasicBlock *HaltLoopBB = MF->CreateMachineBasicBlock();
1977
1978 if (!MBB.succ_empty() || std::next(x: MI.getIterator()) != MBB.end()) {
1979 MBB.splitAt(SplitInst&: MI, /*UpdateLiveIns=*/false);
1980 TrapBB = MF->CreateMachineBasicBlock();
1981 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CBRANCH_EXECNZ)).addMBB(MBB: TrapBB);
1982 MF->push_back(MBB: TrapBB);
1983 MBB.addSuccessor(Succ: TrapBB);
1984 }
1985 // Start with a `s_trap 2`, if we're in PRIV=1 and we need the workaround this
1986 // will be a nop.
1987 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_TRAP))
1988 .addImm(Val: static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));
1989 Register DoorbellReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1990 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SENDMSG_RTN_B32),
1991 DestReg: DoorbellReg)
1992 .addImm(Val: AMDGPU::SendMsg::ID_RTN_GET_DOORBELL);
1993 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::TTMP2)
1994 .addUse(RegNo: AMDGPU::M0);
1995 Register DoorbellRegMasked =
1996 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1997 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_AND_B32), DestReg: DoorbellRegMasked)
1998 .addUse(RegNo: DoorbellReg)
1999 .addImm(Val: DoorbellIDMask);
2000 Register SetWaveAbortBit =
2001 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2002 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_OR_B32), DestReg: SetWaveAbortBit)
2003 .addUse(RegNo: DoorbellRegMasked)
2004 .addImm(Val: ECQueueWaveAbort);
2005 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
2006 .addUse(RegNo: SetWaveAbortBit);
2007 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SENDMSG))
2008 .addImm(Val: AMDGPU::SendMsg::ID_INTERRUPT);
2009 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
2010 .addUse(RegNo: AMDGPU::TTMP2);
2011 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH)).addMBB(MBB: HaltLoopBB);
2012 TrapBB->addSuccessor(Succ: HaltLoopBB);
2013
2014 BuildMI(BB&: *HaltLoopBB, I: HaltLoopBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETHALT)).addImm(Val: 5);
2015 BuildMI(BB&: *HaltLoopBB, I: HaltLoopBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
2016 .addMBB(MBB: HaltLoopBB);
2017 MF->push_back(MBB: HaltLoopBB);
2018 HaltLoopBB->addSuccessor(Succ: HaltLoopBB);
2019
2020 return MBB.getNextNode();
2021}
2022
2023unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) {
2024 switch (MI.getOpcode()) {
2025 default:
2026 if (MI.isMetaInstruction())
2027 return 0;
2028 return 1; // FIXME: Do wait states equal cycles?
2029
2030 case AMDGPU::S_NOP:
2031 return MI.getOperand(i: 0).getImm() + 1;
2032 // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The
2033 // hazard, even if one exist, won't really be visible. Should we handle it?
2034 }
2035}
2036
2037bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2038 MachineBasicBlock &MBB = *MI.getParent();
2039 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2040 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
2041
2042 switch (MI.getOpcode()) {
2043 default: return TargetInstrInfo::expandPostRAPseudo(MI);
2044 case AMDGPU::S_MOV_B64_term:
2045 // This is only a terminator to get the correct spill code placement during
2046 // register allocation.
2047 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2048 break;
2049
2050 case AMDGPU::S_MOV_B32_term:
2051 // This is only a terminator to get the correct spill code placement during
2052 // register allocation.
2053 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B32));
2054 break;
2055
2056 case AMDGPU::S_XOR_B64_term:
2057 // This is only a terminator to get the correct spill code placement during
2058 // register allocation.
2059 MI.setDesc(get(Opcode: AMDGPU::S_XOR_B64));
2060 break;
2061
2062 case AMDGPU::S_XOR_B32_term:
2063 // This is only a terminator to get the correct spill code placement during
2064 // register allocation.
2065 MI.setDesc(get(Opcode: AMDGPU::S_XOR_B32));
2066 break;
2067 case AMDGPU::S_OR_B64_term:
2068 // This is only a terminator to get the correct spill code placement during
2069 // register allocation.
2070 MI.setDesc(get(Opcode: AMDGPU::S_OR_B64));
2071 break;
2072 case AMDGPU::S_OR_B32_term:
2073 // This is only a terminator to get the correct spill code placement during
2074 // register allocation.
2075 MI.setDesc(get(Opcode: AMDGPU::S_OR_B32));
2076 break;
2077
2078 case AMDGPU::S_ANDN2_B64_term:
2079 // This is only a terminator to get the correct spill code placement during
2080 // register allocation.
2081 MI.setDesc(get(Opcode: AMDGPU::S_ANDN2_B64));
2082 break;
2083
2084 case AMDGPU::S_ANDN2_B32_term:
2085 // This is only a terminator to get the correct spill code placement during
2086 // register allocation.
2087 MI.setDesc(get(Opcode: AMDGPU::S_ANDN2_B32));
2088 break;
2089
2090 case AMDGPU::S_AND_B64_term:
2091 // This is only a terminator to get the correct spill code placement during
2092 // register allocation.
2093 MI.setDesc(get(Opcode: AMDGPU::S_AND_B64));
2094 break;
2095
2096 case AMDGPU::S_AND_B32_term:
2097 // This is only a terminator to get the correct spill code placement during
2098 // register allocation.
2099 MI.setDesc(get(Opcode: AMDGPU::S_AND_B32));
2100 break;
2101
2102 case AMDGPU::S_AND_SAVEEXEC_B64_term:
2103 // This is only a terminator to get the correct spill code placement during
2104 // register allocation.
2105 MI.setDesc(get(Opcode: AMDGPU::S_AND_SAVEEXEC_B64));
2106 break;
2107
2108 case AMDGPU::S_AND_SAVEEXEC_B32_term:
2109 // This is only a terminator to get the correct spill code placement during
2110 // register allocation.
2111 MI.setDesc(get(Opcode: AMDGPU::S_AND_SAVEEXEC_B32));
2112 break;
2113
2114 case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
2115 MI.setDesc(get(Opcode: AMDGPU::V_CMPX_EQ_U32_nosdst_e32));
2116 break;
2117 case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
2118 MI.setDesc(get(Opcode: AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
2119 break;
2120
2121 case AMDGPU::SI_SPILL_S32_TO_VGPR:
2122 MI.setDesc(get(Opcode: AMDGPU::V_WRITELANE_B32));
2123 break;
2124
2125 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
2126 MI.setDesc(get(Opcode: AMDGPU::V_READLANE_B32));
2127 break;
2128 case AMDGPU::AV_MOV_B32_IMM_PSEUDO: {
2129 Register Dst = MI.getOperand(i: 0).getReg();
2130 bool IsAGPR = SIRegisterInfo::isAGPRClass(RC: RI.getPhysRegBaseClass(Reg: Dst));
2131 MI.setDesc(
2132 get(Opcode: IsAGPR ? AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::V_MOV_B32_e32));
2133 break;
2134 }
2135 case AMDGPU::AV_MOV_B64_IMM_PSEUDO: {
2136 Register Dst = MI.getOperand(i: 0).getReg();
2137 if (SIRegisterInfo::isAGPRClass(RC: RI.getPhysRegBaseClass(Reg: Dst))) {
2138 int64_t Imm = MI.getOperand(i: 1).getImm();
2139
2140 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2141 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2142 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: DstLo)
2143 .addImm(Val: SignExtend64<32>(x: Imm));
2144 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: DstHi)
2145 .addImm(Val: SignExtend64<32>(x: Imm >> 32));
2146 MI.eraseFromParent();
2147 break;
2148 }
2149
2150 [[fallthrough]];
2151 }
2152 case AMDGPU::V_MOV_B64_PSEUDO: {
2153 Register Dst = MI.getOperand(i: 0).getReg();
2154 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2155 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2156
2157 const MCInstrDesc &Mov64Desc = get(Opcode: AMDGPU::V_MOV_B64_e32);
2158 const TargetRegisterClass *Mov64RC = getRegClass(MCID: Mov64Desc, /*OpNum=*/0);
2159
2160 const MachineOperand &SrcOp = MI.getOperand(i: 1);
2161 // FIXME: Will this work for 64-bit floating point immediates?
2162 assert(!SrcOp.isFPImm());
2163 if (ST.hasVMovB64Inst() && Mov64RC->contains(Reg: Dst)) {
2164 MI.setDesc(Mov64Desc);
2165 if (SrcOp.isReg() || isInlineConstant(MI, OpIdx: 1) ||
2166 (SrcOp.isImm() &&
2167 (isUInt<32>(x: SrcOp.getImm()) || ST.has64BitLiterals())) ||
2168 (SrcOp.isGlobal() && ST.has64BitLiterals()))
2169 break;
2170 }
2171 if (SrcOp.isGlobal()) {
2172 // The address is unknown until link time, so the PK_MOV inline-constant
2173 // shortcut cannot apply.
2174 const GlobalValue *GV = SrcOp.getGlobal();
2175 int64_t Offset = SrcOp.getOffset();
2176 unsigned BaseFlags, LoReloc, HiReloc;
2177 std::tie(args&: BaseFlags, args&: LoReloc, args&: HiReloc) =
2178 splitGlobalAddressRelocFlags(ST, SrcOp);
2179
2180 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2181 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | LoReloc);
2182 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2183 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | HiReloc);
2184 } else if (SrcOp.isImm()) {
2185 APInt Imm(64, SrcOp.getImm());
2186 APInt Lo(32, Imm.getLoBits(numBits: 32).getZExtValue());
2187 APInt Hi(32, Imm.getHiBits(numBits: 32).getZExtValue());
2188 const MCInstrDesc &PkMovDesc = get(Opcode: AMDGPU::V_PK_MOV_B32);
2189 const TargetRegisterClass *PkMovRC = getRegClass(MCID: PkMovDesc, /*OpNum=*/0);
2190
2191 if (ST.hasPkMovB32() && Lo == Hi && isInlineConstant(Imm: Lo) &&
2192 PkMovRC->contains(Reg: Dst)) {
2193 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: PkMovDesc, DestReg: Dst)
2194 .addImm(Val: SISrcMods::OP_SEL_1)
2195 .addImm(Val: Lo.getSExtValue())
2196 .addImm(Val: SISrcMods::OP_SEL_1)
2197 .addImm(Val: Lo.getSExtValue())
2198 .addImm(Val: 0) // op_sel_lo
2199 .addImm(Val: 0) // op_sel_hi
2200 .addImm(Val: 0) // neg_lo
2201 .addImm(Val: 0) // neg_hi
2202 .addImm(Val: 0); // clamp
2203 } else {
2204 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2205 .addImm(Val: Lo.getSExtValue());
2206 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2207 .addImm(Val: Hi.getSExtValue());
2208 }
2209 } else {
2210 assert(SrcOp.isReg());
2211 if (ST.hasPkMovB32() &&
2212 !RI.isAGPR(MRI: MBB.getParent()->getRegInfo(), Reg: SrcOp.getReg())) {
2213 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg: Dst)
2214 .addImm(Val: SISrcMods::OP_SEL_1) // src0_mod
2215 .addReg(RegNo: SrcOp.getReg())
2216 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) // src1_mod
2217 .addReg(RegNo: SrcOp.getReg())
2218 .addImm(Val: 0) // op_sel_lo
2219 .addImm(Val: 0) // op_sel_hi
2220 .addImm(Val: 0) // neg_lo
2221 .addImm(Val: 0) // neg_hi
2222 .addImm(Val: 0); // clamp
2223 } else {
2224 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2225 .addReg(RegNo: RI.getSubReg(Reg: SrcOp.getReg(), Idx: AMDGPU::sub0));
2226 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2227 .addReg(RegNo: RI.getSubReg(Reg: SrcOp.getReg(), Idx: AMDGPU::sub1));
2228 }
2229 }
2230 MI.eraseFromParent();
2231 break;
2232 }
2233 case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
2234 expandMovDPP64(MI);
2235 break;
2236 }
2237 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2238 const MachineOperand &SrcOp = MI.getOperand(i: 1);
2239 assert(!SrcOp.isFPImm());
2240
2241 if (ST.has64BitLiterals()) {
2242 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2243 break;
2244 }
2245
2246 if (SrcOp.isGlobal()) {
2247 Register Dst = MI.getOperand(i: 0).getReg();
2248 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2249 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2250 const GlobalValue *GV = SrcOp.getGlobal();
2251 int64_t Offset = SrcOp.getOffset();
2252 unsigned BaseFlags, LoReloc, HiReloc;
2253 std::tie(args&: BaseFlags, args&: LoReloc, args&: HiReloc) =
2254 splitGlobalAddressRelocFlags(ST, SrcOp);
2255
2256 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstLo)
2257 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | LoReloc);
2258 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi)
2259 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | HiReloc);
2260 MI.eraseFromParent();
2261 break;
2262 }
2263
2264 // SrcOp is immediate
2265 APInt Imm(64, SrcOp.getImm());
2266 if (Imm.isIntN(N: 32) || isInlineConstant(Imm)) {
2267 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2268 break;
2269 }
2270
2271 Register Dst = MI.getOperand(i: 0).getReg();
2272 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2273 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2274
2275 APInt Lo(32, Imm.getLoBits(numBits: 32).getZExtValue());
2276 APInt Hi(32, Imm.getHiBits(numBits: 32).getZExtValue());
2277 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstLo)
2278 .addImm(Val: Lo.getSExtValue());
2279 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi)
2280 .addImm(Val: Hi.getSExtValue());
2281 MI.eraseFromParent();
2282 break;
2283 }
2284 case AMDGPU::V_SET_INACTIVE_B32: {
2285 // Lower V_SET_INACTIVE_B32 to V_CNDMASK_B32.
2286 Register DstReg = MI.getOperand(i: 0).getReg();
2287 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
2288 .add(MO: MI.getOperand(i: 3))
2289 .add(MO: MI.getOperand(i: 4))
2290 .add(MO: MI.getOperand(i: 1))
2291 .add(MO: MI.getOperand(i: 2))
2292 .add(MO: MI.getOperand(i: 5));
2293 MI.eraseFromParent();
2294 break;
2295 }
2296 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2297 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2298 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2299 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2300 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2301 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V6:
2302 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V7:
2303 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2304 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2305 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2306 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2307 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2308 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2309 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2310 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2311 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2312 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2313 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2314 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2315 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V6:
2316 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V7:
2317 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2318 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2319 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2320 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2321 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2322 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2323 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2324 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:
2325 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:
2326 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:
2327 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:
2328 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {
2329 const TargetRegisterClass *EltRC = getOpRegClass(MI, OpNo: 2);
2330
2331 unsigned Opc;
2332 if (RI.hasVGPRs(RC: EltRC)) {
2333 Opc = AMDGPU::V_MOVRELD_B32_e32;
2334 } else {
2335 Opc = RI.getRegSizeInBits(RC: *EltRC) == 64 ? AMDGPU::S_MOVRELD_B64
2336 : AMDGPU::S_MOVRELD_B32;
2337 }
2338
2339 const MCInstrDesc &OpDesc = get(Opcode: Opc);
2340 Register VecReg = MI.getOperand(i: 0).getReg();
2341 bool IsUndef = MI.getOperand(i: 1).isUndef();
2342 unsigned SubReg = MI.getOperand(i: 3).getImm();
2343 assert(VecReg == MI.getOperand(1).getReg());
2344
2345 MachineInstrBuilder MIB =
2346 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: OpDesc)
2347 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2348 .add(MO: MI.getOperand(i: 2))
2349 .addReg(RegNo: VecReg, Flags: RegState::ImplicitDefine)
2350 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2351
2352 const int ImpDefIdx =
2353 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2354 const int ImpUseIdx = ImpDefIdx + 1;
2355 MIB->tieOperands(DefIdx: ImpDefIdx, UseIdx: ImpUseIdx);
2356 MI.eraseFromParent();
2357 break;
2358 }
2359 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:
2360 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:
2361 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:
2362 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:
2363 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:
2364 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V6:
2365 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V7:
2366 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:
2367 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9:
2368 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10:
2369 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11:
2370 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12:
2371 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:
2372 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {
2373 assert(ST.useVGPRIndexMode());
2374 Register VecReg = MI.getOperand(i: 0).getReg();
2375 bool IsUndef = MI.getOperand(i: 1).isUndef();
2376 MachineOperand &Idx = MI.getOperand(i: 3);
2377 Register SubReg = MI.getOperand(i: 4).getImm();
2378
2379 MachineInstr *SetOn = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_ON))
2380 .add(MO: Idx)
2381 .addImm(Val: AMDGPU::VGPRIndexMode::DST_ENABLE);
2382 SetOn->getOperand(i: 3).setIsUndef();
2383
2384 const MCInstrDesc &OpDesc = get(Opcode: AMDGPU::V_MOV_B32_indirect_write);
2385 MachineInstrBuilder MIB =
2386 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: OpDesc)
2387 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2388 .add(MO: MI.getOperand(i: 2))
2389 .addReg(RegNo: VecReg, Flags: RegState::ImplicitDefine)
2390 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2391
2392 const int ImpDefIdx =
2393 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2394 const int ImpUseIdx = ImpDefIdx + 1;
2395 MIB->tieOperands(DefIdx: ImpDefIdx, UseIdx: ImpUseIdx);
2396
2397 MachineInstr *SetOff = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_OFF));
2398
2399 finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2400
2401 MI.eraseFromParent();
2402 break;
2403 }
2404 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:
2405 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:
2406 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:
2407 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:
2408 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:
2409 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V6:
2410 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V7:
2411 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:
2412 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9:
2413 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10:
2414 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11:
2415 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12:
2416 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:
2417 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {
2418 assert(ST.useVGPRIndexMode());
2419 Register Dst = MI.getOperand(i: 0).getReg();
2420 Register VecReg = MI.getOperand(i: 1).getReg();
2421 bool IsUndef = MI.getOperand(i: 1).isUndef();
2422 Register SubReg = MI.getOperand(i: 3).getImm();
2423
2424 MachineInstr *SetOn = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_ON))
2425 .add(MO: MI.getOperand(i: 2))
2426 .addImm(Val: AMDGPU::VGPRIndexMode::SRC0_ENABLE);
2427 SetOn->getOperand(i: 3).setIsUndef();
2428
2429 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_indirect_read))
2430 .addDef(RegNo: Dst)
2431 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2432 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2433
2434 MachineInstr *SetOff = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_OFF));
2435
2436 finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2437
2438 MI.eraseFromParent();
2439 break;
2440 }
2441 case AMDGPU::SI_PC_ADD_REL_OFFSET: {
2442 MachineFunction &MF = *MBB.getParent();
2443 Register Reg = MI.getOperand(i: 0).getReg();
2444 Register RegLo = RI.getSubReg(Reg, Idx: AMDGPU::sub0);
2445 Register RegHi = RI.getSubReg(Reg, Idx: AMDGPU::sub1);
2446 MachineOperand OpLo = MI.getOperand(i: 1);
2447 MachineOperand OpHi = MI.getOperand(i: 2);
2448
2449 // Create a bundle so these instructions won't be re-ordered by the
2450 // post-RA scheduler.
2451 MIBundleBuilder Bundler(MBB, MI);
2452 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: Reg));
2453
2454 // What we want here is an offset from the value returned by s_getpc (which
2455 // is the address of the s_add_u32 instruction) to the global variable, but
2456 // since the encoding of $symbol starts 4 bytes after the start of the
2457 // s_add_u32 instruction, we end up with an offset that is 4 bytes too
2458 // small. This requires us to add 4 to the global variable offset in order
2459 // to compute the correct address. Similarly for the s_addc_u32 instruction,
2460 // the encoding of $symbol starts 12 bytes after the start of the s_add_u32
2461 // instruction.
2462
2463 int64_t Adjust = 0;
2464 if (ST.hasGetPCZeroExtension()) {
2465 // Fix up hardware that does not sign-extend the 48-bit PC value by
2466 // inserting: s_sext_i32_i16 reghi, reghi
2467 Bundler.append(
2468 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SEXT_I32_I16), DestReg: RegHi).addReg(RegNo: RegHi));
2469 Adjust += 4;
2470 }
2471
2472 if (OpLo.isGlobal())
2473 OpLo.setOffset(OpLo.getOffset() + Adjust + 4);
2474 Bundler.append(
2475 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U32), DestReg: RegLo).addReg(RegNo: RegLo).add(MO: OpLo));
2476
2477 if (OpHi.isGlobal())
2478 OpHi.setOffset(OpHi.getOffset() + Adjust + 12);
2479 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADDC_U32), DestReg: RegHi)
2480 .addReg(RegNo: RegHi)
2481 .add(MO: OpHi));
2482
2483 finalizeBundle(MBB, FirstMI: Bundler.begin());
2484
2485 MI.eraseFromParent();
2486 break;
2487 }
2488 case AMDGPU::SI_PC_ADD_REL_OFFSET64: {
2489 MachineFunction &MF = *MBB.getParent();
2490 Register Reg = MI.getOperand(i: 0).getReg();
2491 MachineOperand Op = MI.getOperand(i: 1);
2492
2493 // Create a bundle so these instructions won't be re-ordered by the
2494 // post-RA scheduler.
2495 MIBundleBuilder Bundler(MBB, MI);
2496 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: Reg));
2497 if (Op.isGlobal())
2498 Op.setOffset(Op.getOffset() + 4);
2499 Bundler.append(
2500 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U64), DestReg: Reg).addReg(RegNo: Reg).add(MO: Op));
2501
2502 finalizeBundle(MBB, FirstMI: Bundler.begin());
2503
2504 MI.eraseFromParent();
2505 break;
2506 }
2507 case AMDGPU::ENTER_STRICT_WWM: {
2508 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2509 // Whole Wave Mode is entered.
2510 MI.setDesc(get(Opcode: LMC.OrSaveExecOpc));
2511 break;
2512 }
2513 case AMDGPU::ENTER_STRICT_WQM: {
2514 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2515 // STRICT_WQM is entered.
2516 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: LMC.MovOpc), DestReg: MI.getOperand(i: 0).getReg())
2517 .addReg(RegNo: LMC.ExecReg);
2518 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: LMC.WQMOpc), DestReg: LMC.ExecReg).addReg(RegNo: LMC.ExecReg);
2519
2520 MI.eraseFromParent();
2521 break;
2522 }
2523 case AMDGPU::EXIT_STRICT_WWM:
2524 case AMDGPU::EXIT_STRICT_WQM: {
2525 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2526 // WWM/STICT_WQM is exited.
2527 MI.setDesc(get(Opcode: LMC.MovOpc));
2528 break;
2529 }
2530 case AMDGPU::SI_RETURN: {
2531 const MachineFunction *MF = MBB.getParent();
2532 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2533 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2534 // Hiding the return address use with SI_RETURN may lead to extra kills in
2535 // the function and missing live-ins. We are fine in practice because callee
2536 // saved register handling ensures the register value is restored before
2537 // RET, but we need the undef flag here to appease the MachineVerifier
2538 // liveness checks.
2539 MachineInstrBuilder MIB =
2540 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETPC_B64_return))
2541 .addReg(RegNo: TRI->getReturnAddressReg(MF: *MF), Flags: RegState::Undef);
2542
2543 MIB.copyImplicitOps(OtherMI: MI);
2544 MI.eraseFromParent();
2545 break;
2546 }
2547
2548 case AMDGPU::S_MUL_U64_U32_PSEUDO:
2549 case AMDGPU::S_MUL_I64_I32_PSEUDO:
2550 MI.setDesc(get(Opcode: AMDGPU::S_MUL_U64));
2551 break;
2552
2553 case AMDGPU::S_GETPC_B64_pseudo:
2554 MI.setDesc(get(Opcode: AMDGPU::S_GETPC_B64));
2555 if (ST.hasGetPCZeroExtension()) {
2556 Register Dst = MI.getOperand(i: 0).getReg();
2557 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2558 // Fix up hardware that does not sign-extend the 48-bit PC value by
2559 // inserting: s_sext_i32_i16 dsthi, dsthi
2560 BuildMI(BB&: MBB, I: std::next(x: MI.getIterator()), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SEXT_I32_I16),
2561 DestReg: DstHi)
2562 .addReg(RegNo: DstHi);
2563 }
2564 break;
2565
2566 case AMDGPU::V_MAX_BF16_PSEUDO_e64: {
2567 assert(ST.hasBF16PackedInsts());
2568 MI.setDesc(get(Opcode: AMDGPU::V_PK_MAX_NUM_BF16));
2569 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // op_sel
2570 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // neg_lo
2571 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // neg_hi
2572 auto Op0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
2573 Op0->setImm(Op0->getImm() | SISrcMods::OP_SEL_1);
2574 auto Op1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
2575 Op1->setImm(Op1->getImm() | SISrcMods::OP_SEL_1);
2576 break;
2577 }
2578
2579 case AMDGPU::GET_STACK_BASE:
2580 // The stack starts at offset 0 unless we need to reserve some space at the
2581 // bottom.
2582 if (ST.getFrameLowering()->mayReserveScratchForCWSR(MF: *MBB.getParent())) {
2583 // When CWSR is used in dynamic VGPR mode, the trap handler needs to save
2584 // some of the VGPRs. The size of the required scratch space has already
2585 // been computed by prolog epilog insertion.
2586 const SIMachineFunctionInfo *MFI =
2587 MBB.getParent()->getInfo<SIMachineFunctionInfo>();
2588 unsigned VGPRSize = MFI->getScratchReservedForDynamicVGPRs();
2589 Register DestReg = MI.getOperand(i: 0).getReg();
2590 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETREG_B32), DestReg)
2591 .addImm(Val: AMDGPU::Hwreg::HwregEncoding::encode(
2592 Values: AMDGPU::Hwreg::ID_HW_ID2, Values: AMDGPU::Hwreg::OFFSET_ME_ID, Values: 2));
2593 // The MicroEngine ID is 0 for the graphics queue, and 1 or 2 for compute
2594 // (3 is unused, so we ignore it). Unfortunately, S_GETREG doesn't set
2595 // SCC, so we need to check for 0 manually.
2596 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U32)).addImm(Val: 0).addReg(RegNo: DestReg);
2597 // Change the implicif-def of SCC to an explicit use (but first remove
2598 // the dead flag if present).
2599 MI.getOperand(i: MI.getNumExplicitOperands()).setIsDead(false);
2600 MI.getOperand(i: MI.getNumExplicitOperands()).setIsUse();
2601 MI.setDesc(get(Opcode: AMDGPU::S_CMOVK_I32));
2602 MI.addOperand(Op: MachineOperand::CreateImm(Val: VGPRSize));
2603 } else {
2604 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B32));
2605 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2606 MI.removeOperand(
2607 OpNo: MI.getNumExplicitOperands()); // Drop implicit def of SCC.
2608 }
2609 break;
2610 }
2611
2612 return true;
2613}
2614
2615void SIInstrInfo::reMaterialize(MachineBasicBlock &MBB,
2616 MachineBasicBlock::iterator I, Register DestReg,
2617 unsigned SubIdx, const MachineInstr &Orig,
2618 LaneBitmask UsedLanes) const {
2619
2620 // Try shrinking the instruction to remat only the part needed for current
2621 // context.
2622 // TODO: Handle more cases.
2623 unsigned Opcode = Orig.getOpcode();
2624 switch (Opcode) {
2625 case AMDGPU::S_MOV_B64:
2626 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2627 if (SubIdx != 0)
2628 break;
2629
2630 if (!Orig.getOperand(i: 1).isImm())
2631 break;
2632
2633 // Shrink S_MOV_B64 to S_MOV_B32 when UsedLanes indicates only a single
2634 // 32-bit lane of the 64-bit value is live at the rematerialization point.
2635 if (UsedLanes.all())
2636 break;
2637
2638 // Determine which half of the 64-bit immediate corresponds to the use.
2639 unsigned OrigSubReg = Orig.getOperand(i: 0).getSubReg();
2640 unsigned LoSubReg = RI.composeSubRegIndices(a: OrigSubReg, b: AMDGPU::sub0);
2641 unsigned HiSubReg = RI.composeSubRegIndices(a: OrigSubReg, b: AMDGPU::sub1);
2642
2643 bool NeedLo = (UsedLanes & RI.getSubRegIndexLaneMask(SubIdx: LoSubReg)).any();
2644 bool NeedHi = (UsedLanes & RI.getSubRegIndexLaneMask(SubIdx: HiSubReg)).any();
2645
2646 if (NeedLo && NeedHi)
2647 break;
2648
2649 int64_t Imm64 = Orig.getOperand(i: 1).getImm();
2650 int32_t Imm32 = NeedLo ? Lo_32(Value: Imm64) : Hi_32(Value: Imm64);
2651
2652 unsigned UseSubReg = NeedLo ? LoSubReg : HiSubReg;
2653
2654 // Emit S_MOV_B32 defining just the needed 32-bit subreg of DestReg.
2655 BuildMI(BB&: MBB, I, MIMD: Orig.getDebugLoc(), MCID: get(Opcode: AMDGPU::S_MOV_B32))
2656 .addReg(RegNo: DestReg, Flags: RegState::Define | RegState::Undef, SubReg: UseSubReg)
2657 .addImm(Val: Imm32);
2658 return;
2659 }
2660
2661 case AMDGPU::S_LOAD_DWORDX16_IMM:
2662 case AMDGPU::S_LOAD_DWORDX8_IMM: {
2663 if (SubIdx != 0)
2664 break;
2665
2666 if (I == MBB.end())
2667 break;
2668
2669 if (I->isBundled())
2670 break;
2671
2672 // Look for a single use of the register that is also a subreg.
2673 Register RegToFind = Orig.getOperand(i: 0).getReg();
2674 MachineOperand *UseMO = nullptr;
2675 for (auto &CandMO : I->operands()) {
2676 if (!CandMO.isReg() || CandMO.getReg() != RegToFind || CandMO.isDef())
2677 continue;
2678 if (UseMO) {
2679 UseMO = nullptr;
2680 break;
2681 }
2682 UseMO = &CandMO;
2683 }
2684 if (!UseMO || UseMO->getSubReg() == AMDGPU::NoSubRegister)
2685 break;
2686
2687 unsigned Offset = RI.getSubRegIdxOffset(Idx: UseMO->getSubReg());
2688 unsigned SubregSize = RI.getSubRegIdxSize(Idx: UseMO->getSubReg());
2689
2690 MachineFunction *MF = MBB.getParent();
2691 MachineRegisterInfo &MRI = MF->getRegInfo();
2692 assert(MRI.use_nodbg_empty(DestReg) && "DestReg should have no users yet.");
2693
2694 unsigned NewOpcode = -1;
2695 if (SubregSize == 256)
2696 NewOpcode = AMDGPU::S_LOAD_DWORDX8_IMM;
2697 else if (SubregSize == 128)
2698 NewOpcode = AMDGPU::S_LOAD_DWORDX4_IMM;
2699 else
2700 break;
2701
2702 const MCInstrDesc &TID = get(Opcode: NewOpcode);
2703 const TargetRegisterClass *NewRC =
2704 RI.getAllocatableClass(RC: getRegClass(MCID: TID, OpNum: 0));
2705 MRI.setRegClass(Reg: DestReg, RC: NewRC);
2706
2707 UseMO->setReg(DestReg);
2708 UseMO->setSubReg(AMDGPU::NoSubRegister);
2709
2710 // Use a smaller load with the desired size, possibly with updated offset.
2711 MachineInstr *MI = MF->CloneMachineInstr(Orig: &Orig);
2712 MI->setDesc(TID);
2713 MI->getOperand(i: 0).setReg(DestReg);
2714 MI->getOperand(i: 0).setSubReg(AMDGPU::NoSubRegister);
2715 if (Offset) {
2716 MachineOperand *OffsetMO = getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::offset);
2717 int64_t FinalOffset = OffsetMO->getImm() + Offset / 8;
2718 OffsetMO->setImm(FinalOffset);
2719 }
2720 SmallVector<MachineMemOperand *> NewMMOs;
2721 for (const MachineMemOperand *MemOp : Orig.memoperands())
2722 NewMMOs.push_back(Elt: MF->getMachineMemOperand(MMO: MemOp, PtrInfo: MemOp->getPointerInfo(),
2723 Size: SubregSize / 8));
2724 MI->setMemRefs(MF&: *MF, MemRefs: NewMMOs);
2725
2726 MBB.insert(I, MI);
2727 return;
2728 }
2729
2730 default:
2731 break;
2732 }
2733
2734 TargetInstrInfo::reMaterialize(MBB, MI: I, DestReg, SubIdx, Orig, UsedLanes);
2735}
2736
2737std::pair<MachineInstr*, MachineInstr*>
2738SIInstrInfo::expandMovDPP64(MachineInstr &MI) const {
2739 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
2740
2741 if (ST.hasVMovB64Inst() && ST.hasFeature(Feature: AMDGPU::FeatureDPALU_DPP) &&
2742 AMDGPU::isLegalDPALU_DPPControl(
2743 ST, DC: getNamedOperand(MI, OperandName: AMDGPU::OpName::dpp_ctrl)->getImm())) {
2744 MI.setDesc(get(Opcode: AMDGPU::V_MOV_B64_dpp));
2745 return std::pair(&MI, nullptr);
2746 }
2747
2748 MachineBasicBlock &MBB = *MI.getParent();
2749 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2750 MachineFunction *MF = MBB.getParent();
2751 MachineRegisterInfo &MRI = MF->getRegInfo();
2752 Register Dst = MI.getOperand(i: 0).getReg();
2753 unsigned Part = 0;
2754 MachineInstr *Split[2];
2755
2756 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
2757 auto MovDPP = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_dpp));
2758 if (Dst.isPhysical()) {
2759 MovDPP.addDef(RegNo: RI.getSubReg(Reg: Dst, Idx: Sub));
2760 } else {
2761 assert(MRI.isSSA());
2762 auto Tmp = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2763 MovDPP.addDef(RegNo: Tmp);
2764 }
2765
2766 for (unsigned I = 1; I <= 2; ++I) { // old and src operands.
2767 const MachineOperand &SrcOp = MI.getOperand(i: I);
2768 assert(!SrcOp.isFPImm());
2769 if (SrcOp.isImm()) {
2770 APInt Imm(64, SrcOp.getImm());
2771 Imm.ashrInPlace(ShiftAmt: Part * 32);
2772 MovDPP.addImm(Val: Imm.getLoBits(numBits: 32).getZExtValue());
2773 } else {
2774 assert(SrcOp.isReg());
2775 Register Src = SrcOp.getReg();
2776 if (Src.isPhysical())
2777 MovDPP.addReg(RegNo: RI.getSubReg(Reg: Src, Idx: Sub));
2778 else
2779 MovDPP.addReg(RegNo: Src, Flags: getUndefRegState(B: SrcOp.isUndef()), SubReg: Sub);
2780 }
2781 }
2782
2783 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.explicit_operands(), N: 3))
2784 MovDPP.addImm(Val: MO.getImm());
2785
2786 Split[Part] = MovDPP;
2787 ++Part;
2788 }
2789
2790 if (Dst.isVirtual())
2791 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
2792 .addReg(RegNo: Split[0]->getOperand(i: 0).getReg())
2793 .addImm(Val: AMDGPU::sub0)
2794 .addReg(RegNo: Split[1]->getOperand(i: 0).getReg())
2795 .addImm(Val: AMDGPU::sub1);
2796
2797 MI.eraseFromParent();
2798 return std::pair(Split[0], Split[1]);
2799}
2800
2801std::optional<DestSourcePair>
2802SIInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
2803 if (MI.getOpcode() == AMDGPU::WWM_COPY)
2804 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2805
2806 return std::nullopt;
2807}
2808
2809bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,
2810 AMDGPU::OpName Src0OpName,
2811 MachineOperand &Src1,
2812 AMDGPU::OpName Src1OpName) const {
2813 MachineOperand *Src0Mods = getNamedOperand(MI, OperandName: Src0OpName);
2814 if (!Src0Mods)
2815 return false;
2816
2817 MachineOperand *Src1Mods = getNamedOperand(MI, OperandName: Src1OpName);
2818 assert(Src1Mods &&
2819 "All commutable instructions have both src0 and src1 modifiers");
2820
2821 int Src0ModsVal = Src0Mods->getImm();
2822 int Src1ModsVal = Src1Mods->getImm();
2823
2824 Src1Mods->setImm(Src0ModsVal);
2825 Src0Mods->setImm(Src1ModsVal);
2826 return true;
2827}
2828
2829static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI,
2830 MachineOperand &RegOp,
2831 MachineOperand &NonRegOp) {
2832 Register Reg = RegOp.getReg();
2833 unsigned SubReg = RegOp.getSubReg();
2834 bool IsKill = RegOp.isKill();
2835 bool IsDead = RegOp.isDead();
2836 bool IsUndef = RegOp.isUndef();
2837 bool IsDebug = RegOp.isDebug();
2838
2839 if (NonRegOp.isImm())
2840 RegOp.ChangeToImmediate(ImmVal: NonRegOp.getImm());
2841 else if (NonRegOp.isFI())
2842 RegOp.ChangeToFrameIndex(Idx: NonRegOp.getIndex());
2843 else if (NonRegOp.isGlobal()) {
2844 RegOp.ChangeToGA(GV: NonRegOp.getGlobal(), Offset: NonRegOp.getOffset(),
2845 TargetFlags: NonRegOp.getTargetFlags());
2846 } else
2847 return nullptr;
2848
2849 // Make sure we don't reinterpret a subreg index in the target flags.
2850 RegOp.setTargetFlags(NonRegOp.getTargetFlags());
2851
2852 NonRegOp.ChangeToRegister(Reg, isDef: false, isImp: false, isKill: IsKill, isDead: IsDead, isUndef: IsUndef, isDebug: IsDebug);
2853 NonRegOp.setSubReg(SubReg);
2854
2855 return &MI;
2856}
2857
2858static MachineInstr *swapImmOperands(MachineInstr &MI,
2859 MachineOperand &NonRegOp1,
2860 MachineOperand &NonRegOp2) {
2861 unsigned TargetFlags = NonRegOp1.getTargetFlags();
2862 int64_t NonRegVal = NonRegOp1.getImm();
2863
2864 NonRegOp1.setImm(NonRegOp2.getImm());
2865 NonRegOp2.setImm(NonRegVal);
2866 NonRegOp1.setTargetFlags(NonRegOp2.getTargetFlags());
2867 NonRegOp2.setTargetFlags(TargetFlags);
2868 return &MI;
2869}
2870
2871bool SIInstrInfo::isLegalToSwap(const MachineInstr &MI, unsigned OpIdx0,
2872 unsigned OpIdx1) const {
2873 const MCInstrDesc &InstDesc = MI.getDesc();
2874 const MCOperandInfo &OpInfo0 = InstDesc.operands()[OpIdx0];
2875 const MCOperandInfo &OpInfo1 = InstDesc.operands()[OpIdx1];
2876
2877 unsigned Opc = MI.getOpcode();
2878 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
2879
2880 const MachineOperand &MO0 = MI.getOperand(i: OpIdx0);
2881 const MachineOperand &MO1 = MI.getOperand(i: OpIdx1);
2882
2883 // Swap doesn't breach constant bus or literal limits
2884 // It may move literal to position other than src0, this is not allowed
2885 // pre-gfx10 However, most test cases need literals in Src0 for VOP
2886 // FIXME: After gfx9, literal can be in place other than Src0
2887 if (isVALU(MI, /*AllowLDSDMA=*/true)) {
2888 if ((int)OpIdx0 == Src0Idx && !MO0.isReg() &&
2889 !isInlineConstant(MO: MO0, OpInfo: OpInfo1))
2890 return false;
2891 if ((int)OpIdx1 == Src0Idx && !MO1.isReg() &&
2892 !isInlineConstant(MO: MO1, OpInfo: OpInfo0))
2893 return false;
2894 }
2895
2896 if ((int)OpIdx1 != Src0Idx && MO0.isReg()) {
2897 if (OpInfo1.RegClass == -1)
2898 return OpInfo1.OperandType == MCOI::OPERAND_UNKNOWN;
2899 return isLegalRegOperand(MI, OpIdx: OpIdx1, MO: MO0) &&
2900 (!MO1.isReg() || isLegalRegOperand(MI, OpIdx: OpIdx0, MO: MO1));
2901 }
2902 if ((int)OpIdx0 != Src0Idx && MO1.isReg()) {
2903 if (OpInfo0.RegClass == -1)
2904 return OpInfo0.OperandType == MCOI::OPERAND_UNKNOWN;
2905 return (!MO0.isReg() || isLegalRegOperand(MI, OpIdx: OpIdx1, MO: MO0)) &&
2906 isLegalRegOperand(MI, OpIdx: OpIdx0, MO: MO1);
2907 }
2908
2909 // No need to check 64-bit literals since swapping does not bring new
2910 // 64-bit literals into current instruction to fold to 32-bit
2911
2912 return isImmOperandLegal(MI, OpNo: OpIdx1, MO: MO0);
2913}
2914
2915MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
2916 unsigned Src0Idx,
2917 unsigned Src1Idx) const {
2918 assert(!NewMI && "this should never be used");
2919
2920 unsigned Opc = MI.getOpcode();
2921 int CommutedOpcode = commuteOpcode(Opcode: Opc);
2922 if (CommutedOpcode == -1)
2923 return nullptr;
2924
2925 if (Src0Idx > Src1Idx)
2926 std::swap(a&: Src0Idx, b&: Src1Idx);
2927
2928 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
2929 static_cast<int>(Src0Idx) &&
2930 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
2931 static_cast<int>(Src1Idx) &&
2932 "inconsistency with findCommutedOpIndices");
2933
2934 if (!isLegalToSwap(MI, OpIdx0: Src0Idx, OpIdx1: Src1Idx))
2935 return nullptr;
2936
2937 MachineInstr *CommutedMI = nullptr;
2938 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
2939 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
2940 if (Src0.isReg() && Src1.isReg()) {
2941 // Be sure to copy the source modifiers to the right place.
2942 CommutedMI =
2943 TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1: Src0Idx, OpIdx2: Src1Idx);
2944 } else if (Src0.isReg() && !Src1.isReg()) {
2945 CommutedMI = swapRegAndNonRegOperand(MI, RegOp&: Src0, NonRegOp&: Src1);
2946 } else if (!Src0.isReg() && Src1.isReg()) {
2947 CommutedMI = swapRegAndNonRegOperand(MI, RegOp&: Src1, NonRegOp&: Src0);
2948 } else if (Src0.isImm() && Src1.isImm()) {
2949 CommutedMI = swapImmOperands(MI, NonRegOp1&: Src0, NonRegOp2&: Src1);
2950 } else {
2951 // FIXME: Found two non registers to commute. This does happen.
2952 return nullptr;
2953 }
2954
2955 if (CommutedMI) {
2956 swapSourceModifiers(MI, Src0, Src0OpName: AMDGPU::OpName::src0_modifiers,
2957 Src1, Src1OpName: AMDGPU::OpName::src1_modifiers);
2958
2959 swapSourceModifiers(MI, Src0, Src0OpName: AMDGPU::OpName::src0_sel, Src1,
2960 Src1OpName: AMDGPU::OpName::src1_sel);
2961
2962 CommutedMI->setDesc(get(Opcode: CommutedOpcode));
2963 }
2964
2965 return CommutedMI;
2966}
2967
2968// This needs to be implemented because the source modifiers may be inserted
2969// between the true commutable operands, and the base
2970// TargetInstrInfo::commuteInstruction uses it.
2971bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
2972 unsigned &SrcOpIdx0,
2973 unsigned &SrcOpIdx1) const {
2974 return findCommutedOpIndices(Desc: MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
2975}
2976
2977bool SIInstrInfo::findCommutedOpIndices(const MCInstrDesc &Desc,
2978 unsigned &SrcOpIdx0,
2979 unsigned &SrcOpIdx1) const {
2980 if (!Desc.isCommutable())
2981 return false;
2982
2983 unsigned Opc = Desc.getOpcode();
2984 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
2985 if (Src0Idx == -1)
2986 return false;
2987
2988 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
2989 if (Src1Idx == -1)
2990 return false;
2991
2992 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx0, ResultIdx2&: SrcOpIdx1, CommutableOpIdx1: Src0Idx, CommutableOpIdx2: Src1Idx);
2993}
2994
2995bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
2996 int64_t BrOffset) const {
2997 // BranchRelaxation should never have to check s_setpc_b64 or s_add_pc_i64
2998 // because its dest block is unanalyzable.
2999 assert(isSOPP(BranchOp) || isSOPK(BranchOp));
3000
3001 // Convert to dwords.
3002 BrOffset /= 4;
3003
3004 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
3005 // from the next instruction.
3006 BrOffset -= 1;
3007
3008 return isIntN(N: BranchOffsetBits, x: BrOffset);
3009}
3010
3011MachineBasicBlock *
3012SIInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
3013 return MI.getOperand(i: 0).getMBB();
3014}
3015
3016bool SIInstrInfo::hasDivergentBranch(const MachineBasicBlock *MBB) const {
3017 for (const MachineInstr &MI : MBB->terminators()) {
3018 if (MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE ||
3019 MI.getOpcode() == AMDGPU::SI_LOOP)
3020 return true;
3021 }
3022 return false;
3023}
3024
3025void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
3026 MachineBasicBlock &DestBB,
3027 MachineBasicBlock &RestoreBB,
3028 const DebugLoc &DL, int64_t BrOffset,
3029 RegScavenger *RS) const {
3030 assert(MBB.empty() &&
3031 "new block should be inserted for expanding unconditional branch");
3032 assert(MBB.pred_size() == 1);
3033 assert(RestoreBB.empty() &&
3034 "restore block should be inserted for restoring clobbered registers");
3035
3036 MachineFunction *MF = MBB.getParent();
3037 MachineRegisterInfo &MRI = MF->getRegInfo();
3038 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
3039 auto I = MBB.end();
3040 auto &MCCtx = MF->getContext();
3041
3042 if (ST.useAddPC64Inst()) {
3043 MCSymbol *Offset =
3044 MCCtx.createTempSymbol(Name: "offset", /*AlwaysAddSuffix=*/true);
3045 auto AddPC = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_PC_I64))
3046 .addSym(Sym: Offset, TargetFlags: MO_FAR_BRANCH_OFFSET);
3047 MCSymbol *PostAddPCLabel =
3048 MCCtx.createTempSymbol(Name: "post_addpc", /*AlwaysAddSuffix=*/true);
3049 AddPC->setPostInstrSymbol(MF&: *MF, Symbol: PostAddPCLabel);
3050 auto *OffsetExpr = MCBinaryExpr::createSub(
3051 LHS: MCSymbolRefExpr::create(Symbol: DestBB.getSymbol(), Ctx&: MCCtx),
3052 RHS: MCSymbolRefExpr::create(Symbol: PostAddPCLabel, Ctx&: MCCtx), Ctx&: MCCtx);
3053 Offset->setVariableValue(OffsetExpr);
3054 return;
3055 }
3056
3057 assert(RS && "RegScavenger required for long branching");
3058
3059 // FIXME: Virtual register workaround for RegScavenger not working with empty
3060 // blocks.
3061 Register PCReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
3062
3063 // Note: as this is used after hazard recognizer we need to apply some hazard
3064 // workarounds directly.
3065 const bool FlushSGPRWrites = (ST.isWave64() && ST.hasVALUMaskWriteHazard()) ||
3066 ST.hasVALUReadSGPRHazard();
3067 auto ApplyHazardWorkarounds = [this, &MBB, &I, &DL, FlushSGPRWrites]() {
3068 if (FlushSGPRWrites)
3069 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
3070 .addImm(Val: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST));
3071 };
3072
3073 // We need to compute the offset relative to the instruction immediately after
3074 // s_getpc_b64. Insert pc arithmetic code before last terminator.
3075 MachineInstr *GetPC = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: PCReg);
3076 ApplyHazardWorkarounds();
3077
3078 MCSymbol *PostGetPCLabel =
3079 MCCtx.createTempSymbol(Name: "post_getpc", /*AlwaysAddSuffix=*/true);
3080 GetPC->setPostInstrSymbol(MF&: *MF, Symbol: PostGetPCLabel);
3081
3082 MCSymbol *OffsetLo =
3083 MCCtx.createTempSymbol(Name: "offset_lo", /*AlwaysAddSuffix=*/true);
3084 MCSymbol *OffsetHi =
3085 MCCtx.createTempSymbol(Name: "offset_hi", /*AlwaysAddSuffix=*/true);
3086 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U32))
3087 .addReg(RegNo: PCReg, Flags: RegState::Define, SubReg: AMDGPU::sub0)
3088 .addReg(RegNo: PCReg, Flags: {}, SubReg: AMDGPU::sub0)
3089 .addSym(Sym: OffsetLo, TargetFlags: MO_FAR_BRANCH_OFFSET);
3090 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADDC_U32))
3091 .addReg(RegNo: PCReg, Flags: RegState::Define, SubReg: AMDGPU::sub1)
3092 .addReg(RegNo: PCReg, Flags: {}, SubReg: AMDGPU::sub1)
3093 .addSym(Sym: OffsetHi, TargetFlags: MO_FAR_BRANCH_OFFSET);
3094 ApplyHazardWorkarounds();
3095
3096 // Insert the indirect branch after the other terminator.
3097 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETPC_B64))
3098 .addReg(RegNo: PCReg);
3099
3100 // If a spill is needed for the pc register pair, we need to insert a spill
3101 // restore block right before the destination block, and insert a short branch
3102 // into the old destination block's fallthrough predecessor.
3103 // e.g.:
3104 //
3105 // s_cbranch_scc0 skip_long_branch:
3106 //
3107 // long_branch_bb:
3108 // spill s[8:9]
3109 // s_getpc_b64 s[8:9]
3110 // s_add_u32 s8, s8, restore_bb
3111 // s_addc_u32 s9, s9, 0
3112 // s_setpc_b64 s[8:9]
3113 //
3114 // skip_long_branch:
3115 // foo;
3116 //
3117 // .....
3118 //
3119 // dest_bb_fallthrough_predecessor:
3120 // bar;
3121 // s_branch dest_bb
3122 //
3123 // restore_bb:
3124 // restore s[8:9]
3125 // fallthrough dest_bb
3126 ///
3127 // dest_bb:
3128 // buzz;
3129
3130 Register LongBranchReservedReg = MFI->getLongBranchReservedReg();
3131 Register Scav;
3132
3133 // If we've previously reserved a register for long branches
3134 // avoid running the scavenger and just use those registers
3135 if (LongBranchReservedReg) {
3136 RS->enterBasicBlock(MBB);
3137 Scav = LongBranchReservedReg;
3138 } else {
3139 RS->enterBasicBlockEnd(MBB);
3140 Scav = RS->scavengeRegisterBackwards(
3141 RC: AMDGPU::SReg_64RegClass, To: MachineBasicBlock::iterator(GetPC),
3142 /* RestoreAfter */ false, SPAdj: 0, /* AllowSpill */ false);
3143 }
3144 if (Scav) {
3145 RS->setRegUsed(Reg: Scav);
3146 MRI.replaceRegWith(FromReg: PCReg, ToReg: Scav);
3147 MRI.clearVirtRegs();
3148 } else {
3149 // As SGPR needs VGPR to be spilled, we reuse the slot of temporary VGPR for
3150 // SGPR spill.
3151 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3152 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3153 TRI->spillEmergencySGPR(MI: GetPC, RestoreMBB&: RestoreBB, SGPR: AMDGPU::SGPR0_SGPR1, RS);
3154 MRI.replaceRegWith(FromReg: PCReg, ToReg: AMDGPU::SGPR0_SGPR1);
3155 MRI.clearVirtRegs();
3156 }
3157
3158 MCSymbol *DestLabel = Scav ? DestBB.getSymbol() : RestoreBB.getSymbol();
3159 // Now, the distance could be defined.
3160 auto *Offset = MCBinaryExpr::createSub(
3161 LHS: MCSymbolRefExpr::create(Symbol: DestLabel, Ctx&: MCCtx),
3162 RHS: MCSymbolRefExpr::create(Symbol: PostGetPCLabel, Ctx&: MCCtx), Ctx&: MCCtx);
3163 // Add offset assignments.
3164 auto *Mask = MCConstantExpr::create(Value: 0xFFFFFFFFULL, Ctx&: MCCtx);
3165 OffsetLo->setVariableValue(MCBinaryExpr::createAnd(LHS: Offset, RHS: Mask, Ctx&: MCCtx));
3166 auto *ShAmt = MCConstantExpr::create(Value: 32, Ctx&: MCCtx);
3167 OffsetHi->setVariableValue(MCBinaryExpr::createAShr(LHS: Offset, RHS: ShAmt, Ctx&: MCCtx));
3168}
3169
3170unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
3171 switch (Cond) {
3172 case SIInstrInfo::SCC_TRUE:
3173 return AMDGPU::S_CBRANCH_SCC1;
3174 case SIInstrInfo::SCC_FALSE:
3175 return AMDGPU::S_CBRANCH_SCC0;
3176 case SIInstrInfo::VCCNZ:
3177 return AMDGPU::S_CBRANCH_VCCNZ;
3178 case SIInstrInfo::VCCZ:
3179 return AMDGPU::S_CBRANCH_VCCZ;
3180 case SIInstrInfo::EXECNZ:
3181 return AMDGPU::S_CBRANCH_EXECNZ;
3182 case SIInstrInfo::EXECZ:
3183 return AMDGPU::S_CBRANCH_EXECZ;
3184 default:
3185 llvm_unreachable("invalid branch predicate");
3186 }
3187}
3188
3189SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
3190 switch (Opcode) {
3191 case AMDGPU::S_CBRANCH_SCC0:
3192 return SCC_FALSE;
3193 case AMDGPU::S_CBRANCH_SCC1:
3194 return SCC_TRUE;
3195 case AMDGPU::S_CBRANCH_VCCNZ:
3196 return VCCNZ;
3197 case AMDGPU::S_CBRANCH_VCCZ:
3198 return VCCZ;
3199 case AMDGPU::S_CBRANCH_EXECNZ:
3200 return EXECNZ;
3201 case AMDGPU::S_CBRANCH_EXECZ:
3202 return EXECZ;
3203 default:
3204 return INVALID_BR;
3205 }
3206}
3207
3208bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB,
3209 MachineBasicBlock::iterator I,
3210 MachineBasicBlock *&TBB,
3211 MachineBasicBlock *&FBB,
3212 SmallVectorImpl<MachineOperand> &Cond,
3213 bool AllowModify) const {
3214 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3215 // Unconditional Branch
3216 TBB = I->getOperand(i: 0).getMBB();
3217 return false;
3218 }
3219
3220 BranchPredicate Pred = getBranchPredicate(Opcode: I->getOpcode());
3221 if (Pred == INVALID_BR)
3222 return true;
3223
3224 MachineBasicBlock *CondBB = I->getOperand(i: 0).getMBB();
3225 Cond.push_back(Elt: MachineOperand::CreateImm(Val: Pred));
3226 Cond.push_back(Elt: I->getOperand(i: 1)); // Save the branch register.
3227
3228 ++I;
3229
3230 if (I == MBB.end()) {
3231 // Conditional branch followed by fall-through.
3232 TBB = CondBB;
3233 return false;
3234 }
3235
3236 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3237 TBB = CondBB;
3238 FBB = I->getOperand(i: 0).getMBB();
3239 return false;
3240 }
3241
3242 return true;
3243}
3244
3245bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
3246 MachineBasicBlock *&FBB,
3247 SmallVectorImpl<MachineOperand> &Cond,
3248 bool AllowModify) const {
3249 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
3250 auto E = MBB.end();
3251 if (I == E)
3252 return false;
3253
3254 // Skip over the instructions that are artificially terminators for special
3255 // exec management.
3256 while (I != E && !I->isBranch() && !I->isReturn()) {
3257 switch (I->getOpcode()) {
3258 case AMDGPU::S_MOV_B64_term:
3259 case AMDGPU::S_XOR_B64_term:
3260 case AMDGPU::S_OR_B64_term:
3261 case AMDGPU::S_ANDN2_B64_term:
3262 case AMDGPU::S_AND_B64_term:
3263 case AMDGPU::S_AND_SAVEEXEC_B64_term:
3264 case AMDGPU::S_MOV_B32_term:
3265 case AMDGPU::S_XOR_B32_term:
3266 case AMDGPU::S_OR_B32_term:
3267 case AMDGPU::S_ANDN2_B32_term:
3268 case AMDGPU::S_AND_B32_term:
3269 case AMDGPU::S_AND_SAVEEXEC_B32_term:
3270 case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
3271 case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
3272 break;
3273 case AMDGPU::SI_IF:
3274 case AMDGPU::SI_ELSE:
3275 case AMDGPU::SI_KILL_I1_TERMINATOR:
3276 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
3277 // FIXME: It's messy that these need to be considered here at all.
3278 return true;
3279 default:
3280 llvm_unreachable("unexpected non-branch terminator inst");
3281 }
3282
3283 ++I;
3284 }
3285
3286 if (I == E)
3287 return false;
3288
3289 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
3290}
3291
3292unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB,
3293 int *BytesRemoved) const {
3294 unsigned Count = 0;
3295 unsigned RemovedSize = 0;
3296 for (MachineInstr &MI : llvm::make_early_inc_range(Range: MBB.terminators())) {
3297 // Skip over artificial terminators when removing instructions.
3298 if (MI.isBranch() || MI.isReturn()) {
3299 RemovedSize += getInstSizeInBytes(MI);
3300 MI.eraseFromParent();
3301 ++Count;
3302 }
3303 }
3304
3305 if (BytesRemoved)
3306 *BytesRemoved = RemovedSize;
3307
3308 return Count;
3309}
3310
3311// Copy the flags onto the implicit condition register operand.
3312static void preserveCondRegFlags(MachineOperand &CondReg,
3313 const MachineOperand &OrigCond) {
3314 CondReg.setIsUndef(OrigCond.isUndef());
3315 CondReg.setIsKill(OrigCond.isKill());
3316}
3317
3318unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,
3319 MachineBasicBlock *TBB,
3320 MachineBasicBlock *FBB,
3321 ArrayRef<MachineOperand> Cond,
3322 const DebugLoc &DL,
3323 int *BytesAdded) const {
3324 if (!FBB && Cond.empty()) {
3325 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
3326 .addMBB(MBB: TBB);
3327 if (BytesAdded)
3328 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3329 return 1;
3330 }
3331
3332 assert(TBB && Cond[0].isImm());
3333
3334 unsigned Opcode
3335 = getBranchOpcode(Cond: static_cast<BranchPredicate>(Cond[0].getImm()));
3336
3337 if (!FBB) {
3338 MachineInstr *CondBr =
3339 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode))
3340 .addMBB(MBB: TBB);
3341
3342 // Copy the flags onto the implicit condition register operand.
3343 preserveCondRegFlags(CondReg&: CondBr->getOperand(i: 1), OrigCond: Cond[1]);
3344 fixImplicitOperands(MI&: *CondBr);
3345
3346 if (BytesAdded)
3347 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3348 return 1;
3349 }
3350
3351 assert(TBB && FBB);
3352
3353 MachineInstr *CondBr =
3354 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode))
3355 .addMBB(MBB: TBB);
3356 fixImplicitOperands(MI&: *CondBr);
3357 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
3358 .addMBB(MBB: FBB);
3359
3360 MachineOperand &CondReg = CondBr->getOperand(i: 1);
3361 CondReg.setIsUndef(Cond[1].isUndef());
3362 CondReg.setIsKill(Cond[1].isKill());
3363
3364 if (BytesAdded)
3365 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8;
3366
3367 return 2;
3368}
3369
3370bool SIInstrInfo::reverseBranchCondition(
3371 SmallVectorImpl<MachineOperand> &Cond) const {
3372 if (Cond.size() != 2) {
3373 return true;
3374 }
3375
3376 if (Cond[0].isImm()) {
3377 Cond[0].setImm(-Cond[0].getImm());
3378 return false;
3379 }
3380
3381 return true;
3382}
3383
3384namespace {
3385class AMDGPUPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
3386private:
3387 /// The compare instruction for loop control
3388 const MachineInstr *CmpInst = nullptr;
3389 /// The normalized condition used by createTripCountGreaterCondition()
3390 SmallVector<MachineOperand, 2> Cond;
3391
3392public:
3393 AMDGPUPipelinerLoopInfo(MachineInstr *CmpInst,
3394 const SmallVectorImpl<MachineOperand> &Cond)
3395 : CmpInst(CmpInst), Cond(Cond.begin(), Cond.end()) {}
3396
3397 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
3398 return CmpInst && MI == CmpInst;
3399 }
3400
3401 std::optional<bool> createTripCountGreaterCondition(
3402 int TC, MachineBasicBlock &MBB,
3403 SmallVectorImpl<MachineOperand> &CondParam) override {
3404 CondParam = this->Cond;
3405 return {};
3406 }
3407
3408 void adjustTripCount(int TripCountAdjust) override {}
3409
3410 void setPreheader(MachineBasicBlock *NewPreheader) override {}
3411};
3412} // namespace
3413
3414std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
3415SIInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
3416 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
3417 SmallVector<MachineOperand, 2> Cond;
3418 // Unanalyzable terminator.
3419 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))
3420 return nullptr;
3421
3422 // Infinite loops are not supported.
3423 if (TBB == LoopBB && FBB == LoopBB)
3424 return nullptr;
3425
3426 // Must be conditional branch.
3427 if (FBB == nullptr)
3428 return nullptr;
3429
3430 assert((TBB == LoopBB || FBB == LoopBB) &&
3431 "The Loop must be a single-basic-block loop");
3432
3433 // Divergent (VCC/EXEC) back-edge is not supported.
3434 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
3435 if (Pred != SCC_TRUE && Pred != SCC_FALSE)
3436 return nullptr;
3437
3438 // Calls and inline assembly are not supported.
3439 for (const MachineInstr &MI : *LoopBB)
3440 if (MI.isCall() || MI.isInlineAsm())
3441 return nullptr;
3442
3443 // Normalization for createTripCountGreaterCondition(): make Cond mean
3444 // "exit the loop" so the expander emits correct prolog guard branches.
3445 if (TBB == LoopBB)
3446 reverseBranchCondition(Cond);
3447
3448 auto Instructions = make_range(
3449 x: MachineBasicBlock::reverse_iterator(LoopBB->getFirstTerminator()),
3450 y: LoopBB->rend());
3451 auto CmpI = llvm::find_if(Range&: Instructions, P: [&](const MachineInstr &MI) {
3452 return MI.modifiesRegister(Reg: Cond[1].getReg(), TRI: &RI);
3453 });
3454
3455 if (CmpI == Instructions.end() || CmpI->isPHI())
3456 return nullptr;
3457 MachineInstr *CmpInst = &*CmpI;
3458
3459 return std::make_unique<AMDGPUPipelinerLoopInfo>(args&: CmpInst, args&: Cond);
3460}
3461
3462bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
3463 ArrayRef<MachineOperand> Cond,
3464 Register DstReg, Register TrueReg,
3465 Register FalseReg, int &CondCycles,
3466 int &TrueCycles, int &FalseCycles) const {
3467 switch (Cond[0].getImm()) {
3468 case VCCNZ:
3469 case VCCZ: {
3470 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3471 const TargetRegisterClass *RC = MRI.getRegClass(Reg: TrueReg);
3472 if (MRI.getRegClass(Reg: FalseReg) != RC)
3473 return false;
3474
3475 int NumInsts = AMDGPU::getRegBitWidth(RC: *RC) / 32;
3476 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
3477
3478 // Limit to equal cost for branch vs. N v_cndmask_b32s.
3479 return RI.hasVGPRs(RC) && NumInsts <= 6;
3480 }
3481 case SCC_TRUE:
3482 case SCC_FALSE: {
3483 // FIXME: We could insert for VGPRs if we could replace the original compare
3484 // with a vector one.
3485 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3486 const TargetRegisterClass *RC = MRI.getRegClass(Reg: TrueReg);
3487 if (MRI.getRegClass(Reg: FalseReg) != RC)
3488 return false;
3489
3490 int NumInsts = AMDGPU::getRegBitWidth(RC: *RC) / 32;
3491
3492 // Multiples of 8 can do s_cselect_b64
3493 if (NumInsts % 2 == 0)
3494 NumInsts /= 2;
3495
3496 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
3497 return RI.isSGPRClass(RC);
3498 }
3499 default:
3500 return false;
3501 }
3502}
3503
3504void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,
3505 MachineBasicBlock::iterator I, const DebugLoc &DL,
3506 Register DstReg, ArrayRef<MachineOperand> Cond,
3507 Register TrueReg, Register FalseReg) const {
3508 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
3509 if (Pred == VCCZ || Pred == SCC_FALSE) {
3510 Pred = static_cast<BranchPredicate>(-Pred);
3511 std::swap(a&: TrueReg, b&: FalseReg);
3512 }
3513
3514 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3515 const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: DstReg);
3516 unsigned DstSize = RI.getRegSizeInBits(RC: *DstRC);
3517
3518 if (DstSize == 32) {
3519 MachineInstr *Select;
3520 if (Pred == SCC_TRUE) {
3521 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: DstReg)
3522 .addReg(RegNo: TrueReg)
3523 .addReg(RegNo: FalseReg);
3524 } else {
3525 // Instruction's operands are backwards from what is expected.
3526 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e32), DestReg: DstReg)
3527 .addReg(RegNo: FalseReg)
3528 .addReg(RegNo: TrueReg);
3529 }
3530
3531 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3532 return;
3533 }
3534
3535 if (DstSize == 64 && Pred == SCC_TRUE) {
3536 MachineInstr *Select =
3537 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B64), DestReg: DstReg)
3538 .addReg(RegNo: TrueReg)
3539 .addReg(RegNo: FalseReg);
3540
3541 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3542 return;
3543 }
3544
3545 static const int16_t Sub0_15[] = {
3546 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
3547 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
3548 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
3549 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
3550 };
3551
3552 static const int16_t Sub0_15_64[] = {
3553 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
3554 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
3555 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
3556 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
3557 };
3558
3559 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
3560 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
3561 const int16_t *SubIndices = Sub0_15;
3562 int NElts = DstSize / 32;
3563
3564 // 64-bit select is only available for SALU.
3565 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
3566 if (Pred == SCC_TRUE) {
3567 if (NElts % 2) {
3568 SelOp = AMDGPU::S_CSELECT_B32;
3569 EltRC = &AMDGPU::SGPR_32RegClass;
3570 } else {
3571 SelOp = AMDGPU::S_CSELECT_B64;
3572 EltRC = &AMDGPU::SGPR_64RegClass;
3573 SubIndices = Sub0_15_64;
3574 NElts /= 2;
3575 }
3576 }
3577
3578 MachineInstrBuilder MIB = BuildMI(
3579 BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
3580
3581 I = MIB->getIterator();
3582
3583 SmallVector<Register, 8> Regs;
3584 for (int Idx = 0; Idx != NElts; ++Idx) {
3585 Register DstElt = MRI.createVirtualRegister(RegClass: EltRC);
3586 Regs.push_back(Elt: DstElt);
3587
3588 unsigned SubIdx = SubIndices[Idx];
3589
3590 MachineInstr *Select;
3591 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) {
3592 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SelOp), DestReg: DstElt)
3593 .addReg(RegNo: FalseReg, Flags: {}, SubReg: SubIdx)
3594 .addReg(RegNo: TrueReg, Flags: {}, SubReg: SubIdx);
3595 } else {
3596 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SelOp), DestReg: DstElt)
3597 .addReg(RegNo: TrueReg, Flags: {}, SubReg: SubIdx)
3598 .addReg(RegNo: FalseReg, Flags: {}, SubReg: SubIdx);
3599 }
3600
3601 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3602 fixImplicitOperands(MI&: *Select);
3603
3604 MIB.addReg(RegNo: DstElt)
3605 .addImm(Val: SubIdx);
3606 }
3607}
3608
3609bool SIInstrInfo::isXcntDrain(const MachineInstr &MI) {
3610
3611 if (MI.isBranch() || MI.isCall() || MI.isReturn() || MI.isIndirectBranch())
3612 return true;
3613
3614 switch (MI.getOpcode()) {
3615 case AMDGPU::S_ENDPGM:
3616 case AMDGPU::S_ENDPGM_SAVED:
3617 case AMDGPU::S_TRAP:
3618 case AMDGPU::S_GETREG_B32:
3619 case AMDGPU::S_SETREG_B32:
3620 case AMDGPU::S_SETREG_B32_mode:
3621 case AMDGPU::S_SETREG_IMM32_B32:
3622 case AMDGPU::S_SETREG_IMM32_B32_mode:
3623 case AMDGPU::S_SENDMSG:
3624 case AMDGPU::S_SENDMSGHALT:
3625 case AMDGPU::S_SENDMSG_RTN_B32:
3626 case AMDGPU::S_SENDMSG_RTN_B64:
3627 case AMDGPU::S_BARRIER_WAIT:
3628 case AMDGPU::S_BARRIER_SIGNAL_M0:
3629 case AMDGPU::S_BARRIER_SIGNAL_IMM:
3630 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0:
3631 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM:
3632 return true;
3633 default:
3634 return false;
3635 }
3636}
3637
3638bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) {
3639 switch (MI.getOpcode()) {
3640 case AMDGPU::V_MOV_B16_t16_e32:
3641 case AMDGPU::V_MOV_B16_t16_e64:
3642 case AMDGPU::V_MOV_B32_e32:
3643 case AMDGPU::V_MOV_B32_e64:
3644 case AMDGPU::V_MOV_B64_PSEUDO:
3645 case AMDGPU::V_MOV_B64_e32:
3646 case AMDGPU::V_MOV_B64_e64:
3647 case AMDGPU::S_MOV_B32:
3648 case AMDGPU::S_MOV_B64:
3649 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3650 case AMDGPU::COPY:
3651 case AMDGPU::WWM_COPY:
3652 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3653 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3654 case AMDGPU::V_ACCVGPR_MOV_B32:
3655 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
3656 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
3657 return true;
3658 default:
3659 return false;
3660 }
3661}
3662
3663unsigned SIInstrInfo::getFoldableCopySrcIdx(const MachineInstr &MI) {
3664 switch (MI.getOpcode()) {
3665 case AMDGPU::V_MOV_B16_t16_e32:
3666 case AMDGPU::V_MOV_B16_t16_e64:
3667 return 2;
3668 case AMDGPU::V_MOV_B32_e32:
3669 case AMDGPU::V_MOV_B32_e64:
3670 case AMDGPU::V_MOV_B64_PSEUDO:
3671 case AMDGPU::V_MOV_B64_e32:
3672 case AMDGPU::V_MOV_B64_e64:
3673 case AMDGPU::S_MOV_B32:
3674 case AMDGPU::S_MOV_B64:
3675 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3676 case AMDGPU::COPY:
3677 case AMDGPU::WWM_COPY:
3678 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3679 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3680 case AMDGPU::V_ACCVGPR_MOV_B32:
3681 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
3682 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
3683 return 1;
3684 default:
3685 llvm_unreachable("MI is not a foldable copy");
3686 }
3687}
3688
3689static constexpr AMDGPU::OpName ModifierOpNames[] = {
3690 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
3691 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
3692 AMDGPU::OpName::omod, AMDGPU::OpName::op_sel};
3693
3694void SIInstrInfo::removeModOperands(MachineInstr &MI) const {
3695 unsigned Opc = MI.getOpcode();
3696 for (AMDGPU::OpName Name : reverse(C: ModifierOpNames)) {
3697 int Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name);
3698 if (Idx >= 0)
3699 MI.removeOperand(OpNo: Idx);
3700 }
3701}
3702
3703void SIInstrInfo::mutateAndCleanupImplicit(MachineInstr &MI,
3704 const MCInstrDesc &NewDesc) const {
3705 MI.setDesc(NewDesc);
3706
3707 // Remove any leftover implicit operands from mutating the instruction. e.g.
3708 // if we replace an s_and_b32 with a copy, we don't need the implicit scc def
3709 // anymore.
3710 const MCInstrDesc &Desc = MI.getDesc();
3711 unsigned NumOps = Desc.getNumOperands() + Desc.implicit_uses().size() +
3712 Desc.implicit_defs().size();
3713
3714 for (unsigned I = MI.getNumOperands() - 1; I >= NumOps; --I)
3715 MI.removeOperand(OpNo: I);
3716}
3717
3718std::optional<int64_t> SIInstrInfo::extractSubregFromImm(int64_t Imm,
3719 unsigned SubRegIndex) {
3720 switch (SubRegIndex) {
3721 case AMDGPU::NoSubRegister:
3722 return Imm;
3723 case AMDGPU::sub0:
3724 return SignExtend64<32>(x: Imm);
3725 case AMDGPU::sub1:
3726 return SignExtend64<32>(x: Imm >> 32);
3727 case AMDGPU::lo16:
3728 return SignExtend64<16>(x: Imm);
3729 case AMDGPU::hi16:
3730 return SignExtend64<16>(x: Imm >> 16);
3731 case AMDGPU::sub1_lo16:
3732 return SignExtend64<16>(x: Imm >> 32);
3733 case AMDGPU::sub1_hi16:
3734 return SignExtend64<16>(x: Imm >> 48);
3735 default:
3736 return std::nullopt;
3737 }
3738
3739 llvm_unreachable("covered subregister switch");
3740}
3741
3742static unsigned getNewFMAAKInst(const GCNSubtarget &ST, unsigned Opc) {
3743 switch (Opc) {
3744 case AMDGPU::V_MAC_F16_e32:
3745 case AMDGPU::V_MAC_F16_e64:
3746 case AMDGPU::V_MAD_F16_e64:
3747 return AMDGPU::V_MADAK_F16;
3748 case AMDGPU::V_MAC_F32_e32:
3749 case AMDGPU::V_MAC_F32_e64:
3750 case AMDGPU::V_MAD_F32_e64:
3751 return AMDGPU::V_MADAK_F32;
3752 case AMDGPU::V_FMAC_F32_e32:
3753 case AMDGPU::V_FMAC_F32_e64:
3754 case AMDGPU::V_FMA_F32_e64:
3755 return AMDGPU::V_FMAAK_F32;
3756 case AMDGPU::V_FMAC_F16_e32:
3757 case AMDGPU::V_FMAC_F16_e64:
3758 case AMDGPU::V_FMAC_F16_t16_e64:
3759 case AMDGPU::V_FMAC_F16_fake16_e64:
3760 case AMDGPU::V_FMAC_F16_t16_e32:
3761 case AMDGPU::V_FMAC_F16_fake16_e32:
3762 case AMDGPU::V_FMA_F16_e64:
3763 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
3764 ? AMDGPU::V_FMAAK_F16_t16
3765 : AMDGPU::V_FMAAK_F16_fake16
3766 : AMDGPU::V_FMAAK_F16;
3767 case AMDGPU::V_FMAC_F64_e32:
3768 case AMDGPU::V_FMAC_F64_e64:
3769 case AMDGPU::V_FMA_F64_e64:
3770 return AMDGPU::V_FMAAK_F64;
3771 default:
3772 llvm_unreachable("invalid instruction");
3773 }
3774}
3775
3776static unsigned getNewFMAMKInst(const GCNSubtarget &ST, unsigned Opc) {
3777 switch (Opc) {
3778 case AMDGPU::V_MAC_F16_e32:
3779 case AMDGPU::V_MAC_F16_e64:
3780 case AMDGPU::V_MAD_F16_e64:
3781 return AMDGPU::V_MADMK_F16;
3782 case AMDGPU::V_MAC_F32_e32:
3783 case AMDGPU::V_MAC_F32_e64:
3784 case AMDGPU::V_MAD_F32_e64:
3785 return AMDGPU::V_MADMK_F32;
3786 case AMDGPU::V_FMAC_F32_e32:
3787 case AMDGPU::V_FMAC_F32_e64:
3788 case AMDGPU::V_FMA_F32_e64:
3789 return AMDGPU::V_FMAMK_F32;
3790 case AMDGPU::V_FMAC_F16_e32:
3791 case AMDGPU::V_FMAC_F16_e64:
3792 case AMDGPU::V_FMAC_F16_t16_e64:
3793 case AMDGPU::V_FMAC_F16_fake16_e64:
3794 case AMDGPU::V_FMAC_F16_t16_e32:
3795 case AMDGPU::V_FMAC_F16_fake16_e32:
3796 case AMDGPU::V_FMA_F16_e64:
3797 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
3798 ? AMDGPU::V_FMAMK_F16_t16
3799 : AMDGPU::V_FMAMK_F16_fake16
3800 : AMDGPU::V_FMAMK_F16;
3801 case AMDGPU::V_FMAC_F64_e32:
3802 case AMDGPU::V_FMAC_F64_e64:
3803 case AMDGPU::V_FMA_F64_e64:
3804 return AMDGPU::V_FMAMK_F64;
3805 default:
3806 llvm_unreachable("invalid instruction");
3807 }
3808}
3809
3810bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
3811 Register Reg, MachineRegisterInfo *MRI) const {
3812 int64_t Imm;
3813 if (!getConstValDefinedInReg(MI: DefMI, Reg, ImmVal&: Imm))
3814 return false;
3815
3816 const bool HasMultipleUses = !MRI->hasOneNonDBGUse(RegNo: Reg);
3817
3818 assert(!DefMI.getOperand(0).getSubReg() && "Expected SSA form");
3819
3820 unsigned Opc = UseMI.getOpcode();
3821 if (Opc == AMDGPU::COPY) {
3822 assert(!UseMI.getOperand(0).getSubReg() && "Expected SSA form");
3823
3824 Register DstReg = UseMI.getOperand(i: 0).getReg();
3825 Register UseSubReg = UseMI.getOperand(i: 1).getSubReg();
3826
3827 const TargetRegisterClass *DstRC = RI.getRegClassForReg(MRI: *MRI, Reg: DstReg);
3828
3829 if (HasMultipleUses) {
3830 // TODO: This should fold in more cases with multiple use, but we need to
3831 // more carefully consider what those uses are.
3832 unsigned ImmDefSize = RI.getRegSizeInBits(RC: *MRI->getRegClass(Reg));
3833
3834 // Avoid breaking up a 64-bit inline immediate into a subregister extract.
3835 if (UseSubReg != AMDGPU::NoSubRegister && ImmDefSize == 64)
3836 return false;
3837
3838 // Most of the time folding a 32-bit inline constant is free (though this
3839 // might not be true if we can't later fold it into a real user).
3840 //
3841 // FIXME: This isInlineConstant check is imprecise if
3842 // getConstValDefinedInReg handled the tricky non-mov cases.
3843 if (ImmDefSize == 32 &&
3844 !isInlineConstant(ImmVal: Imm, OperandType: AMDGPU::OPERAND_REG_IMM_INT32))
3845 return false;
3846 }
3847
3848 bool Is16Bit = UseSubReg != AMDGPU::NoSubRegister &&
3849 RI.getSubRegIdxSize(Idx: UseSubReg) == 16;
3850
3851 if (Is16Bit) {
3852 if (RI.hasVGPRs(RC: DstRC))
3853 return false; // Do not clobber vgpr_hi16
3854
3855 if (DstReg.isVirtual() && UseSubReg != AMDGPU::lo16)
3856 return false;
3857 }
3858
3859 MachineFunction *MF = UseMI.getMF();
3860
3861 unsigned NewOpc = AMDGPU::INSTRUCTION_LIST_END;
3862 MCRegister MovDstPhysReg =
3863 DstReg.isPhysical() ? DstReg.asMCReg() : MCRegister();
3864
3865 std::optional<int64_t> SubRegImm = extractSubregFromImm(Imm, SubRegIndex: UseSubReg);
3866
3867 // TODO: Try to fold with AMDGPU::V_MOV_B16_t16_e64
3868 for (unsigned MovOp :
3869 {AMDGPU::S_MOV_B32, AMDGPU::V_MOV_B32_e32, AMDGPU::S_MOV_B64,
3870 AMDGPU::V_MOV_B64_PSEUDO, AMDGPU::V_ACCVGPR_WRITE_B32_e64}) {
3871 const MCInstrDesc &MovDesc = get(Opcode: MovOp);
3872
3873 const TargetRegisterClass *MovDstRC = getRegClass(MCID: MovDesc, OpNum: 0);
3874 if (Is16Bit) {
3875 // We just need to find a correctly sized register class, so the
3876 // subregister index compatibility doesn't matter since we're statically
3877 // extracting the immediate value.
3878 MovDstRC = RI.getMatchingSuperRegClass(A: MovDstRC, B: DstRC, Idx: AMDGPU::lo16);
3879 if (!MovDstRC)
3880 continue;
3881
3882 if (MovDstPhysReg) {
3883 // FIXME: We probably should not do this. If there is a live value in
3884 // the high half of the register, it will be corrupted.
3885 MovDstPhysReg =
3886 RI.getMatchingSuperReg(Reg: MovDstPhysReg, SubIdx: AMDGPU::lo16, RC: MovDstRC);
3887 if (!MovDstPhysReg)
3888 continue;
3889 }
3890 }
3891
3892 // Result class isn't the right size, try the next instruction.
3893 if (MovDstPhysReg) {
3894 if (!MovDstRC->contains(Reg: MovDstPhysReg))
3895 return false;
3896 } else if (!MRI->constrainRegClass(Reg: DstReg, RC: MovDstRC)) {
3897 // TODO: This will be overly conservative in the case of 16-bit virtual
3898 // SGPRs. We could hack up the virtual register uses to use a compatible
3899 // 32-bit class.
3900 continue;
3901 }
3902
3903 const MCOperandInfo &OpInfo = MovDesc.operands()[1];
3904
3905 // Ensure the interpreted immediate value is a valid operand in the new
3906 // mov.
3907 //
3908 // FIXME: isImmOperandLegal should have form that doesn't require existing
3909 // MachineInstr or MachineOperand
3910 if (!RI.opCanUseLiteralConstant(OpType: OpInfo.OperandType) &&
3911 !isInlineConstant(ImmVal: *SubRegImm, OperandType: OpInfo.OperandType))
3912 break;
3913
3914 NewOpc = MovOp;
3915 break;
3916 }
3917
3918 if (NewOpc == AMDGPU::INSTRUCTION_LIST_END)
3919 return false;
3920
3921 if (Is16Bit) {
3922 UseMI.getOperand(i: 0).setSubReg(AMDGPU::NoSubRegister);
3923 if (MovDstPhysReg)
3924 UseMI.getOperand(i: 0).setReg(MovDstPhysReg);
3925 assert(UseMI.getOperand(1).getReg().isVirtual());
3926 }
3927
3928 const MCInstrDesc &NewMCID = get(Opcode: NewOpc);
3929 UseMI.setDesc(NewMCID);
3930 UseMI.getOperand(i: 1).ChangeToImmediate(ImmVal: *SubRegImm);
3931 UseMI.addImplicitDefUseOperands(MF&: *MF);
3932 return true;
3933 }
3934
3935 if (HasMultipleUses)
3936 return false;
3937
3938 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
3939 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3940 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3941 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3942 Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
3943 Opc == AMDGPU::V_FMAC_F16_fake16_e64 || Opc == AMDGPU::V_FMA_F64_e64 ||
3944 Opc == AMDGPU::V_FMAC_F64_e64) {
3945 // Don't fold if we are using source or output modifiers. The new VOP2
3946 // instructions don't have them.
3947 if (hasAnyModifiersSet(MI: UseMI))
3948 return false;
3949
3950 // If this is a free constant, there's no reason to do this.
3951 // TODO: We could fold this here instead of letting SIFoldOperands do it
3952 // later.
3953 int Src0Idx = getNamedOperandIdx(Opcode: UseMI.getOpcode(), Name: AMDGPU::OpName::src0);
3954
3955 // Any src operand can be used for the legality check.
3956 if (isInlineConstant(MI: UseMI, OpIdx: Src0Idx, ImmVal: Imm))
3957 return false;
3958
3959 MachineOperand *Src0 = &UseMI.getOperand(i: Src0Idx);
3960
3961 MachineOperand *Src1 = getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src1);
3962 MachineOperand *Src2 = getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src2);
3963
3964 auto CopyRegOperandToNarrowerRC =
3965 [MRI, this](MachineInstr &MI, unsigned OpNo,
3966 const TargetRegisterClass *NewRC) -> void {
3967 if (!MI.getOperand(i: OpNo).isReg())
3968 return;
3969 Register Reg = MI.getOperand(i: OpNo).getReg();
3970 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI: *MRI, Reg);
3971 if (RI.getCommonSubClass(A: RC, B: NewRC) != NewRC)
3972 return;
3973 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
3974 BuildMI(BB&: *MI.getParent(), I: MI.getIterator(), MIMD: MI.getDebugLoc(),
3975 MCID: get(Opcode: AMDGPU::COPY), DestReg: Tmp)
3976 .addReg(RegNo: Reg);
3977 MI.getOperand(i: OpNo).setReg(Tmp);
3978 MI.getOperand(i: OpNo).setIsKill();
3979 };
3980
3981 // Multiplied part is the constant: Use v_madmk_{f16, f32}.
3982 if ((Src0->isReg() && Src0->getReg() == Reg) ||
3983 (Src1->isReg() && Src1->getReg() == Reg)) {
3984 MachineOperand *RegSrc =
3985 Src1->isReg() && Src1->getReg() == Reg ? Src0 : Src1;
3986 if (!RegSrc->isReg())
3987 return false;
3988 if (RI.isSGPRClass(RC: MRI->getRegClass(Reg: RegSrc->getReg())) &&
3989 ST.getConstantBusLimit(Opcode: Opc) < 2)
3990 return false;
3991
3992 if (!Src2->isReg() || RI.isSGPRClass(RC: MRI->getRegClass(Reg: Src2->getReg())))
3993 return false;
3994
3995 // If src2 is also a literal constant then we have to choose which one to
3996 // fold. In general it is better to choose madak so that the other literal
3997 // can be materialized in an sgpr instead of a vgpr:
3998 // s_mov_b32 s0, literal
3999 // v_madak_f32 v0, s0, v0, literal
4000 // Instead of:
4001 // v_mov_b32 v1, literal
4002 // v_madmk_f32 v0, v0, literal, v1
4003 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src2->getReg());
4004 if (Def && Def->isMoveImmediate() &&
4005 !isInlineConstant(MO: Def->getOperand(i: 1)))
4006 return false;
4007
4008 unsigned NewOpc = getNewFMAMKInst(ST, Opc);
4009 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
4010 return false;
4011
4012 const std::optional<int64_t> SubRegImm = extractSubregFromImm(
4013 Imm, SubRegIndex: RegSrc == Src1 ? Src0->getSubReg() : Src1->getSubReg());
4014
4015 // FIXME: This would be a lot easier if we could return a new instruction
4016 // instead of having to modify in place.
4017
4018 Register SrcReg = RegSrc->getReg();
4019 unsigned SrcSubReg = RegSrc->getSubReg();
4020 Src0->setReg(SrcReg);
4021 Src0->setSubReg(SrcSubReg);
4022 Src0->setIsKill(RegSrc->isKill());
4023
4024 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
4025 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4026 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
4027 Opc == AMDGPU::V_FMAC_F16_e64 || Opc == AMDGPU::V_FMAC_F64_e64)
4028 UseMI.untieRegOperand(
4029 OpIdx: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2));
4030
4031 Src1->ChangeToImmediate(ImmVal: *SubRegImm);
4032
4033 removeModOperands(MI&: UseMI);
4034 UseMI.setDesc(get(Opcode: NewOpc));
4035
4036 if (NewOpc == AMDGPU::V_FMAMK_F16_t16 ||
4037 NewOpc == AMDGPU::V_FMAMK_F16_fake16) {
4038 const TargetRegisterClass *NewRC = getRegClass(MCID: get(Opcode: NewOpc), OpNum: 0);
4039 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
4040 BuildMI(BB&: *UseMI.getParent(), I: std::next(x: UseMI.getIterator()),
4041 MIMD: UseMI.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
4042 DestReg: UseMI.getOperand(i: 0).getReg())
4043 .addReg(RegNo: Tmp, Flags: RegState::Kill);
4044 UseMI.getOperand(i: 0).setReg(Tmp);
4045 CopyRegOperandToNarrowerRC(UseMI, 1, NewRC);
4046 CopyRegOperandToNarrowerRC(UseMI, 3, NewRC);
4047 }
4048
4049 bool DeleteDef = MRI->use_nodbg_empty(RegNo: Reg);
4050 if (DeleteDef)
4051 DefMI.eraseFromParent();
4052
4053 return true;
4054 }
4055
4056 // Added part is the constant: Use v_madak_{f16, f32}.
4057 if (Src2->isReg() && Src2->getReg() == Reg) {
4058 if (ST.getConstantBusLimit(Opcode: Opc) < 2) {
4059 // Not allowed to use constant bus for another operand.
4060 // We can however allow an inline immediate as src0.
4061 bool Src0Inlined = false;
4062 if (Src0->isReg()) {
4063 // Try to inline constant if possible.
4064 // If the Def moves immediate and the use is single
4065 // We are saving VGPR here.
4066 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src0->getReg());
4067 if (Def && Def->isMoveImmediate() &&
4068 isInlineConstant(MO: Def->getOperand(i: 1)) &&
4069 MRI->hasOneNonDBGUse(RegNo: Src0->getReg())) {
4070 Src0->ChangeToImmediate(ImmVal: Def->getOperand(i: 1).getImm());
4071 Src0Inlined = true;
4072 } else if (ST.getConstantBusLimit(Opcode: Opc) <= 1 &&
4073 RI.isSGPRReg(MRI: *MRI, Reg: Src0->getReg())) {
4074 return false;
4075 }
4076 // VGPR is okay as Src0 - fallthrough
4077 }
4078
4079 if (Src1->isReg() && !Src0Inlined) {
4080 // We have one slot for inlinable constant so far - try to fill it
4081 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src1->getReg());
4082 if (Def && Def->isMoveImmediate() &&
4083 isInlineConstant(MO: Def->getOperand(i: 1)) &&
4084 MRI->hasOneNonDBGUse(RegNo: Src1->getReg()) && commuteInstruction(MI&: UseMI))
4085 Src0->ChangeToImmediate(ImmVal: Def->getOperand(i: 1).getImm());
4086 else if (RI.isSGPRReg(MRI: *MRI, Reg: Src1->getReg()))
4087 return false;
4088 // VGPR is okay as Src1 - fallthrough
4089 }
4090 }
4091
4092 unsigned NewOpc = getNewFMAAKInst(ST, Opc);
4093 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
4094 return false;
4095
4096 // FIXME: This would be a lot easier if we could return a new instruction
4097 // instead of having to modify in place.
4098
4099 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
4100 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4101 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
4102 Opc == AMDGPU::V_FMAC_F16_e64 || Opc == AMDGPU::V_FMAC_F64_e64)
4103 UseMI.untieRegOperand(
4104 OpIdx: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2));
4105
4106 const std::optional<int64_t> SubRegImm =
4107 extractSubregFromImm(Imm, SubRegIndex: Src2->getSubReg());
4108
4109 // ChangingToImmediate adds Src2 back to the instruction.
4110 Src2->ChangeToImmediate(ImmVal: *SubRegImm);
4111
4112 // These come before src2.
4113 removeModOperands(MI&: UseMI);
4114 UseMI.setDesc(get(Opcode: NewOpc));
4115
4116 if (NewOpc == AMDGPU::V_FMAAK_F16_t16 ||
4117 NewOpc == AMDGPU::V_FMAAK_F16_fake16) {
4118 const TargetRegisterClass *NewRC = getRegClass(MCID: get(Opcode: NewOpc), OpNum: 0);
4119 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
4120 BuildMI(BB&: *UseMI.getParent(), I: std::next(x: UseMI.getIterator()),
4121 MIMD: UseMI.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
4122 DestReg: UseMI.getOperand(i: 0).getReg())
4123 .addReg(RegNo: Tmp, Flags: RegState::Kill);
4124 UseMI.getOperand(i: 0).setReg(Tmp);
4125 CopyRegOperandToNarrowerRC(UseMI, 1, NewRC);
4126 CopyRegOperandToNarrowerRC(UseMI, 2, NewRC);
4127 }
4128
4129 // It might happen that UseMI was commuted
4130 // and we now have SGPR as SRC1. If so 2 inlined
4131 // constant and SGPR are illegal.
4132 legalizeOperands(MI&: UseMI);
4133
4134 bool DeleteDef = MRI->use_nodbg_empty(RegNo: Reg);
4135 if (DeleteDef)
4136 DefMI.eraseFromParent();
4137
4138 return true;
4139 }
4140 }
4141
4142 return false;
4143}
4144
4145static bool
4146memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1,
4147 ArrayRef<const MachineOperand *> BaseOps2) {
4148 if (BaseOps1.size() != BaseOps2.size())
4149 return false;
4150 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) {
4151 if (!BaseOps1[I]->isIdenticalTo(Other: *BaseOps2[I]))
4152 return false;
4153 }
4154 return true;
4155}
4156
4157static bool offsetsDoNotOverlap(LocationSize WidthA, int OffsetA,
4158 LocationSize WidthB, int OffsetB) {
4159 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
4160 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
4161 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
4162 return LowWidth.hasValue() &&
4163 LowOffset + (int)LowWidth.getValue() <= HighOffset;
4164}
4165
4166bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
4167 const MachineInstr &MIb) const {
4168 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;
4169 int64_t Offset0, Offset1;
4170 LocationSize Dummy0 = LocationSize::precise(Value: 0);
4171 LocationSize Dummy1 = LocationSize::precise(Value: 0);
4172 bool Offset0IsScalable, Offset1IsScalable;
4173 if (!getMemOperandsWithOffsetWidth(LdSt: MIa, BaseOps&: BaseOps0, Offset&: Offset0, OffsetIsScalable&: Offset0IsScalable,
4174 Width&: Dummy0, TRI: &RI) ||
4175 !getMemOperandsWithOffsetWidth(LdSt: MIb, BaseOps&: BaseOps1, Offset&: Offset1, OffsetIsScalable&: Offset1IsScalable,
4176 Width&: Dummy1, TRI: &RI))
4177 return false;
4178
4179 if (!memOpsHaveSameBaseOperands(BaseOps1: BaseOps0, BaseOps2: BaseOps1))
4180 return false;
4181
4182 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
4183 // FIXME: Handle ds_read2 / ds_write2.
4184 return false;
4185 }
4186 LocationSize Width0 = MIa.memoperands().front()->getSize();
4187 LocationSize Width1 = MIb.memoperands().front()->getSize();
4188 return offsetsDoNotOverlap(WidthA: Width0, OffsetA: Offset0, WidthB: Width1, OffsetB: Offset1);
4189}
4190
4191bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
4192 const MachineInstr &MIb) const {
4193 assert(MIa.mayLoadOrStore() &&
4194 "MIa must load from or modify a memory location");
4195 assert(MIb.mayLoadOrStore() &&
4196 "MIb must load from or modify a memory location");
4197
4198 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects())
4199 return false;
4200
4201 // XXX - Can we relax this between address spaces?
4202 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
4203 return false;
4204
4205 if (isLDSDMA(MI: MIa) || isLDSDMA(MI: MIb))
4206 return false;
4207
4208 if (MIa.isBundle() || MIb.isBundle())
4209 return false;
4210
4211 // TODO: Should we check the address space from the MachineMemOperand? That
4212 // would allow us to distinguish objects we know don't alias based on the
4213 // underlying address space, even if it was lowered to a different one,
4214 // e.g. private accesses lowered to use MUBUF instructions on a scratch
4215 // buffer.
4216 if (isDS(MI: MIa)) {
4217 if (isDS(MI: MIb))
4218 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4219
4220 return !isFLAT(MI: MIb) || isSegmentSpecificFLAT(MI: MIb);
4221 }
4222
4223 if (isMUBUF(MI: MIa) || isMTBUF(MI: MIa)) {
4224 if (isMUBUF(MI: MIb) || isMTBUF(MI: MIb))
4225 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4226
4227 if (isFLAT(MI: MIb))
4228 return isFLATScratch(MI: MIb);
4229
4230 return !isSMRD(MI: MIb);
4231 }
4232
4233 if (isSMRD(MI: MIa)) {
4234 if (isSMRD(MI: MIb))
4235 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4236
4237 if (isFLAT(MI: MIb))
4238 return isFLATScratch(MI: MIb);
4239
4240 return !isMUBUF(MI: MIb) && !isMTBUF(MI: MIb);
4241 }
4242
4243 if (isFLAT(MI: MIa)) {
4244 if (isFLAT(MI: MIb)) {
4245 if ((isFLATScratch(MI: MIa) && isFLATGlobal(MI: MIb)) ||
4246 (isFLATGlobal(MI: MIa) && isFLATScratch(MI: MIb)))
4247 return true;
4248
4249 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4250 }
4251
4252 return false;
4253 }
4254
4255 return false;
4256}
4257
4258static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI,
4259 MachineInstr &NewMI) {
4260 if (LV) {
4261 unsigned NumOps = MI.getNumOperands();
4262 for (unsigned I = 1; I < NumOps; ++I) {
4263 MachineOperand &Op = MI.getOperand(i: I);
4264 if (Op.isReg() && Op.isKill())
4265 LV->replaceKillInstruction(Reg: Op.getReg(), OldMI&: MI, NewMI);
4266 }
4267 }
4268}
4269
4270static unsigned getNewFMAInst(const GCNSubtarget &ST, unsigned Opc) {
4271 switch (Opc) {
4272 case AMDGPU::V_MAC_F16_e32:
4273 case AMDGPU::V_MAC_F16_e64:
4274 return AMDGPU::V_MAD_F16_e64;
4275 case AMDGPU::V_MAC_F32_e32:
4276 case AMDGPU::V_MAC_F32_e64:
4277 return AMDGPU::V_MAD_F32_e64;
4278 case AMDGPU::V_MAC_LEGACY_F32_e32:
4279 case AMDGPU::V_MAC_LEGACY_F32_e64:
4280 return AMDGPU::V_MAD_LEGACY_F32_e64;
4281 case AMDGPU::V_FMAC_LEGACY_F32_e32:
4282 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4283 return AMDGPU::V_FMA_LEGACY_F32_e64;
4284 case AMDGPU::V_FMAC_F16_e32:
4285 case AMDGPU::V_FMAC_F16_e64:
4286 case AMDGPU::V_FMAC_F16_t16_e64:
4287 case AMDGPU::V_FMAC_F16_fake16_e64:
4288 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
4289 ? AMDGPU::V_FMA_F16_gfx9_t16_e64
4290 : AMDGPU::V_FMA_F16_gfx9_fake16_e64
4291 : AMDGPU::V_FMA_F16_gfx9_e64;
4292 case AMDGPU::V_FMAC_F32_e32:
4293 case AMDGPU::V_FMAC_F32_e64:
4294 return AMDGPU::V_FMA_F32_e64;
4295 case AMDGPU::V_FMAC_F64_e32:
4296 case AMDGPU::V_FMAC_F64_e64:
4297 return AMDGPU::V_FMA_F64_e64;
4298 default:
4299 llvm_unreachable("invalid instruction");
4300 }
4301}
4302
4303/// Helper struct for the implementation of 3-address conversion to communicate
4304/// updates made to instruction operands.
4305struct SIInstrInfo::ThreeAddressUpdates {
4306 /// Other instruction whose def is no longer used by the converted
4307 /// instruction.
4308 MachineInstr *RemoveMIUse = nullptr;
4309};
4310
4311MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI,
4312 LiveVariables *LV,
4313 LiveIntervals *LIS) const {
4314 MachineBasicBlock &MBB = *MI.getParent();
4315 MachineInstr *CandidateMI = &MI;
4316
4317 if (MI.isBundle()) {
4318 // This is a temporary placeholder for bundle handling that enables us to
4319 // exercise the relevant code paths in the two-address instruction pass.
4320 if (MI.getBundleSize() != 1)
4321 return nullptr;
4322 CandidateMI = MI.getNextNode();
4323 }
4324
4325 ThreeAddressUpdates U;
4326 MachineInstr *NewMI = convertToThreeAddressImpl(MI&: *CandidateMI, Updates&: U);
4327 if (!NewMI)
4328 return nullptr;
4329
4330 if (MI.isBundle()) {
4331 CandidateMI->eraseFromBundle();
4332
4333 for (MachineOperand &MO : MI.all_defs()) {
4334 if (MO.isTied())
4335 MI.untieRegOperand(OpIdx: MO.getOperandNo());
4336 }
4337 } else {
4338 updateLiveVariables(LV, MI, NewMI&: *NewMI);
4339 if (LIS) {
4340 LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *NewMI);
4341 // SlotIndex of defs needs to be updated when converting to early-clobber
4342 MachineOperand &Def = NewMI->getOperand(i: 0);
4343 if (Def.isEarlyClobber() && Def.isReg() &&
4344 LIS->hasInterval(Reg: Def.getReg())) {
4345 SlotIndex OldIndex = LIS->getInstructionIndex(Instr: *NewMI).getRegSlot(EC: false);
4346 SlotIndex NewIndex = LIS->getInstructionIndex(Instr: *NewMI).getRegSlot(EC: true);
4347 auto &LI = LIS->getInterval(Reg: Def.getReg());
4348 auto UpdateDefIndex = [&](LiveRange &LR) {
4349 auto *S = LR.find(Pos: OldIndex);
4350 if (S != LR.end() && S->start == OldIndex) {
4351 assert(S->valno && S->valno->def == OldIndex);
4352 S->start = NewIndex;
4353 S->valno->def = NewIndex;
4354 }
4355 };
4356 UpdateDefIndex(LI);
4357 for (auto &SR : LI.subranges())
4358 UpdateDefIndex(SR);
4359 }
4360 }
4361 }
4362
4363 if (U.RemoveMIUse) {
4364 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4365 // The only user is the instruction which will be killed.
4366 Register DefReg = U.RemoveMIUse->getOperand(i: 0).getReg();
4367
4368 if (MRI.hasOneNonDBGUse(RegNo: DefReg)) {
4369 // We cannot just remove the DefMI here, calling pass will crash.
4370 U.RemoveMIUse->setDesc(get(Opcode: AMDGPU::IMPLICIT_DEF));
4371 U.RemoveMIUse->getOperand(i: 0).setIsDead(true);
4372 for (unsigned I = U.RemoveMIUse->getNumOperands() - 1; I != 0; --I)
4373 U.RemoveMIUse->removeOperand(OpNo: I);
4374 if (LV)
4375 LV->getVarInfo(Reg: DefReg).AliveBlocks.clear();
4376 }
4377
4378 if (MI.isBundle()) {
4379 VirtRegInfo VRI = AnalyzeVirtRegInBundle(MI, Reg: DefReg);
4380 if (!VRI.Reads && !VRI.Writes) {
4381 for (MachineOperand &MO : MI.all_uses()) {
4382 if (MO.isReg() && MO.getReg() == DefReg) {
4383 assert(MO.getSubReg() == 0 &&
4384 "tied sub-registers in bundles currently not supported");
4385 MI.removeOperand(OpNo: MO.getOperandNo());
4386 break;
4387 }
4388 }
4389
4390 if (LIS)
4391 LIS->shrinkToUses(li: &LIS->getInterval(Reg: DefReg));
4392 }
4393 } else if (LIS) {
4394 LiveInterval &DefLI = LIS->getInterval(Reg: DefReg);
4395
4396 // We cannot delete the original instruction here, so hack out the use
4397 // in the original instruction with a dummy register so we can use
4398 // shrinkToUses to deal with any multi-use edge cases. Other targets do
4399 // not have the complexity of deleting a use to consider here.
4400 Register DummyReg = MRI.cloneVirtualRegister(VReg: DefReg);
4401 for (MachineOperand &MIOp : MI.uses()) {
4402 if (MIOp.isReg() && MIOp.getReg() == DefReg) {
4403 MIOp.setIsUndef(true);
4404 MIOp.setReg(DummyReg);
4405 }
4406 }
4407
4408 if (MI.isBundle()) {
4409 VirtRegInfo VRI = AnalyzeVirtRegInBundle(MI, Reg: DefReg);
4410 if (!VRI.Reads && !VRI.Writes) {
4411 for (MachineOperand &MIOp : MI.uses()) {
4412 if (MIOp.isReg() && MIOp.getReg() == DefReg) {
4413 MIOp.setIsUndef(true);
4414 MIOp.setReg(DummyReg);
4415 }
4416 }
4417 }
4418
4419 MI.addOperand(Op: MachineOperand::CreateReg(Reg: DummyReg, isDef: false, isImp: false, isKill: false,
4420 isDead: false, /*isUndef=*/true));
4421 }
4422
4423 LIS->shrinkToUses(li: &DefLI);
4424 }
4425 }
4426
4427 return MI.isBundle() ? &MI : NewMI;
4428}
4429
4430MachineInstr *
4431SIInstrInfo::convertToThreeAddressImpl(MachineInstr &MI,
4432 ThreeAddressUpdates &U) const {
4433 MachineBasicBlock &MBB = *MI.getParent();
4434 unsigned Opc = MI.getOpcode();
4435
4436 // Handle MFMA.
4437 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opcode: Opc);
4438 if (NewMFMAOpc != -1) {
4439 MachineInstrBuilder MIB =
4440 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewMFMAOpc));
4441 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I)
4442 MIB.add(MO: MI.getOperand(i: I));
4443 return MIB;
4444 }
4445
4446 if (SIInstrInfo::isWMMA(MI)) {
4447 unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(Opc: MI.getOpcode());
4448 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4449 .setMIFlags(MI.getFlags());
4450 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I)
4451 MIB->addOperand(Op: MI.getOperand(i: I));
4452 return MIB;
4453 }
4454
4455 assert(Opc != AMDGPU::V_FMAC_F16_t16_e32 &&
4456 Opc != AMDGPU::V_FMAC_F16_fake16_e32 &&
4457 "V_FMAC_F16_t16/fake16_e32 is not supported and not expected to be "
4458 "present pre-RA");
4459
4460 // Handle MAC/FMAC.
4461 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
4462 bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 ||
4463 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 ||
4464 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
4465 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64;
4466 bool Src0Literal = false;
4467
4468 switch (Opc) {
4469 default:
4470 return nullptr;
4471 case AMDGPU::V_MAC_F16_e64:
4472 case AMDGPU::V_FMAC_F16_e64:
4473 case AMDGPU::V_FMAC_F16_t16_e64:
4474 case AMDGPU::V_FMAC_F16_fake16_e64:
4475 case AMDGPU::V_MAC_F32_e64:
4476 case AMDGPU::V_MAC_LEGACY_F32_e64:
4477 case AMDGPU::V_FMAC_F32_e64:
4478 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4479 case AMDGPU::V_FMAC_F64_e64:
4480 break;
4481 case AMDGPU::V_MAC_F16_e32:
4482 case AMDGPU::V_FMAC_F16_e32:
4483 case AMDGPU::V_MAC_F32_e32:
4484 case AMDGPU::V_MAC_LEGACY_F32_e32:
4485 case AMDGPU::V_FMAC_F32_e32:
4486 case AMDGPU::V_FMAC_LEGACY_F32_e32:
4487 case AMDGPU::V_FMAC_F64_e32: {
4488 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
4489 Name: AMDGPU::OpName::src0);
4490 const MachineOperand *Src0 = &MI.getOperand(i: Src0Idx);
4491 if (!Src0->isReg() && !Src0->isImm())
4492 return nullptr;
4493
4494 if (Src0->isImm() && !isInlineConstant(MI, OpIdx: Src0Idx, MO: *Src0))
4495 Src0Literal = true;
4496
4497 break;
4498 }
4499 }
4500
4501 MachineInstrBuilder MIB;
4502 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
4503 const MachineOperand *Src0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
4504 const MachineOperand *Src0Mods =
4505 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
4506 const MachineOperand *Src1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
4507 const MachineOperand *Src1Mods =
4508 getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
4509 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
4510 const MachineOperand *Src2Mods =
4511 getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers);
4512 const MachineOperand *Clamp = getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
4513 const MachineOperand *Omod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
4514 const MachineOperand *OpSel = getNamedOperand(MI, OperandName: AMDGPU::OpName::op_sel);
4515
4516 if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsLegacy &&
4517 (!IsF64 || ST.hasFmaakFmamkF64Insts()) &&
4518 // If we have an SGPR input, we will violate the constant bus restriction.
4519 (ST.getConstantBusLimit(Opcode: Opc) > 1 || !Src0->isReg() ||
4520 !RI.isSGPRReg(MRI: MBB.getParent()->getRegInfo(), Reg: Src0->getReg()))) {
4521 MachineInstr *DefMI = nullptr;
4522 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4523 std::optional<int64_t> ImmOpt;
4524 int64_t Imm;
4525
4526 if (!Src0Literal &&
4527 (ImmOpt = getImmOrMaterializedImm(MRI, Op: *Src2, DefMI: &DefMI))) {
4528 unsigned NewOpc = getNewFMAAKInst(ST, Opc);
4529 if (pseudoToMCOpcode(Opcode: NewOpc) != -1) {
4530 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4531 .add(MO: *Dst)
4532 .add(MO: *Src0)
4533 .add(MO: *Src1)
4534 .addImm(Val: *ImmOpt)
4535 .setMIFlags(MI.getFlags());
4536 U.RemoveMIUse = DefMI;
4537 return MIB;
4538 }
4539 }
4540 unsigned NewOpc = getNewFMAMKInst(ST, Opc);
4541 if (!Src0Literal &&
4542 (ImmOpt = getImmOrMaterializedImm(MRI, Op: *Src1, DefMI: &DefMI))) {
4543 if (pseudoToMCOpcode(Opcode: NewOpc) != -1) {
4544 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4545 .add(MO: *Dst)
4546 .add(MO: *Src0)
4547 .addImm(Val: *ImmOpt)
4548 .add(MO: *Src2)
4549 .setMIFlags(MI.getFlags());
4550 U.RemoveMIUse = DefMI;
4551 return MIB;
4552 }
4553 }
4554 if ((ImmOpt = getImmOrMaterializedImm(MRI, Op: *Src0, DefMI: &DefMI))) {
4555 Imm = *ImmOpt;
4556 if (pseudoToMCOpcode(Opcode: NewOpc) != -1 &&
4557 isOperandLegal(
4558 MI, OpIdx: AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::src0),
4559 MO: Src1)) {
4560 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4561 .add(MO: *Dst)
4562 .add(MO: *Src1)
4563 .addImm(Val: Imm)
4564 .add(MO: *Src2)
4565 .setMIFlags(MI.getFlags());
4566 U.RemoveMIUse = DefMI;
4567 return MIB;
4568 }
4569 }
4570 }
4571
4572 // VOP2 mac/fmac with a literal operand cannot be converted to VOP3 mad/fma
4573 // if VOP3 does not allow a literal operand.
4574 if (Src0Literal && !ST.hasVOP3Literal())
4575 return nullptr;
4576
4577 unsigned NewOpc = getNewFMAInst(ST, Opc);
4578
4579 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
4580 return nullptr;
4581
4582 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4583 .add(MO: *Dst)
4584 .addImm(Val: Src0Mods ? Src0Mods->getImm() : 0)
4585 .add(MO: *Src0)
4586 .addImm(Val: Src1Mods ? Src1Mods->getImm() : 0)
4587 .add(MO: *Src1)
4588 .addImm(Val: Src2Mods ? Src2Mods->getImm() : 0)
4589 .add(MO: *Src2)
4590 .addImm(Val: Clamp ? Clamp->getImm() : 0)
4591 .addImm(Val: Omod ? Omod->getImm() : 0)
4592 .setMIFlags(MI.getFlags());
4593 if (AMDGPU::hasNamedOperand(Opcode: NewOpc, NamedIdx: AMDGPU::OpName::op_sel))
4594 MIB.addImm(Val: OpSel ? OpSel->getImm() : 0);
4595 return MIB;
4596}
4597
4598// It's not generally safe to move VALU instructions across these since it will
4599// start using the register as a base index rather than directly.
4600// XXX - Why isn't hasSideEffects sufficient for these?
4601static bool changesVGPRIndexingMode(const MachineInstr &MI) {
4602 switch (MI.getOpcode()) {
4603 case AMDGPU::S_SET_GPR_IDX_ON:
4604 case AMDGPU::S_SET_GPR_IDX_MODE:
4605 case AMDGPU::S_SET_GPR_IDX_OFF:
4606 return true;
4607 default:
4608 return false;
4609 }
4610}
4611
4612bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
4613 const MachineBasicBlock *MBB,
4614 const MachineFunction &MF) const {
4615 // Skipping the check for SP writes in the base implementation. The reason it
4616 // was added was apparently due to compile time concerns.
4617 //
4618 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops
4619 // but is probably avoidable.
4620
4621 // Copied from base implementation.
4622 // Terminators and labels can't be scheduled around.
4623 if (MI.isTerminator() || MI.isPosition())
4624 return true;
4625
4626 // INLINEASM_BR can jump to another block
4627 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
4628 return true;
4629
4630 if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(i: 0).getImm() == 0)
4631 return true;
4632
4633 // Target-independent instructions do not have an implicit-use of EXEC, even
4634 // when they operate on VGPRs. Treating EXEC modifications as scheduling
4635 // boundaries prevents incorrect movements of such instructions.
4636 return MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &RI) ||
4637 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
4638 MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
4639 MI.getOpcode() == AMDGPU::S_SETPRIO ||
4640 MI.getOpcode() == AMDGPU::S_SETPRIO_INC_WG ||
4641 changesVGPRIndexingMode(MI);
4642}
4643
4644bool SIInstrInfo::isAlwaysGDS(uint32_t Opcode) const {
4645 return Opcode == AMDGPU::DS_ORDERED_COUNT ||
4646 Opcode == AMDGPU::DS_ADD_GS_REG_RTN ||
4647 Opcode == AMDGPU::DS_SUB_GS_REG_RTN || isGWS(Opcode);
4648}
4649
4650bool SIInstrInfo::mayAccessScratch(const MachineInstr &MI) const {
4651 // Instructions that access scratch use FLAT encoding or BUF encodings.
4652 if ((!isFLAT(MI) || isFLATGlobal(MI)) && !isBUF(MI))
4653 return false;
4654
4655 // SCRATCH instructions always access scratch.
4656 if (isFLATScratch(MI))
4657 return true;
4658
4659 // If FLAT_SCRATCH registers are not initialized, we can never access scratch
4660 // via the aperture.
4661 if (MI.getMF()->getFunction().hasFnAttribute(Kind: "amdgpu-no-flat-scratch-init"))
4662 return false;
4663
4664 // If there are no memory operands then conservatively assume the flat
4665 // operation may access scratch.
4666 if (MI.memoperands_empty())
4667 return true;
4668
4669 // See if any memory operand specifies an address space that involves scratch.
4670 return any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *Memop) {
4671 unsigned AS = Memop->getAddrSpace();
4672 if (AS == AMDGPUAS::FLAT_ADDRESS) {
4673 const MDNode *MD = Memop->getAAInfo().NoAliasAddrSpace;
4674 return !MD || !AMDGPU::hasValueInRangeLikeMetadata(
4675 MD: *MD, Val: AMDGPUAS::PRIVATE_ADDRESS);
4676 }
4677 return AS == AMDGPUAS::PRIVATE_ADDRESS;
4678 });
4679}
4680
4681bool SIInstrInfo::mayAccessVMEMThroughFlat(const MachineInstr &MI) const {
4682 assert(isFLAT(MI));
4683
4684 // All flat instructions use the VMEM counter except prefetch.
4685 if (!usesVM_CNT(MI))
4686 return false;
4687
4688 // If there are no memory operands then conservatively assume the flat
4689 // operation may access VMEM.
4690 if (MI.memoperands_empty())
4691 return true;
4692
4693 // See if any memory operand specifies an address space that involves VMEM.
4694 // Flat operations only supported FLAT, LOCAL (LDS), or address spaces
4695 // involving VMEM such as GLOBAL, CONSTANT, PRIVATE (SCRATCH), etc. The REGION
4696 // (GDS) address space is not supported by flat operations. Therefore, simply
4697 // return true unless only the LDS address space is found.
4698 for (const MachineMemOperand *Memop : MI.memoperands()) {
4699 unsigned AS = Memop->getAddrSpace();
4700 assert(AS != AMDGPUAS::REGION_ADDRESS);
4701 if (AS != AMDGPUAS::LOCAL_ADDRESS)
4702 return true;
4703 }
4704
4705 return false;
4706}
4707
4708bool SIInstrInfo::mayAccessLDSThroughFlat(const MachineInstr &MI,
4709 bool TgSplit) const {
4710 assert(isFLAT(MI));
4711
4712 // Flat instruction such as SCRATCH and GLOBAL do not use the lgkm counter.
4713 if (!usesLGKM_CNT(MI))
4714 return false;
4715
4716 // If in tgsplit mode then there can be no use of LDS.
4717 if (TgSplit)
4718 return false;
4719
4720 // If there are no memory operands then conservatively assume the flat
4721 // operation may access LDS.
4722 if (MI.memoperands_empty())
4723 return true;
4724
4725 // See if any memory operand specifies an address space that involves LDS.
4726 for (const MachineMemOperand *Memop : MI.memoperands()) {
4727 unsigned AS = Memop->getAddrSpace();
4728 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)
4729 return true;
4730 }
4731
4732 return false;
4733}
4734
4735bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) {
4736 // Skip the full operand and register alias search modifiesRegister
4737 // does. There's only a handful of instructions that touch this, it's only an
4738 // implicit def, and doesn't alias any other registers.
4739 return is_contained(Range: MI.getDesc().implicit_defs(), Element: AMDGPU::MODE);
4740}
4741
4742bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const {
4743 unsigned Opcode = MI.getOpcode();
4744
4745 if (MI.mayStore() && isSMRD(MI))
4746 return true; // scalar store or atomic
4747
4748 // This will terminate the function when other lanes may need to continue.
4749 if (MI.isReturn())
4750 return true;
4751
4752 // These instructions cause shader I/O that may cause hardware lockups
4753 // when executed with an empty EXEC mask.
4754 //
4755 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
4756 // EXEC = 0, but checking for that case here seems not worth it
4757 // given the typical code patterns.
4758 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
4759 isEXP(Opcode) || Opcode == AMDGPU::DS_ORDERED_COUNT ||
4760 Opcode == AMDGPU::S_TRAP || Opcode == AMDGPU::S_WAIT_EVENT ||
4761 Opcode == AMDGPU::S_SETHALT)
4762 return true;
4763
4764 if (MI.isCall() || MI.isInlineAsm())
4765 return true; // conservative assumption
4766
4767 // Assume that barrier interactions are only intended with active lanes.
4768 if (isBarrier(Opcode))
4769 return true;
4770
4771 // A mode change is a scalar operation that influences vector instructions.
4772 if (modifiesModeRegister(MI))
4773 return true;
4774
4775 // These are like SALU instructions in terms of effects, so it's questionable
4776 // whether we should return true for those.
4777 //
4778 // However, executing them with EXEC = 0 causes them to operate on undefined
4779 // data, which we avoid by returning true here.
4780 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 ||
4781 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32 ||
4782 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
4783 Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR)
4784 return true;
4785
4786 return false;
4787}
4788
4789bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI,
4790 const MachineInstr &MI) const {
4791 if (MI.isMetaInstruction())
4792 return false;
4793
4794 // This won't read exec if this is an SGPR->SGPR copy.
4795 if (MI.isCopyLike()) {
4796 if (!RI.isSGPRReg(MRI, Reg: MI.getOperand(i: 0).getReg()))
4797 return true;
4798
4799 // Make sure this isn't copying exec as a normal operand
4800 return MI.readsRegister(Reg: AMDGPU::EXEC, TRI: &RI);
4801 }
4802
4803 // Make a conservative assumption about the callee.
4804 if (MI.isCall())
4805 return true;
4806
4807 // Be conservative with any unhandled generic opcodes.
4808 if (!isTargetSpecificOpcode(Opcode: MI.getOpcode()))
4809 return true;
4810
4811 return !isSALU(MI) || MI.readsRegister(Reg: AMDGPU::EXEC, TRI: &RI);
4812}
4813
4814bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
4815 switch (Imm.getBitWidth()) {
4816 case 1: // This likely will be a condition code mask.
4817 return true;
4818
4819 case 32:
4820 return AMDGPU::isInlinableLiteral32(Literal: Imm.getSExtValue(),
4821 HasInv2Pi: ST.hasInv2PiInlineImm());
4822 case 64:
4823 return AMDGPU::isInlinableLiteral64(Literal: Imm.getSExtValue(),
4824 HasInv2Pi: ST.hasInv2PiInlineImm());
4825 case 16:
4826 return ST.has16BitInsts() &&
4827 AMDGPU::isInlinableLiteralI16(Literal: Imm.getSExtValue(),
4828 HasInv2Pi: ST.hasInv2PiInlineImm());
4829 default:
4830 llvm_unreachable("invalid bitwidth");
4831 }
4832}
4833
4834bool SIInstrInfo::isInlineConstant(const APFloat &Imm) const {
4835 APInt IntImm = Imm.bitcastToAPInt();
4836 int64_t IntImmVal = IntImm.getSExtValue();
4837 bool HasInv2Pi = ST.hasInv2PiInlineImm();
4838 switch (APFloat::SemanticsToEnum(Sem: Imm.getSemantics())) {
4839 default:
4840 llvm_unreachable("invalid fltSemantics");
4841 case APFloatBase::S_IEEEsingle:
4842 case APFloatBase::S_IEEEdouble:
4843 return isInlineConstant(Imm: IntImm);
4844 case APFloatBase::S_BFloat:
4845 return ST.has16BitInsts() &&
4846 AMDGPU::isInlinableLiteralBF16(Literal: IntImmVal, HasInv2Pi);
4847 case APFloatBase::S_IEEEhalf:
4848 return ST.has16BitInsts() &&
4849 AMDGPU::isInlinableLiteralFP16(Literal: IntImmVal, HasInv2Pi);
4850 }
4851}
4852
4853bool SIInstrInfo::isInlineConstant(int64_t Imm, uint8_t OperandType) const {
4854 // MachineOperand provides no way to tell the true operand size, since it only
4855 // records a 64-bit value. We need to know the size to determine if a 32-bit
4856 // floating point immediate bit pattern is legal for an integer immediate. It
4857 // would be for any 32-bit integer operand, but would not be for a 64-bit one.
4858 switch (OperandType) {
4859 case AMDGPU::OPERAND_REG_IMM_INT32:
4860 case AMDGPU::OPERAND_REG_IMM_FP32:
4861 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
4862 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
4863 case AMDGPU::OPERAND_REG_IMM_V2FP32:
4864 case AMDGPU::OPERAND_REG_IMM_V2INT32:
4865 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
4866 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
4867 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: {
4868 int32_t Trunc = static_cast<int32_t>(Imm);
4869 return AMDGPU::isInlinableLiteral32(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
4870 }
4871 case AMDGPU::OPERAND_REG_IMM_INT64:
4872 case AMDGPU::OPERAND_REG_IMM_FP64:
4873 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
4874 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
4875 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
4876 case AMDGPU::OPERAND_REG_IMM_V2FP64:
4877 case AMDGPU::OPERAND_REG_IMM_V2INT64:
4878 return AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: ST.hasInv2PiInlineImm());
4879 case AMDGPU::OPERAND_REG_IMM_INT16:
4880 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
4881 // We would expect inline immediates to not be concerned with an integer/fp
4882 // distinction. However, in the case of 16-bit integer operations, the
4883 // "floating point" values appear to not work. It seems read the low 16-bits
4884 // of 32-bit immediates, which happens to always work for the integer
4885 // values.
4886 //
4887 // See llvm bugzilla 46302.
4888 //
4889 // TODO: Theoretically we could use op-sel to use the high bits of the
4890 // 32-bit FP values.
4891 return AMDGPU::isInlinableIntLiteral(Literal: Imm);
4892 case AMDGPU::OPERAND_REG_IMM_V2INT16:
4893 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
4894 return AMDGPU::isInlinableLiteralV2I16(Literal: Imm);
4895 case AMDGPU::OPERAND_REG_IMM_V2FP16:
4896 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
4897 return AMDGPU::isInlinableLiteralV2F16(Literal: Imm);
4898 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
4899 return AMDGPU::isPKFMACF16InlineConstant(Literal: Imm, IsGFX11Plus: ST.isGFX11Plus());
4900 case AMDGPU::OPERAND_REG_IMM_V2BF16:
4901 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
4902 return AMDGPU::isInlinableLiteralV2BF16(Literal: Imm);
4903 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
4904 return false;
4905 case AMDGPU::OPERAND_REG_IMM_FP16:
4906 case AMDGPU::OPERAND_REG_INLINE_C_FP16: {
4907 if (isInt<16>(x: Imm) || isUInt<16>(x: Imm)) {
4908 // A few special case instructions have 16-bit operands on subtargets
4909 // where 16-bit instructions are not legal.
4910 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
4911 // constants in these cases
4912 int16_t Trunc = static_cast<int16_t>(Imm);
4913 return ST.has16BitInsts() &&
4914 AMDGPU::isInlinableLiteralFP16(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
4915 }
4916
4917 return false;
4918 }
4919 case AMDGPU::OPERAND_REG_IMM_BF16:
4920 case AMDGPU::OPERAND_REG_INLINE_C_BF16: {
4921 if (isInt<16>(x: Imm) || isUInt<16>(x: Imm)) {
4922 int16_t Trunc = static_cast<int16_t>(Imm);
4923 return ST.has16BitInsts() &&
4924 AMDGPU::isInlinableLiteralBF16(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
4925 }
4926 return false;
4927 }
4928 case AMDGPU::OPERAND_KIMM32:
4929 case AMDGPU::OPERAND_KIMM16:
4930 case AMDGPU::OPERAND_KIMM64:
4931 return false;
4932 case AMDGPU::OPERAND_INLINE_C_AV64_PSEUDO:
4933 return isLegalAV64PseudoImm(Imm);
4934 case AMDGPU::OPERAND_INPUT_MODS:
4935 case MCOI::OPERAND_IMMEDIATE:
4936 // Always embedded in the instruction for free.
4937 return true;
4938 case MCOI::OPERAND_UNKNOWN:
4939 case MCOI::OPERAND_REGISTER:
4940 case MCOI::OPERAND_PCREL:
4941 case MCOI::OPERAND_GENERIC_0:
4942 case MCOI::OPERAND_GENERIC_1:
4943 case MCOI::OPERAND_GENERIC_2:
4944 case MCOI::OPERAND_GENERIC_3:
4945 case MCOI::OPERAND_GENERIC_4:
4946 case MCOI::OPERAND_GENERIC_5:
4947 // Just ignore anything else.
4948 return false;
4949 default:
4950 llvm_unreachable("invalid operand type");
4951 }
4952}
4953
4954static bool compareMachineOp(const MachineOperand &Op0,
4955 const MachineOperand &Op1) {
4956 if (Op0.getType() != Op1.getType())
4957 return false;
4958
4959 switch (Op0.getType()) {
4960 case MachineOperand::MO_Register:
4961 return Op0.getReg() == Op1.getReg();
4962 case MachineOperand::MO_Immediate:
4963 return Op0.getImm() == Op1.getImm();
4964 default:
4965 llvm_unreachable("Didn't expect to be comparing these operand types");
4966 }
4967}
4968
4969bool SIInstrInfo::isLiteralOperandLegal(const MCInstrDesc &InstDesc,
4970 const MCOperandInfo &OpInfo) const {
4971 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
4972 return true;
4973
4974 if (!RI.opCanUseLiteralConstant(OpType: OpInfo.OperandType))
4975 return false;
4976
4977 if (!isVOP3(Desc: InstDesc) || !AMDGPU::isSISrcOperand(OpInfo))
4978 return true;
4979
4980 return ST.hasVOP3Literal();
4981}
4982
4983bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
4984 int64_t ImmVal) const {
4985 const unsigned Opc = InstDesc.getOpcode();
4986 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
4987 if (Src1Idx != -1 && isDPP(Opcode: Opc) && !ST.hasDPPSrc1SGPR() &&
4988 OpNo == static_cast<unsigned>(Src1Idx))
4989 return false;
4990
4991 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
4992 if (isInlineConstant(Imm: ImmVal, OperandType: OpInfo.OperandType)) {
4993 if (isMAI(Desc: InstDesc) && ST.hasMFMAInlineLiteralBug() &&
4994 OpNo == (unsigned)AMDGPU::getNamedOperandIdx(Opcode: InstDesc.getOpcode(),
4995 Name: AMDGPU::OpName::src2))
4996 return false;
4997
4998 if (ST.hasBF16InlineConstFromUpperFP32() && isVOP1(Opcode: Opc)) {
4999 if ((OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_BF16 ||
5000 OpInfo.OperandType == AMDGPU::OPERAND_REG_INLINE_C_BF16) &&
5001 isInlineConstant(Imm: ImmVal, OperandType: OpInfo.OperandType))
5002 return false;
5003 }
5004
5005 return RI.opCanUseInlineConstant(OpType: OpInfo.OperandType);
5006 }
5007
5008 return isLiteralOperandLegal(InstDesc, OpInfo);
5009}
5010
5011bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
5012 const MachineOperand &MO) const {
5013 if (MO.isImm())
5014 return isImmOperandLegal(InstDesc, OpNo, ImmVal: MO.getImm());
5015
5016 assert((MO.isTargetIndex() || MO.isFI() || MO.isGlobal()) &&
5017 "unexpected imm-like operand kind");
5018 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
5019 return isLiteralOperandLegal(InstDesc, OpInfo);
5020}
5021
5022bool SIInstrInfo::isLegalAV64PseudoImm(uint64_t Imm) const {
5023 // 2 32-bit inline constants packed into one.
5024 return AMDGPU::isInlinableLiteral32(Literal: Lo_32(Value: Imm), HasInv2Pi: ST.hasInv2PiInlineImm()) &&
5025 AMDGPU::isInlinableLiteral32(Literal: Hi_32(Value: Imm), HasInv2Pi: ST.hasInv2PiInlineImm());
5026}
5027
5028bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
5029 // GFX90A does not have V_MUL_LEGACY_F32_e32.
5030 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())
5031 return false;
5032
5033 int Op32 = AMDGPU::getVOPe32(Opcode);
5034 if (Op32 == -1)
5035 return false;
5036
5037 return pseudoToMCOpcode(Opcode: Op32) != -1;
5038}
5039
5040bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
5041 // The src0_modifier operand is present on all instructions
5042 // that have modifiers.
5043
5044 return AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0_modifiers);
5045}
5046
5047bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI,
5048 AMDGPU::OpName OpName) const {
5049 const MachineOperand *Mods = getNamedOperand(MI, OperandName: OpName);
5050 return Mods && Mods->getImm();
5051}
5052
5053bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const {
5054 return any_of(Range: ModifierOpNames,
5055 P: [&](AMDGPU::OpName Name) { return hasModifiersSet(MI, OpName: Name); });
5056}
5057
5058bool SIInstrInfo::canShrink(const MachineInstr &MI,
5059 const MachineRegisterInfo &MRI) const {
5060 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
5061 // Can't shrink instruction with three operands.
5062 if (Src2) {
5063 switch (MI.getOpcode()) {
5064 default: return false;
5065
5066 case AMDGPU::V_ADDC_U32_e64:
5067 case AMDGPU::V_SUBB_U32_e64:
5068 case AMDGPU::V_SUBBREV_U32_e64: {
5069 const MachineOperand *Src1
5070 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
5071 if (!Src1->isReg() || !RI.isVGPR(MRI, Reg: Src1->getReg()))
5072 return false;
5073 // Additional verification is needed for sdst/src2.
5074 return true;
5075 }
5076 case AMDGPU::V_MAC_F16_e64:
5077 case AMDGPU::V_MAC_F32_e64:
5078 case AMDGPU::V_MAC_LEGACY_F32_e64:
5079 case AMDGPU::V_FMAC_F16_e64:
5080 case AMDGPU::V_FMAC_F16_t16_e64:
5081 case AMDGPU::V_FMAC_F16_fake16_e64:
5082 case AMDGPU::V_FMAC_F32_e64:
5083 case AMDGPU::V_FMAC_F64_e64:
5084 case AMDGPU::V_FMAC_LEGACY_F32_e64:
5085 if (!Src2->isReg() || !RI.isVGPR(MRI, Reg: Src2->getReg()) ||
5086 hasModifiersSet(MI, OpName: AMDGPU::OpName::src2_modifiers))
5087 return false;
5088 break;
5089
5090 case AMDGPU::V_CNDMASK_B32_e64:
5091 break;
5092 }
5093 }
5094
5095 const MachineOperand *Src1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
5096 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Reg: Src1->getReg()) ||
5097 hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers)))
5098 return false;
5099
5100 // Make sure src0 isn't using any modifiers.
5101 if (hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers))
5102 return false;
5103
5104 // Can it be shrunk to a valid 32 bit opcode?
5105 if (!hasVALU32BitEncoding(Opcode: MI.getOpcode()))
5106 return false;
5107
5108 const MachineOperand *Src0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
5109 if (Src0 && Src0->isImm()) {
5110 unsigned Op32 = AMDGPU::getVOPe32(Opcode: MI.getOpcode());
5111 if (!isImmOperandLegal(
5112 InstDesc: get(Opcode: Op32), OpNo: AMDGPU::getNamedOperandIdx(Opcode: Op32, Name: AMDGPU::OpName::src0),
5113 MO: *Src0))
5114 return false;
5115 }
5116
5117 // Check output modifiers
5118 return !hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) &&
5119 !hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) &&
5120 !hasModifiersSet(MI, OpName: AMDGPU::OpName::byte_sel) &&
5121 // TODO: Can we avoid checking bound_ctrl/fi here?
5122 // They are only used by permlane*_swap special case.
5123 !hasModifiersSet(MI, OpName: AMDGPU::OpName::bound_ctrl) &&
5124 !hasModifiersSet(MI, OpName: AMDGPU::OpName::fi);
5125}
5126
5127// Set VCC operand with all flags from \p Orig, except for setting it as
5128// implicit.
5129static void copyFlagsToImplicitVCC(MachineInstr &MI,
5130 const MachineOperand &Orig) {
5131
5132 for (MachineOperand &Use : MI.implicit_operands()) {
5133 if (Use.isUse() &&
5134 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) {
5135 Use.setIsUndef(Orig.isUndef());
5136 Use.setIsKill(Orig.isKill());
5137 return;
5138 }
5139 }
5140}
5141
5142MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI,
5143 unsigned Op32) const {
5144 MachineBasicBlock *MBB = MI.getParent();
5145
5146 const MCInstrDesc &Op32Desc = get(Opcode: Op32);
5147 MachineInstrBuilder Inst32 =
5148 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: Op32Desc)
5149 .setMIFlags(MI.getFlags());
5150
5151 // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
5152 // For VOPC instructions, this is replaced by an implicit def of vcc.
5153
5154 // We assume the defs of the shrunk opcode are in the same order, and the
5155 // shrunk opcode loses the last def (SGPR def, in the VOP3->VOPC case).
5156 for (int I = 0, E = Op32Desc.getNumDefs(); I != E; ++I)
5157 Inst32.add(MO: MI.getOperand(i: I));
5158
5159 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
5160
5161 int Idx = MI.getNumExplicitDefs();
5162 for (const MachineOperand &Use : MI.explicit_uses()) {
5163 int OpTy = MI.getDesc().operands()[Idx++].OperandType;
5164 if (OpTy == AMDGPU::OPERAND_INPUT_MODS || OpTy == MCOI::OPERAND_IMMEDIATE)
5165 continue;
5166
5167 if (&Use == Src2) {
5168 if (AMDGPU::getNamedOperandIdx(Opcode: Op32, Name: AMDGPU::OpName::src2) == -1) {
5169 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
5170 // replaced with an implicit read of vcc or vcc_lo. The implicit read
5171 // of vcc was already added during the initial BuildMI, but we
5172 // 1) may need to change vcc to vcc_lo to preserve the original register
5173 // 2) have to preserve the original flags.
5174 copyFlagsToImplicitVCC(MI&: *Inst32, Orig: *Src2);
5175 continue;
5176 }
5177 }
5178
5179 Inst32.add(MO: Use);
5180 }
5181
5182 // FIXME: Losing implicit operands
5183 fixImplicitOperands(MI&: *Inst32);
5184 return Inst32;
5185}
5186
5187bool SIInstrInfo::physRegUsesConstantBus(const MachineOperand &RegOp) const {
5188 // Null is free
5189 Register Reg = RegOp.getReg();
5190 if (Reg == AMDGPU::SGPR_NULL || Reg == AMDGPU::SGPR_NULL64)
5191 return false;
5192
5193 // SGPRs use the constant bus
5194
5195 // FIXME: implicit registers that are not part of the MCInstrDesc's implicit
5196 // physical register operands should also count, except for exec.
5197 if (RegOp.isImplicit())
5198 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::M0;
5199
5200 // SGPRs use the constant bus
5201 return AMDGPU::SReg_32RegClass.contains(Reg) ||
5202 AMDGPU::SReg_64RegClass.contains(Reg);
5203}
5204
5205bool SIInstrInfo::regUsesConstantBus(const MachineOperand &RegOp,
5206 const MachineRegisterInfo &MRI) const {
5207 Register Reg = RegOp.getReg();
5208 return Reg.isVirtual() ? RI.isSGPRClass(RC: MRI.getRegClass(Reg))
5209 : physRegUsesConstantBus(RegOp);
5210}
5211
5212bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,
5213 const MachineOperand &MO,
5214 const MCOperandInfo &OpInfo) const {
5215 // Literal constants use the constant bus.
5216 if (!MO.isReg())
5217 return !isInlineConstant(MO, OpInfo);
5218
5219 Register Reg = MO.getReg();
5220 return Reg.isVirtual() ? RI.isSGPRClass(RC: MRI.getRegClass(Reg))
5221 : physRegUsesConstantBus(RegOp: MO);
5222}
5223
5224static Register findImplicitSGPRRead(const MachineInstr &MI) {
5225 for (const MachineOperand &MO : MI.implicit_operands()) {
5226 // We only care about reads.
5227 if (MO.isDef())
5228 continue;
5229
5230 switch (MO.getReg()) {
5231 case AMDGPU::VCC:
5232 case AMDGPU::VCC_LO:
5233 case AMDGPU::VCC_HI:
5234 case AMDGPU::M0:
5235 case AMDGPU::FLAT_SCR:
5236 return MO.getReg();
5237
5238 default:
5239 break;
5240 }
5241 }
5242
5243 return Register();
5244}
5245
5246static bool shouldReadExec(const MachineInstr &MI) {
5247 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) {
5248 switch (MI.getOpcode()) {
5249 case AMDGPU::V_READLANE_B32:
5250 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
5251 case AMDGPU::V_WRITELANE_B32:
5252 case AMDGPU::SI_SPILL_S32_TO_VGPR:
5253 return false;
5254 }
5255
5256 return true;
5257 }
5258
5259 if (MI.isPreISelOpcode() ||
5260 SIInstrInfo::isGenericOpcode(Opc: MI.getOpcode()) ||
5261 SIInstrInfo::isSALU(MI) ||
5262 SIInstrInfo::isSMRD(MI))
5263 return false;
5264
5265 return true;
5266}
5267
5268static bool isRegOrFI(const MachineOperand &MO) {
5269 return MO.isReg() || MO.isFI();
5270}
5271
5272static bool isSubRegOf(const SIRegisterInfo &TRI,
5273 const MachineOperand &SuperVec,
5274 const MachineOperand &SubReg) {
5275 if (SubReg.getReg().isPhysical())
5276 return TRI.isSubRegister(RegA: SuperVec.getReg(), RegB: SubReg.getReg());
5277
5278 return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
5279 SubReg.getReg() == SuperVec.getReg();
5280}
5281
5282// Verify the illegal copy from vector register to SGPR for generic opcode COPY
5283bool SIInstrInfo::verifyCopy(const MachineInstr &MI,
5284 const MachineRegisterInfo &MRI,
5285 StringRef &ErrInfo) const {
5286 Register DstReg = MI.getOperand(i: 0).getReg();
5287 Register SrcReg = MI.getOperand(i: 1).getReg();
5288 // This is a check for copy from vector register to SGPR
5289 if (RI.isVectorRegister(MRI, Reg: SrcReg) && RI.isSGPRReg(MRI, Reg: DstReg)) {
5290 ErrInfo = "illegal copy from vector register to SGPR";
5291 return false;
5292 }
5293 return true;
5294}
5295
5296bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
5297 StringRef &ErrInfo) const {
5298 uint32_t Opcode = MI.getOpcode();
5299 const MachineFunction *MF = MI.getMF();
5300 const MachineRegisterInfo &MRI = MF->getRegInfo();
5301
5302 // FIXME: At this point the COPY verify is done only for non-ssa forms.
5303 // Find a better property to recognize the point where instruction selection
5304 // is just done.
5305 // We can only enforce this check after SIFixSGPRCopies pass so that the
5306 // illegal copies are legalized and thereafter we don't expect a pass
5307 // inserting similar copies.
5308 if (!MRI.isSSA() && MI.isCopy())
5309 return verifyCopy(MI, MRI, ErrInfo);
5310
5311 if (SIInstrInfo::isGenericOpcode(Opc: Opcode))
5312 return true;
5313
5314 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0);
5315 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src1);
5316 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src2);
5317 int Src3Idx = -1;
5318 if (Src0Idx == -1) {
5319 // VOPD V_DUAL_* instructions use different operand names.
5320 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0X);
5321 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vsrc1X);
5322 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0Y);
5323 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vsrc1Y);
5324 }
5325
5326 // Make sure the number of operands is correct.
5327 const MCInstrDesc &Desc = get(Opcode);
5328 if (!Desc.isVariadic() &&
5329 Desc.getNumOperands() != MI.getNumExplicitOperands()) {
5330 ErrInfo = "Instruction has wrong number of operands.";
5331 return false;
5332 }
5333
5334 if (MI.isInlineAsm()) {
5335 // Verify register classes for inlineasm constraints.
5336 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
5337 I != E; ++I) {
5338 const TargetRegisterClass *RC = MI.getRegClassConstraint(OpIdx: I, TII: this, TRI: &RI);
5339 if (!RC)
5340 continue;
5341
5342 const MachineOperand &Op = MI.getOperand(i: I);
5343 if (!Op.isReg())
5344 continue;
5345
5346 Register Reg = Op.getReg();
5347 if (!Reg.isVirtual() && !RC->contains(Reg)) {
5348 ErrInfo = "inlineasm operand has incorrect register class.";
5349 return false;
5350 }
5351 }
5352
5353 return true;
5354 }
5355
5356 if (isImage(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) {
5357 ErrInfo = "missing memory operand from image instruction.";
5358 return false;
5359 }
5360
5361 // Make sure the register classes are correct.
5362 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
5363 const MachineOperand &MO = MI.getOperand(i);
5364 if (MO.isFPImm()) {
5365 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
5366 "all fp values to integers.";
5367 return false;
5368 }
5369
5370 const MCOperandInfo &OpInfo = Desc.operands()[i];
5371 int16_t RegClass = getOpRegClassID(OpInfo);
5372
5373 switch (OpInfo.OperandType) {
5374 case MCOI::OPERAND_REGISTER:
5375 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
5376 ErrInfo = "Illegal immediate value for operand.";
5377 return false;
5378 }
5379 break;
5380 case AMDGPU::OPERAND_REG_IMM_INT32:
5381 case AMDGPU::OPERAND_REG_IMM_INT16:
5382 case AMDGPU::OPERAND_REG_IMM_FP32:
5383 case AMDGPU::OPERAND_REG_IMM_BF16:
5384 case AMDGPU::OPERAND_REG_IMM_FP16:
5385 case AMDGPU::OPERAND_REG_IMM_V2FP16:
5386 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
5387 case AMDGPU::OPERAND_REG_IMM_V2INT16:
5388 case AMDGPU::OPERAND_REG_IMM_V2BF16:
5389 case AMDGPU::OPERAND_REG_IMM_V2FP64:
5390 case AMDGPU::OPERAND_REG_IMM_V2INT64:
5391 break;
5392 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
5393 break;
5394 break;
5395 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
5396 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
5397 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
5398 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
5399 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
5400 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
5401 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
5402 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
5403 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
5404 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
5405 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
5406 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
5407 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: {
5408 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, OpIdx: i))) {
5409 ErrInfo = "Illegal immediate value for operand.";
5410 return false;
5411 }
5412 break;
5413 }
5414 case AMDGPU::OPERAND_REG_IMM_FP64:
5415 case AMDGPU::OPERAND_REG_IMM_INT64:
5416 case AMDGPU::OPERAND_REG_IMM_V2INT32:
5417 case AMDGPU::OPERAND_REG_IMM_V2FP32:
5418 if (ST.has64BitLiterals() && Desc.getSize() != 4 && MO.isImm() &&
5419 !isInlineConstant(MI, OpIdx: i) &&
5420 !AMDGPU::isValid32BitLiteral(Val: MO.getImm(),
5421 IsFP64: OpInfo.OperandType ==
5422 AMDGPU::OPERAND_REG_IMM_FP64)) {
5423 ErrInfo = "illegal 64-bit immediate value for operand.";
5424 return false;
5425 }
5426 break;
5427 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
5428 case AMDGPU::OPERAND_INPUT_MODS:
5429 if (!MI.getOperand(i).isImm() || !isInlineConstant(MI, OpIdx: i)) {
5430 ErrInfo = "Expected inline constant for operand.";
5431 return false;
5432 }
5433 break;
5434 case AMDGPU::OPERAND_SDWA_VOPC_DST:
5435 case AMDGPU::OPERAND_KIMM16:
5436 break;
5437 case MCOI::OPERAND_IMMEDIATE:
5438 case AMDGPU::OPERAND_KIMM32:
5439 case AMDGPU::OPERAND_KIMM64:
5440 case AMDGPU::OPERAND_INLINE_C_AV64_PSEUDO:
5441 // Check if this operand is an immediate.
5442 // FrameIndex operands will be replaced by immediates, so they are
5443 // allowed.
5444 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
5445 ErrInfo = "Expected immediate, but got non-immediate";
5446 return false;
5447 }
5448 break;
5449 case MCOI::OPERAND_UNKNOWN:
5450 case MCOI::OPERAND_MEMORY:
5451 case MCOI::OPERAND_PCREL:
5452 break;
5453 default:
5454 if (OpInfo.isGenericType())
5455 continue;
5456 break;
5457 }
5458
5459 if (!MO.isReg())
5460 continue;
5461 Register Reg = MO.getReg();
5462 if (!Reg)
5463 continue;
5464
5465 // FIXME: Ideally we would have separate instruction definitions with the
5466 // aligned register constraint.
5467 // FIXME: We do not verify inline asm operands, but custom inline asm
5468 // verification is broken anyway
5469 if (ST.needsAlignedVGPRs() && Opcode != AMDGPU::AV_MOV_B64_IMM_PSEUDO &&
5470 Opcode != AMDGPU::V_MOV_B64_PSEUDO && !isSpill(MI)) {
5471 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI, Reg);
5472 if (RI.hasVectorRegisters(RC) && MO.getSubReg()) {
5473 if (const TargetRegisterClass *SubRC =
5474 RI.getSubRegisterClass(RC, MO.getSubReg())) {
5475 RC = RI.getCompatibleSubRegClass(SuperRC: RC, SubRC, SubIdx: MO.getSubReg());
5476 if (RC)
5477 RC = SubRC;
5478 }
5479 }
5480
5481 // Check that this is the aligned version of the class.
5482 if (!RC || !RI.isProperlyAlignedRC(RC: *RC)) {
5483 ErrInfo = "Subtarget requires even aligned vector registers";
5484 return false;
5485 }
5486 }
5487
5488 if (RegClass != -1) {
5489 if (Reg.isVirtual())
5490 continue;
5491
5492 const TargetRegisterClass *RC = RI.getRegClass(i: RegClass);
5493 if (!RC->contains(Reg)) {
5494 ErrInfo = "Operand has incorrect register class.";
5495 return false;
5496 }
5497 }
5498 }
5499
5500 // Verify SDWA
5501 if (isSDWA(MI)) {
5502 if (!ST.hasSDWA()) {
5503 ErrInfo = "SDWA is not supported on this target";
5504 return false;
5505 }
5506
5507 for (auto Op : {AMDGPU::OpName::src0_sel, AMDGPU::OpName::src1_sel,
5508 AMDGPU::OpName::dst_sel}) {
5509 const MachineOperand *MO = getNamedOperand(MI, OperandName: Op);
5510 if (!MO)
5511 continue;
5512 int64_t Imm = MO->getImm();
5513 if (Imm < 0 || Imm > AMDGPU::SDWA::SdwaSel::DWORD) {
5514 ErrInfo = "Invalid SDWA selection";
5515 return false;
5516 }
5517 }
5518
5519 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdst);
5520
5521 for (int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) {
5522 if (OpIdx == -1)
5523 continue;
5524 const MachineOperand &MO = MI.getOperand(i: OpIdx);
5525
5526 if (!ST.hasSDWAScalar()) {
5527 // Only VGPRS on VI
5528 if (!MO.isReg() || !RI.hasVGPRs(RC: RI.getRegClassForReg(MRI, Reg: MO.getReg()))) {
5529 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
5530 return false;
5531 }
5532 } else {
5533 // No immediates on GFX9
5534 if (!MO.isReg()) {
5535 ErrInfo =
5536 "Only reg allowed as operands in SDWA instructions on GFX9+";
5537 return false;
5538 }
5539 }
5540 }
5541
5542 if (!ST.hasSDWAOmod()) {
5543 // No omod allowed on VI
5544 const MachineOperand *OMod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
5545 if (OMod != nullptr &&
5546 (!OMod->isImm() || OMod->getImm() != 0)) {
5547 ErrInfo = "OMod not allowed in SDWA instructions on VI";
5548 return false;
5549 }
5550 }
5551
5552 if (Opcode == AMDGPU::V_CVT_F32_FP8_sdwa ||
5553 Opcode == AMDGPU::V_CVT_F32_BF8_sdwa ||
5554 Opcode == AMDGPU::V_CVT_PK_F32_FP8_sdwa ||
5555 Opcode == AMDGPU::V_CVT_PK_F32_BF8_sdwa) {
5556 const MachineOperand *Src0ModsMO =
5557 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
5558 unsigned Mods = Src0ModsMO->getImm();
5559 if (Mods & SISrcMods::ABS || Mods & SISrcMods::NEG ||
5560 Mods & SISrcMods::SEXT) {
5561 ErrInfo = "sext, abs and neg are not allowed on this instruction";
5562 return false;
5563 }
5564 }
5565
5566 uint32_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
5567 if (isVOPC(Opcode: BasicOpcode)) {
5568 if (!ST.hasSDWASdst() && DstIdx != -1) {
5569 // Only vcc allowed as dst on VI for VOPC
5570 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5571 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
5572 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
5573 return false;
5574 }
5575 } else if (!ST.hasSDWAOutModsVOPC()) {
5576 // No clamp allowed on GFX9 for VOPC
5577 const MachineOperand *Clamp = getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
5578 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
5579 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
5580 return false;
5581 }
5582
5583 // No omod allowed on GFX9 for VOPC
5584 const MachineOperand *OMod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
5585 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
5586 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
5587 return false;
5588 }
5589 }
5590 }
5591
5592 const MachineOperand *DstUnused = getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_unused);
5593 if (DstUnused && DstUnused->isImm() &&
5594 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
5595 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5596 if (!Dst.isReg() || !Dst.isTied()) {
5597 ErrInfo = "Dst register should have tied register";
5598 return false;
5599 }
5600
5601 const MachineOperand &TiedMO =
5602 MI.getOperand(i: MI.findTiedOperandIdx(OpIdx: DstIdx));
5603 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
5604 ErrInfo =
5605 "Dst register should be tied to implicit use of preserved register";
5606 return false;
5607 }
5608 if (TiedMO.getReg().isPhysical() && Dst.getReg() != TiedMO.getReg()) {
5609 ErrInfo = "Dst register should use same physical register as preserved";
5610 return false;
5611 }
5612 }
5613 }
5614
5615 if (isDPP(MI) && !ST.hasDPPSrc1SGPR() && Src1Idx != -1) {
5616 const MachineOperand &Src1MO = MI.getOperand(i: Src1Idx);
5617 if (Src1MO.isReg() && RI.isSGPRReg(MRI, Reg: Src1MO.getReg())) {
5618 ErrInfo = "DPP src1 cannot be SGPR on this subtarget";
5619 return false;
5620 }
5621 if (Src1MO.isImm()) {
5622 ErrInfo = "DPP src1 cannot be an immediate on this subtarget";
5623 return false;
5624 }
5625 }
5626
5627 // Verify MIMG / VIMAGE / VSAMPLE
5628 if (isImage(Opcode) && !MI.mayStore()) {
5629 // Ensure that the return type used is large enough for all the options
5630 // being used TFE/LWE require an extra result register.
5631 const MachineOperand *DMask = getNamedOperand(MI, OperandName: AMDGPU::OpName::dmask);
5632 if (DMask) {
5633 uint64_t DMaskImm = DMask->getImm();
5634 uint32_t RegCount = isGather4(Opcode) ? 4 : llvm::popcount(Value: DMaskImm);
5635 const MachineOperand *TFE = getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe);
5636 const MachineOperand *LWE = getNamedOperand(MI, OperandName: AMDGPU::OpName::lwe);
5637 const MachineOperand *D16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::d16);
5638
5639 // Adjust for packed 16 bit values
5640 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
5641 RegCount = divideCeil(Numerator: RegCount, Denominator: 2);
5642
5643 // Adjust if using LWE or TFE
5644 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
5645 RegCount += 1;
5646
5647 const uint32_t DstIdx =
5648 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdata);
5649 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5650 if (Dst.isReg()) {
5651 const TargetRegisterClass *DstRC = getOpRegClass(MI, OpNo: DstIdx);
5652 uint32_t DstSize = RI.getRegSizeInBits(RC: *DstRC) / 32;
5653 if (RegCount > DstSize) {
5654 ErrInfo = "Image instruction returns too many registers for dst "
5655 "register class";
5656 return false;
5657 }
5658 }
5659 }
5660 }
5661
5662 // Verify VOP*. Ignore multiple sgpr operands on writelane.
5663 if (isVALU(MI, /*AllowLDSDMA=*/true) &&
5664 Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
5665 unsigned ConstantBusCount = 0;
5666 bool UsesLiteral = false;
5667 const MachineOperand *LiteralVal = nullptr;
5668
5669 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::imm);
5670 if (ImmIdx != -1) {
5671 ++ConstantBusCount;
5672 UsesLiteral = true;
5673 LiteralVal = &MI.getOperand(i: ImmIdx);
5674 }
5675
5676 SmallVector<Register, 2> SGPRsUsed;
5677 Register SGPRUsed;
5678
5679 // Only look at the true operands. Only a real operand can use the constant
5680 // bus, and we don't want to check pseudo-operands like the source modifier
5681 // flags.
5682 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) {
5683 if (OpIdx == -1)
5684 continue;
5685 const MachineOperand &MO = MI.getOperand(i: OpIdx);
5686 if (usesConstantBus(MRI, MO, OpInfo: MI.getDesc().operands()[OpIdx])) {
5687 if (MO.isReg()) {
5688 SGPRUsed = MO.getReg();
5689 if (!llvm::is_contained(Range&: SGPRsUsed, Element: SGPRUsed)) {
5690 ++ConstantBusCount;
5691 SGPRsUsed.push_back(Elt: SGPRUsed);
5692 }
5693 } else if (!MO.isFI()) { // Treat FI like a register.
5694 if (!UsesLiteral) {
5695 ++ConstantBusCount;
5696 UsesLiteral = true;
5697 LiteralVal = &MO;
5698 } else if (!MO.isIdenticalTo(Other: *LiteralVal)) {
5699 assert(isVOP2(MI) || isVOP3(MI));
5700 ErrInfo = "VOP2/VOP3 instruction uses more than one literal";
5701 return false;
5702 }
5703 }
5704 }
5705 }
5706
5707 SGPRUsed = findImplicitSGPRRead(MI);
5708 if (SGPRUsed) {
5709 // Implicit uses may safely overlap true operands
5710 if (llvm::all_of(Range&: SGPRsUsed, P: [this, SGPRUsed](unsigned SGPR) {
5711 return !RI.regsOverlap(RegA: SGPRUsed, RegB: SGPR);
5712 })) {
5713 ++ConstantBusCount;
5714 SGPRsUsed.push_back(Elt: SGPRUsed);
5715 }
5716 }
5717
5718 // v_writelane_b32 is an exception from constant bus restriction:
5719 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
5720 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
5721 Opcode != AMDGPU::V_WRITELANE_B32) {
5722 ErrInfo = "VOP* instruction violates constant bus restriction";
5723 return false;
5724 }
5725
5726 if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) {
5727 ErrInfo = "VOP3 instruction uses literal";
5728 return false;
5729 }
5730 }
5731
5732 // Special case for writelane - this can break the multiple constant bus rule,
5733 // but still can't use more than one SGPR register
5734 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
5735 unsigned SGPRCount = 0;
5736 Register SGPRUsed;
5737
5738 for (int OpIdx : {Src0Idx, Src1Idx}) {
5739 if (OpIdx == -1)
5740 break;
5741
5742 const MachineOperand &MO = MI.getOperand(i: OpIdx);
5743
5744 if (usesConstantBus(MRI, MO, OpInfo: MI.getDesc().operands()[OpIdx])) {
5745 if (MO.isReg() && MO.getReg() != AMDGPU::M0) {
5746 if (MO.getReg() != SGPRUsed)
5747 ++SGPRCount;
5748 SGPRUsed = MO.getReg();
5749 }
5750 }
5751 if (SGPRCount > ST.getConstantBusLimit(Opcode)) {
5752 ErrInfo = "WRITELANE instruction violates constant bus restriction";
5753 return false;
5754 }
5755 }
5756 }
5757
5758 // Verify misc. restrictions on specific instructions.
5759 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
5760 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
5761 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5762 const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
5763 const MachineOperand &Src2 = MI.getOperand(i: Src2Idx);
5764 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
5765 if (!compareMachineOp(Op0: Src0, Op1: Src1) &&
5766 !compareMachineOp(Op0: Src0, Op1: Src2)) {
5767 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
5768 return false;
5769 }
5770 }
5771 if ((getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers)->getImm() &
5772 SISrcMods::ABS) ||
5773 (getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers)->getImm() &
5774 SISrcMods::ABS) ||
5775 (getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers)->getImm() &
5776 SISrcMods::ABS)) {
5777 ErrInfo = "ABS not allowed in VOP3B instructions";
5778 return false;
5779 }
5780 }
5781
5782 if (isSOP2(MI) || isSOPC(MI)) {
5783 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5784 const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
5785
5786 if (!isRegOrFI(MO: Src0) && !isRegOrFI(MO: Src1) &&
5787 !isInlineConstant(MO: Src0, OpInfo: Desc.operands()[Src0Idx]) &&
5788 !isInlineConstant(MO: Src1, OpInfo: Desc.operands()[Src1Idx]) &&
5789 !Src0.isIdenticalTo(Other: Src1)) {
5790 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
5791 return false;
5792 }
5793 }
5794
5795 if (isSOPK(MI)) {
5796 const auto *Op = getNamedOperand(MI, OperandName: AMDGPU::OpName::simm16);
5797 if (Desc.isBranch()) {
5798 if (!Op->isMBB()) {
5799 ErrInfo = "invalid branch target for SOPK instruction";
5800 return false;
5801 }
5802 } else {
5803 uint64_t Imm = Op->getImm();
5804 if (sopkIsZext(Opcode)) {
5805 if (!isUInt<16>(x: Imm)) {
5806 ErrInfo = "invalid immediate for SOPK instruction";
5807 return false;
5808 }
5809 } else {
5810 if (!isInt<16>(x: Imm)) {
5811 ErrInfo = "invalid immediate for SOPK instruction";
5812 return false;
5813 }
5814 }
5815 }
5816 }
5817
5818 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
5819 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
5820 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
5821 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
5822 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
5823 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
5824
5825 const unsigned StaticNumOps =
5826 Desc.getNumOperands() + Desc.implicit_uses().size();
5827 const unsigned NumImplicitOps = IsDst ? 2 : 1;
5828
5829 // Require additional implicit operands. This allows a fixup done by the
5830 // post RA scheduler where the main implicit operand is killed and
5831 // implicit-defs are added for sub-registers that remain live after this
5832 // instruction.
5833 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
5834 ErrInfo = "missing implicit register operands";
5835 return false;
5836 }
5837
5838 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
5839 if (IsDst) {
5840 if (!Dst->isUse()) {
5841 ErrInfo = "v_movreld_b32 vdst should be a use operand";
5842 return false;
5843 }
5844
5845 unsigned UseOpIdx;
5846 if (!MI.isRegTiedToUseOperand(DefOpIdx: StaticNumOps, UseOpIdx: &UseOpIdx) ||
5847 UseOpIdx != StaticNumOps + 1) {
5848 ErrInfo = "movrel implicit operands should be tied";
5849 return false;
5850 }
5851 }
5852
5853 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5854 const MachineOperand &ImpUse
5855 = MI.getOperand(i: StaticNumOps + NumImplicitOps - 1);
5856 if (!ImpUse.isReg() || !ImpUse.isUse() ||
5857 !isSubRegOf(TRI: RI, SuperVec: ImpUse, SubReg: IsDst ? *Dst : Src0)) {
5858 ErrInfo = "src0 should be subreg of implicit vector use";
5859 return false;
5860 }
5861 }
5862
5863 // Make sure we aren't losing exec uses in the td files. This mostly requires
5864 // being careful when using let Uses to try to add other use registers.
5865 if (shouldReadExec(MI)) {
5866 if (!MI.hasRegisterImplicitUseOperand(Reg: AMDGPU::EXEC)) {
5867 ErrInfo = "VALU instruction does not implicitly read exec mask";
5868 return false;
5869 }
5870 }
5871
5872 if (isSMRD(MI)) {
5873 if (MI.mayStore() &&
5874 ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) {
5875 // The register offset form of scalar stores may only use m0 as the
5876 // soffset register.
5877 const MachineOperand *Soff = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
5878 if (Soff && Soff->getReg() != AMDGPU::M0) {
5879 ErrInfo = "scalar stores must use m0 as offset register";
5880 return false;
5881 }
5882 }
5883 }
5884
5885 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) {
5886 const MachineOperand *Offset = getNamedOperand(MI, OperandName: AMDGPU::OpName::offset);
5887 if (Offset->getImm() != 0) {
5888 ErrInfo = "subtarget does not support offsets in flat instructions";
5889 return false;
5890 }
5891 }
5892
5893 if (isDS(MI) && !ST.hasGDS()) {
5894 const MachineOperand *GDSOp = getNamedOperand(MI, OperandName: AMDGPU::OpName::gds);
5895 if (GDSOp && GDSOp->getImm() != 0) {
5896 ErrInfo = "GDS is not supported on this subtarget";
5897 return false;
5898 }
5899 }
5900
5901 if (isImage(MI)) {
5902 const MachineOperand *DimOp = getNamedOperand(MI, OperandName: AMDGPU::OpName::dim);
5903 if (DimOp) {
5904 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
5905 Name: AMDGPU::OpName::vaddr0);
5906 AMDGPU::OpName RSrcOpName =
5907 isMIMG(MI) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
5908 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: RSrcOpName);
5909 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: Opcode);
5910 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
5911 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
5912 const AMDGPU::MIMGDimInfo *Dim =
5913 AMDGPU::getMIMGDimInfoByEncoding(DimEnc: DimOp->getImm());
5914
5915 if (!Dim) {
5916 ErrInfo = "dim is out of range";
5917 return false;
5918 }
5919
5920 bool IsA16 = false;
5921 if (ST.hasR128A16()) {
5922 const MachineOperand *R128A16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::r128);
5923 IsA16 = R128A16->getImm() != 0;
5924 } else if (ST.hasA16()) {
5925 const MachineOperand *A16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::a16);
5926 IsA16 = A16->getImm() != 0;
5927 }
5928
5929 bool IsNSA = RsrcIdx - VAddr0Idx > 1;
5930
5931 unsigned AddrWords =
5932 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, IsG16Supported: ST.hasG16());
5933
5934 unsigned VAddrWords;
5935 if (IsNSA) {
5936 VAddrWords = RsrcIdx - VAddr0Idx;
5937 if (ST.hasPartialNSAEncoding() &&
5938 AddrWords > ST.getNSAMaxSize(HasSampler: isVSAMPLE(MI))) {
5939 unsigned LastVAddrIdx = RsrcIdx - 1;
5940 VAddrWords += getOpSize(MI, OpNo: LastVAddrIdx) / 4 - 1;
5941 }
5942 } else {
5943 VAddrWords = getOpSize(MI, OpNo: VAddr0Idx) / 4;
5944 if (AddrWords > 12)
5945 AddrWords = 16;
5946 }
5947
5948 if (VAddrWords != AddrWords) {
5949 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords
5950 << " but got " << VAddrWords << "\n");
5951 ErrInfo = "bad vaddr size";
5952 return false;
5953 }
5954 }
5955 }
5956
5957 const MachineOperand *DppCt = getNamedOperand(MI, OperandName: AMDGPU::OpName::dpp_ctrl);
5958 if (DppCt) {
5959 using namespace AMDGPU::DPP;
5960
5961 unsigned DC = DppCt->getImm();
5962 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
5963 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
5964 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) ||
5965 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) ||
5966 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) ||
5967 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) ||
5968 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) {
5969 ErrInfo = "Invalid dpp_ctrl value";
5970 return false;
5971 }
5972 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
5973 !ST.hasDPPWavefrontShifts()) {
5974 ErrInfo = "Invalid dpp_ctrl value: "
5975 "wavefront shifts are not supported on GFX10+";
5976 return false;
5977 }
5978 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
5979 !ST.hasDPPBroadcasts()) {
5980 ErrInfo = "Invalid dpp_ctrl value: "
5981 "broadcasts are not supported on GFX10+";
5982 return false;
5983 }
5984 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST &&
5985 ST.getGeneration() < AMDGPUSubtarget::GFX10) {
5986 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST &&
5987 DC <= DppCtrl::ROW_NEWBCAST_LAST &&
5988 !ST.hasGFX90AInsts()) {
5989 ErrInfo = "Invalid dpp_ctrl value: "
5990 "row_newbroadcast/row_share is not supported before "
5991 "GFX90A/GFX10";
5992 return false;
5993 }
5994 if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) {
5995 ErrInfo = "Invalid dpp_ctrl value: "
5996 "row_share and row_xmask are not supported before GFX10";
5997 return false;
5998 }
5999 }
6000
6001 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&
6002 !AMDGPU::isLegalDPALU_DPPControl(ST, DC) &&
6003 AMDGPU::isDPALU_DPP(OpDesc: Desc, MII: *this, ST)) {
6004 ErrInfo = "Invalid dpp_ctrl value: "
6005 "DP ALU dpp only support row_newbcast";
6006 return false;
6007 }
6008 }
6009
6010 if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) {
6011 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
6012 AMDGPU::OpName DataName =
6013 isDS(Opcode) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata;
6014 const MachineOperand *Data = getNamedOperand(MI, OperandName: DataName);
6015 const MachineOperand *Data2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::data1);
6016 if (Data && !Data->isReg())
6017 Data = nullptr;
6018
6019 if (!ST.hasGFX90AInsts()) {
6020 if ((Dst && RI.isAGPR(MRI, Reg: Dst->getReg())) ||
6021 (Data && RI.isAGPR(MRI, Reg: Data->getReg())) ||
6022 (Data2 && RI.isAGPR(MRI, Reg: Data2->getReg()))) {
6023 ErrInfo = "Invalid register class: "
6024 "agpr loads and stores not supported on this GPU";
6025 return false;
6026 }
6027 }
6028 }
6029
6030 if (ST.needsAlignedVGPRs()) {
6031 const auto isAlignedReg = [&MI, &MRI, this](AMDGPU::OpName OpName) -> bool {
6032 const MachineOperand *Op = getNamedOperand(MI, OperandName: OpName);
6033 if (!Op)
6034 return true;
6035 Register Reg = Op->getReg();
6036 if (Reg.isPhysical())
6037 return !(RI.getHWRegIndex(Reg) & 1);
6038 const TargetRegisterClass &RC = *MRI.getRegClass(Reg);
6039 return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
6040 !(RI.getChannelFromSubReg(SubReg: Op->getSubReg()) & 1);
6041 };
6042
6043 if (Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_SEMA_BR ||
6044 Opcode == AMDGPU::DS_GWS_BARRIER) {
6045
6046 if (!isAlignedReg(AMDGPU::OpName::data0)) {
6047 ErrInfo = "Subtarget requires even aligned vector registers "
6048 "for DS_GWS instructions";
6049 return false;
6050 }
6051 }
6052
6053 if (isMIMG(MI)) {
6054 if (!isAlignedReg(AMDGPU::OpName::vaddr)) {
6055 ErrInfo = "Subtarget requires even aligned vector registers "
6056 "for vaddr operand of image instructions";
6057 return false;
6058 }
6059 }
6060 }
6061
6062 if (Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts()) {
6063 const MachineOperand *Src = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
6064 if (Src->isReg() && RI.isSGPRReg(MRI, Reg: Src->getReg())) {
6065 ErrInfo = "Invalid register class: "
6066 "v_accvgpr_write with an SGPR is not supported on this GPU";
6067 return false;
6068 }
6069 }
6070
6071 if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) {
6072 const MachineOperand &SrcOp = MI.getOperand(i: 1);
6073 if (!SrcOp.isReg() || SrcOp.getReg().isVirtual()) {
6074 ErrInfo = "pseudo expects only physical SGPRs";
6075 return false;
6076 }
6077 }
6078
6079 if (const MachineOperand *CPol = getNamedOperand(MI, OperandName: AMDGPU::OpName::cpol)) {
6080 if (CPol->getImm() & AMDGPU::CPol::SCAL) {
6081 if (!ST.hasScaleOffset()) {
6082 ErrInfo = "Subtarget does not support offset scaling";
6083 return false;
6084 }
6085 if (!AMDGPU::supportsScaleOffset(MII: *this, Opcode: MI.getOpcode())) {
6086 ErrInfo = "Instruction does not support offset scaling";
6087 return false;
6088 }
6089 }
6090 }
6091
6092 // See SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
6093 if (AMDGPU::isSingleSGPRReadInst(Opc: Opcode)) {
6094 for (unsigned I = 0; I < 3; ++I) {
6095 if (!isLegalSingleSGPRReadInstOperand(MRI, MI, SrcN: I))
6096 return false;
6097 }
6098 }
6099
6100 if (ST.hasFlatScratchHiInB64InstHazard() && isSALU(MI) &&
6101 MI.readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, TRI: nullptr)) {
6102 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
6103 if ((Dst && RI.getRegClassForReg(MRI, Reg: Dst->getReg()) ==
6104 &AMDGPU::SReg_64RegClass) ||
6105 Opcode == AMDGPU::S_BITCMP0_B64 || Opcode == AMDGPU::S_BITCMP1_B64) {
6106 ErrInfo = "Instruction cannot read flat_scratch_base_hi";
6107 return false;
6108 }
6109 }
6110
6111 return true;
6112}
6113
6114unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
6115 if (MI.getOpcode() == AMDGPU::S_MOV_B32) {
6116 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6117 return MI.getOperand(i: 1).isReg() || RI.isAGPR(MRI, Reg: MI.getOperand(i: 0).getReg())
6118 ? AMDGPU::COPY
6119 : AMDGPU::V_MOV_B32_e32;
6120 }
6121 return getVALUOp(Opc: MI.getOpcode());
6122}
6123
6124// It is more readable to list mapped opcodes on the same line.
6125// clang-format off
6126
6127unsigned SIInstrInfo::getVALUOp(unsigned Opc) const {
6128 switch (Opc) {
6129 default: return AMDGPU::INSTRUCTION_LIST_END;
6130 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
6131 case AMDGPU::COPY: return AMDGPU::COPY;
6132 case AMDGPU::PHI: return AMDGPU::PHI;
6133 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
6134 case AMDGPU::WQM: return AMDGPU::WQM;
6135 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
6136 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM;
6137 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM;
6138 case AMDGPU::S_ADD_I32:
6139 return ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
6140 case AMDGPU::S_ADDC_U32:
6141 return AMDGPU::V_ADDC_U32_e32;
6142 case AMDGPU::S_SUB_I32:
6143 return ST.hasAddNoCarryInsts() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32;
6144 // FIXME: These are not consistently handled, and selected when the carry is
6145 // used.
6146 case AMDGPU::S_ADD_U32:
6147 return AMDGPU::V_ADD_CO_U32_e32;
6148 case AMDGPU::S_SUB_U32:
6149 return AMDGPU::V_SUB_CO_U32_e32;
6150 case AMDGPU::S_ADD_U64_PSEUDO:
6151 return AMDGPU::V_ADD_U64_PSEUDO;
6152 case AMDGPU::S_SUB_U64_PSEUDO:
6153 return AMDGPU::V_SUB_U64_PSEUDO;
6154 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
6155 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64;
6156 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64;
6157 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64;
6158 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
6159 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
6160 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
6161 case AMDGPU::S_XNOR_B32:
6162 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
6163 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
6164 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
6165 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
6166 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
6167 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
6168 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64;
6169 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
6170 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64;
6171 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
6172 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64;
6173 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64;
6174 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64;
6175 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64;
6176 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64;
6177 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
6178 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
6179 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
6180 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
6181 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64;
6182 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64;
6183 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64;
6184 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64;
6185 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64;
6186 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64;
6187 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64;
6188 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64;
6189 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64;
6190 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64;
6191 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64;
6192 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64;
6193 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64;
6194 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64;
6195 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
6196 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
6197 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
6198 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
6199 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
6200 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
6201 case AMDGPU::S_CVT_F32_I32: return AMDGPU::V_CVT_F32_I32_e64;
6202 case AMDGPU::S_CVT_F32_U32: return AMDGPU::V_CVT_F32_U32_e64;
6203 case AMDGPU::S_CVT_I32_F32: return AMDGPU::V_CVT_I32_F32_e64;
6204 case AMDGPU::S_CVT_U32_F32: return AMDGPU::V_CVT_U32_F32_e64;
6205 case AMDGPU::S_CVT_F32_F16:
6206 case AMDGPU::S_CVT_HI_F32_F16:
6207 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F32_F16_t16_e64
6208 : AMDGPU::V_CVT_F32_F16_fake16_e64;
6209 case AMDGPU::S_CVT_F16_F32:
6210 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F16_F32_t16_e64
6211 : AMDGPU::V_CVT_F16_F32_fake16_e64;
6212 case AMDGPU::S_CEIL_F32: return AMDGPU::V_CEIL_F32_e64;
6213 case AMDGPU::S_FLOOR_F32: return AMDGPU::V_FLOOR_F32_e64;
6214 case AMDGPU::S_TRUNC_F32: return AMDGPU::V_TRUNC_F32_e64;
6215 case AMDGPU::S_RNDNE_F32: return AMDGPU::V_RNDNE_F32_e64;
6216 case AMDGPU::S_CEIL_F16:
6217 return ST.useRealTrue16Insts() ? AMDGPU::V_CEIL_F16_t16_e64
6218 : AMDGPU::V_CEIL_F16_fake16_e64;
6219 case AMDGPU::S_FLOOR_F16:
6220 return ST.useRealTrue16Insts() ? AMDGPU::V_FLOOR_F16_t16_e64
6221 : AMDGPU::V_FLOOR_F16_fake16_e64;
6222 case AMDGPU::S_TRUNC_F16:
6223 return ST.useRealTrue16Insts() ? AMDGPU::V_TRUNC_F16_t16_e64
6224 : AMDGPU::V_TRUNC_F16_fake16_e64;
6225 case AMDGPU::S_RNDNE_F16:
6226 return ST.useRealTrue16Insts() ? AMDGPU::V_RNDNE_F16_t16_e64
6227 : AMDGPU::V_RNDNE_F16_fake16_e64;
6228 case AMDGPU::S_ADD_F32: return AMDGPU::V_ADD_F32_e64;
6229 case AMDGPU::S_SUB_F32: return AMDGPU::V_SUB_F32_e64;
6230 case AMDGPU::S_MIN_F32: return AMDGPU::V_MIN_F32_e64;
6231 case AMDGPU::S_MAX_F32: return AMDGPU::V_MAX_F32_e64;
6232 case AMDGPU::S_MINIMUM_F32: return AMDGPU::V_MINIMUM_F32_e64;
6233 case AMDGPU::S_MAXIMUM_F32: return AMDGPU::V_MAXIMUM_F32_e64;
6234 case AMDGPU::S_MUL_F32: return AMDGPU::V_MUL_F32_e64;
6235 case AMDGPU::S_ADD_F16:
6236 return ST.useRealTrue16Insts() ? AMDGPU::V_ADD_F16_t16_e64
6237 : AMDGPU::V_ADD_F16_fake16_e64;
6238 case AMDGPU::S_SUB_F16:
6239 return ST.useRealTrue16Insts() ? AMDGPU::V_SUB_F16_t16_e64
6240 : AMDGPU::V_SUB_F16_fake16_e64;
6241 case AMDGPU::S_MIN_F16:
6242 return ST.useRealTrue16Insts() ? AMDGPU::V_MIN_F16_t16_e64
6243 : AMDGPU::V_MIN_F16_fake16_e64;
6244 case AMDGPU::S_MAX_F16:
6245 return ST.useRealTrue16Insts() ? AMDGPU::V_MAX_F16_t16_e64
6246 : AMDGPU::V_MAX_F16_fake16_e64;
6247 case AMDGPU::S_MINIMUM_F16:
6248 return ST.useRealTrue16Insts() ? AMDGPU::V_MINIMUM_F16_t16_e64
6249 : AMDGPU::V_MINIMUM_F16_fake16_e64;
6250 case AMDGPU::S_MAXIMUM_F16:
6251 return ST.useRealTrue16Insts() ? AMDGPU::V_MAXIMUM_F16_t16_e64
6252 : AMDGPU::V_MAXIMUM_F16_fake16_e64;
6253 case AMDGPU::S_MUL_F16:
6254 return ST.useRealTrue16Insts() ? AMDGPU::V_MUL_F16_t16_e64
6255 : AMDGPU::V_MUL_F16_fake16_e64;
6256 case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64;
6257 case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64;
6258 case AMDGPU::S_FMAC_F16:
6259 return ST.useRealTrue16Insts() ? AMDGPU::V_FMAC_F16_t16_e64
6260 : AMDGPU::V_FMAC_F16_fake16_e64;
6261 case AMDGPU::S_FMAMK_F32: return AMDGPU::V_FMAMK_F32;
6262 case AMDGPU::S_FMAAK_F32: return AMDGPU::V_FMAAK_F32;
6263 case AMDGPU::S_CMP_LT_F32: return AMDGPU::V_CMP_LT_F32_e64;
6264 case AMDGPU::S_CMP_EQ_F32: return AMDGPU::V_CMP_EQ_F32_e64;
6265 case AMDGPU::S_CMP_LE_F32: return AMDGPU::V_CMP_LE_F32_e64;
6266 case AMDGPU::S_CMP_GT_F32: return AMDGPU::V_CMP_GT_F32_e64;
6267 case AMDGPU::S_CMP_LG_F32: return AMDGPU::V_CMP_LG_F32_e64;
6268 case AMDGPU::S_CMP_GE_F32: return AMDGPU::V_CMP_GE_F32_e64;
6269 case AMDGPU::S_CMP_O_F32: return AMDGPU::V_CMP_O_F32_e64;
6270 case AMDGPU::S_CMP_U_F32: return AMDGPU::V_CMP_U_F32_e64;
6271 case AMDGPU::S_CMP_NGE_F32: return AMDGPU::V_CMP_NGE_F32_e64;
6272 case AMDGPU::S_CMP_NLG_F32: return AMDGPU::V_CMP_NLG_F32_e64;
6273 case AMDGPU::S_CMP_NGT_F32: return AMDGPU::V_CMP_NGT_F32_e64;
6274 case AMDGPU::S_CMP_NLE_F32: return AMDGPU::V_CMP_NLE_F32_e64;
6275 case AMDGPU::S_CMP_NEQ_F32: return AMDGPU::V_CMP_NEQ_F32_e64;
6276 case AMDGPU::S_CMP_NLT_F32: return AMDGPU::V_CMP_NLT_F32_e64;
6277 case AMDGPU::S_CMP_LT_F16:
6278 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LT_F16_t16_e64
6279 : AMDGPU::V_CMP_LT_F16_fake16_e64;
6280 case AMDGPU::S_CMP_EQ_F16:
6281 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_EQ_F16_t16_e64
6282 : AMDGPU::V_CMP_EQ_F16_fake16_e64;
6283 case AMDGPU::S_CMP_LE_F16:
6284 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LE_F16_t16_e64
6285 : AMDGPU::V_CMP_LE_F16_fake16_e64;
6286 case AMDGPU::S_CMP_GT_F16:
6287 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GT_F16_t16_e64
6288 : AMDGPU::V_CMP_GT_F16_fake16_e64;
6289 case AMDGPU::S_CMP_LG_F16:
6290 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LG_F16_t16_e64
6291 : AMDGPU::V_CMP_LG_F16_fake16_e64;
6292 case AMDGPU::S_CMP_GE_F16:
6293 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GE_F16_t16_e64
6294 : AMDGPU::V_CMP_GE_F16_fake16_e64;
6295 case AMDGPU::S_CMP_O_F16:
6296 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_O_F16_t16_e64
6297 : AMDGPU::V_CMP_O_F16_fake16_e64;
6298 case AMDGPU::S_CMP_U_F16:
6299 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_U_F16_t16_e64
6300 : AMDGPU::V_CMP_U_F16_fake16_e64;
6301 case AMDGPU::S_CMP_NGE_F16:
6302 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGE_F16_t16_e64
6303 : AMDGPU::V_CMP_NGE_F16_fake16_e64;
6304 case AMDGPU::S_CMP_NLG_F16:
6305 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLG_F16_t16_e64
6306 : AMDGPU::V_CMP_NLG_F16_fake16_e64;
6307 case AMDGPU::S_CMP_NGT_F16:
6308 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGT_F16_t16_e64
6309 : AMDGPU::V_CMP_NGT_F16_fake16_e64;
6310 case AMDGPU::S_CMP_NLE_F16:
6311 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLE_F16_t16_e64
6312 : AMDGPU::V_CMP_NLE_F16_fake16_e64;
6313 case AMDGPU::S_CMP_NEQ_F16:
6314 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NEQ_F16_t16_e64
6315 : AMDGPU::V_CMP_NEQ_F16_fake16_e64;
6316 case AMDGPU::S_CMP_NLT_F16:
6317 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLT_F16_t16_e64
6318 : AMDGPU::V_CMP_NLT_F16_fake16_e64;
6319 case AMDGPU::V_S_EXP_F32_e64: return AMDGPU::V_EXP_F32_e64;
6320 case AMDGPU::V_S_EXP_F16_e64:
6321 return ST.useRealTrue16Insts() ? AMDGPU::V_EXP_F16_t16_e64
6322 : AMDGPU::V_EXP_F16_fake16_e64;
6323 case AMDGPU::V_S_LOG_F32_e64: return AMDGPU::V_LOG_F32_e64;
6324 case AMDGPU::V_S_LOG_F16_e64:
6325 return ST.useRealTrue16Insts() ? AMDGPU::V_LOG_F16_t16_e64
6326 : AMDGPU::V_LOG_F16_fake16_e64;
6327 case AMDGPU::V_S_RCP_F32_e64: return AMDGPU::V_RCP_F32_e64;
6328 case AMDGPU::V_S_RCP_F16_e64:
6329 return ST.useRealTrue16Insts() ? AMDGPU::V_RCP_F16_t16_e64
6330 : AMDGPU::V_RCP_F16_fake16_e64;
6331 case AMDGPU::V_S_RSQ_F32_e64: return AMDGPU::V_RSQ_F32_e64;
6332 case AMDGPU::V_S_RSQ_F16_e64:
6333 return ST.useRealTrue16Insts() ? AMDGPU::V_RSQ_F16_t16_e64
6334 : AMDGPU::V_RSQ_F16_fake16_e64;
6335 case AMDGPU::V_S_SQRT_F32_e64: return AMDGPU::V_SQRT_F32_e64;
6336 case AMDGPU::V_S_SQRT_F16_e64:
6337 return ST.useRealTrue16Insts() ? AMDGPU::V_SQRT_F16_t16_e64
6338 : AMDGPU::V_SQRT_F16_fake16_e64;
6339 }
6340 llvm_unreachable(
6341 "Unexpected scalar opcode without corresponding vector one!");
6342}
6343
6344// clang-format on
6345
6346void SIInstrInfo::insertScratchExecCopy(MachineFunction &MF,
6347 MachineBasicBlock &MBB,
6348 MachineBasicBlock::iterator MBBI,
6349 const DebugLoc &DL, Register Reg,
6350 bool IsSCCLive,
6351 SlotIndexes *Indexes) const {
6352 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
6353 const SIInstrInfo *TII = ST.getInstrInfo();
6354 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
6355 if (IsSCCLive) {
6356 // Insert two move instructions, one to save the original value of EXEC and
6357 // the other to turn on all bits in EXEC. This is required as we can't use
6358 // the single instruction S_OR_SAVEEXEC that clobbers SCC.
6359 auto StoreExecMI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: Reg)
6360 .addReg(RegNo: LMC.ExecReg, Flags: RegState::Kill);
6361 auto FlipExecMI =
6362 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg).addImm(Val: -1);
6363 if (Indexes) {
6364 Indexes->insertMachineInstrInMaps(MI&: *StoreExecMI);
6365 Indexes->insertMachineInstrInMaps(MI&: *FlipExecMI);
6366 }
6367 } else {
6368 auto SaveExec =
6369 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.OrSaveExecOpc), DestReg: Reg).addImm(Val: -1);
6370 SaveExec->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
6371 if (Indexes)
6372 Indexes->insertMachineInstrInMaps(MI&: *SaveExec);
6373 }
6374}
6375
6376void SIInstrInfo::restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
6377 MachineBasicBlock::iterator MBBI,
6378 const DebugLoc &DL, Register Reg,
6379 SlotIndexes *Indexes) const {
6380 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
6381 auto ExecRestoreMI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
6382 .addReg(RegNo: Reg, Flags: RegState::Kill);
6383 if (Indexes)
6384 Indexes->insertMachineInstrInMaps(MI&: *ExecRestoreMI);
6385}
6386
6387MachineInstr *
6388SIInstrInfo::getWholeWaveFunctionSetup(MachineFunction &MF) const {
6389 assert(MF.getInfo<SIMachineFunctionInfo>()->isWholeWaveFunction() &&
6390 "Not a whole wave func");
6391 MachineBasicBlock &MBB = *MF.begin();
6392 for (MachineInstr &MI : MBB)
6393 if (MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_SETUP ||
6394 MI.getOpcode() == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
6395 return &MI;
6396
6397 llvm_unreachable("Couldn't find SI_SETUP_WHOLE_WAVE_FUNC instruction");
6398}
6399
6400const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
6401 unsigned OpNo) const {
6402 const MCInstrDesc &Desc = get(Opcode: MI.getOpcode());
6403 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
6404 Desc.operands()[OpNo].RegClass == -1) {
6405 Register Reg = MI.getOperand(i: OpNo).getReg();
6406
6407 if (Reg.isVirtual()) {
6408 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6409 return MRI.getRegClass(Reg);
6410 }
6411 return RI.getPhysRegBaseClass(Reg);
6412 }
6413
6414 int16_t RegClass = getOpRegClassID(OpInfo: Desc.operands()[OpNo]);
6415 return RegClass < 0 ? nullptr : RI.getRegClass(i: RegClass);
6416}
6417
6418// Convert VOP3 operand index to source number.
6419static unsigned VOP3OpIdxToSrcN(const MachineInstr &MI, unsigned OpIdx) {
6420 constexpr AMDGPU::OpName OpNames[] = {
6421 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};
6422
6423 for (auto [I, OpName] : enumerate(First: OpNames)) {
6424 int SrcIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[I]);
6425 if (static_cast<unsigned>(SrcIdx) == OpIdx)
6426 return I;
6427 }
6428
6429 return UINT_MAX;
6430}
6431
6432void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
6433 MachineBasicBlock::iterator I = MI;
6434 MachineBasicBlock *MBB = MI.getParent();
6435 MachineOperand &MO = MI.getOperand(i: OpIdx);
6436 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
6437 unsigned RCID = getOpRegClassID(OpInfo: get(Opcode: MI.getOpcode()).operands()[OpIdx]);
6438 const TargetRegisterClass *RC = RI.getRegClass(i: RCID);
6439 unsigned Size = RI.getRegSizeInBits(RC: *RC);
6440 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO
6441 : Size == 16 ? AMDGPU::V_MOV_B16_t16_e64
6442 : AMDGPU::V_MOV_B32_e32;
6443 if (MO.isReg())
6444 Opcode = AMDGPU::COPY;
6445 else if (RI.isSGPRClass(RC))
6446 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
6447
6448 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(SRC: RC);
6449 Register Reg = MRI.createVirtualRegister(RegClass: VRC);
6450 DebugLoc DL = MBB->findDebugLoc(MBBI: I);
6451
6452 if (Size == 128 && AMDGPU::isPackedSingleSGPR64BitInst(Opc: MI.getOpcode()) &&
6453 isLegalSingleSGPRReadInstOperand(MRI, MI, SrcN: VOP3OpIdxToSrcN(MI, OpIdx))) {
6454 // Special case for V_PK_*64 instructions: these do not have OPSEL but SGPR
6455 // sources behave like OPSEL is set replicating low 64-bits into high. VGPR
6456 // sources in turn read actual 4 registers. To move operand from an SGPR to
6457 // a VGPR we need to replicate low half.
6458 // We also do not select immediates for these instructions so it always has
6459 // to be an SGPR register here.
6460 // Operands which are not legal as per isLegalSingleSGPRReadInstOperand()
6461 // sent here specifically to fix a non-splat SGPR and shall perform a full
6462 // copy.
6463
6464 const TargetRegisterClass *VRC64 = RI.getVGPRClassForBitWidth(BitWidth: 64);
6465 Register Low64 = MRI.createVirtualRegister(RegClass: VRC64);
6466 assert(MO.isReg() && RI.isSGPRReg(MRI, MO.getReg()));
6467 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Low64)
6468 .addReg(RegNo: MO.getReg(), Flags: {}, SubReg: AMDGPU::sub0_sub1);
6469 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Reg)
6470 .addReg(RegNo: Low64)
6471 .addImm(Val: AMDGPU::sub0_sub1)
6472 .addReg(RegNo: Low64, Flags: RegState::Kill)
6473 .addImm(Val: AMDGPU::sub2_sub3);
6474 } else {
6475 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: Reg).add(MO);
6476 }
6477
6478 MO.ChangeToRegister(Reg, isDef: false);
6479}
6480
6481unsigned SIInstrInfo::buildExtractSubReg(
6482 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
6483 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
6484 unsigned SubIdx, const TargetRegisterClass *SubRC) const {
6485 if (!SuperReg.getReg().isVirtual())
6486 return RI.getSubReg(Reg: SuperReg.getReg(), Idx: SubIdx);
6487
6488 MachineBasicBlock *MBB = MI->getParent();
6489 const DebugLoc &DL = MI->getDebugLoc();
6490 Register SubReg = MRI.createVirtualRegister(RegClass: SubRC);
6491
6492 unsigned NewSubIdx = RI.composeSubRegIndices(a: SuperReg.getSubReg(), b: SubIdx);
6493 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: SubReg)
6494 .addReg(RegNo: SuperReg.getReg(), Flags: {}, SubReg: NewSubIdx);
6495 return SubReg;
6496}
6497
6498MachineOperand SIInstrInfo::buildExtractSubRegOrImm(
6499 MachineBasicBlock::iterator MII, MachineRegisterInfo &MRI,
6500 const MachineOperand &Op, const TargetRegisterClass *SuperRC,
6501 unsigned SubIdx, const TargetRegisterClass *SubRC) const {
6502 if (Op.isImm()) {
6503 if (SubIdx == AMDGPU::sub0)
6504 return MachineOperand::CreateImm(Val: static_cast<int32_t>(Op.getImm()));
6505 if (SubIdx == AMDGPU::sub1)
6506 return MachineOperand::CreateImm(Val: static_cast<int32_t>(Op.getImm() >> 32));
6507
6508 llvm_unreachable("Unhandled register index for immediate");
6509 }
6510
6511 unsigned SubReg = buildExtractSubReg(MI: MII, MRI, SuperReg: Op, SuperRC,
6512 SubIdx, SubRC);
6513 return MachineOperand::CreateReg(Reg: SubReg, isDef: false);
6514}
6515
6516// Change the order of operands from (0, 1, 2) to (0, 2, 1)
6517void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
6518 assert(Inst.getNumExplicitOperands() == 3);
6519 MachineOperand Op1 = Inst.getOperand(i: 1);
6520 Inst.removeOperand(OpNo: 1);
6521 Inst.addOperand(Op: Op1);
6522}
6523
6524bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
6525 const MCOperandInfo &OpInfo,
6526 const MachineOperand &MO) const {
6527 if (!MO.isReg())
6528 return false;
6529
6530 Register Reg = MO.getReg();
6531
6532 const TargetRegisterClass *DRC = RI.getRegClass(i: getOpRegClassID(OpInfo));
6533 if (Reg.isPhysical())
6534 return DRC->contains(Reg);
6535
6536 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
6537
6538 if (MO.getSubReg()) {
6539 const TargetRegisterClass *SuperRC =
6540 RI.getLargestLegalSuperClass(RC, MF: MRI.getMF());
6541 if (!SuperRC)
6542 return false;
6543 return RI.getMatchingSuperRegClass(A: SuperRC, B: DRC, Idx: MO.getSubReg()) != nullptr;
6544 }
6545
6546 return RI.getCommonSubClass(A: DRC, B: RC) != nullptr;
6547}
6548
6549bool SIInstrInfo::isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
6550 const MachineOperand &MO) const {
6551 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6552 const MCOperandInfo OpInfo = MI.getDesc().operands()[OpIdx];
6553 unsigned Opc = MI.getOpcode();
6554
6555 // See SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
6556 if (MO.isReg() && RI.isSGPRReg(MRI, Reg: MO.getReg()) &&
6557 AMDGPU::isSingleSGPRReadInst(Opc: MI.getOpcode()) &&
6558 !isLegalSingleSGPRReadInstOperand(MRI, MI, SrcN: VOP3OpIdxToSrcN(MI, OpIdx),
6559 MO: &MO))
6560 return false;
6561
6562 if (!isLegalRegOperand(MRI, OpInfo, MO))
6563 return false;
6564
6565 // check Accumulate GPR operand
6566 bool IsAGPR = RI.isAGPR(MRI, Reg: MO.getReg());
6567 if (IsAGPR && !ST.hasMAIInsts())
6568 return false;
6569 if (IsAGPR && (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
6570 (MI.mayLoad() || MI.mayStore() || isDS(Opcode: Opc) || isMIMG(Opcode: Opc)))
6571 return false;
6572 // Atomics should have both vdst and vdata either vgpr or agpr.
6573 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
6574 const int DataIdx = AMDGPU::getNamedOperandIdx(
6575 Opcode: Opc, Name: isDS(Opcode: Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata);
6576 if ((int)OpIdx == VDstIdx && DataIdx != -1 &&
6577 MI.getOperand(i: DataIdx).isReg() &&
6578 RI.isAGPR(MRI, Reg: MI.getOperand(i: DataIdx).getReg()) != IsAGPR)
6579 return false;
6580 if ((int)OpIdx == DataIdx) {
6581 if (VDstIdx != -1 &&
6582 RI.isAGPR(MRI, Reg: MI.getOperand(i: VDstIdx).getReg()) != IsAGPR)
6583 return false;
6584 // DS instructions with 2 src operands also must have tied RC.
6585 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data1);
6586 if (Data1Idx != -1 && MI.getOperand(i: Data1Idx).isReg() &&
6587 RI.isAGPR(MRI, Reg: MI.getOperand(i: Data1Idx).getReg()) != IsAGPR)
6588 return false;
6589 }
6590
6591 // Check V_ACCVGPR_WRITE_B32_e64
6592 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() &&
6593 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0) &&
6594 RI.isSGPRReg(MRI, Reg: MO.getReg()))
6595 return false;
6596
6597 if (ST.hasFlatScratchHiInB64InstHazard() &&
6598 MO.getReg() == AMDGPU::SRC_FLAT_SCRATCH_BASE_HI && isSALU(MI)) {
6599 if (const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst)) {
6600 if (AMDGPU::getRegBitWidth(RC: *RI.getRegClassForReg(MRI, Reg: Dst->getReg())) ==
6601 64)
6602 return false;
6603 }
6604 if (Opc == AMDGPU::S_BITCMP0_B64 || Opc == AMDGPU::S_BITCMP1_B64)
6605 return false;
6606 }
6607 if (!ST.hasDPPSrc1SGPR() && isDPP(MI) && RI.isSGPRReg(MRI, Reg: MO.getReg()) &&
6608 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1))
6609 return false;
6610
6611 return true;
6612}
6613
6614bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI,
6615 const MCOperandInfo &OpInfo,
6616 const MachineOperand &MO) const {
6617 if (MO.isReg())
6618 return isLegalRegOperand(MRI, OpInfo, MO);
6619
6620 // Handle non-register types that are treated like immediates.
6621 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
6622 return true;
6623}
6624
6625bool SIInstrInfo::isLegalSingleSGPRReadInstOperand(
6626 const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,
6627 const MachineOperand *MO) const {
6628 constexpr unsigned NumOps = 3;
6629 constexpr AMDGPU::OpName OpNames[NumOps * 2] = {
6630 AMDGPU::OpName::src0, AMDGPU::OpName::src1,
6631 AMDGPU::OpName::src2, AMDGPU::OpName::src0_modifiers,
6632 AMDGPU::OpName::src1_modifiers, AMDGPU::OpName::src2_modifiers};
6633
6634 assert(SrcN < NumOps);
6635
6636 if (!MO) {
6637 int SrcIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[SrcN]);
6638 if (SrcIdx == -1)
6639 return true;
6640 MO = &MI.getOperand(i: SrcIdx);
6641 }
6642
6643 if (!MO->isReg() || !RI.isSGPRReg(MRI, Reg: MO->getReg()))
6644 return true;
6645
6646 int ModsIdx =
6647 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[NumOps + SrcN]);
6648 if (ModsIdx == -1)
6649 return false;
6650
6651 unsigned Mods = MI.getOperand(i: ModsIdx).getImm();
6652 bool OpSel = Mods & SISrcMods::OP_SEL_0;
6653 bool OpSelHi = Mods & SISrcMods::OP_SEL_1;
6654
6655 return !OpSel && !OpSelHi;
6656}
6657
6658bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
6659 const MachineOperand *MO) const {
6660 const MachineFunction &MF = *MI.getMF();
6661 const MachineRegisterInfo &MRI = MF.getRegInfo();
6662 const MCInstrDesc &InstDesc = MI.getDesc();
6663 const MCOperandInfo &OpInfo = InstDesc.operands()[OpIdx];
6664 int64_t RegClass = getOpRegClassID(OpInfo);
6665 const TargetRegisterClass *DefinedRC =
6666 RegClass != -1 ? RI.getRegClass(i: RegClass) : nullptr;
6667 if (!MO)
6668 MO = &MI.getOperand(i: OpIdx);
6669
6670 const bool IsInlineConst = !MO->isReg() && isInlineConstant(MO: *MO, OpInfo);
6671
6672 if (isVALU(MI, /*AllowLDSDMA=*/true) && !IsInlineConst &&
6673 usesConstantBus(MRI, MO: *MO, OpInfo)) {
6674 const MachineOperand *UsedLiteral = nullptr;
6675
6676 int ConstantBusLimit = ST.getConstantBusLimit(Opcode: MI.getOpcode());
6677 int LiteralLimit = !isVOP3(MI) || ST.hasVOP3Literal() ? 1 : 0;
6678
6679 // TODO: Be more permissive with frame indexes.
6680 if (!MO->isReg() && !isInlineConstant(MO: *MO, OpInfo)) {
6681 if (!LiteralLimit--)
6682 return false;
6683
6684 UsedLiteral = MO;
6685 }
6686
6687 SmallDenseSet<RegSubRegPair> SGPRsUsed;
6688 if (MO->isReg())
6689 SGPRsUsed.insert(V: RegSubRegPair(MO->getReg(), MO->getSubReg()));
6690
6691 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
6692 if (i == OpIdx)
6693 continue;
6694 const MachineOperand &Op = MI.getOperand(i);
6695 if (Op.isReg()) {
6696 if (Op.isUse()) {
6697 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
6698 if (regUsesConstantBus(RegOp: Op, MRI) && SGPRsUsed.insert(V: SGPR).second) {
6699 if (--ConstantBusLimit <= 0)
6700 return false;
6701 }
6702 }
6703 } else if (AMDGPU::isSISrcOperand(OpInfo: InstDesc.operands()[i]) &&
6704 !isInlineConstant(MO: Op, OpInfo: InstDesc.operands()[i])) {
6705 // The same literal may be used multiple times.
6706 if (!UsedLiteral)
6707 UsedLiteral = &Op;
6708 else if (UsedLiteral->isIdenticalTo(Other: Op))
6709 continue;
6710
6711 if (!LiteralLimit--)
6712 return false;
6713 if (--ConstantBusLimit <= 0)
6714 return false;
6715 }
6716 }
6717 } else if (!IsInlineConst && !MO->isReg() && isSALU(MI)) {
6718 // There can be at most one literal operand, but it can be repeated.
6719 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
6720 if (i == OpIdx)
6721 continue;
6722 const MachineOperand &Op = MI.getOperand(i);
6723 if (!Op.isReg() && !Op.isFI() && !Op.isRegMask() &&
6724 !isInlineConstant(MO: Op, OpInfo: InstDesc.operands()[i]) &&
6725 !Op.isIdenticalTo(Other: *MO))
6726 return false;
6727
6728 // Do not fold a non-inlineable and non-register operand into an
6729 // instruction that already has a frame index. The frame index handling
6730 // code could not handle well when a frame index co-exists with another
6731 // non-register operand, unless that operand is an inlineable immediate.
6732 if (Op.isFI())
6733 return false;
6734 }
6735 } else if (IsInlineConst && ST.hasNoF16PseudoScalarTransInlineConstants() &&
6736 isF16PseudoScalarTrans(Opcode: MI.getOpcode())) {
6737 return false;
6738 }
6739
6740 if (MO->isReg()) {
6741 if (!DefinedRC)
6742 return OpInfo.OperandType == MCOI::OPERAND_UNKNOWN;
6743 return isLegalRegOperand(MI, OpIdx, MO: *MO);
6744 }
6745
6746 if (MO->isImm()) {
6747 uint64_t Imm = MO->getImm();
6748 bool Is64BitFPOp = OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_FP64 ||
6749 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP64;
6750 bool Is64BitOp = Is64BitFPOp ||
6751 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64 ||
6752 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT32 ||
6753 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32 ||
6754 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT64;
6755 if (Is64BitOp &&
6756 !AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: ST.hasInv2PiInlineImm())) {
6757 if (!AMDGPU::isValid32BitLiteral(Val: Imm, IsFP64: Is64BitFPOp) &&
6758 (!ST.has64BitLiterals() || InstDesc.getSize() != 4))
6759 return false;
6760
6761 // FIXME: We can use sign extended 64-bit literals, but only for signed
6762 // operands. At the moment we do not know if an operand is signed.
6763 // Such operand will be encoded as its low 32 bits and then either
6764 // correctly sign extended or incorrectly zero extended by HW.
6765 // If 64-bit literals are supported and the literal will be encoded
6766 // as full 64 bit we still can use it.
6767 if (!Is64BitFPOp && (int32_t)Imm < 0 &&
6768 (!ST.has64BitLiterals() || AMDGPU::isValid32BitLiteral(Val: Imm, IsFP64: false)))
6769 return false;
6770 }
6771 }
6772
6773 // Handle non-register types that are treated like immediates.
6774 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
6775
6776 if (!DefinedRC) {
6777 // This operand expects an immediate.
6778 return true;
6779 }
6780
6781 return isImmOperandLegal(MI, OpNo: OpIdx, MO: *MO);
6782}
6783
6784bool SIInstrInfo::isNeverCoissue(MachineInstr &MI) const {
6785 bool IsGFX950Only = ST.hasGFX950Insts();
6786 bool IsGFX940Only = ST.hasGFX940Insts();
6787
6788 if (!IsGFX950Only && !IsGFX940Only)
6789 return false;
6790
6791 if (!isVALU(MI, /*AllowLDSDMA=*/true))
6792 return false;
6793
6794 // V_COS, V_EXP, V_RCP, etc.
6795 if (isTRANS(MI))
6796 return true;
6797
6798 // DOT2, DOT2C, DOT4, etc.
6799 if (isDOT(MI))
6800 return true;
6801
6802 // MFMA, SMFMA
6803 if (isMFMA(MI))
6804 return true;
6805
6806 unsigned Opcode = MI.getOpcode();
6807 switch (Opcode) {
6808 case AMDGPU::V_CVT_PK_BF8_F32_e64:
6809 case AMDGPU::V_CVT_PK_FP8_F32_e64:
6810 case AMDGPU::V_MQSAD_PK_U16_U8_e64:
6811 case AMDGPU::V_MQSAD_U32_U8_e64:
6812 case AMDGPU::V_PK_ADD_F16:
6813 case AMDGPU::V_PK_ADD_F32:
6814 case AMDGPU::V_PK_ADD_I16:
6815 case AMDGPU::V_PK_ADD_U16:
6816 case AMDGPU::V_PK_ASHRREV_I16:
6817 case AMDGPU::V_PK_FMA_F16:
6818 case AMDGPU::V_PK_FMA_F32:
6819 case AMDGPU::V_PK_FMAC_F16_e32:
6820 case AMDGPU::V_PK_FMAC_F16_e64:
6821 case AMDGPU::V_PK_LSHLREV_B16:
6822 case AMDGPU::V_PK_LSHRREV_B16:
6823 case AMDGPU::V_PK_MAD_I16:
6824 case AMDGPU::V_PK_MAD_U16:
6825 case AMDGPU::V_PK_MAX_F16:
6826 case AMDGPU::V_PK_MAX_I16:
6827 case AMDGPU::V_PK_MAX_U16:
6828 case AMDGPU::V_PK_MIN_F16:
6829 case AMDGPU::V_PK_MIN_I16:
6830 case AMDGPU::V_PK_MIN_U16:
6831 case AMDGPU::V_PK_MOV_B32:
6832 case AMDGPU::V_PK_MUL_F16:
6833 case AMDGPU::V_PK_MUL_F32:
6834 case AMDGPU::V_PK_MUL_LO_U16:
6835 case AMDGPU::V_PK_SUB_I16:
6836 case AMDGPU::V_PK_SUB_U16:
6837 case AMDGPU::V_QSAD_PK_U16_U8_e64:
6838 return true;
6839 default:
6840 return false;
6841 }
6842}
6843
6844void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
6845 MachineInstr &MI) const {
6846 unsigned Opc = MI.getOpcode();
6847 const MCInstrDesc &InstrDesc = get(Opcode: Opc);
6848
6849 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
6850 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
6851
6852 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
6853 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
6854
6855 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
6856 // we need to only have one constant bus use before GFX10.
6857 bool HasImplicitSGPR = findImplicitSGPRRead(MI);
6858 if (HasImplicitSGPR && ST.getConstantBusLimit(Opcode: Opc) <= 1 && Src0.isReg() &&
6859 RI.isSGPRReg(MRI, Reg: Src0.getReg()))
6860 legalizeOpWithMove(MI, OpIdx: Src0Idx);
6861
6862 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
6863 // both the value to write (src0) and lane select (src1). Fix up non-SGPR
6864 // src0/src1 with V_READFIRSTLANE.
6865 if (Opc == AMDGPU::V_WRITELANE_B32) {
6866 const DebugLoc &DL = MI.getDebugLoc();
6867 if (Src0.isReg() && RI.isVGPR(MRI, Reg: Src0.getReg())) {
6868 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6869 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
6870 .add(MO: Src0);
6871 Src0.ChangeToRegister(Reg, isDef: false);
6872 }
6873 if (Src1.isReg() && RI.isVGPR(MRI, Reg: Src1.getReg())) {
6874 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6875 const DebugLoc &DL = MI.getDebugLoc();
6876 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
6877 .add(MO: Src1);
6878 Src1.ChangeToRegister(Reg, isDef: false);
6879 }
6880 return;
6881 }
6882
6883 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2.
6884 if (Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F16_e32) {
6885 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
6886 if (!RI.isVGPR(MRI, Reg: MI.getOperand(i: Src2Idx).getReg()))
6887 legalizeOpWithMove(MI, OpIdx: Src2Idx);
6888 }
6889
6890 // VOP2 src0 instructions support all operand types, so we don't need to check
6891 // their legality. If src1 is already legal, we don't need to do anything.
6892 if (isLegalRegOperand(MRI, OpInfo: InstrDesc.operands()[Src1Idx], MO: Src1))
6893 return;
6894
6895 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
6896 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
6897 // select is uniform.
6898 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
6899 RI.isVGPR(MRI, Reg: Src1.getReg())) {
6900 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6901 const DebugLoc &DL = MI.getDebugLoc();
6902 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
6903 .add(MO: Src1);
6904 Src1.ChangeToRegister(Reg, isDef: false);
6905 return;
6906 }
6907
6908 // We do not use commuteInstruction here because it is too aggressive and will
6909 // commute if it is possible. We only want to commute here if it improves
6910 // legality. This can be called a fairly large number of times so don't waste
6911 // compile time pointlessly swapping and checking legality again.
6912 if (HasImplicitSGPR || !MI.isCommutable()) {
6913 legalizeOpWithMove(MI, OpIdx: Src1Idx);
6914 return;
6915 }
6916
6917 // If src0 can be used as src1, commuting will make the operands legal.
6918 // Otherwise we have to give up and insert a move.
6919 //
6920 // TODO: Other immediate-like operand kinds could be commuted if there was a
6921 // MachineOperand::ChangeTo* for them.
6922 if ((!Src1.isImm() && !Src1.isReg()) ||
6923 !isLegalRegOperand(MRI, OpInfo: InstrDesc.operands()[Src1Idx], MO: Src0)) {
6924 legalizeOpWithMove(MI, OpIdx: Src1Idx);
6925 return;
6926 }
6927
6928 int CommutedOpc = commuteOpcode(MI);
6929 if (CommutedOpc == -1) {
6930 legalizeOpWithMove(MI, OpIdx: Src1Idx);
6931 return;
6932 }
6933
6934 MI.setDesc(get(Opcode: CommutedOpc));
6935
6936 Register Src0Reg = Src0.getReg();
6937 unsigned Src0SubReg = Src0.getSubReg();
6938 bool Src0Kill = Src0.isKill();
6939
6940 if (Src1.isImm())
6941 Src0.ChangeToImmediate(ImmVal: Src1.getImm());
6942 else if (Src1.isReg()) {
6943 Src0.ChangeToRegister(Reg: Src1.getReg(), isDef: false, isImp: false, isKill: Src1.isKill());
6944 Src0.setSubReg(Src1.getSubReg());
6945 } else
6946 llvm_unreachable("Should only have register or immediate operands");
6947
6948 Src1.ChangeToRegister(Reg: Src0Reg, isDef: false, isImp: false, isKill: Src0Kill);
6949 Src1.setSubReg(Src0SubReg);
6950 fixImplicitOperands(MI);
6951}
6952
6953// Legalize VOP3 operands. All operand types are supported for any operand
6954// but only one literal constant and only starting from GFX10.
6955void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
6956 MachineInstr &MI) const {
6957 unsigned Opc = MI.getOpcode();
6958
6959 int VOP3Idx[3] = {
6960 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0),
6961 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1),
6962 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2)
6963 };
6964
6965 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 ||
6966 Opc == AMDGPU::V_PERMLANEX16_B32_e64 ||
6967 Opc == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
6968 Opc == AMDGPU::V_PERMLANE_UP_B32_e64 ||
6969 Opc == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
6970 Opc == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
6971 Opc == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64) {
6972 // src1 and src2 must be scalar
6973 MachineOperand &Src1 = MI.getOperand(i: VOP3Idx[1]);
6974 const DebugLoc &DL = MI.getDebugLoc();
6975 if (Src1.isReg() && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src1.getReg()))) {
6976 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6977 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
6978 .add(MO: Src1);
6979 Src1.ChangeToRegister(Reg, isDef: false);
6980 }
6981 if (VOP3Idx[2] != -1) {
6982 MachineOperand &Src2 = MI.getOperand(i: VOP3Idx[2]);
6983 if (Src2.isReg() && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src2.getReg()))) {
6984 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6985 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
6986 .add(MO: Src2);
6987 Src2.ChangeToRegister(Reg, isDef: false);
6988 }
6989 }
6990 }
6991
6992 // Find the one SGPR operand we are allowed to use.
6993 int ConstantBusLimit = ST.getConstantBusLimit(Opcode: Opc);
6994 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
6995 SmallDenseSet<unsigned> SGPRsUsed;
6996 Register SGPRReg = findUsedSGPR(MI, OpIndices: VOP3Idx);
6997 if (SGPRReg) {
6998 SGPRsUsed.insert(V: SGPRReg);
6999 --ConstantBusLimit;
7000 }
7001
7002 for (int Idx : VOP3Idx) {
7003 if (Idx == -1)
7004 break;
7005 MachineOperand &MO = MI.getOperand(i: Idx);
7006
7007 if (!MO.isReg()) {
7008 if (isInlineConstant(MO, OpInfo: get(Opcode: Opc).operands()[Idx]))
7009 continue;
7010
7011 if (LiteralLimit > 0 && ConstantBusLimit > 0) {
7012 --LiteralLimit;
7013 --ConstantBusLimit;
7014 continue;
7015 }
7016
7017 --LiteralLimit;
7018 --ConstantBusLimit;
7019 legalizeOpWithMove(MI, OpIdx: Idx);
7020 continue;
7021 }
7022
7023 if (!RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg: MO.getReg())))
7024 continue; // VGPRs are legal
7025
7026 // We can use one SGPR in each VOP3 instruction prior to GFX10
7027 // and two starting from GFX10.
7028 if (SGPRsUsed.count(V: MO.getReg()))
7029 continue;
7030 if (ConstantBusLimit > 0) {
7031 SGPRsUsed.insert(V: MO.getReg());
7032 --ConstantBusLimit;
7033 continue;
7034 }
7035
7036 // If we make it this far, then the operand is not legal and we must
7037 // legalize it.
7038 legalizeOpWithMove(MI, OpIdx: Idx);
7039 }
7040
7041 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2 tied to vdst.
7042 if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64) &&
7043 !RI.isVGPR(MRI, Reg: MI.getOperand(i: VOP3Idx[2]).getReg()))
7044 legalizeOpWithMove(MI, OpIdx: VOP3Idx[2]);
7045
7046 // Fix the register class of single-sgpr-read instructions on gfx12+. See
7047 // SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
7048 if (AMDGPU::isSingleSGPRReadInst(Opc)) {
7049 for (unsigned I = 0; I < 3; ++I) {
7050 if (!isLegalSingleSGPRReadInstOperand(MRI, MI, /*SrcN=*/I))
7051 legalizeOpWithMove(MI, OpIdx: VOP3Idx[I]);
7052 }
7053 }
7054}
7055
7056Register SIInstrInfo::readlaneVGPRToSGPR(
7057 Register SrcReg, MachineInstr &UseMI, MachineRegisterInfo &MRI,
7058 const TargetRegisterClass *DstRC /*=nullptr*/) const {
7059 const TargetRegisterClass *VRC = MRI.getRegClass(Reg: SrcReg);
7060 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
7061 if (DstRC)
7062 SRC = RI.getCommonSubClass(A: SRC, B: DstRC);
7063
7064 Register DstReg = MRI.createVirtualRegister(RegClass: SRC);
7065 unsigned SubRegs = RI.getRegSizeInBits(RC: *VRC) / 32;
7066
7067 if (RI.hasAGPRs(RC: VRC)) {
7068 VRC = RI.getEquivalentVGPRClass(SRC: VRC);
7069 Register NewSrcReg = MRI.createVirtualRegister(RegClass: VRC);
7070 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7071 MCID: get(Opcode: TargetOpcode::COPY), DestReg: NewSrcReg)
7072 .addReg(RegNo: SrcReg);
7073 SrcReg = NewSrcReg;
7074 }
7075
7076 if (SubRegs == 1) {
7077 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7078 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
7079 .addReg(RegNo: SrcReg);
7080 return DstReg;
7081 }
7082
7083 SmallVector<Register, 8> SRegs;
7084 for (unsigned i = 0; i < SubRegs; ++i) {
7085 Register SGPR = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
7086 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7087 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: SGPR)
7088 .addReg(RegNo: SrcReg, Flags: {}, SubReg: RI.getSubRegFromChannel(Channel: i));
7089 SRegs.push_back(Elt: SGPR);
7090 }
7091
7092 MachineInstrBuilder MIB =
7093 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7094 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
7095 for (unsigned i = 0; i < SubRegs; ++i) {
7096 MIB.addReg(RegNo: SRegs[i]);
7097 MIB.addImm(Val: RI.getSubRegFromChannel(Channel: i));
7098 }
7099 return DstReg;
7100}
7101
7102void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,
7103 MachineInstr &MI) const {
7104
7105 // If the pointer is store in VGPRs, then we need to move them to
7106 // SGPRs using v_readfirstlane. This is safe because we only select
7107 // loads with uniform pointers to SMRD instruction so we know the
7108 // pointer value is uniform.
7109 MachineOperand *SBase = getNamedOperand(MI, OperandName: AMDGPU::OpName::sbase);
7110 if (SBase && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SBase->getReg()))) {
7111 Register SGPR = readlaneVGPRToSGPR(SrcReg: SBase->getReg(), UseMI&: MI, MRI);
7112 SBase->setReg(SGPR);
7113 }
7114 MachineOperand *SOff = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
7115 if (SOff && !RI.isSGPRReg(MRI, Reg: SOff->getReg())) {
7116 Register SGPR = readlaneVGPRToSGPR(SrcReg: SOff->getReg(), UseMI&: MI, MRI);
7117 SOff->setReg(SGPR);
7118 }
7119}
7120
7121bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const {
7122 unsigned Opc = Inst.getOpcode();
7123 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::saddr);
7124 if (OldSAddrIdx < 0)
7125 return false;
7126
7127 assert(isSegmentSpecificFLAT(Inst) || (isFLAT(Inst) && ST.hasFlatGVSMode()));
7128
7129 int NewOpc = AMDGPU::getGlobalVaddrOp(Opcode: Opc);
7130 if (NewOpc < 0)
7131 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opcode: Opc);
7132 if (NewOpc < 0)
7133 return false;
7134
7135 MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
7136 MachineOperand &SAddr = Inst.getOperand(i: OldSAddrIdx);
7137 if (RI.isSGPRReg(MRI, Reg: SAddr.getReg()))
7138 return false;
7139
7140 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::vaddr);
7141 if (NewVAddrIdx < 0)
7142 return false;
7143
7144 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
7145
7146 // Check vaddr, it shall be zero or absent.
7147 MachineInstr *VAddrDef = nullptr;
7148 if (OldVAddrIdx >= 0) {
7149 MachineOperand &VAddr = Inst.getOperand(i: OldVAddrIdx);
7150 VAddrDef = MRI.getUniqueVRegDef(Reg: VAddr.getReg());
7151 if (!VAddrDef || !VAddrDef->isMoveImmediate() ||
7152 !VAddrDef->getOperand(i: 1).isImm() ||
7153 VAddrDef->getOperand(i: 1).getImm() != 0)
7154 return false;
7155 }
7156
7157 const MCInstrDesc &NewDesc = get(Opcode: NewOpc);
7158 Inst.setDesc(NewDesc);
7159
7160 // Callers expect iterator to be valid after this call, so modify the
7161 // instruction in place.
7162 if (OldVAddrIdx == NewVAddrIdx) {
7163 MachineOperand &NewVAddr = Inst.getOperand(i: NewVAddrIdx);
7164 // Clear use list from the old vaddr holding a zero register.
7165 MRI.removeRegOperandFromUseList(MO: &NewVAddr);
7166 MRI.moveOperands(Dst: &NewVAddr, Src: &SAddr, NumOps: 1);
7167 Inst.removeOperand(OpNo: OldSAddrIdx);
7168 // Update the use list with the pointer we have just moved from vaddr to
7169 // saddr position. Otherwise new vaddr will be missing from the use list.
7170 MRI.removeRegOperandFromUseList(MO: &NewVAddr);
7171 MRI.addRegOperandToUseList(MO: &NewVAddr);
7172 } else {
7173 assert(OldSAddrIdx == NewVAddrIdx);
7174
7175 if (OldVAddrIdx >= 0) {
7176 int NewVDstIn = AMDGPU::getNamedOperandIdx(Opcode: NewOpc,
7177 Name: AMDGPU::OpName::vdst_in);
7178
7179 // removeOperand doesn't try to fixup tied operand indexes at it goes, so
7180 // it asserts. Untie the operands for now and retie them afterwards.
7181 if (NewVDstIn != -1) {
7182 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst_in);
7183 Inst.untieRegOperand(OpIdx: OldVDstIn);
7184 }
7185
7186 Inst.removeOperand(OpNo: OldVAddrIdx);
7187
7188 if (NewVDstIn != -1) {
7189 int NewVDst = AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::vdst);
7190 Inst.tieOperands(DefIdx: NewVDst, UseIdx: NewVDstIn);
7191 }
7192 }
7193 }
7194
7195 if (VAddrDef && MRI.use_nodbg_empty(RegNo: VAddrDef->getOperand(i: 0).getReg()))
7196 VAddrDef->eraseFromParent();
7197
7198 return true;
7199}
7200
7201// FIXME: Remove this when SelectionDAG is obsoleted.
7202void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI,
7203 MachineInstr &MI) const {
7204 if (!isSegmentSpecificFLAT(MI) && !ST.hasFlatGVSMode())
7205 return;
7206
7207 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence
7208 // thinks they are uniform, so a readfirstlane should be valid.
7209 MachineOperand *SAddr = getNamedOperand(MI, OperandName: AMDGPU::OpName::saddr);
7210 if (!SAddr || RI.isSGPRClass(RC: MRI.getRegClass(Reg: SAddr->getReg())))
7211 return;
7212
7213 if (moveFlatAddrToVGPR(Inst&: MI))
7214 return;
7215
7216 const TargetRegisterClass *DeclaredRC =
7217 getRegClass(MCID: MI.getDesc(), OpNum: SAddr->getOperandNo());
7218
7219 Register ToSGPR = readlaneVGPRToSGPR(SrcReg: SAddr->getReg(), UseMI&: MI, MRI, DstRC: DeclaredRC);
7220 SAddr->setReg(ToSGPR);
7221}
7222
7223void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
7224 MachineBasicBlock::iterator I,
7225 const TargetRegisterClass *DstRC,
7226 MachineOperand &Op,
7227 MachineRegisterInfo &MRI,
7228 const DebugLoc &DL) const {
7229 Register OpReg = Op.getReg();
7230 unsigned OpSubReg = Op.getSubReg();
7231
7232 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
7233 RI.getRegClassForReg(MRI, Reg: OpReg), OpSubReg);
7234
7235 // Check if operand is already the correct register class.
7236 if (DstRC == OpRC)
7237 return;
7238
7239 Register DstReg = MRI.createVirtualRegister(RegClass: DstRC);
7240 auto Copy = BuildMI(BB&: InsertMBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: DstReg)
7241 .addReg(RegNo: OpReg, Flags: {}, SubReg: OpSubReg);
7242 Op.setReg(DstReg);
7243 Op.setSubReg(AMDGPU::NoSubRegister);
7244
7245 MachineInstr *Def = MRI.getVRegDef(Reg: OpReg);
7246 if (!Def)
7247 return;
7248
7249 // Try to eliminate the copy if it is copying an immediate value.
7250 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass)
7251 foldImmediate(UseMI&: *Copy, DefMI&: *Def, Reg: OpReg, MRI: &MRI);
7252
7253 bool ImpDef = Def->isImplicitDef();
7254 while (!ImpDef && Def && Def->isCopy()) {
7255 if (Def->getOperand(i: 1).getReg().isPhysical())
7256 break;
7257 Def = MRI.getUniqueVRegDef(Reg: Def->getOperand(i: 1).getReg());
7258 ImpDef = Def && Def->isImplicitDef();
7259 }
7260 if (!RI.isSGPRClass(RC: DstRC) && !Copy->readsRegister(Reg: AMDGPU::EXEC, TRI: &RI) &&
7261 !ImpDef)
7262 Copy.addReg(RegNo: AMDGPU::EXEC, Flags: RegState::Implicit);
7263}
7264
7265// Emit the actual waterfall loop, executing the wrapped instruction for each
7266// unique value of \p ScalarOps across all lanes. In the best case we execute 1
7267// iteration, in the worst case we execute 64 (once per lane).
7268static void emitLoadScalarOpsFromVGPRLoop(
7269 const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &PredBB,
7270 MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
7271 ArrayRef<MachineOperand *> ScalarOps, ArrayRef<Register> PhySGPRs = {}) {
7272 MachineFunction &MF = *LoopBB.getParent();
7273 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
7274 const SIRegisterInfo *TRI = ST.getRegisterInfo();
7275 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
7276 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
7277
7278 // Emit v_cmpx_eq and s_andn2_wrexec when both instructions are
7279 // available. Otherwise, use the previous pattern of v_cmp_eq,
7280 // s_and_saveexec, and s_xor.
7281 bool UseNewExecInstructions =
7282 ST.hasNoSdstCMPX() && TII.pseudoToMCOpcode(Opcode: LMC.AndN2WrExecOpc) != -1;
7283
7284 MachineBasicBlock::iterator I = LoopBB.begin();
7285 Register CondReg;
7286
7287 Register PhiExec;
7288 Register NewExec;
7289
7290 if (UseNewExecInstructions) {
7291 PhiExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7292 NewExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7293 Register InitExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7294 BuildMI(BB&: PredBB, I: PredBB.end(), MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: InitExec)
7295 .addReg(RegNo: LMC.ExecReg);
7296
7297 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::PHI), DestReg: PhiExec)
7298 .addReg(RegNo: InitExec)
7299 .addMBB(MBB: &PredBB)
7300 .addReg(RegNo: NewExec)
7301 .addMBB(MBB: &BodyBB);
7302 }
7303
7304 // Placement of v_cmpx instructions (when index is longer than 64 bit)
7305 // involves a trade-off between register pressure and latency:
7306 // (a) Defering all v_cmpx after all v_readfirstlane may increase
7307 // register pressure because arguments and results of all
7308 // v_readfirstlane instructions must stay live until deferred v_cmpx use them.
7309 // (b) Interleaving v_cmpx with v_readfirstlanes may reduce live ranges and
7310 // increase latency by placing v_readfirstlane instructions
7311 // immediately before v_cmpx instruction that directly depend on it.
7312 ///
7313 // Emitting interleaved v_cmpx and v_readfirstlane requires
7314 // block splitting because v_cmpx changes EXEC mask and therefore for safety
7315 // v_cmpx needs to be treated as terminator until after register allocation
7316 // (spill placement) and instruction reordering.
7317 //
7318 // Current implementation defers v_cmpx and leaves other instruction
7319 // scheduling decisions to later passes, where register pressure is known or
7320 // easier to approximate.
7321 // Non-terminators (V_READFIRSTLANE and REG_SEQUENCE) are inserted before I;
7322 // v_cmpx instructions are inserted at the end of LoopBB.
7323 // After the first v_cmpx is emitted, I is updated to point to it
7324 // so subsequent non-terminators are inserted before all v_cmpx instructions.
7325 for (auto [Idx, ScalarOp] : enumerate(First&: ScalarOps)) {
7326 unsigned RegSize = TRI->getRegSizeInBits(Reg: ScalarOp->getReg(), MRI);
7327 unsigned NumSubRegs = RegSize / 32;
7328 Register VScalarOp = ScalarOp->getReg();
7329
7330 const TargetRegisterClass *RFLSrcRC =
7331 TII.getRegClass(MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), OpNum: 1);
7332
7333 if (NumSubRegs == 1) {
7334 const TargetRegisterClass *VScalarOpRC = MRI.getRegClass(Reg: VScalarOp);
7335 if (const TargetRegisterClass *Common =
7336 TRI->getCommonSubClass(A: VScalarOpRC, B: RFLSrcRC);
7337 Common != VScalarOpRC) {
7338 Register VRReg = MRI.createVirtualRegister(RegClass: Common);
7339 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VRReg).addReg(RegNo: VScalarOp);
7340 VScalarOp = VRReg;
7341 }
7342 Register CurReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7343
7344 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurReg)
7345 .addReg(RegNo: VScalarOp);
7346
7347 if (UseNewExecInstructions) {
7348 auto CmpxMI = BuildMI(BB&: LoopBB, I: LoopBB.end(), MIMD: DL,
7349 MCID: TII.get(Opcode: AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term))
7350 .addReg(RegNo: CurReg)
7351 .addReg(RegNo: VScalarOp);
7352 if (I == LoopBB.end())
7353 I = CmpxMI.getInstr()->getIterator();
7354 } else {
7355 Register NewCondReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7356
7357 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: NewCondReg)
7358 .addReg(RegNo: CurReg)
7359 .addReg(RegNo: VScalarOp);
7360
7361 // Combine the comparison results with AND.
7362 if (!CondReg) { // First.
7363 CondReg = NewCondReg;
7364 } else { // If not the first, we create an AND.
7365 Register AndReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7366 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndOpc), DestReg: AndReg)
7367 .addReg(RegNo: CondReg)
7368 .addReg(RegNo: NewCondReg);
7369 CondReg = AndReg;
7370 }
7371 }
7372
7373 // Update ScalarOp operand to use the SGPR ScalarOp.
7374 if (PhySGPRs.empty() || !PhySGPRs[Idx].isValid())
7375 ScalarOp->setReg(CurReg);
7376 else {
7377 // Insert into the same block of use
7378 BuildMI(BB&: *ScalarOp->getParent()->getParent(), I: ScalarOp->getParent(), MIMD: DL,
7379 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: PhySGPRs[Idx])
7380 .addReg(RegNo: CurReg);
7381 ScalarOp->setReg(PhySGPRs[Idx]);
7382 }
7383 ScalarOp->setIsKill();
7384 } else {
7385 SmallVector<Register, 8> ReadlanePieces;
7386 RegState VScalarOpUndef = getUndefRegState(B: ScalarOp->isUndef());
7387 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 &&
7388 "Unhandled register size");
7389
7390 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) {
7391 Register CurRegLo =
7392 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7393 Register CurRegHi =
7394 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7395
7396 // Read the next variant <- also loop target.
7397 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurRegLo)
7398 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg: TRI->getSubRegFromChannel(Channel: Idx));
7399
7400 // Read the next variant <- also loop target.
7401 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurRegHi)
7402 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef,
7403 SubReg: TRI->getSubRegFromChannel(Channel: Idx + 1));
7404
7405 ReadlanePieces.push_back(Elt: CurRegLo);
7406 ReadlanePieces.push_back(Elt: CurRegHi);
7407
7408 // Comparison is to be done as 64-bit.
7409 Register CurReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_64RegClass);
7410 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: CurReg)
7411 .addReg(RegNo: CurRegLo)
7412 .addImm(Val: AMDGPU::sub0)
7413 .addReg(RegNo: CurRegHi)
7414 .addImm(Val: AMDGPU::sub1);
7415
7416 unsigned SubReg =
7417 NumSubRegs <= 2 ? 0 : TRI->getSubRegFromChannel(Channel: Idx, NumRegs: 2);
7418
7419 if (UseNewExecInstructions) {
7420 auto CmpxMI = BuildMI(BB&: LoopBB, I: LoopBB.end(), MIMD: DL,
7421 MCID: TII.get(Opcode: AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term))
7422 .addReg(RegNo: CurReg)
7423 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg);
7424 if (I == LoopBB.end())
7425 I = CmpxMI.getInstr()->getIterator();
7426 } else {
7427 Register NewCondReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7428 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U64_e64), DestReg: NewCondReg)
7429 .addReg(RegNo: CurReg)
7430 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg);
7431
7432 // Combine the comparison results with AND.
7433 if (!CondReg) { // First.
7434 CondReg = NewCondReg;
7435 } else { // If not the first, we create an AND.
7436 Register AndReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7437 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndOpc), DestReg: AndReg)
7438 .addReg(RegNo: CondReg)
7439 .addReg(RegNo: NewCondReg);
7440 CondReg = AndReg;
7441 }
7442 }
7443 } // End for loop.
7444
7445 const auto *SScalarOpRC =
7446 TRI->getEquivalentSGPRClass(VRC: MRI.getRegClass(Reg: VScalarOp));
7447 Register SScalarOp = MRI.createVirtualRegister(RegClass: SScalarOpRC);
7448
7449 // Build scalar ScalarOp.
7450 auto Merge =
7451 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: SScalarOp);
7452 unsigned Channel = 0;
7453 for (Register Piece : ReadlanePieces) {
7454 Merge.addReg(RegNo: Piece).addImm(Val: TRI->getSubRegFromChannel(Channel: Channel++));
7455 }
7456
7457 // Update ScalarOp operand to use the SGPR ScalarOp.
7458 if (PhySGPRs.empty() || !PhySGPRs[Idx].isValid())
7459 ScalarOp->setReg(SScalarOp);
7460 else {
7461 BuildMI(BB&: *ScalarOp->getParent()->getParent(), I: ScalarOp->getParent(), MIMD: DL,
7462 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: PhySGPRs[Idx])
7463 .addReg(RegNo: SScalarOp);
7464 ScalarOp->setReg(PhySGPRs[Idx]);
7465 }
7466 ScalarOp->setIsKill();
7467 }
7468 }
7469
7470 // Instructions AndSaveExecOpc and AndN2WrExecOpc that modify EXEC mask
7471 // should have isTerminator=1 but terminators that define
7472 // virtual registers are not supported.
7473 Register SaveExec;
7474 if (!UseNewExecInstructions) {
7475 SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7476 MRI.setSimpleHint(VReg: SaveExec, PrefReg: CondReg);
7477
7478 // Update EXEC to matching lanes, saving original to SaveExec.
7479 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndSaveExecOpc), DestReg: SaveExec)
7480 .addReg(RegNo: CondReg, Flags: RegState::Kill);
7481 }
7482
7483 // The original instruction is here; we insert the terminators after it.
7484 I = BodyBB.end();
7485
7486 if (UseNewExecInstructions) {
7487 MRI.setSimpleHint(VReg: NewExec, PrefReg: PhiExec);
7488 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndN2WrExecOpc), DestReg: NewExec)
7489 .addReg(RegNo: PhiExec);
7490 } else {
7491 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
7492 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.XorTermOpc), DestReg: LMC.ExecReg)
7493 .addReg(RegNo: LMC.ExecReg)
7494 .addReg(RegNo: SaveExec);
7495 }
7496
7497 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::SI_WATERFALL_LOOP)).addMBB(MBB: &LoopBB);
7498}
7499
7500// Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register
7501// with SGPRs by iterating over all unique values across all lanes.
7502// Returns the loop basic block that now contains \p MI.
7503static MachineBasicBlock *
7504generateWaterFallLoop(const SIInstrInfo &TII, MachineInstr &MI,
7505 ArrayRef<MachineOperand *> ScalarOps,
7506 MachineDominatorTree *MDT,
7507 MachineBasicBlock::iterator Begin = nullptr,
7508 MachineBasicBlock::iterator End = nullptr,
7509 ArrayRef<Register> PhySGPRs = {}) {
7510 assert((PhySGPRs.empty() || PhySGPRs.size() == ScalarOps.size()) &&
7511 "Physical SGPRs must be empty or match the number of scalar operands");
7512 MachineBasicBlock &MBB = *MI.getParent();
7513 MachineFunction &MF = *MBB.getParent();
7514 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
7515 const SIRegisterInfo *TRI = ST.getRegisterInfo();
7516 MachineRegisterInfo &MRI = MF.getRegInfo();
7517 if (!Begin.isValid())
7518 Begin = &MI;
7519 if (!End.isValid()) {
7520 End = &MI;
7521 ++End;
7522 }
7523 const DebugLoc &DL = MI.getDebugLoc();
7524 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
7525 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
7526
7527 // Save SCC. Waterfall Loop may overwrite SCC.
7528 Register SaveSCCReg;
7529
7530 // FIXME: We should maintain SCC liveness while doing the FixSGPRCopies walk
7531 // rather than unlimited scan everywhere
7532 bool SCCNotDead =
7533 MBB.computeRegisterLiveness(TRI, Reg: AMDGPU::SCC, Before: MI,
7534 Neighborhood: std::numeric_limits<unsigned>::max()) !=
7535 MachineBasicBlock::LQR_Dead;
7536 if (SCCNotDead) {
7537 SaveSCCReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7538 BuildMI(BB&: MBB, I: Begin, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: SaveSCCReg)
7539 .addImm(Val: 1)
7540 .addImm(Val: 0);
7541 }
7542
7543 Register SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7544
7545 // Save the EXEC mask
7546 BuildMI(BB&: MBB, I: Begin, MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: SaveExec).addReg(RegNo: LMC.ExecReg);
7547
7548 // Killed uses in the instruction we are waterfalling around will be
7549 // incorrect due to the added control-flow.
7550 MachineBasicBlock::iterator AfterMI = MI;
7551 ++AfterMI;
7552 for (auto I = Begin; I != AfterMI; I++) {
7553 for (auto &MO : I->all_uses())
7554 MRI.clearKillFlags(Reg: MO.getReg());
7555 }
7556
7557 // To insert the loop we need to split the block. Move everything after this
7558 // point to a new block, and insert a new empty block between the two.
7559 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
7560 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock();
7561 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
7562 MachineFunction::iterator MBBI(MBB);
7563 ++MBBI;
7564
7565 MF.insert(MBBI, MBB: LoopBB);
7566 MF.insert(MBBI, MBB: BodyBB);
7567 MF.insert(MBBI, MBB: RemainderBB);
7568
7569 LoopBB->addSuccessor(Succ: BodyBB);
7570 BodyBB->addSuccessor(Succ: LoopBB);
7571 BodyBB->addSuccessor(Succ: RemainderBB);
7572
7573 // Move Begin to MI to the BodyBB, and the remainder of the block to
7574 // RemainderBB.
7575 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
7576 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: End, To: MBB.end());
7577 BodyBB->splice(Where: BodyBB->begin(), Other: &MBB, From: Begin, To: MBB.end());
7578
7579 MBB.addSuccessor(Succ: LoopBB);
7580
7581 // Update dominators. We know that MBB immediately dominates LoopBB, that
7582 // LoopBB immediately dominates BodyBB, and BodyBB immediately dominates
7583 // RemainderBB. RemainderBB immediately dominates all of the successors
7584 // transferred to it from MBB that MBB used to properly dominate.
7585 if (MDT) {
7586 MDT->addNewBlock(BB: LoopBB, DomBB: &MBB);
7587 MDT->addNewBlock(BB: BodyBB, DomBB: LoopBB);
7588 MDT->addNewBlock(BB: RemainderBB, DomBB: BodyBB);
7589 for (auto &Succ : RemainderBB->successors()) {
7590 if (MDT->properlyDominates(A: &MBB, B: Succ)) {
7591 MDT->changeImmediateDominator(BB: Succ, NewBB: RemainderBB);
7592 }
7593 }
7594 }
7595
7596 emitLoadScalarOpsFromVGPRLoop(TII, MRI, PredBB&: MBB, LoopBB&: *LoopBB, BodyBB&: *BodyBB, DL, ScalarOps,
7597 PhySGPRs);
7598
7599 MachineBasicBlock::iterator First = RemainderBB->begin();
7600 // Restore SCC
7601 if (SCCNotDead) {
7602 BuildMI(BB&: *RemainderBB, I: First, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CMP_LG_U32))
7603 .addReg(RegNo: SaveSCCReg, Flags: RegState::Kill)
7604 .addImm(Val: 0);
7605 }
7606
7607 // Restore the EXEC mask
7608 BuildMI(BB&: *RemainderBB, I: First, MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
7609 .addReg(RegNo: SaveExec);
7610 return BodyBB;
7611}
7612
7613// Extract pointer from Rsrc and return a zero-value Rsrc replacement.
7614static std::tuple<unsigned, unsigned>
7615extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) {
7616 MachineBasicBlock &MBB = *MI.getParent();
7617 MachineFunction &MF = *MBB.getParent();
7618 MachineRegisterInfo &MRI = MF.getRegInfo();
7619
7620 // Extract the ptr from the resource descriptor.
7621 unsigned RsrcPtr =
7622 TII.buildExtractSubReg(MI, MRI, SuperReg: Rsrc, SuperRC: &AMDGPU::VReg_128RegClass,
7623 SubIdx: AMDGPU::sub0_sub1, SubRC: &AMDGPU::VReg_64RegClass);
7624
7625 // Create an empty resource descriptor
7626 Register Zero64 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
7627 Register SRsrcFormatLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
7628 Register SRsrcFormatHi = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
7629 Register NewSRsrc = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_128RegClass);
7630 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
7631
7632 // Zero64 = 0
7633 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B64), DestReg: Zero64)
7634 .addImm(Val: 0);
7635
7636 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
7637 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: SRsrcFormatLo)
7638 .addImm(Val: Lo_32(Value: RsrcDataFormat));
7639
7640 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
7641 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: SRsrcFormatHi)
7642 .addImm(Val: Hi_32(Value: RsrcDataFormat));
7643
7644 // NewSRsrc = {Zero64, SRsrcFormat}
7645 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewSRsrc)
7646 .addReg(RegNo: Zero64)
7647 .addImm(Val: AMDGPU::sub0_sub1)
7648 .addReg(RegNo: SRsrcFormatLo)
7649 .addImm(Val: AMDGPU::sub2)
7650 .addReg(RegNo: SRsrcFormatHi)
7651 .addImm(Val: AMDGPU::sub3);
7652
7653 return std::tuple(RsrcPtr, NewSRsrc);
7654}
7655
7656MachineBasicBlock *
7657SIInstrInfo::legalizeOperands(MachineInstr &MI,
7658 MachineDominatorTree *MDT) const {
7659 MachineFunction &MF = *MI.getMF();
7660 MachineRegisterInfo &MRI = MF.getRegInfo();
7661 MachineBasicBlock *CreatedBB = nullptr;
7662
7663 // Legalize True16
7664 if (ST.useRealTrue16Insts())
7665 legalizeOperandsVALUt16(Inst&: MI, MRI);
7666
7667 // Legalize VOP2
7668 if (isVOP2(MI) || isVOPC(MI)) {
7669 legalizeOperandsVOP2(MRI, MI);
7670 return CreatedBB;
7671 }
7672
7673 // Legalize VOP3
7674 if (isVOP3(MI)) {
7675 legalizeOperandsVOP3(MRI, MI);
7676 return CreatedBB;
7677 }
7678
7679 // Legalize SMRD
7680 if (isSMRD(MI)) {
7681 legalizeOperandsSMRD(MRI, MI);
7682 return CreatedBB;
7683 }
7684
7685 // Legalize FLAT
7686 if (isFLAT(MI)) {
7687 legalizeOperandsFLAT(MRI, MI);
7688 return CreatedBB;
7689 }
7690
7691 // Legalize PHI
7692 // The register class of the operands must be the same type as the register
7693 // class of the output.
7694 if (MI.getOpcode() == AMDGPU::PHI) {
7695 const TargetRegisterClass *VRC = getOpRegClass(MI, OpNo: 0);
7696 assert(!RI.isSGPRClass(VRC));
7697
7698 // Update all the operands so they have the same type.
7699 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
7700 MachineOperand &Op = MI.getOperand(i: I);
7701 if (!Op.isReg() || !Op.getReg().isVirtual())
7702 continue;
7703
7704 // MI is a PHI instruction.
7705 MachineBasicBlock *InsertBB = MI.getOperand(i: I + 1).getMBB();
7706 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
7707
7708 // Avoid creating no-op copies with the same src and dst reg class. These
7709 // confuse some of the machine passes.
7710 legalizeGenericOperand(InsertMBB&: *InsertBB, I: Insert, DstRC: VRC, Op, MRI, DL: MI.getDebugLoc());
7711 }
7712 }
7713
7714 // REG_SEQUENCE doesn't really require operand legalization, but if one has a
7715 // VGPR dest type and SGPR sources, insert copies so all operands are
7716 // VGPRs. This seems to help operand folding / the register coalescer.
7717 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
7718 MachineBasicBlock *MBB = MI.getParent();
7719 const TargetRegisterClass *DstRC = getOpRegClass(MI, OpNo: 0);
7720 if (RI.hasVGPRs(RC: DstRC)) {
7721 // Update all the operands so they are VGPR register classes. These may
7722 // not be the same register class because REG_SEQUENCE supports mixing
7723 // subregister index types e.g. sub0_sub1 + sub2 + sub3
7724 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
7725 MachineOperand &Op = MI.getOperand(i: I);
7726 if (!Op.isReg() || !Op.getReg().isVirtual())
7727 continue;
7728
7729 const TargetRegisterClass *OpRC = MRI.getRegClass(Reg: Op.getReg());
7730 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(SRC: OpRC);
7731 if (VRC == OpRC)
7732 continue;
7733
7734 legalizeGenericOperand(InsertMBB&: *MBB, I: MI, DstRC: VRC, Op, MRI, DL: MI.getDebugLoc());
7735 Op.setIsKill();
7736 }
7737 }
7738
7739 return CreatedBB;
7740 }
7741
7742 // Legalize INSERT_SUBREG
7743 // src0 must have the same register class as dst
7744 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
7745 Register Dst = MI.getOperand(i: 0).getReg();
7746 Register Src0 = MI.getOperand(i: 1).getReg();
7747 const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: Dst);
7748 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0);
7749 if (DstRC != Src0RC) {
7750 MachineBasicBlock *MBB = MI.getParent();
7751 MachineOperand &Op = MI.getOperand(i: 1);
7752 legalizeGenericOperand(InsertMBB&: *MBB, I: MI, DstRC, Op, MRI, DL: MI.getDebugLoc());
7753 }
7754 return CreatedBB;
7755 }
7756
7757 // Legalize SI_INIT_M0
7758 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
7759 MachineOperand &Src = MI.getOperand(i: 0);
7760 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
7761 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
7762 return CreatedBB;
7763 }
7764
7765 // Legalize S_BITREPLICATE, S_QUADMASK and S_WQM
7766 if (MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 ||
7767 MI.getOpcode() == AMDGPU::S_QUADMASK_B32 ||
7768 MI.getOpcode() == AMDGPU::S_QUADMASK_B64 ||
7769 MI.getOpcode() == AMDGPU::S_WQM_B32 ||
7770 MI.getOpcode() == AMDGPU::S_WQM_B64 ||
7771 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U32 ||
7772 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U64) {
7773 MachineOperand &Src = MI.getOperand(i: 1);
7774 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
7775 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
7776 return CreatedBB;
7777 }
7778
7779 // Legalize MIMG/VIMAGE/VSAMPLE and MUBUF/MTBUF for shaders.
7780 //
7781 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
7782 // scratch memory access. In both cases, the legalization never involves
7783 // conversion to the addr64 form.
7784 if (isImage(MI) || (AMDGPU::isGraphics(CC: MF.getFunction().getCallingConv()) &&
7785 (isMUBUF(MI) || isMTBUF(MI)))) {
7786 AMDGPU::OpName RSrcOpName = (isVIMAGE(MI) || isVSAMPLE(MI))
7787 ? AMDGPU::OpName::rsrc
7788 : AMDGPU::OpName::srsrc;
7789 MachineOperand *SRsrc = getNamedOperand(MI, OperandName: RSrcOpName);
7790 if (SRsrc && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SRsrc->getReg())))
7791 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {SRsrc}, MDT);
7792
7793 AMDGPU::OpName SampOpName =
7794 isMIMG(MI) ? AMDGPU::OpName::ssamp : AMDGPU::OpName::samp;
7795 MachineOperand *SSamp = getNamedOperand(MI, OperandName: SampOpName);
7796 if (SSamp && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SSamp->getReg())))
7797 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {SSamp}, MDT);
7798
7799 return CreatedBB;
7800 }
7801
7802 // Legalize SI_CALL
7803 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) {
7804 MachineOperand *Dest = &MI.getOperand(i: 0);
7805 if (!RI.isSGPRClass(RC: MRI.getRegClass(Reg: Dest->getReg()))) {
7806 createWaterFallForSiCall(MI: &MI, MDT, ScalarOps: {Dest});
7807 }
7808 }
7809
7810 // Legalize s_sleep_var.
7811 if (MI.getOpcode() == AMDGPU::S_SLEEP_VAR) {
7812 const DebugLoc &DL = MI.getDebugLoc();
7813 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7814 int Src0Idx =
7815 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
7816 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
7817 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7818 .add(MO: Src0);
7819 Src0.ChangeToRegister(Reg, isDef: false);
7820 return nullptr;
7821 }
7822
7823 // Legalize TENSOR_LOAD_TO_LDS_d2/_d4, TENSOR_STORE_FROM_LDS_d2/_d4. All their
7824 // operands are scalar.
7825 if (MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_d2 ||
7826 MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_d4 ||
7827 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_d2 ||
7828 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_d4) {
7829 for (MachineOperand &Src : MI.explicit_operands()) {
7830 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
7831 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
7832 }
7833 return CreatedBB;
7834 }
7835
7836 // Legalize MUBUF instructions.
7837 bool isSoffsetLegal = true;
7838 int SoffsetIdx =
7839 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::soffset);
7840 if (SoffsetIdx != -1) {
7841 MachineOperand *Soffset = &MI.getOperand(i: SoffsetIdx);
7842 if (Soffset->isReg() && Soffset->getReg().isVirtual() &&
7843 !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Soffset->getReg()))) {
7844 isSoffsetLegal = false;
7845 }
7846 }
7847
7848 bool isRsrcLegal = true;
7849 int RsrcIdx =
7850 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::srsrc);
7851 if (RsrcIdx != -1) {
7852 MachineOperand *Rsrc = &MI.getOperand(i: RsrcIdx);
7853 if (Rsrc->isReg() && !RI.isSGPRReg(MRI, Reg: Rsrc->getReg()))
7854 isRsrcLegal = false;
7855 }
7856
7857 // The operands are legal.
7858 if (isRsrcLegal && isSoffsetLegal)
7859 return CreatedBB;
7860
7861 if (!isRsrcLegal) {
7862 // Legalize a VGPR Rsrc
7863 //
7864 // If the instruction is _ADDR64, we can avoid a waterfall by extracting
7865 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
7866 // a zero-value SRsrc.
7867 //
7868 // If the instruction is _OFFSET (both idxen and offen disabled), and we
7869 // support ADDR64 instructions, we can convert to ADDR64 and do the same as
7870 // above.
7871 //
7872 // Otherwise we are on non-ADDR64 hardware, and/or we have
7873 // idxen/offen/bothen and we fall back to a waterfall loop.
7874
7875 MachineOperand *Rsrc = &MI.getOperand(i: RsrcIdx);
7876 MachineBasicBlock &MBB = *MI.getParent();
7877
7878 MachineOperand *VAddr = getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
7879 if (VAddr && AMDGPU::getIfAddr64Inst(Opcode: MI.getOpcode()) != -1) {
7880 // This is already an ADDR64 instruction so we need to add the pointer
7881 // extracted from the resource descriptor to the current value of VAddr.
7882 Register NewVAddrLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7883 Register NewVAddrHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7884 Register NewVAddr = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
7885
7886 const auto *BoolXExecRC = RI.getWaveMaskRegClass();
7887 Register CondReg0 = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7888 Register CondReg1 = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7889
7890 unsigned RsrcPtr, NewSRsrc;
7891 std::tie(args&: RsrcPtr, args&: NewSRsrc) = extractRsrcPtr(TII: *this, MI, Rsrc&: *Rsrc);
7892
7893 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
7894 const DebugLoc &DL = MI.getDebugLoc();
7895 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg: NewVAddrLo)
7896 .addDef(RegNo: CondReg0)
7897 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub0)
7898 .addReg(RegNo: VAddr->getReg(), Flags: {}, SubReg: AMDGPU::sub0)
7899 .addImm(Val: 0);
7900
7901 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
7902 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADDC_U32_e64), DestReg: NewVAddrHi)
7903 .addDef(RegNo: CondReg1, Flags: RegState::Dead)
7904 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub1)
7905 .addReg(RegNo: VAddr->getReg(), Flags: {}, SubReg: AMDGPU::sub1)
7906 .addReg(RegNo: CondReg0, Flags: RegState::Kill)
7907 .addImm(Val: 0);
7908
7909 // NewVaddr = {NewVaddrHi, NewVaddrLo}
7910 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewVAddr)
7911 .addReg(RegNo: NewVAddrLo)
7912 .addImm(Val: AMDGPU::sub0)
7913 .addReg(RegNo: NewVAddrHi)
7914 .addImm(Val: AMDGPU::sub1);
7915
7916 VAddr->setReg(NewVAddr);
7917 Rsrc->setReg(NewSRsrc);
7918 } else if (!VAddr && ST.hasAddr64()) {
7919 // This instructions is the _OFFSET variant, so we need to convert it to
7920 // ADDR64.
7921 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS &&
7922 "FIXME: Need to emit flat atomics here");
7923
7924 unsigned RsrcPtr, NewSRsrc;
7925 std::tie(args&: RsrcPtr, args&: NewSRsrc) = extractRsrcPtr(TII: *this, MI, Rsrc&: *Rsrc);
7926
7927 Register NewVAddr = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
7928 MachineOperand *VData = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata);
7929 MachineOperand *Offset = getNamedOperand(MI, OperandName: AMDGPU::OpName::offset);
7930 MachineOperand *SOffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
7931 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(Opcode: MI.getOpcode());
7932
7933 // Atomics with return have an additional tied operand and are
7934 // missing some of the special bits.
7935 MachineOperand *VDataIn = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata_in);
7936 MachineInstr *Addr64;
7937
7938 if (!VDataIn) {
7939 // Regular buffer load / store.
7940 MachineInstrBuilder MIB =
7941 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Addr64Opcode))
7942 .add(MO: *VData)
7943 .addReg(RegNo: NewVAddr)
7944 .addReg(RegNo: NewSRsrc)
7945 .add(MO: *SOffset)
7946 .add(MO: *Offset);
7947
7948 if (const MachineOperand *CPol =
7949 getNamedOperand(MI, OperandName: AMDGPU::OpName::cpol)) {
7950 MIB.addImm(Val: CPol->getImm());
7951 }
7952
7953 if (const MachineOperand *TFE =
7954 getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe)) {
7955 MIB.addImm(Val: TFE->getImm());
7956 }
7957
7958 MIB.addImm(Val: getNamedImmOperand(MI, OperandName: AMDGPU::OpName::swz));
7959
7960 MIB.cloneMemRefs(OtherMI: MI);
7961 Addr64 = MIB;
7962 } else {
7963 // Atomics with return.
7964 Addr64 = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Addr64Opcode))
7965 .add(MO: *VData)
7966 .add(MO: *VDataIn)
7967 .addReg(RegNo: NewVAddr)
7968 .addReg(RegNo: NewSRsrc)
7969 .add(MO: *SOffset)
7970 .add(MO: *Offset)
7971 .addImm(Val: getNamedImmOperand(MI, OperandName: AMDGPU::OpName::cpol))
7972 .cloneMemRefs(OtherMI: MI);
7973 }
7974
7975 MI.removeFromParent();
7976
7977 // NewVaddr = {NewVaddrHi, NewVaddrLo}
7978 BuildMI(BB&: MBB, I: Addr64, MIMD: Addr64->getDebugLoc(), MCID: get(Opcode: AMDGPU::REG_SEQUENCE),
7979 DestReg: NewVAddr)
7980 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub0)
7981 .addImm(Val: AMDGPU::sub0)
7982 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub1)
7983 .addImm(Val: AMDGPU::sub1);
7984 } else {
7985 // Legalize a VGPR Rsrc and soffset together.
7986 if (!isSoffsetLegal) {
7987 MachineOperand *Soffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
7988 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Rsrc, Soffset}, MDT);
7989 return CreatedBB;
7990 }
7991 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Rsrc}, MDT);
7992 return CreatedBB;
7993 }
7994 }
7995
7996 // Legalize a VGPR soffset.
7997 if (!isSoffsetLegal) {
7998 MachineOperand *Soffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
7999 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Soffset}, MDT);
8000 return CreatedBB;
8001 }
8002 return CreatedBB;
8003}
8004
8005void SIInstrWorklist::insert(MachineInstr *MI) {
8006 if (InSet.insert(Ptr: MI).second)
8007 InstrList.push_back(Elt: MI);
8008 // Add MBUF instructiosn to deferred list.
8009 int RsrcIdx =
8010 AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(), Name: AMDGPU::OpName::srsrc);
8011 if (RsrcIdx != -1) {
8012 DeferredList.insert(X: MI);
8013 }
8014}
8015
8016bool SIInstrWorklist::isDeferred(MachineInstr *MI) {
8017 return DeferredList.contains(key: MI);
8018}
8019
8020// Legalize size mismatches between 16bit and 32bit registers in v2s copy
8021// lowering (change sgpr to vgpr).
8022// This is mainly caused by 16bit SALU and 16bit VALU using reg with different
8023// size. Need to legalize the size of the operands during the vgpr lowering
8024// chain. This can be removed after we have sgpr16 in place
8025void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &MI, unsigned OpIdx,
8026 MachineRegisterInfo &MRI) const {
8027 if (!ST.useRealTrue16Insts())
8028 return;
8029
8030 unsigned Opcode = MI.getOpcode();
8031 MachineBasicBlock *MBB = MI.getParent();
8032 // Legalize operands and check for size mismatch
8033 if (OpIdx >= MI.getNumExplicitOperands() ||
8034 OpIdx >= get(Opcode).getNumOperands() ||
8035 get(Opcode).operands()[OpIdx].RegClass == -1)
8036 return;
8037
8038 MachineOperand &Op = MI.getOperand(i: OpIdx);
8039 if (!Op.isReg() || !Op.getReg().isVirtual() || Op.isDef())
8040 return;
8041
8042 const TargetRegisterClass *CurrRC = MRI.getRegClass(Reg: Op.getReg());
8043 if (!RI.isVGPRClass(RC: CurrRC))
8044 return;
8045
8046 int16_t RCID = getOpRegClassID(OpInfo: get(Opcode).operands()[OpIdx]);
8047 const TargetRegisterClass *ExpectedRC = RI.getRegClass(i: RCID);
8048 if (RI.getMatchingSuperRegClass(A: CurrRC, B: ExpectedRC, Idx: AMDGPU::lo16)) {
8049 // Default to the lo16 only if the subregister is not specified.
8050 if (Op.getSubReg() == AMDGPU::NoSubRegister)
8051 Op.setSubReg(AMDGPU::lo16);
8052 return;
8053 }
8054
8055 const TargetRegisterClass *CurrSRC =
8056 RI.getSubRegisterClass(CurrRC, Op.getSubReg());
8057 if (RI.getMatchingSuperRegClass(A: ExpectedRC, B: CurrSRC, Idx: AMDGPU::lo16)) {
8058 const DebugLoc &DL = MI.getDebugLoc();
8059 Register NewDstReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8060 Register Undef = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
8061 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
8062 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewDstReg)
8063 .addReg(RegNo: Op.getReg(), Flags: {}, SubReg: Op.getSubReg())
8064 .addImm(Val: AMDGPU::lo16)
8065 .addReg(RegNo: Undef)
8066 .addImm(Val: AMDGPU::hi16);
8067 Op.setReg(NewDstReg);
8068 Op.setSubReg(AMDGPU::NoSubRegister);
8069 }
8070}
8071void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &MI,
8072 MachineRegisterInfo &MRI) const {
8073 for (unsigned OpIdx = 0; OpIdx < MI.getNumExplicitOperands(); OpIdx++)
8074 legalizeOperandsVALUt16(MI, OpIdx, MRI);
8075}
8076
8077void SIInstrInfo::createWaterFallForSiCall(MachineInstr *MI,
8078 MachineDominatorTree *MDT,
8079 ArrayRef<MachineOperand *> ScalarOps,
8080 ArrayRef<Register> PhySGPRs) const {
8081 assert(MI->getOpcode() == AMDGPU::SI_CALL_ISEL &&
8082 "This only handle waterfall for SI_CALL_ISEL");
8083 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and
8084 // following copies, we also need to move copies from and to physical
8085 // registers into the loop block.
8086 // Also move the copies to physical registers into the loop block
8087 MachineBasicBlock &MBB = *MI->getParent();
8088 MachineBasicBlock::iterator Start(MI);
8089 while (Start->getOpcode() != AMDGPU::ADJCALLSTACKUP)
8090 --Start;
8091 MachineBasicBlock::iterator End(MI);
8092 while (End->getOpcode() != AMDGPU::ADJCALLSTACKDOWN)
8093 ++End;
8094
8095 // Also include following copies of the return value
8096 ++End;
8097 while (End != MBB.end() && End->isCopy() &&
8098 MI->definesRegister(Reg: End->getOperand(i: 1).getReg(), TRI: &RI))
8099 ++End;
8100
8101 generateWaterFallLoop(TII: *this, MI&: *MI, ScalarOps, MDT, Begin: Start, End, PhySGPRs);
8102}
8103
8104void SIInstrInfo::moveToVALU(SIInstrWorklist &Worklist,
8105 MachineDominatorTree *MDT) const {
8106 DenseMap<MachineInstr *, V2PhysSCopyInfo> WaterFalls;
8107 DenseMap<MachineInstr *, bool> V2SPhyCopiesToErase;
8108 while (!Worklist.empty()) {
8109 MachineInstr &Inst = *Worklist.top();
8110 Worklist.erase_top();
8111 // Skip MachineInstr in the deferred list.
8112 if (Worklist.isDeferred(MI: &Inst))
8113 continue;
8114 moveToVALUImpl(Worklist, MDT, Inst, WaterFalls, V2SPhyCopiesToErase);
8115 }
8116
8117 // Deferred list of instructions will be processed once
8118 // all the MachineInstr in the worklist are done.
8119 for (MachineInstr *Inst : Worklist.getDeferredList()) {
8120 moveToVALUImpl(Worklist, MDT, Inst&: *Inst, WaterFalls, V2SPhyCopiesToErase);
8121 assert(Worklist.empty() &&
8122 "Deferred MachineInstr are not supposed to re-populate worklist");
8123 }
8124
8125 for (std::pair<MachineInstr *, V2PhysSCopyInfo> &Entry : WaterFalls) {
8126 if (Entry.first->getOpcode() == AMDGPU::SI_CALL_ISEL)
8127 createWaterFallForSiCall(MI: Entry.first, MDT, ScalarOps: Entry.second.MOs,
8128 PhySGPRs: Entry.second.SGPRs);
8129 }
8130
8131 for (std::pair<MachineInstr *, bool> Entry : V2SPhyCopiesToErase)
8132 if (Entry.second)
8133 Entry.first->eraseFromParent();
8134}
8135void SIInstrInfo::createReadFirstLaneFromCopyToPhysReg(
8136 MachineRegisterInfo &MRI, Register DstReg, MachineInstr &Inst) const {
8137 // If it's a copy of a VGPR to a physical SGPR, insert a V_READFIRSTLANE and
8138 // hope for the best.
8139 const TargetRegisterClass *DstRC = RI.getRegClassForReg(MRI, Reg: DstReg);
8140 ArrayRef<int16_t> SubRegIndices = RI.getRegSplitParts(RC: DstRC, EltSize: 4);
8141 if (SubRegIndices.size() <= 1) {
8142 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
8143 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8144 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: NewDst)
8145 .add(MO: Inst.getOperand(i: 1));
8146 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
8147 DestReg: DstReg)
8148 .addReg(RegNo: NewDst);
8149 } else {
8150 SmallVector<Register, 8> DstRegs;
8151 for (int16_t Indice : SubRegIndices) {
8152 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
8153 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8154 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: NewDst)
8155 .addReg(RegNo: Inst.getOperand(i: 1).getReg(), Flags: {}, SubReg: Indice);
8156
8157 DstRegs.push_back(Elt: NewDst);
8158 }
8159 MachineInstrBuilder MIB =
8160 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8161 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
8162 for (unsigned i = 0; i < SubRegIndices.size(); ++i) {
8163 MIB.addReg(RegNo: DstRegs[i]);
8164 MIB.addImm(Val: RI.getSubRegFromChannel(Channel: i));
8165 }
8166 }
8167}
8168
8169void SIInstrInfo::handleCopyToPhysHelper(
8170 SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst,
8171 MachineRegisterInfo &MRI,
8172 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
8173 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const {
8174 if (DstReg == AMDGPU::M0) {
8175 createReadFirstLaneFromCopyToPhysReg(MRI, DstReg, Inst);
8176 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
8177 return;
8178 }
8179 Register SrcReg = Inst.getOperand(i: 1).getReg();
8180 MachineBasicBlock::iterator I = Inst.getIterator();
8181 MachineBasicBlock::iterator E = Inst.getParent()->end();
8182 // Only search current block since phyreg's def & use cannot cross
8183 // blocks when MF.NoPhi = false.
8184 while (++I != E) {
8185 // For SI_CALL_ISEL users, replace the phys SGPR with the VGPR source
8186 // and record the operand for later waterfall loop generation.
8187 if (I->getOpcode() == AMDGPU::SI_CALL_ISEL) {
8188 MachineInstr *UseMI = &*I;
8189 for (unsigned i = 0; i < UseMI->getNumOperands(); ++i) {
8190 if (UseMI->getOperand(i).isReg() &&
8191 UseMI->getOperand(i).getReg() == DstReg) {
8192 MachineOperand *MO = &UseMI->getOperand(i);
8193 MO->setReg(SrcReg);
8194 V2PhysSCopyInfo &V2SCopyInfo = WaterFalls[UseMI];
8195 V2SCopyInfo.MOs.push_back(Elt: MO);
8196 V2SCopyInfo.SGPRs.push_back(Elt: DstReg);
8197 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
8198 }
8199 }
8200 } else if (I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG &&
8201 I->getOperand(i: 0).isReg() &&
8202 I->getOperand(i: 0).getReg() == DstReg) {
8203 createReadFirstLaneFromCopyToPhysReg(MRI, DstReg, Inst);
8204 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
8205 } else if (I->readsRegister(Reg: DstReg, TRI: &RI)) {
8206 // COPY cannot be erased if other type of inst uses it.
8207 V2SPhyCopiesToErase[&Inst] = false;
8208 }
8209 if (I->findRegisterDefOperand(Reg: DstReg, TRI: &RI))
8210 break;
8211 }
8212}
8213
8214void SIInstrInfo::moveToVALUImpl(
8215 SIInstrWorklist &Worklist, MachineDominatorTree *MDT, MachineInstr &Inst,
8216 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
8217 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const {
8218
8219 MachineBasicBlock *MBB = Inst.getParent();
8220 if (!MBB)
8221 return;
8222 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
8223 unsigned Opcode = Inst.getOpcode();
8224 unsigned NewOpcode = getVALUOp(MI: Inst);
8225 const DebugLoc &DL = Inst.getDebugLoc();
8226
8227 // Handle some special cases
8228 switch (Opcode) {
8229 default:
8230 break;
8231 case AMDGPU::S_ADD_I32:
8232 case AMDGPU::S_SUB_I32: {
8233 // FIXME: The u32 versions currently selected use the carry.
8234 bool Changed;
8235 MachineBasicBlock *CreatedBBTmp = nullptr;
8236 std::tie(args&: Changed, args&: CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT);
8237 if (Changed)
8238 return;
8239
8240 // Default handling
8241 break;
8242 }
8243
8244 case AMDGPU::S_MUL_U64:
8245 if (ST.hasVMulU64Inst()) {
8246 NewOpcode = AMDGPU::V_MUL_U64_e64;
8247 break;
8248 }
8249 // Split s_mul_u64 in 32-bit vector multiplications.
8250 splitScalarSMulU64(Worklist, Inst, MDT);
8251 Inst.eraseFromParent();
8252 return;
8253
8254 case AMDGPU::S_MUL_U64_U32_PSEUDO:
8255 case AMDGPU::S_MUL_I64_I32_PSEUDO:
8256 // This is a special case of s_mul_u64 where all the operands are either
8257 // zero extended or sign extended.
8258 splitScalarSMulPseudo(Worklist, Inst, MDT);
8259 Inst.eraseFromParent();
8260 return;
8261
8262 case AMDGPU::S_AND_B64:
8263 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_AND_B32, MDT);
8264 Inst.eraseFromParent();
8265 return;
8266
8267 case AMDGPU::S_OR_B64:
8268 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_OR_B32, MDT);
8269 Inst.eraseFromParent();
8270 return;
8271
8272 case AMDGPU::S_XOR_B64:
8273 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_XOR_B32, MDT);
8274 Inst.eraseFromParent();
8275 return;
8276
8277 case AMDGPU::S_NAND_B64:
8278 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_NAND_B32, MDT);
8279 Inst.eraseFromParent();
8280 return;
8281
8282 case AMDGPU::S_NOR_B64:
8283 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_NOR_B32, MDT);
8284 Inst.eraseFromParent();
8285 return;
8286
8287 case AMDGPU::S_XNOR_B64:
8288 if (ST.hasDLInsts())
8289 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_XNOR_B32, MDT);
8290 else
8291 splitScalar64BitXnor(Worklist, Inst, MDT);
8292 Inst.eraseFromParent();
8293 return;
8294
8295 case AMDGPU::S_ANDN2_B64:
8296 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_ANDN2_B32, MDT);
8297 Inst.eraseFromParent();
8298 return;
8299
8300 case AMDGPU::S_ORN2_B64:
8301 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_ORN2_B32, MDT);
8302 Inst.eraseFromParent();
8303 return;
8304
8305 case AMDGPU::S_BREV_B64:
8306 splitScalar64BitUnaryOp(Worklist, Inst, Opcode: AMDGPU::S_BREV_B32, Swap: true);
8307 Inst.eraseFromParent();
8308 return;
8309
8310 case AMDGPU::S_NOT_B64:
8311 splitScalar64BitUnaryOp(Worklist, Inst, Opcode: AMDGPU::S_NOT_B32);
8312 Inst.eraseFromParent();
8313 return;
8314
8315 case AMDGPU::S_BCNT1_I32_B64:
8316 splitScalar64BitBCNT(Worklist, Inst);
8317 Inst.eraseFromParent();
8318 return;
8319
8320 case AMDGPU::S_BFE_I64:
8321 splitScalar64BitBFE(Worklist, Inst);
8322 Inst.eraseFromParent();
8323 return;
8324
8325 case AMDGPU::S_FLBIT_I32_B64:
8326 splitScalar64BitCountOp(Worklist, Inst, Opcode: AMDGPU::V_FFBH_U32_e32);
8327 Inst.eraseFromParent();
8328 return;
8329 case AMDGPU::S_FF1_I32_B64:
8330 splitScalar64BitCountOp(Worklist, Inst, Opcode: AMDGPU::V_FFBL_B32_e32);
8331 Inst.eraseFromParent();
8332 return;
8333
8334 case AMDGPU::S_LSHL_B32:
8335 if (ST.hasOnlyRevVALUShifts()) {
8336 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
8337 swapOperands(Inst);
8338 }
8339 break;
8340 case AMDGPU::S_ASHR_I32:
8341 if (ST.hasOnlyRevVALUShifts()) {
8342 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
8343 swapOperands(Inst);
8344 }
8345 break;
8346 case AMDGPU::S_LSHR_B32:
8347 if (ST.hasOnlyRevVALUShifts()) {
8348 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
8349 swapOperands(Inst);
8350 }
8351 break;
8352 case AMDGPU::S_LSHL_B64:
8353 if (ST.hasOnlyRevVALUShifts()) {
8354 NewOpcode = ST.getGeneration() >= AMDGPUSubtarget::GFX12
8355 ? AMDGPU::V_LSHLREV_B64_pseudo_e64
8356 : AMDGPU::V_LSHLREV_B64_e64;
8357 swapOperands(Inst);
8358 }
8359 break;
8360 case AMDGPU::S_ASHR_I64:
8361 if (ST.hasOnlyRevVALUShifts()) {
8362 NewOpcode = AMDGPU::V_ASHRREV_I64_e64;
8363 swapOperands(Inst);
8364 }
8365 break;
8366 case AMDGPU::S_LSHR_B64:
8367 if (ST.hasOnlyRevVALUShifts()) {
8368 NewOpcode = AMDGPU::V_LSHRREV_B64_e64;
8369 swapOperands(Inst);
8370 }
8371 break;
8372
8373 case AMDGPU::S_ABS_I32:
8374 lowerScalarAbs(Worklist, Inst);
8375 Inst.eraseFromParent();
8376 return;
8377
8378 case AMDGPU::S_ABSDIFF_I32:
8379 lowerScalarAbsDiff(Worklist, Inst);
8380 Inst.eraseFromParent();
8381 return;
8382
8383 case AMDGPU::S_CBRANCH_SCC0:
8384 case AMDGPU::S_CBRANCH_SCC1: {
8385 // Clear unused bits of vcc
8386 Register CondReg = Inst.getOperand(i: 1).getReg();
8387 bool IsSCC = CondReg == AMDGPU::SCC;
8388 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
8389 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: LMC.AndOpc), DestReg: LMC.VccReg)
8390 .addReg(RegNo: LMC.ExecReg)
8391 .addReg(RegNo: IsSCC ? LMC.VccReg : CondReg);
8392 Inst.removeOperand(OpNo: 1);
8393 } break;
8394
8395 case AMDGPU::S_BFE_U64:
8396 case AMDGPU::S_BFM_B64:
8397 llvm_unreachable("Moving this op to VALU not implemented");
8398
8399 case AMDGPU::S_PACK_LL_B32_B16:
8400 case AMDGPU::S_PACK_LH_B32_B16:
8401 case AMDGPU::S_PACK_HL_B32_B16:
8402 case AMDGPU::S_PACK_HH_B32_B16:
8403 movePackToVALU(Worklist, MRI, Inst);
8404 Inst.eraseFromParent();
8405 return;
8406
8407 case AMDGPU::S_XNOR_B32:
8408 lowerScalarXnor(Worklist, Inst);
8409 Inst.eraseFromParent();
8410 return;
8411
8412 case AMDGPU::S_NAND_B32:
8413 splitScalarNotBinop(Worklist, Inst, Opcode: AMDGPU::S_AND_B32);
8414 Inst.eraseFromParent();
8415 return;
8416
8417 case AMDGPU::S_NOR_B32:
8418 splitScalarNotBinop(Worklist, Inst, Opcode: AMDGPU::S_OR_B32);
8419 Inst.eraseFromParent();
8420 return;
8421
8422 case AMDGPU::S_ANDN2_B32:
8423 splitScalarBinOpN2(Worklist, Inst, Opcode: AMDGPU::S_AND_B32);
8424 Inst.eraseFromParent();
8425 return;
8426
8427 case AMDGPU::S_ORN2_B32:
8428 splitScalarBinOpN2(Worklist, Inst, Opcode: AMDGPU::S_OR_B32);
8429 Inst.eraseFromParent();
8430 return;
8431
8432 // TODO: remove as soon as everything is ready
8433 // to replace VGPR to SGPR copy with V_READFIRSTLANEs.
8434 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO
8435 // can only be selected from the uniform SDNode.
8436 case AMDGPU::S_ADD_CO_PSEUDO:
8437 case AMDGPU::S_SUB_CO_PSEUDO: {
8438 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
8439 ? AMDGPU::V_ADDC_U32_e64
8440 : AMDGPU::V_SUBB_U32_e64;
8441 const auto *CarryRC = RI.getWaveMaskRegClass();
8442
8443 Register CarryInReg = Inst.getOperand(i: 4).getReg();
8444 if (!MRI.constrainRegClass(Reg: CarryInReg, RC: CarryRC)) {
8445 Register NewCarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
8446 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY), DestReg: NewCarryReg)
8447 .addReg(RegNo: CarryInReg);
8448 }
8449
8450 Register CarryOutReg = Inst.getOperand(i: 1).getReg();
8451
8452 Register DestReg = MRI.createVirtualRegister(RegClass: RI.getEquivalentVGPRClass(
8453 SRC: MRI.getRegClass(Reg: Inst.getOperand(i: 0).getReg())));
8454 MachineInstr *CarryOp =
8455 BuildMI(BB&: *MBB, I: &Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: Opc), DestReg)
8456 .addReg(RegNo: CarryOutReg, Flags: RegState::Define)
8457 .add(MO: Inst.getOperand(i: 2))
8458 .add(MO: Inst.getOperand(i: 3))
8459 .addReg(RegNo: CarryInReg)
8460 .addImm(Val: 0);
8461 legalizeOperands(MI&: *CarryOp);
8462 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: DestReg);
8463 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8464 Inst.eraseFromParent();
8465 }
8466 return;
8467 case AMDGPU::S_UADDO_PSEUDO:
8468 case AMDGPU::S_USUBO_PSEUDO: {
8469 MachineOperand &Dest0 = Inst.getOperand(i: 0);
8470 MachineOperand &Dest1 = Inst.getOperand(i: 1);
8471 MachineOperand &Src0 = Inst.getOperand(i: 2);
8472 MachineOperand &Src1 = Inst.getOperand(i: 3);
8473
8474 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
8475 ? AMDGPU::V_ADD_CO_U32_e64
8476 : AMDGPU::V_SUB_CO_U32_e64;
8477 const TargetRegisterClass *NewRC =
8478 RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest0.getReg()));
8479 Register DestReg = MRI.createVirtualRegister(RegClass: NewRC);
8480 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I: &Inst, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
8481 .addReg(RegNo: Dest1.getReg(), Flags: RegState::Define)
8482 .add(MO: Src0)
8483 .add(MO: Src1)
8484 .addImm(Val: 0); // clamp bit
8485
8486 legalizeOperands(MI&: *NewInstr, MDT);
8487 MRI.replaceRegWith(FromReg: Dest0.getReg(), ToReg: DestReg);
8488 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8489 Inst.eraseFromParent();
8490 }
8491 return;
8492 case AMDGPU::S_LSHL1_ADD_U32:
8493 case AMDGPU::S_LSHL2_ADD_U32:
8494 case AMDGPU::S_LSHL3_ADD_U32:
8495 case AMDGPU::S_LSHL4_ADD_U32: {
8496 MachineOperand &Dest = Inst.getOperand(i: 0);
8497 MachineOperand &Src0 = Inst.getOperand(i: 1);
8498 MachineOperand &Src1 = Inst.getOperand(i: 2);
8499 unsigned ShiftAmt = (Opcode == AMDGPU::S_LSHL1_ADD_U32 ? 1
8500 : Opcode == AMDGPU::S_LSHL2_ADD_U32 ? 2
8501 : Opcode == AMDGPU::S_LSHL3_ADD_U32 ? 3
8502 : 4);
8503
8504 const TargetRegisterClass *NewRC =
8505 RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest.getReg()));
8506 Register DestReg = MRI.createVirtualRegister(RegClass: NewRC);
8507 MachineInstr *NewInstr =
8508 BuildMI(BB&: *MBB, I: &Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_ADD_U32_e64), DestReg)
8509 .add(MO: Src0)
8510 .addImm(Val: ShiftAmt)
8511 .add(MO: Src1);
8512
8513 legalizeOperands(MI&: *NewInstr, MDT);
8514 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: DestReg);
8515 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8516 Inst.eraseFromParent();
8517 }
8518 return;
8519 case AMDGPU::S_CSELECT_B32:
8520 case AMDGPU::S_CSELECT_B64:
8521 lowerSelect(Worklist, Inst, MDT);
8522 Inst.eraseFromParent();
8523 return;
8524 case AMDGPU::S_CMP_EQ_I32:
8525 case AMDGPU::S_CMP_LG_I32:
8526 case AMDGPU::S_CMP_GT_I32:
8527 case AMDGPU::S_CMP_GE_I32:
8528 case AMDGPU::S_CMP_LT_I32:
8529 case AMDGPU::S_CMP_LE_I32:
8530 case AMDGPU::S_CMP_EQ_U32:
8531 case AMDGPU::S_CMP_LG_U32:
8532 case AMDGPU::S_CMP_GT_U32:
8533 case AMDGPU::S_CMP_GE_U32:
8534 case AMDGPU::S_CMP_LT_U32:
8535 case AMDGPU::S_CMP_LE_U32:
8536 case AMDGPU::S_CMP_EQ_U64:
8537 case AMDGPU::S_CMP_LG_U64:
8538 case AMDGPU::S_CMP_LT_F32:
8539 case AMDGPU::S_CMP_EQ_F32:
8540 case AMDGPU::S_CMP_LE_F32:
8541 case AMDGPU::S_CMP_GT_F32:
8542 case AMDGPU::S_CMP_LG_F32:
8543 case AMDGPU::S_CMP_GE_F32:
8544 case AMDGPU::S_CMP_O_F32:
8545 case AMDGPU::S_CMP_U_F32:
8546 case AMDGPU::S_CMP_NGE_F32:
8547 case AMDGPU::S_CMP_NLG_F32:
8548 case AMDGPU::S_CMP_NGT_F32:
8549 case AMDGPU::S_CMP_NLE_F32:
8550 case AMDGPU::S_CMP_NEQ_F32:
8551 case AMDGPU::S_CMP_NLT_F32: {
8552 Register CondReg = MRI.createVirtualRegister(RegClass: RI.getWaveMaskRegClass());
8553 auto NewInstr =
8554 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode), DestReg: CondReg)
8555 .setMIFlags(Inst.getFlags());
8556 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src0_modifiers) >=
8557 0) {
8558 NewInstr
8559 .addImm(Val: 0) // src0_modifiers
8560 .add(MO: Inst.getOperand(i: 0)) // src0
8561 .addImm(Val: 0) // src1_modifiers
8562 .add(MO: Inst.getOperand(i: 1)) // src1
8563 .addImm(Val: 0); // clamp
8564 } else {
8565 NewInstr.add(MO: Inst.getOperand(i: 0)).add(MO: Inst.getOperand(i: 1));
8566 }
8567 legalizeOperands(MI&: *NewInstr, MDT);
8568 int SCCIdx = Inst.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
8569 const MachineOperand &SCCOp = Inst.getOperand(i: SCCIdx);
8570 addSCCDefUsersToVALUWorklist(Op: SCCOp, SCCDefInst&: Inst, Worklist, NewCond: CondReg);
8571 Inst.eraseFromParent();
8572 return;
8573 }
8574 case AMDGPU::S_CMP_LT_F16:
8575 case AMDGPU::S_CMP_EQ_F16:
8576 case AMDGPU::S_CMP_LE_F16:
8577 case AMDGPU::S_CMP_GT_F16:
8578 case AMDGPU::S_CMP_LG_F16:
8579 case AMDGPU::S_CMP_GE_F16:
8580 case AMDGPU::S_CMP_O_F16:
8581 case AMDGPU::S_CMP_U_F16:
8582 case AMDGPU::S_CMP_NGE_F16:
8583 case AMDGPU::S_CMP_NLG_F16:
8584 case AMDGPU::S_CMP_NGT_F16:
8585 case AMDGPU::S_CMP_NLE_F16:
8586 case AMDGPU::S_CMP_NEQ_F16:
8587 case AMDGPU::S_CMP_NLT_F16: {
8588 Register CondReg = MRI.createVirtualRegister(RegClass: RI.getWaveMaskRegClass());
8589 auto NewInstr =
8590 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode), DestReg: CondReg)
8591 .setMIFlags(Inst.getFlags());
8592 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::src0_modifiers)) {
8593 NewInstr
8594 .addImm(Val: 0) // src0_modifiers
8595 .add(MO: Inst.getOperand(i: 0)) // src0
8596 .addImm(Val: 0) // src1_modifiers
8597 .add(MO: Inst.getOperand(i: 1)) // src1
8598 .addImm(Val: 0); // clamp
8599 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::op_sel))
8600 NewInstr.addImm(Val: 0); // op_sel0
8601 } else {
8602 NewInstr
8603 .add(MO: Inst.getOperand(i: 0))
8604 .add(MO: Inst.getOperand(i: 1));
8605 }
8606 legalizeOperands(MI&: *NewInstr, MDT);
8607 int SCCIdx = Inst.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
8608 const MachineOperand &SCCOp = Inst.getOperand(i: SCCIdx);
8609 addSCCDefUsersToVALUWorklist(Op: SCCOp, SCCDefInst&: Inst, Worklist, NewCond: CondReg);
8610 Inst.eraseFromParent();
8611 return;
8612 }
8613 case AMDGPU::S_CVT_HI_F32_F16: {
8614 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8615 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8616 if (ST.useRealTrue16Insts()) {
8617 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: TmpReg)
8618 .add(MO: Inst.getOperand(i: 1));
8619 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8620 .addImm(Val: 0) // src0_modifiers
8621 .addReg(RegNo: TmpReg, Flags: {}, SubReg: AMDGPU::hi16)
8622 .addImm(Val: 0) // clamp
8623 .addImm(Val: 0) // omod
8624 .addImm(Val: 0); // op_sel0
8625 } else {
8626 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
8627 .addImm(Val: 16)
8628 .add(MO: Inst.getOperand(i: 1));
8629 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8630 .addImm(Val: 0) // src0_modifiers
8631 .addReg(RegNo: TmpReg)
8632 .addImm(Val: 0) // clamp
8633 .addImm(Val: 0); // omod
8634 }
8635
8636 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8637 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8638 Inst.eraseFromParent();
8639 return;
8640 }
8641 case AMDGPU::S_MINIMUM_F32:
8642 case AMDGPU::S_MAXIMUM_F32: {
8643 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8644 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8645 .addImm(Val: 0) // src0_modifiers
8646 .add(MO: Inst.getOperand(i: 1))
8647 .addImm(Val: 0) // src1_modifiers
8648 .add(MO: Inst.getOperand(i: 2))
8649 .addImm(Val: 0) // clamp
8650 .addImm(Val: 0); // omod
8651 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8652
8653 legalizeOperands(MI&: *NewInstr, MDT);
8654 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8655 Inst.eraseFromParent();
8656 return;
8657 }
8658 case AMDGPU::S_MINIMUM_F16:
8659 case AMDGPU::S_MAXIMUM_F16: {
8660 Register NewDst = MRI.createVirtualRegister(RegClass: ST.useRealTrue16Insts()
8661 ? &AMDGPU::VGPR_16RegClass
8662 : &AMDGPU::VGPR_32RegClass);
8663 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8664 .addImm(Val: 0) // src0_modifiers
8665 .add(MO: Inst.getOperand(i: 1))
8666 .addImm(Val: 0) // src1_modifiers
8667 .add(MO: Inst.getOperand(i: 2))
8668 .addImm(Val: 0) // clamp
8669 .addImm(Val: 0) // omod
8670 .addImm(Val: 0); // opsel0
8671 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8672 legalizeOperands(MI&: *NewInstr, MDT);
8673 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8674 Inst.eraseFromParent();
8675 return;
8676 }
8677 case AMDGPU::V_S_EXP_F16_e64:
8678 case AMDGPU::V_S_LOG_F16_e64:
8679 case AMDGPU::V_S_RCP_F16_e64:
8680 case AMDGPU::V_S_RSQ_F16_e64:
8681 case AMDGPU::V_S_SQRT_F16_e64: {
8682 Register NewDst = MRI.createVirtualRegister(RegClass: ST.useRealTrue16Insts()
8683 ? &AMDGPU::VGPR_16RegClass
8684 : &AMDGPU::VGPR_32RegClass);
8685 auto NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8686 .add(MO: Inst.getOperand(i: 1)) // src0_modifiers
8687 .add(MO: Inst.getOperand(i: 2))
8688 .add(MO: Inst.getOperand(i: 3)) // clamp
8689 .add(MO: Inst.getOperand(i: 4)) // omod
8690 .setMIFlags(Inst.getFlags());
8691 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::op_sel))
8692 NewInstr.addImm(Val: 0); // opsel0
8693 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8694 legalizeOperands(MI&: *NewInstr, MDT);
8695 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8696 Inst.eraseFromParent();
8697 return;
8698 }
8699 }
8700
8701 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
8702 // We cannot move this instruction to the VALU, so we should try to
8703 // legalize its operands instead.
8704 legalizeOperands(MI&: Inst, MDT);
8705 return;
8706 }
8707 // Handle converting generic instructions like COPY-to-SGPR into
8708 // COPY-to-VGPR.
8709 if (NewOpcode == Opcode) {
8710 Register DstReg = Inst.getOperand(i: 0).getReg();
8711 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
8712
8713 if (Inst.isCopy() && DstReg.isPhysical() &&
8714 Inst.getOperand(i: 1).getReg().isVirtual()) {
8715 handleCopyToPhysHelper(Worklist, DstReg, Inst, MRI, WaterFalls,
8716 V2SPhyCopiesToErase);
8717 return;
8718 }
8719
8720 if (Inst.isCopy() && Inst.getOperand(i: 1).getReg().isVirtual()) {
8721 Register NewDstReg = Inst.getOperand(i: 1).getReg();
8722 const TargetRegisterClass *SrcRC = RI.getRegClassForReg(MRI, Reg: NewDstReg);
8723 if (const TargetRegisterClass *CommonRC =
8724 RI.getCommonSubClass(A: NewDstRC, B: SrcRC)) {
8725 // Instead of creating a copy where src and dst are the same register
8726 // class, we just replace all uses of dst with src. These kinds of
8727 // copies interfere with the heuristics MachineSink uses to decide
8728 // whether or not to split a critical edge. Since the pass assumes
8729 // that copies will end up as machine instructions and not be
8730 // eliminated.
8731 addUsersToMoveToVALUWorklist(Reg: DstReg, MRI, Worklist);
8732 unsigned SrcSubReg = Inst.getOperand(i: 1).getSubReg();
8733 bool IsUndef = Inst.getOperand(i: 1).isUndef();
8734 for (MachineOperand &UseMO :
8735 make_early_inc_range(Range: MRI.use_operands(Reg: DstReg))) {
8736 UseMO.setSubReg(
8737 RI.composeSubRegIndices(a: SrcSubReg, b: UseMO.getSubReg()));
8738 UseMO.setReg(NewDstReg);
8739 if (IsUndef)
8740 UseMO.setIsUndef();
8741 }
8742 MRI.clearKillFlags(Reg: NewDstReg);
8743
8744 if (!MRI.constrainRegClass(Reg: NewDstReg, RC: CommonRC))
8745 llvm_unreachable("failed to constrain register");
8746
8747 Inst.eraseFromParent();
8748
8749 for (MachineOperand &UseMO :
8750 make_early_inc_range(Range: MRI.use_operands(Reg: NewDstReg))) {
8751 MachineInstr &UseMI = *UseMO.getParent();
8752
8753 // Legalize t16 operands since replaceReg is called after
8754 // addUsersToVALU.
8755 legalizeOperandsVALUt16(MI&: UseMI, MRI);
8756
8757 unsigned OpIdx = UseMI.getOperandNo(I: &UseMO);
8758 if (const TargetRegisterClass *OpRC =
8759 getRegClass(MCID: UseMI.getDesc(), OpNum: OpIdx))
8760 MRI.constrainRegClass(Reg: NewDstReg, RC: OpRC);
8761 }
8762
8763 return;
8764 }
8765 }
8766
8767 // If this is a v2s copy between 16bit and 32bit reg,
8768 // replace vgpr copy to reg_sequence/extract_subreg
8769 // This can be remove after we have sgpr16 in place
8770 if (ST.useRealTrue16Insts() && Inst.isCopy() &&
8771 Inst.getOperand(i: 1).getReg().isVirtual() &&
8772 RI.isVGPR(MRI, Reg: Inst.getOperand(i: 1).getReg())) {
8773 const TargetRegisterClass *SrcRegRC = getOpRegClass(MI: Inst, OpNo: 1);
8774 if (RI.getMatchingSuperRegClass(A: NewDstRC, B: SrcRegRC, Idx: AMDGPU::lo16)) {
8775 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
8776 Register Undef = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
8777 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8778 MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
8779 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8780 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewDstReg)
8781 .addReg(RegNo: Inst.getOperand(i: 1).getReg())
8782 .addImm(Val: AMDGPU::lo16)
8783 .addReg(RegNo: Undef)
8784 .addImm(Val: AMDGPU::hi16);
8785 Inst.eraseFromParent();
8786 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
8787 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
8788 return;
8789 } else if (RI.getMatchingSuperRegClass(A: SrcRegRC, B: NewDstRC,
8790 Idx: AMDGPU::lo16)) {
8791 Inst.getOperand(i: 1).setSubReg(AMDGPU::lo16);
8792 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
8793 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
8794 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
8795 return;
8796 }
8797 }
8798
8799 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
8800 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
8801 legalizeOperands(MI&: Inst, MDT);
8802 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
8803 return;
8804 }
8805
8806 // Use the new VALU Opcode.
8807 auto NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode))
8808 .setMIFlags(Inst.getFlags());
8809 if (isVOP3(Opcode: NewOpcode) && !isVOP3(Opcode)) {
8810 // Intersperse VOP3 modifiers among the SALU operands.
8811 NewInstr->addOperand(Op: Inst.getOperand(i: 0));
8812 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
8813 Name: AMDGPU::OpName::src0_modifiers) >= 0)
8814 NewInstr.addImm(Val: 0);
8815 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::src0)) {
8816 const MachineOperand &Src = Inst.getOperand(i: 1);
8817 NewInstr->addOperand(Op: Src);
8818 }
8819
8820 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
8821 // We are converting these to a BFE, so we need to add the missing
8822 // operands for the size and offset.
8823 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
8824 NewInstr.addImm(Val: 0);
8825 NewInstr.addImm(Val: Size);
8826 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
8827 // The VALU version adds the second operand to the result, so insert an
8828 // extra 0 operand.
8829 NewInstr.addImm(Val: 0);
8830 } else if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
8831 const MachineOperand &OffsetWidthOp = Inst.getOperand(i: 2);
8832 // If we need to move this to VGPRs, we need to unpack the second
8833 // operand back into the 2 separate ones for bit offset and width.
8834 assert(OffsetWidthOp.isImm() &&
8835 "Scalar BFE is only implemented for constant width and offset");
8836 uint32_t Imm = OffsetWidthOp.getImm();
8837
8838 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
8839 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
8840 NewInstr.addImm(Val: Offset);
8841 NewInstr.addImm(Val: BitWidth);
8842 } else {
8843 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
8844 Name: AMDGPU::OpName::src1_modifiers) >= 0)
8845 NewInstr.addImm(Val: 0);
8846 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src1) >= 0)
8847 NewInstr->addOperand(Op: Inst.getOperand(i: 2));
8848 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
8849 Name: AMDGPU::OpName::src2_modifiers) >= 0)
8850 NewInstr.addImm(Val: 0);
8851 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src2) >= 0)
8852 NewInstr->addOperand(Op: Inst.getOperand(i: 3));
8853 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::clamp) >= 0)
8854 NewInstr.addImm(Val: 0);
8855 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::omod) >= 0)
8856 NewInstr.addImm(Val: 0);
8857 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::op_sel) >= 0)
8858 NewInstr.addImm(Val: 0);
8859 }
8860 } else {
8861 // Just copy the SALU operands.
8862 for (const MachineOperand &Op : Inst.explicit_operands())
8863 NewInstr->addOperand(Op);
8864 }
8865
8866 // Remove any references to SCC. Vector instructions can't read from it, and
8867 // We're just about to add the implicit use / defs of VCC, and we don't want
8868 // both.
8869 for (MachineOperand &Op : Inst.implicit_operands()) {
8870 if (Op.getReg() == AMDGPU::SCC) {
8871 // Only propagate through live-def of SCC.
8872 if (Op.isDef() && !Op.isDead())
8873 addSCCDefUsersToVALUWorklist(Op, SCCDefInst&: Inst, Worklist);
8874 if (Op.isUse())
8875 addSCCDefsToVALUWorklist(SCCUseInst: NewInstr, Worklist);
8876 }
8877 }
8878 Inst.eraseFromParent();
8879 Register NewDstReg;
8880 if (NewInstr->getOperand(i: 0).isReg() && NewInstr->getOperand(i: 0).isDef()) {
8881 Register DstReg = NewInstr->getOperand(i: 0).getReg();
8882 assert(DstReg.isVirtual());
8883 // Update the destination register class.
8884 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst: *NewInstr);
8885 assert(NewDstRC);
8886 NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
8887 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
8888 }
8889 fixImplicitOperands(MI&: *NewInstr);
8890
8891 // Legalize the operands
8892 legalizeOperands(MI&: *NewInstr, MDT);
8893 if (NewDstReg)
8894 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
8895}
8896
8897// Add/sub require special handling to deal with carry outs.
8898std::pair<bool, MachineBasicBlock *>
8899SIInstrInfo::moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
8900 MachineDominatorTree *MDT) const {
8901 if (ST.hasAddNoCarryInsts()) {
8902 // Assume there is no user of scc since we don't select this in that case.
8903 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
8904 // is used.
8905
8906 MachineBasicBlock &MBB = *Inst.getParent();
8907 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8908
8909 Register OldDstReg = Inst.getOperand(i: 0).getReg();
8910 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8911
8912 unsigned Opc = Inst.getOpcode();
8913 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
8914
8915 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
8916 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
8917
8918 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
8919 Inst.removeOperand(OpNo: 3);
8920
8921 Inst.setDesc(get(Opcode: NewOpc));
8922 Inst.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // clamp bit
8923 Inst.addImplicitDefUseOperands(MF&: *MBB.getParent());
8924 MRI.replaceRegWith(FromReg: OldDstReg, ToReg: ResultReg);
8925 MachineBasicBlock *NewBB = legalizeOperands(MI&: Inst, MDT);
8926
8927 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
8928 return std::pair(true, NewBB);
8929 }
8930
8931 return std::pair(false, nullptr);
8932}
8933
8934void SIInstrInfo::lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
8935 MachineDominatorTree *MDT) const {
8936
8937 MachineBasicBlock &MBB = *Inst.getParent();
8938 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8939 MachineBasicBlock::iterator MII = Inst;
8940 const DebugLoc &DL = Inst.getDebugLoc();
8941
8942 MachineOperand &Dest = Inst.getOperand(i: 0);
8943 MachineOperand &Src0 = Inst.getOperand(i: 1);
8944 MachineOperand &Src1 = Inst.getOperand(i: 2);
8945 MachineOperand &Cond = Inst.getOperand(i: 3);
8946
8947 Register CondReg = Cond.getReg();
8948 bool IsSCC = (CondReg == AMDGPU::SCC);
8949
8950 // Remove S_CSELECT instructions that we previously inserted to feed the SCC
8951 // condition output from S_CMP into the SGPR condition input of V_CNDMASK. If
8952 // the S_CMP has been promoted to V_CMP then we can feed its SGPR condition
8953 // output directly into the V_CNDMASK.
8954 if (!IsSCC && Src0.isImm() && (Src0.getImm() == -1) && Src1.isImm() &&
8955 (Src1.getImm() == 0)) {
8956 for (MachineOperand &UseMO :
8957 make_early_inc_range(Range: MRI.use_nodbg_operands(Reg: Dest.getReg()))) {
8958 MachineInstr &UseMI = *UseMO.getParent();
8959 switch (UseMI.getOpcode()) {
8960 case AMDGPU::V_CNDMASK_B16_fake16_e32:
8961 case AMDGPU::V_CNDMASK_B16_fake16_e64:
8962 case AMDGPU::V_CNDMASK_B16_t16_e32:
8963 case AMDGPU::V_CNDMASK_B16_t16_e64:
8964 case AMDGPU::V_CNDMASK_B32_e32:
8965 case AMDGPU::V_CNDMASK_B32_e64:
8966 case AMDGPU::V_CNDMASK_B64_PSEUDO:
8967 if (UseMO.isImplicit() ||
8968 &UseMO == getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src2))
8969 UseMO.setReg(CondReg);
8970 }
8971 }
8972 if (MRI.use_nodbg_empty(RegNo: Dest.getReg()))
8973 return;
8974 }
8975
8976 Register NewCondReg = CondReg;
8977 if (IsSCC) {
8978 const TargetRegisterClass *TC = RI.getWaveMaskRegClass();
8979 NewCondReg = MRI.createVirtualRegister(RegClass: TC);
8980
8981 // Now look for the closest SCC def if it is a copy
8982 // replacing the CondReg with the COPY source register
8983 bool CopyFound = false;
8984 for (MachineInstr &CandI :
8985 make_range(x: std::next(x: MachineBasicBlock::reverse_iterator(Inst)),
8986 y: Inst.getParent()->rend())) {
8987 if (CandI.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isDead: false, Overlap: false) !=
8988 -1) {
8989 if (CandI.isCopy() && CandI.getOperand(i: 0).getReg() == AMDGPU::SCC) {
8990 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: NewCondReg)
8991 .addReg(RegNo: CandI.getOperand(i: 1).getReg());
8992 CopyFound = true;
8993 }
8994 break;
8995 }
8996 }
8997 if (!CopyFound) {
8998 // SCC def is not a copy
8999 // Insert a trivial select instead of creating a copy, because a copy from
9000 // SCC would semantically mean just copying a single bit, but we may need
9001 // the result to be a vector condition mask that needs preserving.
9002 unsigned Opcode =
9003 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
9004 auto NewSelect =
9005 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: NewCondReg).addImm(Val: -1).addImm(Val: 0);
9006 NewSelect->getOperand(i: 3).setIsUndef(Cond.isUndef());
9007 }
9008 }
9009
9010 Register NewDestReg = MRI.createVirtualRegister(
9011 RegClass: RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest.getReg())));
9012 MachineInstr *NewInst;
9013 if (Inst.getOpcode() == AMDGPU::S_CSELECT_B32) {
9014 NewInst = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: NewDestReg)
9015 .addImm(Val: 0)
9016 .add(MO: Src1) // False
9017 .addImm(Val: 0)
9018 .add(MO: Src0) // True
9019 .addReg(RegNo: NewCondReg);
9020 } else {
9021 NewInst =
9022 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B64_PSEUDO), DestReg: NewDestReg)
9023 .add(MO: Src1) // False
9024 .add(MO: Src0) // True
9025 .addReg(RegNo: NewCondReg);
9026 }
9027 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDestReg);
9028 legalizeOperands(MI&: *NewInst, MDT);
9029 addUsersToMoveToVALUWorklist(Reg: NewDestReg, MRI, Worklist);
9030}
9031
9032void SIInstrInfo::lowerScalarAbs(SIInstrWorklist &Worklist,
9033 MachineInstr &Inst) const {
9034 MachineBasicBlock &MBB = *Inst.getParent();
9035 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9036 MachineBasicBlock::iterator MII = Inst;
9037 const DebugLoc &DL = Inst.getDebugLoc();
9038
9039 MachineOperand &Dest = Inst.getOperand(i: 0);
9040 MachineOperand &Src = Inst.getOperand(i: 1);
9041 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9042 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9043
9044 unsigned SubOp = ST.hasAddNoCarryInsts() ? AMDGPU::V_SUB_U32_e32
9045 : AMDGPU::V_SUB_CO_U32_e32;
9046
9047 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: TmpReg)
9048 .addImm(Val: 0)
9049 .addReg(RegNo: Src.getReg());
9050
9051 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MAX_I32_e64), DestReg: ResultReg)
9052 .addReg(RegNo: Src.getReg())
9053 .addReg(RegNo: TmpReg);
9054
9055 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9056 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9057}
9058
9059void SIInstrInfo::lowerScalarAbsDiff(SIInstrWorklist &Worklist,
9060 MachineInstr &Inst) const {
9061 MachineBasicBlock &MBB = *Inst.getParent();
9062 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9063 MachineBasicBlock::iterator MII = Inst;
9064 const DebugLoc &DL = Inst.getDebugLoc();
9065
9066 MachineOperand &Dest = Inst.getOperand(i: 0);
9067 MachineOperand &Src1 = Inst.getOperand(i: 1);
9068 MachineOperand &Src2 = Inst.getOperand(i: 2);
9069 Register SubResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9070 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9071 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9072
9073 unsigned SubOp = ST.hasAddNoCarryInsts() ? AMDGPU::V_SUB_U32_e32
9074 : AMDGPU::V_SUB_CO_U32_e32;
9075
9076 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: SubResultReg)
9077 .addReg(RegNo: Src1.getReg())
9078 .addReg(RegNo: Src2.getReg());
9079
9080 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: TmpReg).addImm(Val: 0).addReg(RegNo: SubResultReg);
9081
9082 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MAX_I32_e64), DestReg: ResultReg)
9083 .addReg(RegNo: SubResultReg)
9084 .addReg(RegNo: TmpReg);
9085
9086 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9087 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9088}
9089
9090void SIInstrInfo::lowerScalarXnor(SIInstrWorklist &Worklist,
9091 MachineInstr &Inst) const {
9092 MachineBasicBlock &MBB = *Inst.getParent();
9093 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9094 MachineBasicBlock::iterator MII = Inst;
9095 const DebugLoc &DL = Inst.getDebugLoc();
9096
9097 MachineOperand &Dest = Inst.getOperand(i: 0);
9098 MachineOperand &Src0 = Inst.getOperand(i: 1);
9099 MachineOperand &Src1 = Inst.getOperand(i: 2);
9100
9101 if (ST.hasDLInsts()) {
9102 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9103 legalizeGenericOperand(InsertMBB&: MBB, I: MII, DstRC: &AMDGPU::VGPR_32RegClass, Op&: Src0, MRI, DL);
9104 legalizeGenericOperand(InsertMBB&: MBB, I: MII, DstRC: &AMDGPU::VGPR_32RegClass, Op&: Src1, MRI, DL);
9105
9106 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_XNOR_B32_e64), DestReg: NewDest)
9107 .add(MO: Src0)
9108 .add(MO: Src1);
9109
9110 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9111 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9112 } else {
9113 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
9114 // invert either source and then perform the XOR. If either source is a
9115 // scalar register, then we can leave the inversion on the scalar unit to
9116 // achieve a better distribution of scalar and vector instructions.
9117 bool Src0IsSGPR = Src0.isReg() &&
9118 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src0.getReg()));
9119 bool Src1IsSGPR = Src1.isReg() &&
9120 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src1.getReg()));
9121 MachineInstr *Xor;
9122 Register Temp = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9123 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9124
9125 // Build a pair of scalar instructions and add them to the work list.
9126 // The next iteration over the work list will lower these to the vector
9127 // unit as necessary.
9128 if (Src0IsSGPR) {
9129 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Temp).add(MO: Src0);
9130 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: NewDest)
9131 .addReg(RegNo: Temp)
9132 .add(MO: Src1);
9133 } else if (Src1IsSGPR) {
9134 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Temp).add(MO: Src1);
9135 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: NewDest)
9136 .add(MO: Src0)
9137 .addReg(RegNo: Temp);
9138 } else {
9139 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: Temp)
9140 .add(MO: Src0)
9141 .add(MO: Src1);
9142 MachineInstr *Not =
9143 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: NewDest).addReg(RegNo: Temp);
9144 Worklist.insert(MI: Not);
9145 }
9146
9147 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9148
9149 Worklist.insert(MI: Xor);
9150
9151 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9152 }
9153}
9154
9155void SIInstrInfo::splitScalarNotBinop(SIInstrWorklist &Worklist,
9156 MachineInstr &Inst,
9157 unsigned Opcode) const {
9158 MachineBasicBlock &MBB = *Inst.getParent();
9159 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9160 MachineBasicBlock::iterator MII = Inst;
9161 const DebugLoc &DL = Inst.getDebugLoc();
9162
9163 MachineOperand &Dest = Inst.getOperand(i: 0);
9164 MachineOperand &Src0 = Inst.getOperand(i: 1);
9165 MachineOperand &Src1 = Inst.getOperand(i: 2);
9166
9167 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9168 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9169
9170 MachineInstr &Op = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: Interm)
9171 .add(MO: Src0)
9172 .add(MO: Src1);
9173
9174 MachineInstr &Not = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: NewDest)
9175 .addReg(RegNo: Interm);
9176
9177 Worklist.insert(MI: &Op);
9178 Worklist.insert(MI: &Not);
9179
9180 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9181 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9182}
9183
9184void SIInstrInfo::splitScalarBinOpN2(SIInstrWorklist &Worklist,
9185 MachineInstr &Inst,
9186 unsigned Opcode) const {
9187 MachineBasicBlock &MBB = *Inst.getParent();
9188 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9189 MachineBasicBlock::iterator MII = Inst;
9190 const DebugLoc &DL = Inst.getDebugLoc();
9191
9192 MachineOperand &Dest = Inst.getOperand(i: 0);
9193 MachineOperand &Src0 = Inst.getOperand(i: 1);
9194 MachineOperand &Src1 = Inst.getOperand(i: 2);
9195
9196 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
9197 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
9198
9199 MachineInstr &Not = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Interm)
9200 .add(MO: Src1);
9201
9202 MachineInstr &Op = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: NewDest)
9203 .add(MO: Src0)
9204 .addReg(RegNo: Interm);
9205
9206 Worklist.insert(MI: &Not);
9207 Worklist.insert(MI: &Op);
9208
9209 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9210 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9211}
9212
9213void SIInstrInfo::splitScalar64BitUnaryOp(SIInstrWorklist &Worklist,
9214 MachineInstr &Inst, unsigned Opcode,
9215 bool Swap) const {
9216 MachineBasicBlock &MBB = *Inst.getParent();
9217 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9218
9219 MachineOperand &Dest = Inst.getOperand(i: 0);
9220 MachineOperand &Src0 = Inst.getOperand(i: 1);
9221 const DebugLoc &DL = Inst.getDebugLoc();
9222
9223 MachineBasicBlock::iterator MII = Inst;
9224
9225 const MCInstrDesc &InstDesc = get(Opcode);
9226 const TargetRegisterClass *Src0RC = Src0.isReg() ?
9227 MRI.getRegClass(Reg: Src0.getReg()) :
9228 &AMDGPU::SGPR_32RegClass;
9229
9230 const TargetRegisterClass *Src0SubRC =
9231 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9232
9233 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9234 SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9235
9236 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9237 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(SRC: DestRC);
9238 const TargetRegisterClass *NewDestSubRC =
9239 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
9240
9241 Register DestSub0 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9242 MachineInstr &LoHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub0).add(MO: SrcReg0Sub0);
9243
9244 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9245 SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9246
9247 Register DestSub1 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9248 MachineInstr &HiHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub1).add(MO: SrcReg0Sub1);
9249
9250 if (Swap)
9251 std::swap(a&: DestSub0, b&: DestSub1);
9252
9253 Register FullDestReg = MRI.createVirtualRegister(RegClass: NewDestRC);
9254 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9255 .addReg(RegNo: DestSub0)
9256 .addImm(Val: AMDGPU::sub0)
9257 .addReg(RegNo: DestSub1)
9258 .addImm(Val: AMDGPU::sub1);
9259
9260 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9261
9262 Worklist.insert(MI: &LoHalf);
9263 Worklist.insert(MI: &HiHalf);
9264
9265 // We don't need to legalizeOperands here because for a single operand, src0
9266 // will support any kind of input.
9267
9268 // Move all users of this moved value.
9269 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9270}
9271
9272// There is not a vector equivalent of s_mul_u64. For this reason, we need to
9273// split the s_mul_u64 in 32-bit vector multiplications.
9274void SIInstrInfo::splitScalarSMulU64(SIInstrWorklist &Worklist,
9275 MachineInstr &Inst,
9276 MachineDominatorTree *MDT) const {
9277 MachineBasicBlock &MBB = *Inst.getParent();
9278 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9279
9280 Register FullDestReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9281 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9282 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9283
9284 MachineOperand &Dest = Inst.getOperand(i: 0);
9285 MachineOperand &Src0 = Inst.getOperand(i: 1);
9286 MachineOperand &Src1 = Inst.getOperand(i: 2);
9287 const DebugLoc &DL = Inst.getDebugLoc();
9288 MachineBasicBlock::iterator MII = Inst;
9289
9290 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0.getReg());
9291 const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: Src1.getReg());
9292 const TargetRegisterClass *Src0SubRC =
9293 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9294 if (RI.isSGPRClass(RC: Src0SubRC))
9295 Src0SubRC = RI.getEquivalentVGPRClass(SRC: Src0SubRC);
9296 const TargetRegisterClass *Src1SubRC =
9297 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9298 if (RI.isSGPRClass(RC: Src1SubRC))
9299 Src1SubRC = RI.getEquivalentVGPRClass(SRC: Src1SubRC);
9300
9301 // First, we extract the low 32-bit and high 32-bit values from each of the
9302 // operands.
9303 MachineOperand Op0L =
9304 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9305 MachineOperand Op1L =
9306 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9307 MachineOperand Op0H =
9308 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9309 MachineOperand Op1H =
9310 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
9311
9312 // The multilication is done as follows:
9313 //
9314 // Op1H Op1L
9315 // * Op0H Op0L
9316 // --------------------
9317 // Op1H*Op0L Op1L*Op0L
9318 // + Op1H*Op0H Op1L*Op0H
9319 // -----------------------------------------
9320 // (Op1H*Op0L + Op1L*Op0H + carry) Op1L*Op0L
9321 //
9322 // We drop Op1H*Op0H because the result of the multiplication is a 64-bit
9323 // value and that would overflow.
9324 // The low 32-bit value is Op1L*Op0L.
9325 // The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from Op1L*Op0L).
9326
9327 Register Op1L_Op0H_Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9328 MachineInstr *Op1L_Op0H =
9329 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: Op1L_Op0H_Reg)
9330 .add(MO: Op1L)
9331 .add(MO: Op0H);
9332
9333 Register Op1H_Op0L_Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9334 MachineInstr *Op1H_Op0L =
9335 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: Op1H_Op0L_Reg)
9336 .add(MO: Op1H)
9337 .add(MO: Op0L);
9338
9339 Register CarryReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9340 MachineInstr *Carry =
9341 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_HI_U32_e64), DestReg: CarryReg)
9342 .add(MO: Op1L)
9343 .add(MO: Op0L);
9344
9345 MachineInstr *LoHalf =
9346 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: DestSub0)
9347 .add(MO: Op1L)
9348 .add(MO: Op0L);
9349
9350 Register AddReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9351 MachineInstr *Add = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg: AddReg)
9352 .addReg(RegNo: Op1L_Op0H_Reg)
9353 .addReg(RegNo: Op1H_Op0L_Reg);
9354
9355 MachineInstr *HiHalf =
9356 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg: DestSub1)
9357 .addReg(RegNo: AddReg)
9358 .addReg(RegNo: CarryReg);
9359
9360 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9361 .addReg(RegNo: DestSub0)
9362 .addImm(Val: AMDGPU::sub0)
9363 .addReg(RegNo: DestSub1)
9364 .addImm(Val: AMDGPU::sub1);
9365
9366 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9367
9368 // Try to legalize the operands in case we need to swap the order to keep it
9369 // valid.
9370 legalizeOperands(MI&: *Op1L_Op0H, MDT);
9371 legalizeOperands(MI&: *Op1H_Op0L, MDT);
9372 legalizeOperands(MI&: *Carry, MDT);
9373 legalizeOperands(MI&: *LoHalf, MDT);
9374 legalizeOperands(MI&: *Add, MDT);
9375 legalizeOperands(MI&: *HiHalf, MDT);
9376
9377 // Move all users of this moved value.
9378 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9379}
9380
9381// Lower S_MUL_U64_U32_PSEUDO/S_MUL_I64_I32_PSEUDO in two 32-bit vector
9382// multiplications.
9383void SIInstrInfo::splitScalarSMulPseudo(SIInstrWorklist &Worklist,
9384 MachineInstr &Inst,
9385 MachineDominatorTree *MDT) const {
9386 MachineBasicBlock &MBB = *Inst.getParent();
9387 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9388
9389 Register FullDestReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9390 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9391 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9392
9393 MachineOperand &Dest = Inst.getOperand(i: 0);
9394 MachineOperand &Src0 = Inst.getOperand(i: 1);
9395 MachineOperand &Src1 = Inst.getOperand(i: 2);
9396 const DebugLoc &DL = Inst.getDebugLoc();
9397 MachineBasicBlock::iterator MII = Inst;
9398
9399 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0.getReg());
9400 const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: Src1.getReg());
9401 const TargetRegisterClass *Src0SubRC =
9402 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9403 if (RI.isSGPRClass(RC: Src0SubRC))
9404 Src0SubRC = RI.getEquivalentVGPRClass(SRC: Src0SubRC);
9405 const TargetRegisterClass *Src1SubRC =
9406 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9407 if (RI.isSGPRClass(RC: Src1SubRC))
9408 Src1SubRC = RI.getEquivalentVGPRClass(SRC: Src1SubRC);
9409
9410 // First, we extract the low 32-bit and high 32-bit values from each of the
9411 // operands.
9412 MachineOperand Op0L =
9413 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9414 MachineOperand Op1L =
9415 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9416
9417 unsigned Opc = Inst.getOpcode();
9418 unsigned NewOpc = Opc == AMDGPU::S_MUL_U64_U32_PSEUDO
9419 ? AMDGPU::V_MUL_HI_U32_e64
9420 : AMDGPU::V_MUL_HI_I32_e64;
9421 MachineInstr *HiHalf =
9422 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: NewOpc), DestReg: DestSub1).add(MO: Op1L).add(MO: Op0L);
9423
9424 MachineInstr *LoHalf =
9425 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: DestSub0)
9426 .add(MO: Op1L)
9427 .add(MO: Op0L);
9428
9429 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9430 .addReg(RegNo: DestSub0)
9431 .addImm(Val: AMDGPU::sub0)
9432 .addReg(RegNo: DestSub1)
9433 .addImm(Val: AMDGPU::sub1);
9434
9435 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9436
9437 // Try to legalize the operands in case we need to swap the order to keep it
9438 // valid.
9439 legalizeOperands(MI&: *HiHalf, MDT);
9440 legalizeOperands(MI&: *LoHalf, MDT);
9441
9442 // Move all users of this moved value.
9443 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9444}
9445
9446void SIInstrInfo::splitScalar64BitBinaryOp(SIInstrWorklist &Worklist,
9447 MachineInstr &Inst, unsigned Opcode,
9448 MachineDominatorTree *MDT) const {
9449 MachineBasicBlock &MBB = *Inst.getParent();
9450 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9451
9452 MachineOperand &Dest = Inst.getOperand(i: 0);
9453 MachineOperand &Src0 = Inst.getOperand(i: 1);
9454 MachineOperand &Src1 = Inst.getOperand(i: 2);
9455 const DebugLoc &DL = Inst.getDebugLoc();
9456
9457 MachineBasicBlock::iterator MII = Inst;
9458
9459 const MCInstrDesc &InstDesc = get(Opcode);
9460 const TargetRegisterClass *Src0RC = Src0.isReg() ?
9461 MRI.getRegClass(Reg: Src0.getReg()) :
9462 &AMDGPU::SGPR_32RegClass;
9463
9464 const TargetRegisterClass *Src0SubRC =
9465 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9466 const TargetRegisterClass *Src1RC = Src1.isReg() ?
9467 MRI.getRegClass(Reg: Src1.getReg()) :
9468 &AMDGPU::SGPR_32RegClass;
9469
9470 const TargetRegisterClass *Src1SubRC =
9471 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9472
9473 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9474 SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9475 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC,
9476 SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9477 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9478 SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9479 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC,
9480 SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
9481
9482 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9483 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(SRC: DestRC);
9484 const TargetRegisterClass *NewDestSubRC =
9485 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
9486
9487 Register DestSub0 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9488 MachineInstr &LoHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub0)
9489 .add(MO: SrcReg0Sub0)
9490 .add(MO: SrcReg1Sub0);
9491
9492 Register DestSub1 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9493 MachineInstr &HiHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub1)
9494 .add(MO: SrcReg0Sub1)
9495 .add(MO: SrcReg1Sub1);
9496
9497 Register FullDestReg = MRI.createVirtualRegister(RegClass: NewDestRC);
9498 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9499 .addReg(RegNo: DestSub0)
9500 .addImm(Val: AMDGPU::sub0)
9501 .addReg(RegNo: DestSub1)
9502 .addImm(Val: AMDGPU::sub1);
9503
9504 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9505
9506 Worklist.insert(MI: &LoHalf);
9507 Worklist.insert(MI: &HiHalf);
9508
9509 // Move all users of this moved value.
9510 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9511}
9512
9513void SIInstrInfo::splitScalar64BitXnor(SIInstrWorklist &Worklist,
9514 MachineInstr &Inst,
9515 MachineDominatorTree *MDT) const {
9516 MachineBasicBlock &MBB = *Inst.getParent();
9517 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9518
9519 MachineOperand &Dest = Inst.getOperand(i: 0);
9520 MachineOperand &Src0 = Inst.getOperand(i: 1);
9521 MachineOperand &Src1 = Inst.getOperand(i: 2);
9522 const DebugLoc &DL = Inst.getDebugLoc();
9523
9524 MachineBasicBlock::iterator MII = Inst;
9525
9526 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9527
9528 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
9529
9530 MachineOperand* Op0;
9531 MachineOperand* Op1;
9532
9533 if (Src0.isReg() && RI.isSGPRReg(MRI, Reg: Src0.getReg())) {
9534 Op0 = &Src0;
9535 Op1 = &Src1;
9536 } else {
9537 Op0 = &Src1;
9538 Op1 = &Src0;
9539 }
9540
9541 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B64), DestReg: Interm)
9542 .add(MO: *Op0);
9543
9544 Register NewDest = MRI.createVirtualRegister(RegClass: DestRC);
9545
9546 MachineInstr &Xor = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B64), DestReg: NewDest)
9547 .addReg(RegNo: Interm)
9548 .add(MO: *Op1);
9549
9550 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9551
9552 Worklist.insert(MI: &Xor);
9553}
9554
9555void SIInstrInfo::splitScalar64BitBCNT(SIInstrWorklist &Worklist,
9556 MachineInstr &Inst) const {
9557 MachineBasicBlock &MBB = *Inst.getParent();
9558 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9559
9560 MachineBasicBlock::iterator MII = Inst;
9561 const DebugLoc &DL = Inst.getDebugLoc();
9562
9563 MachineOperand &Dest = Inst.getOperand(i: 0);
9564 MachineOperand &Src = Inst.getOperand(i: 1);
9565
9566 const MCInstrDesc &InstDesc = get(Opcode: AMDGPU::V_BCNT_U32_B32_e64);
9567 const TargetRegisterClass *SrcRC = Src.isReg() ?
9568 MRI.getRegClass(Reg: Src.getReg()) :
9569 &AMDGPU::SGPR_32RegClass;
9570
9571 Register MidReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9572 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9573
9574 const TargetRegisterClass *SrcSubRC =
9575 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
9576
9577 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC,
9578 SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
9579 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC,
9580 SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
9581
9582 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg).add(MO: SrcRegSub0).addImm(Val: 0);
9583
9584 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: ResultReg).add(MO: SrcRegSub1).addReg(RegNo: MidReg);
9585
9586 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9587
9588 // We don't need to legalize operands here. src0 for either instruction can be
9589 // an SGPR, and the second input is unused or determined here.
9590 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9591}
9592
9593void SIInstrInfo::splitScalar64BitBFE(SIInstrWorklist &Worklist,
9594 MachineInstr &Inst) const {
9595 MachineBasicBlock &MBB = *Inst.getParent();
9596 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9597 MachineBasicBlock::iterator MII = Inst;
9598 const DebugLoc &DL = Inst.getDebugLoc();
9599
9600 MachineOperand &Dest = Inst.getOperand(i: 0);
9601 uint32_t Imm = Inst.getOperand(i: 2).getImm();
9602 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
9603 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
9604
9605 (void) Offset;
9606
9607 // Only sext_inreg cases handled.
9608 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
9609 Offset == 0 && "Not implemented");
9610
9611 if (BitWidth < 32) {
9612 Register MidRegLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9613 Register MidRegHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9614 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9615
9616 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_BFE_I32_e64), DestReg: MidRegLo)
9617 .addReg(RegNo: Inst.getOperand(i: 1).getReg(), Flags: {}, SubReg: AMDGPU::sub0)
9618 .addImm(Val: 0)
9619 .addImm(Val: BitWidth);
9620
9621 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ASHRREV_I32_e32), DestReg: MidRegHi)
9622 .addImm(Val: 31)
9623 .addReg(RegNo: MidRegLo);
9624
9625 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: ResultReg)
9626 .addReg(RegNo: MidRegLo)
9627 .addImm(Val: AMDGPU::sub0)
9628 .addReg(RegNo: MidRegHi)
9629 .addImm(Val: AMDGPU::sub1);
9630
9631 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9632 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9633 return;
9634 }
9635
9636 MachineOperand &Src = Inst.getOperand(i: 1);
9637 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9638 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9639
9640 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ASHRREV_I32_e64), DestReg: TmpReg)
9641 .addImm(Val: 31)
9642 .addReg(RegNo: Src.getReg(), Flags: {}, SubReg: AMDGPU::sub0);
9643
9644 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: ResultReg)
9645 .addReg(RegNo: Src.getReg(), Flags: {}, SubReg: AMDGPU::sub0)
9646 .addImm(Val: AMDGPU::sub0)
9647 .addReg(RegNo: TmpReg)
9648 .addImm(Val: AMDGPU::sub1);
9649
9650 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9651 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9652}
9653
9654void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist,
9655 MachineInstr &Inst, unsigned Opcode,
9656 MachineDominatorTree *MDT) const {
9657 // (S_FLBIT_I32_B64 hi:lo) ->
9658 // -> (umin (V_FFBH_U32_e32 hi), (or (V_FFBH_U32_e32 lo), 32))
9659 // (S_FF1_I32_B64 hi:lo) ->
9660 // ->(umin (or (V_FFBL_B32_e32 hi), 32) (V_FFBL_B32_e32 lo))
9661
9662 MachineBasicBlock &MBB = *Inst.getParent();
9663 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9664 MachineBasicBlock::iterator MII = Inst;
9665 const DebugLoc &DL = Inst.getDebugLoc();
9666
9667 MachineOperand &Dest = Inst.getOperand(i: 0);
9668 MachineOperand &Src = Inst.getOperand(i: 1);
9669
9670 const MCInstrDesc &InstDesc = get(Opcode);
9671
9672 bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32;
9673
9674 const TargetRegisterClass *SrcRC =
9675 Src.isReg() ? MRI.getRegClass(Reg: Src.getReg()) : &AMDGPU::SGPR_32RegClass;
9676 const TargetRegisterClass *SrcSubRC =
9677 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
9678
9679 MachineOperand SrcRegSub0 =
9680 buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC, SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
9681 MachineOperand SrcRegSub1 =
9682 buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC, SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
9683
9684 Register MidReg1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9685 Register MidReg2 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9686 Register MidReg3 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9687 Register MidReg4 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9688
9689 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg1).add(MO: SrcRegSub0);
9690
9691 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg2).add(MO: SrcRegSub1);
9692
9693 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_OR_B32_e32), DestReg: MidReg3)
9694 .addImm(Val: 32)
9695 .addReg(RegNo: IsCtlz ? MidReg1 : MidReg2);
9696
9697 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MIN_U32_e64), DestReg: MidReg4)
9698 .addReg(RegNo: MidReg3)
9699 .addReg(RegNo: IsCtlz ? MidReg2 : MidReg1);
9700
9701 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: MidReg4);
9702
9703 addUsersToMoveToVALUWorklist(Reg: MidReg4, MRI, Worklist);
9704}
9705
9706void SIInstrInfo::addUsersToMoveToVALUWorklist(
9707 Register DstReg, MachineRegisterInfo &MRI,
9708 SIInstrWorklist &Worklist) const {
9709 for (MachineOperand &MO : make_early_inc_range(Range: MRI.use_operands(Reg: DstReg))) {
9710 MachineInstr &UseMI = *MO.getParent();
9711
9712 unsigned OpNo = 0;
9713
9714 switch (UseMI.getOpcode()) {
9715 case AMDGPU::COPY:
9716 case AMDGPU::WQM:
9717 case AMDGPU::SOFT_WQM:
9718 case AMDGPU::STRICT_WWM:
9719 case AMDGPU::STRICT_WQM:
9720 case AMDGPU::REG_SEQUENCE:
9721 case AMDGPU::PHI:
9722 case AMDGPU::INSERT_SUBREG:
9723 break;
9724 default:
9725 OpNo = MO.getOperandNo();
9726 break;
9727 }
9728
9729 const TargetRegisterClass *OpRC = getOpRegClass(MI: UseMI, OpNo);
9730 MRI.constrainRegClass(Reg: DstReg, RC: OpRC);
9731
9732 if (!RI.hasVectorRegisters(RC: OpRC))
9733 Worklist.insert(MI: &UseMI);
9734 else
9735 // Legalization could change user list.
9736 legalizeOperandsVALUt16(MI&: UseMI, OpIdx: OpNo, MRI);
9737 }
9738}
9739
9740void SIInstrInfo::movePackToVALU(SIInstrWorklist &Worklist,
9741 MachineRegisterInfo &MRI,
9742 MachineInstr &Inst) const {
9743 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9744 MachineBasicBlock *MBB = Inst.getParent();
9745 MachineOperand &Src0 = Inst.getOperand(i: 1);
9746 MachineOperand &Src1 = Inst.getOperand(i: 2);
9747 const DebugLoc &DL = Inst.getDebugLoc();
9748
9749 if (ST.useRealTrue16Insts()) {
9750 Register SrcReg0, SrcReg1;
9751 if (!Src0.isReg() || !RI.isVGPR(MRI, Reg: Src0.getReg())) {
9752 SrcReg0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9753 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL,
9754 MCID: get(Opcode: Src0.isImm() ? AMDGPU::V_MOV_B32_e32 : AMDGPU::COPY), DestReg: SrcReg0)
9755 .add(MO: Src0);
9756 } else {
9757 SrcReg0 = Src0.getReg();
9758 }
9759
9760 if (!Src1.isReg() || !RI.isVGPR(MRI, Reg: Src1.getReg())) {
9761 SrcReg1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9762 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL,
9763 MCID: get(Opcode: Src1.isImm() ? AMDGPU::V_MOV_B32_e32 : AMDGPU::COPY), DestReg: SrcReg1)
9764 .add(MO: Src1);
9765 } else {
9766 SrcReg1 = Src1.getReg();
9767 }
9768
9769 bool isSrc0Reg16 = MRI.constrainRegClass(Reg: SrcReg0, RC: &AMDGPU::VGPR_16RegClass);
9770 bool isSrc1Reg16 = MRI.constrainRegClass(Reg: SrcReg1, RC: &AMDGPU::VGPR_16RegClass);
9771
9772 auto NewMI = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: ResultReg);
9773 switch (Inst.getOpcode()) {
9774 case AMDGPU::S_PACK_LL_B32_B16:
9775 NewMI
9776 .addReg(RegNo: SrcReg0, Flags: {},
9777 SubReg: isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
9778 .addImm(Val: AMDGPU::lo16)
9779 .addReg(RegNo: SrcReg1, Flags: {},
9780 SubReg: isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
9781 .addImm(Val: AMDGPU::hi16);
9782 break;
9783 case AMDGPU::S_PACK_LH_B32_B16:
9784 NewMI
9785 .addReg(RegNo: SrcReg0, Flags: {},
9786 SubReg: isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
9787 .addImm(Val: AMDGPU::lo16)
9788 .addReg(RegNo: SrcReg1, Flags: {}, SubReg: AMDGPU::hi16)
9789 .addImm(Val: AMDGPU::hi16);
9790 break;
9791 case AMDGPU::S_PACK_HL_B32_B16:
9792 NewMI.addReg(RegNo: SrcReg0, Flags: {}, SubReg: AMDGPU::hi16)
9793 .addImm(Val: AMDGPU::lo16)
9794 .addReg(RegNo: SrcReg1, Flags: {},
9795 SubReg: isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
9796 .addImm(Val: AMDGPU::hi16);
9797 break;
9798 case AMDGPU::S_PACK_HH_B32_B16:
9799 NewMI.addReg(RegNo: SrcReg0, Flags: {}, SubReg: AMDGPU::hi16)
9800 .addImm(Val: AMDGPU::lo16)
9801 .addReg(RegNo: SrcReg1, Flags: {}, SubReg: AMDGPU::hi16)
9802 .addImm(Val: AMDGPU::hi16);
9803 break;
9804 default:
9805 llvm_unreachable("unhandled s_pack_* instruction");
9806 }
9807
9808 MachineOperand &Dest = Inst.getOperand(i: 0);
9809 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9810 addUsersToMoveToVALUWorklist(DstReg: ResultReg, MRI, Worklist);
9811 return;
9812 }
9813
9814 switch (Inst.getOpcode()) {
9815 case AMDGPU::S_PACK_LL_B32_B16: {
9816 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9817 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9818
9819 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are
9820 // 0.
9821 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
9822 .addImm(Val: 0xffff);
9823
9824 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_AND_B32_e64), DestReg: TmpReg)
9825 .addReg(RegNo: ImmReg, Flags: RegState::Kill)
9826 .add(MO: Src0);
9827
9828 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: ResultReg)
9829 .add(MO: Src1)
9830 .addImm(Val: 16)
9831 .addReg(RegNo: TmpReg, Flags: RegState::Kill);
9832 break;
9833 }
9834 case AMDGPU::S_PACK_LH_B32_B16: {
9835 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9836 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
9837 .addImm(Val: 0xffff);
9838 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_BFI_B32_e64), DestReg: ResultReg)
9839 .addReg(RegNo: ImmReg, Flags: RegState::Kill)
9840 .add(MO: Src0)
9841 .add(MO: Src1);
9842 break;
9843 }
9844 case AMDGPU::S_PACK_HL_B32_B16: {
9845 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9846 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
9847 .addImm(Val: 16)
9848 .add(MO: Src0);
9849 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: ResultReg)
9850 .add(MO: Src1)
9851 .addImm(Val: 16)
9852 .addReg(RegNo: TmpReg, Flags: RegState::Kill);
9853 break;
9854 }
9855 case AMDGPU::S_PACK_HH_B32_B16: {
9856 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9857 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9858 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
9859 .addImm(Val: 16)
9860 .add(MO: Src0);
9861 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
9862 .addImm(Val: 0xffff0000);
9863 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_AND_OR_B32_e64), DestReg: ResultReg)
9864 .add(MO: Src1)
9865 .addReg(RegNo: ImmReg, Flags: RegState::Kill)
9866 .addReg(RegNo: TmpReg, Flags: RegState::Kill);
9867 break;
9868 }
9869 default:
9870 llvm_unreachable("unhandled s_pack_* instruction");
9871 }
9872
9873 MachineOperand &Dest = Inst.getOperand(i: 0);
9874 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9875 addUsersToMoveToVALUWorklist(DstReg: ResultReg, MRI, Worklist);
9876}
9877
9878void SIInstrInfo::addSCCDefUsersToVALUWorklist(const MachineOperand &Op,
9879 MachineInstr &SCCDefInst,
9880 SIInstrWorklist &Worklist,
9881 Register NewCond) const {
9882
9883 // Ensure that def inst defines SCC, which is still live.
9884 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() &&
9885 !Op.isDead() && Op.getParent() == &SCCDefInst);
9886 SmallVector<MachineInstr *, 4> CopyToDelete;
9887 // This assumes that all the users of SCC are in the same block
9888 // as the SCC def.
9889 for (MachineInstr &MI : // Skip the def inst itself.
9890 make_range(x: std::next(x: MachineBasicBlock::iterator(SCCDefInst)),
9891 y: SCCDefInst.getParent()->end())) {
9892 // Check if SCC is used first.
9893 int SCCIdx = MI.findRegisterUseOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isKill: false);
9894 if (SCCIdx != -1) {
9895 if (MI.isCopy()) {
9896 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
9897 Register DestReg = MI.getOperand(i: 0).getReg();
9898
9899 MRI.replaceRegWith(FromReg: DestReg, ToReg: NewCond);
9900 CopyToDelete.push_back(Elt: &MI);
9901 } else {
9902
9903 if (NewCond.isValid())
9904 MI.getOperand(i: SCCIdx).setReg(NewCond);
9905
9906 Worklist.insert(MI: &MI);
9907 }
9908 }
9909 // Exit if we find another SCC def.
9910 if (MI.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isDead: false, Overlap: false) != -1)
9911 break;
9912 }
9913 for (auto &Copy : CopyToDelete)
9914 Copy->eraseFromParent();
9915}
9916
9917// Instructions that use SCC may be converted to VALU instructions. When that
9918// happens, the SCC register is changed to VCC_LO. The instruction that defines
9919// SCC must be changed to an instruction that defines VCC. This function makes
9920// sure that the instruction that defines SCC is added to the moveToVALU
9921// worklist.
9922void SIInstrInfo::addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
9923 SIInstrWorklist &Worklist) const {
9924 // Look for a preceding instruction that either defines VCC or SCC. If VCC
9925 // then there is nothing to do because the defining instruction has been
9926 // converted to a VALU already. If SCC then that instruction needs to be
9927 // converted to a VALU.
9928 for (MachineInstr &MI :
9929 make_range(x: std::next(x: MachineBasicBlock::reverse_iterator(SCCUseInst)),
9930 y: SCCUseInst->getParent()->rend())) {
9931 if (MI.modifiesRegister(Reg: AMDGPU::VCC, TRI: &RI))
9932 break;
9933 if (MI.definesRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
9934 Worklist.insert(MI: &MI);
9935 break;
9936 }
9937 }
9938}
9939
9940const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
9941 const MachineInstr &Inst) const {
9942 const TargetRegisterClass *NewDstRC = getOpRegClass(MI: Inst, OpNo: 0);
9943
9944 switch (Inst.getOpcode()) {
9945 // For target instructions, getOpRegClass just returns the virtual register
9946 // class associated with the operand, so we need to find an equivalent VGPR
9947 // register class in order to move the instruction to the VALU.
9948 case AMDGPU::COPY:
9949 case AMDGPU::PHI:
9950 case AMDGPU::REG_SEQUENCE:
9951 case AMDGPU::INSERT_SUBREG:
9952 case AMDGPU::WQM:
9953 case AMDGPU::SOFT_WQM:
9954 case AMDGPU::STRICT_WWM:
9955 case AMDGPU::STRICT_WQM: {
9956 const TargetRegisterClass *SrcRC = getOpRegClass(MI: Inst, OpNo: 1);
9957 if (RI.isAGPRClass(RC: SrcRC)) {
9958 if (RI.isAGPRClass(RC: NewDstRC))
9959 return nullptr;
9960
9961 switch (Inst.getOpcode()) {
9962 case AMDGPU::PHI:
9963 case AMDGPU::REG_SEQUENCE:
9964 case AMDGPU::INSERT_SUBREG:
9965 NewDstRC = RI.getEquivalentAGPRClass(SRC: NewDstRC);
9966 break;
9967 default:
9968 NewDstRC = RI.getEquivalentVGPRClass(SRC: NewDstRC);
9969 }
9970
9971 if (!NewDstRC)
9972 return nullptr;
9973 } else {
9974 if (!RI.isSGPRClass(RC: NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass)
9975 return nullptr;
9976
9977 NewDstRC = RI.getEquivalentVGPRClass(SRC: NewDstRC);
9978 if (!NewDstRC)
9979 return nullptr;
9980 }
9981
9982 return NewDstRC;
9983 }
9984 default:
9985 return NewDstRC;
9986 }
9987}
9988
9989// Find the one SGPR operand we are allowed to use.
9990Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
9991 int OpIndices[3]) const {
9992 const MCInstrDesc &Desc = MI.getDesc();
9993
9994 // Find the one SGPR operand we are allowed to use.
9995 //
9996 // First we need to consider the instruction's operand requirements before
9997 // legalizing. Some operands are required to be SGPRs, such as implicit uses
9998 // of VCC, but we are still bound by the constant bus requirement to only use
9999 // one.
10000 //
10001 // If the operand's class is an SGPR, we can never move it.
10002
10003 Register SGPRReg = findImplicitSGPRRead(MI);
10004 if (SGPRReg)
10005 return SGPRReg;
10006
10007 Register UsedSGPRs[3] = {Register()};
10008 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
10009
10010 for (unsigned i = 0; i < 3; ++i) {
10011 int Idx = OpIndices[i];
10012 if (Idx == -1)
10013 break;
10014
10015 const MachineOperand &MO = MI.getOperand(i: Idx);
10016 if (!MO.isReg())
10017 continue;
10018
10019 // Is this operand statically required to be an SGPR based on the operand
10020 // constraints?
10021 const TargetRegisterClass *OpRC =
10022 RI.getRegClass(i: getOpRegClassID(OpInfo: Desc.operands()[Idx]));
10023 bool IsRequiredSGPR = RI.isSGPRClass(RC: OpRC);
10024 if (IsRequiredSGPR)
10025 return MO.getReg();
10026
10027 // If this could be a VGPR or an SGPR, Check the dynamic register class.
10028 Register Reg = MO.getReg();
10029 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
10030 if (RI.isSGPRClass(RC: RegRC))
10031 UsedSGPRs[i] = Reg;
10032 }
10033
10034 // We don't have a required SGPR operand, so we have a bit more freedom in
10035 // selecting operands to move.
10036
10037 // Try to select the most used SGPR. If an SGPR is equal to one of the
10038 // others, we choose that.
10039 //
10040 // e.g.
10041 // V_FMA_F32 v0, s0, s0, s0 -> No moves
10042 // V_FMA_F32 v0, s0, s1, s0 -> Move s1
10043
10044 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
10045 // prefer those.
10046
10047 if (UsedSGPRs[0]) {
10048 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
10049 SGPRReg = UsedSGPRs[0];
10050 }
10051
10052 if (!SGPRReg && UsedSGPRs[1]) {
10053 if (UsedSGPRs[1] == UsedSGPRs[2])
10054 SGPRReg = UsedSGPRs[1];
10055 }
10056
10057 return SGPRReg;
10058}
10059
10060MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI,
10061 AMDGPU::OpName OperandName) const {
10062 if (OperandName == AMDGPU::OpName::NUM_OPERAND_NAMES)
10063 return nullptr;
10064
10065 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OperandName);
10066 if (Idx == -1)
10067 return nullptr;
10068
10069 return &MI.getOperand(i: Idx);
10070}
10071
10072uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {
10073 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
10074 int64_t Format = ST.getGeneration() >= AMDGPUSubtarget::GFX11
10075 ? (int64_t)AMDGPU::UfmtGFX11::UFMT_32_FLOAT
10076 : (int64_t)AMDGPU::UfmtGFX10::UFMT_32_FLOAT;
10077 return (Format << 44) |
10078 (1ULL << 56) | // RESOURCE_LEVEL = 1
10079 (3ULL << 60); // OOB_SELECT = 3
10080 }
10081
10082 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
10083 if (ST.isAmdHsaOS()) {
10084 // Set ATC = 1. GFX9 doesn't have this bit.
10085 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10086 RsrcDataFormat |= (1ULL << 56);
10087
10088 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
10089 // BTW, it disables TC L2 and therefore decreases performance.
10090 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)
10091 RsrcDataFormat |= (2ULL << 59);
10092 }
10093
10094 return RsrcDataFormat;
10095}
10096
10097uint64_t SIInstrInfo::getScratchRsrcWords23() const {
10098 uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
10099 AMDGPU::RSRC_TID_ENABLE |
10100 0xffffffff; // Size;
10101
10102 // GFX9 doesn't have ELEMENT_SIZE.
10103 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
10104 uint64_t EltSizeValue = Log2_32(Value: ST.getMaxPrivateElementSize(ForBufferRSrc: true)) - 1;
10105 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
10106 }
10107
10108 // IndexStride = 64 / 32.
10109 uint64_t IndexStride = ST.isWave64() ? 3 : 2;
10110 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
10111
10112 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
10113 // Clear them unless we want a huge stride.
10114 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
10115 ST.getGeneration() <= AMDGPUSubtarget::GFX9)
10116 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
10117
10118 return Rsrc23;
10119}
10120
10121bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const {
10122 unsigned Opc = MI.getOpcode();
10123
10124 return isSMRD(Opcode: Opc);
10125}
10126
10127bool SIInstrInfo::isHighLatencyDef(int Opc) const {
10128 return get(Opcode: Opc).mayLoad() &&
10129 (isMUBUF(Opcode: Opc) || isMTBUF(Opcode: Opc) || isMIMG(Opcode: Opc) || isFLAT(Opcode: Opc));
10130}
10131
10132Register SIInstrInfo::isStackAccess(const MachineInstr &MI, int &FrameIndex,
10133 TypeSize &MemBytes) const {
10134 const MachineOperand *Addr = getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
10135 if (!Addr || !Addr->isFI())
10136 return Register();
10137
10138 assert(!MI.memoperands_empty() &&
10139 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS);
10140
10141 FrameIndex = Addr->getIndex();
10142
10143 int VDataIdx =
10144 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdata);
10145 MemBytes = TypeSize::getFixed(ExactSize: getOpSize(Opcode: MI.getOpcode(), OpNo: VDataIdx));
10146 return MI.getOperand(i: VDataIdx).getReg();
10147}
10148
10149Register SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex,
10150 TypeSize &MemBytes) const {
10151 const MachineOperand *Addr = getNamedOperand(MI, OperandName: AMDGPU::OpName::addr);
10152 assert(Addr && Addr->isFI());
10153 FrameIndex = Addr->getIndex();
10154
10155 int DataIdx =
10156 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::data);
10157 MemBytes = TypeSize::getFixed(ExactSize: getOpSize(Opcode: MI.getOpcode(), OpNo: DataIdx));
10158 return MI.getOperand(i: DataIdx).getReg();
10159}
10160
10161Register SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
10162 int &FrameIndex,
10163 TypeSize &MemBytes) const {
10164 if (!MI.mayLoad())
10165 return Register();
10166
10167 if (isMUBUF(MI) || isVGPRSpill(MI))
10168 return isStackAccess(MI, FrameIndex, MemBytes);
10169
10170 if (isSGPRSpill(MI))
10171 return isSGPRStackAccess(MI, FrameIndex, MemBytes);
10172
10173 return Register();
10174}
10175
10176Register SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
10177 int &FrameIndex,
10178 TypeSize &MemBytes) const {
10179 if (!MI.mayStore())
10180 return Register();
10181
10182 if (isMUBUF(MI) || isVGPRSpill(MI))
10183 return isStackAccess(MI, FrameIndex, MemBytes);
10184
10185 if (isSGPRSpill(MI))
10186 return isSGPRStackAccess(MI, FrameIndex, MemBytes);
10187
10188 return Register();
10189}
10190
10191unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
10192 unsigned Opc = MI.getOpcode();
10193 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: Opc);
10194 unsigned DescSize = Desc.getSize();
10195
10196 // If we have a definitive size, we can use it. Otherwise we need to inspect
10197 // the operands to know the size.
10198 if (isFixedSize(MI)) {
10199 unsigned Size = DescSize;
10200
10201 // If we hit the buggy offset, an extra nop will be inserted in MC so
10202 // estimate the worst case.
10203 if (MI.isBranch() && ST.hasOffset3fBug())
10204 Size += 4;
10205
10206 return Size;
10207 }
10208
10209 // Instructions may have a 32-bit literal encoded after them. Check
10210 // operands that could ever be literals.
10211 if (isVALU(MI, /*AllowLDSDMA=*/true) || isSALU(MI)) {
10212 if (isDPP(MI))
10213 return DescSize;
10214 bool HasLiteral = false;
10215 unsigned LiteralSize = 4;
10216 for (int I = 0, E = MI.getNumExplicitOperands(); I != E; ++I) {
10217 const MachineOperand &Op = MI.getOperand(i: I);
10218 const MCOperandInfo &OpInfo = Desc.operands()[I];
10219 if (!Op.isReg() && !isInlineConstant(MO: Op, OpInfo)) {
10220 HasLiteral = true;
10221 if (ST.has64BitLiterals()) {
10222 switch (OpInfo.OperandType) {
10223 default:
10224 break;
10225 case AMDGPU::OPERAND_REG_IMM_FP64:
10226 case AMDGPU::OPERAND_REG_IMM_V2FP64:
10227 if (!AMDGPU::isValid32BitLiteral(Val: Op.getImm(), IsFP64: true))
10228 LiteralSize = 8;
10229 break;
10230 case AMDGPU::OPERAND_REG_IMM_INT64:
10231 case AMDGPU::OPERAND_REG_IMM_V2INT64:
10232 // A 32-bit literal is only valid when the value fits in BOTH signed
10233 // and unsigned 32-bit ranges [0, 2^31-1], matching the MC code
10234 // emitter's getLit64Encoding logic. This is because of the lack of
10235 // abilility to tell signedness of the literal, therefore we need to
10236 // be conservative and assume values outside this range require a
10237 // 64-bit literal encoding (8 bytes).
10238 if (!Op.isImm() || !isInt<32>(x: Op.getImm()) ||
10239 !isUInt<32>(x: Op.getImm()))
10240 LiteralSize = 8;
10241 break;
10242 }
10243 }
10244 break;
10245 }
10246 }
10247 return HasLiteral ? DescSize + LiteralSize : DescSize;
10248 }
10249
10250 // Check whether we have extra NSA words.
10251 if (isMIMG(MI)) {
10252 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0);
10253 if (VAddr0Idx < 0)
10254 return 8;
10255
10256 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::srsrc);
10257 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4);
10258 }
10259
10260 switch (Opc) {
10261 case TargetOpcode::BUNDLE:
10262 return getInstBundleSize(MI);
10263 case TargetOpcode::INLINEASM:
10264 case TargetOpcode::INLINEASM_BR: {
10265 const MachineFunction *MF = MI.getMF();
10266 const char *AsmStr = MI.getOperand(i: 0).getSymbolName();
10267 return getInlineAsmLength(Str: AsmStr, MAI: MF->getTarget().getMCAsmInfo(), STI: &ST);
10268 }
10269 default:
10270 if (MI.isMetaInstruction())
10271 return 0;
10272
10273 // If D16 Pseudo inst, get correct MC code size
10274 const auto *D16Info = AMDGPU::getT16D16Helper(T16Op: Opc);
10275 if (D16Info) {
10276 // Assume d16_lo/hi inst are always in same size
10277 unsigned LoInstOpcode = D16Info->LoOp;
10278 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: LoInstOpcode);
10279 DescSize = Desc.getSize();
10280 }
10281
10282 // If FMA Pseudo inst, get correct MC code size
10283 if (Opc == AMDGPU::V_FMA_MIX_F16_t16 || Opc == AMDGPU::V_FMA_MIX_BF16_t16) {
10284 // All potential lowerings are the same size; arbitrarily pick one.
10285 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: AMDGPU::V_FMA_MIXLO_F16);
10286 DescSize = Desc.getSize();
10287 }
10288
10289 return DescSize;
10290 }
10291}
10292
10293TargetInstrInfo::InstSizeVerifyMode
10294SIInstrInfo::getInstSizeVerifyMode(const MachineInstr &MI) const {
10295 if (MI.isBranch() && ST.hasOffset3fBug())
10296 return InstSizeVerifyMode::NoVerify;
10297 return InstSizeVerifyMode::ExactSize;
10298}
10299
10300bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const {
10301 if (!isFLAT(MI))
10302 return false;
10303
10304 if (MI.memoperands_empty())
10305 return true;
10306
10307 for (const MachineMemOperand *MMO : MI.memoperands()) {
10308 if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)
10309 return true;
10310 }
10311 return false;
10312}
10313
10314ArrayRef<std::pair<int, const char *>>
10315SIInstrInfo::getSerializableTargetIndices() const {
10316 static const std::pair<int, const char *> TargetIndices[] = {
10317 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
10318 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
10319 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
10320 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
10321 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
10322 return ArrayRef(TargetIndices);
10323}
10324
10325/// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The
10326/// post-RA version of misched uses CreateTargetMIHazardRecognizer.
10327ScheduleHazardRecognizer *
10328SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
10329 const ScheduleDAG *DAG) const {
10330 return new GCNHazardRecognizer(DAG->MF);
10331}
10332
10333/// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer
10334/// pass.
10335ScheduleHazardRecognizer *
10336SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF,
10337 MachineLoopInfo *MLI) const {
10338 return new GCNHazardRecognizer(
10339 MF, GCNHazardRecognizer::OperatingMode::HazardRecognizerMode, MLI);
10340}
10341
10342// Called during:
10343// - pre-RA scheduling and post-RA scheduling
10344ScheduleHazardRecognizer *
10345SIInstrInfo::CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
10346 const ScheduleDAGMI *DAG) const {
10347 // Borrowed from Arm Target
10348 // We would like to restrict this hazard recognizer to only
10349 // post-RA scheduling; we can tell that we're post-RA because we don't
10350 // track VRegLiveness.
10351 if (!DAG->hasVRegLiveness())
10352 return new GCNHazardRecognizer(DAG->MF);
10353 return TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG);
10354}
10355
10356std::pair<unsigned, unsigned>
10357SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
10358 return std::pair(TF & MO_MASK, TF & ~MO_MASK);
10359}
10360
10361ArrayRef<std::pair<unsigned, const char *>>
10362SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
10363 static const std::pair<unsigned, const char *> TargetFlags[] = {
10364 {MO_GOTPCREL, "amdgpu-gotprel"},
10365 {MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo"},
10366 {MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi"},
10367 {MO_GOTPCREL64, "amdgpu-gotprel64"},
10368 {MO_REL32_LO, "amdgpu-rel32-lo"},
10369 {MO_REL32_HI, "amdgpu-rel32-hi"},
10370 {MO_REL64, "amdgpu-rel64"},
10371 {MO_ABS32_LO, "amdgpu-abs32-lo"},
10372 {MO_ABS32_HI, "amdgpu-abs32-hi"},
10373 {MO_ABS64, "amdgpu-abs64"},
10374 };
10375
10376 return ArrayRef(TargetFlags);
10377}
10378
10379ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
10380SIInstrInfo::getSerializableMachineMemOperandTargetFlags() const {
10381 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
10382 {
10383 {MONoClobber, "amdgpu-noclobber"},
10384 {MOLastUse, "amdgpu-last-use"},
10385 {MOCooperative, "amdgpu-cooperative"},
10386 {MOThreadPrivate, "amdgpu-thread-private"},
10387 };
10388
10389 return ArrayRef(TargetFlags);
10390}
10391
10392unsigned SIInstrInfo::getLiveRangeSplitOpcode(Register SrcReg,
10393 const MachineFunction &MF) const {
10394 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
10395 assert(SrcReg.isVirtual());
10396 if (MFI->checkFlag(Reg: SrcReg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
10397 return AMDGPU::WWM_COPY;
10398
10399 return AMDGPU::COPY;
10400}
10401
10402bool SIInstrInfo::canAddToBBProlog(const MachineInstr &MI) const {
10403 uint32_t Opcode = MI.getOpcode();
10404 // Check if it is SGPR spill or wwm-register spill Opcode.
10405 if (isSGPRSpill(Opcode) || isWWMRegSpillOpcode(Opcode))
10406 return true;
10407
10408 const MachineFunction *MF = MI.getMF();
10409 const MachineRegisterInfo &MRI = MF->getRegInfo();
10410 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
10411
10412 // See if this is Liverange split instruction inserted for SGPR or
10413 // wwm-register. The implicit def inserted for wwm-registers should also be
10414 // included as they can appear at the bb begin.
10415 bool IsLRSplitInst = MI.getFlag(Flag: MachineInstr::LRSplit);
10416 if (!IsLRSplitInst && Opcode != AMDGPU::IMPLICIT_DEF)
10417 return false;
10418
10419 Register Reg = MI.getOperand(i: 0).getReg();
10420 if (RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg)))
10421 return IsLRSplitInst;
10422
10423 return MFI->isWWMReg(Reg);
10424}
10425
10426bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI,
10427 Register Reg) const {
10428 // We need to handle instructions which may be inserted during register
10429 // allocation to handle the prolog. The initial prolog instruction may have
10430 // been separated from the start of the block by spills and copies inserted
10431 // needed by the prolog. However, the insertions for scalar registers can
10432 // always be placed at the BB top as they are independent of the exec mask
10433 // value.
10434 bool IsNullOrVectorRegister = true;
10435 if (Reg) {
10436 const MachineFunction *MF = MI.getMF();
10437 const MachineRegisterInfo &MRI = MF->getRegInfo();
10438 IsNullOrVectorRegister = !RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg));
10439 }
10440
10441 return IsNullOrVectorRegister &&
10442 (canAddToBBProlog(MI) ||
10443 (!MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY &&
10444 MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &RI)));
10445}
10446
10447MachineInstrBuilder
10448SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
10449 MachineBasicBlock::iterator I,
10450 const DebugLoc &DL,
10451 Register DestReg) const {
10452 if (ST.hasAddNoCarryInsts())
10453 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e64), DestReg);
10454
10455 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
10456 Register UnusedCarry = MRI.createVirtualRegister(RegClass: RI.getBoolRC());
10457 MRI.setRegAllocationHint(VReg: UnusedCarry, Type: 0, PrefReg: RI.getVCC());
10458
10459 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg)
10460 .addReg(RegNo: UnusedCarry, Flags: RegState::Define | RegState::Dead);
10461}
10462
10463MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
10464 MachineBasicBlock::iterator I,
10465 const DebugLoc &DL,
10466 Register DestReg,
10467 RegScavenger &RS) const {
10468 if (ST.hasAddNoCarryInsts())
10469 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg);
10470
10471 // If available, prefer to use vcc.
10472 Register UnusedCarry = !RS.isRegUsed(Reg: AMDGPU::VCC)
10473 ? Register(RI.getVCC())
10474 : RS.scavengeRegisterBackwards(
10475 RC: *RI.getBoolRC(), To: I, /* RestoreAfter */ false,
10476 SPAdj: 0, /* AllowSpill */ false);
10477
10478 // TODO: Users need to deal with this.
10479 if (!UnusedCarry.isValid())
10480 return MachineInstrBuilder();
10481
10482 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg)
10483 .addReg(RegNo: UnusedCarry, Flags: RegState::Define | RegState::Dead);
10484}
10485
10486bool SIInstrInfo::isKillTerminator(unsigned Opcode) {
10487 switch (Opcode) {
10488 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
10489 case AMDGPU::SI_KILL_I1_TERMINATOR:
10490 return true;
10491 default:
10492 return false;
10493 }
10494}
10495
10496const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const {
10497 switch (Opcode) {
10498 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
10499 return get(Opcode: AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);
10500 case AMDGPU::SI_KILL_I1_PSEUDO:
10501 return get(Opcode: AMDGPU::SI_KILL_I1_TERMINATOR);
10502 default:
10503 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO");
10504 }
10505}
10506
10507bool SIInstrInfo::isLegalMUBUFImmOffset(unsigned Imm) const {
10508 return Imm <= getMaxMUBUFImmOffset(ST);
10509}
10510
10511unsigned SIInstrInfo::getMaxMUBUFImmOffset(const GCNSubtarget &ST) {
10512 // GFX12 field is non-negative 24-bit signed byte offset.
10513 const unsigned OffsetBits =
10514 ST.getGeneration() >= AMDGPUSubtarget::GFX12 ? 23 : 12;
10515 return (1 << OffsetBits) - 1;
10516}
10517
10518void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const {
10519 if (!ST.isWave32())
10520 return;
10521
10522 if (MI.isInlineAsm())
10523 return;
10524
10525 if (MI.getNumOperands() < MI.getNumExplicitOperands())
10526 return;
10527
10528 for (auto &Op : MI.implicit_operands()) {
10529 if (Op.isReg() && Op.getReg() == AMDGPU::VCC)
10530 Op.setReg(AMDGPU::VCC_LO);
10531 }
10532}
10533
10534bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const {
10535 if (!isSMRD(MI))
10536 return false;
10537
10538 // Check that it is using a buffer resource.
10539 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::sbase);
10540 if (Idx == -1) // e.g. s_memtime
10541 return false;
10542
10543 const int16_t RCID = getOpRegClassID(OpInfo: MI.getDesc().operands()[Idx]);
10544 return RI.getRegClass(i: RCID)->hasSubClassEq(RC: &AMDGPU::SGPR_128RegClass);
10545}
10546
10547// Given Imm, split it into the values to put into the SOffset and ImmOffset
10548// fields in an MUBUF instruction. Return false if it is not possible (due to a
10549// hardware bug needing a workaround).
10550//
10551// The required alignment ensures that individual address components remain
10552// aligned if they are aligned to begin with. It also ensures that additional
10553// offsets within the given alignment can be added to the resulting ImmOffset.
10554bool SIInstrInfo::splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset,
10555 uint32_t &ImmOffset, Align Alignment) const {
10556 const uint64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST);
10557 const uint32_t MaxImm = alignDown(Value: MaxOffset, Align: Alignment.value());
10558 uint32_t Overflow = 0;
10559
10560 if (Imm > MaxImm) {
10561 if (Imm <= MaxImm + 64) {
10562 // Use an SOffset inline constant for 4..64
10563 Overflow = Imm - MaxImm;
10564 Imm = MaxImm;
10565 } else {
10566 // Try to keep the same value in SOffset for adjacent loads, so that
10567 // the corresponding register contents can be re-used.
10568 //
10569 // Load values with all low-bits (except for alignment bits) set into
10570 // SOffset, so that a larger range of values can be covered using
10571 // s_movk_i32.
10572 //
10573 // Atomic operations fail to work correctly when individual address
10574 // components are unaligned, even if their sum is aligned.
10575 uint32_t High = (Imm + Alignment.value()) & ~MaxOffset;
10576 uint32_t Low = (Imm + Alignment.value()) & MaxOffset;
10577 Imm = Low;
10578 Overflow = High - Alignment.value();
10579 }
10580 }
10581
10582 if (Overflow > 0) {
10583 // There is a hardware bug in SI and CI which prevents address clamping in
10584 // MUBUF instructions from working correctly with SOffsets. The immediate
10585 // offset is unaffected.
10586 if (ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
10587 return false;
10588
10589 // It is not possible to set immediate in SOffset field on some targets.
10590 if (ST.hasRestrictedSOffset())
10591 return false;
10592 }
10593
10594 ImmOffset = Imm;
10595 SOffset = Overflow;
10596 return true;
10597}
10598
10599// Depending on the used address space and instructions, some immediate offsets
10600// are allowed and some are not.
10601// Pre-GFX12, flat instruction offsets can only be non-negative, global and
10602// scratch instruction offsets can also be negative. On GFX12, offsets can be
10603// negative for all variants.
10604//
10605// There are several bugs related to these offsets:
10606// On gfx10.1, flat instructions that go into the global address space cannot
10607// use an offset.
10608//
10609// For scratch instructions, the address can be either an SGPR or a VGPR.
10610// The following offsets can be used, depending on the architecture (x means
10611// cannot be used):
10612// +----------------------------+------+------+
10613// | Address-Mode | SGPR | VGPR |
10614// +----------------------------+------+------+
10615// | gfx9 | | |
10616// | negative, 4-aligned offset | x | ok |
10617// | negative, unaligned offset | x | ok |
10618// +----------------------------+------+------+
10619// | gfx10 | | |
10620// | negative, 4-aligned offset | ok | ok |
10621// | negative, unaligned offset | ok | x |
10622// +----------------------------+------+------+
10623// | gfx10.3 | | |
10624// | negative, 4-aligned offset | ok | ok |
10625// | negative, unaligned offset | ok | ok |
10626// +----------------------------+------+------+
10627//
10628// This function ignores the addressing mode, so if an offset cannot be used in
10629// one addressing mode, it is considered illegal.
10630bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
10631 AMDGPU::FlatAddrSpace FlatVariant) const {
10632 // TODO: Should 0 be special cased?
10633 if (!ST.hasFlatInstOffsets())
10634 return false;
10635
10636 using AMDGPU::FlatAddrSpace;
10637 if (ST.hasFlatSegmentOffsetBug() && FlatVariant == FlatAddrSpace::FLAT &&
10638 (AddrSpace == AMDGPUAS::FLAT_ADDRESS ||
10639 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS))
10640 return false;
10641
10642 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
10643 FlatVariant == FlatAddrSpace::FlatScratch && Offset < 0 &&
10644 (Offset % 4) != 0) {
10645 return false;
10646 }
10647
10648 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
10649 unsigned N = AMDGPU::getNumFlatOffsetBits(ST);
10650 return isIntN(N, x: Offset) && (AllowNegative || Offset >= 0);
10651}
10652
10653// See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not.
10654std::pair<int64_t, int64_t>
10655SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,
10656 AMDGPU::FlatAddrSpace FlatVariant) const {
10657 int64_t RemainderOffset = COffsetVal;
10658 int64_t ImmField = 0;
10659
10660 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
10661 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST) - 1;
10662
10663 if (AllowNegative) {
10664 // Use signed division by a power of two to truncate towards 0.
10665 int64_t D = 1LL << NumBits;
10666 RemainderOffset = (COffsetVal / D) * D;
10667 ImmField = COffsetVal - RemainderOffset;
10668
10669 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
10670 FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch && ImmField < 0 &&
10671 (ImmField % 4) != 0) {
10672 // Make ImmField a multiple of 4
10673 RemainderOffset += ImmField % 4;
10674 ImmField -= ImmField % 4;
10675 }
10676 } else if (COffsetVal >= 0) {
10677 ImmField = COffsetVal & maskTrailingOnes<uint64_t>(N: NumBits);
10678 RemainderOffset = COffsetVal - ImmField;
10679 }
10680
10681 assert(isLegalFLATOffset(ImmField, AddrSpace, FlatVariant));
10682 assert(RemainderOffset + ImmField == COffsetVal);
10683 return {ImmField, RemainderOffset};
10684}
10685
10686bool SIInstrInfo::allowNegativeFlatOffset(
10687 AMDGPU::FlatAddrSpace FlatVariant) const {
10688 if (ST.hasNegativeScratchOffsetBug() &&
10689 FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch)
10690 return false;
10691
10692 return FlatVariant != AMDGPU::FlatAddrSpace::FLAT || AMDGPU::isGFX12Plus(STI: ST);
10693}
10694
10695static unsigned subtargetEncodingFamily(const GCNSubtarget &ST) {
10696 switch (ST.getGeneration()) {
10697 default:
10698 break;
10699 case AMDGPUSubtarget::SOUTHERN_ISLANDS:
10700 case AMDGPUSubtarget::SEA_ISLANDS:
10701 return SIEncodingFamily::SI;
10702 case AMDGPUSubtarget::VOLCANIC_ISLANDS:
10703 case AMDGPUSubtarget::GFX9:
10704 return SIEncodingFamily::VI;
10705 case AMDGPUSubtarget::GFX10:
10706 return SIEncodingFamily::GFX10;
10707 case AMDGPUSubtarget::GFX11:
10708 return ST.hasGFX11_7Insts() ? SIEncodingFamily::GFX1170
10709 : SIEncodingFamily::GFX11;
10710 case AMDGPUSubtarget::GFX12:
10711 return ST.hasGFX1250Insts() ? SIEncodingFamily::GFX1250
10712 : SIEncodingFamily::GFX12;
10713 case AMDGPUSubtarget::GFX13:
10714 return SIEncodingFamily::GFX13;
10715 }
10716 llvm_unreachable("Unknown subtarget generation!");
10717}
10718
10719bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const {
10720 switch(MCOp) {
10721 // These opcodes use indirect register addressing so
10722 // they need special handling by codegen (currently missing).
10723 // Therefore it is too risky to allow these opcodes
10724 // to be selected by dpp combiner or sdwa peepholer.
10725 case AMDGPU::V_MOVRELS_B32_dpp_gfx10:
10726 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
10727 case AMDGPU::V_MOVRELD_B32_dpp_gfx10:
10728 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10:
10729 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10:
10730 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
10731 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10:
10732 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
10733 return true;
10734 default:
10735 return false;
10736 }
10737}
10738
10739#define GENERATE_RENAMED_GFX9_CASES(OPCODE) \
10740 case OPCODE##_dpp: \
10741 case OPCODE##_e32: \
10742 case OPCODE##_e64: \
10743 case OPCODE##_e64_dpp: \
10744 case OPCODE##_sdwa:
10745
10746static bool isRenamedInGFX9(int Opcode) {
10747 switch (Opcode) {
10748 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADDC_U32)
10749 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_CO_U32)
10750 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_U32)
10751 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBBREV_U32)
10752 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBB_U32)
10753 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_CO_U32)
10754 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_U32)
10755 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_CO_U32)
10756 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_U32)
10757 //
10758 case AMDGPU::V_DIV_FIXUP_F16_gfx9_e64:
10759 case AMDGPU::V_DIV_FIXUP_F16_gfx9_fake16_e64:
10760 case AMDGPU::V_FMA_F16_gfx9_e64:
10761 case AMDGPU::V_FMA_F16_gfx9_fake16_e64:
10762 case AMDGPU::V_INTERP_P2_F16:
10763 case AMDGPU::V_MAD_F16_e64:
10764 case AMDGPU::V_MAD_U16_e64:
10765 case AMDGPU::V_MAD_I16_e64:
10766 return true;
10767 default:
10768 return false;
10769 }
10770}
10771
10772int SIInstrInfo::pseudoToMCOpcode(int Opcode) const {
10773 assert(Opcode == (int)SIInstrInfo::getNonSoftWaitcntOpcode(Opcode) &&
10774 "SIInsertWaitcnts should have promoted soft waitcnt instructions!");
10775
10776 unsigned Gen = subtargetEncodingFamily(ST);
10777
10778 if (ST.getGeneration() == AMDGPUSubtarget::GFX9 && isRenamedInGFX9(Opcode))
10779 Gen = SIEncodingFamily::GFX9;
10780
10781 // Adjust the encoding family to GFX80 for D16 buffer instructions when the
10782 // subtarget has UnpackedD16VMem feature.
10783 // TODO: remove this when we discard GFX80 encoding.
10784 if (ST.hasUnpackedD16VMem() && SIInstrFlags::isD16Buf(O: get(Opcode)))
10785 Gen = SIEncodingFamily::GFX80;
10786
10787 if (SIInstrFlags::isSDWA(O: get(Opcode))) {
10788 switch (ST.getGeneration()) {
10789 default:
10790 Gen = SIEncodingFamily::SDWA;
10791 break;
10792 case AMDGPUSubtarget::GFX9:
10793 Gen = SIEncodingFamily::SDWA9;
10794 break;
10795 case AMDGPUSubtarget::GFX10:
10796 Gen = SIEncodingFamily::SDWA10;
10797 break;
10798 }
10799 }
10800
10801 if (isMAI(Opcode)) {
10802 int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode);
10803 if (MFMAOp != -1)
10804 Opcode = MFMAOp;
10805 }
10806
10807 int32_t MCOp = AMDGPU::getMCOpcode(Opcode, Gen);
10808
10809 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && ST.hasGFX11_7Insts())
10810 MCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX11);
10811
10812 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && ST.hasGFX1250Insts())
10813 MCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX12);
10814
10815 // -1 means that Opcode is already a native instruction.
10816 if (MCOp == -1)
10817 return Opcode;
10818
10819 if (ST.hasGFX90AInsts()) {
10820 uint32_t NMCOp = AMDGPU::INSTRUCTION_LIST_END;
10821 if (ST.hasGFX940Insts())
10822 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX940);
10823 if (NMCOp == AMDGPU::INSTRUCTION_LIST_END)
10824 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX90A);
10825 if (NMCOp == AMDGPU::INSTRUCTION_LIST_END)
10826 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX9);
10827 if (NMCOp != AMDGPU::INSTRUCTION_LIST_END)
10828 MCOp = NMCOp;
10829 }
10830
10831 // INSTRUCTION_LIST_END means that Opcode is a pseudo instruction that has no
10832 // encoding in the given subtarget generation.
10833 if (MCOp == AMDGPU::INSTRUCTION_LIST_END)
10834 return -1;
10835
10836 if (isAsmOnlyOpcode(MCOp))
10837 return -1;
10838
10839 return MCOp;
10840}
10841
10842static
10843TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) {
10844 assert(RegOpnd.isReg());
10845 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() :
10846 getRegSubRegPair(O: RegOpnd);
10847}
10848
10849TargetInstrInfo::RegSubRegPair
10850llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) {
10851 assert(MI.isRegSequence());
10852 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I)
10853 if (MI.getOperand(i: 1 + 2 * I + 1).getImm() == SubReg) {
10854 auto &RegOp = MI.getOperand(i: 1 + 2 * I);
10855 return getRegOrUndef(RegOpnd: RegOp);
10856 }
10857 return TargetInstrInfo::RegSubRegPair();
10858}
10859
10860// Try to find the definition of reg:subreg in subreg-manipulation pseudos
10861// Following a subreg of reg:subreg isn't supported
10862static bool followSubRegDef(MachineInstr &MI,
10863 TargetInstrInfo::RegSubRegPair &RSR) {
10864 if (!RSR.SubReg)
10865 return false;
10866 switch (MI.getOpcode()) {
10867 default: break;
10868 case AMDGPU::REG_SEQUENCE:
10869 RSR = getRegSequenceSubReg(MI, SubReg: RSR.SubReg);
10870 return true;
10871 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg
10872 case AMDGPU::INSERT_SUBREG:
10873 if (RSR.SubReg == (unsigned)MI.getOperand(i: 3).getImm())
10874 // inserted the subreg we're looking for
10875 RSR = getRegOrUndef(RegOpnd: MI.getOperand(i: 2));
10876 else { // the subreg in the rest of the reg
10877 auto R1 = getRegOrUndef(RegOpnd: MI.getOperand(i: 1));
10878 if (R1.SubReg) // subreg of subreg isn't supported
10879 return false;
10880 RSR.Reg = R1.Reg;
10881 }
10882 return true;
10883 }
10884 return false;
10885}
10886
10887MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
10888 const MachineRegisterInfo &MRI) {
10889 assert(MRI.isSSA());
10890 if (!P.Reg.isVirtual())
10891 return nullptr;
10892
10893 auto RSR = P;
10894 auto *DefInst = MRI.getVRegDef(Reg: RSR.Reg);
10895 while (auto *MI = DefInst) {
10896 DefInst = nullptr;
10897 switch (MI->getOpcode()) {
10898 case AMDGPU::COPY:
10899 case AMDGPU::V_MOV_B32_e32: {
10900 auto &Op1 = MI->getOperand(i: 1);
10901 if (Op1.isReg() && Op1.getReg().isVirtual()) {
10902 if (Op1.isUndef())
10903 return nullptr;
10904 RSR = getRegSubRegPair(O: Op1);
10905 DefInst = MRI.getVRegDef(Reg: RSR.Reg);
10906 }
10907 break;
10908 }
10909 default:
10910 if (followSubRegDef(MI&: *MI, RSR)) {
10911 if (!RSR.Reg)
10912 return nullptr;
10913 DefInst = MRI.getVRegDef(Reg: RSR.Reg);
10914 }
10915 }
10916 if (!DefInst)
10917 return MI;
10918 }
10919 return nullptr;
10920}
10921
10922bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
10923 Register VReg,
10924 const MachineInstr &DefMI,
10925 const MachineInstr &UseMI) {
10926 assert(MRI.isSSA() && "Must be run on SSA");
10927
10928 auto *TRI = MRI.getTargetRegisterInfo();
10929 auto *DefBB = DefMI.getParent();
10930
10931 // Don't bother searching between blocks, although it is possible this block
10932 // doesn't modify exec.
10933 if (UseMI.getParent() != DefBB)
10934 return true;
10935
10936 const int MaxInstScan = 20;
10937 int NumInst = 0;
10938
10939 // Stop scan at the use.
10940 auto E = UseMI.getIterator();
10941 for (auto I = std::next(x: DefMI.getIterator()); I != E; ++I) {
10942 if (I->isDebugInstr())
10943 continue;
10944
10945 if (++NumInst > MaxInstScan)
10946 return true;
10947
10948 if (I->modifiesRegister(Reg: AMDGPU::EXEC, TRI))
10949 return true;
10950 }
10951
10952 return false;
10953}
10954
10955bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
10956 Register VReg,
10957 const MachineInstr &DefMI) {
10958 assert(MRI.isSSA() && "Must be run on SSA");
10959
10960 auto *TRI = MRI.getTargetRegisterInfo();
10961 auto *DefBB = DefMI.getParent();
10962
10963 const int MaxUseScan = 10;
10964 int NumUse = 0;
10965
10966 for (auto &Use : MRI.use_nodbg_operands(Reg: VReg)) {
10967 auto &UseInst = *Use.getParent();
10968 // Don't bother searching between blocks, although it is possible this block
10969 // doesn't modify exec.
10970 if (UseInst.getParent() != DefBB || UseInst.isPHI())
10971 return true;
10972
10973 if (++NumUse > MaxUseScan)
10974 return true;
10975 }
10976
10977 if (NumUse == 0)
10978 return false;
10979
10980 const int MaxInstScan = 20;
10981 int NumInst = 0;
10982
10983 // Stop scan when we have seen all the uses.
10984 for (auto I = std::next(x: DefMI.getIterator()); ; ++I) {
10985 assert(I != DefBB->end());
10986
10987 if (I->isDebugInstr())
10988 continue;
10989
10990 if (++NumInst > MaxInstScan)
10991 return true;
10992
10993 for (const MachineOperand &Op : I->operands()) {
10994 // We don't check reg masks here as they're used only on calls:
10995 // 1. EXEC is only considered const within one BB
10996 // 2. Call should be a terminator instruction if present in a BB
10997
10998 if (!Op.isReg())
10999 continue;
11000
11001 Register Reg = Op.getReg();
11002 if (Op.isUse()) {
11003 if (Reg == VReg && --NumUse == 0)
11004 return false;
11005 } else if (TRI->regsOverlap(RegA: Reg, RegB: AMDGPU::EXEC))
11006 return true;
11007 }
11008 }
11009}
11010
11011MachineInstr *SIInstrInfo::createPHIDestinationCopy(
11012 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt,
11013 const DebugLoc &DL, Register Src, Register Dst) const {
11014 auto Cur = MBB.begin();
11015 if (Cur != MBB.end())
11016 do {
11017 if (!Cur->isPHI() && Cur->readsRegister(Reg: Dst, /*TRI=*/nullptr))
11018 return BuildMI(BB&: MBB, I: Cur, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Dst).addReg(RegNo: Src);
11019 ++Cur;
11020 } while (Cur != MBB.end() && Cur != LastPHIIt);
11021
11022 return TargetInstrInfo::createPHIDestinationCopy(MBB, InsPt: LastPHIIt, DL, Src,
11023 Dst);
11024}
11025
11026MachineInstr *SIInstrInfo::createPHISourceCopy(
11027 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt,
11028 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const {
11029 if (InsPt != MBB.end() &&
11030 (InsPt->getOpcode() == AMDGPU::SI_IF ||
11031 InsPt->getOpcode() == AMDGPU::SI_ELSE ||
11032 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) &&
11033 InsPt->definesRegister(Reg: Src, /*TRI=*/nullptr)) {
11034 InsPt++;
11035 return BuildMI(BB&: MBB, I: InsPt, MIMD: DL,
11036 MCID: get(Opcode: AMDGPU::LaneMaskConstants::get(ST).MovTermOpc), DestReg: Dst)
11037 .addReg(RegNo: Src, Flags: {}, SubReg: SrcSubReg)
11038 .addReg(RegNo: AMDGPU::EXEC, Flags: RegState::Implicit);
11039 }
11040 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg,
11041 Dst);
11042}
11043
11044bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); }
11045
11046bool SIInstrInfo::hasRAWDependency(const MachineInstr &FirstMI,
11047 const MachineInstr &SecondMI) const {
11048 for (const auto &Use : SecondMI.all_uses()) {
11049 if (Use.isReg() && FirstMI.modifiesRegister(Reg: Use.getReg(), TRI: &RI))
11050 return true;
11051 }
11052 return false;
11053}
11054
11055/// If OpX is multicycle, anti-dependencies are not allowed.
11056/// isDPMACCInstruction was not designed for VOPD, but it is fit for the
11057/// purpose.
11058bool llvm::SIInstrInfo::isVOPDAntidependencyAllowed(
11059 const MachineInstr &OpX) const {
11060 return !AMDGPU::isDPMACCInstruction(Opc: OpX.getOpcode());
11061}
11062
11063MachineInstr *
11064SIInstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
11065 ArrayRef<unsigned> Ops, int FrameIndex,
11066 MachineInstr *&CopyMI, LiveIntervals *LIS,
11067 VirtRegMap *VRM) const {
11068 // This is a bit of a hack (copied from AArch64). Consider this instruction:
11069 //
11070 // %0:sreg_32 = COPY $m0
11071 //
11072 // We explicitly chose SReg_32 for the virtual register so such a copy might
11073 // be eliminated by RegisterCoalescer. However, that may not be possible, and
11074 // %0 may even spill. We can't spill $m0 normally (it would require copying to
11075 // a numbered SGPR anyway), and since it is in the SReg_32 register class,
11076 // TargetInstrInfo::foldMemoryOperand() is going to try.
11077 // A similar issue also exists with spilling and reloading $exec registers.
11078 //
11079 // To prevent that, constrain the %0 register class here.
11080 if (isFullCopyInstr(MI)) {
11081 Register DstReg = MI.getOperand(i: 0).getReg();
11082 Register SrcReg = MI.getOperand(i: 1).getReg();
11083 if ((DstReg.isVirtual() || SrcReg.isVirtual()) &&
11084 (DstReg.isVirtual() != SrcReg.isVirtual())) {
11085 MachineRegisterInfo &MRI = MF.getRegInfo();
11086 Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg;
11087 const TargetRegisterClass *RC = MRI.getRegClass(Reg: VirtReg);
11088 if (RC->hasSuperClassEq(RC: &AMDGPU::SReg_32RegClass)) {
11089 MRI.constrainRegClass(Reg: VirtReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
11090 return nullptr;
11091 }
11092 if (RC->hasSuperClassEq(RC: &AMDGPU::SReg_64RegClass)) {
11093 MRI.constrainRegClass(Reg: VirtReg, RC: &AMDGPU::SReg_64_XEXECRegClass);
11094 return nullptr;
11095 }
11096 }
11097 }
11098
11099 return nullptr;
11100}
11101
11102unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
11103 const MachineInstr &MI,
11104 unsigned *PredCost) const {
11105 if (MI.isBundle()) {
11106 MachineBasicBlock::const_instr_iterator I(MI.getIterator());
11107 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end());
11108 unsigned Lat = 0, Count = 0;
11109 for (++I; I != E && I->isBundledWithPred(); ++I) {
11110 ++Count;
11111 Lat = std::max(a: Lat, b: SchedModel.computeInstrLatency(MI: &*I));
11112 }
11113 return Lat + Count - 1;
11114 }
11115
11116 return SchedModel.computeInstrLatency(MI: &MI);
11117}
11118
11119const MachineOperand &
11120SIInstrInfo::getCalleeOperand(const MachineInstr &MI) const {
11121 if (const MachineOperand *CallAddrOp =
11122 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0))
11123 return *CallAddrOp;
11124 return TargetInstrInfo::getCalleeOperand(MI);
11125}
11126
11127ValueUniformity
11128SIInstrInfo::getGenericValueUniformity(const MachineInstr &MI) const {
11129 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
11130 unsigned Opcode = MI.getOpcode();
11131
11132 auto HandleAddrSpaceCast = [this, &MRI](const MachineInstr &MI) {
11133 Register Dst = MI.getOperand(i: 0).getReg();
11134 Register Src = isa<GIntrinsic>(Val: MI) ? MI.getOperand(i: 2).getReg()
11135 : MI.getOperand(i: 1).getReg();
11136 LLT DstTy = MRI.getType(Reg: Dst);
11137 LLT SrcTy = MRI.getType(Reg: Src);
11138 unsigned DstAS = DstTy.getAddressSpace();
11139 unsigned SrcAS = SrcTy.getAddressSpace();
11140 return SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
11141 DstAS == AMDGPUAS::FLAT_ADDRESS &&
11142 ST.hasGloballyAddressableScratch()
11143 ? ValueUniformity::NeverUniform
11144 : ValueUniformity::Default;
11145 };
11146
11147 // If the target supports globally addressable scratch, the mapping from
11148 // scratch memory to the flat aperture changes therefore an address space cast
11149 // is no longer uniform.
11150 if (Opcode == TargetOpcode::G_ADDRSPACE_CAST)
11151 return HandleAddrSpaceCast(MI);
11152
11153 if (auto *GI = dyn_cast<GIntrinsic>(Val: &MI)) {
11154 auto IID = GI->getIntrinsicID();
11155 if (AMDGPU::isIntrinsicSourceOfDivergence(IntrID: IID))
11156 return ValueUniformity::NeverUniform;
11157 if (AMDGPU::isIntrinsicAlwaysUniform(IntrID: IID))
11158 return ValueUniformity::AlwaysUniform;
11159
11160 switch (IID) {
11161 case Intrinsic::amdgcn_addrspacecast_nonnull:
11162 return HandleAddrSpaceCast(MI);
11163 case Intrinsic::amdgcn_if:
11164 case Intrinsic::amdgcn_else:
11165 // FIXME: Uniform if second result
11166 break;
11167 }
11168
11169 return ValueUniformity::Default;
11170 }
11171
11172 // Loads from the private and flat address spaces are divergent, because
11173 // threads can execute the load instruction with the same inputs and get
11174 // different results.
11175 //
11176 // All other loads are not divergent, because if threads issue loads with the
11177 // same arguments, they will always get the same result.
11178 if (Opcode == AMDGPU::G_LOAD || Opcode == AMDGPU::G_ZEXTLOAD ||
11179 Opcode == AMDGPU::G_SEXTLOAD) {
11180 if (MI.memoperands_empty())
11181 return ValueUniformity::NeverUniform; // conservative assumption
11182
11183 if (llvm::any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *mmo) {
11184 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
11185 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
11186 })) {
11187 // At least one MMO in a non-global address space.
11188 return ValueUniformity::NeverUniform;
11189 }
11190 return ValueUniformity::Default;
11191 }
11192
11193 if (SIInstrInfo::isGenericAtomicRMWOpcode(Opc: Opcode) ||
11194 Opcode == AMDGPU::G_ATOMIC_CMPXCHG ||
11195 Opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS ||
11196 AMDGPU::isGenericAtomic(Opc: Opcode)) {
11197 return ValueUniformity::NeverUniform;
11198 }
11199
11200 // Result is computed from uniform SP and uniform wave-wide max size.
11201 if (Opcode == TargetOpcode::G_DYN_STACKALLOC)
11202 return ValueUniformity::AlwaysUniform;
11203
11204 if (Opcode == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
11205 return ValueUniformity::NeverUniform;
11206
11207 return ValueUniformity::Default;
11208}
11209
11210const MIRFormatter *SIInstrInfo::getMIRFormatter() const {
11211 if (!Formatter)
11212 Formatter = std::make_unique<AMDGPUMIRFormatter>(args: ST);
11213 return Formatter.get();
11214}
11215
11216ValueUniformity SIInstrInfo::getValueUniformity(const MachineInstr &MI) const {
11217
11218 if (isNeverUniform(MI))
11219 return ValueUniformity::NeverUniform;
11220
11221 unsigned opcode = MI.getOpcode();
11222 if (opcode == AMDGPU::V_READLANE_B32 ||
11223 opcode == AMDGPU::V_READFIRSTLANE_B32 ||
11224 opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR)
11225 return ValueUniformity::AlwaysUniform;
11226
11227 // If any of defs is divergent, report as NeverUniform. isUniformReg will
11228 // calculate in more detail for each def from its reg class, if available.
11229 if (MI.isInlineAsm()) {
11230 for (const MachineOperand &MO : MI.operands()) {
11231 if (!MO.isReg() || !MO.isDef())
11232 continue;
11233 const TargetRegisterClass *RC =
11234 MI.getRegClassConstraint(OpIdx: MO.getOperandNo(), TII: this, TRI: &RI);
11235 if (!RC || !RI.isSGPRClass(RC))
11236 return ValueUniformity::NeverUniform;
11237 }
11238 }
11239
11240 if (isCopyInstr(MI)) {
11241 const MachineOperand &srcOp = MI.getOperand(i: 1);
11242 if (srcOp.isReg() && srcOp.getReg().isPhysical()) {
11243 const TargetRegisterClass *regClass =
11244 RI.getPhysRegBaseClass(Reg: srcOp.getReg());
11245 return RI.isSGPRClass(RC: regClass) ? ValueUniformity::AlwaysUniform
11246 : ValueUniformity::NeverUniform;
11247 }
11248 return ValueUniformity::Default;
11249 }
11250
11251 // GMIR handling
11252 if (MI.isPreISelOpcode())
11253 return SIInstrInfo::getGenericValueUniformity(MI);
11254
11255 // Atomics are divergent because they are executed sequentially: when an
11256 // atomic operation refers to the same address in each thread, then each
11257 // thread after the first sees the value written by the previous thread as
11258 // original value.
11259
11260 if (isAtomic(MI))
11261 return ValueUniformity::NeverUniform;
11262
11263 // Loads from the private and flat address spaces are divergent, because
11264 // threads can execute the load instruction with the same inputs and get
11265 // different results.
11266 if (isFLAT(MI) && MI.mayLoad()) {
11267 if (MI.memoperands_empty())
11268 return ValueUniformity::NeverUniform; // conservative assumption
11269
11270 if (llvm::any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *mmo) {
11271 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
11272 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
11273 })) {
11274 // At least one MMO in a non-global address space.
11275 return ValueUniformity::NeverUniform;
11276 }
11277
11278 return ValueUniformity::Default;
11279 }
11280
11281 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
11282 const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo();
11283
11284 // FIXME: It's conceptually broken to report this for an instruction, and not
11285 // a specific def operand. For inline asm in particular, there could be mixed
11286 // uniform and divergent results.
11287 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
11288 const MachineOperand &SrcOp = MI.getOperand(i: I);
11289 if (!SrcOp.isReg())
11290 continue;
11291
11292 Register Reg = SrcOp.getReg();
11293 if (!Reg || !SrcOp.readsReg())
11294 continue;
11295
11296 // If RegBank is null, this is unassigned or an unallocatable special
11297 // register, which are all scalars.
11298 const RegisterBank *RegBank = RBI->getRegBank(Reg, MRI, TRI: RI);
11299 if (RegBank && RegBank->getID() != AMDGPU::SGPRRegBankID)
11300 return ValueUniformity::NeverUniform;
11301 }
11302
11303 // TODO: Uniformity check condtions above can be rearranged for more
11304 // redability
11305
11306 // TODO: amdgcn.{ballot, [if]cmp} should be AlwaysUniform, but they are
11307 // currently turned into no-op COPYs by SelectionDAG ISel and are
11308 // therefore no longer recognizable.
11309
11310 return ValueUniformity::Default;
11311}
11312
11313unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) {
11314 switch (MF.getFunction().getCallingConv()) {
11315 case CallingConv::AMDGPU_PS:
11316 return 1;
11317 case CallingConv::AMDGPU_VS:
11318 return 2;
11319 case CallingConv::AMDGPU_GS:
11320 return 3;
11321 case CallingConv::AMDGPU_HS:
11322 case CallingConv::AMDGPU_LS:
11323 case CallingConv::AMDGPU_ES: {
11324 const Function &F = MF.getFunction();
11325 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
11326 F, "ds_ordered_count unsupported for this calling conv"));
11327 [[fallthrough]];
11328 }
11329 case CallingConv::AMDGPU_CS:
11330 case CallingConv::AMDGPU_KERNEL:
11331 case CallingConv::C:
11332 case CallingConv::Fast:
11333 default:
11334 // Assume other calling conventions are various compute callable functions
11335 return 0;
11336 }
11337}
11338
11339bool SIInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
11340 Register &SrcReg2, int64_t &CmpMask,
11341 int64_t &CmpValue) const {
11342 if (!MI.getOperand(i: 0).isReg() || MI.getOperand(i: 0).getSubReg())
11343 return false;
11344
11345 switch (MI.getOpcode()) {
11346 default:
11347 break;
11348 case AMDGPU::S_CMP_EQ_U32:
11349 case AMDGPU::S_CMP_EQ_I32:
11350 case AMDGPU::S_CMP_LG_U32:
11351 case AMDGPU::S_CMP_LG_I32:
11352 case AMDGPU::S_CMP_LT_U32:
11353 case AMDGPU::S_CMP_LT_I32:
11354 case AMDGPU::S_CMP_GT_U32:
11355 case AMDGPU::S_CMP_GT_I32:
11356 case AMDGPU::S_CMP_LE_U32:
11357 case AMDGPU::S_CMP_LE_I32:
11358 case AMDGPU::S_CMP_GE_U32:
11359 case AMDGPU::S_CMP_GE_I32:
11360 case AMDGPU::S_CMP_EQ_U64:
11361 case AMDGPU::S_CMP_LG_U64:
11362 SrcReg = MI.getOperand(i: 0).getReg();
11363 if (MI.getOperand(i: 1).isReg()) {
11364 if (MI.getOperand(i: 1).getSubReg())
11365 return false;
11366 SrcReg2 = MI.getOperand(i: 1).getReg();
11367 CmpValue = 0;
11368 } else if (MI.getOperand(i: 1).isImm()) {
11369 SrcReg2 = Register();
11370 CmpValue = MI.getOperand(i: 1).getImm();
11371 } else {
11372 return false;
11373 }
11374 CmpMask = ~0;
11375 return true;
11376 case AMDGPU::S_CMPK_EQ_U32:
11377 case AMDGPU::S_CMPK_EQ_I32:
11378 case AMDGPU::S_CMPK_LG_U32:
11379 case AMDGPU::S_CMPK_LG_I32:
11380 case AMDGPU::S_CMPK_LT_U32:
11381 case AMDGPU::S_CMPK_LT_I32:
11382 case AMDGPU::S_CMPK_GT_U32:
11383 case AMDGPU::S_CMPK_GT_I32:
11384 case AMDGPU::S_CMPK_LE_U32:
11385 case AMDGPU::S_CMPK_LE_I32:
11386 case AMDGPU::S_CMPK_GE_U32:
11387 case AMDGPU::S_CMPK_GE_I32:
11388 SrcReg = MI.getOperand(i: 0).getReg();
11389 SrcReg2 = Register();
11390 CmpValue = MI.getOperand(i: 1).getImm();
11391 CmpMask = ~0;
11392 return true;
11393 }
11394
11395 return false;
11396}
11397
11398static bool isSCCDeadOnExit(MachineBasicBlock *MBB) {
11399 for (MachineBasicBlock *S : MBB->successors()) {
11400 if (S->isLiveIn(Reg: AMDGPU::SCC))
11401 return false;
11402 }
11403 return true;
11404}
11405
11406// Invert all uses of SCC following SCCDef because SCCDef may be deleted and
11407// (incoming SCC) = !(SCC defined by SCCDef).
11408// Return true if all uses can be re-written, false otherwise.
11409bool SIInstrInfo::invertSCCUse(MachineInstr *SCCDef) const {
11410 MachineBasicBlock *MBB = SCCDef->getParent();
11411 SmallVector<MachineInstr *> InvertInstr;
11412 bool SCCIsDead = false;
11413
11414 // Scan instructions for SCC uses that need to be inverted until SCC is dead.
11415 constexpr unsigned ScanLimit = 12;
11416 unsigned Count = 0;
11417 for (MachineInstr &MI :
11418 make_range(x: std::next(x: MachineBasicBlock::iterator(SCCDef)), y: MBB->end())) {
11419 if (++Count > ScanLimit)
11420 return false;
11421 if (MI.readsRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
11422 if (MI.getOpcode() == AMDGPU::S_CSELECT_B32 ||
11423 MI.getOpcode() == AMDGPU::S_CSELECT_B64 ||
11424 MI.getOpcode() == AMDGPU::S_CBRANCH_SCC0 ||
11425 MI.getOpcode() == AMDGPU::S_CBRANCH_SCC1)
11426 InvertInstr.push_back(Elt: &MI);
11427 else
11428 return false;
11429 }
11430 if (MI.definesRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
11431 SCCIsDead = true;
11432 break;
11433 }
11434 }
11435 if (!SCCIsDead && isSCCDeadOnExit(MBB))
11436 SCCIsDead = true;
11437
11438 // SCC may have more uses. Can't invert all of them.
11439 if (!SCCIsDead)
11440 return false;
11441
11442 // Invert uses
11443 for (MachineInstr *MI : InvertInstr) {
11444 if (MI->getOpcode() == AMDGPU::S_CSELECT_B32 ||
11445 MI->getOpcode() == AMDGPU::S_CSELECT_B64) {
11446 swapOperands(Inst&: *MI);
11447 } else if (MI->getOpcode() == AMDGPU::S_CBRANCH_SCC0 ||
11448 MI->getOpcode() == AMDGPU::S_CBRANCH_SCC1) {
11449 MI->setDesc(get(Opcode: MI->getOpcode() == AMDGPU::S_CBRANCH_SCC0
11450 ? AMDGPU::S_CBRANCH_SCC1
11451 : AMDGPU::S_CBRANCH_SCC0));
11452 } else {
11453 llvm_unreachable("SCC used but no inversion handling");
11454 }
11455 }
11456 return true;
11457}
11458
11459// SCC is already valid after SCCValid.
11460// SCCRedefine will redefine SCC to the same value already available after
11461// SCCValid. If there are no intervening SCC conflicts delete SCCRedefine and
11462// update kill/dead flags if necessary.
11463bool SIInstrInfo::optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,
11464 bool NeedInversion) const {
11465 MachineInstr *KillsSCC = nullptr;
11466 if (SCCValid->getParent() != SCCRedefine->getParent())
11467 return false;
11468 for (MachineInstr &MI : make_range(x: std::next(x: SCCValid->getIterator()),
11469 y: SCCRedefine->getIterator())) {
11470 if (MI.modifiesRegister(Reg: AMDGPU::SCC, TRI: &RI))
11471 return false;
11472 if (MI.killsRegister(Reg: AMDGPU::SCC, TRI: &RI))
11473 KillsSCC = &MI;
11474 }
11475 if (NeedInversion && !invertSCCUse(SCCDef: SCCRedefine))
11476 return false;
11477 if (MachineOperand *SccDef =
11478 SCCValid->findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr))
11479 SccDef->setIsDead(false);
11480 if (KillsSCC)
11481 KillsSCC->clearRegisterKills(Reg: AMDGPU::SCC, /*TRI=*/RegInfo: nullptr);
11482 SCCRedefine->eraseFromParent();
11483 return true;
11484}
11485
11486static bool foldableSelect(const MachineInstr &Def) {
11487 if (Def.getOpcode() != AMDGPU::S_CSELECT_B32 &&
11488 Def.getOpcode() != AMDGPU::S_CSELECT_B64)
11489 return false;
11490 bool Op1IsNonZeroImm =
11491 Def.getOperand(i: 1).isImm() && Def.getOperand(i: 1).getImm() != 0;
11492 bool Op2IsZeroImm =
11493 Def.getOperand(i: 2).isImm() && Def.getOperand(i: 2).getImm() == 0;
11494 if (!Op1IsNonZeroImm || !Op2IsZeroImm)
11495 return false;
11496 return true;
11497}
11498
11499static bool setsSCCIfResultIsZero(const MachineInstr &Def, bool &NeedInversion,
11500 unsigned &NewDefOpc) {
11501 // S_ADD_U32 X, 1 sets SCC on carryout which can only happen if result==0.
11502 // S_ADD_I32 X, 1 can be converted to S_ADD_U32 X, 1 if SCC is dead.
11503 if (Def.getOpcode() != AMDGPU::S_ADD_I32 &&
11504 Def.getOpcode() != AMDGPU::S_ADD_U32)
11505 return false;
11506 const MachineOperand &AddSrc1 = Def.getOperand(i: 1);
11507 const MachineOperand &AddSrc2 = Def.getOperand(i: 2);
11508 const MachineRegisterInfo &MRI = Def.getMF()->getRegInfo();
11509 const SIInstrInfo *TII = static_cast<const SIInstrInfo *>(
11510 Def.getMF()->getSubtarget().getInstrInfo());
11511
11512 auto Imm1 = TII->getImmOrMaterializedImm(MRI, Op: AddSrc1);
11513 auto Imm2 = TII->getImmOrMaterializedImm(MRI, Op: AddSrc2);
11514 if ((!Imm1 || *Imm1 != 1) && (!Imm2 || *Imm2 != 1))
11515 return false;
11516
11517 if (Def.getOpcode() == AMDGPU::S_ADD_I32) {
11518 const MachineOperand *SccDef =
11519 Def.findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
11520 if (!SccDef->isDead())
11521 return false;
11522 NewDefOpc = AMDGPU::S_ADD_U32;
11523 }
11524 NeedInversion = !NeedInversion;
11525 return true;
11526}
11527
11528bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
11529 Register SrcReg2, int64_t CmpMask,
11530 int64_t CmpValue,
11531 const MachineRegisterInfo *MRI) const {
11532 if (!SrcReg || SrcReg.isPhysical())
11533 return false;
11534
11535 if (SrcReg2) {
11536 auto ImmOpt = getImmOrMaterializedImm(MRI: *MRI, Reg: SrcReg2);
11537 if (!ImmOpt)
11538 return false;
11539 CmpValue = *ImmOpt;
11540 }
11541
11542 const auto optimizeCmpSelect = [&CmpInstr, SrcReg, CmpValue, MRI,
11543 this](bool NeedInversion) -> bool {
11544 if (CmpValue != 0)
11545 return false;
11546
11547 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
11548 if (!Def)
11549 return false;
11550
11551 // For S_OP that set SCC = DST!=0, do the transformation
11552 //
11553 // s_cmp_[lg|eq]_* (S_OP ...), 0 => (S_OP ...)
11554 //
11555 // For (S_OP ...) that set SCC = DST==0, invert NeedInversion and
11556 // do the transformation:
11557 //
11558 // s_cmp_[lg|eq]_* (S_OP ...), 0 => (S_OP ...)
11559 //
11560 // If foldableSelect, s_cmp_lg_* is redundant because the SCC input value
11561 // for S_CSELECT* already has the same value that will be calculated by
11562 // s_cmp_lg_*
11563 //
11564 // s_cmp_[lg|eq]_* (S_CSELECT* (non-zero imm), 0), 0 => (S_CSELECT*
11565 // (non-zero imm), 0)
11566
11567 unsigned NewDefOpc = Def->getOpcode();
11568 if (!setsSCCIfResultIsNonZero(*Def) &&
11569 !setsSCCIfResultIsZero(Def: *Def, NeedInversion, NewDefOpc) &&
11570 !foldableSelect(Def: *Def))
11571 return false;
11572
11573 if (!optimizeSCC(SCCValid: Def, SCCRedefine: &CmpInstr, NeedInversion))
11574 return false;
11575
11576 if (NewDefOpc != Def->getOpcode())
11577 Def->setDesc(get(Opcode: NewDefOpc));
11578
11579 // If s_or_b32 result, sY, is unused (i.e. it is effectively a 64-bit
11580 // s_cmp_lg of a register pair) and the inputs are the hi and lo-halves of a
11581 // 64-bit foldableSelect then delete s_or_b32 in the sequence:
11582 // sX = s_cselect_b64 (non-zero imm), 0
11583 // sLo = copy sX.sub0
11584 // sHi = copy sX.sub1
11585 // sY = s_or_b32 sLo, sHi
11586 if (Def->getOpcode() == AMDGPU::S_OR_B32 &&
11587 MRI->use_nodbg_empty(RegNo: Def->getOperand(i: 0).getReg())) {
11588 const MachineOperand &OrOpnd1 = Def->getOperand(i: 1);
11589 const MachineOperand &OrOpnd2 = Def->getOperand(i: 2);
11590 if (OrOpnd1.isReg() && OrOpnd2.isReg()) {
11591 MachineInstr *Def1 = MRI->getVRegDef(Reg: OrOpnd1.getReg());
11592 MachineInstr *Def2 = MRI->getVRegDef(Reg: OrOpnd2.getReg());
11593 if (Def1 && Def1->getOpcode() == AMDGPU::COPY && Def2 &&
11594 Def2->getOpcode() == AMDGPU::COPY && Def1->getOperand(i: 1).isReg() &&
11595 Def2->getOperand(i: 1).isReg() &&
11596 Def1->getOperand(i: 1).getSubReg() == AMDGPU::sub0 &&
11597 Def2->getOperand(i: 1).getSubReg() == AMDGPU::sub1 &&
11598 Def1->getOperand(i: 1).getReg() == Def2->getOperand(i: 1).getReg()) {
11599 MachineInstr *Select = MRI->getVRegDef(Reg: Def1->getOperand(i: 1).getReg());
11600 if (Select && foldableSelect(Def: *Select))
11601 optimizeSCC(SCCValid: Select, SCCRedefine: Def, /*NeedInversion=*/false);
11602 }
11603 }
11604 }
11605 return true;
11606 };
11607
11608 const auto optimizeCmpAnd = [&CmpInstr, SrcReg, CmpValue, MRI,
11609 this](int64_t ExpectedValue, unsigned SrcSize,
11610 bool IsReversible, bool IsSigned) -> bool {
11611 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11612 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11613 // s_cmp_ge_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11614 // s_cmp_ge_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11615 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 1 << n => s_and_b64 $src, 1 << n
11616 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11617 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11618 // s_cmp_gt_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11619 // s_cmp_gt_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11620 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 0 => s_and_b64 $src, 1 << n
11621 //
11622 // Signed ge/gt are not used for the sign bit.
11623 //
11624 // If result of the AND is unused except in the compare:
11625 // s_and_b(32|64) $src, 1 << n => s_bitcmp1_b(32|64) $src, n
11626 //
11627 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
11628 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
11629 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 0 => s_bitcmp0_b64 $src, n
11630 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
11631 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
11632 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 1 << n => s_bitcmp0_b64 $src, n
11633
11634 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
11635 if (!Def)
11636 return false;
11637
11638 if (Def->getOpcode() != AMDGPU::S_AND_B32 &&
11639 Def->getOpcode() != AMDGPU::S_AND_B64)
11640 return false;
11641
11642 int64_t Mask;
11643 const auto isMask = [&Mask, SrcSize, MRI,
11644 this](const MachineOperand *MO) -> bool {
11645 auto ImmOpt = this->getImmOrMaterializedImm(MRI: *MRI, Op: *MO);
11646 if (!ImmOpt)
11647 return false;
11648 Mask = *ImmOpt;
11649 Mask &= maxUIntN(N: SrcSize);
11650 return isPowerOf2_64(Value: Mask);
11651 };
11652
11653 MachineOperand *SrcOp = &Def->getOperand(i: 1);
11654 if (isMask(SrcOp))
11655 SrcOp = &Def->getOperand(i: 2);
11656 else if (isMask(&Def->getOperand(i: 2)))
11657 SrcOp = &Def->getOperand(i: 1);
11658 else
11659 return false;
11660
11661 // A valid Mask is required to have a single bit set, hence a non-zero and
11662 // power-of-two value. This verifies that we will not do 64-bit shift below.
11663 assert(llvm::has_single_bit<uint64_t>(Mask) && "Invalid mask.");
11664 unsigned BitNo = llvm::countr_zero(Val: (uint64_t)Mask);
11665 if (IsSigned && BitNo == SrcSize - 1)
11666 return false;
11667
11668 ExpectedValue <<= BitNo;
11669
11670 bool IsReversedCC = false;
11671 if (CmpValue != ExpectedValue) {
11672 if (!IsReversible)
11673 return false;
11674 IsReversedCC = CmpValue == (ExpectedValue ^ Mask);
11675 if (!IsReversedCC)
11676 return false;
11677 }
11678
11679 Register DefReg = Def->getOperand(i: 0).getReg();
11680 if (IsReversedCC && !MRI->hasOneNonDBGUse(RegNo: DefReg))
11681 return false;
11682
11683 if (!optimizeSCC(SCCValid: Def, SCCRedefine: &CmpInstr, /*NeedInversion=*/false))
11684 return false;
11685
11686 if (!MRI->use_nodbg_empty(RegNo: DefReg)) {
11687 assert(!IsReversedCC);
11688 return true;
11689 }
11690
11691 // Replace AND with unused result with a S_BITCMP.
11692 MachineBasicBlock *MBB = Def->getParent();
11693
11694 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32
11695 : AMDGPU::S_BITCMP1_B32
11696 : IsReversedCC ? AMDGPU::S_BITCMP0_B64
11697 : AMDGPU::S_BITCMP1_B64;
11698
11699 BuildMI(BB&: *MBB, I: Def, MIMD: Def->getDebugLoc(), MCID: get(Opcode: NewOpc))
11700 .add(MO: *SrcOp)
11701 .addImm(Val: BitNo);
11702 Def->eraseFromParent();
11703
11704 return true;
11705 };
11706
11707 switch (CmpInstr.getOpcode()) {
11708 default:
11709 break;
11710 case AMDGPU::S_CMP_EQ_U32:
11711 case AMDGPU::S_CMP_EQ_I32:
11712 case AMDGPU::S_CMPK_EQ_U32:
11713 case AMDGPU::S_CMPK_EQ_I32:
11714 return optimizeCmpAnd(1, 32, true, false) ||
11715 optimizeCmpSelect(/*NeedInversion=*/true);
11716 case AMDGPU::S_CMP_GE_U32:
11717 case AMDGPU::S_CMPK_GE_U32:
11718 return optimizeCmpAnd(1, 32, false, false);
11719 case AMDGPU::S_CMP_GE_I32:
11720 case AMDGPU::S_CMPK_GE_I32:
11721 return optimizeCmpAnd(1, 32, false, true);
11722 case AMDGPU::S_CMP_EQ_U64:
11723 return optimizeCmpAnd(1, 64, true, false);
11724 case AMDGPU::S_CMP_LG_U32:
11725 case AMDGPU::S_CMP_LG_I32:
11726 case AMDGPU::S_CMPK_LG_U32:
11727 case AMDGPU::S_CMPK_LG_I32:
11728 return optimizeCmpAnd(0, 32, true, false) ||
11729 optimizeCmpSelect(/*NeedInversion=*/false);
11730 case AMDGPU::S_CMP_GT_U32:
11731 case AMDGPU::S_CMPK_GT_U32:
11732 return optimizeCmpAnd(0, 32, false, false);
11733 case AMDGPU::S_CMP_GT_I32:
11734 case AMDGPU::S_CMPK_GT_I32:
11735 return optimizeCmpAnd(0, 32, false, true);
11736 case AMDGPU::S_CMP_LG_U64:
11737 return optimizeCmpAnd(0, 64, true, false) ||
11738 optimizeCmpSelect(/*NeedInversion=*/false);
11739 }
11740
11741 return false;
11742}
11743
11744void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI,
11745 AMDGPU::OpName OpName) const {
11746 if (!ST.needsAlignedVGPRs())
11747 return;
11748
11749 int OpNo = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpName);
11750 if (OpNo < 0)
11751 return;
11752 MachineOperand &Op = MI.getOperand(i: OpNo);
11753 if (getOpSize(MI, OpNo) > 4)
11754 return;
11755
11756 // Add implicit aligned super-reg to force alignment on the data operand.
11757 const DebugLoc &DL = MI.getDebugLoc();
11758 MachineBasicBlock *BB = MI.getParent();
11759 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
11760 Register DataReg = Op.getReg();
11761 bool IsAGPR = RI.isAGPR(MRI, Reg: DataReg);
11762 Register Undef = MRI.createVirtualRegister(
11763 RegClass: IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);
11764 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
11765 Register NewVR =
11766 MRI.createVirtualRegister(RegClass: IsAGPR ? &AMDGPU::AReg_64_Align2RegClass
11767 : &AMDGPU::VReg_64_Align2RegClass);
11768 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewVR)
11769 .addReg(RegNo: DataReg, Flags: {}, SubReg: Op.getSubReg())
11770 .addImm(Val: AMDGPU::sub0)
11771 .addReg(RegNo: Undef)
11772 .addImm(Val: AMDGPU::sub1);
11773 Op.setReg(NewVR);
11774 Op.setSubReg(AMDGPU::sub0);
11775 MI.addOperand(Op: MachineOperand::CreateReg(Reg: NewVR, isDef: false, isImp: true));
11776}
11777
11778unsigned SIInstrInfo::getRepeatRate(const MachineInstr &MI) const {
11779 if (!SchedModel.hasInstrSchedModel())
11780 return 0;
11781
11782 // The repeat rate is the throughput-limiting resource occupancy: the largest
11783 // number of cycles any written processor resource is held.
11784 const MCSchedClassDesc *SCDesc = SchedModel.resolveSchedClass(MI: &MI);
11785 unsigned RepeatRate = 0;
11786 for (TargetSchedModel::ProcResIter
11787 PI = SchedModel.getWriteProcResBegin(SC: SCDesc),
11788 PE = SchedModel.getWriteProcResEnd(SC: SCDesc);
11789 PI != PE; ++PI) {
11790 RepeatRate = std::max(a: RepeatRate, b: (unsigned)PI->ReleaseAtCycle);
11791 }
11792
11793 return RepeatRate;
11794}
11795
11796bool SIInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const {
11797 if (isIGLP(MI: *MI))
11798 return false;
11799
11800 return TargetInstrInfo::isGlobalMemoryObject(MI);
11801}
11802
11803bool SIInstrInfo::isXDLWMMA(const MachineInstr &MI) const {
11804 if (!isWMMA(MI) && !isSWMMAC(MI))
11805 return false;
11806
11807 if (ST.hasGFX1250Insts())
11808 return AMDGPU::getWMMAIsXDL(Opc: MI.getOpcode());
11809
11810 return true;
11811}
11812
11813bool SIInstrInfo::isXDL(const MachineInstr &MI) const {
11814 unsigned Opcode = MI.getOpcode();
11815
11816 if (AMDGPU::isGFX12Plus(STI: ST))
11817 return isDOT(MI) || isXDLWMMA(MI);
11818
11819 if (!isMAI(MI) || isDGEMM(Opcode) ||
11820 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
11821 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
11822 return false;
11823
11824 if (!ST.hasGFX940Insts())
11825 return true;
11826
11827 return AMDGPU::getMAIIsGFX940XDL(Opc: Opcode);
11828}
11829