1//===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// SI Implementation of TargetInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIInstrInfo.h"
15#include "AMDGPU.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPULaneMaskUtils.h"
18#include "AMDGPUMachineInstrs.h"
19#include "GCNHazardRecognizer.h"
20#include "GCNSubtarget.h"
21#include "SIMachineFunctionInfo.h"
22#include "Utils/AMDGPUBaseInfo.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
26#include "llvm/CodeGen/LiveIntervals.h"
27#include "llvm/CodeGen/MachineCycleAnalysis.h"
28#include "llvm/CodeGen/MachineDominators.h"
29#include "llvm/CodeGen/MachineFrameInfo.h"
30#include "llvm/CodeGen/MachineScheduler.h"
31#include "llvm/CodeGen/RegisterScavenging.h"
32#include "llvm/CodeGen/ScheduleDAG.h"
33#include "llvm/IR/DiagnosticInfo.h"
34#include "llvm/IR/IntrinsicsAMDGPU.h"
35#include "llvm/MC/MCContext.h"
36#include "llvm/Support/AMDGPUAsyncStages.h"
37#include "llvm/Target/TargetMachine.h"
38#include <tuple>
39
40using namespace llvm;
41
42#define DEBUG_TYPE "si-instr-info"
43
44#define GET_INSTRINFO_CTOR_DTOR
45#include "AMDGPUGenInstrInfo.inc"
46
47namespace llvm::AMDGPU {
48#define GET_ImageDimIntrinsicTable_IMPL
49#define GET_RsrcIntrinsics_IMPL
50#define GET_BufferIntrinsicsTable_IMPL
51#define GET_GFX1250BlockingCyclesTable_DECL
52#define GET_GFX1250BlockingCyclesTable_IMPL
53
54struct AMDGPUBlockingCyclesInfo {
55 uint16_t Opcode;
56 uint8_t GFX1250BlockingCycles;
57};
58
59#include "AMDGPUGenSearchableTables.inc"
60} // namespace llvm::AMDGPU
61
62SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
63 : AMDGPUGenInstrInfo(ST, RI, AMDGPU::ADJCALLSTACKUP,
64 AMDGPU::ADJCALLSTACKDOWN),
65 RI(ST), ST(ST) {
66 SchedModel.init(TSInfo: &ST);
67}
68
69//===----------------------------------------------------------------------===//
70// TargetInstrInfo callbacks
71//===----------------------------------------------------------------------===//
72
73static unsigned getNumOperandsNoGlue(SDNode *Node) {
74 unsigned N = Node->getNumOperands();
75 while (N && Node->getOperand(Num: N - 1).getValueType() == MVT::Glue)
76 --N;
77 return N;
78}
79
80/// Returns true if both nodes have the same value for the given
81/// operand \p Op, or if both nodes do not have this operand.
82static bool nodesHaveSameOperandValue(SDNode *N0, SDNode *N1,
83 AMDGPU::OpName OpName) {
84 unsigned Opc0 = N0->getMachineOpcode();
85 unsigned Opc1 = N1->getMachineOpcode();
86
87 int Op0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: OpName);
88 int Op1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: OpName);
89
90 if (Op0Idx == -1 && Op1Idx == -1)
91 return true;
92
93
94 if ((Op0Idx == -1 && Op1Idx != -1) ||
95 (Op1Idx == -1 && Op0Idx != -1))
96 return false;
97
98 // getNamedOperandIdx returns the index for the MachineInstr's operands,
99 // which includes the result as the first operand. We are indexing into the
100 // MachineSDNode's operands, so we need to skip the result operand to get
101 // the real index.
102 --Op0Idx;
103 --Op1Idx;
104
105 return N0->getOperand(Num: Op0Idx) == N1->getOperand(Num: Op1Idx);
106}
107
108static bool canRemat(const MachineInstr &MI) {
109
110 if (SIInstrInfo::isVOP1(MI) || SIInstrInfo::isVOP2(MI) ||
111 SIInstrInfo::isVOP3(MI) || SIInstrInfo::isSDWA(MI) ||
112 SIInstrInfo::isSALU(MI))
113 return true;
114
115 if (SIInstrInfo::isSMRD(MI)) {
116 return !MI.memoperands_empty() &&
117 llvm::all_of(Range: MI.memoperands(), P: [](const MachineMemOperand *MMO) {
118 return MMO->isLoad() && MMO->isInvariant();
119 });
120 }
121
122 return false;
123}
124
125// Split relocation flags for 64-bit global-address materialization into the
126// flags for the lo and hi halves.
127static std::pair<unsigned, unsigned>
128splitGlobalAddressRelocFlags(const MachineOperand &SrcOp) {
129 const unsigned Flags = SrcOp.getTargetFlags();
130
131 // Infer the relocation type from the existing flags on the global operand.
132 // The relocation type should have been determined earlier in the pipeline.
133 switch (Flags & SIInstrInfo::MO_MASK) {
134 case SIInstrInfo::MO_GOTPCREL:
135 case SIInstrInfo::MO_GOTPCREL64: {
136 // For 64-bit GOT-relative, use the 64-bit relocation.
137 const unsigned GotFlags =
138 (Flags & ~SIInstrInfo::MO_MASK) | SIInstrInfo::MO_GOTPCREL64;
139 return {GotFlags, GotFlags};
140 }
141 default:
142 break;
143 }
144
145 std::optional<unsigned> LoFlags =
146 SIInstrInfo::extractSubregFromGlobalAddressFlags(TargetFlags: Flags, SubRegIndex: AMDGPU::sub0);
147 std::optional<unsigned> HiFlags =
148 SIInstrInfo::extractSubregFromGlobalAddressFlags(TargetFlags: Flags, SubRegIndex: AMDGPU::sub1);
149 if (!LoFlags || !HiFlags)
150 llvm_unreachable("unknown relocation type for global address");
151 return {*LoFlags, *HiFlags};
152}
153
154bool SIInstrInfo::isReMaterializableImpl(
155 const MachineInstr &MI) const {
156
157 if (canRemat(MI)) {
158 // Normally VALU use of exec would block the rematerialization, but that
159 // is OK in this case to have an implicit exec read as all VALU do.
160 // We really want all of the generic logic for this except for this.
161
162 // Another potential implicit use is mode register. The core logic of
163 // the RA will not attempt rematerialization if mode is set anywhere
164 // in the function, otherwise it is safe since mode is not changed.
165
166 // There is difference to generic method which does not allow
167 // rematerialization if there are virtual register uses. We allow this,
168 // therefore this method includes SOP instructions as well.
169 if (!MI.hasImplicitDef() &&
170 MI.getNumImplicitOperands() == MI.getDesc().implicit_uses().size() &&
171 !MI.mayRaiseFPException())
172 return true;
173 }
174
175 // Everything below copied from TargetInstrInfo::isReMaterializableImpl. The
176 // only difference is that we allow operations that perform read-modify-write
177 // on sub-registers.
178
179 // Remat clients assume operand 0 is the defined register.
180 if (!MI.getNumOperands() || !MI.getOperand(i: 0).isReg())
181 return false;
182 Register DefReg = MI.getOperand(i: 0).getReg();
183
184 const MachineFunction &MF = *MI.getMF();
185
186 // A load from a fixed stack slot can be rematerialized. This may be
187 // redundant with subsequent checks, but it's target-independent,
188 // simple, and a common case.
189 int FrameIdx = 0;
190 if (isLoadFromStackSlot(MI, FrameIndex&: FrameIdx) &&
191 MF.getFrameInfo().isImmutableObjectIndex(ObjectIdx: FrameIdx))
192 return true;
193
194 // Avoid instructions obviously unsafe for remat.
195 if (MI.isNotDuplicable() || MI.mayStore() || MI.mayRaiseFPException() ||
196 MI.hasUnmodeledSideEffects())
197 return false;
198
199 // Don't remat inline asm. We have no idea how expensive it is
200 // even if it's side effect free.
201 if (MI.isInlineAsm())
202 return false;
203
204 // Avoid instructions which load from potentially varying memory.
205 if (MI.mayLoad() && !MI.isDereferenceableInvariantLoad())
206 return false;
207
208 const MachineRegisterInfo &MRI = MF.getRegInfo();
209
210 // If any of the registers accessed are non-constant, conservatively assume
211 // the instruction is not rematerializable.
212 for (const MachineOperand &MO : MI.operands()) {
213 if (!MO.isReg())
214 continue;
215 Register Reg = MO.getReg();
216 if (Reg == 0)
217 continue;
218
219 // Check for a well-behaved physical register.
220 if (Reg.isPhysical()) {
221 if (MO.isUse()) {
222 // If the physreg has no defs anywhere, it's just an ambient register
223 // and we can freely move its uses. Alternatively, if it's allocatable,
224 // it could get allocated to something with a def during allocation.
225 if (!MRI.isConstantPhysReg(PhysReg: Reg))
226 return false;
227 } else {
228 // A physreg def. We can't remat it.
229 return false;
230 }
231 continue;
232 }
233
234 // Only allow one virtual-register def. There may be multiple defs of the
235 // same virtual register, though.
236 if (MO.isDef() && Reg != DefReg)
237 return false;
238 }
239
240 return true;
241}
242
243bool SIInstrInfo::isSrc1DPPRevOpcode(const GCNSubtarget &ST, uint32_t Opcode) {
244 switch (Opcode) {
245 // v_subrev_u16 (gfx9)
246 case AMDGPU::V_SUBREV_U16_e32:
247 case AMDGPU::V_SUBREV_U16_e64:
248 // v_subrev_u32 (gfx9) / v_subrev_nc_u32 (gfx10+)
249 case AMDGPU::V_SUBREV_U32_e32:
250 case AMDGPU::V_SUBREV_U32_e64:
251 // v_subrev_co_u32
252 case AMDGPU::V_SUBREV_CO_U32_e32:
253 case AMDGPU::V_SUBREV_CO_U32_e64:
254 // v_subbrev_u32 (gfx9) / v_subrev_co_ci_u32 (gfx10+)
255 case AMDGPU::V_SUBBREV_U32_e32:
256 case AMDGPU::V_SUBBREV_U32_e64:
257 return true;
258 // REV shift opcodes worked this way before GFX11, verified on hardware
259 case AMDGPU::V_ASHRREV_I16_e32:
260 case AMDGPU::V_ASHRREV_I16_e64:
261 case AMDGPU::V_ASHRREV_I32_e32:
262 case AMDGPU::V_ASHRREV_I32_e64:
263 case AMDGPU::V_ASHRREV_I64_e64:
264 case AMDGPU::V_LSHLREV_B16_e32:
265 case AMDGPU::V_LSHLREV_B16_e64:
266 case AMDGPU::V_LSHLREV_B32_e32:
267 case AMDGPU::V_LSHLREV_B32_e64:
268 case AMDGPU::V_LSHLREV_B64_e64:
269 case AMDGPU::V_LSHRREV_B16_e32:
270 case AMDGPU::V_LSHRREV_B16_e64:
271 case AMDGPU::V_LSHRREV_B32_e32:
272 case AMDGPU::V_LSHRREV_B32_e64:
273 case AMDGPU::V_LSHRREV_B64_e64:
274 return !ST.hasGFX11Insts();
275 default:
276 return false;
277 }
278}
279
280// Returns true if the result of a VALU instruction depends on exec.
281bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {
282 assert(isVALU(MI, /*AllowLDSDMA=*/true));
283
284 // If it is convergent it depends on EXEC.
285 if (MI.isConvergent())
286 return true;
287
288 // If it defines an SGPR it depends on EXEC, unless it's dead.
289 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
290 for (const MachineOperand &Def : MI.defs()) {
291 if (Def.isDead())
292 continue;
293
294 Register Reg = Def.getReg();
295 if (Reg && RI.isSGPRReg(MRI, Reg))
296 return true;
297 }
298
299 return false;
300}
301
302bool SIInstrInfo::isIgnorableUse(const MachineInstr &MI, unsigned OpIdx) const {
303 const MachineOperand &MO = MI.getOperand(i: OpIdx);
304 // Any implicit use of exec by VALU is not a real register read.
305 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&
306 isVALU(MI, /*AllowLDSDMA=*/true) && !resultDependsOnExec(MI);
307}
308
309bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
310 MachineBasicBlock *SuccToSinkTo,
311 MachineCycleInfo *CI) const {
312 // Allow sinking if MI edits lane mask (divergent i1 in sgpr).
313 if (MI.getOpcode() == AMDGPU::SI_IF_BREAK)
314 return true;
315
316 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
317 // Check if sinking of MI would create temporal divergent use.
318 for (auto Op : MI.uses()) {
319 if (Op.isReg() && Op.getReg().isVirtual() &&
320 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Op.getReg()))) {
321 MachineInstr *SgprDef = MRI.getVRegDef(Reg: Op.getReg());
322 if (!SgprDef)
323 continue;
324
325 // SgprDef defined inside cycle
326 CycleRef FromCycle = CI->getCycle(Block: SgprDef->getParent());
327 if (!FromCycle)
328 continue;
329
330 CycleRef ToCycle = CI->getCycle(Block: SuccToSinkTo);
331 // Check if there is a FromCycle that contains SgprDef's basic block but
332 // does not contain SuccToSinkTo and also has divergent exit condition.
333 while (FromCycle && !(ToCycle && CI->contains(Outer: FromCycle, Inner: ToCycle))) {
334 SmallVector<MachineBasicBlock *, 1> ExitingBlocks;
335 CI->getExitingBlocks(C: FromCycle, TmpStorage&: ExitingBlocks);
336
337 // FromCycle has divergent exit condition.
338 for (MachineBasicBlock *ExitingBlock : ExitingBlocks) {
339 if (hasDivergentBranch(MBB: ExitingBlock))
340 return false;
341 }
342
343 FromCycle = CI->getParentCycle(C: FromCycle);
344 }
345 }
346 }
347
348 return true;
349}
350
351bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,
352 int64_t &Offset0,
353 int64_t &Offset1) const {
354 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
355 return false;
356
357 unsigned Opc0 = Load0->getMachineOpcode();
358 unsigned Opc1 = Load1->getMachineOpcode();
359
360 // Make sure both are actually loads.
361 if (!get(Opcode: Opc0).mayLoad() || !get(Opcode: Opc1).mayLoad())
362 return false;
363
364 // A mayLoad instruction without a def is not a load. Likely a prefetch.
365 if (!get(Opcode: Opc0).getNumDefs() || !get(Opcode: Opc1).getNumDefs())
366 return false;
367
368 if (isDS(Opcode: Opc0) && isDS(Opcode: Opc1)) {
369
370 // FIXME: Handle this case:
371 if (getNumOperandsNoGlue(Node: Load0) != getNumOperandsNoGlue(Node: Load1))
372 return false;
373
374 // Check base reg.
375 if (Load0->getOperand(Num: 0) != Load1->getOperand(Num: 0))
376 return false;
377
378 // Skip read2 / write2 variants for simplicity.
379 // TODO: We should report true if the used offsets are adjacent (excluded
380 // st64 versions).
381 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: AMDGPU::OpName::offset);
382 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: AMDGPU::OpName::offset);
383 if (Offset0Idx == -1 || Offset1Idx == -1)
384 return false;
385
386 // XXX - be careful of dataless loads
387 // getNamedOperandIdx returns the index for MachineInstrs. Since they
388 // include the output in the operand list, but SDNodes don't, we need to
389 // subtract the index by one.
390 Offset0Idx -= get(Opcode: Opc0).NumDefs;
391 Offset1Idx -= get(Opcode: Opc1).NumDefs;
392 Offset0 = Load0->getConstantOperandVal(Num: Offset0Idx);
393 Offset1 = Load1->getConstantOperandVal(Num: Offset1Idx);
394 return true;
395 }
396
397 if (isSMRD(Opcode: Opc0) && isSMRD(Opcode: Opc1)) {
398 // Skip time and cache invalidation instructions.
399 if (!AMDGPU::hasNamedOperand(Opcode: Opc0, NamedIdx: AMDGPU::OpName::sbase) ||
400 !AMDGPU::hasNamedOperand(Opcode: Opc1, NamedIdx: AMDGPU::OpName::sbase))
401 return false;
402
403 unsigned NumOps = getNumOperandsNoGlue(Node: Load0);
404 if (NumOps != getNumOperandsNoGlue(Node: Load1))
405 return false;
406
407 // Check base reg.
408 if (Load0->getOperand(Num: 0) != Load1->getOperand(Num: 0))
409 return false;
410
411 // Match register offsets, if both register and immediate offsets present.
412 assert(NumOps == 4 || NumOps == 5);
413 if (NumOps == 5 && Load0->getOperand(Num: 1) != Load1->getOperand(Num: 1))
414 return false;
415
416 const ConstantSDNode *Load0Offset =
417 dyn_cast<ConstantSDNode>(Val: Load0->getOperand(Num: NumOps - 3));
418 const ConstantSDNode *Load1Offset =
419 dyn_cast<ConstantSDNode>(Val: Load1->getOperand(Num: NumOps - 3));
420
421 if (!Load0Offset || !Load1Offset)
422 return false;
423
424 Offset0 = Load0Offset->getZExtValue();
425 Offset1 = Load1Offset->getZExtValue();
426 return true;
427 }
428
429 // MUBUF and MTBUF can access the same addresses.
430 if ((isMUBUF(Opcode: Opc0) || isMTBUF(Opcode: Opc0)) && (isMUBUF(Opcode: Opc1) || isMTBUF(Opcode: Opc1))) {
431
432 // MUBUF and MTBUF have vaddr at different indices.
433 if (!nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::soffset) ||
434 !nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::vaddr) ||
435 !nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::srsrc))
436 return false;
437
438 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: AMDGPU::OpName::offset);
439 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: AMDGPU::OpName::offset);
440
441 if (OffIdx0 == -1 || OffIdx1 == -1)
442 return false;
443
444 // getNamedOperandIdx returns the index for MachineInstrs. Since they
445 // include the output in the operand list, but SDNodes don't, we need to
446 // subtract the index by one.
447 OffIdx0 -= get(Opcode: Opc0).NumDefs;
448 OffIdx1 -= get(Opcode: Opc1).NumDefs;
449
450 SDValue Off0 = Load0->getOperand(Num: OffIdx0);
451 SDValue Off1 = Load1->getOperand(Num: OffIdx1);
452
453 // The offset might be a FrameIndexSDNode.
454 if (!isa<ConstantSDNode>(Val: Off0) || !isa<ConstantSDNode>(Val: Off1))
455 return false;
456
457 Offset0 = Off0->getAsZExtVal();
458 Offset1 = Off1->getAsZExtVal();
459 return true;
460 }
461
462 return false;
463}
464
465static bool isStride64(unsigned Opc) {
466 switch (Opc) {
467 case AMDGPU::DS_READ2ST64_B32:
468 case AMDGPU::DS_READ2ST64_B64:
469 case AMDGPU::DS_WRITE2ST64_B32:
470 case AMDGPU::DS_WRITE2ST64_B64:
471 return true;
472 default:
473 return false;
474 }
475}
476
477bool SIInstrInfo::getMemOperandsWithOffsetWidth(
478 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
479 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
480 if (!LdSt.mayLoadOrStore())
481 return false;
482
483 unsigned Opc = LdSt.getOpcode();
484 OffsetIsScalable = false;
485 const MachineOperand *BaseOp, *OffsetOp;
486 int DataOpIdx;
487
488 if (isDS(MI: LdSt)) {
489 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::addr);
490 OffsetOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
491 if (OffsetOp) {
492 // Normal, single offset LDS instruction.
493 if (!BaseOp) {
494 // DS_CONSUME/DS_APPEND use M0 for the base address.
495 // TODO: find the implicit use operand for M0 and use that as BaseOp?
496 return false;
497 }
498 BaseOps.push_back(Elt: BaseOp);
499 Offset = OffsetOp->getImm();
500 // Get appropriate operand, and compute width accordingly.
501 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
502 if (DataOpIdx == -1)
503 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
504 if (Opc == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
505 Width = LocationSize::precise(Value: 64);
506 else
507 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
508 } else {
509 // The 2 offset instructions use offset0 and offset1 instead. We can treat
510 // these as a load with a single offset if the 2 offsets are consecutive.
511 // We will use this for some partially aligned loads.
512 const MachineOperand *Offset0Op =
513 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset0);
514 const MachineOperand *Offset1Op =
515 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset1);
516
517 unsigned Offset0 = Offset0Op->getImm() & 0xff;
518 unsigned Offset1 = Offset1Op->getImm() & 0xff;
519 if (Offset0 + 1 != Offset1)
520 return false;
521
522 // Each of these offsets is in element sized units, so we need to convert
523 // to bytes of the individual reads.
524
525 unsigned EltSize;
526 if (LdSt.mayLoad())
527 EltSize = RI.getRegSizeInBits(RC: *getOpRegClass(MI: LdSt, OpNo: 0)) / 16;
528 else {
529 assert(LdSt.mayStore());
530 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
531 EltSize = RI.getRegSizeInBits(RC: *getOpRegClass(MI: LdSt, OpNo: Data0Idx)) / 8;
532 }
533
534 if (isStride64(Opc))
535 EltSize *= 64;
536
537 BaseOps.push_back(Elt: BaseOp);
538 Offset = EltSize * Offset0;
539 // Get appropriate operand(s), and compute width accordingly.
540 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
541 if (DataOpIdx == -1) {
542 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
543 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
544 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data1);
545 Width = LocationSize::precise(
546 Value: Width.getValue() + TypeSize::getFixed(ExactSize: getOpSize(MI: LdSt, OpNo: DataOpIdx)));
547 } else {
548 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
549 }
550 }
551 return true;
552 }
553
554 if (isMUBUF(MI: LdSt) || isMTBUF(MI: LdSt)) {
555 const MachineOperand *RSrc = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::srsrc);
556 if (!RSrc) // e.g. BUFFER_WBINVL1_VOL
557 return false;
558 BaseOps.push_back(Elt: RSrc);
559 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr);
560 if (BaseOp && !BaseOp->isFI())
561 BaseOps.push_back(Elt: BaseOp);
562 const MachineOperand *OffsetImm =
563 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
564 Offset = OffsetImm->getImm();
565 const MachineOperand *SOffset =
566 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::soffset);
567 if (SOffset) {
568 if (SOffset->isReg())
569 BaseOps.push_back(Elt: SOffset);
570 else
571 Offset += SOffset->getImm();
572 }
573 // Get appropriate operand, and compute width accordingly.
574 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
575 if (DataOpIdx == -1)
576 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
577 if (DataOpIdx == -1) // LDS DMA
578 return false;
579 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
580 return true;
581 }
582
583 if (isImage(MI: LdSt)) {
584 auto RsrcOpName =
585 isMIMG(MI: LdSt) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
586 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: RsrcOpName);
587 BaseOps.push_back(Elt: &LdSt.getOperand(i: SRsrcIdx));
588 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0);
589 if (VAddr0Idx >= 0) {
590 // GFX10 possible NSA encoding.
591 for (int I = VAddr0Idx; I < SRsrcIdx; ++I)
592 BaseOps.push_back(Elt: &LdSt.getOperand(i: I));
593 } else {
594 BaseOps.push_back(Elt: getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr));
595 }
596 Offset = 0;
597 // Get appropriate operand, and compute width accordingly.
598 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
599 if (DataOpIdx == -1)
600 return false; // no return sampler
601 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
602 return true;
603 }
604
605 if (isSMRD(MI: LdSt)) {
606 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::sbase);
607 if (!BaseOp) // e.g. S_MEMTIME
608 return false;
609 BaseOps.push_back(Elt: BaseOp);
610 OffsetOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
611 Offset = OffsetOp ? OffsetOp->getImm() : 0;
612 // Get appropriate operand, and compute width accordingly.
613 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::sdst);
614 if (DataOpIdx == -1)
615 return false;
616 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
617 return true;
618 }
619
620 if (isFLAT(MI: LdSt)) {
621 // Instructions have either vaddr or saddr or both or none.
622 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr);
623 if (BaseOp)
624 BaseOps.push_back(Elt: BaseOp);
625 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::saddr);
626 if (BaseOp)
627 BaseOps.push_back(Elt: BaseOp);
628 Offset = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset)->getImm();
629 // Get appropriate operand, and compute width accordingly.
630 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
631 if (DataOpIdx == -1)
632 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
633 if (DataOpIdx == -1) // LDS DMA
634 return false;
635 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
636 return true;
637 }
638
639 if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(Val: &LdSt)) {
640 // The movrel form's index is whatever M0 holds, which can change between
641 // accesses, so it has no base operand to report.
642 if (!LdStIdx->isGPRIdx())
643 return false;
644 BaseOp = &LdStIdx->getIdxOp();
645 OffsetOp = &LdStIdx->getOffsetOp();
646 if (!BaseOp->getReg().isVirtual())
647 return false;
648
649 BaseOps.push_back(Elt: BaseOp);
650 Offset = OffsetOp->getImm() * 4; // Offset has units of dwords.
651 Width = LocationSize::precise(Value: LdStIdx->getBitWidth() / 8);
652 return true;
653 }
654
655 return false;
656}
657
658static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
659 ArrayRef<const MachineOperand *> BaseOps1,
660 const MachineInstr &MI2,
661 ArrayRef<const MachineOperand *> BaseOps2) {
662 // Only examine the first "base" operand of each instruction, on the
663 // assumption that it represents the real base address of the memory access.
664 // Other operands are typically offsets or indices from this base address.
665 if (BaseOps1.front()->isIdenticalTo(Other: *BaseOps2.front()))
666 return true;
667
668 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
669 return false;
670
671 auto *MO1 = *MI1.memoperands_begin();
672 auto *MO2 = *MI2.memoperands_begin();
673 if (MO1->getAddrSpace() != MO2->getAddrSpace())
674 return false;
675
676 const auto *Base1 = MO1->getValue();
677 const auto *Base2 = MO2->getValue();
678 if (!Base1 || !Base2)
679 return false;
680 Base1 = getUnderlyingObject(V: Base1);
681 Base2 = getUnderlyingObject(V: Base2);
682
683 if (isa<UndefValue>(Val: Base1) || isa<UndefValue>(Val: Base2))
684 return false;
685
686 return Base1 == Base2;
687}
688
689bool SIInstrInfo::memOpsHaveSameBase(
690 ArrayRef<const MachineOperand *> BaseOps1,
691 ArrayRef<const MachineOperand *> BaseOps2) const {
692 // Accesses without base operands (e.g. scratch addressed only by an
693 // immediate offset) share the same implicit base.
694 if (BaseOps1.empty() || BaseOps2.empty())
695 return BaseOps1.empty() && BaseOps2.empty();
696 return memOpsHaveSameBasePtr(MI1: *BaseOps1.front()->getParent(), BaseOps1,
697 MI2: *BaseOps2.front()->getParent(), BaseOps2);
698}
699
700bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
701 int64_t Offset1, bool OffsetIsScalable1,
702 ArrayRef<const MachineOperand *> BaseOps2,
703 int64_t Offset2, bool OffsetIsScalable2,
704 unsigned ClusterSize,
705 unsigned NumBytes) const {
706 // If the mem ops (to be clustered) do not have the same base ptr, then they
707 // should not be clustered
708 if (!memOpsHaveSameBase(BaseOps1, BaseOps2))
709 return false;
710
711 // Both lists are empty or neither is, so BaseOps1 alone decides whether
712 // there is an instruction to read the budget from.
713 assert(BaseOps1.empty() == BaseOps2.empty());
714 unsigned MaxMemoryClusterDWords = DefaultMemoryClusterDWordsLimit;
715 if (!BaseOps1.empty()) {
716 const MachineFunction *MF = BaseOps1.front()->getParent()->getMF();
717 MaxMemoryClusterDWords =
718 MF->getInfo<SIMachineFunctionInfo>()->getMaxMemoryClusterDWords();
719 }
720
721 // In order to avoid register pressure, on an average, the number of DWORDS
722 // loaded together by all clustered mem ops should not exceed
723 // MaxMemoryClusterDWords. This is an empirical value based on certain
724 // observations and performance related experiments.
725 // The good thing about this heuristic is - it avoids clustering of too many
726 // sub-word loads, and also avoids clustering of wide loads. Below is the
727 // brief summary of how the heuristic behaves for various `LoadSize` when
728 // MaxMemoryClusterDWords is 8.
729 //
730 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops
731 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops
732 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops
733 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops
734 // (5) LoadSize >= 17: do not cluster
735 const unsigned LoadSize = NumBytes / ClusterSize;
736 const unsigned NumDWords = ((LoadSize + 3) / 4) * ClusterSize;
737 return NumDWords <= MaxMemoryClusterDWords;
738}
739
740// FIXME: This behaves strangely. If, for example, you have 32 load + stores,
741// the first 16 loads will be interleaved with the stores, and the next 16 will
742// be clustered as expected. It should really split into 2 16 store batches.
743//
744// Loads are clustered until this returns false, rather than trying to schedule
745// groups of stores. This also means we have to deal with saying different
746// address space loads should be clustered, and ones which might cause bank
747// conflicts.
748//
749// This might be deprecated so it might not be worth that much effort to fix.
750bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1,
751 int64_t Offset0, int64_t Offset1,
752 unsigned NumLoads) const {
753 assert(Offset1 > Offset0 &&
754 "Second offset should be larger than first offset!");
755 // If we have less than 16 loads in a row, and the offsets are within 64
756 // bytes, then schedule together.
757
758 // A cacheline is 64 bytes (for global memory).
759 return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
760}
761
762static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB,
763 MachineBasicBlock::iterator MI,
764 const DebugLoc &DL, MCRegister DestReg,
765 MCRegister SrcReg, bool KillSrc,
766 const char *Msg = "illegal VGPR to SGPR copy") {
767 MachineFunction *MF = MBB.getParent();
768
769 LLVMContext &C = MF->getFunction().getContext();
770 C.diagnose(DI: DiagnosticInfoUnsupported(MF->getFunction(), Msg, DL, DS_Error));
771
772 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_ILLEGAL_COPY), DestReg)
773 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
774}
775
776/// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not
777/// possible to have a direct copy in these cases on GFX908, so an intermediate
778/// VGPR copy is required.
779static void indirectCopyToAGPR(const SIInstrInfo &TII, MachineBasicBlock &MBB,
780 MachineBasicBlock::iterator MI,
781 const DebugLoc &DL, MCRegister DestReg,
782 MCRegister SrcReg, bool KillSrc,
783 RegScavenger &RS, bool RegsOverlap,
784 Register ImpUseSuperReg = Register()) {
785 assert((TII.getSubtarget().hasMAIInsts() &&
786 !TII.getSubtarget().hasGFX90AInsts()) &&
787 "Expected GFX908 subtarget.");
788
789 assert((AMDGPU::SReg_32RegClass.contains(SrcReg) ||
790 AMDGPU::AGPR_32RegClass.contains(SrcReg)) &&
791 "Source register of the copy should be either an SGPR or an AGPR.");
792
793 assert(AMDGPU::AGPR_32RegClass.contains(DestReg) &&
794 "Destination register of the copy should be an AGPR.");
795
796 const SIRegisterInfo &RI = TII.getRegisterInfo();
797
798 // First try to find defining accvgpr_write to avoid temporary registers.
799 // In the case of copies of overlapping AGPRs, we conservatively do not
800 // reuse previous accvgpr_writes. Otherwise, we may incorrectly pick up
801 // an accvgpr_write used for this same copy due to implicit-defs
802 if (!RegsOverlap) {
803 for (auto Def = MI, E = MBB.begin(); Def != E; ) {
804 --Def;
805
806 if (!Def->modifiesRegister(Reg: SrcReg, TRI: &RI))
807 continue;
808
809 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
810 Def->getOperand(i: 0).getReg() != SrcReg)
811 break;
812
813 MachineOperand &DefOp = Def->getOperand(i: 1);
814 assert(DefOp.isReg() || DefOp.isImm());
815
816 if (DefOp.isReg()) {
817 bool SafeToPropagate = true;
818 // Check that register source operand is not clobbered before MI.
819 // Immediate operands are always safe to propagate.
820 for (auto I = Def; I != MI && SafeToPropagate; ++I)
821 if (I->modifiesRegister(Reg: DefOp.getReg(), TRI: &RI))
822 SafeToPropagate = false;
823
824 if (!SafeToPropagate)
825 break;
826
827 for (auto I = Def; I != MI; ++I)
828 I->clearRegisterKills(Reg: DefOp.getReg(), RegInfo: &RI);
829 }
830
831 MachineInstrBuilder Builder =
832 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64),
833 DestReg)
834 .add(MO: DefOp);
835
836 if (ImpUseSuperReg) {
837 Builder.addReg(RegNo: ImpUseSuperReg,
838 Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
839 }
840
841 return;
842 }
843 }
844
845 RS.enterBasicBlockEnd(MBB);
846 RS.backward(I: std::next(x: MI));
847
848 // Ideally we want to have three registers for a long reg_sequence copy
849 // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
850 unsigned MaxVGPRs = RI.getRegPressureLimit(RC: &AMDGPU::VGPR_32RegClass,
851 MF&: *MBB.getParent());
852
853 // Registers in the sequence are allocated contiguously so we can just
854 // use register number to pick one of three round-robin temps.
855 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3;
856 Register Tmp =
857 MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getVGPRForAGPRCopy();
858 assert(MBB.getParent()->getRegInfo().isReserved(Tmp) &&
859 "VGPR used for an intermediate copy should have been reserved.");
860
861 // Only loop through if there are any free registers left. We don't want to
862 // spill.
863 while (RegNo--) {
864 Register Tmp2 = RS.scavengeRegisterBackwards(RC: AMDGPU::VGPR_32RegClass, To: MI,
865 /* RestoreAfter */ false, SPAdj: 0,
866 /* AllowSpill */ false);
867 if (!Tmp2 || RI.getHWRegIndex(Reg: Tmp2) >= MaxVGPRs)
868 break;
869 Tmp = Tmp2;
870 RS.setRegUsed(Reg: Tmp);
871 }
872
873 // Insert copy to temporary VGPR.
874 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;
875 if (AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg)) {
876 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64;
877 } else {
878 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
879 }
880
881 MachineInstrBuilder UseBuilder = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: TmpCopyOp), DestReg: Tmp)
882 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
883 if (ImpUseSuperReg) {
884 UseBuilder.addReg(RegNo: ImpUseSuperReg,
885 Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
886 }
887
888 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
889 .addReg(RegNo: Tmp, Flags: RegState::Kill);
890}
891
892static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB,
893 MachineBasicBlock::iterator MI, const DebugLoc &DL,
894 MCRegister DestReg, MCRegister SrcReg, bool KillSrc,
895 const TargetRegisterClass *RC, bool Forward) {
896 const SIRegisterInfo &RI = TII.getRegisterInfo();
897 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, EltSize: 4);
898 MachineBasicBlock::iterator I = MI;
899 MachineInstr *FirstMI = nullptr, *LastMI = nullptr;
900
901 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) {
902 int16_t SubIdx = BaseIndices[Idx];
903 Register DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
904 Register SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
905 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
906 unsigned Opcode = AMDGPU::S_MOV_B32;
907
908 // Is SGPR aligned? If so try to combine with next.
909 bool AlignedDest = ((DestSubReg - AMDGPU::SGPR0) % 2) == 0;
910 bool AlignedSrc = ((SrcSubReg - AMDGPU::SGPR0) % 2) == 0;
911 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) {
912 // Can use SGPR64 copy
913 unsigned Channel = RI.getChannelFromSubReg(SubReg: SubIdx);
914 SubIdx = RI.getSubRegFromChannel(Channel, NumRegs: 2);
915 DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
916 SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
917 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
918 Opcode = AMDGPU::S_MOV_B64;
919 Idx++;
920 }
921
922 LastMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode), DestReg: DestSubReg)
923 .addReg(RegNo: SrcSubReg)
924 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
925
926 if (!FirstMI)
927 FirstMI = LastMI;
928
929 if (!Forward)
930 I--;
931 }
932
933 assert(FirstMI && LastMI);
934 if (!Forward)
935 std::swap(a&: FirstMI, b&: LastMI);
936
937 if (KillSrc)
938 LastMI->addRegisterKilled(IncomingReg: SrcReg, RegInfo: &RI);
939}
940
941void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
942 MachineBasicBlock::iterator MI,
943 const DebugLoc &DL, Register DestReg,
944 Register SrcReg, bool KillSrc, bool RenamableDest,
945 bool RenamableSrc) const {
946 const TargetRegisterClass *RC = RI.getPhysRegBaseClass(Reg: DestReg);
947 unsigned Size = RI.getRegSizeInBits(RC: *RC);
948 const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(Reg: SrcReg);
949 unsigned SrcSize = RI.getRegSizeInBits(RC: *SrcRC);
950
951 // The rest of copyPhysReg assumes Src and Dst size are the same size.
952 // TODO-GFX11_16BIT If all true 16 bit instruction patterns are completed can
953 // we remove fix_16_bit_physreg_copies and this code block?
954 if (ST.getCLOpts().fix_16_bit_physreg_copies) {
955 if (((Size == 16) != (SrcSize == 16))) {
956 // Non-VGPR Src and Dst will later be expanded back to 32 bits.
957 assert(ST.useRealTrue16Insts());
958 Register &RegToFix = (Size == 32) ? DestReg : SrcReg;
959 MCRegister SubReg = RI.getSubReg(Reg: RegToFix, Idx: AMDGPU::lo16);
960 RegToFix = SubReg;
961
962 if (DestReg == SrcReg) {
963 // Identity copy. Insert empty bundle since ExpandPostRA expects an
964 // instruction here.
965 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::BUNDLE));
966 return;
967 }
968 RC = RI.getPhysRegBaseClass(Reg: DestReg);
969 Size = RI.getRegSizeInBits(RC: *RC);
970 SrcRC = RI.getPhysRegBaseClass(Reg: SrcReg);
971 SrcSize = RI.getRegSizeInBits(RC: *SrcRC);
972 }
973 }
974
975 if (RC == &AMDGPU::VGPR_32RegClass) {
976 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
977 AMDGPU::SReg_32RegClass.contains(SrcReg) ||
978 AMDGPU::AGPR_32RegClass.contains(SrcReg));
979 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg) ?
980 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;
981 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
982 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
983 return;
984 }
985
986 if (RC == &AMDGPU::SReg_32_XM0RegClass ||
987 RC == &AMDGPU::SReg_32RegClass) {
988 if (SrcReg == AMDGPU::SCC) {
989 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B32), DestReg)
990 .addImm(Val: 1)
991 .addImm(Val: 0);
992 return;
993 }
994
995 if (!AMDGPU::SReg_32RegClass.contains(Reg: SrcReg)) {
996 if (DestReg == AMDGPU::VCC_LO) {
997 // FIXME: Hack until VReg_1 removed.
998 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
999 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CMP_NE_U32_e32))
1000 .addImm(Val: 0)
1001 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1002 return;
1003 }
1004
1005 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1006 return;
1007 }
1008
1009 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg)
1010 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1011 return;
1012 }
1013
1014 if (RC == &AMDGPU::SReg_64RegClass) {
1015 if (SrcReg == AMDGPU::SCC) {
1016 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B64), DestReg)
1017 .addImm(Val: 1)
1018 .addImm(Val: 0);
1019 return;
1020 }
1021
1022 if (!AMDGPU::SReg_64_EncodableRegClass.contains(Reg: SrcReg)) {
1023 if (DestReg == AMDGPU::VCC) {
1024 // FIXME: Hack until VReg_1 removed.
1025 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
1026 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CMP_NE_U32_e32))
1027 .addImm(Val: 0)
1028 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1029 return;
1030 }
1031
1032 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1033 return;
1034 }
1035
1036 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B64), DestReg)
1037 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1038 return;
1039 }
1040
1041 if (DestReg == AMDGPU::SCC) {
1042 // Copying 64-bit or 32-bit sources to SCC barely makes sense,
1043 // but SelectionDAG emits such copies for i1 sources.
1044 if (AMDGPU::SReg_64RegClass.contains(Reg: SrcReg)) {
1045 // This copy can only be produced by patterns
1046 // with explicit SCC, which are known to be enabled
1047 // only for subtargets with S_CMP_LG_U64 present.
1048 assert(ST.hasScalarCompareEq64());
1049 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U64))
1050 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
1051 .addImm(Val: 0);
1052 } else {
1053 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
1054 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U32))
1055 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
1056 .addImm(Val: 0);
1057 }
1058
1059 return;
1060 }
1061
1062 if (RC == &AMDGPU::AGPR_32RegClass) {
1063 if (AMDGPU::VGPR_32RegClass.contains(Reg: SrcReg) ||
1064 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(Reg: SrcReg))) {
1065 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
1066 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1067 return;
1068 }
1069
1070 if (AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg) && ST.hasGFX90AInsts()) {
1071 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_MOV_B32), DestReg)
1072 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1073 return;
1074 }
1075
1076 // FIXME: Pass should maintain scavenger to avoid scan through the block on
1077 // every AGPR spill.
1078 RegScavenger RS;
1079 const bool Overlap = RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1080 indirectCopyToAGPR(TII: *this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS, RegsOverlap: Overlap);
1081 return;
1082 }
1083
1084 if (Size == 16) {
1085 assert(AMDGPU::VGPR_16RegClass.contains(SrcReg) ||
1086 AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
1087 AMDGPU::AGPR_LO16RegClass.contains(SrcReg));
1088
1089 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(Reg: DestReg);
1090 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(Reg: SrcReg);
1091 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(Reg: DestReg);
1092 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(Reg: SrcReg);
1093 bool DstLow = !AMDGPU::isHi16Reg(Reg: DestReg, MRI: RI);
1094 bool SrcLow = !AMDGPU::isHi16Reg(Reg: SrcReg, MRI: RI);
1095 MCRegister NewDestReg = RI.get32BitRegister(Reg: DestReg);
1096 MCRegister NewSrcReg = RI.get32BitRegister(Reg: SrcReg);
1097
1098 if (IsSGPRDst) {
1099 if (!IsSGPRSrc) {
1100 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1101 return;
1102 }
1103
1104 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: NewDestReg)
1105 .addReg(RegNo: NewSrcReg, Flags: getKillRegState(B: KillSrc));
1106 return;
1107 }
1108
1109 if (IsAGPRDst || IsAGPRSrc) {
1110 if (!DstLow || !SrcLow) {
1111 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
1112 Msg: "Cannot use hi16 subreg with an AGPR!");
1113 }
1114
1115 copyPhysReg(MBB, MI, DL, DestReg: NewDestReg, SrcReg: NewSrcReg, KillSrc);
1116 return;
1117 }
1118
1119 if (ST.useRealTrue16Insts()) {
1120 if (IsSGPRSrc) {
1121 assert(SrcLow);
1122 SrcReg = NewSrcReg;
1123 }
1124 // Use the smaller instruction encoding if possible.
1125 if (AMDGPU::VGPR_16_Lo128RegClass.contains(Reg: DestReg) &&
1126 (IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.contains(Reg: SrcReg))) {
1127 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B16_t16_e32), DestReg)
1128 .addReg(RegNo: SrcReg);
1129 } else {
1130 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B16_t16_e64), DestReg)
1131 .addImm(Val: 0) // src0_modifiers
1132 .addReg(RegNo: SrcReg)
1133 .addImm(Val: 0); // op_sel
1134 }
1135 return;
1136 }
1137
1138 if (IsSGPRSrc && !ST.hasSDWAScalar()) {
1139 if (!DstLow || !SrcLow) {
1140 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
1141 Msg: "Cannot use hi16 subreg on VI!");
1142 }
1143
1144 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: NewDestReg)
1145 .addReg(RegNo: NewSrcReg, Flags: getKillRegState(B: KillSrc));
1146 return;
1147 }
1148
1149 auto MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_sdwa), DestReg: NewDestReg)
1150 .addImm(Val: 0) // src0_modifiers
1151 .addReg(RegNo: NewSrcReg)
1152 .addImm(Val: 0) // clamp
1153 .addImm(Val: DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1154 : AMDGPU::SDWA::SdwaSel::WORD_1)
1155 .addImm(Val: AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE)
1156 .addImm(Val: SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1157 : AMDGPU::SDWA::SdwaSel::WORD_1)
1158 .addReg(RegNo: NewDestReg, Flags: RegState::Implicit | RegState::Undef);
1159 // First implicit operand is $exec.
1160 MIB->tieOperands(DefIdx: 0, UseIdx: MIB->getNumOperands() - 1);
1161 return;
1162 }
1163
1164 // Returns true if Dst and Src are in Opc's HwMode-resolved destination and
1165 // source operand classes.
1166 auto CanCopyWith = [&](unsigned Opc, MCRegister Dst, MCRegister Src,
1167 unsigned SrcOp = 1) {
1168 const MCInstrDesc &Desc = get(Opcode: Opc);
1169 const TargetRegisterClass *DstOpRC = getRegClass(MCID: Desc, OpNum: 0);
1170 const TargetRegisterClass *SrcOpRC = getRegClass(MCID: Desc, OpNum: SrcOp);
1171 return DstOpRC && SrcOpRC && DstOpRC->contains(Reg: Dst) &&
1172 SrcOpRC->contains(Reg: Src);
1173 };
1174
1175 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(RC: SrcRC))) {
1176 if (ST.hasVMovB64Inst() &&
1177 CanCopyWith(AMDGPU::V_MOV_B64_e32, DestReg, SrcReg)) {
1178 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B64_e32), DestReg)
1179 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1180 return;
1181 }
1182 if (ST.hasPkMovB32() &&
1183 CanCopyWith(AMDGPU::V_PK_MOV_B32, DestReg, SrcReg, /*SrcOp=*/2)) {
1184 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg)
1185 .addImm(Val: SISrcMods::OP_SEL_1)
1186 .addReg(RegNo: SrcReg)
1187 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
1188 .addReg(RegNo: SrcReg)
1189 .addImm(Val: 0) // op_sel_lo
1190 .addImm(Val: 0) // op_sel_hi
1191 .addImm(Val: 0) // neg_lo
1192 .addImm(Val: 0) // neg_hi
1193 .addImm(Val: 0) // clamp
1194 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
1195 return;
1196 }
1197 }
1198
1199 const bool Forward = RI.getHWRegIndex(Reg: DestReg) <= RI.getHWRegIndex(Reg: SrcReg);
1200 if (RI.isSGPRClass(RC)) {
1201 if (!RI.isSGPRClass(RC: SrcRC)) {
1202 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1203 return;
1204 }
1205 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1206 expandSGPRCopy(TII: *this, MBB, MI, DL, DestReg, SrcReg, KillSrc: CanKillSuperReg, RC,
1207 Forward);
1208 return;
1209 }
1210
1211 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1212 unsigned WideOpcode = AMDGPU::INSTRUCTION_LIST_END;
1213 if (RI.isAGPRClass(RC)) {
1214 if (ST.hasGFX90AInsts() && RI.isAGPRClass(RC: SrcRC))
1215 Opcode = AMDGPU::V_ACCVGPR_MOV_B32;
1216 else if (RI.hasVGPRs(RC: SrcRC) ||
1217 (ST.hasGFX90AInsts() && RI.isSGPRClass(RC: SrcRC)))
1218 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
1219 else
1220 Opcode = AMDGPU::INSTRUCTION_LIST_END;
1221 } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(RC: SrcRC)) {
1222 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;
1223 } else if (RI.isVGPRClass(RC)) {
1224 if (ST.hasVMovB64Inst())
1225 WideOpcode = AMDGPU::V_MOV_B64_e32;
1226 else if (ST.hasPkMovB32())
1227 WideOpcode = AMDGPU::V_PK_MOV_B32;
1228 }
1229
1230 const TargetRegisterClass *WideDstRC{}, *WideSrcRC{};
1231 if (WideOpcode != AMDGPU::INSTRUCTION_LIST_END) {
1232 const MCInstrDesc &Desc = get(Opcode: WideOpcode);
1233 unsigned SrcOp = WideOpcode == AMDGPU::V_PK_MOV_B32 ? 2 : 1;
1234 WideDstRC = getRegClass(MCID: Desc, OpNum: 0);
1235 WideSrcRC = getRegClass(MCID: Desc, OpNum: SrcOp);
1236 }
1237
1238 // If there is an overlap, we can't kill the super-register on the last
1239 // instruction, since it will also kill the components made live by this def.
1240 const bool Overlap = RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1241 const bool CanKillSuperReg = KillSrc && !Overlap;
1242
1243 // For the cases where we need an intermediate instruction/temporary register
1244 // (destination is an AGPR), we need a scavenger.
1245 //
1246 // FIXME: The pass should maintain this for us so we don't have to re-scan the
1247 // whole block for every handled copy.
1248 std::unique_ptr<RegScavenger> RS;
1249 if (Opcode == AMDGPU::INSTRUCTION_LIST_END)
1250 RS = std::make_unique<RegScavenger>();
1251
1252 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize: 4);
1253
1254 for (unsigned Idx{}; Idx < SubIndices.size();) {
1255 unsigned NumRegs = 1;
1256 unsigned ThisOpcode = Opcode;
1257 unsigned SubIdx =
1258 Forward ? SubIndices[Idx] : SubIndices[SubIndices.size() - Idx - 1];
1259
1260 if (WideDstRC && WideSrcRC && Idx + 1 < SubIndices.size()) {
1261 unsigned Channel = RI.getChannelFromSubReg(SubReg: SubIdx);
1262 if (!Forward)
1263 --Channel;
1264
1265 unsigned WideSubIdx = RI.getSubRegFromChannel(Channel, NumRegs: 2);
1266 Register WideDst = RI.getSubReg(Reg: DestReg, Idx: WideSubIdx);
1267 Register WideSrc = RI.getSubReg(Reg: SrcReg, Idx: WideSubIdx);
1268
1269 if (WideDst && WideSrc && WideDstRC->contains(Reg: WideDst) &&
1270 WideSrcRC->contains(Reg: WideSrc)) {
1271 SubIdx = WideSubIdx;
1272 NumRegs = 2;
1273 ThisOpcode = WideOpcode;
1274 }
1275 }
1276
1277 Register DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
1278 Register SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
1279 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
1280
1281 Idx += NumRegs;
1282 bool UseKill = CanKillSuperReg && Idx == SubIndices.size();
1283
1284 if (ThisOpcode == AMDGPU::INSTRUCTION_LIST_END) {
1285 Register ImpUseSuper = SrcReg;
1286 indirectCopyToAGPR(TII: *this, MBB, MI, DL, DestReg: DestSubReg, SrcReg: SrcSubReg, KillSrc: UseKill,
1287 RS&: *RS, RegsOverlap: Overlap, ImpUseSuperReg: ImpUseSuper);
1288 } else if (ThisOpcode == AMDGPU::V_PK_MOV_B32) {
1289 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg: DestSubReg)
1290 .addImm(Val: SISrcMods::OP_SEL_1)
1291 .addReg(RegNo: SrcSubReg)
1292 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
1293 .addReg(RegNo: SrcSubReg)
1294 .addImm(Val: 0) // op_sel_lo
1295 .addImm(Val: 0) // op_sel_hi
1296 .addImm(Val: 0) // neg_lo
1297 .addImm(Val: 0) // neg_hi
1298 .addImm(Val: 0) // clamp
1299 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: UseKill) | RegState::Implicit);
1300 } else {
1301 MachineInstrBuilder Builder =
1302 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: ThisOpcode), DestReg: DestSubReg).addReg(RegNo: SrcSubReg);
1303
1304 Builder.addReg(RegNo: SrcReg, Flags: getKillRegState(B: UseKill) | RegState::Implicit);
1305 }
1306 }
1307}
1308
1309int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
1310 int32_t NewOpc;
1311
1312 // Try to map original to commuted opcode
1313 NewOpc = AMDGPU::getCommuteRev(Opcode);
1314 if (NewOpc != -1)
1315 // Check if the commuted (REV) opcode exists on the target.
1316 return pseudoToMCOpcode(Opcode: NewOpc) != -1 ? NewOpc : -1;
1317
1318 // Try to map commuted to original opcode
1319 NewOpc = AMDGPU::getCommuteOrig(Opcode);
1320 if (NewOpc != -1)
1321 // Check if the original (non-REV) opcode exists on the target.
1322 return pseudoToMCOpcode(Opcode: NewOpc) != -1 ? NewOpc : -1;
1323
1324 return Opcode;
1325}
1326
1327bool SIInstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
1328 const Register Reg,
1329 int64_t &ImmVal) const {
1330 switch (MI.getOpcode()) {
1331 case AMDGPU::V_MOV_B32_e32:
1332 case AMDGPU::S_MOV_B32:
1333 case AMDGPU::S_MOVK_I32:
1334 case AMDGPU::S_MOV_B64:
1335 case AMDGPU::V_MOV_B64_e32:
1336 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
1337 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
1338 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
1339 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
1340 case AMDGPU::V_MOV_B64_PSEUDO:
1341 case AMDGPU::V_MOV_B16_t16_e32: {
1342 const MachineOperand &Src0 = MI.getOperand(i: 1);
1343 if (Src0.isImm()) {
1344 ImmVal = Src0.getImm();
1345 return MI.getOperand(i: 0).getReg() == Reg;
1346 }
1347
1348 return false;
1349 }
1350 case AMDGPU::V_MOV_B16_t16_e64: {
1351 const MachineOperand &Src0 = MI.getOperand(i: 2);
1352 if (Src0.isImm() && !MI.getOperand(i: 1).getImm()) {
1353 ImmVal = Src0.getImm();
1354 return MI.getOperand(i: 0).getReg() == Reg;
1355 }
1356
1357 return false;
1358 }
1359 case AMDGPU::S_BREV_B32:
1360 case AMDGPU::V_BFREV_B32_e32:
1361 case AMDGPU::V_BFREV_B32_e64: {
1362 const MachineOperand &Src0 = MI.getOperand(i: 1);
1363 if (Src0.isImm()) {
1364 ImmVal = static_cast<int64_t>(reverseBits<int32_t>(Val: Src0.getImm()));
1365 return MI.getOperand(i: 0).getReg() == Reg;
1366 }
1367
1368 return false;
1369 }
1370 case AMDGPU::S_NOT_B32:
1371 case AMDGPU::V_NOT_B32_e32:
1372 case AMDGPU::V_NOT_B32_e64: {
1373 const MachineOperand &Src0 = MI.getOperand(i: 1);
1374 if (Src0.isImm()) {
1375 ImmVal = static_cast<int64_t>(~static_cast<int32_t>(Src0.getImm()));
1376 return MI.getOperand(i: 0).getReg() == Reg;
1377 }
1378
1379 return false;
1380 }
1381 default:
1382 return false;
1383 }
1384}
1385
1386std::optional<int64_t>
1387SIInstrInfo::getImmOrMaterializedImm(const MachineRegisterInfo &MRI,
1388 const MachineOperand &Op,
1389 MachineInstr **DefMI) const {
1390 if (DefMI)
1391 *DefMI = nullptr;
1392
1393 if (Op.isImm())
1394 return Op.getImm();
1395
1396 if (!Op.isReg() || !Op.getReg().isVirtual())
1397 return std::nullopt;
1398 MachineInstr *Def = MRI.getUniqueVRegDef(Reg: Op.getReg());
1399 if (Def && Def->isMoveImmediate()) {
1400 const MachineOperand &ImmSrc = Def->getOperand(i: 1);
1401 if (ImmSrc.isImm()) {
1402 if (DefMI)
1403 *DefMI = Def;
1404 return extractSubregFromImm(ImmVal: ImmSrc.getImm(), SubRegIndex: Op.getSubReg());
1405 }
1406 }
1407
1408 return std::nullopt;
1409}
1410
1411std::optional<int64_t>
1412SIInstrInfo::getImmOrMaterializedImm(const MachineRegisterInfo &MRI,
1413 Register Reg, MachineInstr **DefMI) const {
1414 MachineOperand Op = MachineOperand::CreateReg(Reg, isDef: false);
1415 return getImmOrMaterializedImm(MRI, Op, DefMI);
1416}
1417
1418unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
1419
1420 if (RI.isAGPRClass(RC: DstRC))
1421 return AMDGPU::COPY;
1422 if (RI.getRegSizeInBits(RC: *DstRC) == 16) {
1423 // Assume hi bits are unneeded. Only _e64 true16 instructions are legal
1424 // before RA.
1425 return RI.isSGPRClass(RC: DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64;
1426 }
1427 if (RI.getRegSizeInBits(RC: *DstRC) == 32)
1428 return RI.isSGPRClass(RC: DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1429 if (RI.getRegSizeInBits(RC: *DstRC) == 64 && RI.isSGPRClass(RC: DstRC))
1430 return AMDGPU::S_MOV_B64;
1431 if (RI.getRegSizeInBits(RC: *DstRC) == 64 && !RI.isSGPRClass(RC: DstRC))
1432 return AMDGPU::V_MOV_B64_PSEUDO;
1433 return AMDGPU::COPY;
1434}
1435
1436const MCInstrDesc &
1437SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize,
1438 bool IsIndirectSrc) const {
1439 if (IsIndirectSrc) {
1440 if (VecSize <= 32) // 4 bytes
1441 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);
1442 if (VecSize <= 64) // 8 bytes
1443 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);
1444 if (VecSize <= 96) // 12 bytes
1445 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);
1446 if (VecSize <= 128) // 16 bytes
1447 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);
1448 if (VecSize <= 160) // 20 bytes
1449 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);
1450 if (VecSize <= 192) // 24 bytes
1451 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V6);
1452 if (VecSize <= 224) // 28 bytes
1453 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V7);
1454 if (VecSize <= 256) // 32 bytes
1455 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);
1456 if (VecSize <= 288) // 36 bytes
1457 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9);
1458 if (VecSize <= 320) // 40 bytes
1459 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10);
1460 if (VecSize <= 352) // 44 bytes
1461 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11);
1462 if (VecSize <= 384) // 48 bytes
1463 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12);
1464 if (VecSize <= 512) // 64 bytes
1465 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);
1466 if (VecSize <= 1024) // 128 bytes
1467 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);
1468
1469 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos");
1470 }
1471
1472 if (VecSize <= 32) // 4 bytes
1473 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);
1474 if (VecSize <= 64) // 8 bytes
1475 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);
1476 if (VecSize <= 96) // 12 bytes
1477 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);
1478 if (VecSize <= 128) // 16 bytes
1479 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);
1480 if (VecSize <= 160) // 20 bytes
1481 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);
1482 if (VecSize <= 192) // 24 bytes
1483 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V6);
1484 if (VecSize <= 224) // 28 bytes
1485 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V7);
1486 if (VecSize <= 256) // 32 bytes
1487 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);
1488 if (VecSize <= 288) // 36 bytes
1489 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9);
1490 if (VecSize <= 320) // 40 bytes
1491 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10);
1492 if (VecSize <= 352) // 44 bytes
1493 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11);
1494 if (VecSize <= 384) // 48 bytes
1495 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12);
1496 if (VecSize <= 512) // 64 bytes
1497 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);
1498 if (VecSize <= 1024) // 128 bytes
1499 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);
1500
1501 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos");
1502}
1503
1504static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) {
1505 if (VecSize <= 32) // 4 bytes
1506 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1507 if (VecSize <= 64) // 8 bytes
1508 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1509 if (VecSize <= 96) // 12 bytes
1510 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1511 if (VecSize <= 128) // 16 bytes
1512 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1513 if (VecSize <= 160) // 20 bytes
1514 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1515 if (VecSize <= 192) // 24 bytes
1516 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V6;
1517 if (VecSize <= 224) // 28 bytes
1518 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V7;
1519 if (VecSize <= 256) // 32 bytes
1520 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1521 if (VecSize <= 288) // 36 bytes
1522 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1523 if (VecSize <= 320) // 40 bytes
1524 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1525 if (VecSize <= 352) // 44 bytes
1526 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1527 if (VecSize <= 384) // 48 bytes
1528 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1529 if (VecSize <= 512) // 64 bytes
1530 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1531 if (VecSize <= 1024) // 128 bytes
1532 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1533
1534 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1535}
1536
1537static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) {
1538 if (VecSize <= 32) // 4 bytes
1539 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1540 if (VecSize <= 64) // 8 bytes
1541 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1542 if (VecSize <= 96) // 12 bytes
1543 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1544 if (VecSize <= 128) // 16 bytes
1545 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1546 if (VecSize <= 160) // 20 bytes
1547 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1548 if (VecSize <= 192) // 24 bytes
1549 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V6;
1550 if (VecSize <= 224) // 28 bytes
1551 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V7;
1552 if (VecSize <= 256) // 32 bytes
1553 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1554 if (VecSize <= 288) // 36 bytes
1555 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1556 if (VecSize <= 320) // 40 bytes
1557 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1558 if (VecSize <= 352) // 44 bytes
1559 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1560 if (VecSize <= 384) // 48 bytes
1561 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1562 if (VecSize <= 512) // 64 bytes
1563 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1564 if (VecSize <= 1024) // 128 bytes
1565 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1566
1567 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1568}
1569
1570static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) {
1571 if (VecSize <= 64) // 8 bytes
1572 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;
1573 if (VecSize <= 128) // 16 bytes
1574 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;
1575 if (VecSize <= 256) // 32 bytes
1576 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;
1577 if (VecSize <= 512) // 64 bytes
1578 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;
1579 if (VecSize <= 1024) // 128 bytes
1580 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;
1581
1582 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1583}
1584
1585const MCInstrDesc &
1586SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize,
1587 bool IsSGPR) const {
1588 if (IsSGPR) {
1589 switch (EltSize) {
1590 case 32:
1591 return get(Opcode: getIndirectSGPRWriteMovRelPseudo32(VecSize));
1592 case 64:
1593 return get(Opcode: getIndirectSGPRWriteMovRelPseudo64(VecSize));
1594 default:
1595 llvm_unreachable("invalid reg indexing elt size");
1596 }
1597 }
1598
1599 assert(EltSize == 32 && "invalid reg indexing elt size");
1600 return get(Opcode: getIndirectVGPRWriteMovRelPseudoOpc(VecSize));
1601}
1602
1603static unsigned getSGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1604 switch (Size) {
1605 case 4:
1606 return NeedsCFI ? AMDGPU::SI_SPILL_S32_CFI_SAVE : AMDGPU::SI_SPILL_S32_SAVE;
1607 case 8:
1608 return NeedsCFI ? AMDGPU::SI_SPILL_S64_CFI_SAVE : AMDGPU::SI_SPILL_S64_SAVE;
1609 case 12:
1610 return NeedsCFI ? AMDGPU::SI_SPILL_S96_CFI_SAVE : AMDGPU::SI_SPILL_S96_SAVE;
1611 case 16:
1612 return NeedsCFI ? AMDGPU::SI_SPILL_S128_CFI_SAVE
1613 : AMDGPU::SI_SPILL_S128_SAVE;
1614 case 20:
1615 return NeedsCFI ? AMDGPU::SI_SPILL_S160_CFI_SAVE
1616 : AMDGPU::SI_SPILL_S160_SAVE;
1617 case 24:
1618 return NeedsCFI ? AMDGPU::SI_SPILL_S192_CFI_SAVE
1619 : AMDGPU::SI_SPILL_S192_SAVE;
1620 case 28:
1621 return NeedsCFI ? AMDGPU::SI_SPILL_S224_CFI_SAVE
1622 : AMDGPU::SI_SPILL_S224_SAVE;
1623 case 32:
1624 return AMDGPU::SI_SPILL_S256_SAVE;
1625 case 36:
1626 return AMDGPU::SI_SPILL_S288_SAVE;
1627 case 40:
1628 return AMDGPU::SI_SPILL_S320_SAVE;
1629 case 44:
1630 return AMDGPU::SI_SPILL_S352_SAVE;
1631 case 48:
1632 return AMDGPU::SI_SPILL_S384_SAVE;
1633 case 64:
1634 return NeedsCFI ? AMDGPU::SI_SPILL_S512_CFI_SAVE
1635 : AMDGPU::SI_SPILL_S512_SAVE;
1636 case 128:
1637 return NeedsCFI ? AMDGPU::SI_SPILL_S1024_CFI_SAVE
1638 : AMDGPU::SI_SPILL_S1024_SAVE;
1639 default:
1640 llvm_unreachable("unknown register size");
1641 }
1642}
1643
1644static unsigned getVGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1645 switch (Size) {
1646 case 2:
1647 return AMDGPU::SI_SPILL_V16_SAVE;
1648 case 4:
1649 return NeedsCFI ? AMDGPU::SI_SPILL_V32_CFI_SAVE : AMDGPU::SI_SPILL_V32_SAVE;
1650 case 8:
1651 return NeedsCFI ? AMDGPU::SI_SPILL_V64_CFI_SAVE : AMDGPU::SI_SPILL_V64_SAVE;
1652 case 12:
1653 return NeedsCFI ? AMDGPU::SI_SPILL_V96_CFI_SAVE : AMDGPU::SI_SPILL_V96_SAVE;
1654 case 16:
1655 return NeedsCFI ? AMDGPU::SI_SPILL_V128_CFI_SAVE
1656 : AMDGPU::SI_SPILL_V128_SAVE;
1657 case 20:
1658 return NeedsCFI ? AMDGPU::SI_SPILL_V160_CFI_SAVE
1659 : AMDGPU::SI_SPILL_V160_SAVE;
1660 case 24:
1661 return NeedsCFI ? AMDGPU::SI_SPILL_V192_CFI_SAVE
1662 : AMDGPU::SI_SPILL_V192_SAVE;
1663 case 28:
1664 return NeedsCFI ? AMDGPU::SI_SPILL_V224_CFI_SAVE
1665 : AMDGPU::SI_SPILL_V224_SAVE;
1666 case 32:
1667 return NeedsCFI ? AMDGPU::SI_SPILL_V256_CFI_SAVE
1668 : AMDGPU::SI_SPILL_V256_SAVE;
1669 case 36:
1670 return NeedsCFI ? AMDGPU::SI_SPILL_V288_CFI_SAVE
1671 : AMDGPU::SI_SPILL_V288_SAVE;
1672 case 40:
1673 return NeedsCFI ? AMDGPU::SI_SPILL_V320_CFI_SAVE
1674 : AMDGPU::SI_SPILL_V320_SAVE;
1675 case 44:
1676 return NeedsCFI ? AMDGPU::SI_SPILL_V352_CFI_SAVE
1677 : AMDGPU::SI_SPILL_V352_SAVE;
1678 case 48:
1679 return NeedsCFI ? AMDGPU::SI_SPILL_V384_CFI_SAVE
1680 : AMDGPU::SI_SPILL_V384_SAVE;
1681 case 64:
1682 return NeedsCFI ? AMDGPU::SI_SPILL_V512_CFI_SAVE
1683 : AMDGPU::SI_SPILL_V512_SAVE;
1684 case 128:
1685 return NeedsCFI ? AMDGPU::SI_SPILL_V1024_CFI_SAVE
1686 : AMDGPU::SI_SPILL_V1024_SAVE;
1687 default:
1688 llvm_unreachable("unknown register size");
1689 }
1690}
1691
1692static unsigned getAVSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1693 switch (Size) {
1694 case 4:
1695 return NeedsCFI ? AMDGPU::SI_SPILL_AV32_CFI_SAVE
1696 : AMDGPU::SI_SPILL_AV32_SAVE;
1697 case 8:
1698 return NeedsCFI ? AMDGPU::SI_SPILL_AV64_CFI_SAVE
1699 : AMDGPU::SI_SPILL_AV64_SAVE;
1700 case 12:
1701 return NeedsCFI ? AMDGPU::SI_SPILL_AV96_CFI_SAVE
1702 : AMDGPU::SI_SPILL_AV96_SAVE;
1703 case 16:
1704 return NeedsCFI ? AMDGPU::SI_SPILL_AV128_CFI_SAVE
1705 : AMDGPU::SI_SPILL_AV128_SAVE;
1706 case 20:
1707 return NeedsCFI ? AMDGPU::SI_SPILL_AV160_CFI_SAVE
1708 : AMDGPU::SI_SPILL_AV160_SAVE;
1709 case 24:
1710 return NeedsCFI ? AMDGPU::SI_SPILL_AV192_CFI_SAVE
1711 : AMDGPU::SI_SPILL_AV192_SAVE;
1712 case 28:
1713 return NeedsCFI ? AMDGPU::SI_SPILL_AV224_CFI_SAVE
1714 : AMDGPU::SI_SPILL_AV224_SAVE;
1715 case 32:
1716 return NeedsCFI ? AMDGPU::SI_SPILL_AV256_CFI_SAVE
1717 : AMDGPU::SI_SPILL_AV256_SAVE;
1718 case 36:
1719 return AMDGPU::SI_SPILL_AV288_SAVE;
1720 case 40:
1721 return AMDGPU::SI_SPILL_AV320_SAVE;
1722 case 44:
1723 return AMDGPU::SI_SPILL_AV352_SAVE;
1724 case 48:
1725 return AMDGPU::SI_SPILL_AV384_SAVE;
1726 case 64:
1727 return NeedsCFI ? AMDGPU::SI_SPILL_AV512_CFI_SAVE
1728 : AMDGPU::SI_SPILL_AV512_SAVE;
1729 case 128:
1730 return NeedsCFI ? AMDGPU::SI_SPILL_AV1024_CFI_SAVE
1731 : AMDGPU::SI_SPILL_AV1024_SAVE;
1732 default:
1733 llvm_unreachable("unknown register size");
1734 }
1735}
1736
1737static unsigned getWWMRegSpillSaveOpcode(unsigned Size,
1738 bool IsVectorSuperClass) {
1739 // Currently, there is only 32-bit WWM register spills needed.
1740 if (Size != 4)
1741 llvm_unreachable("unknown wwm register spill size");
1742
1743 if (IsVectorSuperClass)
1744 return AMDGPU::SI_SPILL_WWM_AV32_SAVE;
1745
1746 return AMDGPU::SI_SPILL_WWM_V32_SAVE;
1747}
1748
1749unsigned SIInstrInfo::getVectorRegSpillSaveOpcode(
1750 Register Reg, const TargetRegisterClass *RC, unsigned Size,
1751 const SIMachineFunctionInfo &MFI, bool NeedsCFI) const {
1752 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
1753
1754 // Choose the right opcode if spilling a WWM register.
1755 if (MFI.checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
1756 return getWWMRegSpillSaveOpcode(Size, IsVectorSuperClass);
1757
1758 // TODO: Check if AGPRs are available
1759 if (ST.hasMAIInsts())
1760 return getAVSpillSaveOpcode(Size, NeedsCFI);
1761
1762 return getVGPRSpillSaveOpcode(Size, NeedsCFI);
1763}
1764
1765void SIInstrInfo::storeRegToStackSlotImpl(
1766 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
1767 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
1768 MachineInstr::MIFlag Flags, bool NeedsCFI) const {
1769 MachineFunction *MF = MBB.getParent();
1770 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1771 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1772 const DebugLoc &DL = MBB.findDebugLoc(MBBI: MI);
1773
1774 MachinePointerInfo PtrInfo
1775 = MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
1776 MachineMemOperand *MMO = MF->getMachineMemOperand(
1777 PtrInfo, F: MachineMemOperand::MOStore, Size: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
1778 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
1779 unsigned SpillSize = RI.getSpillSize(RC: *RC);
1780
1781 MachineRegisterInfo &MRI = MF->getRegInfo();
1782 if (RI.isSGPRClass(RC)) {
1783 if (FrameInfo.getStackID(ObjectIdx: FrameIndex) == TargetStackID::SGPRSpill)
1784 MFI->setHasSpilledSGPRs();
1785 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled");
1786 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI &&
1787 SrcReg != AMDGPU::EXEC && "exec should not be spilled");
1788
1789 // We are only allowed to create one new instruction when spilling
1790 // registers, so we need to use pseudo instruction for spilling SGPRs.
1791 const MCInstrDesc &OpDesc =
1792 get(Opcode: getSGPRSpillSaveOpcode(Size: SpillSize, NeedsCFI));
1793
1794 // The SGPR spill/restore instructions only work on number sgprs, so we need
1795 // to make sure we are using the correct register class.
1796 if (SrcReg.isVirtual() && SpillSize == 4) {
1797 MRI.constrainRegClass(Reg: SrcReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
1798 }
1799
1800 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: OpDesc)
1801 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill)) // data
1802 .addFrameIndex(Idx: FrameIndex) // addr
1803 .addMemOperand(MMO)
1804 .addReg(RegNo: MFI->getStackPtrOffsetReg(), Flags: RegState::Implicit);
1805
1806 return;
1807 }
1808
1809 unsigned Opcode = getVectorRegSpillSaveOpcode(Reg: VReg ? VReg : SrcReg, RC,
1810 Size: SpillSize, MFI: *MFI, NeedsCFI);
1811 MFI->setHasSpilledVGPRs();
1812
1813 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode))
1814 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill)) // data
1815 .addFrameIndex(Idx: FrameIndex) // addr
1816 .addReg(RegNo: MFI->getStackPtrOffsetReg()) // scratch_offset
1817 .addImm(Val: 0) // offset
1818 .addMemOperand(MMO);
1819}
1820
1821void SIInstrInfo::storeRegToStackSlot(
1822 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
1823 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
1824 MachineInstr::MIFlag Flags) const {
1825 storeRegToStackSlotImpl(MBB, MI, SrcReg, isKill, FrameIndex, RC, VReg, Flags,
1826 NeedsCFI: false);
1827}
1828
1829void SIInstrInfo::storeRegToStackSlotCFI(MachineBasicBlock &MBB,
1830 MachineBasicBlock::iterator MI,
1831 Register SrcReg, bool isKill,
1832 int FrameIndex,
1833 const TargetRegisterClass *RC) const {
1834 storeRegToStackSlotImpl(MBB, MI, SrcReg, isKill, FrameIndex, RC, VReg: Register(),
1835 Flags: MachineInstr::NoFlags, NeedsCFI: true);
1836}
1837
1838static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1839 switch (Size) {
1840 case 4:
1841 return AMDGPU::SI_SPILL_S32_RESTORE;
1842 case 8:
1843 return AMDGPU::SI_SPILL_S64_RESTORE;
1844 case 12:
1845 return AMDGPU::SI_SPILL_S96_RESTORE;
1846 case 16:
1847 return AMDGPU::SI_SPILL_S128_RESTORE;
1848 case 20:
1849 return AMDGPU::SI_SPILL_S160_RESTORE;
1850 case 24:
1851 return AMDGPU::SI_SPILL_S192_RESTORE;
1852 case 28:
1853 return AMDGPU::SI_SPILL_S224_RESTORE;
1854 case 32:
1855 return AMDGPU::SI_SPILL_S256_RESTORE;
1856 case 36:
1857 return AMDGPU::SI_SPILL_S288_RESTORE;
1858 case 40:
1859 return AMDGPU::SI_SPILL_S320_RESTORE;
1860 case 44:
1861 return AMDGPU::SI_SPILL_S352_RESTORE;
1862 case 48:
1863 return AMDGPU::SI_SPILL_S384_RESTORE;
1864 case 64:
1865 return AMDGPU::SI_SPILL_S512_RESTORE;
1866 case 128:
1867 return AMDGPU::SI_SPILL_S1024_RESTORE;
1868 default:
1869 llvm_unreachable("unknown register size");
1870 }
1871}
1872
1873static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1874 switch (Size) {
1875 case 2:
1876 return AMDGPU::SI_SPILL_V16_RESTORE;
1877 case 4:
1878 return AMDGPU::SI_SPILL_V32_RESTORE;
1879 case 8:
1880 return AMDGPU::SI_SPILL_V64_RESTORE;
1881 case 12:
1882 return AMDGPU::SI_SPILL_V96_RESTORE;
1883 case 16:
1884 return AMDGPU::SI_SPILL_V128_RESTORE;
1885 case 20:
1886 return AMDGPU::SI_SPILL_V160_RESTORE;
1887 case 24:
1888 return AMDGPU::SI_SPILL_V192_RESTORE;
1889 case 28:
1890 return AMDGPU::SI_SPILL_V224_RESTORE;
1891 case 32:
1892 return AMDGPU::SI_SPILL_V256_RESTORE;
1893 case 36:
1894 return AMDGPU::SI_SPILL_V288_RESTORE;
1895 case 40:
1896 return AMDGPU::SI_SPILL_V320_RESTORE;
1897 case 44:
1898 return AMDGPU::SI_SPILL_V352_RESTORE;
1899 case 48:
1900 return AMDGPU::SI_SPILL_V384_RESTORE;
1901 case 64:
1902 return AMDGPU::SI_SPILL_V512_RESTORE;
1903 case 128:
1904 return AMDGPU::SI_SPILL_V1024_RESTORE;
1905 default:
1906 llvm_unreachable("unknown register size");
1907 }
1908}
1909
1910static unsigned getAVSpillRestoreOpcode(unsigned Size) {
1911 switch (Size) {
1912 case 4:
1913 return AMDGPU::SI_SPILL_AV32_RESTORE;
1914 case 8:
1915 return AMDGPU::SI_SPILL_AV64_RESTORE;
1916 case 12:
1917 return AMDGPU::SI_SPILL_AV96_RESTORE;
1918 case 16:
1919 return AMDGPU::SI_SPILL_AV128_RESTORE;
1920 case 20:
1921 return AMDGPU::SI_SPILL_AV160_RESTORE;
1922 case 24:
1923 return AMDGPU::SI_SPILL_AV192_RESTORE;
1924 case 28:
1925 return AMDGPU::SI_SPILL_AV224_RESTORE;
1926 case 32:
1927 return AMDGPU::SI_SPILL_AV256_RESTORE;
1928 case 36:
1929 return AMDGPU::SI_SPILL_AV288_RESTORE;
1930 case 40:
1931 return AMDGPU::SI_SPILL_AV320_RESTORE;
1932 case 44:
1933 return AMDGPU::SI_SPILL_AV352_RESTORE;
1934 case 48:
1935 return AMDGPU::SI_SPILL_AV384_RESTORE;
1936 case 64:
1937 return AMDGPU::SI_SPILL_AV512_RESTORE;
1938 case 128:
1939 return AMDGPU::SI_SPILL_AV1024_RESTORE;
1940 default:
1941 llvm_unreachable("unknown register size");
1942 }
1943}
1944
1945static unsigned getWWMRegSpillRestoreOpcode(unsigned Size,
1946 bool IsVectorSuperClass) {
1947 // Currently, there is only 32-bit WWM register spills needed.
1948 if (Size != 4)
1949 llvm_unreachable("unknown wwm register spill size");
1950
1951 if (IsVectorSuperClass) // TODO: Always use this if there are AGPRs
1952 return AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
1953
1954 return AMDGPU::SI_SPILL_WWM_V32_RESTORE;
1955}
1956
1957unsigned SIInstrInfo::getVectorRegSpillRestoreOpcode(
1958 Register Reg, const TargetRegisterClass *RC, unsigned Size,
1959 const SIMachineFunctionInfo &MFI) const {
1960 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
1961
1962 // Choose the right opcode if restoring a WWM register.
1963 if (MFI.checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
1964 return getWWMRegSpillRestoreOpcode(Size, IsVectorSuperClass);
1965
1966 // TODO: Check if AGPRs are available
1967 if (ST.hasMAIInsts())
1968 return getAVSpillRestoreOpcode(Size);
1969
1970 assert(!RI.isAGPRClass(RC));
1971 return getVGPRSpillRestoreOpcode(Size);
1972}
1973
1974void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1975 MachineBasicBlock::iterator MI,
1976 Register DestReg, int FrameIndex,
1977 const TargetRegisterClass *RC,
1978 Register VReg, unsigned SubReg,
1979 MachineInstr::MIFlag Flags) const {
1980 MachineFunction *MF = MBB.getParent();
1981 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1982 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1983 const DebugLoc &DL = MBB.findDebugLoc(MBBI: MI);
1984 unsigned SpillSize = RI.getSpillSize(RC: *RC);
1985
1986 MachinePointerInfo PtrInfo
1987 = MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
1988
1989 MachineMemOperand *MMO = MF->getMachineMemOperand(
1990 PtrInfo, F: MachineMemOperand::MOLoad, Size: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
1991 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
1992
1993 if (RI.isSGPRClass(RC)) {
1994 if (FrameInfo.getStackID(ObjectIdx: FrameIndex) == TargetStackID::SGPRSpill)
1995 MFI->setHasSpilledSGPRs();
1996 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into");
1997 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI &&
1998 DestReg != AMDGPU::EXEC && "exec should not be spilled");
1999
2000 // FIXME: Maybe this should not include a memoperand because it will be
2001 // lowered to non-memory instructions.
2002 const MCInstrDesc &OpDesc = get(Opcode: getSGPRSpillRestoreOpcode(Size: SpillSize));
2003 if (DestReg.isVirtual() && SpillSize == 4) {
2004 MachineRegisterInfo &MRI = MF->getRegInfo();
2005 MRI.constrainRegClass(Reg: DestReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
2006 }
2007
2008 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: OpDesc, DestReg)
2009 .addFrameIndex(Idx: FrameIndex) // addr
2010 .addMemOperand(MMO)
2011 .addReg(RegNo: MFI->getStackPtrOffsetReg(), Flags: RegState::Implicit);
2012
2013 return;
2014 }
2015
2016 unsigned Opcode = getVectorRegSpillRestoreOpcode(Reg: VReg ? VReg : DestReg, RC,
2017 Size: SpillSize, MFI: *MFI);
2018 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode), DestReg)
2019 .addFrameIndex(Idx: FrameIndex) // vaddr
2020 .addReg(RegNo: MFI->getStackPtrOffsetReg()) // scratch_offset
2021 .addImm(Val: 0) // offset
2022 .addMemOperand(MMO);
2023}
2024
2025void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
2026 MachineBasicBlock::iterator MI) const {
2027 insertNoops(MBB, MI, Quantity: 1);
2028}
2029
2030void SIInstrInfo::insertNoops(MachineBasicBlock &MBB,
2031 MachineBasicBlock::iterator MI,
2032 unsigned Quantity) const {
2033 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2034 unsigned MaxSNopCount = 1u << ST.getSNopBits();
2035 while (Quantity > 0) {
2036 unsigned Arg = std::min(a: Quantity, b: MaxSNopCount);
2037 Quantity -= Arg;
2038 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOP)).addImm(Val: Arg - 1);
2039 }
2040}
2041
2042MachineBasicBlock *SIInstrInfo::insertSimulatedTrap(MachineRegisterInfo &MRI,
2043 MachineBasicBlock &MBB,
2044 MachineInstr &MI,
2045 const DebugLoc &DL) const {
2046 MachineFunction *MF = MBB.getParent();
2047 constexpr unsigned DoorbellIDMask = 0x3ff;
2048 constexpr unsigned ECQueueWaveAbort = 0x400;
2049
2050 MachineBasicBlock *TrapBB = &MBB;
2051 MachineBasicBlock *HaltLoopBB = MF->CreateMachineBasicBlock();
2052
2053 if (!MBB.succ_empty() ||
2054 next_nodbg(It: MachineBasicBlock::iterator(MI), End: MBB.end(),
2055 /*SkipPseudoOp=*/false) != MBB.end()) {
2056 MBB.splitAt(SplitInst&: MI, /*UpdateLiveIns=*/false);
2057 TrapBB = MF->CreateMachineBasicBlock();
2058 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CBRANCH_EXECNZ)).addMBB(MBB: TrapBB);
2059 MF->push_back(MBB: TrapBB);
2060 MBB.addSuccessor(Succ: TrapBB);
2061 }
2062 // Start with a `s_trap 2`, if we're in PRIV=1 and we need the workaround this
2063 // will be a nop.
2064 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_TRAP))
2065 .addImm(Val: static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));
2066 Register DoorbellReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2067 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SENDMSG_RTN_B32),
2068 DestReg: DoorbellReg)
2069 .addImm(Val: AMDGPU::SendMsg::ID_RTN_GET_DOORBELL);
2070 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::TTMP2)
2071 .addUse(RegNo: AMDGPU::M0);
2072 Register DoorbellRegMasked =
2073 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2074 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_AND_B32), DestReg: DoorbellRegMasked)
2075 .addUse(RegNo: DoorbellReg)
2076 .addImm(Val: DoorbellIDMask)
2077 .setOperandDead(3); // implicit-def $scc
2078 Register SetWaveAbortBit =
2079 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2080 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_OR_B32), DestReg: SetWaveAbortBit)
2081 .addUse(RegNo: DoorbellRegMasked)
2082 .addImm(Val: ECQueueWaveAbort)
2083 .setOperandDead(3); // implicit-def $scc
2084 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
2085 .addUse(RegNo: SetWaveAbortBit);
2086 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SENDMSG))
2087 .addImm(Val: AMDGPU::SendMsg::ID_INTERRUPT);
2088 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
2089 .addUse(RegNo: AMDGPU::TTMP2);
2090 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH)).addMBB(MBB: HaltLoopBB);
2091 TrapBB->addSuccessor(Succ: HaltLoopBB);
2092
2093 BuildMI(BB&: *HaltLoopBB, I: HaltLoopBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETHALT)).addImm(Val: 5);
2094 BuildMI(BB&: *HaltLoopBB, I: HaltLoopBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
2095 .addMBB(MBB: HaltLoopBB);
2096 MF->push_back(MBB: HaltLoopBB);
2097 HaltLoopBB->addSuccessor(Succ: HaltLoopBB);
2098
2099 return MBB.getNextNode();
2100}
2101
2102unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) {
2103 switch (MI.getOpcode()) {
2104 default:
2105 if (MI.isMetaInstruction())
2106 return 0;
2107 return 1; // FIXME: Do wait states equal cycles?
2108
2109 case AMDGPU::S_NOP:
2110 return MI.getOperand(i: 0).getImm() + 1;
2111 // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The
2112 // hazard, even if one exist, won't really be visible. Should we handle it?
2113 }
2114}
2115
2116// Expand a VGPR "as memory" access into one indexed move per dword.
2117static void expandLoadStoreIdx(const SIInstrInfo &TII, MachineInstr &MI) {
2118 auto &LdSt = cast<AMDGPUMI::VLoadStoreIdxInst>(Val&: MI);
2119 MachineBasicBlock &MBB = *MI.getParent();
2120 MachineFunction &MF = *MBB.getParent();
2121 const DebugLoc &DL = MI.getDebugLoc();
2122 const SIRegisterInfo &RI = TII.getRegisterInfo();
2123 const bool IsStore = LdSt.mayStore();
2124
2125 Register Data = LdSt.getDataOp().getReg();
2126 unsigned Offset = LdSt.getOffsetOp().getImm();
2127 unsigned NumDwords = LdSt.getBitWidth() / 32;
2128
2129 const bool UseGPRIdxMode = LdSt.isGPRIdx();
2130 assert(UseGPRIdxMode == TII.getSubtarget().useVGPRIndexMode());
2131
2132 MachineInstr *SetOn = nullptr;
2133 if (UseGPRIdxMode) {
2134 SetOn = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_SET_GPR_IDX_ON))
2135 .add(MO: LdSt.getIdxOp())
2136 .addImm(Val: IsStore ? AMDGPU::VGPRIndexMode::DST_ENABLE
2137 : AMDGPU::VGPRIndexMode::SRC0_ENABLE);
2138 }
2139
2140 unsigned Opcode;
2141 if (UseGPRIdxMode) {
2142 Opcode = IsStore ? AMDGPU::V_MOV_B32_indirect_write
2143 : AMDGPU::V_MOV_B32_indirect_read;
2144 } else {
2145 Opcode =
2146 IsStore ? AMDGPU::V_MOVRELD_B32_as_mem : AMDGPU::V_MOVRELS_B32_as_mem;
2147 }
2148
2149 // A move touches its base plus the run-time index, which no operand can name,
2150 // so the base is undef and the liveness of those registers is not expressed.
2151 const RegState DataFlags = IsStore
2152 ? getUndefRegState(B: LdSt.getDataOp().isUndef())
2153 : RegState::NoFlags;
2154 for (unsigned i = 0; i < NumDwords; ++i) {
2155 Register Base = AMDGPU::VGPR_32RegClass.getRegister(i: Offset + i);
2156 Register Sub = Data;
2157 if (NumDwords != 1)
2158 Sub = RI.getSubReg(Reg: Data, Idx: RI.getSubRegFromChannel(Channel: i));
2159
2160 MachineInstrBuilder Move;
2161 if (IsStore) {
2162 Move = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode))
2163 .addReg(RegNo: Base, Flags: RegState::Undef)
2164 .addReg(RegNo: Sub, Flags: DataFlags);
2165 // As in copyPhysReg, the implicit use of the whole tuple keeps a partly
2166 // undefined value a valid operand.
2167 if (NumDwords != 1)
2168 Move.addReg(RegNo: Data, Flags: RegState::Implicit | DataFlags);
2169 } else {
2170 Move = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode), DestReg: Sub)
2171 .addReg(RegNo: Base, Flags: RegState::Undef);
2172 }
2173 // Each mode switch defines M0, which keeps the index-mode moves in order. A
2174 // movrel move only reads M0, so it is a load or store, ordered by the
2175 // memory operand of the dword it moves.
2176 if (!UseGPRIdxMode) {
2177 for (const MachineMemOperand *MMO : MI.memoperands())
2178 Move.addMemOperand(MMO: MF.getMachineMemOperand(MMO, Offset: 4 * i, Size: 4));
2179 }
2180 }
2181
2182 // Bundled so the post-RA scheduler keeps everything else out of the mode
2183 // switch.
2184 if (SetOn) {
2185 MachineInstr *SetOff =
2186 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_SET_GPR_IDX_OFF));
2187 finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2188 }
2189
2190 MI.eraseFromParent();
2191}
2192
2193bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2194 if (isa<AMDGPUMI::VLoadStoreIdxInst>(Val: MI)) {
2195 expandLoadStoreIdx(TII: *this, MI);
2196 return true;
2197 }
2198
2199 MachineBasicBlock &MBB = *MI.getParent();
2200 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2201 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
2202
2203 switch (MI.getOpcode()) {
2204 default: return TargetInstrInfo::expandPostRAPseudo(MI);
2205 case AMDGPU::S_MOV_B64_term:
2206 // This is only a terminator to get the correct spill code placement during
2207 // register allocation.
2208 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2209 break;
2210
2211 case AMDGPU::S_MOV_B32_term:
2212 // This is only a terminator to get the correct spill code placement during
2213 // register allocation.
2214 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B32));
2215 break;
2216
2217 case AMDGPU::S_XOR_B64_term:
2218 // This is only a terminator to get the correct spill code placement during
2219 // register allocation.
2220 MI.setDesc(get(Opcode: AMDGPU::S_XOR_B64));
2221 break;
2222
2223 case AMDGPU::S_XOR_B32_term:
2224 // This is only a terminator to get the correct spill code placement during
2225 // register allocation.
2226 MI.setDesc(get(Opcode: AMDGPU::S_XOR_B32));
2227 break;
2228 case AMDGPU::S_OR_B64_term:
2229 // This is only a terminator to get the correct spill code placement during
2230 // register allocation.
2231 MI.setDesc(get(Opcode: AMDGPU::S_OR_B64));
2232 break;
2233 case AMDGPU::S_OR_B32_term:
2234 // This is only a terminator to get the correct spill code placement during
2235 // register allocation.
2236 MI.setDesc(get(Opcode: AMDGPU::S_OR_B32));
2237 break;
2238
2239 case AMDGPU::S_ANDN2_B64_term:
2240 // This is only a terminator to get the correct spill code placement during
2241 // register allocation.
2242 MI.setDesc(get(Opcode: AMDGPU::S_ANDN2_B64));
2243 break;
2244
2245 case AMDGPU::S_ANDN2_B32_term:
2246 // This is only a terminator to get the correct spill code placement during
2247 // register allocation.
2248 MI.setDesc(get(Opcode: AMDGPU::S_ANDN2_B32));
2249 break;
2250
2251 case AMDGPU::S_AND_B64_term:
2252 // This is only a terminator to get the correct spill code placement during
2253 // register allocation.
2254 MI.setDesc(get(Opcode: AMDGPU::S_AND_B64));
2255 break;
2256
2257 case AMDGPU::S_AND_B32_term:
2258 // This is only a terminator to get the correct spill code placement during
2259 // register allocation.
2260 MI.setDesc(get(Opcode: AMDGPU::S_AND_B32));
2261 break;
2262
2263 case AMDGPU::S_AND_SAVEEXEC_B64_term:
2264 // This is only a terminator to get the correct spill code placement during
2265 // register allocation.
2266 MI.setDesc(get(Opcode: AMDGPU::S_AND_SAVEEXEC_B64));
2267 break;
2268
2269 case AMDGPU::S_AND_SAVEEXEC_B32_term:
2270 // This is only a terminator to get the correct spill code placement during
2271 // register allocation.
2272 MI.setDesc(get(Opcode: AMDGPU::S_AND_SAVEEXEC_B32));
2273 break;
2274
2275 case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
2276 MI.setDesc(get(Opcode: AMDGPU::V_CMPX_EQ_U32_nosdst_e32));
2277 break;
2278 case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
2279 MI.setDesc(get(Opcode: AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
2280 break;
2281
2282 case AMDGPU::SI_SPILL_S32_TO_VGPR:
2283 MI.setDesc(get(Opcode: AMDGPU::V_WRITELANE_B32));
2284 break;
2285
2286 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
2287 MI.setDesc(get(Opcode: AMDGPU::V_READLANE_B32));
2288 break;
2289 case AMDGPU::AV_MOV_B32_IMM_PSEUDO: {
2290 Register Dst = MI.getOperand(i: 0).getReg();
2291 bool IsAGPR = SIRegisterInfo::isAGPRClass(RC: RI.getPhysRegBaseClass(Reg: Dst));
2292 MI.setDesc(
2293 get(Opcode: IsAGPR ? AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::V_MOV_B32_e32));
2294 break;
2295 }
2296 case AMDGPU::AV_MOV_B64_IMM_PSEUDO: {
2297 Register Dst = MI.getOperand(i: 0).getReg();
2298 if (SIRegisterInfo::isAGPRClass(RC: RI.getPhysRegBaseClass(Reg: Dst))) {
2299 int64_t Imm = MI.getOperand(i: 1).getImm();
2300
2301 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2302 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2303 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: DstLo)
2304 .addImm(Val: SignExtend64<32>(x: Imm));
2305 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: DstHi)
2306 .addImm(Val: SignExtend64<32>(x: Imm >> 32));
2307 MI.eraseFromParent();
2308 break;
2309 }
2310
2311 [[fallthrough]];
2312 }
2313 case AMDGPU::V_MOV_B64_PSEUDO: {
2314 Register Dst = MI.getOperand(i: 0).getReg();
2315 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2316 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2317
2318 const MCInstrDesc &Mov64Desc = get(Opcode: AMDGPU::V_MOV_B64_e32);
2319 const TargetRegisterClass *Mov64RC = getRegClass(MCID: Mov64Desc, /*OpNum=*/0);
2320
2321 const MachineOperand &SrcOp = MI.getOperand(i: 1);
2322 // FIXME: Will this work for 64-bit floating point immediates?
2323 assert(!SrcOp.isFPImm());
2324 if (ST.hasVMovB64Inst() && Mov64RC->contains(Reg: Dst)) {
2325 MI.setDesc(Mov64Desc);
2326 if (SrcOp.isReg() || isInlineConstant(MI, OpIdx: 1) ||
2327 (SrcOp.isImm() &&
2328 (isUInt<32>(x: SrcOp.getImm()) || ST.has64BitLiterals())) ||
2329 (SrcOp.isGlobal() && ST.has64BitLiterals()))
2330 break;
2331 }
2332 if (SrcOp.isGlobal()) {
2333 // The address is unknown until link time, so the PK_MOV inline-constant
2334 // shortcut cannot apply.
2335 const GlobalValue *GV = SrcOp.getGlobal();
2336 int64_t Offset = SrcOp.getOffset();
2337 auto [LoFlags, HiFlags] = splitGlobalAddressRelocFlags(SrcOp);
2338
2339 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2340 .addGlobalAddress(GV, Offset, TargetFlags: LoFlags);
2341 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2342 .addGlobalAddress(GV, Offset, TargetFlags: HiFlags);
2343 } else if (SrcOp.isImm()) {
2344 APInt Imm(64, SrcOp.getImm());
2345 APInt Lo(32, Imm.getLoBits(numBits: 32).getZExtValue());
2346 APInt Hi(32, Imm.getHiBits(numBits: 32).getZExtValue());
2347 const MCInstrDesc &PkMovDesc = get(Opcode: AMDGPU::V_PK_MOV_B32);
2348 const TargetRegisterClass *PkMovRC = getRegClass(MCID: PkMovDesc, /*OpNum=*/0);
2349
2350 if (ST.hasPkMovB32() && Lo == Hi && isInlineConstant(Imm: Lo) &&
2351 PkMovRC->contains(Reg: Dst)) {
2352 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: PkMovDesc, DestReg: Dst)
2353 .addImm(Val: SISrcMods::OP_SEL_1)
2354 .addImm(Val: Lo.getSExtValue())
2355 .addImm(Val: SISrcMods::OP_SEL_1)
2356 .addImm(Val: Lo.getSExtValue())
2357 .addImm(Val: 0) // op_sel_lo
2358 .addImm(Val: 0) // op_sel_hi
2359 .addImm(Val: 0) // neg_lo
2360 .addImm(Val: 0) // neg_hi
2361 .addImm(Val: 0); // clamp
2362 } else {
2363 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2364 .addImm(Val: Lo.getSExtValue());
2365 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2366 .addImm(Val: Hi.getSExtValue());
2367 }
2368 } else {
2369 assert(SrcOp.isReg());
2370 if (ST.hasPkMovB32() &&
2371 !RI.isAGPR(MRI: MBB.getParent()->getRegInfo(), Reg: SrcOp.getReg())) {
2372 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg: Dst)
2373 .addImm(Val: SISrcMods::OP_SEL_1) // src0_mod
2374 .addReg(RegNo: SrcOp.getReg())
2375 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) // src1_mod
2376 .addReg(RegNo: SrcOp.getReg())
2377 .addImm(Val: 0) // op_sel_lo
2378 .addImm(Val: 0) // op_sel_hi
2379 .addImm(Val: 0) // neg_lo
2380 .addImm(Val: 0) // neg_hi
2381 .addImm(Val: 0); // clamp
2382 } else {
2383 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2384 .addReg(RegNo: RI.getSubReg(Reg: SrcOp.getReg(), Idx: AMDGPU::sub0));
2385 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2386 .addReg(RegNo: RI.getSubReg(Reg: SrcOp.getReg(), Idx: AMDGPU::sub1));
2387 }
2388 }
2389 MI.eraseFromParent();
2390 break;
2391 }
2392 case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
2393 expandMovDPP64(MI);
2394 break;
2395 }
2396 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2397 const MachineOperand &SrcOp = MI.getOperand(i: 1);
2398 assert(!SrcOp.isFPImm());
2399
2400 if (ST.has64BitLiterals()) {
2401 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2402 break;
2403 }
2404
2405 if (SrcOp.isGlobal()) {
2406 Register Dst = MI.getOperand(i: 0).getReg();
2407 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2408 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2409 const GlobalValue *GV = SrcOp.getGlobal();
2410 int64_t Offset = SrcOp.getOffset();
2411 auto [LoFlags, HiFlags] = splitGlobalAddressRelocFlags(SrcOp);
2412
2413 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstLo)
2414 .addGlobalAddress(GV, Offset, TargetFlags: LoFlags);
2415 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi)
2416 .addGlobalAddress(GV, Offset, TargetFlags: HiFlags);
2417 MI.eraseFromParent();
2418 break;
2419 }
2420
2421 // SrcOp is immediate
2422 APInt Imm(64, SrcOp.getImm());
2423 if (Imm.isIntN(N: 32) || isInlineConstant(Imm)) {
2424 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2425 break;
2426 }
2427
2428 Register Dst = MI.getOperand(i: 0).getReg();
2429 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2430 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2431
2432 APInt Lo(32, Imm.getLoBits(numBits: 32).getZExtValue());
2433 APInt Hi(32, Imm.getHiBits(numBits: 32).getZExtValue());
2434 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstLo)
2435 .addImm(Val: Lo.getSExtValue());
2436 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi)
2437 .addImm(Val: Hi.getSExtValue());
2438 MI.eraseFromParent();
2439 break;
2440 }
2441 case AMDGPU::V_SET_INACTIVE_B32: {
2442 // Lower V_SET_INACTIVE_B32 to V_CNDMASK_B32.
2443 Register DstReg = MI.getOperand(i: 0).getReg();
2444 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
2445 .add(MO: MI.getOperand(i: 3))
2446 .add(MO: MI.getOperand(i: 4))
2447 .add(MO: MI.getOperand(i: 1))
2448 .add(MO: MI.getOperand(i: 2))
2449 .add(MO: MI.getOperand(i: 5));
2450 MI.eraseFromParent();
2451 break;
2452 }
2453 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2454 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2455 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2456 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2457 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2458 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V6:
2459 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V7:
2460 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2461 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2462 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2463 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2464 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2465 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2466 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2467 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2468 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2469 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2470 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2471 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2472 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V6:
2473 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V7:
2474 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2475 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2476 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2477 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2478 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2479 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2480 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2481 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:
2482 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:
2483 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:
2484 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:
2485 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {
2486 const TargetRegisterClass *EltRC = getOpRegClass(MI, OpNo: 2);
2487
2488 unsigned Opc;
2489 if (RI.hasVGPRs(RC: EltRC)) {
2490 Opc = AMDGPU::V_MOVRELD_B32_e32;
2491 } else {
2492 Opc = RI.getRegSizeInBits(RC: *EltRC) == 64 ? AMDGPU::S_MOVRELD_B64
2493 : AMDGPU::S_MOVRELD_B32;
2494 }
2495
2496 const MCInstrDesc &OpDesc = get(Opcode: Opc);
2497 Register VecReg = MI.getOperand(i: 0).getReg();
2498 bool IsUndef = MI.getOperand(i: 1).isUndef();
2499 unsigned SubReg = MI.getOperand(i: 3).getImm();
2500 assert(VecReg == MI.getOperand(1).getReg());
2501
2502 MachineInstrBuilder MIB =
2503 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: OpDesc)
2504 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2505 .add(MO: MI.getOperand(i: 2))
2506 .addReg(RegNo: VecReg, Flags: RegState::ImplicitDefine)
2507 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2508
2509 const int ImpDefIdx =
2510 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2511 const int ImpUseIdx = ImpDefIdx + 1;
2512 MIB->tieOperands(DefIdx: ImpDefIdx, UseIdx: ImpUseIdx);
2513 MI.eraseFromParent();
2514 break;
2515 }
2516 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:
2517 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:
2518 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:
2519 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:
2520 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:
2521 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V6:
2522 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V7:
2523 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:
2524 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9:
2525 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10:
2526 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11:
2527 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12:
2528 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:
2529 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {
2530 assert(ST.useVGPRIndexMode());
2531 Register VecReg = MI.getOperand(i: 0).getReg();
2532 bool IsUndef = MI.getOperand(i: 1).isUndef();
2533 MachineOperand &Idx = MI.getOperand(i: 3);
2534 Register SubReg = MI.getOperand(i: 4).getImm();
2535
2536 MachineInstr *SetOn = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_ON))
2537 .add(MO: Idx)
2538 .addImm(Val: AMDGPU::VGPRIndexMode::DST_ENABLE);
2539 SetOn->getOperand(i: 3).setIsUndef();
2540
2541 const MCInstrDesc &OpDesc = get(Opcode: AMDGPU::V_MOV_B32_indirect_write);
2542 MachineInstrBuilder MIB =
2543 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: OpDesc)
2544 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2545 .add(MO: MI.getOperand(i: 2))
2546 .addReg(RegNo: VecReg, Flags: RegState::ImplicitDefine)
2547 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2548
2549 const int ImpDefIdx =
2550 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2551 const int ImpUseIdx = ImpDefIdx + 1;
2552 MIB->tieOperands(DefIdx: ImpDefIdx, UseIdx: ImpUseIdx);
2553
2554 MachineInstr *SetOff = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_OFF));
2555
2556 finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2557
2558 MI.eraseFromParent();
2559 break;
2560 }
2561 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:
2562 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:
2563 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:
2564 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:
2565 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:
2566 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V6:
2567 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V7:
2568 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:
2569 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9:
2570 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10:
2571 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11:
2572 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12:
2573 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:
2574 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {
2575 assert(ST.useVGPRIndexMode());
2576 Register Dst = MI.getOperand(i: 0).getReg();
2577 Register VecReg = MI.getOperand(i: 1).getReg();
2578 bool IsUndef = MI.getOperand(i: 1).isUndef();
2579 Register SubReg = MI.getOperand(i: 3).getImm();
2580
2581 MachineInstr *SetOn = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_ON))
2582 .add(MO: MI.getOperand(i: 2))
2583 .addImm(Val: AMDGPU::VGPRIndexMode::SRC0_ENABLE);
2584 SetOn->getOperand(i: 3).setIsUndef();
2585
2586 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_indirect_read))
2587 .addDef(RegNo: Dst)
2588 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2589 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2590
2591 MachineInstr *SetOff = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_OFF));
2592
2593 finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2594
2595 MI.eraseFromParent();
2596 break;
2597 }
2598 case AMDGPU::SI_PC_ADD_REL_OFFSET: {
2599 MachineFunction &MF = *MBB.getParent();
2600 Register Reg = MI.getOperand(i: 0).getReg();
2601 Register RegLo = RI.getSubReg(Reg, Idx: AMDGPU::sub0);
2602 Register RegHi = RI.getSubReg(Reg, Idx: AMDGPU::sub1);
2603 MachineOperand OpLo = MI.getOperand(i: 1);
2604 MachineOperand OpHi = MI.getOperand(i: 2);
2605
2606 // Create a bundle so these instructions won't be re-ordered by the
2607 // post-RA scheduler.
2608 MIBundleBuilder Bundler(MBB, MI);
2609 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: Reg));
2610
2611 // What we want here is an offset from the value returned by s_getpc (which
2612 // is the address of the s_add_u32 instruction) to the global variable, but
2613 // since the encoding of $symbol starts 4 bytes after the start of the
2614 // s_add_u32 instruction, we end up with an offset that is 4 bytes too
2615 // small. This requires us to add 4 to the global variable offset in order
2616 // to compute the correct address. Similarly for the s_addc_u32 instruction,
2617 // the encoding of $symbol starts 12 bytes after the start of the s_add_u32
2618 // instruction.
2619
2620 int64_t Adjust = 0;
2621 if (ST.hasGetPCZeroExtension()) {
2622 // Fix up hardware that does not sign-extend the 48-bit PC value by
2623 // inserting: s_sext_i32_i16 reghi, reghi
2624 Bundler.append(
2625 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SEXT_I32_I16), DestReg: RegHi).addReg(RegNo: RegHi));
2626 Adjust += 4;
2627 }
2628
2629 if (OpLo.isGlobal())
2630 OpLo.setOffset(OpLo.getOffset() + Adjust + 4);
2631 Bundler.append(
2632 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U32), DestReg: RegLo).addReg(RegNo: RegLo).add(MO: OpLo));
2633
2634 if (OpHi.isGlobal())
2635 OpHi.setOffset(OpHi.getOffset() + Adjust + 12);
2636 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADDC_U32), DestReg: RegHi)
2637 .addReg(RegNo: RegHi)
2638 .add(MO: OpHi));
2639
2640 finalizeBundle(MBB, FirstMI: Bundler.begin());
2641
2642 MI.eraseFromParent();
2643 break;
2644 }
2645 case AMDGPU::SI_PC_ADD_REL_OFFSET64: {
2646 MachineFunction &MF = *MBB.getParent();
2647 Register Reg = MI.getOperand(i: 0).getReg();
2648 MachineOperand Op = MI.getOperand(i: 1);
2649
2650 // Create a bundle so these instructions won't be re-ordered by the
2651 // post-RA scheduler.
2652 MIBundleBuilder Bundler(MBB, MI);
2653 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: Reg));
2654 if (Op.isGlobal())
2655 Op.setOffset(Op.getOffset() + 4);
2656 Bundler.append(
2657 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U64), DestReg: Reg).addReg(RegNo: Reg).add(MO: Op));
2658
2659 finalizeBundle(MBB, FirstMI: Bundler.begin());
2660
2661 MI.eraseFromParent();
2662 break;
2663 }
2664 case AMDGPU::ENTER_STRICT_WWM: {
2665 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2666 // Whole Wave Mode is entered.
2667 MI.setDesc(get(Opcode: LMC.OrSaveExecOpc));
2668 break;
2669 }
2670 case AMDGPU::ENTER_STRICT_WQM: {
2671 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2672 // STRICT_WQM is entered.
2673 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: LMC.MovOpc), DestReg: MI.getOperand(i: 0).getReg())
2674 .addReg(RegNo: LMC.ExecReg);
2675 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: LMC.WQMOpc), DestReg: LMC.ExecReg).addReg(RegNo: LMC.ExecReg);
2676
2677 MI.eraseFromParent();
2678 break;
2679 }
2680 case AMDGPU::EXIT_STRICT_WWM:
2681 case AMDGPU::EXIT_STRICT_WQM: {
2682 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2683 // WWM/STICT_WQM is exited.
2684 MI.setDesc(get(Opcode: LMC.MovOpc));
2685 break;
2686 }
2687 case AMDGPU::SI_RETURN: {
2688 const MachineFunction *MF = MBB.getParent();
2689 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2690 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2691 // Hiding the return address use with SI_RETURN may lead to extra kills in
2692 // the function and missing live-ins. We are fine in practice because callee
2693 // saved register handling ensures the register value is restored before
2694 // RET, but we need the undef flag here to appease the MachineVerifier
2695 // liveness checks.
2696 MachineInstrBuilder MIB =
2697 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETPC_B64_return))
2698 .addReg(RegNo: TRI->getReturnAddressReg(MF: *MF), Flags: RegState::Undef);
2699
2700 MIB.copyImplicitOps(OtherMI: MI);
2701 MI.eraseFromParent();
2702 break;
2703 }
2704
2705 case AMDGPU::S_MUL_U64_U32_PSEUDO:
2706 case AMDGPU::S_MUL_I64_I32_PSEUDO:
2707 MI.setDesc(get(Opcode: AMDGPU::S_MUL_U64));
2708 break;
2709
2710 case AMDGPU::S_GETPC_B64_pseudo:
2711 MI.setDesc(get(Opcode: AMDGPU::S_GETPC_B64));
2712 if (ST.hasGetPCZeroExtension()) {
2713 Register Dst = MI.getOperand(i: 0).getReg();
2714 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2715 // Fix up hardware that does not sign-extend the 48-bit PC value by
2716 // inserting: s_sext_i32_i16 dsthi, dsthi
2717 BuildMI(BB&: MBB, I: std::next(x: MI.getIterator()), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SEXT_I32_I16),
2718 DestReg: DstHi)
2719 .addReg(RegNo: DstHi);
2720 }
2721 break;
2722
2723 case AMDGPU::V_MAX_BF16_PSEUDO_e64: {
2724 assert(ST.hasBF16PackedInsts());
2725 MI.setDesc(get(Opcode: AMDGPU::V_PK_MAX_NUM_BF16));
2726 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // op_sel
2727 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // neg_lo
2728 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // neg_hi
2729 auto Op0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
2730 Op0->setImm(Op0->getImm() | SISrcMods::OP_SEL_1);
2731 auto Op1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
2732 Op1->setImm(Op1->getImm() | SISrcMods::OP_SEL_1);
2733 break;
2734 }
2735
2736 case AMDGPU::GET_STACK_BASE:
2737 // The stack starts at offset 0 unless we need to reserve some space at the
2738 // bottom.
2739 if (ST.getFrameLowering()->mayReserveScratchForCWSR(MF: *MBB.getParent())) {
2740 // When CWSR is used in dynamic VGPR mode, the trap handler needs to save
2741 // some of the VGPRs. The size of the required scratch space has already
2742 // been computed by prolog epilog insertion.
2743 const SIMachineFunctionInfo *MFI =
2744 MBB.getParent()->getInfo<SIMachineFunctionInfo>();
2745 unsigned VGPRSize = MFI->getScratchReservedForDynamicVGPRs();
2746 Register DestReg = MI.getOperand(i: 0).getReg();
2747 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETREG_B32), DestReg)
2748 .addImm(Val: AMDGPU::Hwreg::HwregEncoding::encode(
2749 Values: AMDGPU::Hwreg::ID_HW_ID2, Values: AMDGPU::Hwreg::OFFSET_ME_ID, Values: 2));
2750 // The MicroEngine ID is 0 for the graphics queue, and 1 or 2 for compute
2751 // (3 is unused, so we ignore it). Unfortunately, S_GETREG doesn't set
2752 // SCC, so we need to check for 0 manually.
2753 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U32)).addImm(Val: 0).addReg(RegNo: DestReg);
2754 // Change the implicif-def of SCC to an explicit use (but first remove
2755 // the dead flag if present).
2756 MI.getOperand(i: MI.getNumExplicitOperands()).setIsDead(false);
2757 MI.getOperand(i: MI.getNumExplicitOperands()).setIsUse();
2758 MI.setDesc(get(Opcode: AMDGPU::S_CMOVK_I32));
2759 MI.addOperand(Op: MachineOperand::CreateImm(Val: VGPRSize));
2760 } else {
2761 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B32));
2762 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2763 MI.removeOperand(
2764 OpNo: MI.getNumExplicitOperands()); // Drop implicit def of SCC.
2765 }
2766 break;
2767 }
2768
2769 return true;
2770}
2771
2772void SIInstrInfo::reMaterialize(MachineBasicBlock &MBB,
2773 MachineBasicBlock::iterator I, Register DestReg,
2774 unsigned SubIdx, const MachineInstr &Orig,
2775 LaneBitmask UsedLanes) const {
2776
2777 // Try shrinking the instruction to remat only the part needed for current
2778 // context.
2779 // TODO: Handle more cases.
2780 unsigned Opcode = Orig.getOpcode();
2781 switch (Opcode) {
2782 case AMDGPU::S_MOV_B64:
2783 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2784 if (SubIdx != 0)
2785 break;
2786
2787 if (!Orig.getOperand(i: 1).isImm())
2788 break;
2789
2790 // Shrink S_MOV_B64 to S_MOV_B32 when UsedLanes indicates only a single
2791 // 32-bit lane of the 64-bit value is live at the rematerialization point.
2792 if (UsedLanes.all())
2793 break;
2794
2795 // Determine which half of the 64-bit immediate corresponds to the use.
2796 unsigned OrigSubReg = Orig.getOperand(i: 0).getSubReg();
2797 unsigned LoSubReg = RI.composeSubRegIndices(a: OrigSubReg, b: AMDGPU::sub0);
2798 unsigned HiSubReg = RI.composeSubRegIndices(a: OrigSubReg, b: AMDGPU::sub1);
2799
2800 bool NeedLo = (UsedLanes & RI.getSubRegIndexLaneMask(SubIdx: LoSubReg)).any();
2801 bool NeedHi = (UsedLanes & RI.getSubRegIndexLaneMask(SubIdx: HiSubReg)).any();
2802
2803 if (NeedLo && NeedHi)
2804 break;
2805
2806 int64_t Imm64 = Orig.getOperand(i: 1).getImm();
2807 int32_t Imm32 = NeedLo ? Lo_32(Value: Imm64) : Hi_32(Value: Imm64);
2808
2809 unsigned UseSubReg = NeedLo ? LoSubReg : HiSubReg;
2810
2811 // Emit S_MOV_B32 defining just the needed 32-bit subreg of DestReg.
2812 BuildMI(BB&: MBB, I, MIMD: Orig.getDebugLoc(), MCID: get(Opcode: AMDGPU::S_MOV_B32))
2813 .addReg(RegNo: DestReg, Flags: RegState::Define | RegState::Undef, SubReg: UseSubReg)
2814 .addImm(Val: Imm32);
2815 return;
2816 }
2817
2818 case AMDGPU::S_LOAD_DWORDX16_IMM:
2819 case AMDGPU::S_LOAD_DWORDX8_IMM: {
2820 if (SubIdx != 0 || UsedLanes.all())
2821 break;
2822
2823 if (I == MBB.end())
2824 break;
2825
2826 if (I->isBundled())
2827 break;
2828
2829 MachineFunction *MF = MBB.getParent();
2830 MachineRegisterInfo &MRI = MF->getRegInfo();
2831 assert(MRI.use_nodbg_empty(DestReg) && "DestReg should have no users yet.");
2832
2833 // Look for a single use of the register at the insertion point that is also
2834 // a subreg.
2835 Register RegToFind = Orig.getOperand(i: 0).getReg();
2836 MachineOperand *UseMO = MRI.getOneNonDBGUse(RegNo: RegToFind);
2837 if (!UseMO ||
2838 UseMO != I->findRegisterUseOperand(Reg: RegToFind, /*TRI=*/nullptr) ||
2839 UseMO->getSubReg() == AMDGPU::NoSubRegister)
2840 break;
2841
2842 unsigned Offset = RI.getSubRegIdxOffset(Idx: UseMO->getSubReg());
2843 unsigned SubregSize = RI.getSubRegIdxSize(Idx: UseMO->getSubReg());
2844
2845 unsigned NewOpcode = -1;
2846 if (SubregSize == 256)
2847 NewOpcode = AMDGPU::S_LOAD_DWORDX8_IMM;
2848 else if (SubregSize == 128)
2849 NewOpcode = AMDGPU::S_LOAD_DWORDX4_IMM;
2850 else
2851 break;
2852
2853 const MCInstrDesc &TID = get(Opcode: NewOpcode);
2854 const TargetRegisterClass *NewRC =
2855 RI.getAllocatableClass(RC: getRegClass(MCID: TID, OpNum: 0));
2856 MRI.setRegClass(Reg: DestReg, RC: NewRC);
2857
2858 UseMO->setReg(DestReg);
2859 UseMO->setSubReg(AMDGPU::NoSubRegister);
2860
2861 // Use a smaller load with the desired size, possibly with updated offset.
2862 MachineInstr *MI = MF->CloneMachineInstr(Orig: &Orig);
2863 MI->setDesc(TID);
2864 MI->getOperand(i: 0).setReg(DestReg);
2865 MI->getOperand(i: 0).setSubReg(AMDGPU::NoSubRegister);
2866 if (Offset) {
2867 MachineOperand *OffsetMO = getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::offset);
2868 int64_t FinalOffset = OffsetMO->getImm() + Offset / 8;
2869 OffsetMO->setImm(FinalOffset);
2870 }
2871 SmallVector<MachineMemOperand *> NewMMOs;
2872 for (const MachineMemOperand *MemOp : Orig.memoperands())
2873 NewMMOs.push_back(Elt: MF->getMachineMemOperand(MMO: MemOp, PtrInfo: MemOp->getPointerInfo(),
2874 Size: SubregSize / 8));
2875 MI->setMemRefs(MF&: *MF, MemRefs: NewMMOs);
2876
2877 MBB.insert(I, MI);
2878 return;
2879 }
2880
2881 default:
2882 break;
2883 }
2884
2885 TargetInstrInfo::reMaterialize(MBB, MI: I, DestReg, SubIdx, Orig, UsedLanes);
2886}
2887
2888std::pair<MachineInstr*, MachineInstr*>
2889SIInstrInfo::expandMovDPP64(MachineInstr &MI) const {
2890 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
2891
2892 if (ST.hasVMovB64Inst() && ST.hasFeature(Feature: AMDGPU::FeatureDPALU_DPP) &&
2893 AMDGPU::isLegalDPALU_DPPControl(
2894 ST, DC: getNamedOperand(MI, OperandName: AMDGPU::OpName::dpp_ctrl)->getImm())) {
2895 MI.setDesc(get(Opcode: AMDGPU::V_MOV_B64_dpp));
2896 return std::pair(&MI, nullptr);
2897 }
2898
2899 MachineBasicBlock &MBB = *MI.getParent();
2900 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2901 MachineFunction *MF = MBB.getParent();
2902 MachineRegisterInfo &MRI = MF->getRegInfo();
2903 Register Dst = MI.getOperand(i: 0).getReg();
2904 unsigned Part = 0;
2905 MachineInstr *Split[2];
2906
2907 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
2908 auto MovDPP = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_dpp));
2909 if (Dst.isPhysical()) {
2910 MovDPP.addDef(RegNo: RI.getSubReg(Reg: Dst, Idx: Sub));
2911 } else {
2912 assert(MRI.isSSA());
2913 auto Tmp = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2914 MovDPP.addDef(RegNo: Tmp);
2915 }
2916
2917 for (unsigned I = 1; I <= 2; ++I) { // old and src operands.
2918 const MachineOperand &SrcOp = MI.getOperand(i: I);
2919 assert(!SrcOp.isFPImm());
2920 if (SrcOp.isImm()) {
2921 APInt Imm(64, SrcOp.getImm());
2922 Imm.ashrInPlace(ShiftAmt: Part * 32);
2923 MovDPP.addImm(Val: Imm.getLoBits(numBits: 32).getZExtValue());
2924 } else {
2925 assert(SrcOp.isReg());
2926 Register Src = SrcOp.getReg();
2927 if (Src.isPhysical())
2928 MovDPP.addReg(RegNo: RI.getSubReg(Reg: Src, Idx: Sub));
2929 else
2930 MovDPP.addReg(RegNo: Src, Flags: getUndefRegState(B: SrcOp.isUndef()), SubReg: Sub);
2931 }
2932 }
2933
2934 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.explicit_operands(), N: 3))
2935 MovDPP.addImm(Val: MO.getImm());
2936
2937 Split[Part] = MovDPP;
2938 ++Part;
2939 }
2940
2941 if (Dst.isVirtual())
2942 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
2943 .addReg(RegNo: Split[0]->getOperand(i: 0).getReg())
2944 .addImm(Val: AMDGPU::sub0)
2945 .addReg(RegNo: Split[1]->getOperand(i: 0).getReg())
2946 .addImm(Val: AMDGPU::sub1);
2947
2948 MI.eraseFromParent();
2949 return std::pair(Split[0], Split[1]);
2950}
2951
2952std::optional<DestSourcePair>
2953SIInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
2954 if (MI.getOpcode() == AMDGPU::WWM_COPY)
2955 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2956
2957 return std::nullopt;
2958}
2959
2960bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,
2961 AMDGPU::OpName Src0OpName,
2962 MachineOperand &Src1,
2963 AMDGPU::OpName Src1OpName) const {
2964 MachineOperand *Src0Mods = getNamedOperand(MI, OperandName: Src0OpName);
2965 if (!Src0Mods)
2966 return false;
2967
2968 MachineOperand *Src1Mods = getNamedOperand(MI, OperandName: Src1OpName);
2969 assert(Src1Mods &&
2970 "All commutable instructions have both src0 and src1 modifiers");
2971
2972 int Src0ModsVal = Src0Mods->getImm();
2973 int Src1ModsVal = Src1Mods->getImm();
2974
2975 Src1Mods->setImm(Src0ModsVal);
2976 Src0Mods->setImm(Src1ModsVal);
2977 return true;
2978}
2979
2980static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI,
2981 MachineOperand &RegOp,
2982 MachineOperand &NonRegOp) {
2983 Register Reg = RegOp.getReg();
2984 unsigned SubReg = RegOp.getSubReg();
2985 bool IsKill = RegOp.isKill();
2986 bool IsDead = RegOp.isDead();
2987 bool IsUndef = RegOp.isUndef();
2988 bool IsDebug = RegOp.isDebug();
2989
2990 if (NonRegOp.isImm())
2991 RegOp.ChangeToImmediate(ImmVal: NonRegOp.getImm());
2992 else if (NonRegOp.isFI())
2993 RegOp.ChangeToFrameIndex(Idx: NonRegOp.getIndex());
2994 else if (NonRegOp.isGlobal()) {
2995 RegOp.ChangeToGA(GV: NonRegOp.getGlobal(), Offset: NonRegOp.getOffset(),
2996 TargetFlags: NonRegOp.getTargetFlags());
2997 } else
2998 return nullptr;
2999
3000 // Make sure we don't reinterpret a subreg index in the target flags.
3001 RegOp.setTargetFlags(NonRegOp.getTargetFlags());
3002
3003 NonRegOp.ChangeToRegister(Reg, isDef: false, isImp: false, isKill: IsKill, isDead: IsDead, isUndef: IsUndef, isDebug: IsDebug);
3004 NonRegOp.setSubReg(SubReg);
3005
3006 return &MI;
3007}
3008
3009static MachineInstr *swapImmOperands(MachineInstr &MI,
3010 MachineOperand &NonRegOp1,
3011 MachineOperand &NonRegOp2) {
3012 unsigned TargetFlags = NonRegOp1.getTargetFlags();
3013 int64_t NonRegVal = NonRegOp1.getImm();
3014
3015 NonRegOp1.setImm(NonRegOp2.getImm());
3016 NonRegOp2.setImm(NonRegVal);
3017 NonRegOp1.setTargetFlags(NonRegOp2.getTargetFlags());
3018 NonRegOp2.setTargetFlags(TargetFlags);
3019 return &MI;
3020}
3021
3022// GFX950 v_pk_{add,mul}_f32 must not have opsel[1:0] == 2'b10, i.e. OP_SEL_0 on
3023// src1 but not on src0, and v_pk_fma_f32 must not have OP_SEL_0 on src1 or
3024// src2.
3025static bool isLegalPkF32OpSel(const SIInstrInfo &TII, const MachineInstr &MI,
3026 bool CheckSwappedSrc = false) {
3027 unsigned Opc = MI.getOpcode();
3028 if (!TII.getSubtarget().hasPkF32InvalidOpSel() ||
3029 (Opc != AMDGPU::V_PK_ADD_F32 && Opc != AMDGPU::V_PK_MUL_F32 &&
3030 Opc != AMDGPU::V_PK_FMA_F32))
3031 return true;
3032
3033 auto Mods = [&](AMDGPU::OpName Name) {
3034 const MachineOperand *Op = TII.getNamedOperand(MI, OperandName: Name);
3035 return Op ? static_cast<unsigned>(Op->getImm()) : 0u;
3036 };
3037 unsigned Src0Mods = Mods(AMDGPU::OpName::src0_modifiers);
3038 unsigned Src1Mods = Mods(AMDGPU::OpName::src1_modifiers);
3039 unsigned Src2Mods = Mods(AMDGPU::OpName::src2_modifiers);
3040
3041 if (CheckSwappedSrc)
3042 std::swap(a&: Src0Mods, b&: Src1Mods);
3043
3044 if (Opc == AMDGPU::V_PK_FMA_F32)
3045 return !((Src1Mods | Src2Mods) & SISrcMods::OP_SEL_0);
3046 return (Src0Mods & SISrcMods::OP_SEL_0) || !(Src1Mods & SISrcMods::OP_SEL_0);
3047}
3048
3049bool SIInstrInfo::isLegalToSwap(const MachineInstr &MI, unsigned OpIdx0,
3050 unsigned OpIdx1) const {
3051 const MCInstrDesc &InstDesc = MI.getDesc();
3052 const MCOperandInfo &OpInfo0 = InstDesc.operands()[OpIdx0];
3053 const MCOperandInfo &OpInfo1 = InstDesc.operands()[OpIdx1];
3054
3055 unsigned Opc = MI.getOpcode();
3056 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
3057 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
3058
3059 // Swapping src0 and src1 swaps their modifiers; refuse a swap that would
3060 // leave an invalid GFX950 packed FP32 op_sel.
3061 bool IsSrc01Swap = ((int)OpIdx0 == Src0Idx && (int)OpIdx1 == Src1Idx) ||
3062 ((int)OpIdx0 == Src1Idx && (int)OpIdx1 == Src0Idx);
3063 if (IsSrc01Swap && !isLegalPkF32OpSel(TII: *this, MI, CheckSwappedSrc: true))
3064 return false;
3065
3066 const MachineOperand &MO0 = MI.getOperand(i: OpIdx0);
3067 const MachineOperand &MO1 = MI.getOperand(i: OpIdx1);
3068
3069 // Swap doesn't breach constant bus or literal limits, but it may move a
3070 // constant out of src0. Allow that for immediates where the destination
3071 // operand can hold them, e.g. VOP3 sources on targets with VOP3 literal
3072 // support. Other non-register operands stay in src0.
3073 if (isVALU(MI, /*AllowLDSDMA=*/false)) {
3074 if ((int)OpIdx0 == Src0Idx && !MO0.isReg() &&
3075 (!MO0.isImm() || !isImmOperandLegal(MI, OpNo: OpIdx1, MO: MO0)))
3076 return false;
3077 if ((int)OpIdx1 == Src0Idx && !MO1.isReg() &&
3078 (!MO1.isImm() || !isImmOperandLegal(MI, OpNo: OpIdx0, MO: MO1)))
3079 return false;
3080 }
3081
3082 if ((int)OpIdx1 != Src0Idx && MO0.isReg()) {
3083 if (OpInfo1.RegClass == -1)
3084 return OpInfo1.OperandType == MCOI::OPERAND_UNKNOWN;
3085 return isLegalRegOperand(MI, OpIdx: OpIdx1, MO: MO0) &&
3086 (!MO1.isReg() || isLegalRegOperand(MI, OpIdx: OpIdx0, MO: MO1));
3087 }
3088 if ((int)OpIdx0 != Src0Idx && MO1.isReg()) {
3089 if (OpInfo0.RegClass == -1)
3090 return OpInfo0.OperandType == MCOI::OPERAND_UNKNOWN;
3091 return (!MO0.isReg() || isLegalRegOperand(MI, OpIdx: OpIdx1, MO: MO0)) &&
3092 isLegalRegOperand(MI, OpIdx: OpIdx0, MO: MO1);
3093 }
3094
3095 // No need to check 64-bit literals since swapping does not bring new
3096 // 64-bit literals into current instruction to fold to 32-bit
3097
3098 return isImmOperandLegal(MI, OpNo: OpIdx1, MO: MO0);
3099}
3100
3101bool SIInstrInfo::isNonCommutableDPP(const MachineInstr &MI) const {
3102 if (!isDPP(MI))
3103 return false;
3104 const MachineOperand *DppCtrl = getNamedOperand(MI, OperandName: AMDGPU::OpName::dpp_ctrl);
3105 return !DppCtrl || DppCtrl->getImm() != AMDGPU::DPP::QUAD_PERM_ID;
3106}
3107
3108MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
3109 unsigned Src0Idx,
3110 unsigned Src1Idx) const {
3111 assert(!NewMI && "this should never be used");
3112
3113 if (isNonCommutableDPP(MI))
3114 return nullptr;
3115
3116 unsigned Opc = MI.getOpcode();
3117 int CommutedOpcode = commuteOpcode(Opcode: Opc);
3118 if (CommutedOpcode == -1)
3119 return nullptr;
3120
3121 if (Src0Idx > Src1Idx)
3122 std::swap(a&: Src0Idx, b&: Src1Idx);
3123
3124 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
3125 static_cast<int>(Src0Idx) &&
3126 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
3127 static_cast<int>(Src1Idx) &&
3128 "inconsistency with findCommutedOpIndices");
3129
3130 if (!isLegalToSwap(MI, OpIdx0: Src0Idx, OpIdx1: Src1Idx))
3131 return nullptr;
3132
3133 MachineInstr *CommutedMI = nullptr;
3134 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
3135 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
3136 if (Src0.isReg() && Src1.isReg()) {
3137 // Be sure to copy the source modifiers to the right place.
3138 CommutedMI =
3139 TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1: Src0Idx, OpIdx2: Src1Idx);
3140 } else if (Src0.isReg() && !Src1.isReg()) {
3141 CommutedMI = swapRegAndNonRegOperand(MI, RegOp&: Src0, NonRegOp&: Src1);
3142 } else if (!Src0.isReg() && Src1.isReg()) {
3143 CommutedMI = swapRegAndNonRegOperand(MI, RegOp&: Src1, NonRegOp&: Src0);
3144 } else if (Src0.isImm() && Src1.isImm()) {
3145 CommutedMI = swapImmOperands(MI, NonRegOp1&: Src0, NonRegOp2&: Src1);
3146 } else {
3147 // FIXME: Found two non registers to commute. This does happen.
3148 return nullptr;
3149 }
3150
3151 if (CommutedMI) {
3152 swapSourceModifiers(MI, Src0, Src0OpName: AMDGPU::OpName::src0_modifiers,
3153 Src1, Src1OpName: AMDGPU::OpName::src1_modifiers);
3154
3155 swapSourceModifiers(MI, Src0, Src0OpName: AMDGPU::OpName::src0_sel, Src1,
3156 Src1OpName: AMDGPU::OpName::src1_sel);
3157
3158 CommutedMI->setDesc(get(Opcode: CommutedOpcode));
3159 }
3160
3161 return CommutedMI;
3162}
3163
3164// This needs to be implemented because the source modifiers may be inserted
3165// between the true commutable operands, and the base
3166// TargetInstrInfo::commuteInstruction uses it.
3167bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
3168 unsigned &SrcOpIdx0,
3169 unsigned &SrcOpIdx1) const {
3170 if (isNonCommutableDPP(MI))
3171 return false;
3172
3173 return findCommutedOpIndices(Desc: MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
3174}
3175
3176bool SIInstrInfo::findCommutedOpIndices(const MCInstrDesc &Desc,
3177 unsigned &SrcOpIdx0,
3178 unsigned &SrcOpIdx1) const {
3179 if (!Desc.isCommutable())
3180 return false;
3181
3182 unsigned Opc = Desc.getOpcode();
3183 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
3184 if (Src0Idx == -1)
3185 return false;
3186
3187 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
3188 if (Src1Idx == -1)
3189 return false;
3190
3191 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx0, ResultIdx2&: SrcOpIdx1, CommutableOpIdx1: Src0Idx, CommutableOpIdx2: Src1Idx);
3192}
3193
3194bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
3195 int64_t BrOffset) const {
3196 // BranchRelaxation should never have to check s_setpc_b64 or s_add_pc_i64
3197 // because its dest block is unanalyzable.
3198 assert(isSOPP(BranchOp) || isSOPK(BranchOp));
3199
3200 // Convert to dwords.
3201 BrOffset /= 4;
3202
3203 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
3204 // from the next instruction.
3205 BrOffset -= 1;
3206
3207 return isIntN(N: ST.getCLOpts().s_branch_bits, x: BrOffset);
3208}
3209
3210MachineBasicBlock *
3211SIInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
3212 return MI.getOperand(i: 0).getMBB();
3213}
3214
3215bool SIInstrInfo::hasDivergentBranch(const MachineBasicBlock *MBB) const {
3216 for (const MachineInstr &MI : MBB->terminators()) {
3217 if (MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE ||
3218 MI.getOpcode() == AMDGPU::SI_LOOP ||
3219 MI.getOpcode() == AMDGPU::SI_WATERFALL_LOOP)
3220 return true;
3221 }
3222 return false;
3223}
3224
3225void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
3226 MachineBasicBlock &DestBB,
3227 MachineBasicBlock &RestoreBB,
3228 const DebugLoc &DL, int64_t BrOffset,
3229 RegScavenger *RS) const {
3230 assert(MBB.empty() &&
3231 "new block should be inserted for expanding unconditional branch");
3232 assert(MBB.pred_size() == 1);
3233 assert(RestoreBB.empty() &&
3234 "restore block should be inserted for restoring clobbered registers");
3235
3236 MachineFunction *MF = MBB.getParent();
3237 MachineRegisterInfo &MRI = MF->getRegInfo();
3238 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
3239 auto I = MBB.end();
3240 auto &MCCtx = MF->getContext();
3241
3242 if (ST.useAddPCI64Inst()) {
3243 MCSymbol *Offset =
3244 MCCtx.createTempSymbol(Name: "offset", /*AlwaysAddSuffix=*/true);
3245 auto AddPC = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_PC_I64))
3246 .addSym(Sym: Offset, TargetFlags: MO_FAR_BRANCH_OFFSET);
3247 MCSymbol *PostAddPCLabel =
3248 MCCtx.createTempSymbol(Name: "post_addpc", /*AlwaysAddSuffix=*/true);
3249 AddPC->setPostInstrSymbol(MF&: *MF, Symbol: PostAddPCLabel);
3250 auto *OffsetExpr = MCBinaryExpr::createSub(
3251 LHS: MCSymbolRefExpr::create(Symbol: DestBB.getSymbol(), Ctx&: MCCtx),
3252 RHS: MCSymbolRefExpr::create(Symbol: PostAddPCLabel, Ctx&: MCCtx), Ctx&: MCCtx);
3253 Offset->setVariableValue(OffsetExpr);
3254 return;
3255 }
3256
3257 assert(RS && "RegScavenger required for long branching");
3258
3259 // FIXME: Virtual register workaround for RegScavenger not working with empty
3260 // blocks.
3261 Register PCReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
3262
3263 // Note: as this is used after hazard recognizer we need to apply some hazard
3264 // workarounds directly.
3265 const bool FlushSGPRWrites = (ST.isWave64() && ST.hasVALUMaskWriteHazard()) ||
3266 ST.hasVALUReadSGPRHazard();
3267 auto ApplyHazardWorkarounds = [this, &MBB, &I, &DL, FlushSGPRWrites]() {
3268 if (FlushSGPRWrites)
3269 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
3270 .addImm(Val: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST));
3271 };
3272
3273 // We need to compute the offset relative to the instruction immediately after
3274 // s_getpc_b64. Insert pc arithmetic code before last terminator.
3275 MachineInstr *GetPC = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: PCReg);
3276 ApplyHazardWorkarounds();
3277
3278 MCSymbol *PostGetPCLabel =
3279 MCCtx.createTempSymbol(Name: "post_getpc", /*AlwaysAddSuffix=*/true);
3280 GetPC->setPostInstrSymbol(MF&: *MF, Symbol: PostGetPCLabel);
3281
3282 MCSymbol *OffsetLo =
3283 MCCtx.createTempSymbol(Name: "offset_lo", /*AlwaysAddSuffix=*/true);
3284 MCSymbol *OffsetHi =
3285 MCCtx.createTempSymbol(Name: "offset_hi", /*AlwaysAddSuffix=*/true);
3286 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U32))
3287 .addReg(RegNo: PCReg, Flags: RegState::Define, SubReg: AMDGPU::sub0)
3288 .addReg(RegNo: PCReg, Flags: {}, SubReg: AMDGPU::sub0)
3289 .addSym(Sym: OffsetLo, TargetFlags: MO_FAR_BRANCH_OFFSET);
3290 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADDC_U32))
3291 .addReg(RegNo: PCReg, Flags: RegState::Define, SubReg: AMDGPU::sub1)
3292 .addReg(RegNo: PCReg, Flags: {}, SubReg: AMDGPU::sub1)
3293 .addSym(Sym: OffsetHi, TargetFlags: MO_FAR_BRANCH_OFFSET);
3294 ApplyHazardWorkarounds();
3295
3296 // Insert the indirect branch after the other terminator.
3297 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETPC_B64))
3298 .addReg(RegNo: PCReg);
3299
3300 // If a spill is needed for the pc register pair, we need to insert a spill
3301 // restore block right before the destination block, and insert a short branch
3302 // into the old destination block's fallthrough predecessor.
3303 // e.g.:
3304 //
3305 // s_cbranch_scc0 skip_long_branch:
3306 //
3307 // long_branch_bb:
3308 // spill s[8:9]
3309 // s_getpc_b64 s[8:9]
3310 // s_add_u32 s8, s8, restore_bb
3311 // s_addc_u32 s9, s9, 0
3312 // s_setpc_b64 s[8:9]
3313 //
3314 // skip_long_branch:
3315 // foo;
3316 //
3317 // .....
3318 //
3319 // dest_bb_fallthrough_predecessor:
3320 // bar;
3321 // s_branch dest_bb
3322 //
3323 // restore_bb:
3324 // restore s[8:9]
3325 // fallthrough dest_bb
3326 ///
3327 // dest_bb:
3328 // buzz;
3329
3330 Register LongBranchReservedReg = MFI->getLongBranchReservedReg();
3331 Register Scav;
3332
3333 // If we've previously reserved a register for long branches
3334 // avoid running the scavenger and just use those registers
3335 if (LongBranchReservedReg) {
3336 RS->enterBasicBlock(MBB);
3337 Scav = LongBranchReservedReg;
3338 } else {
3339 RS->enterBasicBlockEnd(MBB);
3340 Scav = RS->scavengeRegisterBackwards(
3341 RC: AMDGPU::SReg_64RegClass, To: MachineBasicBlock::iterator(GetPC),
3342 /* RestoreAfter */ false, SPAdj: 0, /* AllowSpill */ false);
3343 }
3344 if (Scav) {
3345 RS->setRegUsed(Reg: Scav);
3346 MRI.replaceRegWith(FromReg: PCReg, ToReg: Scav);
3347 MRI.clearVirtRegs();
3348 } else {
3349 // As SGPR needs VGPR to be spilled, we reuse the slot of temporary VGPR for
3350 // SGPR spill.
3351 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3352 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3353 TRI->spillEmergencySGPR(MI: GetPC, RestoreMBB&: RestoreBB, SGPR: AMDGPU::SGPR0_SGPR1, RS);
3354 MRI.replaceRegWith(FromReg: PCReg, ToReg: AMDGPU::SGPR0_SGPR1);
3355 MRI.clearVirtRegs();
3356 }
3357
3358 MCSymbol *DestLabel = Scav ? DestBB.getSymbol() : RestoreBB.getSymbol();
3359 // Now, the distance could be defined.
3360 auto *Offset = MCBinaryExpr::createSub(
3361 LHS: MCSymbolRefExpr::create(Symbol: DestLabel, Ctx&: MCCtx),
3362 RHS: MCSymbolRefExpr::create(Symbol: PostGetPCLabel, Ctx&: MCCtx), Ctx&: MCCtx);
3363 // Add offset assignments.
3364 auto *Mask = MCConstantExpr::create(Value: 0xFFFFFFFFULL, Ctx&: MCCtx);
3365 OffsetLo->setVariableValue(MCBinaryExpr::createAnd(LHS: Offset, RHS: Mask, Ctx&: MCCtx));
3366 auto *ShAmt = MCConstantExpr::create(Value: 32, Ctx&: MCCtx);
3367 OffsetHi->setVariableValue(MCBinaryExpr::createAShr(LHS: Offset, RHS: ShAmt, Ctx&: MCCtx));
3368}
3369
3370unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
3371 switch (Cond) {
3372 case SIInstrInfo::SCC_TRUE:
3373 return AMDGPU::S_CBRANCH_SCC1;
3374 case SIInstrInfo::SCC_FALSE:
3375 return AMDGPU::S_CBRANCH_SCC0;
3376 case SIInstrInfo::VCCNZ:
3377 return AMDGPU::S_CBRANCH_VCCNZ;
3378 case SIInstrInfo::VCCZ:
3379 return AMDGPU::S_CBRANCH_VCCZ;
3380 case SIInstrInfo::EXECNZ:
3381 return AMDGPU::S_CBRANCH_EXECNZ;
3382 case SIInstrInfo::EXECZ:
3383 return AMDGPU::S_CBRANCH_EXECZ;
3384 default:
3385 llvm_unreachable("invalid branch predicate");
3386 }
3387}
3388
3389SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
3390 switch (Opcode) {
3391 case AMDGPU::S_CBRANCH_SCC0:
3392 return SCC_FALSE;
3393 case AMDGPU::S_CBRANCH_SCC1:
3394 return SCC_TRUE;
3395 case AMDGPU::S_CBRANCH_VCCNZ:
3396 return VCCNZ;
3397 case AMDGPU::S_CBRANCH_VCCZ:
3398 return VCCZ;
3399 case AMDGPU::S_CBRANCH_EXECNZ:
3400 return EXECNZ;
3401 case AMDGPU::S_CBRANCH_EXECZ:
3402 return EXECZ;
3403 default:
3404 return INVALID_BR;
3405 }
3406}
3407
3408bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB,
3409 MachineBasicBlock::iterator I,
3410 MachineBasicBlock *&TBB,
3411 MachineBasicBlock *&FBB,
3412 SmallVectorImpl<MachineOperand> &Cond,
3413 bool AllowModify) const {
3414 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3415 // Unconditional Branch
3416 TBB = I->getOperand(i: 0).getMBB();
3417 return false;
3418 }
3419
3420 BranchPredicate Pred = getBranchPredicate(Opcode: I->getOpcode());
3421 if (Pred == INVALID_BR)
3422 return true;
3423
3424 MachineBasicBlock *CondBB = I->getOperand(i: 0).getMBB();
3425 Cond.push_back(Elt: MachineOperand::CreateImm(Val: Pred));
3426 Cond.push_back(Elt: I->getOperand(i: 1)); // Save the branch register.
3427
3428 ++I;
3429
3430 if (I == MBB.end()) {
3431 // Conditional branch followed by fall-through.
3432 TBB = CondBB;
3433 return false;
3434 }
3435
3436 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3437 TBB = CondBB;
3438 FBB = I->getOperand(i: 0).getMBB();
3439 return false;
3440 }
3441
3442 return true;
3443}
3444
3445bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
3446 MachineBasicBlock *&FBB,
3447 SmallVectorImpl<MachineOperand> &Cond,
3448 bool AllowModify) const {
3449 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
3450 auto E = MBB.end();
3451 if (I == E)
3452 return false;
3453
3454 // Skip over the instructions that are artificially terminators for special
3455 // exec management.
3456 while (I != E && !I->isBranch() && !I->isReturn()) {
3457 switch (I->getOpcode()) {
3458 case AMDGPU::S_MOV_B64_term:
3459 case AMDGPU::S_XOR_B64_term:
3460 case AMDGPU::S_OR_B64_term:
3461 case AMDGPU::S_ANDN2_B64_term:
3462 case AMDGPU::S_AND_B64_term:
3463 case AMDGPU::S_AND_SAVEEXEC_B64_term:
3464 case AMDGPU::S_MOV_B32_term:
3465 case AMDGPU::S_XOR_B32_term:
3466 case AMDGPU::S_OR_B32_term:
3467 case AMDGPU::S_ANDN2_B32_term:
3468 case AMDGPU::S_AND_B32_term:
3469 case AMDGPU::S_AND_SAVEEXEC_B32_term:
3470 case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
3471 case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
3472 break;
3473 case AMDGPU::SI_IF:
3474 case AMDGPU::SI_ELSE:
3475 case AMDGPU::SI_KILL_I1_TERMINATOR:
3476 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
3477 // FIXME: It's messy that these need to be considered here at all.
3478 return true;
3479 default:
3480 llvm_unreachable("unexpected non-branch terminator inst");
3481 }
3482
3483 ++I;
3484 }
3485
3486 if (I == E)
3487 return false;
3488
3489 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
3490}
3491
3492unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB,
3493 int *BytesRemoved) const {
3494 unsigned Count = 0;
3495 unsigned RemovedSize = 0;
3496 for (MachineInstr &MI : llvm::make_early_inc_range(Range: MBB.terminators())) {
3497 // Skip over artificial terminators when removing instructions.
3498 if (MI.isBranch() || MI.isReturn()) {
3499 RemovedSize += getInstSizeInBytes(MI);
3500 MI.eraseFromParent();
3501 ++Count;
3502 }
3503 }
3504
3505 if (BytesRemoved)
3506 *BytesRemoved = RemovedSize;
3507
3508 return Count;
3509}
3510
3511// Copy the flags onto the implicit condition register operand.
3512static void preserveCondRegFlags(MachineOperand &CondReg,
3513 const MachineOperand &OrigCond) {
3514 CondReg.setIsUndef(OrigCond.isUndef());
3515 CondReg.setIsKill(OrigCond.isKill());
3516}
3517
3518unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,
3519 MachineBasicBlock *TBB,
3520 MachineBasicBlock *FBB,
3521 ArrayRef<MachineOperand> Cond,
3522 const DebugLoc &DL,
3523 int *BytesAdded) const {
3524 if (!FBB && Cond.empty()) {
3525 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
3526 .addMBB(MBB: TBB);
3527 if (BytesAdded)
3528 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3529 return 1;
3530 }
3531
3532 assert(TBB && Cond[0].isImm());
3533
3534 unsigned Opcode
3535 = getBranchOpcode(Cond: static_cast<BranchPredicate>(Cond[0].getImm()));
3536
3537 if (!FBB) {
3538 MachineInstr *CondBr =
3539 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode))
3540 .addMBB(MBB: TBB);
3541
3542 // Copy the flags onto the implicit condition register operand.
3543 preserveCondRegFlags(CondReg&: CondBr->getOperand(i: 1), OrigCond: Cond[1]);
3544 fixImplicitOperands(MI&: *CondBr);
3545
3546 if (BytesAdded)
3547 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3548 return 1;
3549 }
3550
3551 assert(TBB && FBB);
3552
3553 MachineInstr *CondBr =
3554 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode))
3555 .addMBB(MBB: TBB);
3556 fixImplicitOperands(MI&: *CondBr);
3557 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
3558 .addMBB(MBB: FBB);
3559
3560 MachineOperand &CondReg = CondBr->getOperand(i: 1);
3561 CondReg.setIsUndef(Cond[1].isUndef());
3562 CondReg.setIsKill(Cond[1].isKill());
3563
3564 if (BytesAdded)
3565 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8;
3566
3567 return 2;
3568}
3569
3570bool SIInstrInfo::reverseBranchCondition(
3571 SmallVectorImpl<MachineOperand> &Cond) const {
3572 if (Cond.size() != 2) {
3573 return true;
3574 }
3575
3576 if (Cond[0].isImm()) {
3577 Cond[0].setImm(-Cond[0].getImm());
3578 return false;
3579 }
3580
3581 return true;
3582}
3583
3584namespace {
3585class AMDGPUPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
3586private:
3587 /// The compare instruction for loop control
3588 const MachineInstr *CmpInst = nullptr;
3589 /// The normalized condition used by createTripCountGreaterCondition()
3590 SmallVector<MachineOperand, 2> Cond;
3591
3592public:
3593 AMDGPUPipelinerLoopInfo(MachineInstr *CmpInst,
3594 const SmallVectorImpl<MachineOperand> &Cond)
3595 : CmpInst(CmpInst), Cond(Cond.begin(), Cond.end()) {}
3596
3597 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
3598 return CmpInst && MI == CmpInst;
3599 }
3600
3601 std::optional<bool> createTripCountGreaterCondition(
3602 int TC, MachineBasicBlock &MBB,
3603 SmallVectorImpl<MachineOperand> &CondParam) override {
3604 CondParam = this->Cond;
3605 return {};
3606 }
3607
3608 void adjustTripCount(int TripCountAdjust) override {}
3609
3610 void setPreheader(MachineBasicBlock *NewPreheader) override {}
3611};
3612} // namespace
3613
3614std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
3615SIInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
3616 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
3617 SmallVector<MachineOperand, 2> Cond;
3618 // Unanalyzable terminator.
3619 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))
3620 return nullptr;
3621
3622 // Infinite loops are not supported.
3623 if (TBB == LoopBB && FBB == LoopBB)
3624 return nullptr;
3625
3626 // Must be conditional branch.
3627 if (FBB == nullptr)
3628 return nullptr;
3629
3630 assert((TBB == LoopBB || FBB == LoopBB) &&
3631 "The Loop must be a single-basic-block loop");
3632
3633 // Divergent (VCC/EXEC) back-edge is not supported.
3634 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
3635 if (Pred != SCC_TRUE && Pred != SCC_FALSE)
3636 return nullptr;
3637
3638 // Calls and inline assembly are not supported.
3639 for (const MachineInstr &MI : *LoopBB)
3640 if (MI.isCall() || MI.isInlineAsm())
3641 return nullptr;
3642
3643 // Normalization for createTripCountGreaterCondition(): make Cond mean
3644 // "exit the loop" so the expander emits correct prolog guard branches.
3645 if (TBB == LoopBB)
3646 reverseBranchCondition(Cond);
3647
3648 auto Instructions = make_range(
3649 x: MachineBasicBlock::reverse_iterator(LoopBB->getFirstTerminator()),
3650 y: LoopBB->rend());
3651 auto CmpI = llvm::find_if(Range&: Instructions, P: [&](const MachineInstr &MI) {
3652 return MI.modifiesRegister(Reg: Cond[1].getReg(), TRI: &RI);
3653 });
3654
3655 if (CmpI == Instructions.end() || CmpI->isPHI())
3656 return nullptr;
3657 MachineInstr *CmpInst = &*CmpI;
3658
3659 return std::make_unique<AMDGPUPipelinerLoopInfo>(args&: CmpInst, args&: Cond);
3660}
3661
3662bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
3663 ArrayRef<MachineOperand> Cond,
3664 Register DstReg, Register TrueReg,
3665 Register FalseReg,
3666 SelectExpansion &Exp) const {
3667 switch (Cond[0].getImm()) {
3668 case VCCNZ:
3669 case VCCZ: {
3670 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3671 const TargetRegisterClass *RC = MRI.getRegClass(Reg: TrueReg);
3672 if (MRI.getRegClass(Reg: FalseReg) != RC)
3673 return false;
3674
3675 int NumInsts = AMDGPU::getRegBitWidth(RC: *RC) / 32;
3676 Exp.CondCycles = Exp.TrueCycles = Exp.FalseCycles = NumInsts; // ???
3677
3678 // Limit to equal cost for branch vs. N v_cndmask_b32s.
3679 return RI.hasVGPRs(RC) && NumInsts <= 6;
3680 }
3681 case SCC_TRUE:
3682 case SCC_FALSE: {
3683 // FIXME: We could insert for VGPRs if we could replace the original compare
3684 // with a vector one.
3685 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3686 const TargetRegisterClass *RC = MRI.getRegClass(Reg: TrueReg);
3687 if (MRI.getRegClass(Reg: FalseReg) != RC)
3688 return false;
3689
3690 int NumInsts = AMDGPU::getRegBitWidth(RC: *RC) / 32;
3691
3692 // Multiples of 8 can do s_cselect_b64
3693 if (NumInsts % 2 == 0)
3694 NumInsts /= 2;
3695
3696 Exp.CondCycles = Exp.TrueCycles = Exp.FalseCycles = NumInsts; // ???
3697 return RI.isSGPRClass(RC);
3698 }
3699 default:
3700 return false;
3701 }
3702}
3703
3704void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,
3705 MachineBasicBlock::iterator I, const DebugLoc &DL,
3706 Register DstReg, ArrayRef<MachineOperand> Cond,
3707 Register TrueReg, Register FalseReg) const {
3708 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
3709 if (Pred == VCCZ || Pred == SCC_FALSE) {
3710 Pred = static_cast<BranchPredicate>(-Pred);
3711 std::swap(a&: TrueReg, b&: FalseReg);
3712 }
3713
3714 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3715 const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: DstReg);
3716 unsigned DstSize = RI.getRegSizeInBits(RC: *DstRC);
3717
3718 if (DstSize == 32) {
3719 MachineInstr *Select;
3720 if (Pred == SCC_TRUE) {
3721 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: DstReg)
3722 .addReg(RegNo: TrueReg)
3723 .addReg(RegNo: FalseReg);
3724 } else {
3725 // Instruction's operands are backwards from what is expected.
3726 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e32), DestReg: DstReg)
3727 .addReg(RegNo: FalseReg)
3728 .addReg(RegNo: TrueReg);
3729 }
3730
3731 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3732 return;
3733 }
3734
3735 if (DstSize == 64 && Pred == SCC_TRUE) {
3736 MachineInstr *Select =
3737 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B64), DestReg: DstReg)
3738 .addReg(RegNo: TrueReg)
3739 .addReg(RegNo: FalseReg);
3740
3741 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3742 return;
3743 }
3744
3745 static const int16_t Sub0_15[] = {
3746 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
3747 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
3748 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
3749 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
3750 };
3751
3752 static const int16_t Sub0_15_64[] = {
3753 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
3754 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
3755 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
3756 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
3757 };
3758
3759 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
3760 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
3761 const int16_t *SubIndices = Sub0_15;
3762 int NElts = DstSize / 32;
3763
3764 // 64-bit select is only available for SALU.
3765 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
3766 if (Pred == SCC_TRUE) {
3767 if (NElts % 2) {
3768 SelOp = AMDGPU::S_CSELECT_B32;
3769 EltRC = &AMDGPU::SGPR_32RegClass;
3770 } else {
3771 SelOp = AMDGPU::S_CSELECT_B64;
3772 EltRC = &AMDGPU::SGPR_64RegClass;
3773 SubIndices = Sub0_15_64;
3774 NElts /= 2;
3775 }
3776 }
3777
3778 MachineInstrBuilder MIB = BuildMI(
3779 BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
3780
3781 I = MIB->getIterator();
3782
3783 SmallVector<Register, 8> Regs;
3784 for (int Idx = 0; Idx != NElts; ++Idx) {
3785 Register DstElt = MRI.createVirtualRegister(RegClass: EltRC);
3786 Regs.push_back(Elt: DstElt);
3787
3788 unsigned SubIdx = SubIndices[Idx];
3789
3790 MachineInstr *Select;
3791 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) {
3792 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SelOp), DestReg: DstElt)
3793 .addReg(RegNo: FalseReg, Flags: {}, SubReg: SubIdx)
3794 .addReg(RegNo: TrueReg, Flags: {}, SubReg: SubIdx);
3795 } else {
3796 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SelOp), DestReg: DstElt)
3797 .addReg(RegNo: TrueReg, Flags: {}, SubReg: SubIdx)
3798 .addReg(RegNo: FalseReg, Flags: {}, SubReg: SubIdx);
3799 }
3800
3801 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3802 fixImplicitOperands(MI&: *Select);
3803
3804 MIB.addReg(RegNo: DstElt)
3805 .addImm(Val: SubIdx);
3806 }
3807}
3808
3809bool SIInstrInfo::isXcntDrain(const MachineInstr &MI) {
3810
3811 if (MI.isBranch() || MI.isCall() || MI.isReturn() || MI.isIndirectBranch())
3812 return true;
3813
3814 switch (MI.getOpcode()) {
3815 case AMDGPU::S_ENDPGM:
3816 case AMDGPU::S_ENDPGM_SAVED:
3817 case AMDGPU::S_TRAP:
3818 case AMDGPU::S_GETREG_B32:
3819 case AMDGPU::S_SETREG_B32:
3820 case AMDGPU::S_SETREG_B32_mode:
3821 case AMDGPU::S_SETREG_IMM32_B32:
3822 case AMDGPU::S_SETREG_IMM32_B32_mode:
3823 case AMDGPU::S_SENDMSG:
3824 case AMDGPU::S_SENDMSGHALT:
3825 case AMDGPU::S_SENDMSG_RTN_B32:
3826 case AMDGPU::S_SENDMSG_RTN_B64:
3827 case AMDGPU::S_BARRIER_WAIT:
3828 case AMDGPU::S_BARRIER_SIGNAL_M0:
3829 case AMDGPU::S_BARRIER_SIGNAL_IMM:
3830 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0:
3831 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM:
3832 return true;
3833 default:
3834 return false;
3835 }
3836}
3837
3838bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) {
3839 switch (MI.getOpcode()) {
3840 case AMDGPU::V_MOV_B16_t16_e32:
3841 case AMDGPU::V_MOV_B16_t16_e64:
3842 case AMDGPU::V_MOV_B32_e32:
3843 case AMDGPU::V_MOV_B32_e64:
3844 case AMDGPU::V_MOV_B64_PSEUDO:
3845 case AMDGPU::V_MOV_B64_e32:
3846 case AMDGPU::V_MOV_B64_e64:
3847 case AMDGPU::S_MOV_B32:
3848 case AMDGPU::S_MOV_B64:
3849 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3850 case AMDGPU::COPY:
3851 case AMDGPU::WWM_COPY:
3852 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3853 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3854 case AMDGPU::V_ACCVGPR_MOV_B32:
3855 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
3856 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
3857 return true;
3858 default:
3859 return false;
3860 }
3861}
3862
3863unsigned SIInstrInfo::getFoldableCopySrcIdx(const MachineInstr &MI) {
3864 switch (MI.getOpcode()) {
3865 case AMDGPU::V_MOV_B16_t16_e32:
3866 case AMDGPU::V_MOV_B16_t16_e64:
3867 return 2;
3868 case AMDGPU::V_MOV_B32_e32:
3869 case AMDGPU::V_MOV_B32_e64:
3870 case AMDGPU::V_MOV_B64_PSEUDO:
3871 case AMDGPU::V_MOV_B64_e32:
3872 case AMDGPU::V_MOV_B64_e64:
3873 case AMDGPU::S_MOV_B32:
3874 case AMDGPU::S_MOV_B64:
3875 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3876 case AMDGPU::COPY:
3877 case AMDGPU::WWM_COPY:
3878 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3879 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3880 case AMDGPU::V_ACCVGPR_MOV_B32:
3881 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
3882 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
3883 return 1;
3884 default:
3885 llvm_unreachable("MI is not a foldable copy");
3886 }
3887}
3888
3889static constexpr AMDGPU::OpName ModifierOpNames[] = {
3890 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
3891 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
3892 AMDGPU::OpName::omod, AMDGPU::OpName::op_sel};
3893
3894void SIInstrInfo::removeModOperands(MachineInstr &MI) const {
3895 unsigned Opc = MI.getOpcode();
3896 for (AMDGPU::OpName Name : reverse(C: ModifierOpNames)) {
3897 int Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name);
3898 if (Idx >= 0)
3899 MI.removeOperand(OpNo: Idx);
3900 }
3901}
3902
3903void SIInstrInfo::mutateAndCleanupImplicit(MachineInstr &MI,
3904 const MCInstrDesc &NewDesc) const {
3905 MI.setDesc(NewDesc);
3906
3907 // Remove any leftover implicit operands from mutating the instruction. e.g.
3908 // if we replace an s_and_b32 with a copy, we don't need the implicit scc def
3909 // anymore.
3910 const MCInstrDesc &Desc = MI.getDesc();
3911 unsigned NumOps = Desc.getNumOperands() + Desc.implicit_uses().size() +
3912 Desc.implicit_defs().size();
3913
3914 for (unsigned I = MI.getNumOperands() - 1; I >= NumOps; --I)
3915 MI.removeOperand(OpNo: I);
3916}
3917
3918std::optional<int64_t> SIInstrInfo::extractSubregFromImm(int64_t Imm,
3919 unsigned SubRegIndex) {
3920 switch (SubRegIndex) {
3921 case AMDGPU::NoSubRegister:
3922 return Imm;
3923 case AMDGPU::sub0:
3924 return SignExtend64<32>(x: Imm);
3925 case AMDGPU::sub1:
3926 return SignExtend64<32>(x: Imm >> 32);
3927 case AMDGPU::lo16:
3928 return SignExtend64<16>(x: Imm);
3929 case AMDGPU::hi16:
3930 return SignExtend64<16>(x: Imm >> 16);
3931 case AMDGPU::sub1_lo16:
3932 return SignExtend64<16>(x: Imm >> 32);
3933 case AMDGPU::sub1_hi16:
3934 return SignExtend64<16>(x: Imm >> 48);
3935 default:
3936 return std::nullopt;
3937 }
3938
3939 llvm_unreachable("covered subregister switch");
3940}
3941
3942std::optional<unsigned>
3943SIInstrInfo::extractSubregFromGlobalAddressFlags(unsigned TargetFlags,
3944 unsigned SubRegIndex) {
3945 if (SubRegIndex == AMDGPU::NoSubRegister)
3946 return TargetFlags;
3947
3948 unsigned LoReloc, HiReloc;
3949 switch (TargetFlags & MO_MASK) {
3950 case MO_REL32_LO:
3951 case MO_REL32_HI:
3952 case MO_REL64:
3953 LoReloc = MO_REL32_LO;
3954 HiReloc = MO_REL32_HI;
3955 break;
3956 case MO_GOTPCREL32_LO:
3957 case MO_GOTPCREL32_HI:
3958 LoReloc = MO_GOTPCREL32_LO;
3959 HiReloc = MO_GOTPCREL32_HI;
3960 break;
3961 case MO_ABS32_LO:
3962 case MO_ABS32_HI:
3963 case MO_ABS64:
3964 LoReloc = MO_ABS32_LO;
3965 HiReloc = MO_ABS32_HI;
3966 break;
3967 default:
3968 return std::nullopt;
3969 }
3970
3971 const unsigned BaseFlags = TargetFlags & ~MO_MASK;
3972 switch (SubRegIndex) {
3973 case AMDGPU::sub0:
3974 return BaseFlags | LoReloc;
3975 case AMDGPU::sub1:
3976 return BaseFlags | HiReloc;
3977 default:
3978 return std::nullopt;
3979 }
3980}
3981
3982static unsigned getNewFMAAKInst(const GCNSubtarget &ST, unsigned Opc) {
3983 switch (Opc) {
3984 case AMDGPU::V_MAC_F16_e32:
3985 case AMDGPU::V_MAC_F16_e64:
3986 case AMDGPU::V_MAD_F16_e64:
3987 return AMDGPU::V_MADAK_F16;
3988 case AMDGPU::V_MAC_F32_e32:
3989 case AMDGPU::V_MAC_F32_e64:
3990 case AMDGPU::V_MAD_F32_e64:
3991 return AMDGPU::V_MADAK_F32;
3992 case AMDGPU::V_FMAC_F32_e32:
3993 case AMDGPU::V_FMAC_F32_e64:
3994 case AMDGPU::V_FMA_F32_e64:
3995 return AMDGPU::V_FMAAK_F32;
3996 case AMDGPU::V_FMAC_F16_e32:
3997 case AMDGPU::V_FMAC_F16_e64:
3998 case AMDGPU::V_FMAC_F16_t16_e64:
3999 case AMDGPU::V_FMAC_F16_fake16_e64:
4000 case AMDGPU::V_FMAC_F16_t16_e32:
4001 case AMDGPU::V_FMAC_F16_fake16_e32:
4002 case AMDGPU::V_FMA_F16_e64:
4003 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
4004 ? AMDGPU::V_FMAAK_F16_t16
4005 : AMDGPU::V_FMAAK_F16_fake16
4006 : AMDGPU::V_FMAAK_F16;
4007 case AMDGPU::V_FMAC_F64_e32:
4008 case AMDGPU::V_FMAC_F64_e64:
4009 case AMDGPU::V_FMA_F64_e64:
4010 return AMDGPU::V_FMAAK_F64;
4011 default:
4012 llvm_unreachable("invalid instruction");
4013 }
4014}
4015
4016static unsigned getNewFMAMKInst(const GCNSubtarget &ST, unsigned Opc) {
4017 switch (Opc) {
4018 case AMDGPU::V_MAC_F16_e32:
4019 case AMDGPU::V_MAC_F16_e64:
4020 case AMDGPU::V_MAD_F16_e64:
4021 return AMDGPU::V_MADMK_F16;
4022 case AMDGPU::V_MAC_F32_e32:
4023 case AMDGPU::V_MAC_F32_e64:
4024 case AMDGPU::V_MAD_F32_e64:
4025 return AMDGPU::V_MADMK_F32;
4026 case AMDGPU::V_FMAC_F32_e32:
4027 case AMDGPU::V_FMAC_F32_e64:
4028 case AMDGPU::V_FMA_F32_e64:
4029 return AMDGPU::V_FMAMK_F32;
4030 case AMDGPU::V_FMAC_F16_e32:
4031 case AMDGPU::V_FMAC_F16_e64:
4032 case AMDGPU::V_FMAC_F16_t16_e64:
4033 case AMDGPU::V_FMAC_F16_fake16_e64:
4034 case AMDGPU::V_FMAC_F16_t16_e32:
4035 case AMDGPU::V_FMAC_F16_fake16_e32:
4036 case AMDGPU::V_FMA_F16_e64:
4037 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
4038 ? AMDGPU::V_FMAMK_F16_t16
4039 : AMDGPU::V_FMAMK_F16_fake16
4040 : AMDGPU::V_FMAMK_F16;
4041 case AMDGPU::V_FMAC_F64_e32:
4042 case AMDGPU::V_FMAC_F64_e64:
4043 case AMDGPU::V_FMA_F64_e64:
4044 return AMDGPU::V_FMAMK_F64;
4045 default:
4046 llvm_unreachable("invalid instruction");
4047 }
4048}
4049
4050bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
4051 Register Reg, MachineRegisterInfo *MRI) const {
4052 int64_t Imm;
4053 if (!getConstValDefinedInReg(MI: DefMI, Reg, ImmVal&: Imm))
4054 return false;
4055
4056 const bool HasMultipleUses = !MRI->hasOneNonDBGUse(RegNo: Reg);
4057
4058 assert(!DefMI.getOperand(0).getSubReg() && "Expected SSA form");
4059
4060 unsigned Opc = UseMI.getOpcode();
4061 if (Opc == AMDGPU::COPY) {
4062 assert(!UseMI.getOperand(0).getSubReg() && "Expected SSA form");
4063
4064 Register DstReg = UseMI.getOperand(i: 0).getReg();
4065 Register UseSubReg = UseMI.getOperand(i: 1).getSubReg();
4066
4067 const TargetRegisterClass *DstRC = RI.getRegClassForReg(MRI: *MRI, Reg: DstReg);
4068
4069 if (HasMultipleUses) {
4070 // TODO: This should fold in more cases with multiple use, but we need to
4071 // more carefully consider what those uses are.
4072 unsigned ImmDefSize = RI.getRegSizeInBits(RC: *MRI->getRegClass(Reg));
4073
4074 // Avoid breaking up a 64-bit inline immediate into a subregister extract.
4075 if (UseSubReg != AMDGPU::NoSubRegister && ImmDefSize == 64)
4076 return false;
4077
4078 // Most of the time folding a 32-bit inline constant is free (though this
4079 // might not be true if we can't later fold it into a real user).
4080 //
4081 // FIXME: This isInlineConstant check is imprecise if
4082 // getConstValDefinedInReg handled the tricky non-mov cases.
4083 if (ImmDefSize == 32 &&
4084 !isInlineConstant(ImmVal: Imm, OperandType: AMDGPU::OPERAND_REG_IMM_INT32))
4085 return false;
4086 }
4087
4088 bool Is16Bit = UseSubReg != AMDGPU::NoSubRegister &&
4089 RI.getSubRegIdxSize(Idx: UseSubReg) == 16;
4090
4091 if (Is16Bit) {
4092 if (RI.hasVGPRs(RC: DstRC))
4093 return false; // Do not clobber vgpr_hi16
4094
4095 if (DstReg.isVirtual() && UseSubReg != AMDGPU::lo16)
4096 return false;
4097 }
4098
4099 MachineFunction *MF = UseMI.getMF();
4100
4101 unsigned NewOpc = AMDGPU::INSTRUCTION_LIST_END;
4102 MCRegister MovDstPhysReg =
4103 DstReg.isPhysical() ? DstReg.asMCReg() : MCRegister();
4104
4105 std::optional<int64_t> SubRegImm = extractSubregFromImm(Imm, SubRegIndex: UseSubReg);
4106
4107 // TODO: Try to fold with AMDGPU::V_MOV_B16_t16_e64
4108 for (unsigned MovOp :
4109 {AMDGPU::S_MOV_B32, AMDGPU::V_MOV_B32_e32, AMDGPU::S_MOV_B64,
4110 AMDGPU::V_MOV_B64_PSEUDO, AMDGPU::V_ACCVGPR_WRITE_B32_e64}) {
4111 const MCInstrDesc &MovDesc = get(Opcode: MovOp);
4112
4113 const TargetRegisterClass *MovDstRC = getRegClass(MCID: MovDesc, OpNum: 0);
4114 if (Is16Bit) {
4115 // We just need to find a correctly sized register class, so the
4116 // subregister index compatibility doesn't matter since we're statically
4117 // extracting the immediate value.
4118 MovDstRC = RI.getMatchingSuperRegClass(A: MovDstRC, B: DstRC, Idx: AMDGPU::lo16);
4119 if (!MovDstRC)
4120 continue;
4121
4122 if (MovDstPhysReg) {
4123 // FIXME: We probably should not do this. If there is a live value in
4124 // the high half of the register, it will be corrupted.
4125 MovDstPhysReg =
4126 RI.getMatchingSuperReg(Reg: MovDstPhysReg, SubIdx: AMDGPU::lo16, RC: MovDstRC);
4127 if (!MovDstPhysReg)
4128 continue;
4129 }
4130 }
4131
4132 // Result class isn't the right size, try the next instruction.
4133 if (MovDstPhysReg) {
4134 if (!MovDstRC->contains(Reg: MovDstPhysReg))
4135 return false;
4136 } else if (!MRI->constrainRegClass(Reg: DstReg, RC: MovDstRC)) {
4137 // TODO: This will be overly conservative in the case of 16-bit virtual
4138 // SGPRs. We could hack up the virtual register uses to use a compatible
4139 // 32-bit class.
4140 continue;
4141 }
4142
4143 const MCOperandInfo &OpInfo = MovDesc.operands()[1];
4144
4145 // Ensure the interpreted immediate value is a valid operand in the new
4146 // mov.
4147 //
4148 // FIXME: isImmOperandLegal should have form that doesn't require existing
4149 // MachineInstr or MachineOperand
4150 if (!RI.opCanUseLiteralConstant(OpType: OpInfo.OperandType) &&
4151 !isInlineConstant(ImmVal: *SubRegImm, OperandType: OpInfo.OperandType))
4152 break;
4153
4154 NewOpc = MovOp;
4155 break;
4156 }
4157
4158 if (NewOpc == AMDGPU::INSTRUCTION_LIST_END)
4159 return false;
4160
4161 if (Is16Bit) {
4162 UseMI.getOperand(i: 0).setSubReg(AMDGPU::NoSubRegister);
4163 if (MovDstPhysReg)
4164 UseMI.getOperand(i: 0).setReg(MovDstPhysReg);
4165 assert(UseMI.getOperand(1).getReg().isVirtual());
4166 }
4167
4168 const MCInstrDesc &NewMCID = get(Opcode: NewOpc);
4169 UseMI.setDesc(NewMCID);
4170 UseMI.getOperand(i: 1).ChangeToImmediate(ImmVal: *SubRegImm);
4171 UseMI.addImplicitDefUseOperands(MF&: *MF);
4172 return true;
4173 }
4174
4175 if (HasMultipleUses)
4176 return false;
4177
4178 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
4179 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
4180 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
4181 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
4182 Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4183 Opc == AMDGPU::V_FMAC_F16_fake16_e64 || Opc == AMDGPU::V_FMA_F64_e64 ||
4184 Opc == AMDGPU::V_FMAC_F64_e64) {
4185 // Don't fold if we are using source or output modifiers. The new VOP2
4186 // instructions don't have them.
4187 if (hasAnyModifiersSet(MI: UseMI))
4188 return false;
4189
4190 // If this is a free constant, there's no reason to do this.
4191 // TODO: We could fold this here instead of letting SIFoldOperands do it
4192 // later.
4193 int Src0Idx = getNamedOperandIdx(Opcode: UseMI.getOpcode(), Name: AMDGPU::OpName::src0);
4194
4195 // Any src operand can be used for the legality check.
4196 if (isInlineConstant(MI: UseMI, OpIdx: Src0Idx, ImmVal: Imm))
4197 return false;
4198
4199 MachineOperand *Src0 = &UseMI.getOperand(i: Src0Idx);
4200
4201 MachineOperand *Src1 = getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src1);
4202 MachineOperand *Src2 = getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src2);
4203
4204 auto CopyRegOperandToNarrowerRC =
4205 [MRI, this](MachineInstr &MI, unsigned OpNo,
4206 const TargetRegisterClass *NewRC) -> void {
4207 if (!MI.getOperand(i: OpNo).isReg())
4208 return;
4209 Register Reg = MI.getOperand(i: OpNo).getReg();
4210 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI: *MRI, Reg);
4211 if (RI.getCommonSubClass(A: RC, B: NewRC) != NewRC)
4212 return;
4213 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
4214 BuildMI(BB&: *MI.getParent(), I: MI.getIterator(), MIMD: MI.getDebugLoc(),
4215 MCID: get(Opcode: AMDGPU::COPY), DestReg: Tmp)
4216 .addReg(RegNo: Reg);
4217 MI.getOperand(i: OpNo).setReg(Tmp);
4218 };
4219
4220 // Multiplied part is the constant: Use v_madmk_{f16, f32}.
4221 if ((Src0->isReg() && Src0->getReg() == Reg) ||
4222 (Src1->isReg() && Src1->getReg() == Reg)) {
4223 MachineOperand *RegSrc =
4224 Src1->isReg() && Src1->getReg() == Reg ? Src0 : Src1;
4225 if (!RegSrc->isReg())
4226 return false;
4227 if (RI.isSGPRClass(RC: MRI->getRegClass(Reg: RegSrc->getReg())) &&
4228 ST.getConstantBusLimit(Opcode: Opc) < 2)
4229 return false;
4230
4231 if (!Src2->isReg() || RI.isSGPRClass(RC: MRI->getRegClass(Reg: Src2->getReg())))
4232 return false;
4233
4234 // If src2 is also a literal constant then we have to choose which one to
4235 // fold. In general it is better to choose madak so that the other literal
4236 // can be materialized in an sgpr instead of a vgpr:
4237 // s_mov_b32 s0, literal
4238 // v_madak_f32 v0, s0, v0, literal
4239 // Instead of:
4240 // v_mov_b32 v1, literal
4241 // v_madmk_f32 v0, v0, literal, v1
4242 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src2->getReg());
4243 if (Def && Def->isMoveImmediate() &&
4244 !isInlineConstant(MO: Def->getOperand(i: 1)))
4245 return false;
4246
4247 unsigned NewOpc = getNewFMAMKInst(ST, Opc);
4248 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
4249 return false;
4250
4251 const std::optional<int64_t> SubRegImm = extractSubregFromImm(
4252 Imm, SubRegIndex: RegSrc == Src1 ? Src0->getSubReg() : Src1->getSubReg());
4253
4254 // FIXME: This would be a lot easier if we could return a new instruction
4255 // instead of having to modify in place.
4256
4257 Register SrcReg = RegSrc->getReg();
4258 unsigned SrcSubReg = RegSrc->getSubReg();
4259 Src0->setReg(SrcReg);
4260 Src0->setSubReg(SrcSubReg);
4261 Src0->setIsKill(RegSrc->isKill());
4262
4263 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
4264 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4265 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
4266 Opc == AMDGPU::V_FMAC_F16_e64 || Opc == AMDGPU::V_FMAC_F64_e64)
4267 UseMI.untieRegOperand(
4268 OpIdx: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2));
4269
4270 Src1->ChangeToImmediate(ImmVal: *SubRegImm);
4271
4272 removeModOperands(MI&: UseMI);
4273 UseMI.setDesc(get(Opcode: NewOpc));
4274
4275 if (NewOpc == AMDGPU::V_FMAMK_F16_t16 ||
4276 NewOpc == AMDGPU::V_FMAMK_F16_fake16) {
4277 const TargetRegisterClass *NewRC = getRegClass(MCID: get(Opcode: NewOpc), OpNum: 0);
4278 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
4279 BuildMI(BB&: *UseMI.getParent(), I: std::next(x: UseMI.getIterator()),
4280 MIMD: UseMI.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
4281 DestReg: UseMI.getOperand(i: 0).getReg())
4282 .addReg(RegNo: Tmp);
4283 UseMI.getOperand(i: 0).setReg(Tmp);
4284 CopyRegOperandToNarrowerRC(UseMI, 1, NewRC);
4285 CopyRegOperandToNarrowerRC(UseMI, 3, NewRC);
4286 }
4287
4288 bool DeleteDef = MRI->use_nodbg_empty(RegNo: Reg);
4289 if (DeleteDef)
4290 DefMI.eraseFromParent();
4291
4292 return true;
4293 }
4294
4295 // Added part is the constant: Use v_madak_{f16, f32}.
4296 if (Src2->isReg() && Src2->getReg() == Reg) {
4297 if (ST.getConstantBusLimit(Opcode: Opc) < 2) {
4298 // Not allowed to use constant bus for another operand.
4299 // We can however allow an inline immediate as src0.
4300 bool Src0Inlined = false;
4301 if (Src0->isReg()) {
4302 // Try to inline constant if possible.
4303 // If the Def moves immediate and the use is single
4304 // We are saving VGPR here.
4305 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src0->getReg());
4306 if (Def && Def->isMoveImmediate() &&
4307 isInlineConstant(MO: Def->getOperand(i: 1)) &&
4308 MRI->hasOneNonDBGUse(RegNo: Src0->getReg())) {
4309 Src0->ChangeToImmediate(ImmVal: Def->getOperand(i: 1).getImm());
4310 Src0Inlined = true;
4311 } else if (ST.getConstantBusLimit(Opcode: Opc) <= 1 &&
4312 RI.isSGPRReg(MRI: *MRI, Reg: Src0->getReg())) {
4313 return false;
4314 }
4315 // VGPR is okay as Src0 - fallthrough
4316 }
4317
4318 if (Src1->isReg() && !Src0Inlined) {
4319 // We have one slot for inlinable constant so far - try to fill it
4320 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src1->getReg());
4321 if (Def && Def->isMoveImmediate() &&
4322 isInlineConstant(MO: Def->getOperand(i: 1)) &&
4323 MRI->hasOneNonDBGUse(RegNo: Src1->getReg()) && commuteInstruction(MI&: UseMI))
4324 Src0->ChangeToImmediate(ImmVal: Def->getOperand(i: 1).getImm());
4325 else if (RI.isSGPRReg(MRI: *MRI, Reg: Src1->getReg()))
4326 return false;
4327 // VGPR is okay as Src1 - fallthrough
4328 }
4329 }
4330
4331 unsigned NewOpc = getNewFMAAKInst(ST, Opc);
4332 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
4333 return false;
4334
4335 // FIXME: This would be a lot easier if we could return a new instruction
4336 // instead of having to modify in place.
4337
4338 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
4339 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4340 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
4341 Opc == AMDGPU::V_FMAC_F16_e64 || Opc == AMDGPU::V_FMAC_F64_e64)
4342 UseMI.untieRegOperand(
4343 OpIdx: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2));
4344
4345 const std::optional<int64_t> SubRegImm =
4346 extractSubregFromImm(Imm, SubRegIndex: Src2->getSubReg());
4347
4348 // ChangingToImmediate adds Src2 back to the instruction.
4349 Src2->ChangeToImmediate(ImmVal: *SubRegImm);
4350
4351 // These come before src2.
4352 removeModOperands(MI&: UseMI);
4353 UseMI.setDesc(get(Opcode: NewOpc));
4354
4355 if (NewOpc == AMDGPU::V_FMAAK_F16_t16 ||
4356 NewOpc == AMDGPU::V_FMAAK_F16_fake16) {
4357 const TargetRegisterClass *NewRC = getRegClass(MCID: get(Opcode: NewOpc), OpNum: 0);
4358 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
4359 BuildMI(BB&: *UseMI.getParent(), I: std::next(x: UseMI.getIterator()),
4360 MIMD: UseMI.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
4361 DestReg: UseMI.getOperand(i: 0).getReg())
4362 .addReg(RegNo: Tmp);
4363 UseMI.getOperand(i: 0).setReg(Tmp);
4364 CopyRegOperandToNarrowerRC(UseMI, 1, NewRC);
4365 CopyRegOperandToNarrowerRC(UseMI, 2, NewRC);
4366 }
4367
4368 // It might happen that UseMI was commuted
4369 // and we now have SGPR as SRC1. If so 2 inlined
4370 // constant and SGPR are illegal.
4371 legalizeOperands(MI&: UseMI);
4372
4373 int NewSrc0Idx =
4374 AMDGPU::getNamedOperandIdx(Opcode: UseMI.getOpcode(), Name: AMDGPU::OpName::src0);
4375 if (!isOperandLegal(MI: UseMI, OpIdx: NewSrc0Idx))
4376 legalizeOpWithMove(MI&: UseMI, OpIdx: NewSrc0Idx);
4377
4378 bool DeleteDef = MRI->use_nodbg_empty(RegNo: Reg);
4379 if (DeleteDef)
4380 DefMI.eraseFromParent();
4381
4382 return true;
4383 }
4384 }
4385
4386 return false;
4387}
4388
4389static bool
4390memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1,
4391 ArrayRef<const MachineOperand *> BaseOps2) {
4392 if (BaseOps1.size() != BaseOps2.size())
4393 return false;
4394 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) {
4395 if (!BaseOps1[I]->isIdenticalTo(Other: *BaseOps2[I]))
4396 return false;
4397 }
4398 return true;
4399}
4400
4401static bool offsetsDoNotOverlap(LocationSize WidthA, int OffsetA,
4402 LocationSize WidthB, int OffsetB) {
4403 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
4404 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
4405 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
4406 return LowWidth.hasValue() &&
4407 LowOffset + (int)LowWidth.getValue() <= HighOffset;
4408}
4409
4410bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
4411 const MachineInstr &MIb) const {
4412 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;
4413 int64_t Offset0, Offset1;
4414 LocationSize Dummy0 = LocationSize::precise(Value: 0);
4415 LocationSize Dummy1 = LocationSize::precise(Value: 0);
4416 bool Offset0IsScalable, Offset1IsScalable;
4417 if (!getMemOperandsWithOffsetWidth(LdSt: MIa, BaseOps&: BaseOps0, Offset&: Offset0, OffsetIsScalable&: Offset0IsScalable,
4418 Width&: Dummy0) ||
4419 !getMemOperandsWithOffsetWidth(LdSt: MIb, BaseOps&: BaseOps1, Offset&: Offset1, OffsetIsScalable&: Offset1IsScalable,
4420 Width&: Dummy1))
4421 return false;
4422
4423 if (!memOpsHaveSameBaseOperands(BaseOps1: BaseOps0, BaseOps2: BaseOps1))
4424 return false;
4425
4426 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
4427 // FIXME: Handle ds_read2 / ds_write2.
4428 return false;
4429 }
4430 LocationSize Width0 = MIa.memoperands().front()->getSize();
4431 LocationSize Width1 = MIb.memoperands().front()->getSize();
4432 return offsetsDoNotOverlap(WidthA: Width0, OffsetA: Offset0, WidthB: Width1, OffsetB: Offset1);
4433}
4434
4435bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
4436 const MachineInstr &MIb) const {
4437 assert(MIa.mayLoadOrStore() &&
4438 "MIa must load from or modify a memory location");
4439 assert(MIb.mayLoadOrStore() &&
4440 "MIb must load from or modify a memory location");
4441
4442 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects())
4443 return false;
4444
4445 // XXX - Can we relax this between address spaces?
4446 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
4447 return false;
4448
4449 if (isLDSDMA(MI: MIa) || isLDSDMA(MI: MIb))
4450 return false;
4451
4452 if (MIa.isBundle() || MIb.isBundle())
4453 return false;
4454
4455 // These only alias each other, and only on overlapping dword ranges.
4456 const bool IsLdStIdxA = isa<AMDGPUMI::VLoadStoreIdxInst>(Val: MIa);
4457 const bool IsLdStIdxB = isa<AMDGPUMI::VLoadStoreIdxInst>(Val: MIb);
4458 if (IsLdStIdxA || IsLdStIdxB) {
4459 if (IsLdStIdxA && IsLdStIdxB)
4460 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4461 return true;
4462 }
4463
4464 // TODO: Should we check the address space from the MachineMemOperand? That
4465 // would allow us to distinguish objects we know don't alias based on the
4466 // underlying address space, even if it was lowered to a different one,
4467 // e.g. private accesses lowered to use MUBUF instructions on a scratch
4468 // buffer.
4469 if (isDS(MI: MIa)) {
4470 if (isDS(MI: MIb))
4471 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4472
4473 return !isFLAT(MI: MIb) || isSegmentSpecificFLAT(MI: MIb);
4474 }
4475
4476 if (isMUBUF(MI: MIa) || isMTBUF(MI: MIa)) {
4477 if (isMUBUF(MI: MIb) || isMTBUF(MI: MIb))
4478 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4479
4480 if (isFLAT(MI: MIb))
4481 return isFLATScratch(MI: MIb);
4482
4483 return !isSMRD(MI: MIb);
4484 }
4485
4486 if (isSMRD(MI: MIa)) {
4487 if (isSMRD(MI: MIb))
4488 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4489
4490 if (isFLAT(MI: MIb))
4491 return isFLATScratch(MI: MIb);
4492
4493 return !isMUBUF(MI: MIb) && !isMTBUF(MI: MIb);
4494 }
4495
4496 if (isFLAT(MI: MIa)) {
4497 if (isFLAT(MI: MIb)) {
4498 if ((isFLATScratch(MI: MIa) && isFLATGlobal(MI: MIb)) ||
4499 (isFLATGlobal(MI: MIa) && isFLATScratch(MI: MIb)))
4500 return true;
4501
4502 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4503 }
4504
4505 return false;
4506 }
4507
4508 return false;
4509}
4510
4511static unsigned getNewFMAInst(const GCNSubtarget &ST, unsigned Opc) {
4512 switch (Opc) {
4513 case AMDGPU::V_MAC_F16_e32:
4514 case AMDGPU::V_MAC_F16_e64:
4515 return AMDGPU::V_MAD_F16_e64;
4516 case AMDGPU::V_MAC_F32_e32:
4517 case AMDGPU::V_MAC_F32_e64:
4518 return AMDGPU::V_MAD_F32_e64;
4519 case AMDGPU::V_MAC_LEGACY_F32_e32:
4520 case AMDGPU::V_MAC_LEGACY_F32_e64:
4521 return AMDGPU::V_MAD_LEGACY_F32_e64;
4522 case AMDGPU::V_FMAC_LEGACY_F32_e32:
4523 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4524 return AMDGPU::V_FMA_LEGACY_F32_e64;
4525 case AMDGPU::V_FMAC_F16_e32:
4526 case AMDGPU::V_FMAC_F16_e64:
4527 case AMDGPU::V_FMAC_F16_t16_e64:
4528 case AMDGPU::V_FMAC_F16_fake16_e64:
4529 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
4530 ? AMDGPU::V_FMA_F16_gfx9_t16_e64
4531 : AMDGPU::V_FMA_F16_gfx9_fake16_e64
4532 : AMDGPU::V_FMA_F16_gfx9_e64;
4533 case AMDGPU::V_FMAC_F32_e32:
4534 case AMDGPU::V_FMAC_F32_e64:
4535 return AMDGPU::V_FMA_F32_e64;
4536 case AMDGPU::V_FMAC_F64_e32:
4537 case AMDGPU::V_FMAC_F64_e64:
4538 return AMDGPU::V_FMA_F64_e64;
4539 default:
4540 llvm_unreachable("invalid instruction");
4541 }
4542}
4543
4544/// Helper struct for the implementation of 3-address conversion to communicate
4545/// updates made to instruction operands.
4546struct SIInstrInfo::ThreeAddressUpdates {
4547 /// Other instruction whose def is no longer used by the converted
4548 /// instruction.
4549 MachineInstr *RemoveMIUse = nullptr;
4550};
4551
4552MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI,
4553 LiveIntervals *LIS) const {
4554 MachineBasicBlock &MBB = *MI.getParent();
4555 MachineInstr *CandidateMI = &MI;
4556
4557 if (MI.isBundle()) {
4558 // This is a temporary placeholder for bundle handling that enables us to
4559 // exercise the relevant code paths in the two-address instruction pass.
4560 if (MI.getBundleSize() != 1)
4561 return nullptr;
4562 CandidateMI = MI.getNextNode();
4563 }
4564
4565 ThreeAddressUpdates U;
4566 MachineInstr *NewMI = convertToThreeAddressImpl(MI&: *CandidateMI, Updates&: U);
4567 if (!NewMI)
4568 return nullptr;
4569
4570 if (MI.isBundle()) {
4571 CandidateMI->eraseFromBundle();
4572
4573 for (MachineOperand &MO : MI.all_defs()) {
4574 if (MO.isTied())
4575 MI.untieRegOperand(OpIdx: MO.getOperandNo());
4576 }
4577 } else {
4578 if (LIS) {
4579 LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *NewMI);
4580 // SlotIndex of defs needs to be updated when converting to early-clobber
4581 MachineOperand &Def = NewMI->getOperand(i: 0);
4582 if (Def.isEarlyClobber() && Def.isReg() &&
4583 LIS->hasInterval(Reg: Def.getReg())) {
4584 SlotIndex OldIndex = LIS->getInstructionIndex(Instr: *NewMI).getRegSlot(EC: false);
4585 SlotIndex NewIndex = LIS->getInstructionIndex(Instr: *NewMI).getRegSlot(EC: true);
4586 auto &LI = LIS->getInterval(Reg: Def.getReg());
4587 auto UpdateDefIndex = [&](LiveRange &LR) {
4588 auto *S = LR.find(Pos: OldIndex);
4589 if (S != LR.end() && S->start == OldIndex) {
4590 assert(S->valno && S->valno->def == OldIndex);
4591 S->start = NewIndex;
4592 S->valno->def = NewIndex;
4593 }
4594 };
4595 UpdateDefIndex(LI);
4596 for (auto &SR : LI.subranges())
4597 UpdateDefIndex(SR);
4598 }
4599
4600 // A use tied to an early-clobber def is read at the early-clobber slot.
4601 // The three-address instruction has no tied operands, so the use is now
4602 // read at the register slot.
4603 const MachineOperand &OldDef = MI.getOperand(i: 0);
4604 if (OldDef.isEarlyClobber()) {
4605 assert(OldDef.isTied() &&
4606 "expected a tied def in the two-address form");
4607 Register UseReg =
4608 MI.getOperand(i: MI.findTiedOperandIdx(OpIdx: OldDef.getOperandNo()))
4609 .getReg();
4610 SlotIndex Idx = LIS->getInstructionIndex(Instr: *NewMI);
4611 SlotIndex OldEnd = Idx.getRegSlot(EC: true);
4612 SlotIndex NewEnd = Idx.getRegSlot();
4613 auto UpdateUseIndex = [&](LiveRange &LR) {
4614 LiveRange::Segment *S = LR.getSegmentContaining(Idx);
4615 if (S && S->end == OldEnd)
4616 S->end = NewEnd;
4617 };
4618 LiveInterval &LI = LIS->getInterval(Reg: UseReg);
4619 UpdateUseIndex(LI);
4620 for (LiveInterval::SubRange &SR : LI.subranges())
4621 UpdateUseIndex(SR);
4622 }
4623 }
4624 }
4625
4626 if (U.RemoveMIUse) {
4627 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4628 // The only user is the instruction which will be killed.
4629 Register DefReg = U.RemoveMIUse->getOperand(i: 0).getReg();
4630
4631 if (MRI.hasOneNonDBGUse(RegNo: DefReg)) {
4632 // We cannot just remove the DefMI here, calling pass will crash.
4633 U.RemoveMIUse->setDesc(get(Opcode: AMDGPU::IMPLICIT_DEF));
4634 U.RemoveMIUse->getOperand(i: 0).setIsDead(true);
4635 for (unsigned I = U.RemoveMIUse->getNumOperands() - 1; I != 0; --I)
4636 U.RemoveMIUse->removeOperand(OpNo: I);
4637 }
4638
4639 if (MI.isBundle()) {
4640 VirtRegInfo VRI = AnalyzeVirtRegInBundle(MI, Reg: DefReg);
4641 if (!VRI.Reads && !VRI.Writes) {
4642 for (MachineOperand &MO : MI.all_uses()) {
4643 if (MO.isReg() && MO.getReg() == DefReg) {
4644 assert(MO.getSubReg() == 0 &&
4645 "tied sub-registers in bundles currently not supported");
4646 MI.removeOperand(OpNo: MO.getOperandNo());
4647 break;
4648 }
4649 }
4650
4651 if (LIS)
4652 LIS->shrinkToUses(li: &LIS->getInterval(Reg: DefReg));
4653 }
4654 } else if (LIS) {
4655 LiveInterval &DefLI = LIS->getInterval(Reg: DefReg);
4656
4657 // We cannot delete the original instruction here, so hack out the use
4658 // in the original instruction with a dummy register so we can use
4659 // shrinkToUses to deal with any multi-use edge cases. Other targets do
4660 // not have the complexity of deleting a use to consider here.
4661 Register DummyReg = MRI.cloneVirtualRegister(VReg: DefReg);
4662 for (MachineOperand &MIOp : MI.uses()) {
4663 if (MIOp.isReg() && MIOp.getReg() == DefReg) {
4664 MIOp.setIsUndef(true);
4665 MIOp.setReg(DummyReg);
4666 }
4667 }
4668
4669 if (MI.isBundle()) {
4670 VirtRegInfo VRI = AnalyzeVirtRegInBundle(MI, Reg: DefReg);
4671 if (!VRI.Reads && !VRI.Writes) {
4672 for (MachineOperand &MIOp : MI.uses()) {
4673 if (MIOp.isReg() && MIOp.getReg() == DefReg) {
4674 MIOp.setIsUndef(true);
4675 MIOp.setReg(DummyReg);
4676 }
4677 }
4678 }
4679
4680 MI.addOperand(Op: MachineOperand::CreateReg(Reg: DummyReg, isDef: false, isImp: false, isKill: false,
4681 isDead: false, /*isUndef=*/true));
4682 }
4683
4684 LIS->shrinkToUses(li: &DefLI);
4685 }
4686 }
4687
4688 return MI.isBundle() ? &MI : NewMI;
4689}
4690
4691MachineInstr *
4692SIInstrInfo::convertToThreeAddressImpl(MachineInstr &MI,
4693 ThreeAddressUpdates &U) const {
4694 MachineBasicBlock &MBB = *MI.getParent();
4695 unsigned Opc = MI.getOpcode();
4696
4697 // Handle MFMA.
4698 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opcode: Opc);
4699 if (NewMFMAOpc != -1) {
4700 MachineInstrBuilder MIB =
4701 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewMFMAOpc));
4702 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I)
4703 MIB.add(MO: MI.getOperand(i: I));
4704 return MIB;
4705 }
4706
4707 if (SIInstrInfo::isWMMA(MI)) {
4708 unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(Opc: MI.getOpcode());
4709 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4710 .setMIFlags(MI.getFlags());
4711 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I)
4712 MIB->addOperand(Op: MI.getOperand(i: I));
4713 return MIB;
4714 }
4715
4716 assert(Opc != AMDGPU::V_FMAC_F16_t16_e32 &&
4717 Opc != AMDGPU::V_FMAC_F16_fake16_e32 &&
4718 "V_FMAC_F16_t16/fake16_e32 is not supported and not expected to be "
4719 "present pre-RA");
4720
4721 // Handle MAC/FMAC.
4722 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
4723 bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 ||
4724 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 ||
4725 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
4726 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64;
4727 bool Src0Literal = false;
4728
4729 switch (Opc) {
4730 default:
4731 return nullptr;
4732 case AMDGPU::V_MAC_F16_e64:
4733 case AMDGPU::V_FMAC_F16_e64:
4734 case AMDGPU::V_FMAC_F16_t16_e64:
4735 case AMDGPU::V_FMAC_F16_fake16_e64:
4736 case AMDGPU::V_MAC_F32_e64:
4737 case AMDGPU::V_MAC_LEGACY_F32_e64:
4738 case AMDGPU::V_FMAC_F32_e64:
4739 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4740 case AMDGPU::V_FMAC_F64_e64:
4741 break;
4742 case AMDGPU::V_MAC_F16_e32:
4743 case AMDGPU::V_FMAC_F16_e32:
4744 case AMDGPU::V_MAC_F32_e32:
4745 case AMDGPU::V_MAC_LEGACY_F32_e32:
4746 case AMDGPU::V_FMAC_F32_e32:
4747 case AMDGPU::V_FMAC_LEGACY_F32_e32:
4748 case AMDGPU::V_FMAC_F64_e32: {
4749 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
4750 Name: AMDGPU::OpName::src0);
4751 const MachineOperand *Src0 = &MI.getOperand(i: Src0Idx);
4752 if (!Src0->isReg() && !Src0->isImm())
4753 return nullptr;
4754
4755 if (Src0->isImm() && !isInlineConstant(MI, OpIdx: Src0Idx, MO: *Src0))
4756 Src0Literal = true;
4757
4758 break;
4759 }
4760 }
4761
4762 MachineInstrBuilder MIB;
4763 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
4764 const MachineOperand *Src0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
4765 const MachineOperand *Src0Mods =
4766 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
4767 const MachineOperand *Src1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
4768 const MachineOperand *Src1Mods =
4769 getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
4770 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
4771 const MachineOperand *Src2Mods =
4772 getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers);
4773 const MachineOperand *Clamp = getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
4774 const MachineOperand *Omod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
4775 const MachineOperand *OpSel = getNamedOperand(MI, OperandName: AMDGPU::OpName::op_sel);
4776
4777 if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsLegacy &&
4778 (!IsF64 || ST.hasFmaakFmamkF64Insts()) &&
4779 // If we have an SGPR input, we will violate the constant bus restriction.
4780 (ST.getConstantBusLimit(Opcode: Opc) > 1 || !Src0->isReg() ||
4781 !RI.isSGPRReg(MRI: MBB.getParent()->getRegInfo(), Reg: Src0->getReg()))) {
4782 MachineInstr *DefMI = nullptr;
4783 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4784 std::optional<int64_t> ImmOpt;
4785 int64_t Imm;
4786
4787 if (!Src0Literal &&
4788 (ImmOpt = getImmOrMaterializedImm(MRI, Op: *Src2, DefMI: &DefMI))) {
4789 unsigned NewOpc = getNewFMAAKInst(ST, Opc);
4790 if (pseudoToMCOpcode(Opcode: NewOpc) != -1) {
4791 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4792 .add(MO: *Dst)
4793 .add(MO: *Src0)
4794 .add(MO: *Src1)
4795 .addImm(Val: *ImmOpt)
4796 .setMIFlags(MI.getFlags());
4797 U.RemoveMIUse = DefMI;
4798 return MIB;
4799 }
4800 }
4801 unsigned NewOpc = getNewFMAMKInst(ST, Opc);
4802 if (!Src0Literal &&
4803 (ImmOpt = getImmOrMaterializedImm(MRI, Op: *Src1, DefMI: &DefMI))) {
4804 if (pseudoToMCOpcode(Opcode: NewOpc) != -1) {
4805 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4806 .add(MO: *Dst)
4807 .add(MO: *Src0)
4808 .addImm(Val: *ImmOpt)
4809 .add(MO: *Src2)
4810 .setMIFlags(MI.getFlags());
4811 U.RemoveMIUse = DefMI;
4812 return MIB;
4813 }
4814 }
4815 if ((ImmOpt = getImmOrMaterializedImm(MRI, Op: *Src0, DefMI: &DefMI))) {
4816 Imm = *ImmOpt;
4817 if (pseudoToMCOpcode(Opcode: NewOpc) != -1 &&
4818 isOperandLegal(
4819 MI, OpIdx: AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::src0),
4820 MO: Src1)) {
4821 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4822 .add(MO: *Dst)
4823 .add(MO: *Src1)
4824 .addImm(Val: Imm)
4825 .add(MO: *Src2)
4826 .setMIFlags(MI.getFlags());
4827 U.RemoveMIUse = DefMI;
4828 return MIB;
4829 }
4830 }
4831 }
4832
4833 // VOP2 mac/fmac with a literal operand cannot be converted to VOP3 mad/fma
4834 // if VOP3 does not allow a literal operand.
4835 if (Src0Literal && !ST.hasVOP3Literal())
4836 return nullptr;
4837
4838 unsigned NewOpc = getNewFMAInst(ST, Opc);
4839
4840 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
4841 return nullptr;
4842
4843 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4844 .add(MO: *Dst)
4845 .addImm(Val: Src0Mods ? Src0Mods->getImm() : 0)
4846 .add(MO: *Src0)
4847 .addImm(Val: Src1Mods ? Src1Mods->getImm() : 0)
4848 .add(MO: *Src1)
4849 .addImm(Val: Src2Mods ? Src2Mods->getImm() : 0)
4850 .add(MO: *Src2)
4851 .addImm(Val: Clamp ? Clamp->getImm() : 0)
4852 .addImm(Val: Omod ? Omod->getImm() : 0)
4853 .setMIFlags(MI.getFlags());
4854 if (AMDGPU::hasNamedOperand(Opcode: NewOpc, NamedIdx: AMDGPU::OpName::op_sel))
4855 MIB.addImm(Val: OpSel ? OpSel->getImm() : 0);
4856 return MIB;
4857}
4858
4859// It's not generally safe to move VALU instructions across these since it will
4860// start using the register as a base index rather than directly.
4861// XXX - Why isn't hasSideEffects sufficient for these?
4862static bool changesVGPRIndexingMode(const MachineInstr &MI) {
4863 switch (MI.getOpcode()) {
4864 case AMDGPU::S_SET_GPR_IDX_ON:
4865 case AMDGPU::S_SET_GPR_IDX_MODE:
4866 case AMDGPU::S_SET_GPR_IDX_OFF:
4867 return true;
4868 default:
4869 return false;
4870 }
4871}
4872
4873bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
4874 const MachineBasicBlock *MBB,
4875 const MachineFunction &MF) const {
4876 // Skipping the check for SP writes in the base implementation. The reason it
4877 // was added was apparently due to compile time concerns.
4878 //
4879 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops
4880 // but is probably avoidable.
4881
4882 // Copied from base implementation.
4883 // Terminators and labels can't be scheduled around.
4884 if (MI.isTerminator() || MI.isPosition())
4885 return true;
4886
4887 // INLINEASM_BR can jump to another block
4888 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
4889 return true;
4890
4891 if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(i: 0).getImm() == 0)
4892 return true;
4893
4894 // Target-independent instructions do not have an implicit-use of EXEC, even
4895 // when they operate on VGPRs. Treating EXEC modifications as scheduling
4896 // boundaries prevents incorrect movements of such instructions.
4897 return MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &RI) ||
4898 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
4899 MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
4900 MI.getOpcode() == AMDGPU::S_SETPRIO ||
4901 MI.getOpcode() == AMDGPU::S_SETPRIO_INC_WG ||
4902 changesVGPRIndexingMode(MI);
4903}
4904
4905bool SIInstrInfo::isAlwaysGDS(uint32_t Opcode) const {
4906 return Opcode == AMDGPU::DS_ORDERED_COUNT ||
4907 Opcode == AMDGPU::DS_ADD_GS_REG_RTN ||
4908 Opcode == AMDGPU::DS_SUB_GS_REG_RTN || isGWS(Opcode);
4909}
4910
4911bool SIInstrInfo::mayAccessScratch(const MachineInstr &MI) const {
4912 // Instructions that access scratch use FLAT encoding or BUF encodings.
4913 if ((!isFLAT(MI) || isFLATGlobal(MI)) && !isBUF(MI))
4914 return false;
4915
4916 // SCRATCH instructions always access scratch.
4917 if (isFLATScratch(MI))
4918 return true;
4919
4920 // If FLAT_SCRATCH registers are not initialized, we can never access scratch
4921 // via the aperture.
4922 if (MI.getMF()->getFunction().hasFnAttribute(Kind: "amdgpu-no-flat-scratch-init"))
4923 return false;
4924
4925 // If there are no memory operands then conservatively assume the flat
4926 // operation may access scratch.
4927 if (MI.memoperands_empty())
4928 return true;
4929
4930 // See if any memory operand specifies an address space that involves scratch.
4931 return any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *Memop) {
4932 unsigned AS = Memop->getAddrSpace();
4933 if (AS == AMDGPUAS::FLAT_ADDRESS) {
4934 const MDNode *MD = Memop->getAAInfo().NoAliasAddrSpace;
4935 return !MD || !AMDGPU::hasValueInRangeLikeMetadata(
4936 MD: *MD, Val: AMDGPUAS::PRIVATE_ADDRESS);
4937 }
4938 return AS == AMDGPUAS::PRIVATE_ADDRESS;
4939 });
4940}
4941
4942bool SIInstrInfo::mayAccessVMEMThroughFlat(const MachineInstr &MI) const {
4943 assert(isFLAT(MI));
4944
4945 // All flat instructions use the VMEM counter except prefetch.
4946 if (!usesVM_CNT(MI))
4947 return false;
4948
4949 // If there are no memory operands then conservatively assume the flat
4950 // operation may access VMEM.
4951 if (MI.memoperands_empty())
4952 return true;
4953
4954 // See if any memory operand specifies an address space that involves VMEM.
4955 // Flat operations only supported FLAT, LOCAL (LDS), or address spaces
4956 // involving VMEM such as GLOBAL, CONSTANT, PRIVATE (SCRATCH), etc. The REGION
4957 // (GDS) address space is not supported by flat operations. Therefore, simply
4958 // return true unless only the LDS address space is found.
4959 for (const MachineMemOperand *Memop : MI.memoperands()) {
4960 unsigned AS = Memop->getAddrSpace();
4961 assert(AS != AMDGPUAS::REGION_ADDRESS);
4962 if (AS != AMDGPUAS::LOCAL_ADDRESS)
4963 return true;
4964 }
4965
4966 return false;
4967}
4968
4969bool SIInstrInfo::mayAccessLDSThroughFlat(const MachineInstr &MI,
4970 bool TgSplit) const {
4971 assert(isFLAT(MI));
4972
4973 // Flat instruction such as SCRATCH and GLOBAL do not use the lgkm counter.
4974 if (!usesLGKM_CNT(MI))
4975 return false;
4976
4977 // If in tgsplit mode then there can be no use of LDS.
4978 if (TgSplit)
4979 return false;
4980
4981 // If there are no memory operands then conservatively assume the flat
4982 // operation may access LDS.
4983 if (MI.memoperands_empty())
4984 return true;
4985
4986 // See if any memory operand specifies an address space that involves LDS.
4987 for (const MachineMemOperand *Memop : MI.memoperands()) {
4988 unsigned AS = Memop->getAddrSpace();
4989 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)
4990 return true;
4991 }
4992
4993 return false;
4994}
4995
4996AMDGPU::AsyncStages SIInstrInfo::getAsyncStage(const MachineInstr &MI) const {
4997 if (!isAsync(MI))
4998 return AMDGPU::AsyncStages::NONE;
4999 if (usesTENSOR_CNT(MI))
5000 return AMDGPU::AsyncStages::TENSOR;
5001 if (!usesASYNC_CNT(MI))
5002 return AMDGPU::AsyncStages::BUFFER_GLOBAL_LOAD;
5003
5004 // TODO: Declare the stage of each async instruction in TableGen alongside
5005 // its definition, instead of listing opcodes here.
5006 switch (MI.getOpcode()) {
5007 // Keep in sync with FLATInstructions.td.
5008 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8:
5009 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay:
5010 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_SADDR:
5011 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B32:
5012 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B32_SADDR:
5013 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B64:
5014 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B64_SADDR:
5015 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B128:
5016 case AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B128_SADDR:
5017 return AMDGPU::AsyncStages::GLOBAL_LOAD_ASYNC_TO_LDS;
5018 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B8:
5019 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B8_SADDR:
5020 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B32:
5021 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B32_SADDR:
5022 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B64:
5023 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B64_SADDR:
5024 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B128:
5025 case AMDGPU::CLUSTER_LOAD_ASYNC_TO_LDS_B128_SADDR:
5026 return AMDGPU::AsyncStages::GLOBAL_LOAD_ASYNC_TO_LDS_MCAST;
5027 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B8:
5028 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B8_SADDR:
5029 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B32:
5030 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B32_SADDR:
5031 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B64:
5032 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B64_SADDR:
5033 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B128:
5034 case AMDGPU::GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR:
5035 return AMDGPU::AsyncStages::GLOBAL_STORE_ASYNC_FROM_LDS;
5036 default:
5037 llvm_unreachable("Async opcode has no associated async stage");
5038 }
5039}
5040
5041bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) {
5042 // Skip the full operand and register alias search modifiesRegister
5043 // does. There's only a handful of instructions that touch this, it's only an
5044 // implicit def, and doesn't alias any other registers.
5045 return is_contained(Range: MI.getDesc().implicit_defs(), Element: AMDGPU::MODE);
5046}
5047
5048bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const {
5049 unsigned Opcode = MI.getOpcode();
5050
5051 if (MI.mayStore() && isSMRD(MI))
5052 return true; // scalar store or atomic
5053
5054 // This will terminate the function when other lanes may need to continue.
5055 if (MI.isReturn())
5056 return true;
5057
5058 // These instructions cause shader I/O that may cause hardware lockups
5059 // when executed with an empty EXEC mask.
5060 //
5061 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
5062 // EXEC = 0, but checking for that case here seems not worth it
5063 // given the typical code patterns.
5064 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
5065 isEXP(Opcode) || Opcode == AMDGPU::DS_ORDERED_COUNT ||
5066 Opcode == AMDGPU::S_TRAP || Opcode == AMDGPU::S_WAIT_EVENT ||
5067 Opcode == AMDGPU::S_SETHALT)
5068 return true;
5069
5070 if (MI.isCall() || MI.isInlineAsm())
5071 return true; // conservative assumption
5072
5073 // V_PERM_PK16 must issue with EXEC != 0 so its follower (or an inserted
5074 // V_NOP) actually runs on the VALU pipe. Returning true here keeps the
5075 // s_cbranch_execz that skips this region when EXEC is empty.
5076 if (ST.hasVPermPk16Hazard() && isVPermPk16(Opcode))
5077 return true;
5078
5079 // Assume that barrier interactions are only intended with active lanes.
5080 if (isBarrier(Opcode))
5081 return true;
5082
5083 // A mode change is a scalar operation that influences vector instructions.
5084 if (modifiesModeRegister(MI))
5085 return true;
5086
5087 // These are like SALU instructions in terms of effects, so it's questionable
5088 // whether we should return true for those.
5089 //
5090 // However, executing them with EXEC = 0 causes them to operate on undefined
5091 // data, which we avoid by returning true here.
5092 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 ||
5093 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32 ||
5094 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
5095 Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR)
5096 return true;
5097
5098 return false;
5099}
5100
5101bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI,
5102 const MachineInstr &MI) const {
5103 if (MI.isMetaInstruction())
5104 return false;
5105
5106 // This won't read exec if this is an SGPR->SGPR copy.
5107 if (MI.isCopyLike()) {
5108 if (!RI.isSGPRReg(MRI, Reg: MI.getOperand(i: 0).getReg()))
5109 return true;
5110
5111 // Make sure this isn't copying exec as a normal operand
5112 return MI.readsRegister(Reg: AMDGPU::EXEC, TRI: &RI);
5113 }
5114
5115 // Make a conservative assumption about the callee.
5116 if (MI.isCall())
5117 return true;
5118
5119 // Be conservative with any unhandled generic opcodes.
5120 if (!isTargetSpecificOpcode(Opcode: MI.getOpcode()))
5121 return true;
5122
5123 return !isSALU(MI) || MI.readsRegister(Reg: AMDGPU::EXEC, TRI: &RI);
5124}
5125
5126bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
5127 switch (Imm.getBitWidth()) {
5128 case 1: // This likely will be a condition code mask.
5129 return true;
5130
5131 case 32:
5132 return AMDGPU::isInlinableLiteral32(Literal: Imm.getSExtValue(),
5133 HasInv2Pi: ST.hasInv2PiInlineImm());
5134 case 64:
5135 return AMDGPU::isInlinableLiteral64(Literal: Imm.getSExtValue(),
5136 HasInv2Pi: ST.hasInv2PiInlineImm());
5137 case 16:
5138 return ST.has16BitInsts() &&
5139 AMDGPU::isInlinableLiteralI16(Literal: Imm.getSExtValue(),
5140 HasInv2Pi: ST.hasInv2PiInlineImm());
5141 default:
5142 llvm_unreachable("invalid bitwidth");
5143 }
5144}
5145
5146bool SIInstrInfo::isInlineConstant(const APFloat &Imm) const {
5147 APInt IntImm = Imm.bitcastToAPInt();
5148 int64_t IntImmVal = IntImm.getSExtValue();
5149 bool HasInv2Pi = ST.hasInv2PiInlineImm();
5150 switch (APFloat::SemanticsToEnum(Sem: Imm.getSemantics())) {
5151 default:
5152 llvm_unreachable("invalid fltSemantics");
5153 case APFloatBase::S_IEEEsingle:
5154 case APFloatBase::S_IEEEdouble:
5155 return isInlineConstant(Imm: IntImm);
5156 case APFloatBase::S_BFloat:
5157 return ST.has16BitInsts() &&
5158 AMDGPU::isInlinableLiteralBF16(Literal: IntImmVal, HasInv2Pi);
5159 case APFloatBase::S_IEEEhalf:
5160 return ST.has16BitInsts() &&
5161 AMDGPU::isInlinableLiteralFP16(Literal: IntImmVal, HasInv2Pi);
5162 }
5163}
5164
5165bool SIInstrInfo::isInlineConstant(int64_t Imm, uint8_t OperandType) const {
5166 // MachineOperand provides no way to tell the true operand size, since it only
5167 // records a 64-bit value. We need to know the size to determine if a 32-bit
5168 // floating point immediate bit pattern is legal for an integer immediate. It
5169 // would be for any 32-bit integer operand, but would not be for a 64-bit one.
5170 switch (OperandType) {
5171 case AMDGPU::OPERAND_REG_IMM_INT32:
5172 case AMDGPU::OPERAND_REG_IMM_FP32:
5173 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
5174 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
5175 case AMDGPU::OPERAND_REG_IMM_V2FP32:
5176 case AMDGPU::OPERAND_REG_IMM_V2INT32:
5177 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
5178 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
5179 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: {
5180 int32_t Trunc = static_cast<int32_t>(Imm);
5181 return AMDGPU::isInlinableLiteral32(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
5182 }
5183 case AMDGPU::OPERAND_REG_IMM_INT64:
5184 case AMDGPU::OPERAND_REG_IMM_FP64:
5185 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
5186 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
5187 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
5188 case AMDGPU::OPERAND_REG_IMM_V2FP64:
5189 case AMDGPU::OPERAND_REG_IMM_V2INT64:
5190 return AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: ST.hasInv2PiInlineImm());
5191 case AMDGPU::OPERAND_REG_IMM_INT16:
5192 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
5193 // We would expect inline immediates to not be concerned with an integer/fp
5194 // distinction. However, in the case of 16-bit integer operations, the
5195 // "floating point" values appear to not work. It seems read the low 16-bits
5196 // of 32-bit immediates, which happens to always work for the integer
5197 // values.
5198 //
5199 // See llvm bugzilla 46302.
5200 //
5201 // TODO: Theoretically we could use op-sel to use the high bits of the
5202 // 32-bit FP values.
5203 return AMDGPU::isInlinableIntLiteral(Literal: Imm);
5204 case AMDGPU::OPERAND_REG_IMM_V2INT16:
5205 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
5206 return AMDGPU::isInlinableLiteralV2I16(Literal: Imm);
5207 case AMDGPU::OPERAND_REG_IMM_V2FP16:
5208 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
5209 return AMDGPU::isInlinableLiteralV2F16(Literal: Imm);
5210 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
5211 return AMDGPU::isPKFMACF16InlineConstant(Literal: Imm, IsGFX11Plus: ST.isGFX11Plus());
5212 case AMDGPU::OPERAND_REG_IMM_V2BF16:
5213 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
5214 return AMDGPU::isInlinableLiteralV2BF16(Literal: Imm);
5215 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
5216 return false;
5217 case AMDGPU::OPERAND_REG_IMM_FP16:
5218 case AMDGPU::OPERAND_REG_INLINE_C_FP16: {
5219 if (isInt<16>(x: Imm) || isUInt<16>(x: Imm)) {
5220 // A few special case instructions have 16-bit operands on subtargets
5221 // where 16-bit instructions are not legal.
5222 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
5223 // constants in these cases
5224 int16_t Trunc = static_cast<int16_t>(Imm);
5225 return ST.has16BitInsts() &&
5226 AMDGPU::isInlinableLiteralFP16(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
5227 }
5228
5229 return false;
5230 }
5231 case AMDGPU::OPERAND_REG_IMM_BF16:
5232 case AMDGPU::OPERAND_REG_INLINE_C_BF16: {
5233 if (isInt<16>(x: Imm) || isUInt<16>(x: Imm)) {
5234 int16_t Trunc = static_cast<int16_t>(Imm);
5235 return ST.has16BitInsts() &&
5236 AMDGPU::isInlinableLiteralBF16(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
5237 }
5238 return false;
5239 }
5240 case AMDGPU::OPERAND_KIMM32:
5241 case AMDGPU::OPERAND_KIMM16:
5242 case AMDGPU::OPERAND_KIMM64:
5243 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
5244 return false;
5245 case AMDGPU::OPERAND_INLINE_C_AV64_PSEUDO:
5246 return isLegalAV64PseudoImm(Imm);
5247 case AMDGPU::OPERAND_INPUT_MODS:
5248 case MCOI::OPERAND_IMMEDIATE:
5249 // Always embedded in the instruction for free.
5250 return true;
5251 case MCOI::OPERAND_UNKNOWN:
5252 case MCOI::OPERAND_REGISTER:
5253 case MCOI::OPERAND_PCREL:
5254 case MCOI::OPERAND_GENERIC_0:
5255 case MCOI::OPERAND_GENERIC_1:
5256 case MCOI::OPERAND_GENERIC_2:
5257 case MCOI::OPERAND_GENERIC_3:
5258 case MCOI::OPERAND_GENERIC_4:
5259 case MCOI::OPERAND_GENERIC_5:
5260 // Just ignore anything else.
5261 return false;
5262 default:
5263 llvm_unreachable("invalid operand type");
5264 }
5265}
5266
5267static bool compareMachineOp(const MachineOperand &Op0,
5268 const MachineOperand &Op1) {
5269 if (Op0.getType() != Op1.getType())
5270 return false;
5271
5272 switch (Op0.getType()) {
5273 case MachineOperand::MO_Register:
5274 return Op0.getReg() == Op1.getReg();
5275 case MachineOperand::MO_Immediate:
5276 return Op0.getImm() == Op1.getImm();
5277 default:
5278 llvm_unreachable("Didn't expect to be comparing these operand types");
5279 }
5280}
5281
5282bool SIInstrInfo::isLiteralOperandLegal(const MCInstrDesc &InstDesc,
5283 const MCOperandInfo &OpInfo) const {
5284 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
5285 return true;
5286
5287 if (!RI.opCanUseLiteralConstant(OpType: OpInfo.OperandType))
5288 return false;
5289
5290 if (!isVOP3(Desc: InstDesc) || !AMDGPU::isSISrcOperand(OpInfo))
5291 return true;
5292
5293 return ST.hasVOP3Literal();
5294}
5295
5296bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
5297 int64_t ImmVal) const {
5298 const unsigned Opc = InstDesc.getOpcode();
5299 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
5300 if (Src1Idx != -1 && isDPP(Opcode: Opc) && !ST.hasDPPSrc1SGPR() &&
5301 OpNo == static_cast<unsigned>(Src1Idx))
5302 return false;
5303
5304 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
5305 if (isInlineConstant(Imm: ImmVal, OperandType: OpInfo.OperandType)) {
5306 if (isMAI(Desc: InstDesc) && ST.hasMFMAInlineLiteralBug() &&
5307 OpNo == (unsigned)AMDGPU::getNamedOperandIdx(Opcode: InstDesc.getOpcode(),
5308 Name: AMDGPU::OpName::src2))
5309 return false;
5310
5311 if (ST.hasBF16InlineConstFromUpperFP32() && isVOP1(Opcode: Opc)) {
5312 if ((OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_BF16 ||
5313 OpInfo.OperandType == AMDGPU::OPERAND_REG_INLINE_C_BF16) &&
5314 isInlineConstant(Imm: ImmVal, OperandType: OpInfo.OperandType))
5315 return false;
5316 }
5317
5318 return RI.opCanUseInlineConstant(OpType: OpInfo.OperandType);
5319 }
5320
5321 return isLiteralOperandLegal(InstDesc, OpInfo);
5322}
5323
5324bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
5325 const MachineOperand &MO) const {
5326 if (MO.isImm())
5327 return isImmOperandLegal(InstDesc, OpNo, ImmVal: MO.getImm());
5328
5329 assert((MO.isTargetIndex() || MO.isFI() || MO.isGlobal()) &&
5330 "unexpected imm-like operand kind");
5331 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
5332 return isLiteralOperandLegal(InstDesc, OpInfo);
5333}
5334
5335bool SIInstrInfo::isLegalAV64PseudoImm(uint64_t Imm) const {
5336 // 2 32-bit inline constants packed into one.
5337 return AMDGPU::isInlinableLiteral32(Literal: Lo_32(Value: Imm), HasInv2Pi: ST.hasInv2PiInlineImm()) &&
5338 AMDGPU::isInlinableLiteral32(Literal: Hi_32(Value: Imm), HasInv2Pi: ST.hasInv2PiInlineImm());
5339}
5340
5341bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
5342 // GFX90A does not have V_MUL_LEGACY_F32_e32.
5343 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())
5344 return false;
5345
5346 int Op32 = AMDGPU::getVOPe32(Opcode);
5347 if (Op32 == -1)
5348 return false;
5349
5350 return pseudoToMCOpcode(Opcode: Op32) != -1;
5351}
5352
5353/// Return true if \p MI is a VALU comparison, i.e. an instruction that writes
5354/// a lane mask with one bit per lane, and zeroes the bits of lanes that were
5355/// inactive when it executed.
5356///
5357/// TODO: Also handle the sdst result of V_ADD_CO_U32 and V_SUB_CO_U32 and
5358/// V_DIV_SCALE_F32.
5359static bool isVCmp(const SIInstrInfo &TII, const MachineInstr &MI) {
5360 if (TII.isVOPC(MI))
5361 return true;
5362 int Op32 = AMDGPU::getVOPe32(Opcode: MI.getOpcode());
5363 return Op32 != -1 && TII.isVOPC(Opcode: Op32);
5364}
5365
5366bool SIInstrInfo::isMaskedByExec(Register Reg, const MachineInstr &Use,
5367 const MachineRegisterInfo &MRI,
5368 unsigned Depth) const {
5369 assert(MRI.isSSA() && "isMaskedByExec requires SSA form");
5370 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
5371 const MachineBasicBlock *MBB = Use.getParent();
5372
5373 // EXEC itself is trivially masked by EXEC.
5374 if (Reg == LMC.ExecReg)
5375 return true;
5376
5377 // Maximum depth of the def-use walk.
5378 constexpr unsigned MaxDepth = 6;
5379 if (Depth >= MaxDepth || !Reg.isVirtual())
5380 return false;
5381
5382 // Only look at definitions that can execute under the same EXEC mask as the
5383 // use.
5384 const MachineInstr *Def = MRI.getVRegDef(Reg);
5385 if (!Def || Def->getParent() != MBB)
5386 return false;
5387
5388 if (isVCmp(TII: *this, MI: *Def))
5389 return true;
5390
5391 // Recurse into an operand, which must be a whole register to say anything
5392 // about the whole lane mask.
5393 auto Recurse = [&](unsigned OpIdx) {
5394 const MachineOperand &MO = Def->getOperand(i: OpIdx);
5395 return MO.isReg() && !MO.getSubReg() &&
5396 isMaskedByExec(Reg: MO.getReg(), Use, MRI, Depth: Depth + 1);
5397 };
5398
5399 unsigned Opc = Def->getOpcode();
5400 if (Opc == AMDGPU::COPY && Recurse(1))
5401 return true;
5402 if (Opc == LMC.AndOpc && (Recurse(1) || Recurse(2)))
5403 return true;
5404 if (Opc == LMC.AndN2Opc && Recurse(1))
5405 return true;
5406 if ((Opc == LMC.OrOpc || Opc == LMC.XorOpc) && Recurse(1) && Recurse(2))
5407 return true;
5408 // TODO: Sometimes we encounter "reg = S_CSELECT -1, 0". If Reg has no other
5409 // uses this could be optimized to "reg = S_CSELECT $exec, 0".
5410
5411 return false;
5412}
5413
5414bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
5415 // The src0_modifier operand is present on all instructions
5416 // that have modifiers.
5417
5418 return AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0_modifiers);
5419}
5420
5421bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI,
5422 AMDGPU::OpName OpName) const {
5423 const MachineOperand *Mods = getNamedOperand(MI, OperandName: OpName);
5424 return Mods && Mods->getImm();
5425}
5426
5427bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const {
5428 return any_of(Range: ModifierOpNames,
5429 P: [&](AMDGPU::OpName Name) { return hasModifiersSet(MI, OpName: Name); });
5430}
5431
5432bool SIInstrInfo::canShrink(const MachineInstr &MI,
5433 const MachineRegisterInfo &MRI) const {
5434 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
5435 // Can't shrink instruction with three operands.
5436 if (Src2) {
5437 switch (MI.getOpcode()) {
5438 default: return false;
5439
5440 case AMDGPU::V_ADDC_U32_e64:
5441 case AMDGPU::V_SUBB_U32_e64:
5442 case AMDGPU::V_SUBBREV_U32_e64: {
5443 const MachineOperand *Src1
5444 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
5445 if (!Src1->isReg() || !RI.isVGPR(MRI, Reg: Src1->getReg()))
5446 return false;
5447 // Additional verification is needed for sdst/src2.
5448 return true;
5449 }
5450 case AMDGPU::V_MAC_F16_e64:
5451 case AMDGPU::V_MAC_F32_e64:
5452 case AMDGPU::V_MAC_LEGACY_F32_e64:
5453 case AMDGPU::V_FMAC_F16_e64:
5454 case AMDGPU::V_FMAC_F16_t16_e64:
5455 case AMDGPU::V_FMAC_F16_fake16_e64:
5456 case AMDGPU::V_FMAC_F32_e64:
5457 case AMDGPU::V_FMAC_F64_e64:
5458 case AMDGPU::V_FMAC_LEGACY_F32_e64:
5459 if (!Src2->isReg() || !RI.isVGPR(MRI, Reg: Src2->getReg()) ||
5460 hasModifiersSet(MI, OpName: AMDGPU::OpName::src2_modifiers))
5461 return false;
5462 break;
5463
5464 case AMDGPU::V_CNDMASK_B32_e64:
5465 break;
5466 }
5467 }
5468
5469 const MachineOperand *Src1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
5470 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Reg: Src1->getReg()) ||
5471 hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers)))
5472 return false;
5473
5474 // Make sure src0 isn't using any modifiers.
5475 if (hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers))
5476 return false;
5477
5478 // Can it be shrunk to a valid 32 bit opcode?
5479 if (!hasVALU32BitEncoding(Opcode: MI.getOpcode()))
5480 return false;
5481
5482 const MachineOperand *Src0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
5483 if (Src0 && Src0->isImm()) {
5484 unsigned Op32 = AMDGPU::getVOPe32(Opcode: MI.getOpcode());
5485 if (!isImmOperandLegal(
5486 InstDesc: get(Opcode: Op32), OpNo: AMDGPU::getNamedOperandIdx(Opcode: Op32, Name: AMDGPU::OpName::src0),
5487 MO: *Src0))
5488 return false;
5489 }
5490
5491 // Check output modifiers
5492 return !hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) &&
5493 !hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) &&
5494 !hasModifiersSet(MI, OpName: AMDGPU::OpName::byte_sel) &&
5495 // TODO: Can we avoid checking bound_ctrl/fi here?
5496 // They are only used by permlane*_swap special case.
5497 !hasModifiersSet(MI, OpName: AMDGPU::OpName::bound_ctrl) &&
5498 !hasModifiersSet(MI, OpName: AMDGPU::OpName::fi);
5499}
5500
5501// Set VCC operand with all flags from \p Orig, except for setting it as
5502// implicit.
5503static void copyFlagsToImplicitVCC(MachineInstr &MI,
5504 const MachineOperand &Orig) {
5505
5506 for (MachineOperand &Use : MI.implicit_operands()) {
5507 if (Use.isUse() &&
5508 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) {
5509 Use.setIsUndef(Orig.isUndef());
5510 Use.setIsKill(Orig.isKill());
5511 return;
5512 }
5513 }
5514}
5515
5516MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI,
5517 unsigned Op32) const {
5518 MachineBasicBlock *MBB = MI.getParent();
5519
5520 const MCInstrDesc &Op32Desc = get(Opcode: Op32);
5521 MachineInstrBuilder Inst32 =
5522 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: Op32Desc)
5523 .setMIFlags(MI.getFlags());
5524
5525 // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
5526 // For VOPC instructions, this is replaced by an implicit def of vcc.
5527
5528 // We assume the defs of the shrunk opcode are in the same order, and the
5529 // shrunk opcode loses the last def (SGPR def, in the VOP3->VOPC case).
5530 for (int I = 0, E = Op32Desc.getNumDefs(); I != E; ++I)
5531 Inst32.add(MO: MI.getOperand(i: I));
5532
5533 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
5534
5535 int Idx = MI.getNumExplicitDefs();
5536 for (const MachineOperand &Use : MI.explicit_uses()) {
5537 int OpTy = MI.getDesc().operands()[Idx++].OperandType;
5538 if (OpTy == AMDGPU::OPERAND_INPUT_MODS || OpTy == MCOI::OPERAND_IMMEDIATE)
5539 continue;
5540
5541 if (&Use == Src2) {
5542 if (AMDGPU::getNamedOperandIdx(Opcode: Op32, Name: AMDGPU::OpName::src2) == -1) {
5543 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
5544 // replaced with an implicit read of vcc or vcc_lo. The implicit read
5545 // of vcc was already added during the initial BuildMI, but we
5546 // 1) may need to change vcc to vcc_lo to preserve the original register
5547 // 2) have to preserve the original flags.
5548 copyFlagsToImplicitVCC(MI&: *Inst32, Orig: *Src2);
5549 continue;
5550 }
5551 }
5552
5553 Inst32.add(MO: Use);
5554 }
5555
5556 // FIXME: Losing implicit operands
5557 fixImplicitOperands(MI&: *Inst32);
5558
5559 // The explicit carry/result def is dropped in favor of an implicit VCC def;
5560 // preserve the dead flag.
5561 const MachineOperand *OldSDst = getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
5562 if (OldSDst && OldSDst->isDead()) {
5563 if (MachineOperand *NewVCC =
5564 Inst32->findRegisterDefOperand(Reg: RI.getVCC(), TRI: &RI))
5565 NewVCC->setIsDead();
5566 }
5567
5568 return Inst32;
5569}
5570
5571bool SIInstrInfo::physRegUsesConstantBus(const MachineOperand &RegOp) const {
5572 // Null is free
5573 Register Reg = RegOp.getReg();
5574 if (Reg == AMDGPU::SGPR_NULL || Reg == AMDGPU::SGPR_NULL64)
5575 return false;
5576
5577 // SGPRs use the constant bus
5578
5579 // FIXME: implicit registers that are not part of the MCInstrDesc's implicit
5580 // physical register operands should also count, except for exec.
5581 if (RegOp.isImplicit())
5582 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::M0;
5583
5584 // SGPRs use the constant bus
5585 return AMDGPU::SReg_32RegClass.contains(Reg) ||
5586 AMDGPU::SReg_64RegClass.contains(Reg);
5587}
5588
5589bool SIInstrInfo::regUsesConstantBus(const MachineOperand &RegOp,
5590 const MachineRegisterInfo &MRI) const {
5591 Register Reg = RegOp.getReg();
5592 return Reg.isVirtual() ? RI.isSGPRClass(RC: MRI.getRegClass(Reg))
5593 : physRegUsesConstantBus(RegOp);
5594}
5595
5596bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,
5597 const MachineOperand &MO,
5598 const MCOperandInfo &OpInfo) const {
5599 // Literal constants use the constant bus.
5600 if (!MO.isReg())
5601 return !isInlineConstant(MO, OpInfo);
5602
5603 Register Reg = MO.getReg();
5604 return Reg.isVirtual() ? RI.isSGPRClass(RC: MRI.getRegClass(Reg))
5605 : physRegUsesConstantBus(RegOp: MO);
5606}
5607
5608static Register findImplicitSGPRRead(const MachineInstr &MI) {
5609 for (const MachineOperand &MO : MI.implicit_operands()) {
5610 // We only care about reads.
5611 if (MO.isDef())
5612 continue;
5613
5614 switch (MO.getReg()) {
5615 case AMDGPU::VCC:
5616 case AMDGPU::VCC_LO:
5617 case AMDGPU::VCC_HI:
5618 case AMDGPU::M0:
5619 case AMDGPU::FLAT_SCR:
5620 return MO.getReg();
5621
5622 default:
5623 break;
5624 }
5625 }
5626
5627 return Register();
5628}
5629
5630static bool shouldReadExec(const MachineInstr &MI) {
5631 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false)) {
5632 switch (MI.getOpcode()) {
5633 case AMDGPU::V_READLANE_B32:
5634 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
5635 case AMDGPU::V_WRITELANE_B32:
5636 case AMDGPU::SI_SPILL_S32_TO_VGPR:
5637 return false;
5638 }
5639
5640 return true;
5641 }
5642
5643 if (MI.isPreISelOpcode() ||
5644 SIInstrInfo::isGenericOpcode(Opc: MI.getOpcode()) ||
5645 SIInstrInfo::isSALU(MI) ||
5646 SIInstrInfo::isSMRD(MI))
5647 return false;
5648
5649 return true;
5650}
5651
5652static bool isRegOrFI(const MachineOperand &MO) {
5653 return MO.isReg() || MO.isFI();
5654}
5655
5656static bool isSubRegOf(const SIRegisterInfo &TRI,
5657 const MachineOperand &SuperVec,
5658 const MachineOperand &SubReg) {
5659 if (SubReg.getReg().isPhysical())
5660 return TRI.isSubRegister(RegA: SuperVec.getReg(), RegB: SubReg.getReg());
5661
5662 return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
5663 SubReg.getReg() == SuperVec.getReg();
5664}
5665
5666// Verify the illegal copy from vector register to SGPR for generic opcode COPY
5667bool SIInstrInfo::verifyCopy(const MachineInstr &MI,
5668 const MachineRegisterInfo &MRI,
5669 StringRef &ErrInfo) const {
5670 Register DstReg = MI.getOperand(i: 0).getReg();
5671 Register SrcReg = MI.getOperand(i: 1).getReg();
5672 // This is a check for copy from vector register to SGPR
5673 if (RI.isVectorRegister(MRI, Reg: SrcReg) && RI.isSGPRReg(MRI, Reg: DstReg)) {
5674 ErrInfo = "illegal copy from vector register to SGPR";
5675 return false;
5676 }
5677 return true;
5678}
5679
5680bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
5681 StringRef &ErrInfo) const {
5682 uint32_t Opcode = MI.getOpcode();
5683 const MachineFunction *MF = MI.getMF();
5684 const MachineRegisterInfo &MRI = MF->getRegInfo();
5685
5686 // FIXME: At this point the COPY verify is done only for non-ssa forms.
5687 // Find a better property to recognize the point where instruction selection
5688 // is just done.
5689 // We can only enforce this check after SIFixSGPRCopies pass so that the
5690 // illegal copies are legalized and thereafter we don't expect a pass
5691 // inserting similar copies.
5692 if (!MRI.isSSA() && MI.isCopy())
5693 return verifyCopy(MI, MRI, ErrInfo);
5694
5695 if (SIInstrInfo::isGenericOpcode(Opc: Opcode) && !MI.isInlineAsm())
5696 return true;
5697
5698 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0);
5699 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src1);
5700 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src2);
5701 int Src3Idx = -1;
5702 if (Src0Idx == -1) {
5703 // VOPD V_DUAL_* instructions use different operand names.
5704 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0X);
5705 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vsrc1X);
5706 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0Y);
5707 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vsrc1Y);
5708 }
5709
5710 // Make sure the number of operands is correct.
5711 const MCInstrDesc &Desc = get(Opcode);
5712 if (!Desc.isVariadic() &&
5713 Desc.getNumOperands() != MI.getNumExplicitOperands()) {
5714 ErrInfo = "Instruction has wrong number of operands.";
5715 return false;
5716 }
5717
5718 if (MI.isInlineAsm()) {
5719 // Verify register classes for inlineasm constraints.
5720 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
5721 I != E; ++I) {
5722 const TargetRegisterClass *RC = MI.getRegClassConstraint(OpIdx: I, TII: this, TRI: &RI);
5723 if (!RC)
5724 continue;
5725
5726 const MachineOperand &Op = MI.getOperand(i: I);
5727 if (!Op.isReg())
5728 continue;
5729
5730 Register Reg = Op.getReg();
5731 if (!Reg.isVirtual() && !RC->contains(Reg)) {
5732 ErrInfo = "inlineasm operand has incorrect register class.";
5733 return false;
5734 }
5735 }
5736
5737 return true;
5738 }
5739
5740 if (isImage(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) {
5741 ErrInfo = "missing memory operand from image instruction.";
5742 return false;
5743 }
5744
5745 // Make sure the register classes are correct.
5746 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
5747 const MachineOperand &MO = MI.getOperand(i);
5748 if (MO.isFPImm()) {
5749 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
5750 "all fp values to integers.";
5751 return false;
5752 }
5753
5754 const MCOperandInfo &OpInfo = Desc.operands()[i];
5755
5756 switch (OpInfo.OperandType) {
5757 case MCOI::OPERAND_REGISTER:
5758 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
5759 ErrInfo = "Illegal immediate value for operand.";
5760 return false;
5761 }
5762 break;
5763 case AMDGPU::OPERAND_REG_IMM_INT32:
5764 case AMDGPU::OPERAND_REG_IMM_INT16:
5765 case AMDGPU::OPERAND_REG_IMM_FP32:
5766 case AMDGPU::OPERAND_REG_IMM_BF16:
5767 case AMDGPU::OPERAND_REG_IMM_FP16:
5768 case AMDGPU::OPERAND_REG_IMM_V2FP16:
5769 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
5770 case AMDGPU::OPERAND_REG_IMM_V2INT16:
5771 case AMDGPU::OPERAND_REG_IMM_V2BF16:
5772 case AMDGPU::OPERAND_REG_IMM_V2FP64:
5773 case AMDGPU::OPERAND_REG_IMM_V2INT64:
5774 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
5775 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
5776 break;
5777 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
5778 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
5779 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
5780 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
5781 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
5782 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
5783 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
5784 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
5785 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
5786 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
5787 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
5788 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
5789 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: {
5790 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, OpIdx: i))) {
5791 ErrInfo = "Illegal immediate value for operand.";
5792 return false;
5793 }
5794 break;
5795 }
5796 case AMDGPU::OPERAND_REG_IMM_FP64:
5797 case AMDGPU::OPERAND_REG_IMM_INT64:
5798 case AMDGPU::OPERAND_REG_IMM_V2INT32:
5799 case AMDGPU::OPERAND_REG_IMM_V2FP32:
5800 if (ST.has64BitLiterals() && Desc.getSize() != 4 && MO.isImm() &&
5801 !isInlineConstant(MI, OpIdx: i) &&
5802 !AMDGPU::isValid32BitLiteral(Val: MO.getImm(),
5803 IsFP64: OpInfo.OperandType ==
5804 AMDGPU::OPERAND_REG_IMM_FP64)) {
5805 ErrInfo = "illegal 64-bit immediate value for operand.";
5806 return false;
5807 }
5808 break;
5809 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
5810 case AMDGPU::OPERAND_INPUT_MODS:
5811 if (!MI.getOperand(i).isImm() || !isInlineConstant(MI, OpIdx: i)) {
5812 ErrInfo = "Expected inline constant for operand.";
5813 return false;
5814 }
5815 break;
5816 case AMDGPU::OPERAND_SDWA_VOPC_DST:
5817 case AMDGPU::OPERAND_KIMM16:
5818 break;
5819 case MCOI::OPERAND_IMMEDIATE:
5820 case AMDGPU::OPERAND_KIMM32:
5821 case AMDGPU::OPERAND_KIMM64:
5822 case AMDGPU::OPERAND_INLINE_C_AV64_PSEUDO:
5823 // Check if this operand is an immediate.
5824 // FrameIndex operands will be replaced by immediates, so they are
5825 // allowed.
5826 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
5827 ErrInfo = "Expected immediate, but got non-immediate";
5828 return false;
5829 }
5830 break;
5831 case MCOI::OPERAND_UNKNOWN:
5832 case MCOI::OPERAND_MEMORY:
5833 case MCOI::OPERAND_PCREL:
5834 break;
5835 default:
5836 if (OpInfo.isGenericType())
5837 continue;
5838 break;
5839 }
5840 }
5841
5842 // Verify SDWA
5843 if (isSDWA(MI)) {
5844 if (!ST.hasSDWA()) {
5845 ErrInfo = "SDWA is not supported on this target";
5846 return false;
5847 }
5848
5849 for (auto Op : {AMDGPU::OpName::src0_sel, AMDGPU::OpName::src1_sel,
5850 AMDGPU::OpName::dst_sel}) {
5851 const MachineOperand *MO = getNamedOperand(MI, OperandName: Op);
5852 if (!MO)
5853 continue;
5854 int64_t Imm = MO->getImm();
5855 if (Imm < 0 || Imm > AMDGPU::SDWA::SdwaSel::DWORD) {
5856 ErrInfo = "Invalid SDWA selection";
5857 return false;
5858 }
5859 }
5860
5861 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdst);
5862
5863 for (int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) {
5864 if (OpIdx == -1)
5865 continue;
5866 const MachineOperand &MO = MI.getOperand(i: OpIdx);
5867
5868 if (!ST.hasSDWAScalar()) {
5869 // Only VGPRS on VI
5870 if (!MO.isReg() || !RI.hasVGPRs(RC: RI.getRegClassForReg(MRI, Reg: MO.getReg()))) {
5871 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
5872 return false;
5873 }
5874 } else {
5875 // No immediates on GFX9
5876 if (!MO.isReg()) {
5877 ErrInfo =
5878 "Only reg allowed as operands in SDWA instructions on GFX9+";
5879 return false;
5880 }
5881 }
5882 }
5883
5884 if (!ST.hasSDWAOmod()) {
5885 // No omod allowed on VI
5886 const MachineOperand *OMod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
5887 if (OMod != nullptr &&
5888 (!OMod->isImm() || OMod->getImm() != 0)) {
5889 ErrInfo = "OMod not allowed in SDWA instructions on VI";
5890 return false;
5891 }
5892 }
5893
5894 if (Opcode == AMDGPU::V_CVT_F32_FP8_sdwa ||
5895 Opcode == AMDGPU::V_CVT_F32_BF8_sdwa ||
5896 Opcode == AMDGPU::V_CVT_PK_F32_FP8_sdwa ||
5897 Opcode == AMDGPU::V_CVT_PK_F32_BF8_sdwa) {
5898 const MachineOperand *Src0ModsMO =
5899 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
5900 unsigned Mods = Src0ModsMO->getImm();
5901 if (Mods & SISrcMods::ABS || Mods & SISrcMods::NEG ||
5902 Mods & SISrcMods::SEXT) {
5903 ErrInfo = "sext, abs and neg are not allowed on this instruction";
5904 return false;
5905 }
5906 }
5907
5908 uint32_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
5909 if (isVOPC(Opcode: BasicOpcode)) {
5910 if (!ST.hasSDWASdst() && DstIdx != -1) {
5911 // Only vcc allowed as dst on VI for VOPC
5912 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5913 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
5914 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
5915 return false;
5916 }
5917 } else if (!ST.hasSDWAOutModsVOPC()) {
5918 // No clamp allowed on GFX9 for VOPC
5919 const MachineOperand *Clamp = getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
5920 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
5921 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
5922 return false;
5923 }
5924
5925 // No omod allowed on GFX9 for VOPC
5926 const MachineOperand *OMod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
5927 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
5928 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
5929 return false;
5930 }
5931 }
5932 }
5933
5934 const MachineOperand *DstUnused = getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_unused);
5935 if (DstUnused && DstUnused->isImm() &&
5936 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
5937 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5938 if (!Dst.isReg() || !Dst.isTied()) {
5939 ErrInfo = "Dst register should have tied register";
5940 return false;
5941 }
5942
5943 const MachineOperand &TiedMO =
5944 MI.getOperand(i: MI.findTiedOperandIdx(OpIdx: DstIdx));
5945 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
5946 ErrInfo =
5947 "Dst register should be tied to implicit use of preserved register";
5948 return false;
5949 }
5950 if (TiedMO.getReg().isPhysical() && Dst.getReg() != TiedMO.getReg()) {
5951 ErrInfo = "Dst register should use same physical register as preserved";
5952 return false;
5953 }
5954 }
5955 }
5956
5957 if (isDPP(MI) && !ST.hasDPPSrc1SGPR() && Src1Idx != -1) {
5958 const MachineOperand &Src1MO = MI.getOperand(i: Src1Idx);
5959 if (Src1MO.isReg() && RI.isSGPRReg(MRI, Reg: Src1MO.getReg())) {
5960 ErrInfo = "DPP src1 cannot be SGPR on this subtarget";
5961 return false;
5962 }
5963 if (Src1MO.isImm()) {
5964 ErrInfo = "DPP src1 cannot be an immediate on this subtarget";
5965 return false;
5966 }
5967 }
5968
5969 // Verify MIMG / VIMAGE / VSAMPLE
5970 if (isImage(Opcode) && !MI.mayStore()) {
5971 // Ensure that the return type used is large enough for all the options
5972 // being used TFE/LWE require an extra result register.
5973 const MachineOperand *DMask = getNamedOperand(MI, OperandName: AMDGPU::OpName::dmask);
5974 if (DMask) {
5975 uint64_t DMaskImm = DMask->getImm();
5976 uint32_t RegCount = isGather4(Opcode) ? 4 : llvm::popcount(Value: DMaskImm);
5977 const MachineOperand *TFE = getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe);
5978 const MachineOperand *LWE = getNamedOperand(MI, OperandName: AMDGPU::OpName::lwe);
5979 const MachineOperand *D16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::d16);
5980
5981 // Adjust for packed 16 bit values
5982 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
5983 RegCount = divideCeil(Numerator: RegCount, Denominator: 2);
5984
5985 // Adjust if using LWE or TFE
5986 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
5987 RegCount += 1;
5988
5989 const uint32_t DstIdx =
5990 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdata);
5991 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5992 if (Dst.isReg()) {
5993 const TargetRegisterClass *DstRC = getOpRegClass(MI, OpNo: DstIdx);
5994 uint32_t DstSize = RI.getRegSizeInBits(RC: *DstRC) / 32;
5995 if (RegCount > DstSize) {
5996 ErrInfo = "Image instruction returns too many registers for dst "
5997 "register class";
5998 return false;
5999 }
6000 }
6001 }
6002 }
6003
6004 // Verify VOP*. Ignore multiple sgpr operands on writelane.
6005 if (isVALU(MI, /*AllowLDSDMA=*/false) &&
6006 Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
6007 unsigned ConstantBusCount = 0;
6008 bool UsesLiteral = false;
6009 const MachineOperand *LiteralVal = nullptr;
6010
6011 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::imm);
6012 if (ImmIdx != -1) {
6013 ++ConstantBusCount;
6014 UsesLiteral = true;
6015 LiteralVal = &MI.getOperand(i: ImmIdx);
6016 }
6017
6018 SmallVector<Register, 2> SGPRsUsed;
6019 Register SGPRUsed;
6020
6021 // Only look at the true operands. Only a real operand can use the constant
6022 // bus, and we don't want to check pseudo-operands like the source modifier
6023 // flags.
6024 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) {
6025 if (OpIdx == -1)
6026 continue;
6027 const MachineOperand &MO = MI.getOperand(i: OpIdx);
6028 if (usesConstantBus(MRI, MO, OpInfo: MI.getDesc().operands()[OpIdx])) {
6029 if (MO.isReg()) {
6030 SGPRUsed = MO.getReg();
6031 if (!llvm::is_contained(Range&: SGPRsUsed, Element: SGPRUsed)) {
6032 ++ConstantBusCount;
6033 SGPRsUsed.push_back(Elt: SGPRUsed);
6034 }
6035 } else if (!MO.isFI()) { // Treat FI like a register.
6036 if (!UsesLiteral) {
6037 ++ConstantBusCount;
6038 UsesLiteral = true;
6039 LiteralVal = &MO;
6040 } else if (!MO.isIdenticalTo(Other: *LiteralVal)) {
6041 assert(isVOP2(MI) || isVOP3(MI));
6042 ErrInfo = "VOP2/VOP3 instruction uses more than one literal";
6043 return false;
6044 }
6045 }
6046 }
6047 }
6048
6049 SGPRUsed = findImplicitSGPRRead(MI);
6050 if (SGPRUsed) {
6051 // Implicit uses may safely overlap true operands
6052 if (llvm::all_of(Range&: SGPRsUsed, P: [this, SGPRUsed](unsigned SGPR) {
6053 return !RI.regsOverlap(RegA: SGPRUsed, RegB: SGPR);
6054 })) {
6055 ++ConstantBusCount;
6056 SGPRsUsed.push_back(Elt: SGPRUsed);
6057 }
6058 }
6059
6060 // v_writelane_b32 is an exception from constant bus restriction:
6061 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
6062 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
6063 Opcode != AMDGPU::V_WRITELANE_B32) {
6064 ErrInfo = "VOP* instruction violates constant bus restriction";
6065 return false;
6066 }
6067
6068 if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) {
6069 ErrInfo = "VOP3 instruction uses literal";
6070 return false;
6071 }
6072 }
6073
6074 // Special case for writelane - this can break the multiple constant bus rule,
6075 // but still can't use more than one SGPR register
6076 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
6077 unsigned SGPRCount = 0;
6078 Register SGPRUsed;
6079
6080 for (int OpIdx : {Src0Idx, Src1Idx}) {
6081 if (OpIdx == -1)
6082 break;
6083
6084 const MachineOperand &MO = MI.getOperand(i: OpIdx);
6085
6086 if (usesConstantBus(MRI, MO, OpInfo: MI.getDesc().operands()[OpIdx])) {
6087 if (MO.isReg() && MO.getReg() != AMDGPU::M0) {
6088 if (MO.getReg() != SGPRUsed)
6089 ++SGPRCount;
6090 SGPRUsed = MO.getReg();
6091 }
6092 }
6093 if (SGPRCount > ST.getConstantBusLimit(Opcode)) {
6094 ErrInfo = "WRITELANE instruction violates constant bus restriction";
6095 return false;
6096 }
6097 }
6098 }
6099
6100 if (Opcode == AMDGPU::ASYNCMARK || Opcode == AMDGPU::WAIT_ASYNCMARK) {
6101 int64_t Mask = MI.getOperand(i: MI.getNumExplicitOperands() - 1).getImm();
6102 if (!AMDGPU::isValidAsyncStageMask(Mask)) {
6103 ErrInfo = "invalid asyncmark stage mask";
6104 return false;
6105 }
6106 }
6107
6108 // An offset past the addressable VGPRs is an out-of-bounds access, which is
6109 // allowed, but each dword must still be a VGPR.
6110 if (const auto *LdSt = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(Val: &MI)) {
6111 const MachineOperand &Offset = LdSt->getOffsetOp();
6112 if (!Offset.isImm() || Offset.getImm() < 0 ||
6113 Offset.getImm() + LdSt->getBitWidth() / 32 >
6114 AMDGPU::VGPR_32RegClass.getNumRegs()) {
6115 ErrInfo = "invalid VGPR address space offset";
6116 return false;
6117 }
6118 }
6119
6120 // Verify misc. restrictions on specific instructions.
6121 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
6122 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
6123 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
6124 const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
6125 const MachineOperand &Src2 = MI.getOperand(i: Src2Idx);
6126 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
6127 if (!compareMachineOp(Op0: Src0, Op1: Src1) &&
6128 !compareMachineOp(Op0: Src0, Op1: Src2)) {
6129 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
6130 return false;
6131 }
6132 }
6133 if ((getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers)->getImm() &
6134 SISrcMods::ABS) ||
6135 (getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers)->getImm() &
6136 SISrcMods::ABS) ||
6137 (getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers)->getImm() &
6138 SISrcMods::ABS)) {
6139 ErrInfo = "ABS not allowed in VOP3B instructions";
6140 return false;
6141 }
6142 }
6143
6144 if (isSOP2(MI) || isSOPC(MI)) {
6145 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
6146 const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
6147
6148 if (!isRegOrFI(MO: Src0) && !isRegOrFI(MO: Src1) &&
6149 !isInlineConstant(MO: Src0, OpInfo: Desc.operands()[Src0Idx]) &&
6150 !isInlineConstant(MO: Src1, OpInfo: Desc.operands()[Src1Idx]) &&
6151 !Src0.isIdenticalTo(Other: Src1)) {
6152 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
6153 return false;
6154 }
6155 }
6156
6157 if (isSOPK(MI)) {
6158 const auto *Op = getNamedOperand(MI, OperandName: AMDGPU::OpName::simm16);
6159 if (Desc.isBranch()) {
6160 if (!Op->isMBB()) {
6161 ErrInfo = "invalid branch target for SOPK instruction";
6162 return false;
6163 }
6164 } else {
6165 uint64_t Imm = Op->getImm();
6166 if (sopkIsZext(Opcode)) {
6167 if (!isUInt<16>(x: Imm)) {
6168 ErrInfo = "invalid immediate for SOPK instruction";
6169 return false;
6170 }
6171 } else {
6172 if (!isInt<16>(x: Imm)) {
6173 ErrInfo = "invalid immediate for SOPK instruction";
6174 return false;
6175 }
6176 }
6177 }
6178 }
6179
6180 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
6181 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
6182 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
6183 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
6184 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
6185 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
6186
6187 const unsigned StaticNumOps =
6188 Desc.getNumOperands() + Desc.implicit_uses().size();
6189 const unsigned NumImplicitOps = IsDst ? 2 : 1;
6190
6191 // Require additional implicit operands. This allows a fixup done by the
6192 // post RA scheduler where the main implicit operand is killed and
6193 // implicit-defs are added for sub-registers that remain live after this
6194 // instruction.
6195 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
6196 ErrInfo = "missing implicit register operands";
6197 return false;
6198 }
6199
6200 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
6201 if (IsDst) {
6202 if (!Dst->isUse()) {
6203 ErrInfo = "v_movreld_b32 vdst should be a use operand";
6204 return false;
6205 }
6206
6207 unsigned UseOpIdx;
6208 if (!MI.isRegTiedToUseOperand(DefOpIdx: StaticNumOps, UseOpIdx: &UseOpIdx) ||
6209 UseOpIdx != StaticNumOps + 1) {
6210 ErrInfo = "movrel implicit operands should be tied";
6211 return false;
6212 }
6213 }
6214
6215 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
6216 const MachineOperand &ImpUse
6217 = MI.getOperand(i: StaticNumOps + NumImplicitOps - 1);
6218 if (!ImpUse.isReg() || !ImpUse.isUse() ||
6219 !isSubRegOf(TRI: RI, SuperVec: ImpUse, SubReg: IsDst ? *Dst : Src0)) {
6220 ErrInfo = "src0 should be subreg of implicit vector use";
6221 return false;
6222 }
6223 }
6224
6225 // Make sure we aren't losing exec uses in the td files. This mostly requires
6226 // being careful when using let Uses to try to add other use registers.
6227 if (shouldReadExec(MI)) {
6228 if (!MI.hasRegisterImplicitUseOperand(Reg: AMDGPU::EXEC)) {
6229 ErrInfo = "VALU instruction does not implicitly read exec mask";
6230 return false;
6231 }
6232 }
6233
6234 if (isSMRD(MI)) {
6235 if (MI.mayStore() &&
6236 ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) {
6237 // The register offset form of scalar stores may only use m0 as the
6238 // soffset register.
6239 const MachineOperand *Soff = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
6240 if (Soff && Soff->getReg() != AMDGPU::M0) {
6241 ErrInfo = "scalar stores must use m0 as offset register";
6242 return false;
6243 }
6244 }
6245 }
6246
6247 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) {
6248 const MachineOperand *Offset = getNamedOperand(MI, OperandName: AMDGPU::OpName::offset);
6249 if (Offset->getImm() != 0) {
6250 ErrInfo = "subtarget does not support offsets in flat instructions";
6251 return false;
6252 }
6253 }
6254
6255 if (isDS(MI) && !ST.hasGDS()) {
6256 const MachineOperand *GDSOp = getNamedOperand(MI, OperandName: AMDGPU::OpName::gds);
6257 if (GDSOp && GDSOp->getImm() != 0) {
6258 ErrInfo = "GDS is not supported on this subtarget";
6259 return false;
6260 }
6261 }
6262
6263 if (isImage(MI)) {
6264 const MachineOperand *DimOp = getNamedOperand(MI, OperandName: AMDGPU::OpName::dim);
6265 if (DimOp) {
6266 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
6267 Name: AMDGPU::OpName::vaddr0);
6268 AMDGPU::OpName RSrcOpName =
6269 isMIMG(MI) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
6270 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: RSrcOpName);
6271 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: Opcode);
6272 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
6273 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
6274 const AMDGPU::MIMGDimInfo *Dim =
6275 AMDGPU::getMIMGDimInfoByEncoding(DimEnc: DimOp->getImm());
6276
6277 if (!Dim) {
6278 ErrInfo = "dim is out of range";
6279 return false;
6280 }
6281
6282 bool IsA16 = false;
6283 if (ST.hasR128A16()) {
6284 const MachineOperand *R128A16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::r128);
6285 IsA16 = R128A16->getImm() != 0;
6286 } else if (ST.hasA16()) {
6287 const MachineOperand *A16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::a16);
6288 IsA16 = A16->getImm() != 0;
6289 }
6290
6291 bool IsNSA = RsrcIdx - VAddr0Idx > 1;
6292
6293 unsigned AddrWords =
6294 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, IsG16Supported: ST.hasG16());
6295
6296 unsigned VAddrWords;
6297 if (IsNSA) {
6298 VAddrWords = RsrcIdx - VAddr0Idx;
6299 if (ST.hasPartialNSAEncoding() &&
6300 AddrWords > ST.getNSAMaxSize(HasSampler: isVSAMPLE(MI))) {
6301 unsigned LastVAddrIdx = RsrcIdx - 1;
6302 VAddrWords += getOpSize(MI, OpNo: LastVAddrIdx) / 4 - 1;
6303 }
6304 } else {
6305 VAddrWords = getOpSize(MI, OpNo: VAddr0Idx) / 4;
6306 if (AddrWords > 12)
6307 AddrWords = 16;
6308 }
6309
6310 if (VAddrWords != AddrWords) {
6311 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords
6312 << " but got " << VAddrWords << "\n");
6313 ErrInfo = "bad vaddr size";
6314 return false;
6315 }
6316 }
6317 }
6318
6319 const MachineOperand *DppCt = getNamedOperand(MI, OperandName: AMDGPU::OpName::dpp_ctrl);
6320 if (DppCt) {
6321 using namespace AMDGPU::DPP;
6322
6323 unsigned DC = DppCt->getImm();
6324 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
6325 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
6326 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) ||
6327 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) ||
6328 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) ||
6329 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) ||
6330 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) {
6331 ErrInfo = "Invalid dpp_ctrl value";
6332 return false;
6333 }
6334 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
6335 !ST.hasDPPWavefrontShifts()) {
6336 ErrInfo = "Invalid dpp_ctrl value: "
6337 "wavefront shifts are not supported on GFX10+";
6338 return false;
6339 }
6340 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
6341 !ST.hasDPPBroadcasts()) {
6342 ErrInfo = "Invalid dpp_ctrl value: "
6343 "broadcasts are not supported on GFX10+";
6344 return false;
6345 }
6346 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST &&
6347 ST.getGeneration() < AMDGPUSubtarget::GFX10) {
6348 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST &&
6349 DC <= DppCtrl::ROW_NEWBCAST_LAST &&
6350 !ST.hasGFX90AInsts()) {
6351 ErrInfo = "Invalid dpp_ctrl value: "
6352 "row_newbroadcast/row_share is not supported before "
6353 "GFX90A/GFX10";
6354 return false;
6355 }
6356 if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) {
6357 ErrInfo = "Invalid dpp_ctrl value: "
6358 "row_share and row_xmask are not supported before GFX10";
6359 return false;
6360 }
6361 }
6362
6363 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&
6364 !AMDGPU::isLegalDPALU_DPPControl(ST, DC) &&
6365 ST.hasFeature(Feature: AMDGPU::FeatureDPALU_DPP) &&
6366 AMDGPU::isDPALU_DPP(OpDesc: Desc, MII: *this, ST)) {
6367 ErrInfo = "Invalid dpp_ctrl value: "
6368 "DP ALU dpp only support row_newbcast";
6369 return false;
6370 }
6371 }
6372
6373 if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) {
6374 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
6375 AMDGPU::OpName DataName =
6376 isDS(Opcode) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata;
6377 const MachineOperand *Data = getNamedOperand(MI, OperandName: DataName);
6378 const MachineOperand *Data2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::data1);
6379 if (Data && !Data->isReg())
6380 Data = nullptr;
6381
6382 if (!ST.hasGFX90AInsts()) {
6383 if ((Dst && RI.isAGPR(MRI, Reg: Dst->getReg())) ||
6384 (Data && RI.isAGPR(MRI, Reg: Data->getReg())) ||
6385 (Data2 && RI.isAGPR(MRI, Reg: Data2->getReg()))) {
6386 ErrInfo = "Invalid register class: "
6387 "agpr loads and stores not supported on this GPU";
6388 return false;
6389 }
6390 }
6391 }
6392
6393 if (ST.needsAlignedVGPRs()) {
6394 const auto isAlignedReg = [&MI, &MRI, this](AMDGPU::OpName OpName) -> bool {
6395 const MachineOperand *Op = getNamedOperand(MI, OperandName: OpName);
6396 if (!Op)
6397 return true;
6398 Register Reg = Op->getReg();
6399 if (Reg.isPhysical())
6400 return !(RI.getHWRegIndex(Reg) & 1);
6401 const TargetRegisterClass &RC = *MRI.getRegClass(Reg);
6402 return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
6403 !(RI.getChannelFromSubReg(SubReg: Op->getSubReg()) & 1);
6404 };
6405
6406 if (isMIMG(MI)) {
6407 if (!isAlignedReg(AMDGPU::OpName::vaddr)) {
6408 ErrInfo = "Subtarget requires even aligned vector registers "
6409 "for vaddr operand of image instructions";
6410 return false;
6411 }
6412 }
6413 }
6414
6415 if (Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts()) {
6416 const MachineOperand *Src = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
6417 if (Src->isReg() && RI.isSGPRReg(MRI, Reg: Src->getReg())) {
6418 ErrInfo = "Invalid register class: "
6419 "v_accvgpr_write with an SGPR is not supported on this GPU";
6420 return false;
6421 }
6422 }
6423
6424 if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) {
6425 const MachineOperand &SrcOp = MI.getOperand(i: 1);
6426 if (!SrcOp.isReg() || SrcOp.getReg().isVirtual()) {
6427 ErrInfo = "pseudo expects only physical SGPRs";
6428 return false;
6429 }
6430 }
6431
6432 if (const MachineOperand *CPol = getNamedOperand(MI, OperandName: AMDGPU::OpName::cpol)) {
6433 if (CPol->getImm() & AMDGPU::CPol::SCAL) {
6434 if (!ST.hasScaleOffset()) {
6435 ErrInfo = "subtarget does not support offset scaling";
6436 return false;
6437 }
6438 if (!AMDGPU::supportsScaleOffset(MII: *this, Opcode: MI.getOpcode())) {
6439 ErrInfo = "instruction does not support offset scaling";
6440 return false;
6441 }
6442 }
6443 }
6444
6445 // See SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
6446 if (AMDGPU::isSingleSGPRReadInst(Opc: Opcode)) {
6447 for (unsigned I = 0; I < 3; ++I) {
6448 if (!isLegalSingleSGPRReadInstOperand(MRI, MI, SrcN: I))
6449 return false;
6450 }
6451 }
6452
6453 if (ST.hasFlatScratchHiInB64InstHazard() && isSALU(MI) &&
6454 MI.readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, TRI: nullptr)) {
6455 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
6456 if ((Dst && RI.getRegClassForReg(MRI, Reg: Dst->getReg()) ==
6457 &AMDGPU::SReg_64RegClass) ||
6458 Opcode == AMDGPU::S_BITCMP0_B64 || Opcode == AMDGPU::S_BITCMP1_B64) {
6459 ErrInfo = "Instruction cannot read flat_scratch_base_hi";
6460 return false;
6461 }
6462 }
6463
6464 // GFX950 v_pk_{add,mul,fma}_f32 must not have an invalid op_sel.
6465 if (!isLegalPkF32OpSel(TII: *this, MI)) {
6466 ErrInfo = "GFX950 v_pk_{add,mul,fma}_f32 has an invalid op_sel.";
6467 return false;
6468 }
6469
6470 return true;
6471}
6472
6473unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
6474 if (MI.getOpcode() == AMDGPU::S_MOV_B32) {
6475 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6476 return MI.getOperand(i: 1).isReg() || RI.isAGPR(MRI, Reg: MI.getOperand(i: 0).getReg())
6477 ? AMDGPU::COPY
6478 : AMDGPU::V_MOV_B32_e32;
6479 }
6480 return getVALUOp(Opc: MI.getOpcode());
6481}
6482
6483// It is more readable to list mapped opcodes on the same line.
6484// clang-format off
6485
6486unsigned SIInstrInfo::getVALUOp(unsigned Opc) const {
6487 switch (Opc) {
6488 default: return AMDGPU::INSTRUCTION_LIST_END;
6489 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
6490 case AMDGPU::COPY: return AMDGPU::COPY;
6491 case AMDGPU::PHI: return AMDGPU::PHI;
6492 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
6493 case AMDGPU::WQM: return AMDGPU::WQM;
6494 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
6495 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM;
6496 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM;
6497 case AMDGPU::S_ADD_I32:
6498 return ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
6499 case AMDGPU::S_ADDC_U32:
6500 return AMDGPU::V_ADDC_U32_e32;
6501 case AMDGPU::S_SUB_I32:
6502 return ST.hasAddNoCarryInsts() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32;
6503 // FIXME: These are not consistently handled, and selected when the carry is
6504 // used.
6505 case AMDGPU::S_ADD_U32:
6506 return AMDGPU::V_ADD_CO_U32_e32;
6507 case AMDGPU::S_SUB_U32:
6508 return AMDGPU::V_SUB_CO_U32_e32;
6509 case AMDGPU::S_ADD_U64_PSEUDO:
6510 return AMDGPU::V_ADD_U64_PSEUDO;
6511 case AMDGPU::S_SUB_U64_PSEUDO:
6512 return AMDGPU::V_SUB_U64_PSEUDO;
6513 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
6514 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64;
6515 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64;
6516 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64;
6517 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
6518 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
6519 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
6520 case AMDGPU::S_XNOR_B32:
6521 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
6522 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
6523 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
6524 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
6525 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
6526 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
6527 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64;
6528 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
6529 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64;
6530 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
6531 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64;
6532 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64;
6533 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64;
6534 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64;
6535 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64;
6536 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
6537 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
6538 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
6539 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
6540 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64;
6541 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64;
6542 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64;
6543 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64;
6544 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64;
6545 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64;
6546 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64;
6547 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64;
6548 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64;
6549 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64;
6550 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64;
6551 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64;
6552 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64;
6553 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64;
6554 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
6555 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
6556 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
6557 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
6558 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
6559 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
6560 case AMDGPU::S_CVT_F32_I32: return AMDGPU::V_CVT_F32_I32_e64;
6561 case AMDGPU::S_CVT_F32_U32: return AMDGPU::V_CVT_F32_U32_e64;
6562 case AMDGPU::S_CVT_I32_F32: return AMDGPU::V_CVT_I32_F32_e64;
6563 case AMDGPU::S_CVT_U32_F32: return AMDGPU::V_CVT_U32_F32_e64;
6564 case AMDGPU::S_CVT_F32_F16:
6565 case AMDGPU::S_CVT_HI_F32_F16:
6566 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F32_F16_t16_e64
6567 : AMDGPU::V_CVT_F32_F16_fake16_e64;
6568 case AMDGPU::S_CVT_F16_F32:
6569 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F16_F32_t16_e64
6570 : AMDGPU::V_CVT_F16_F32_fake16_e64;
6571 case AMDGPU::S_CEIL_F32: return AMDGPU::V_CEIL_F32_e64;
6572 case AMDGPU::S_FLOOR_F32: return AMDGPU::V_FLOOR_F32_e64;
6573 case AMDGPU::S_TRUNC_F32: return AMDGPU::V_TRUNC_F32_e64;
6574 case AMDGPU::S_RNDNE_F32: return AMDGPU::V_RNDNE_F32_e64;
6575 case AMDGPU::S_CEIL_F16:
6576 return ST.useRealTrue16Insts() ? AMDGPU::V_CEIL_F16_t16_e64
6577 : AMDGPU::V_CEIL_F16_fake16_e64;
6578 case AMDGPU::S_FLOOR_F16:
6579 return ST.useRealTrue16Insts() ? AMDGPU::V_FLOOR_F16_t16_e64
6580 : AMDGPU::V_FLOOR_F16_fake16_e64;
6581 case AMDGPU::S_TRUNC_F16:
6582 return ST.useRealTrue16Insts() ? AMDGPU::V_TRUNC_F16_t16_e64
6583 : AMDGPU::V_TRUNC_F16_fake16_e64;
6584 case AMDGPU::S_RNDNE_F16:
6585 return ST.useRealTrue16Insts() ? AMDGPU::V_RNDNE_F16_t16_e64
6586 : AMDGPU::V_RNDNE_F16_fake16_e64;
6587 case AMDGPU::S_ADD_F32: return AMDGPU::V_ADD_F32_e64;
6588 case AMDGPU::S_SUB_F32: return AMDGPU::V_SUB_F32_e64;
6589 case AMDGPU::S_MIN_F32: return AMDGPU::V_MIN_F32_e64;
6590 case AMDGPU::S_MAX_F32: return AMDGPU::V_MAX_F32_e64;
6591 case AMDGPU::S_MINIMUM_F32: return AMDGPU::V_MINIMUM_F32_e64;
6592 case AMDGPU::S_MAXIMUM_F32: return AMDGPU::V_MAXIMUM_F32_e64;
6593 case AMDGPU::S_MUL_F32: return AMDGPU::V_MUL_F32_e64;
6594 case AMDGPU::S_ADD_F16:
6595 return ST.useRealTrue16Insts() ? AMDGPU::V_ADD_F16_t16_e64
6596 : AMDGPU::V_ADD_F16_fake16_e64;
6597 case AMDGPU::S_SUB_F16:
6598 return ST.useRealTrue16Insts() ? AMDGPU::V_SUB_F16_t16_e64
6599 : AMDGPU::V_SUB_F16_fake16_e64;
6600 case AMDGPU::S_MIN_F16:
6601 return ST.useRealTrue16Insts() ? AMDGPU::V_MIN_F16_t16_e64
6602 : AMDGPU::V_MIN_F16_fake16_e64;
6603 case AMDGPU::S_MAX_F16:
6604 return ST.useRealTrue16Insts() ? AMDGPU::V_MAX_F16_t16_e64
6605 : AMDGPU::V_MAX_F16_fake16_e64;
6606 case AMDGPU::S_MINIMUM_F16:
6607 return ST.useRealTrue16Insts() ? AMDGPU::V_MINIMUM_F16_t16_e64
6608 : AMDGPU::V_MINIMUM_F16_fake16_e64;
6609 case AMDGPU::S_MAXIMUM_F16:
6610 return ST.useRealTrue16Insts() ? AMDGPU::V_MAXIMUM_F16_t16_e64
6611 : AMDGPU::V_MAXIMUM_F16_fake16_e64;
6612 case AMDGPU::S_MUL_F16:
6613 return ST.useRealTrue16Insts() ? AMDGPU::V_MUL_F16_t16_e64
6614 : AMDGPU::V_MUL_F16_fake16_e64;
6615 case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64;
6616 case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64;
6617 case AMDGPU::S_FMAC_F16:
6618 return ST.useRealTrue16Insts() ? AMDGPU::V_FMAC_F16_t16_e64
6619 : AMDGPU::V_FMAC_F16_fake16_e64;
6620 case AMDGPU::S_FMAMK_F32: return AMDGPU::V_FMAMK_F32;
6621 case AMDGPU::S_FMAAK_F32: return AMDGPU::V_FMAAK_F32;
6622 case AMDGPU::S_CMP_LT_F32: return AMDGPU::V_CMP_LT_F32_e64;
6623 case AMDGPU::S_CMP_EQ_F32: return AMDGPU::V_CMP_EQ_F32_e64;
6624 case AMDGPU::S_CMP_LE_F32: return AMDGPU::V_CMP_LE_F32_e64;
6625 case AMDGPU::S_CMP_GT_F32: return AMDGPU::V_CMP_GT_F32_e64;
6626 case AMDGPU::S_CMP_LG_F32: return AMDGPU::V_CMP_LG_F32_e64;
6627 case AMDGPU::S_CMP_GE_F32: return AMDGPU::V_CMP_GE_F32_e64;
6628 case AMDGPU::S_CMP_O_F32: return AMDGPU::V_CMP_O_F32_e64;
6629 case AMDGPU::S_CMP_U_F32: return AMDGPU::V_CMP_U_F32_e64;
6630 case AMDGPU::S_CMP_NGE_F32: return AMDGPU::V_CMP_NGE_F32_e64;
6631 case AMDGPU::S_CMP_NLG_F32: return AMDGPU::V_CMP_NLG_F32_e64;
6632 case AMDGPU::S_CMP_NGT_F32: return AMDGPU::V_CMP_NGT_F32_e64;
6633 case AMDGPU::S_CMP_NLE_F32: return AMDGPU::V_CMP_NLE_F32_e64;
6634 case AMDGPU::S_CMP_NEQ_F32: return AMDGPU::V_CMP_NEQ_F32_e64;
6635 case AMDGPU::S_CMP_NLT_F32: return AMDGPU::V_CMP_NLT_F32_e64;
6636 case AMDGPU::S_CMP_LT_F16:
6637 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LT_F16_t16_e64
6638 : AMDGPU::V_CMP_LT_F16_fake16_e64;
6639 case AMDGPU::S_CMP_EQ_F16:
6640 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_EQ_F16_t16_e64
6641 : AMDGPU::V_CMP_EQ_F16_fake16_e64;
6642 case AMDGPU::S_CMP_LE_F16:
6643 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LE_F16_t16_e64
6644 : AMDGPU::V_CMP_LE_F16_fake16_e64;
6645 case AMDGPU::S_CMP_GT_F16:
6646 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GT_F16_t16_e64
6647 : AMDGPU::V_CMP_GT_F16_fake16_e64;
6648 case AMDGPU::S_CMP_LG_F16:
6649 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LG_F16_t16_e64
6650 : AMDGPU::V_CMP_LG_F16_fake16_e64;
6651 case AMDGPU::S_CMP_GE_F16:
6652 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GE_F16_t16_e64
6653 : AMDGPU::V_CMP_GE_F16_fake16_e64;
6654 case AMDGPU::S_CMP_O_F16:
6655 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_O_F16_t16_e64
6656 : AMDGPU::V_CMP_O_F16_fake16_e64;
6657 case AMDGPU::S_CMP_U_F16:
6658 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_U_F16_t16_e64
6659 : AMDGPU::V_CMP_U_F16_fake16_e64;
6660 case AMDGPU::S_CMP_NGE_F16:
6661 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGE_F16_t16_e64
6662 : AMDGPU::V_CMP_NGE_F16_fake16_e64;
6663 case AMDGPU::S_CMP_NLG_F16:
6664 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLG_F16_t16_e64
6665 : AMDGPU::V_CMP_NLG_F16_fake16_e64;
6666 case AMDGPU::S_CMP_NGT_F16:
6667 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGT_F16_t16_e64
6668 : AMDGPU::V_CMP_NGT_F16_fake16_e64;
6669 case AMDGPU::S_CMP_NLE_F16:
6670 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLE_F16_t16_e64
6671 : AMDGPU::V_CMP_NLE_F16_fake16_e64;
6672 case AMDGPU::S_CMP_NEQ_F16:
6673 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NEQ_F16_t16_e64
6674 : AMDGPU::V_CMP_NEQ_F16_fake16_e64;
6675 case AMDGPU::S_CMP_NLT_F16:
6676 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLT_F16_t16_e64
6677 : AMDGPU::V_CMP_NLT_F16_fake16_e64;
6678 case AMDGPU::V_S_EXP_F32_e64: return AMDGPU::V_EXP_F32_e64;
6679 case AMDGPU::V_S_EXP_F16_e64:
6680 return ST.useRealTrue16Insts() ? AMDGPU::V_EXP_F16_t16_e64
6681 : AMDGPU::V_EXP_F16_fake16_e64;
6682 case AMDGPU::V_S_LOG_F32_e64: return AMDGPU::V_LOG_F32_e64;
6683 case AMDGPU::V_S_LOG_F16_e64:
6684 return ST.useRealTrue16Insts() ? AMDGPU::V_LOG_F16_t16_e64
6685 : AMDGPU::V_LOG_F16_fake16_e64;
6686 case AMDGPU::V_S_RCP_F32_e64: return AMDGPU::V_RCP_F32_e64;
6687 case AMDGPU::V_S_RCP_F16_e64:
6688 return ST.useRealTrue16Insts() ? AMDGPU::V_RCP_F16_t16_e64
6689 : AMDGPU::V_RCP_F16_fake16_e64;
6690 case AMDGPU::V_S_RSQ_F32_e64: return AMDGPU::V_RSQ_F32_e64;
6691 case AMDGPU::V_S_RSQ_F16_e64:
6692 return ST.useRealTrue16Insts() ? AMDGPU::V_RSQ_F16_t16_e64
6693 : AMDGPU::V_RSQ_F16_fake16_e64;
6694 case AMDGPU::V_S_SQRT_F32_e64: return AMDGPU::V_SQRT_F32_e64;
6695 case AMDGPU::V_S_SQRT_F16_e64:
6696 return ST.useRealTrue16Insts() ? AMDGPU::V_SQRT_F16_t16_e64
6697 : AMDGPU::V_SQRT_F16_fake16_e64;
6698 }
6699 llvm_unreachable(
6700 "Unexpected scalar opcode without corresponding vector one!");
6701}
6702
6703// clang-format on
6704
6705void SIInstrInfo::insertScratchExecCopy(MachineFunction &MF,
6706 MachineBasicBlock &MBB,
6707 MachineBasicBlock::iterator MBBI,
6708 const DebugLoc &DL, Register Reg,
6709 bool IsSCCLive,
6710 SlotIndexes *Indexes) const {
6711 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
6712 const SIInstrInfo *TII = ST.getInstrInfo();
6713 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
6714 if (IsSCCLive) {
6715 // Insert two move instructions, one to save the original value of EXEC and
6716 // the other to turn on all bits in EXEC. This is required as we can't use
6717 // the single instruction S_OR_SAVEEXEC that clobbers SCC.
6718 auto StoreExecMI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: Reg)
6719 .addReg(RegNo: LMC.ExecReg, Flags: RegState::Kill);
6720 auto FlipExecMI =
6721 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg).addImm(Val: -1);
6722 if (Indexes) {
6723 Indexes->insertMachineInstrInMaps(MI&: *StoreExecMI);
6724 Indexes->insertMachineInstrInMaps(MI&: *FlipExecMI);
6725 }
6726 } else {
6727 auto SaveExec =
6728 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.OrSaveExecOpc), DestReg: Reg).addImm(Val: -1);
6729 SaveExec->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
6730 if (Indexes)
6731 Indexes->insertMachineInstrInMaps(MI&: *SaveExec);
6732 }
6733}
6734
6735void SIInstrInfo::restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
6736 MachineBasicBlock::iterator MBBI,
6737 const DebugLoc &DL, Register Reg,
6738 SlotIndexes *Indexes) const {
6739 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
6740 auto ExecRestoreMI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
6741 .addReg(RegNo: Reg, Flags: RegState::Kill);
6742 if (Indexes)
6743 Indexes->insertMachineInstrInMaps(MI&: *ExecRestoreMI);
6744}
6745
6746MachineInstr *
6747SIInstrInfo::getWholeWaveFunctionSetup(MachineFunction &MF) const {
6748 assert(MF.getInfo<SIMachineFunctionInfo>()->isWholeWaveFunction() &&
6749 "Not a whole wave func");
6750 MachineBasicBlock &MBB = *MF.begin();
6751 for (MachineInstr &MI : MBB)
6752 if (MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_SETUP ||
6753 MI.getOpcode() == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
6754 return &MI;
6755
6756 llvm_unreachable("Couldn't find SI_SETUP_WHOLE_WAVE_FUNC instruction");
6757}
6758
6759const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
6760 unsigned OpNo) const {
6761 const MCInstrDesc &Desc = get(Opcode: MI.getOpcode());
6762 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
6763 Desc.operands()[OpNo].RegClass == -1) {
6764 Register Reg = MI.getOperand(i: OpNo).getReg();
6765
6766 if (Reg.isVirtual()) {
6767 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6768 return MRI.getRegClass(Reg);
6769 }
6770 return RI.getPhysRegBaseClass(Reg);
6771 }
6772
6773 int16_t RegClass = getOpRegClassID(OpInfo: Desc.operands()[OpNo]);
6774 return RegClass < 0 ? nullptr : RI.getRegClass(i: RegClass);
6775}
6776
6777// Convert VOP3 operand index to source number.
6778static unsigned VOP3OpIdxToSrcN(const MachineInstr &MI, unsigned OpIdx) {
6779 constexpr AMDGPU::OpName OpNames[] = {
6780 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};
6781
6782 for (auto [I, OpName] : enumerate(First: OpNames)) {
6783 int SrcIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[I]);
6784 if (static_cast<unsigned>(SrcIdx) == OpIdx)
6785 return I;
6786 }
6787
6788 return UINT_MAX;
6789}
6790
6791void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
6792 MachineBasicBlock::iterator I = MI;
6793 MachineBasicBlock *MBB = MI.getParent();
6794 MachineOperand &MO = MI.getOperand(i: OpIdx);
6795 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
6796 unsigned RCID = getOpRegClassID(OpInfo: get(Opcode: MI.getOpcode()).operands()[OpIdx]);
6797 const TargetRegisterClass *RC = RI.getRegClass(i: RCID);
6798 unsigned Size = RI.getRegSizeInBits(RC: *RC);
6799 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO
6800 : Size == 16 ? AMDGPU::V_MOV_B16_t16_e64
6801 : AMDGPU::V_MOV_B32_e32;
6802 if (MO.isReg())
6803 Opcode = AMDGPU::COPY;
6804 else if (RI.isSGPRClass(RC))
6805 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
6806
6807 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(SRC: RC);
6808 Register Reg = MRI.createVirtualRegister(RegClass: VRC);
6809 DebugLoc DL = MBB->findDebugLoc(MBBI: I);
6810
6811 if (Size == 128 && AMDGPU::isPackedSingleSGPR64BitInst(Opc: MI.getOpcode()) &&
6812 isLegalSingleSGPRReadInstOperand(MRI, MI, SrcN: VOP3OpIdxToSrcN(MI, OpIdx))) {
6813 // Special case for V_PK_*64 instructions: these do not have OPSEL but SGPR
6814 // sources behave like OPSEL is set replicating low 64-bits into high. VGPR
6815 // sources in turn read actual 4 registers. To move operand from an SGPR to
6816 // a VGPR we need to replicate low half.
6817 // We also do not select immediates for these instructions so it always has
6818 // to be an SGPR register here.
6819 // Operands which are not legal as per isLegalSingleSGPRReadInstOperand()
6820 // sent here specifically to fix a non-splat SGPR and shall perform a full
6821 // copy.
6822
6823 const TargetRegisterClass *VRC64 = RI.getVGPRClassForBitWidth(BitWidth: 64);
6824 Register Low64 = MRI.createVirtualRegister(RegClass: VRC64);
6825 assert(MO.isReg() && RI.isSGPRReg(MRI, MO.getReg()));
6826 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Low64)
6827 .addReg(RegNo: MO.getReg(), Flags: {}, SubReg: AMDGPU::sub0_sub1);
6828 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Reg)
6829 .addReg(RegNo: Low64)
6830 .addImm(Val: AMDGPU::sub0_sub1)
6831 .addReg(RegNo: Low64)
6832 .addImm(Val: AMDGPU::sub2_sub3);
6833 } else if (Opcode == AMDGPU::V_MOV_B16_t16_e64) {
6834 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: Reg)
6835 .addImm(Val: 0) // src0_modifiers
6836 .add(MO)
6837 .addImm(Val: 0); // op_sel
6838 } else {
6839 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: Reg).add(MO);
6840 }
6841
6842 MO.ChangeToRegister(Reg, isDef: false);
6843}
6844
6845unsigned SIInstrInfo::buildExtractSubReg(
6846 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
6847 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
6848 unsigned SubIdx, const TargetRegisterClass *SubRC) const {
6849 if (!SuperReg.getReg().isVirtual())
6850 return RI.getSubReg(Reg: SuperReg.getReg(), Idx: SubIdx);
6851
6852 MachineBasicBlock *MBB = MI->getParent();
6853 const DebugLoc &DL = MI->getDebugLoc();
6854 Register SubReg = MRI.createVirtualRegister(RegClass: SubRC);
6855
6856 unsigned NewSubIdx = RI.composeSubRegIndices(a: SuperReg.getSubReg(), b: SubIdx);
6857 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: SubReg)
6858 .addReg(RegNo: SuperReg.getReg(), Flags: {}, SubReg: NewSubIdx);
6859 return SubReg;
6860}
6861
6862MachineOperand SIInstrInfo::buildExtractSubRegOrImm(
6863 MachineBasicBlock::iterator MII, MachineRegisterInfo &MRI,
6864 const MachineOperand &Op, const TargetRegisterClass *SuperRC,
6865 unsigned SubIdx, const TargetRegisterClass *SubRC) const {
6866 if (Op.isImm()) {
6867 if (SubIdx == AMDGPU::sub0)
6868 return MachineOperand::CreateImm(Val: static_cast<int32_t>(Op.getImm()));
6869 if (SubIdx == AMDGPU::sub1)
6870 return MachineOperand::CreateImm(Val: static_cast<int32_t>(Op.getImm() >> 32));
6871
6872 llvm_unreachable("Unhandled register index for immediate");
6873 }
6874
6875 unsigned SubReg = buildExtractSubReg(MI: MII, MRI, SuperReg: Op, SuperRC,
6876 SubIdx, SubRC);
6877 return MachineOperand::CreateReg(Reg: SubReg, isDef: false);
6878}
6879
6880// Change the order of operands from (0, 1, 2) to (0, 2, 1)
6881void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
6882 assert(Inst.getNumExplicitOperands() == 3);
6883 MachineOperand Op1 = Inst.getOperand(i: 1);
6884 Inst.removeOperand(OpNo: 1);
6885 Inst.addOperand(Op: Op1);
6886}
6887
6888bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
6889 const MCOperandInfo &OpInfo,
6890 const MachineOperand &MO) const {
6891 if (!MO.isReg())
6892 return false;
6893
6894 Register Reg = MO.getReg();
6895
6896 const TargetRegisterClass *DRC = RI.getRegClass(i: getOpRegClassID(OpInfo));
6897 if (Reg.isPhysical())
6898 return DRC->contains(Reg);
6899
6900 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
6901
6902 if (MO.getSubReg()) {
6903 const TargetRegisterClass *SuperRC =
6904 RI.getLargestLegalSuperClass(RC, MF: MRI.getMF());
6905 if (!SuperRC)
6906 return false;
6907 return RI.getMatchingSuperRegClass(A: SuperRC, B: DRC, Idx: MO.getSubReg()) != nullptr;
6908 }
6909
6910 return RI.getCommonSubClass(A: DRC, B: RC) != nullptr;
6911}
6912
6913bool SIInstrInfo::isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
6914 const MachineOperand &MO) const {
6915 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6916 const MCOperandInfo OpInfo = MI.getDesc().operands()[OpIdx];
6917 unsigned Opc = MI.getOpcode();
6918
6919 // See SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
6920 if (MO.isReg() && RI.isSGPRReg(MRI, Reg: MO.getReg()) &&
6921 AMDGPU::isSingleSGPRReadInst(Opc: MI.getOpcode()) &&
6922 !isLegalSingleSGPRReadInstOperand(MRI, MI, SrcN: VOP3OpIdxToSrcN(MI, OpIdx),
6923 MO: &MO))
6924 return false;
6925
6926 if (!isLegalRegOperand(MRI, OpInfo, MO))
6927 return false;
6928
6929 // check Accumulate GPR operand
6930 bool IsAGPR = RI.isAGPR(MRI, Reg: MO.getReg());
6931 if (IsAGPR && !ST.hasMAIInsts())
6932 return false;
6933 if (IsAGPR && (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
6934 (MI.mayLoad() || MI.mayStore() || isDS(Opcode: Opc) || isMIMG(Opcode: Opc)))
6935 return false;
6936 // Atomics should have both vdst and vdata either vgpr or agpr.
6937 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
6938 const int DataIdx = AMDGPU::getNamedOperandIdx(
6939 Opcode: Opc, Name: isDS(Opcode: Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata);
6940 if ((int)OpIdx == VDstIdx && DataIdx != -1 &&
6941 MI.getOperand(i: DataIdx).isReg() &&
6942 RI.isAGPR(MRI, Reg: MI.getOperand(i: DataIdx).getReg()) != IsAGPR)
6943 return false;
6944 if ((int)OpIdx == DataIdx) {
6945 if (VDstIdx != -1 &&
6946 RI.isAGPR(MRI, Reg: MI.getOperand(i: VDstIdx).getReg()) != IsAGPR)
6947 return false;
6948 // DS instructions with 2 src operands also must have tied RC.
6949 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data1);
6950 if (Data1Idx != -1 && MI.getOperand(i: Data1Idx).isReg() &&
6951 RI.isAGPR(MRI, Reg: MI.getOperand(i: Data1Idx).getReg()) != IsAGPR)
6952 return false;
6953 }
6954
6955 // Check V_ACCVGPR_WRITE_B32_e64
6956 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() &&
6957 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0) &&
6958 RI.isSGPRReg(MRI, Reg: MO.getReg()))
6959 return false;
6960
6961 if (ST.hasFlatScratchHiInB64InstHazard() &&
6962 MO.getReg() == AMDGPU::SRC_FLAT_SCRATCH_BASE_HI && isSALU(MI)) {
6963 if (const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst)) {
6964 if (AMDGPU::getRegBitWidth(RC: *RI.getRegClassForReg(MRI, Reg: Dst->getReg())) ==
6965 64)
6966 return false;
6967 }
6968 if (Opc == AMDGPU::S_BITCMP0_B64 || Opc == AMDGPU::S_BITCMP1_B64)
6969 return false;
6970 }
6971 if (!ST.hasDPPSrc1SGPR() && isDPP(MI) && RI.isSGPRReg(MRI, Reg: MO.getReg()) &&
6972 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1))
6973 return false;
6974
6975 return true;
6976}
6977
6978bool SIInstrInfo::isLegalSingleSGPRReadInstOperand(
6979 const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,
6980 const MachineOperand *MO) const {
6981 constexpr unsigned NumOps = 3;
6982 constexpr AMDGPU::OpName OpNames[NumOps * 2] = {
6983 AMDGPU::OpName::src0, AMDGPU::OpName::src1,
6984 AMDGPU::OpName::src2, AMDGPU::OpName::src0_modifiers,
6985 AMDGPU::OpName::src1_modifiers, AMDGPU::OpName::src2_modifiers};
6986
6987 assert(SrcN < NumOps);
6988
6989 if (!MO) {
6990 int SrcIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[SrcN]);
6991 if (SrcIdx == -1)
6992 return true;
6993 MO = &MI.getOperand(i: SrcIdx);
6994 }
6995
6996 if (!MO->isReg() || !RI.isSGPRReg(MRI, Reg: MO->getReg()))
6997 return true;
6998
6999 int ModsIdx =
7000 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[NumOps + SrcN]);
7001 if (ModsIdx == -1)
7002 return false;
7003
7004 unsigned Mods = MI.getOperand(i: ModsIdx).getImm();
7005 bool OpSel = Mods & SISrcMods::OP_SEL_0;
7006 bool OpSelHi = Mods & SISrcMods::OP_SEL_1;
7007
7008 return !OpSel && !OpSelHi;
7009}
7010
7011bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
7012 const MachineOperand *MO) const {
7013 const MachineFunction &MF = *MI.getMF();
7014 const MachineRegisterInfo &MRI = MF.getRegInfo();
7015 const MCInstrDesc &InstDesc = MI.getDesc();
7016 const MCOperandInfo &OpInfo = InstDesc.operands()[OpIdx];
7017 int64_t RegClass = getOpRegClassID(OpInfo);
7018 const TargetRegisterClass *DefinedRC =
7019 RegClass != -1 ? RI.getRegClass(i: RegClass) : nullptr;
7020 if (!MO)
7021 MO = &MI.getOperand(i: OpIdx);
7022
7023 const bool IsInlineConst = !MO->isReg() && isInlineConstant(MO: *MO, OpInfo);
7024
7025 if (isVALU(MI, /*AllowLDSDMA=*/false) && !IsInlineConst &&
7026 usesConstantBus(MRI, MO: *MO, OpInfo)) {
7027 const MachineOperand *UsedLiteral = nullptr;
7028
7029 int ConstantBusLimit = ST.getConstantBusLimit(Opcode: MI.getOpcode());
7030 int LiteralLimit = !isVOP3(MI) || ST.hasVOP3Literal() ? 1 : 0;
7031
7032 // TODO: Be more permissive with frame indexes.
7033 if (!MO->isReg() && !isInlineConstant(MO: *MO, OpInfo)) {
7034 if (!LiteralLimit--)
7035 return false;
7036
7037 UsedLiteral = MO;
7038 }
7039
7040 SmallDenseSet<RegSubRegPair> SGPRsUsed;
7041 if (MO->isReg())
7042 SGPRsUsed.insert(V: RegSubRegPair(MO->getReg(), MO->getSubReg()));
7043
7044 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
7045 if (i == OpIdx)
7046 continue;
7047 const MachineOperand &Op = MI.getOperand(i);
7048 if (Op.isReg()) {
7049 if (Op.isUse()) {
7050 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
7051 if (regUsesConstantBus(RegOp: Op, MRI) && SGPRsUsed.insert(V: SGPR).second) {
7052 if (--ConstantBusLimit <= 0)
7053 return false;
7054 }
7055 }
7056 } else if (AMDGPU::isSISrcOperand(OpInfo: InstDesc.operands()[i]) &&
7057 !isInlineConstant(MO: Op, OpInfo: InstDesc.operands()[i])) {
7058 // The same literal may be used multiple times.
7059 if (!UsedLiteral)
7060 UsedLiteral = &Op;
7061 else if (UsedLiteral->isIdenticalTo(Other: Op))
7062 continue;
7063
7064 if (!LiteralLimit--)
7065 return false;
7066 if (--ConstantBusLimit <= 0)
7067 return false;
7068 }
7069 }
7070 } else if (!IsInlineConst && !MO->isReg() && isSALU(MI)) {
7071 // There can be at most one literal operand, but it can be repeated.
7072 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
7073 if (i == OpIdx)
7074 continue;
7075 const MachineOperand &Op = MI.getOperand(i);
7076 if (!Op.isReg() && !Op.isFI() && !Op.isRegMask() &&
7077 !isInlineConstant(MO: Op, OpInfo: InstDesc.operands()[i]) &&
7078 !Op.isIdenticalTo(Other: *MO))
7079 return false;
7080
7081 // Do not fold a non-inlineable and non-register operand into an
7082 // instruction that already has a frame index. The frame index handling
7083 // code could not handle well when a frame index co-exists with another
7084 // non-register operand, unless that operand is an inlineable immediate.
7085 if (Op.isFI())
7086 return false;
7087 }
7088 }
7089
7090 if (MO->isReg()) {
7091 if (!DefinedRC)
7092 return OpInfo.OperandType == MCOI::OPERAND_UNKNOWN;
7093 return isLegalRegOperand(MI, OpIdx, MO: *MO);
7094 }
7095
7096 if (MO->isImm()) {
7097 uint64_t Imm = MO->getImm();
7098 bool Is64BitFPOp = OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_FP64 ||
7099 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP64;
7100 bool Is64BitOp = Is64BitFPOp ||
7101 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64 ||
7102 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT32 ||
7103 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32 ||
7104 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT64;
7105 if (Is64BitOp &&
7106 !AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: ST.hasInv2PiInlineImm())) {
7107 if (!AMDGPU::isValid32BitLiteral(Val: Imm, IsFP64: Is64BitFPOp) &&
7108 (!ST.has64BitLiterals() || InstDesc.getSize() != 4))
7109 return false;
7110
7111 // FIXME: We can use sign extended 64-bit literals, but only for signed
7112 // operands. At the moment we do not know if an operand is signed.
7113 // Such operand will be encoded as its low 32 bits and then either
7114 // correctly sign extended or incorrectly zero extended by HW.
7115 // If 64-bit literals are supported and the literal will be encoded
7116 // as full 64 bit we still can use it.
7117 if (!Is64BitFPOp && (int32_t)Imm < 0 &&
7118 (!ST.has64BitLiterals() || AMDGPU::isValid32BitLiteral(Val: Imm, IsFP64: false)))
7119 return false;
7120 }
7121 }
7122
7123 // Handle non-register types that are treated like immediates.
7124 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
7125
7126 if (!DefinedRC) {
7127 // This operand expects an immediate.
7128 return true;
7129 }
7130
7131 return isImmOperandLegal(MI, OpNo: OpIdx, MO: *MO);
7132}
7133
7134bool SIInstrInfo::isNeverCoissue(MachineInstr &MI) const {
7135 bool IsGFX950Only = ST.hasGFX950Insts();
7136 bool IsGFX940Only = ST.hasGFX940Insts();
7137
7138 if (!IsGFX950Only && !IsGFX940Only)
7139 return false;
7140
7141 if (!isVALU(MI, /*AllowLDSDMA=*/false))
7142 return false;
7143
7144 // V_COS, V_EXP, V_RCP, etc.
7145 if (isTRANS(MI))
7146 return true;
7147
7148 // DOT2, DOT2C, DOT4, etc.
7149 if (isDOT(MI))
7150 return true;
7151
7152 // MFMA, SMFMA
7153 if (isMFMA(MI))
7154 return true;
7155
7156 unsigned Opcode = MI.getOpcode();
7157 switch (Opcode) {
7158 case AMDGPU::V_CVT_PK_BF8_F32_e64:
7159 case AMDGPU::V_CVT_PK_FP8_F32_e64:
7160 case AMDGPU::V_MQSAD_PK_U16_U8_e64:
7161 case AMDGPU::V_MQSAD_U32_U8_e64:
7162 case AMDGPU::V_PK_ADD_F16:
7163 case AMDGPU::V_PK_ADD_F32:
7164 case AMDGPU::V_PK_ADD_I16:
7165 case AMDGPU::V_PK_ADD_U16:
7166 case AMDGPU::V_PK_ASHRREV_I16:
7167 case AMDGPU::V_PK_FMA_F16:
7168 case AMDGPU::V_PK_FMA_F32:
7169 case AMDGPU::V_PK_FMAC_F16_e32:
7170 case AMDGPU::V_PK_FMAC_F16_e64:
7171 case AMDGPU::V_PK_LSHLREV_B16:
7172 case AMDGPU::V_PK_LSHRREV_B16:
7173 case AMDGPU::V_PK_MAD_I16:
7174 case AMDGPU::V_PK_MAD_U16:
7175 case AMDGPU::V_PK_MAX_F16:
7176 case AMDGPU::V_PK_MAX_I16:
7177 case AMDGPU::V_PK_MAX_U16:
7178 case AMDGPU::V_PK_MIN_F16:
7179 case AMDGPU::V_PK_MIN_I16:
7180 case AMDGPU::V_PK_MIN_U16:
7181 case AMDGPU::V_PK_MOV_B32:
7182 case AMDGPU::V_PK_MUL_F16:
7183 case AMDGPU::V_PK_MUL_F32:
7184 case AMDGPU::V_PK_MUL_LO_U16:
7185 case AMDGPU::V_PK_SUB_I16:
7186 case AMDGPU::V_PK_SUB_U16:
7187 case AMDGPU::V_QSAD_PK_U16_U8_e64:
7188 return true;
7189 default:
7190 return false;
7191 }
7192}
7193
7194void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
7195 MachineInstr &MI) const {
7196 unsigned Opc = MI.getOpcode();
7197 const MCInstrDesc &InstrDesc = get(Opcode: Opc);
7198
7199 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
7200 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
7201
7202 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
7203 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
7204
7205 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
7206 // we need to only have one constant bus use before GFX10.
7207 bool HasImplicitSGPR = findImplicitSGPRRead(MI);
7208 if (HasImplicitSGPR && ST.getConstantBusLimit(Opcode: Opc) <= 1 && Src0.isReg() &&
7209 RI.isSGPRReg(MRI, Reg: Src0.getReg()))
7210 legalizeOpWithMove(MI, OpIdx: Src0Idx);
7211
7212 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
7213 // both the value to write (src0) and lane select (src1). Fix up non-SGPR
7214 // src0/src1 with V_READFIRSTLANE.
7215 if (Opc == AMDGPU::V_WRITELANE_B32) {
7216 const DebugLoc &DL = MI.getDebugLoc();
7217 if (Src0.isReg() && RI.isVGPR(MRI, Reg: Src0.getReg())) {
7218 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7219 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7220 .add(MO: Src0);
7221 Src0.ChangeToRegister(Reg, isDef: false);
7222 }
7223 if (Src1.isReg() && RI.isVGPR(MRI, Reg: Src1.getReg())) {
7224 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7225 const DebugLoc &DL = MI.getDebugLoc();
7226 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7227 .add(MO: Src1);
7228 Src1.ChangeToRegister(Reg, isDef: false);
7229 }
7230 return;
7231 }
7232
7233 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2.
7234 if (Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F16_e32) {
7235 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
7236 if (!RI.isVGPR(MRI, Reg: MI.getOperand(i: Src2Idx).getReg()))
7237 legalizeOpWithMove(MI, OpIdx: Src2Idx);
7238 }
7239
7240 // VOP2 src0 instructions support all operand types, so we don't need to check
7241 // their legality. If src1 is already legal, we don't need to do anything.
7242 if (isLegalRegOperand(MRI, OpInfo: InstrDesc.operands()[Src1Idx], MO: Src1))
7243 return;
7244
7245 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
7246 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
7247 // select is uniform.
7248 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
7249 RI.isVGPR(MRI, Reg: Src1.getReg())) {
7250 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7251 const DebugLoc &DL = MI.getDebugLoc();
7252 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7253 .add(MO: Src1);
7254 Src1.ChangeToRegister(Reg, isDef: false);
7255 return;
7256 }
7257
7258 // We do not use commuteInstruction here because it is too aggressive and will
7259 // commute if it is possible. We only want to commute here if it improves
7260 // legality. This can be called a fairly large number of times so don't waste
7261 // compile time pointlessly swapping and checking legality again.
7262 if (HasImplicitSGPR || !MI.isCommutable()) {
7263 legalizeOpWithMove(MI, OpIdx: Src1Idx);
7264 return;
7265 }
7266
7267 // If src0 can be used as src1, commuting will make the operands legal.
7268 // Otherwise we have to give up and insert a move.
7269 //
7270 // TODO: Other immediate-like operand kinds could be commuted if there was a
7271 // MachineOperand::ChangeTo* for them.
7272 if ((!Src1.isImm() && !Src1.isReg()) ||
7273 !isLegalRegOperand(MRI, OpInfo: InstrDesc.operands()[Src1Idx], MO: Src0)) {
7274 legalizeOpWithMove(MI, OpIdx: Src1Idx);
7275 return;
7276 }
7277
7278 int CommutedOpc = commuteOpcode(MI);
7279 if (CommutedOpc == -1) {
7280 legalizeOpWithMove(MI, OpIdx: Src1Idx);
7281 return;
7282 }
7283
7284 MI.setDesc(get(Opcode: CommutedOpc));
7285
7286 Register Src0Reg = Src0.getReg();
7287 unsigned Src0SubReg = Src0.getSubReg();
7288 bool Src0Kill = Src0.isKill();
7289
7290 if (Src1.isImm())
7291 Src0.ChangeToImmediate(ImmVal: Src1.getImm());
7292 else if (Src1.isReg()) {
7293 Src0.ChangeToRegister(Reg: Src1.getReg(), isDef: false, isImp: false, isKill: Src1.isKill());
7294 Src0.setSubReg(Src1.getSubReg());
7295 } else
7296 llvm_unreachable("Should only have register or immediate operands");
7297
7298 Src1.ChangeToRegister(Reg: Src0Reg, isDef: false, isImp: false, isKill: Src0Kill);
7299 Src1.setSubReg(Src0SubReg);
7300 fixImplicitOperands(MI);
7301}
7302
7303// Legalize VOP3 operands. All operand types are supported for any operand
7304// but only one literal constant and only starting from GFX10.
7305void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
7306 MachineInstr &MI) const {
7307 unsigned Opc = MI.getOpcode();
7308
7309 int VOP3Idx[3] = {
7310 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0),
7311 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1),
7312 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2)
7313 };
7314
7315 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 ||
7316 Opc == AMDGPU::V_PERMLANEX16_B32_e64 ||
7317 Opc == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
7318 Opc == AMDGPU::V_PERMLANE_UP_B32_e64 ||
7319 Opc == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
7320 Opc == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
7321 Opc == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64) {
7322 // src1 and src2 must be scalar
7323 MachineOperand &Src1 = MI.getOperand(i: VOP3Idx[1]);
7324 const DebugLoc &DL = MI.getDebugLoc();
7325 if (Src1.isReg() && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src1.getReg()))) {
7326 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7327 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7328 .add(MO: Src1);
7329 Src1.ChangeToRegister(Reg, isDef: false);
7330 }
7331 if (VOP3Idx[2] != -1) {
7332 MachineOperand &Src2 = MI.getOperand(i: VOP3Idx[2]);
7333 if (Src2.isReg() && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src2.getReg()))) {
7334 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7335 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7336 .add(MO: Src2);
7337 Src2.ChangeToRegister(Reg, isDef: false);
7338 }
7339 }
7340 }
7341
7342 // Find the one SGPR operand we are allowed to use.
7343 int ConstantBusLimit = ST.getConstantBusLimit(Opcode: Opc);
7344 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
7345 SmallDenseSet<unsigned> SGPRsUsed;
7346 Register SGPRReg = findUsedSGPR(MI, OpIndices: VOP3Idx);
7347 if (SGPRReg) {
7348 SGPRsUsed.insert(V: SGPRReg);
7349 --ConstantBusLimit;
7350 }
7351
7352 for (int Idx : VOP3Idx) {
7353 if (Idx == -1)
7354 break;
7355 MachineOperand &MO = MI.getOperand(i: Idx);
7356
7357 if (!MO.isReg()) {
7358 if (isInlineConstant(MO, OpInfo: get(Opcode: Opc).operands()[Idx]))
7359 continue;
7360
7361 if (LiteralLimit > 0 && ConstantBusLimit > 0) {
7362 --LiteralLimit;
7363 --ConstantBusLimit;
7364 continue;
7365 }
7366
7367 --LiteralLimit;
7368 --ConstantBusLimit;
7369 legalizeOpWithMove(MI, OpIdx: Idx);
7370 continue;
7371 }
7372
7373 if (!RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg: MO.getReg())))
7374 continue; // VGPRs are legal
7375
7376 // We can use one SGPR in each VOP3 instruction prior to GFX10
7377 // and two starting from GFX10.
7378 if (SGPRsUsed.count(V: MO.getReg()))
7379 continue;
7380 if (ConstantBusLimit > 0) {
7381 SGPRsUsed.insert(V: MO.getReg());
7382 --ConstantBusLimit;
7383 continue;
7384 }
7385
7386 // If we make it this far, then the operand is not legal and we must
7387 // legalize it.
7388 legalizeOpWithMove(MI, OpIdx: Idx);
7389 }
7390
7391 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2 tied to vdst.
7392 if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
7393 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
7394 Opc == AMDGPU::V_FMAC_F16_t16_e64) &&
7395 !RI.isVGPR(MRI, Reg: MI.getOperand(i: VOP3Idx[2]).getReg()))
7396 legalizeOpWithMove(MI, OpIdx: VOP3Idx[2]);
7397
7398 // Fix the register class of single-sgpr-read instructions on gfx12+. See
7399 // SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
7400 if (AMDGPU::isSingleSGPRReadInst(Opc)) {
7401 for (unsigned I = 0; I < 3; ++I) {
7402 if (!isLegalSingleSGPRReadInstOperand(MRI, MI, /*SrcN=*/I))
7403 legalizeOpWithMove(MI, OpIdx: VOP3Idx[I]);
7404 }
7405 }
7406}
7407
7408Register SIInstrInfo::readlaneVGPRToSGPR(
7409 Register SrcReg, MachineInstr &UseMI, MachineRegisterInfo &MRI,
7410 const TargetRegisterClass *DstRC /*=nullptr*/) const {
7411 const TargetRegisterClass *VRC = MRI.getRegClass(Reg: SrcReg);
7412 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
7413 if (DstRC)
7414 SRC = RI.getCommonSubClass(A: SRC, B: DstRC);
7415
7416 Register DstReg = MRI.createVirtualRegister(RegClass: SRC);
7417 unsigned SubRegs = RI.getRegSizeInBits(RC: *VRC) / 32;
7418
7419 if (RI.hasAGPRs(RC: VRC)) {
7420 VRC = RI.getEquivalentVGPRClass(SRC: VRC);
7421 Register NewSrcReg = MRI.createVirtualRegister(RegClass: VRC);
7422 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7423 MCID: get(Opcode: TargetOpcode::COPY), DestReg: NewSrcReg)
7424 .addReg(RegNo: SrcReg);
7425 SrcReg = NewSrcReg;
7426 }
7427
7428 if (SubRegs == 1) {
7429 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7430 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
7431 .addReg(RegNo: SrcReg);
7432 return DstReg;
7433 }
7434
7435 SmallVector<Register, 8> SRegs;
7436 for (unsigned i = 0; i < SubRegs; ++i) {
7437 Register SGPR = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
7438 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7439 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: SGPR)
7440 .addReg(RegNo: SrcReg, Flags: {}, SubReg: RI.getSubRegFromChannel(Channel: i));
7441 SRegs.push_back(Elt: SGPR);
7442 }
7443
7444 MachineInstrBuilder MIB =
7445 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7446 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
7447 for (unsigned i = 0; i < SubRegs; ++i) {
7448 MIB.addReg(RegNo: SRegs[i]);
7449 MIB.addImm(Val: RI.getSubRegFromChannel(Channel: i));
7450 }
7451 return DstReg;
7452}
7453
7454void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,
7455 MachineInstr &MI) const {
7456
7457 // If the pointer is store in VGPRs, then we need to move them to
7458 // SGPRs using v_readfirstlane. This is safe because we only select
7459 // loads with uniform pointers to SMRD instruction so we know the
7460 // pointer value is uniform.
7461 MachineOperand *SBase = getNamedOperand(MI, OperandName: AMDGPU::OpName::sbase);
7462 if (SBase && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SBase->getReg()))) {
7463 Register SGPR = readlaneVGPRToSGPR(SrcReg: SBase->getReg(), UseMI&: MI, MRI);
7464 SBase->setReg(SGPR);
7465 }
7466 MachineOperand *SOff = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
7467 if (SOff && !RI.isSGPRReg(MRI, Reg: SOff->getReg())) {
7468 Register SGPR = readlaneVGPRToSGPR(SrcReg: SOff->getReg(), UseMI&: MI, MRI);
7469 SOff->setReg(SGPR);
7470 }
7471}
7472
7473bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const {
7474 unsigned Opc = Inst.getOpcode();
7475 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::saddr);
7476 if (OldSAddrIdx < 0)
7477 return false;
7478
7479 assert(isSegmentSpecificFLAT(Inst) || (isFLAT(Inst) && ST.hasFlatGVSMode()));
7480
7481 int NewOpc = AMDGPU::getGlobalVaddrOp(Opcode: Opc);
7482 if (NewOpc < 0)
7483 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opcode: Opc);
7484 if (NewOpc < 0)
7485 return false;
7486
7487 MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
7488 MachineOperand &SAddr = Inst.getOperand(i: OldSAddrIdx);
7489 if (RI.isSGPRReg(MRI, Reg: SAddr.getReg()))
7490 return false;
7491
7492 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::vaddr);
7493 if (NewVAddrIdx < 0)
7494 return false;
7495
7496 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
7497
7498 // Check vaddr, it shall be zero or absent.
7499 MachineInstr *VAddrDef = nullptr;
7500 if (OldVAddrIdx >= 0) {
7501 MachineOperand &VAddr = Inst.getOperand(i: OldVAddrIdx);
7502 VAddrDef = MRI.getUniqueVRegDef(Reg: VAddr.getReg());
7503 if (!VAddrDef || !VAddrDef->isMoveImmediate() ||
7504 !VAddrDef->getOperand(i: 1).isImm() ||
7505 VAddrDef->getOperand(i: 1).getImm() != 0)
7506 return false;
7507 }
7508
7509 const MCInstrDesc &NewDesc = get(Opcode: NewOpc);
7510 Inst.setDesc(NewDesc);
7511
7512 // Callers expect iterator to be valid after this call, so modify the
7513 // instruction in place.
7514 if (OldVAddrIdx == NewVAddrIdx) {
7515 MachineOperand &NewVAddr = Inst.getOperand(i: NewVAddrIdx);
7516 // Clear use list from the old vaddr holding a zero register.
7517 MRI.removeRegOperandFromUseList(MO: &NewVAddr);
7518 MRI.moveOperands(Dst: &NewVAddr, Src: &SAddr, NumOps: 1);
7519 Inst.removeOperand(OpNo: OldSAddrIdx);
7520 // Update the use list with the pointer we have just moved from vaddr to
7521 // saddr position. Otherwise new vaddr will be missing from the use list.
7522 MRI.removeRegOperandFromUseList(MO: &NewVAddr);
7523 MRI.addRegOperandToUseList(MO: &NewVAddr);
7524 } else {
7525 assert(OldSAddrIdx == NewVAddrIdx);
7526
7527 if (OldVAddrIdx >= 0) {
7528 int NewVDstIn = AMDGPU::getNamedOperandIdx(Opcode: NewOpc,
7529 Name: AMDGPU::OpName::vdst_in);
7530
7531 // removeOperand doesn't try to fixup tied operand indexes at it goes, so
7532 // it asserts. Untie the operands for now and retie them afterwards.
7533 if (NewVDstIn != -1) {
7534 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst_in);
7535 Inst.untieRegOperand(OpIdx: OldVDstIn);
7536 }
7537
7538 Inst.removeOperand(OpNo: OldVAddrIdx);
7539
7540 if (NewVDstIn != -1) {
7541 int NewVDst = AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::vdst);
7542 Inst.tieOperands(DefIdx: NewVDst, UseIdx: NewVDstIn);
7543 }
7544 }
7545 }
7546
7547 if (VAddrDef && MRI.use_nodbg_empty(RegNo: VAddrDef->getOperand(i: 0).getReg()))
7548 VAddrDef->eraseFromParent();
7549
7550 return true;
7551}
7552
7553// FIXME: Remove this when SelectionDAG is obsoleted.
7554void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI,
7555 MachineInstr &MI) const {
7556 if (!isSegmentSpecificFLAT(MI) && !ST.hasFlatGVSMode())
7557 return;
7558
7559 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence
7560 // thinks they are uniform, so a readfirstlane should be valid.
7561 MachineOperand *SAddr = getNamedOperand(MI, OperandName: AMDGPU::OpName::saddr);
7562 if (!SAddr || RI.isSGPRClass(RC: MRI.getRegClass(Reg: SAddr->getReg())))
7563 return;
7564
7565 if (moveFlatAddrToVGPR(Inst&: MI))
7566 return;
7567
7568 const TargetRegisterClass *DeclaredRC =
7569 getRegClass(MCID: MI.getDesc(), OpNum: SAddr->getOperandNo());
7570
7571 Register ToSGPR = readlaneVGPRToSGPR(SrcReg: SAddr->getReg(), UseMI&: MI, MRI, DstRC: DeclaredRC);
7572 SAddr->setReg(ToSGPR);
7573}
7574
7575void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
7576 MachineBasicBlock::iterator I,
7577 const TargetRegisterClass *DstRC,
7578 MachineOperand &Op,
7579 MachineRegisterInfo &MRI,
7580 const DebugLoc &DL) const {
7581 Register OpReg = Op.getReg();
7582 unsigned OpSubReg = Op.getSubReg();
7583
7584 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
7585 RI.getRegClassForReg(MRI, Reg: OpReg), OpSubReg);
7586
7587 // Check if operand is already the correct register class.
7588 if (DstRC == OpRC)
7589 return;
7590
7591 Register DstReg = MRI.createVirtualRegister(RegClass: DstRC);
7592 auto Copy = BuildMI(BB&: InsertMBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: DstReg)
7593 .addReg(RegNo: OpReg, Flags: {}, SubReg: OpSubReg);
7594 Op.setReg(DstReg);
7595 Op.setSubReg(AMDGPU::NoSubRegister);
7596
7597 MachineInstr *Def = MRI.getVRegDef(Reg: OpReg);
7598 if (!Def)
7599 return;
7600
7601 // Try to eliminate the copy if it is copying an immediate value.
7602 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass)
7603 foldImmediate(UseMI&: *Copy, DefMI&: *Def, Reg: OpReg, MRI: &MRI);
7604
7605 bool ImpDef = Def->isImplicitDef();
7606 while (!ImpDef && Def && Def->isCopy()) {
7607 if (Def->getOperand(i: 1).getReg().isPhysical())
7608 break;
7609 Def = MRI.getUniqueVRegDef(Reg: Def->getOperand(i: 1).getReg());
7610 ImpDef = Def && Def->isImplicitDef();
7611 }
7612 if (!RI.isSGPRClass(RC: DstRC) && !Copy->readsRegister(Reg: AMDGPU::EXEC, TRI: &RI) &&
7613 !ImpDef)
7614 Copy.addReg(RegNo: AMDGPU::EXEC, Flags: RegState::Implicit);
7615}
7616
7617// Emit the actual waterfall loop, executing the wrapped instruction for each
7618// unique value of \p ScalarOps across all lanes. In the best case we execute 1
7619// iteration, in the worst case we execute 64 (once per lane).
7620static void emitLoadScalarOpsFromVGPRLoop(
7621 const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &PredBB,
7622 MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
7623 ArrayRef<MachineOperand *> ScalarOps, ArrayRef<Register> PhySGPRs = {}) {
7624 MachineFunction &MF = *LoopBB.getParent();
7625 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
7626 const SIRegisterInfo *TRI = ST.getRegisterInfo();
7627 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
7628 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
7629
7630 // Emit v_cmpx_eq and s_andn2_wrexec when both instructions are
7631 // available. Otherwise, use the previous pattern of v_cmp_eq,
7632 // s_and_saveexec, and s_xor.
7633 bool UseNewExecInstructions =
7634 ST.hasNoSdstCMPX() && TII.pseudoToMCOpcode(Opcode: LMC.AndN2WrExecOpc) != -1;
7635
7636 MachineBasicBlock::iterator I = LoopBB.begin();
7637 Register CondReg;
7638
7639 Register PhiExec;
7640 Register NewExec;
7641
7642 if (UseNewExecInstructions) {
7643 PhiExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7644 NewExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7645 Register InitExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7646 BuildMI(BB&: PredBB, I: PredBB.end(), MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: InitExec)
7647 .addReg(RegNo: LMC.ExecReg);
7648
7649 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::PHI), DestReg: PhiExec)
7650 .addReg(RegNo: InitExec)
7651 .addMBB(MBB: &PredBB)
7652 .addReg(RegNo: NewExec)
7653 .addMBB(MBB: &BodyBB);
7654 }
7655
7656 // Placement of v_cmpx instructions (when index is longer than 64 bit)
7657 // involves a trade-off between register pressure and latency:
7658 // (a) Defering all v_cmpx after all v_readfirstlane may increase
7659 // register pressure because arguments and results of all
7660 // v_readfirstlane instructions must stay live until deferred v_cmpx use them.
7661 // (b) Interleaving v_cmpx with v_readfirstlanes may reduce live ranges and
7662 // increase latency by placing v_readfirstlane instructions
7663 // immediately before v_cmpx instruction that directly depend on it.
7664 ///
7665 // Emitting interleaved v_cmpx and v_readfirstlane requires
7666 // block splitting because v_cmpx changes EXEC mask and therefore for safety
7667 // v_cmpx needs to be treated as terminator until after register allocation
7668 // (spill placement) and instruction reordering.
7669 //
7670 // Current implementation defers v_cmpx and leaves other instruction
7671 // scheduling decisions to later passes, where register pressure is known or
7672 // easier to approximate.
7673 // Non-terminators (V_READFIRSTLANE and REG_SEQUENCE) are inserted before I;
7674 // v_cmpx instructions are inserted at the end of LoopBB.
7675 // After the first v_cmpx is emitted, I is updated to point to it
7676 // so subsequent non-terminators are inserted before all v_cmpx instructions.
7677 for (auto [Idx, ScalarOp] : enumerate(First&: ScalarOps)) {
7678 unsigned RegSize = TRI->getRegSizeInBits(Reg: ScalarOp->getReg(), MRI);
7679 unsigned NumSubRegs = RegSize / 32;
7680 Register VScalarOp = ScalarOp->getReg();
7681
7682 const TargetRegisterClass *RFLSrcRC =
7683 TII.getRegClass(MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), OpNum: 1);
7684
7685 if (NumSubRegs == 1) {
7686 const TargetRegisterClass *VScalarOpRC = MRI.getRegClass(Reg: VScalarOp);
7687 if (const TargetRegisterClass *Common =
7688 TRI->getCommonSubClass(A: VScalarOpRC, B: RFLSrcRC);
7689 Common != VScalarOpRC) {
7690 Register VRReg = MRI.createVirtualRegister(RegClass: Common);
7691 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VRReg).addReg(RegNo: VScalarOp);
7692 VScalarOp = VRReg;
7693 }
7694 Register CurReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7695
7696 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurReg)
7697 .addReg(RegNo: VScalarOp);
7698
7699 if (UseNewExecInstructions) {
7700 auto CmpxMI = BuildMI(BB&: LoopBB, I: LoopBB.end(), MIMD: DL,
7701 MCID: TII.get(Opcode: AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term))
7702 .addReg(RegNo: CurReg)
7703 .addReg(RegNo: VScalarOp);
7704 if (I == LoopBB.end())
7705 I = CmpxMI.getInstr()->getIterator();
7706 } else {
7707 Register NewCondReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7708
7709 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: NewCondReg)
7710 .addReg(RegNo: CurReg)
7711 .addReg(RegNo: VScalarOp);
7712
7713 // Combine the comparison results with AND.
7714 if (!CondReg) { // First.
7715 CondReg = NewCondReg;
7716 } else { // If not the first, we create an AND.
7717 Register AndReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7718 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndOpc), DestReg: AndReg)
7719 .addReg(RegNo: CondReg)
7720 .addReg(RegNo: NewCondReg)
7721 .setOperandDead(3);
7722 CondReg = AndReg;
7723 }
7724 }
7725
7726 // Update ScalarOp operand to use the SGPR ScalarOp.
7727 if (PhySGPRs.empty() || !PhySGPRs[Idx].isValid())
7728 ScalarOp->setReg(CurReg);
7729 else {
7730 // Insert into the same block of use
7731 BuildMI(BB&: *ScalarOp->getParent()->getParent(), I: ScalarOp->getParent(), MIMD: DL,
7732 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: PhySGPRs[Idx])
7733 .addReg(RegNo: CurReg);
7734 ScalarOp->setReg(PhySGPRs[Idx]);
7735 }
7736 } else {
7737 SmallVector<Register, 8> ReadlanePieces;
7738 RegState VScalarOpUndef = getUndefRegState(B: ScalarOp->isUndef());
7739 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 &&
7740 "Unhandled register size");
7741
7742 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) {
7743 Register CurRegLo =
7744 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7745 Register CurRegHi =
7746 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7747
7748 // Read the next variant <- also loop target.
7749 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurRegLo)
7750 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg: TRI->getSubRegFromChannel(Channel: Idx));
7751
7752 // Read the next variant <- also loop target.
7753 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurRegHi)
7754 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef,
7755 SubReg: TRI->getSubRegFromChannel(Channel: Idx + 1));
7756
7757 ReadlanePieces.push_back(Elt: CurRegLo);
7758 ReadlanePieces.push_back(Elt: CurRegHi);
7759
7760 // Comparison is to be done as 64-bit.
7761 Register CurReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_64RegClass);
7762 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: CurReg)
7763 .addReg(RegNo: CurRegLo)
7764 .addImm(Val: AMDGPU::sub0)
7765 .addReg(RegNo: CurRegHi)
7766 .addImm(Val: AMDGPU::sub1);
7767
7768 unsigned SubReg =
7769 NumSubRegs <= 2 ? 0 : TRI->getSubRegFromChannel(Channel: Idx, NumRegs: 2);
7770
7771 if (UseNewExecInstructions) {
7772 auto CmpxMI = BuildMI(BB&: LoopBB, I: LoopBB.end(), MIMD: DL,
7773 MCID: TII.get(Opcode: AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term))
7774 .addReg(RegNo: CurReg)
7775 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg);
7776 if (I == LoopBB.end())
7777 I = CmpxMI.getInstr()->getIterator();
7778 } else {
7779 Register NewCondReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7780 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U64_e64), DestReg: NewCondReg)
7781 .addReg(RegNo: CurReg)
7782 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg);
7783
7784 // Combine the comparison results with AND.
7785 if (!CondReg) { // First.
7786 CondReg = NewCondReg;
7787 } else { // If not the first, we create an AND.
7788 Register AndReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7789 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndOpc), DestReg: AndReg)
7790 .addReg(RegNo: CondReg)
7791 .addReg(RegNo: NewCondReg)
7792 .setOperandDead(3);
7793 CondReg = AndReg;
7794 }
7795 }
7796 } // End for loop.
7797
7798 const auto *SScalarOpRC =
7799 TRI->getEquivalentSGPRClass(VRC: MRI.getRegClass(Reg: VScalarOp));
7800 Register SScalarOp = MRI.createVirtualRegister(RegClass: SScalarOpRC);
7801
7802 // Build scalar ScalarOp.
7803 auto Merge =
7804 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: SScalarOp);
7805 unsigned Channel = 0;
7806 for (Register Piece : ReadlanePieces) {
7807 Merge.addReg(RegNo: Piece).addImm(Val: TRI->getSubRegFromChannel(Channel: Channel++));
7808 }
7809
7810 // Update ScalarOp operand to use the SGPR ScalarOp.
7811 if (PhySGPRs.empty() || !PhySGPRs[Idx].isValid())
7812 ScalarOp->setReg(SScalarOp);
7813 else {
7814 BuildMI(BB&: *ScalarOp->getParent()->getParent(), I: ScalarOp->getParent(), MIMD: DL,
7815 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: PhySGPRs[Idx])
7816 .addReg(RegNo: SScalarOp);
7817 ScalarOp->setReg(PhySGPRs[Idx]);
7818 }
7819 }
7820 }
7821
7822 // AndSaveExecOpc modifies EXEC but can't be isTerminator=1: terminators
7823 // that define virtual registers aren't supported.
7824 Register SaveExec;
7825 if (!UseNewExecInstructions) {
7826 SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7827 MRI.setSimpleHint(VReg: SaveExec, PrefReg: CondReg);
7828
7829 // Update EXEC to matching lanes, saving original to SaveExec.
7830 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndSaveExecOpc), DestReg: SaveExec)
7831 .addReg(RegNo: CondReg)
7832 .setOperandDead(3);
7833 }
7834
7835 // The original instruction is here; we insert the terminators after it.
7836 I = BodyBB.end();
7837
7838 if (UseNewExecInstructions) {
7839 // Compute the remaining lanes into a plain virtual register and write EXEC
7840 // from a terminator, so spill code for NewExec is placed before EXEC
7841 // changes. SIOptimizeExecMasking opportunistically folds the pair back
7842 // into S_ANDN2_WREXEC after register allocation.
7843 MRI.setSimpleHint(VReg: NewExec, PrefReg: PhiExec);
7844 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndN2Opc), DestReg: NewExec)
7845 .addReg(RegNo: PhiExec)
7846 .addReg(RegNo: LMC.ExecReg)
7847 .setOperandDead(3);
7848 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.MovTermOpc), DestReg: LMC.ExecReg)
7849 .addReg(RegNo: NewExec);
7850 } else {
7851 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
7852 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.XorTermOpc), DestReg: LMC.ExecReg)
7853 .addReg(RegNo: LMC.ExecReg)
7854 .addReg(RegNo: SaveExec)
7855 .setOperandDead(3);
7856 }
7857
7858 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::SI_WATERFALL_LOOP)).addMBB(MBB: &LoopBB);
7859}
7860
7861// Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register
7862// with SGPRs by iterating over all unique values across all lanes.
7863// Returns the loop basic block that now contains \p MI.
7864static MachineBasicBlock *
7865generateWaterFallLoop(const SIInstrInfo &TII, MachineInstr &MI,
7866 ArrayRef<MachineOperand *> ScalarOps,
7867 MachineDominatorTree *MDT,
7868 MachineBasicBlock::iterator Begin = nullptr,
7869 MachineBasicBlock::iterator End = nullptr,
7870 ArrayRef<Register> PhySGPRs = {}) {
7871 assert((PhySGPRs.empty() || PhySGPRs.size() == ScalarOps.size()) &&
7872 "Physical SGPRs must be empty or match the number of scalar operands");
7873 MachineBasicBlock &MBB = *MI.getParent();
7874 MachineFunction &MF = *MBB.getParent();
7875 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
7876 const SIRegisterInfo *TRI = ST.getRegisterInfo();
7877 MachineRegisterInfo &MRI = MF.getRegInfo();
7878 if (!Begin.isValid())
7879 Begin = &MI;
7880 if (!End.isValid()) {
7881 End = &MI;
7882 ++End;
7883 }
7884 const DebugLoc &DL = MI.getDebugLoc();
7885 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
7886 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
7887
7888 // Save SCC. Waterfall Loop may overwrite SCC.
7889 Register SaveSCCReg;
7890
7891 // FIXME: We should maintain SCC liveness while doing the FixSGPRCopies walk
7892 // rather than unlimited scan everywhere
7893 bool SCCNotDead =
7894 MBB.computeRegisterLiveness(TRI, Reg: AMDGPU::SCC, Before: MI,
7895 Neighborhood: std::numeric_limits<unsigned>::max()) !=
7896 MachineBasicBlock::LQR_Dead;
7897 if (SCCNotDead) {
7898 SaveSCCReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7899 BuildMI(BB&: MBB, I: Begin, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: SaveSCCReg)
7900 .addImm(Val: 1)
7901 .addImm(Val: 0);
7902 }
7903
7904 Register SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7905
7906 // Save the EXEC mask
7907 BuildMI(BB&: MBB, I: Begin, MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: SaveExec).addReg(RegNo: LMC.ExecReg);
7908
7909 // To insert the loop we need to split the block. Move everything after this
7910 // point to a new block, and insert a new empty block between the two.
7911 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
7912 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock();
7913 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
7914 MachineFunction::iterator MBBI(MBB);
7915 ++MBBI;
7916
7917 MF.insert(MBBI, MBB: LoopBB);
7918 MF.insert(MBBI, MBB: BodyBB);
7919 MF.insert(MBBI, MBB: RemainderBB);
7920
7921 LoopBB->addSuccessor(Succ: BodyBB);
7922 BodyBB->addSuccessor(Succ: LoopBB);
7923 BodyBB->addSuccessor(Succ: RemainderBB);
7924
7925 // Move Begin to MI to the BodyBB, and the remainder of the block to
7926 // RemainderBB.
7927 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
7928 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: End, To: MBB.end());
7929 BodyBB->splice(Where: BodyBB->begin(), Other: &MBB, From: Begin, To: MBB.end());
7930
7931 MBB.addSuccessor(Succ: LoopBB);
7932
7933 // Update dominators. We know that MBB immediately dominates LoopBB, that
7934 // LoopBB immediately dominates BodyBB, and BodyBB immediately dominates
7935 // RemainderBB. RemainderBB immediately dominates all of the successors
7936 // transferred to it from MBB that MBB used to properly dominate.
7937 if (MDT) {
7938 MDT->addNewBlock(BB: LoopBB, DomBB: &MBB);
7939 MDT->addNewBlock(BB: BodyBB, DomBB: LoopBB);
7940 MDT->addNewBlock(BB: RemainderBB, DomBB: BodyBB);
7941 for (auto &Succ : RemainderBB->successors()) {
7942 if (MDT->properlyDominates(A: &MBB, B: Succ)) {
7943 MDT->changeImmediateDominator(BB: Succ, NewBB: RemainderBB);
7944 }
7945 }
7946 }
7947
7948 emitLoadScalarOpsFromVGPRLoop(TII, MRI, PredBB&: MBB, LoopBB&: *LoopBB, BodyBB&: *BodyBB, DL, ScalarOps,
7949 PhySGPRs);
7950
7951 MachineBasicBlock::iterator First = RemainderBB->begin();
7952 // Restore SCC
7953 if (SCCNotDead) {
7954 BuildMI(BB&: *RemainderBB, I: First, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CMP_LG_U32))
7955 .addReg(RegNo: SaveSCCReg)
7956 .addImm(Val: 0);
7957 }
7958
7959 // Restore the EXEC mask
7960 BuildMI(BB&: *RemainderBB, I: First, MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
7961 .addReg(RegNo: SaveExec);
7962 return BodyBB;
7963}
7964
7965// Extract pointer from Rsrc and return a zero-value Rsrc replacement.
7966static std::tuple<unsigned, unsigned>
7967extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) {
7968 MachineBasicBlock &MBB = *MI.getParent();
7969 MachineFunction &MF = *MBB.getParent();
7970 MachineRegisterInfo &MRI = MF.getRegInfo();
7971
7972 // Extract the ptr from the resource descriptor.
7973 unsigned RsrcPtr =
7974 TII.buildExtractSubReg(MI, MRI, SuperReg: Rsrc, SuperRC: &AMDGPU::VReg_128RegClass,
7975 SubIdx: AMDGPU::sub0_sub1, SubRC: &AMDGPU::VReg_64RegClass);
7976
7977 // Create an empty resource descriptor
7978 Register Zero64 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
7979 Register SRsrcFormatLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
7980 Register SRsrcFormatHi = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
7981 Register NewSRsrc = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_128RegClass);
7982 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
7983
7984 // Zero64 = 0
7985 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B64), DestReg: Zero64)
7986 .addImm(Val: 0);
7987
7988 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
7989 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: SRsrcFormatLo)
7990 .addImm(Val: Lo_32(Value: RsrcDataFormat));
7991
7992 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
7993 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: SRsrcFormatHi)
7994 .addImm(Val: Hi_32(Value: RsrcDataFormat));
7995
7996 // NewSRsrc = {Zero64, SRsrcFormat}
7997 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewSRsrc)
7998 .addReg(RegNo: Zero64)
7999 .addImm(Val: AMDGPU::sub0_sub1)
8000 .addReg(RegNo: SRsrcFormatLo)
8001 .addImm(Val: AMDGPU::sub2)
8002 .addReg(RegNo: SRsrcFormatHi)
8003 .addImm(Val: AMDGPU::sub3);
8004
8005 return std::tuple(RsrcPtr, NewSRsrc);
8006}
8007
8008MachineBasicBlock *
8009SIInstrInfo::legalizeOperands(MachineInstr &MI,
8010 MachineDominatorTree *MDT) const {
8011 MachineFunction &MF = *MI.getMF();
8012 MachineRegisterInfo &MRI = MF.getRegInfo();
8013 MachineBasicBlock *CreatedBB = nullptr;
8014
8015 // Legalize True16
8016 if (ST.useRealTrue16Insts())
8017 legalizeOperandsVALUt16(Inst&: MI, MRI);
8018
8019 // Legalize VOP2
8020 if (isVOP2(MI) || isVOPC(MI)) {
8021 legalizeOperandsVOP2(MRI, MI);
8022 return CreatedBB;
8023 }
8024
8025 // Legalize VOP3
8026 if (isVOP3(MI)) {
8027 legalizeOperandsVOP3(MRI, MI);
8028 return CreatedBB;
8029 }
8030
8031 // Legalize SMRD
8032 if (isSMRD(MI)) {
8033 legalizeOperandsSMRD(MRI, MI);
8034 return CreatedBB;
8035 }
8036
8037 // Legalize FLAT
8038 if (isFLAT(MI)) {
8039 legalizeOperandsFLAT(MRI, MI);
8040 return CreatedBB;
8041 }
8042
8043 // The GPR_IDX form needs its index in an SGPR. For the movrel form, a
8044 // divergent M0 write is waterfalled where the copy into M0 is lowered.
8045 if (auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(Val: &MI)) {
8046 if (!LdStIdx->isGPRIdx())
8047 return CreatedBB;
8048 MachineOperand *Idx = &LdStIdx->getIdxOp();
8049 if (!RI.isSGPRReg(MRI, Reg: Idx->getReg()))
8050 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Idx}, MDT);
8051 return CreatedBB;
8052 }
8053
8054 // REG_SEQUENCE doesn't really require operand legalization, but if one has a
8055 // VGPR dest type and SGPR sources, insert copies so all operands are
8056 // VGPRs. This seems to help operand folding / the register coalescer.
8057 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
8058 MachineBasicBlock *MBB = MI.getParent();
8059 const TargetRegisterClass *DstRC = getOpRegClass(MI, OpNo: 0);
8060 if (RI.hasVGPRs(RC: DstRC)) {
8061 // Update all the operands so they are VGPR register classes. These may
8062 // not be the same register class because REG_SEQUENCE supports mixing
8063 // subregister index types e.g. sub0_sub1 + sub2 + sub3
8064 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
8065 MachineOperand &Op = MI.getOperand(i: I);
8066 if (!Op.isReg() || !Op.getReg().isVirtual())
8067 continue;
8068
8069 const TargetRegisterClass *OpRC = MRI.getRegClass(Reg: Op.getReg());
8070 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(SRC: OpRC);
8071 if (VRC == OpRC)
8072 continue;
8073
8074 legalizeGenericOperand(InsertMBB&: *MBB, I: MI, DstRC: VRC, Op, MRI, DL: MI.getDebugLoc());
8075 }
8076 }
8077
8078 return CreatedBB;
8079 }
8080
8081 // Legalize INSERT_SUBREG
8082 // src0 must have the same register class as dst
8083 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
8084 Register Dst = MI.getOperand(i: 0).getReg();
8085 Register Src0 = MI.getOperand(i: 1).getReg();
8086 const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: Dst);
8087 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0);
8088 if (DstRC != Src0RC) {
8089 MachineBasicBlock *MBB = MI.getParent();
8090 MachineOperand &Op = MI.getOperand(i: 1);
8091 legalizeGenericOperand(InsertMBB&: *MBB, I: MI, DstRC, Op, MRI, DL: MI.getDebugLoc());
8092 }
8093 return CreatedBB;
8094 }
8095
8096 // Legalize SI_INIT_M0
8097 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
8098 MachineOperand &Src = MI.getOperand(i: 0);
8099 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
8100 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
8101 return CreatedBB;
8102 }
8103
8104 // Legalize S_BITREPLICATE, S_QUADMASK and S_WQM
8105 if (MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 ||
8106 MI.getOpcode() == AMDGPU::S_QUADMASK_B32 ||
8107 MI.getOpcode() == AMDGPU::S_QUADMASK_B64 ||
8108 MI.getOpcode() == AMDGPU::S_WQM_B32 ||
8109 MI.getOpcode() == AMDGPU::S_WQM_B64 ||
8110 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U32 ||
8111 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U64) {
8112 MachineOperand &Src = MI.getOperand(i: 1);
8113 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
8114 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
8115 return CreatedBB;
8116 }
8117
8118 // Legalize MIMG/VIMAGE/VSAMPLE and MUBUF/MTBUF for shaders.
8119 //
8120 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
8121 // scratch memory access. In both cases, the legalization never involves
8122 // conversion to the addr64 form.
8123 if (isImage(MI) || (AMDGPU::isGraphics(CC: MF.getFunction().getCallingConv()) &&
8124 (isMUBUF(MI) || isMTBUF(MI)))) {
8125 AMDGPU::OpName RSrcOpName = (isVIMAGE(MI) || isVSAMPLE(MI))
8126 ? AMDGPU::OpName::rsrc
8127 : AMDGPU::OpName::srsrc;
8128 MachineOperand *SRsrc = getNamedOperand(MI, OperandName: RSrcOpName);
8129 if (SRsrc && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SRsrc->getReg())))
8130 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {SRsrc}, MDT);
8131
8132 AMDGPU::OpName SampOpName =
8133 isMIMG(MI) ? AMDGPU::OpName::ssamp : AMDGPU::OpName::samp;
8134 MachineOperand *SSamp = getNamedOperand(MI, OperandName: SampOpName);
8135 if (SSamp && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SSamp->getReg())))
8136 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {SSamp}, MDT);
8137
8138 return CreatedBB;
8139 }
8140
8141 // Legalize SI_CALL
8142 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) {
8143 MachineOperand *Dest = &MI.getOperand(i: 0);
8144 if (!RI.isSGPRClass(RC: MRI.getRegClass(Reg: Dest->getReg()))) {
8145 createWaterFallForSiCall(MI: &MI, MDT, ScalarOps: {Dest});
8146 }
8147 }
8148
8149 // Legalize s_sleep_var.
8150 if (MI.getOpcode() == AMDGPU::S_SLEEP_VAR) {
8151 const DebugLoc &DL = MI.getDebugLoc();
8152 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
8153 int Src0Idx =
8154 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
8155 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
8156 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
8157 .add(MO: Src0);
8158 Src0.ChangeToRegister(Reg, isDef: false);
8159 return nullptr;
8160 }
8161
8162 // Legalize TENSOR_LOAD_TO_LDS_d2/_d4, TENSOR_STORE_FROM_LDS_d2/_d4. All their
8163 // operands are scalar.
8164 if (MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_d2 ||
8165 MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_d4 ||
8166 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_d2 ||
8167 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_d4) {
8168 for (MachineOperand &Src : MI.explicit_operands()) {
8169 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
8170 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
8171 }
8172 return CreatedBB;
8173 }
8174
8175 // Legalize MUBUF instructions.
8176 bool isSoffsetLegal = true;
8177 int SoffsetIdx =
8178 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::soffset);
8179 if (SoffsetIdx != -1) {
8180 MachineOperand *Soffset = &MI.getOperand(i: SoffsetIdx);
8181 if (Soffset->isReg() && Soffset->getReg().isVirtual() &&
8182 !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Soffset->getReg()))) {
8183 isSoffsetLegal = false;
8184 }
8185 }
8186
8187 bool isRsrcLegal = true;
8188 int RsrcIdx =
8189 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::srsrc);
8190 if (RsrcIdx != -1) {
8191 MachineOperand *Rsrc = &MI.getOperand(i: RsrcIdx);
8192 if (Rsrc->isReg() && !RI.isSGPRReg(MRI, Reg: Rsrc->getReg()))
8193 isRsrcLegal = false;
8194 }
8195
8196 // The operands are legal.
8197 if (isRsrcLegal && isSoffsetLegal)
8198 return CreatedBB;
8199
8200 if (!isRsrcLegal) {
8201 // Legalize a VGPR Rsrc
8202 //
8203 // If the instruction is _ADDR64, we can avoid a waterfall by extracting
8204 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
8205 // a zero-value SRsrc.
8206 //
8207 // If the instruction is _OFFSET (both idxen and offen disabled), and we
8208 // support ADDR64 instructions, we can convert to ADDR64 and do the same as
8209 // above.
8210 //
8211 // Otherwise we are on non-ADDR64 hardware, and/or we have
8212 // idxen/offen/bothen and we fall back to a waterfall loop.
8213
8214 MachineOperand *Rsrc = &MI.getOperand(i: RsrcIdx);
8215 MachineBasicBlock &MBB = *MI.getParent();
8216
8217 MachineOperand *VAddr = getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
8218 if (VAddr && AMDGPU::getIfAddr64Inst(Opcode: MI.getOpcode()) != -1) {
8219 // This is already an ADDR64 instruction so we need to add the pointer
8220 // extracted from the resource descriptor to the current value of VAddr.
8221 Register NewVAddrLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8222 Register NewVAddrHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8223 Register NewVAddr = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
8224
8225 const auto *BoolXExecRC = RI.getWaveMaskRegClass();
8226 Register CondReg0 = MRI.createVirtualRegister(RegClass: BoolXExecRC);
8227 Register CondReg1 = MRI.createVirtualRegister(RegClass: BoolXExecRC);
8228
8229 unsigned RsrcPtr, NewSRsrc;
8230 std::tie(args&: RsrcPtr, args&: NewSRsrc) = extractRsrcPtr(TII: *this, MI, Rsrc&: *Rsrc);
8231
8232 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
8233 const DebugLoc &DL = MI.getDebugLoc();
8234 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg: NewVAddrLo)
8235 .addDef(RegNo: CondReg0)
8236 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub0)
8237 .addReg(RegNo: VAddr->getReg(), Flags: {}, SubReg: AMDGPU::sub0)
8238 .addImm(Val: 0);
8239
8240 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
8241 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADDC_U32_e64), DestReg: NewVAddrHi)
8242 .addDef(RegNo: CondReg1, Flags: RegState::Dead)
8243 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub1)
8244 .addReg(RegNo: VAddr->getReg(), Flags: {}, SubReg: AMDGPU::sub1)
8245 .addReg(RegNo: CondReg0)
8246 .addImm(Val: 0);
8247
8248 // NewVaddr = {NewVaddrHi, NewVaddrLo}
8249 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewVAddr)
8250 .addReg(RegNo: NewVAddrLo)
8251 .addImm(Val: AMDGPU::sub0)
8252 .addReg(RegNo: NewVAddrHi)
8253 .addImm(Val: AMDGPU::sub1);
8254
8255 VAddr->setReg(NewVAddr);
8256 Rsrc->setReg(NewSRsrc);
8257 } else if (!VAddr && ST.hasAddr64()) {
8258 // This instructions is the _OFFSET variant, so we need to convert it to
8259 // ADDR64.
8260 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS &&
8261 "FIXME: Need to emit flat atomics here");
8262
8263 unsigned RsrcPtr, NewSRsrc;
8264 std::tie(args&: RsrcPtr, args&: NewSRsrc) = extractRsrcPtr(TII: *this, MI, Rsrc&: *Rsrc);
8265
8266 Register NewVAddr = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
8267 MachineOperand *VData = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata);
8268 MachineOperand *Offset = getNamedOperand(MI, OperandName: AMDGPU::OpName::offset);
8269 MachineOperand *SOffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
8270 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(Opcode: MI.getOpcode());
8271
8272 // Atomics with return have an additional tied operand and are
8273 // missing some of the special bits.
8274 MachineOperand *VDataIn = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata_in);
8275 MachineInstr *Addr64;
8276
8277 if (!VDataIn) {
8278 // Regular buffer load / store.
8279 MachineInstrBuilder MIB =
8280 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Addr64Opcode))
8281 .add(MO: *VData)
8282 .addReg(RegNo: NewVAddr)
8283 .addReg(RegNo: NewSRsrc)
8284 .add(MO: *SOffset)
8285 .add(MO: *Offset);
8286
8287 if (const MachineOperand *CPol =
8288 getNamedOperand(MI, OperandName: AMDGPU::OpName::cpol)) {
8289 MIB.addImm(Val: CPol->getImm());
8290 }
8291
8292 if (const MachineOperand *TFE =
8293 getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe)) {
8294 MIB.addImm(Val: TFE->getImm());
8295 }
8296
8297 MIB.addImm(Val: getNamedImmOperand(MI, OperandName: AMDGPU::OpName::swz));
8298
8299 MIB.cloneMemRefs(OtherMI: MI);
8300 Addr64 = MIB;
8301 } else {
8302 // Atomics with return.
8303 Addr64 = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Addr64Opcode))
8304 .add(MO: *VData)
8305 .add(MO: *VDataIn)
8306 .addReg(RegNo: NewVAddr)
8307 .addReg(RegNo: NewSRsrc)
8308 .add(MO: *SOffset)
8309 .add(MO: *Offset)
8310 .addImm(Val: getNamedImmOperand(MI, OperandName: AMDGPU::OpName::cpol))
8311 .cloneMemRefs(OtherMI: MI);
8312 }
8313
8314 MI.removeFromParent();
8315
8316 // NewVaddr = {NewVaddrHi, NewVaddrLo}
8317 BuildMI(BB&: MBB, I: Addr64, MIMD: Addr64->getDebugLoc(), MCID: get(Opcode: AMDGPU::REG_SEQUENCE),
8318 DestReg: NewVAddr)
8319 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub0)
8320 .addImm(Val: AMDGPU::sub0)
8321 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub1)
8322 .addImm(Val: AMDGPU::sub1);
8323 } else {
8324 // Legalize a VGPR Rsrc and soffset together.
8325 if (!isSoffsetLegal) {
8326 MachineOperand *Soffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
8327 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Rsrc, Soffset}, MDT);
8328 return CreatedBB;
8329 }
8330 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Rsrc}, MDT);
8331 return CreatedBB;
8332 }
8333 }
8334
8335 // Legalize a VGPR soffset.
8336 if (!isSoffsetLegal) {
8337 MachineOperand *Soffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
8338 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Soffset}, MDT);
8339 return CreatedBB;
8340 }
8341 return CreatedBB;
8342}
8343
8344void SIInstrWorklist::insert(MachineInstr *MI) {
8345 if (InSet.insert(Ptr: MI).second)
8346 InstrList.push_back(Elt: MI);
8347 // Add MBUF instructiosn to deferred list.
8348 int RsrcIdx =
8349 AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(), Name: AMDGPU::OpName::srsrc);
8350 if (RsrcIdx != -1) {
8351 DeferredList.insert(X: MI);
8352 }
8353}
8354
8355bool SIInstrWorklist::isDeferred(MachineInstr *MI) {
8356 return DeferredList.contains(key: MI);
8357}
8358
8359// Legalize size mismatches between 16bit and 32bit registers in v2s copy
8360// lowering (change sgpr to vgpr).
8361// This is mainly caused by 16bit SALU and 16bit VALU using reg with different
8362// size. Need to legalize the size of the operands during the vgpr lowering
8363// chain. This can be removed after we have sgpr16 in place
8364void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &MI, unsigned OpIdx,
8365 MachineRegisterInfo &MRI) const {
8366 if (!ST.useRealTrue16Insts())
8367 return;
8368
8369 unsigned Opcode = MI.getOpcode();
8370 MachineBasicBlock *MBB = MI.getParent();
8371 // Legalize operands and check for size mismatch
8372 if (OpIdx >= MI.getNumExplicitOperands() ||
8373 OpIdx >= get(Opcode).getNumOperands() ||
8374 get(Opcode).operands()[OpIdx].RegClass == -1)
8375 return;
8376
8377 MachineOperand &Op = MI.getOperand(i: OpIdx);
8378 if (!Op.isReg() || !Op.getReg().isVirtual() || Op.isDef())
8379 return;
8380
8381 const TargetRegisterClass *CurrRC = MRI.getRegClass(Reg: Op.getReg());
8382 if (!RI.isVGPRClass(RC: CurrRC))
8383 return;
8384
8385 int16_t RCID = getOpRegClassID(OpInfo: get(Opcode).operands()[OpIdx]);
8386 const TargetRegisterClass *ExpectedRC = RI.getRegClass(i: RCID);
8387 if (RI.getMatchingSuperRegClass(A: CurrRC, B: ExpectedRC, Idx: AMDGPU::lo16)) {
8388 // Default to the lo16 only if the subregister is not specified.
8389 if (Op.getSubReg() == AMDGPU::NoSubRegister)
8390 Op.setSubReg(AMDGPU::lo16);
8391 return;
8392 }
8393
8394 const TargetRegisterClass *CurrSRC =
8395 RI.getSubRegisterClass(CurrRC, Op.getSubReg());
8396 if (RI.getMatchingSuperRegClass(A: ExpectedRC, B: CurrSRC, Idx: AMDGPU::lo16)) {
8397 Op.setReg(buildVGPR32FromLo16(MBB&: *MBB, I: MI, DL: MI.getDebugLoc(), Src: Op.getReg(),
8398 SrcSubReg: Op.getSubReg()));
8399 Op.setSubReg(AMDGPU::NoSubRegister);
8400 }
8401}
8402void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &MI,
8403 MachineRegisterInfo &MRI) const {
8404 for (unsigned OpIdx = 0; OpIdx < MI.getNumExplicitOperands(); OpIdx++)
8405 legalizeOperandsVALUt16(MI, OpIdx, MRI);
8406}
8407
8408Register SIInstrInfo::buildVGPR32FromLo16(MachineBasicBlock &MBB,
8409 MachineBasicBlock::iterator I,
8410 const DebugLoc &DL, Register Src,
8411 unsigned SrcSubReg) const {
8412 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8413 Register Dst = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8414 Register Undef = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
8415 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
8416 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
8417 .addReg(RegNo: Src, Flags: {}, SubReg: SrcSubReg)
8418 .addImm(Val: AMDGPU::lo16)
8419 .addReg(RegNo: Undef)
8420 .addImm(Val: AMDGPU::hi16);
8421 return Dst;
8422}
8423
8424void SIInstrInfo::createWaterFallForSiCall(MachineInstr *MI,
8425 MachineDominatorTree *MDT,
8426 ArrayRef<MachineOperand *> ScalarOps,
8427 ArrayRef<Register> PhySGPRs) const {
8428 assert(MI->getOpcode() == AMDGPU::SI_CALL_ISEL &&
8429 "This only handle waterfall for SI_CALL_ISEL");
8430 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and
8431 // following copies, we also need to move copies from and to physical
8432 // registers into the loop block.
8433 // Also move the copies to physical registers into the loop block
8434 MachineBasicBlock &MBB = *MI->getParent();
8435 MachineBasicBlock::iterator Start(MI);
8436 while (Start->getOpcode() != AMDGPU::ADJCALLSTACKUP)
8437 --Start;
8438 MachineBasicBlock::iterator End(MI);
8439 while (End->getOpcode() != AMDGPU::ADJCALLSTACKDOWN)
8440 ++End;
8441
8442 // Also include following copies of the return value
8443 ++End;
8444 while (End != MBB.end() && End->isCopy() &&
8445 MI->definesRegister(Reg: End->getOperand(i: 1).getReg(), TRI: &RI))
8446 ++End;
8447
8448 generateWaterFallLoop(TII: *this, MI&: *MI, ScalarOps, MDT, Begin: Start, End, PhySGPRs);
8449}
8450
8451void SIInstrInfo::moveToVALU(SIInstrWorklist &Worklist,
8452 MachineDominatorTree *MDT) const {
8453 DenseMap<MachineInstr *, V2PhysSCopyInfo> WaterFalls;
8454 DenseMap<MachineInstr *, bool> V2SPhyCopiesToErase;
8455 while (!Worklist.empty()) {
8456 MachineInstr &Inst = *Worklist.top();
8457 Worklist.erase_top();
8458 // Skip MachineInstr in the deferred list.
8459 if (Worklist.isDeferred(MI: &Inst))
8460 continue;
8461 moveToVALUImpl(Worklist, MDT, Inst, WaterFalls, V2SPhyCopiesToErase);
8462 }
8463
8464 // Deferred list of instructions will be processed once
8465 // all the MachineInstr in the worklist are done.
8466 for (MachineInstr *Inst : Worklist.getDeferredList()) {
8467 moveToVALUImpl(Worklist, MDT, Inst&: *Inst, WaterFalls, V2SPhyCopiesToErase);
8468 assert(Worklist.empty() &&
8469 "Deferred MachineInstr are not supposed to re-populate worklist");
8470 }
8471
8472 for (auto &Entry : WaterFalls) {
8473 if (Entry.first->getOpcode() == AMDGPU::SI_CALL_ISEL)
8474 createWaterFallForSiCall(MI: Entry.first, MDT, ScalarOps: Entry.second.MOs,
8475 PhySGPRs: Entry.second.SGPRs);
8476 }
8477
8478 for (std::pair<MachineInstr *, bool> Entry : V2SPhyCopiesToErase)
8479 if (Entry.second)
8480 Entry.first->eraseFromParent();
8481}
8482void SIInstrInfo::createReadFirstLaneFromCopyToPhysReg(
8483 MachineRegisterInfo &MRI, Register DstReg, MachineInstr &Inst) const {
8484 // If it's a copy of a VGPR to a physical SGPR, insert a V_READFIRSTLANE and
8485 // hope for the best.
8486 const TargetRegisterClass *DstRC = RI.getRegClassForReg(MRI, Reg: DstReg);
8487 ArrayRef<int16_t> SubRegIndices = RI.getRegSplitParts(RC: DstRC, EltSize: 4);
8488 if (SubRegIndices.size() <= 1) {
8489 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
8490 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8491 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: NewDst)
8492 .add(MO: Inst.getOperand(i: 1));
8493 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
8494 DestReg: DstReg)
8495 .addReg(RegNo: NewDst);
8496 } else {
8497 SmallVector<Register, 8> DstRegs;
8498 for (int16_t Indice : SubRegIndices) {
8499 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
8500 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8501 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: NewDst)
8502 .addReg(RegNo: Inst.getOperand(i: 1).getReg(), Flags: {}, SubReg: Indice);
8503
8504 DstRegs.push_back(Elt: NewDst);
8505 }
8506 MachineInstrBuilder MIB =
8507 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8508 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
8509 for (unsigned i = 0; i < SubRegIndices.size(); ++i) {
8510 MIB.addReg(RegNo: DstRegs[i]);
8511 MIB.addImm(Val: RI.getSubRegFromChannel(Channel: i));
8512 }
8513 }
8514}
8515
8516// M0 holds one value, but the VGPR copied into it holds one per lane. A movrel
8517// access adds M0 to its base register, so each one that reads the copy is
8518// wrapped in a waterfall loop that puts the value of each group of lanes in M0
8519// in turn. Every other reader gets the first lane, as all of them do when no
8520// access reads the copy.
8521static void lowerVGPRCopyToM0(const SIInstrInfo &TII, MachineInstr &Copy,
8522 ArrayRef<MachineInstr *> Readers,
8523 MachineRegisterInfo &MRI,
8524 MachineDominatorTree *MDT) {
8525 auto IsMovrelAccess = [](const MachineInstr *MI) {
8526 const auto *LdStIdx = dyn_cast<AMDGPUMI::VLoadStoreIdxInst>(Val: MI);
8527 return LdStIdx && !LdStIdx->isGPRIdx();
8528 };
8529 if (none_of(Range&: Readers, P: IsMovrelAccess)) {
8530 TII.createReadFirstLaneFromCopyToPhysReg(MRI, DstReg: AMDGPU::M0, Inst&: Copy);
8531 return;
8532 }
8533
8534 assert(!Copy.getOperand(1).getSubReg() &&
8535 "selection copies a whole register into M0");
8536 Register Idx = Copy.getOperand(i: 1).getReg();
8537 Register FirstLane;
8538 for (MachineInstr *Reader : Readers) {
8539 if (IsMovrelAccess(Reader)) {
8540 MachineOperand *M0Use =
8541 Reader->findRegisterUseOperand(Reg: AMDGPU::M0, TRI: &TII.getRegisterInfo());
8542 M0Use->setReg(Idx);
8543 generateWaterFallLoop(TII, MI&: *Reader, ScalarOps: M0Use, MDT, Begin: nullptr, End: nullptr,
8544 PhySGPRs: {AMDGPU::M0});
8545 continue;
8546 }
8547
8548 if (!FirstLane) {
8549 FirstLane = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
8550 BuildMI(BB&: *Copy.getParent(), I&: Copy, MIMD: Copy.getDebugLoc(),
8551 MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: FirstLane)
8552 .addReg(RegNo: Idx);
8553 }
8554 BuildMI(BB&: *Reader->getParent(), I: Reader, MIMD: Reader->getDebugLoc(),
8555 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
8556 .addReg(RegNo: FirstLane);
8557 }
8558}
8559
8560void SIInstrInfo::handleCopyToPhysHelper(
8561 SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst,
8562 MachineRegisterInfo &MRI, MachineDominatorTree *MDT,
8563 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
8564 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const {
8565 // In SSA a copy into a physical register and its readers share a block, so
8566 // only the rest of this block is searched.
8567 SmallVector<MachineInstr *, 4> Readers;
8568 for (MachineInstr &MI :
8569 instructionsWithoutDebug(It: std::next(x: MachineBasicBlock::iterator(Inst)),
8570 End: Inst.getParent()->end())) {
8571 if (MI.readsRegister(Reg: DstReg, TRI: &RI))
8572 Readers.push_back(Elt: &MI);
8573
8574 if (MI.modifiesRegister(Reg: DstReg, TRI: &RI))
8575 break;
8576 }
8577
8578 if (DstReg == AMDGPU::M0) {
8579 lowerVGPRCopyToM0(TII: *this, Copy&: Inst, Readers, MRI, MDT);
8580 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
8581 return;
8582 }
8583
8584 Register SrcReg = Inst.getOperand(i: 1).getReg();
8585 for (MachineInstr *UseMI : Readers) {
8586 // For SI_CALL_ISEL users, replace the phys SGPR with the VGPR source
8587 // and record the operand for later waterfall loop generation.
8588 if (UseMI->getOpcode() == AMDGPU::SI_CALL_ISEL) {
8589 for (unsigned i = 0; i < UseMI->getNumOperands(); ++i) {
8590 if (UseMI->getOperand(i).isReg() &&
8591 UseMI->getOperand(i).getReg() == DstReg) {
8592 MachineOperand *MO = &UseMI->getOperand(i);
8593 MO->setReg(SrcReg);
8594 V2PhysSCopyInfo &V2SCopyInfo = WaterFalls[UseMI];
8595 V2SCopyInfo.MOs.push_back(Elt: MO);
8596 V2SCopyInfo.SGPRs.push_back(Elt: DstReg);
8597 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
8598 }
8599 }
8600 } else if (UseMI->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG &&
8601 UseMI->getOperand(i: 0).isReg() &&
8602 UseMI->getOperand(i: 0).getReg() == DstReg) {
8603 createReadFirstLaneFromCopyToPhysReg(MRI, DstReg, Inst);
8604 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
8605 } else {
8606 // COPY cannot be erased if other type of inst uses it.
8607 V2SPhyCopiesToErase[&Inst] = false;
8608 }
8609 }
8610}
8611
8612void SIInstrInfo::moveToVALUImpl(
8613 SIInstrWorklist &Worklist, MachineDominatorTree *MDT, MachineInstr &Inst,
8614 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
8615 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const {
8616
8617 MachineBasicBlock *MBB = Inst.getParent();
8618 if (!MBB)
8619 return;
8620 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
8621 unsigned Opcode = Inst.getOpcode();
8622 unsigned NewOpcode = getVALUOp(MI: Inst);
8623 const DebugLoc &DL = Inst.getDebugLoc();
8624
8625 // Handle some special cases
8626 switch (Opcode) {
8627 default:
8628 break;
8629 case AMDGPU::S_ADD_I32:
8630 case AMDGPU::S_SUB_I32: {
8631 // FIXME: The u32 versions currently selected use the carry.
8632 bool Changed;
8633 MachineBasicBlock *CreatedBBTmp = nullptr;
8634 std::tie(args&: Changed, args&: CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT);
8635 if (Changed)
8636 return;
8637
8638 // Default handling
8639 break;
8640 }
8641
8642 case AMDGPU::S_MUL_U64:
8643 if (ST.useVMulU64Inst()) {
8644 NewOpcode = AMDGPU::V_MUL_U64_e64;
8645 break;
8646 }
8647 // Split s_mul_u64 in 32-bit vector multiplications.
8648 splitScalarSMulU64(Worklist, Inst, MDT);
8649 Inst.eraseFromParent();
8650 return;
8651
8652 case AMDGPU::S_MUL_U64_U32_PSEUDO:
8653 case AMDGPU::S_MUL_I64_I32_PSEUDO:
8654 // This is a special case of s_mul_u64 where all the operands are either
8655 // zero extended or sign extended.
8656 splitScalarSMulPseudo(Worklist, Inst, MDT);
8657 Inst.eraseFromParent();
8658 return;
8659
8660 case AMDGPU::S_AND_B64:
8661 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_AND_B32, MDT);
8662 Inst.eraseFromParent();
8663 return;
8664
8665 case AMDGPU::S_OR_B64:
8666 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_OR_B32, MDT);
8667 Inst.eraseFromParent();
8668 return;
8669
8670 case AMDGPU::S_XOR_B64:
8671 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_XOR_B32, MDT);
8672 Inst.eraseFromParent();
8673 return;
8674
8675 case AMDGPU::S_NAND_B64:
8676 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_NAND_B32, MDT);
8677 Inst.eraseFromParent();
8678 return;
8679
8680 case AMDGPU::S_NOR_B64:
8681 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_NOR_B32, MDT);
8682 Inst.eraseFromParent();
8683 return;
8684
8685 case AMDGPU::S_XNOR_B64:
8686 if (ST.hasDLInsts())
8687 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_XNOR_B32, MDT);
8688 else
8689 splitScalar64BitXnor(Worklist, Inst, MDT);
8690 Inst.eraseFromParent();
8691 return;
8692
8693 case AMDGPU::S_ANDN2_B64:
8694 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_ANDN2_B32, MDT);
8695 Inst.eraseFromParent();
8696 return;
8697
8698 case AMDGPU::S_ORN2_B64:
8699 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_ORN2_B32, MDT);
8700 Inst.eraseFromParent();
8701 return;
8702
8703 case AMDGPU::S_BREV_B64:
8704 splitScalar64BitUnaryOp(Worklist, Inst, Opcode: AMDGPU::S_BREV_B32, Swap: true);
8705 Inst.eraseFromParent();
8706 return;
8707
8708 case AMDGPU::S_NOT_B64:
8709 splitScalar64BitUnaryOp(Worklist, Inst, Opcode: AMDGPU::S_NOT_B32);
8710 Inst.eraseFromParent();
8711 return;
8712
8713 case AMDGPU::S_BCNT1_I32_B64:
8714 splitScalar64BitBCNT(Worklist, Inst);
8715 Inst.eraseFromParent();
8716 return;
8717
8718 case AMDGPU::S_BFE_I64:
8719 splitScalar64BitBFE(Worklist, Inst);
8720 Inst.eraseFromParent();
8721 return;
8722
8723 case AMDGPU::S_FLBIT_I32_B64:
8724 splitScalar64BitCountOp(Worklist, Inst, Opcode: AMDGPU::V_FFBH_U32_e32);
8725 Inst.eraseFromParent();
8726 return;
8727 case AMDGPU::S_FF1_I32_B64:
8728 splitScalar64BitCountOp(Worklist, Inst, Opcode: AMDGPU::V_FFBL_B32_e32);
8729 Inst.eraseFromParent();
8730 return;
8731
8732 case AMDGPU::S_LSHL_B32:
8733 if (ST.hasOnlyRevVALUShifts()) {
8734 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
8735 swapOperands(Inst);
8736 }
8737 break;
8738 case AMDGPU::S_ASHR_I32:
8739 if (ST.hasOnlyRevVALUShifts()) {
8740 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
8741 swapOperands(Inst);
8742 }
8743 break;
8744 case AMDGPU::S_LSHR_B32:
8745 if (ST.hasOnlyRevVALUShifts()) {
8746 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
8747 swapOperands(Inst);
8748 }
8749 break;
8750 case AMDGPU::S_LSHL_B64:
8751 if (ST.hasOnlyRevVALUShifts()) {
8752 NewOpcode = ST.getGeneration() >= AMDGPUSubtarget::GFX12
8753 ? AMDGPU::V_LSHLREV_B64_pseudo_e64
8754 : AMDGPU::V_LSHLREV_B64_e64;
8755 swapOperands(Inst);
8756 }
8757 break;
8758 case AMDGPU::S_ASHR_I64:
8759 if (ST.hasOnlyRevVALUShifts()) {
8760 NewOpcode = AMDGPU::V_ASHRREV_I64_e64;
8761 swapOperands(Inst);
8762 }
8763 break;
8764 case AMDGPU::S_LSHR_B64:
8765 if (ST.hasOnlyRevVALUShifts()) {
8766 NewOpcode = AMDGPU::V_LSHRREV_B64_e64;
8767 swapOperands(Inst);
8768 }
8769 break;
8770
8771 case AMDGPU::S_ABS_I32:
8772 lowerScalarAbs(Worklist, Inst);
8773 Inst.eraseFromParent();
8774 return;
8775
8776 case AMDGPU::S_ABSDIFF_I32:
8777 lowerScalarAbsDiff(Worklist, Inst);
8778 Inst.eraseFromParent();
8779 return;
8780
8781 case AMDGPU::S_CBRANCH_SCC0:
8782 case AMDGPU::S_CBRANCH_SCC1: {
8783 // Clear unused bits of vcc
8784 Register CondReg = Inst.getOperand(i: 1).getReg();
8785 bool IsSCC = CondReg == AMDGPU::SCC;
8786 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
8787 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: LMC.AndOpc), DestReg: LMC.VccReg)
8788 .addReg(RegNo: LMC.ExecReg)
8789 .addReg(RegNo: IsSCC ? LMC.VccReg : CondReg)
8790 .setOperandDead(3); // implicit-def $scc
8791 Inst.removeOperand(OpNo: 1);
8792 } break;
8793
8794 case AMDGPU::S_BFE_U64:
8795 case AMDGPU::S_BFM_B64:
8796 llvm_unreachable("Moving this op to VALU not implemented");
8797
8798 case AMDGPU::S_PACK_LL_B32_B16:
8799 case AMDGPU::S_PACK_LH_B32_B16:
8800 case AMDGPU::S_PACK_HL_B32_B16:
8801 case AMDGPU::S_PACK_HH_B32_B16:
8802 movePackToVALU(Worklist, MRI, Inst);
8803 Inst.eraseFromParent();
8804 return;
8805
8806 case AMDGPU::S_XNOR_B32:
8807 lowerScalarXnor(Worklist, Inst);
8808 Inst.eraseFromParent();
8809 return;
8810
8811 case AMDGPU::S_NAND_B32:
8812 splitScalarNotBinop(Worklist, Inst, Opcode: AMDGPU::S_AND_B32);
8813 Inst.eraseFromParent();
8814 return;
8815
8816 case AMDGPU::S_NOR_B32:
8817 splitScalarNotBinop(Worklist, Inst, Opcode: AMDGPU::S_OR_B32);
8818 Inst.eraseFromParent();
8819 return;
8820
8821 case AMDGPU::S_ANDN2_B32:
8822 splitScalarBinOpN2(Worklist, Inst, Opcode: AMDGPU::S_AND_B32);
8823 Inst.eraseFromParent();
8824 return;
8825
8826 case AMDGPU::S_ORN2_B32:
8827 splitScalarBinOpN2(Worklist, Inst, Opcode: AMDGPU::S_OR_B32);
8828 Inst.eraseFromParent();
8829 return;
8830
8831 // TODO: remove as soon as everything is ready
8832 // to replace VGPR to SGPR copy with V_READFIRSTLANEs.
8833 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO
8834 // can only be selected from the uniform SDNode.
8835 case AMDGPU::S_ADD_CO_PSEUDO:
8836 case AMDGPU::S_SUB_CO_PSEUDO: {
8837 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
8838 ? AMDGPU::V_ADDC_U32_e64
8839 : AMDGPU::V_SUBB_U32_e64;
8840 const auto *CarryRC = RI.getWaveMaskRegClass();
8841
8842 Register CarryInReg = Inst.getOperand(i: 4).getReg();
8843 if (!MRI.constrainRegClass(Reg: CarryInReg, RC: CarryRC)) {
8844 Register NewCarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
8845 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY), DestReg: NewCarryReg)
8846 .addReg(RegNo: CarryInReg);
8847 }
8848
8849 Register CarryOutReg = Inst.getOperand(i: 1).getReg();
8850
8851 Register DestReg = MRI.createVirtualRegister(RegClass: RI.getEquivalentVGPRClass(
8852 SRC: MRI.getRegClass(Reg: Inst.getOperand(i: 0).getReg())));
8853 MachineInstr *CarryOp =
8854 BuildMI(BB&: *MBB, I: &Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: Opc), DestReg)
8855 .addReg(RegNo: CarryOutReg, Flags: RegState::Define)
8856 .add(MO: Inst.getOperand(i: 2))
8857 .add(MO: Inst.getOperand(i: 3))
8858 .addReg(RegNo: CarryInReg)
8859 .addImm(Val: 0);
8860 legalizeOperands(MI&: *CarryOp);
8861 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: DestReg);
8862 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8863 Inst.eraseFromParent();
8864 }
8865 return;
8866 case AMDGPU::S_UADDO_PSEUDO:
8867 case AMDGPU::S_USUBO_PSEUDO: {
8868 MachineOperand &Dest0 = Inst.getOperand(i: 0);
8869 MachineOperand &Dest1 = Inst.getOperand(i: 1);
8870 MachineOperand &Src0 = Inst.getOperand(i: 2);
8871 MachineOperand &Src1 = Inst.getOperand(i: 3);
8872
8873 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
8874 ? AMDGPU::V_ADD_CO_U32_e64
8875 : AMDGPU::V_SUB_CO_U32_e64;
8876 const TargetRegisterClass *NewRC =
8877 RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest0.getReg()));
8878 Register DestReg = MRI.createVirtualRegister(RegClass: NewRC);
8879 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I: &Inst, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
8880 .addReg(RegNo: Dest1.getReg(), Flags: RegState::Define)
8881 .add(MO: Src0)
8882 .add(MO: Src1)
8883 .addImm(Val: 0); // clamp bit
8884
8885 legalizeOperands(MI&: *NewInstr, MDT);
8886 MRI.replaceRegWith(FromReg: Dest0.getReg(), ToReg: DestReg);
8887 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8888 Inst.eraseFromParent();
8889 }
8890 return;
8891 case AMDGPU::S_LSHL1_ADD_U32:
8892 case AMDGPU::S_LSHL2_ADD_U32:
8893 case AMDGPU::S_LSHL3_ADD_U32:
8894 case AMDGPU::S_LSHL4_ADD_U32: {
8895 MachineOperand &Dest = Inst.getOperand(i: 0);
8896 MachineOperand &Src0 = Inst.getOperand(i: 1);
8897 MachineOperand &Src1 = Inst.getOperand(i: 2);
8898 unsigned ShiftAmt = (Opcode == AMDGPU::S_LSHL1_ADD_U32 ? 1
8899 : Opcode == AMDGPU::S_LSHL2_ADD_U32 ? 2
8900 : Opcode == AMDGPU::S_LSHL3_ADD_U32 ? 3
8901 : 4);
8902
8903 const TargetRegisterClass *NewRC =
8904 RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest.getReg()));
8905 Register DestReg = MRI.createVirtualRegister(RegClass: NewRC);
8906 MachineInstr *NewInstr =
8907 BuildMI(BB&: *MBB, I: &Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_ADD_U32_e64), DestReg)
8908 .add(MO: Src0)
8909 .addImm(Val: ShiftAmt)
8910 .add(MO: Src1);
8911
8912 legalizeOperands(MI&: *NewInstr, MDT);
8913 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: DestReg);
8914 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8915 Inst.eraseFromParent();
8916 }
8917 return;
8918 case AMDGPU::S_CSELECT_B32:
8919 case AMDGPU::S_CSELECT_B64:
8920 lowerSelect(Worklist, Inst, MDT);
8921 Inst.eraseFromParent();
8922 return;
8923 case AMDGPU::S_CMP_EQ_I32:
8924 case AMDGPU::S_CMP_LG_I32:
8925 case AMDGPU::S_CMP_GT_I32:
8926 case AMDGPU::S_CMP_GE_I32:
8927 case AMDGPU::S_CMP_LT_I32:
8928 case AMDGPU::S_CMP_LE_I32:
8929 case AMDGPU::S_CMP_EQ_U32:
8930 case AMDGPU::S_CMP_LG_U32:
8931 case AMDGPU::S_CMP_GT_U32:
8932 case AMDGPU::S_CMP_GE_U32:
8933 case AMDGPU::S_CMP_LT_U32:
8934 case AMDGPU::S_CMP_LE_U32:
8935 case AMDGPU::S_CMP_EQ_U64:
8936 case AMDGPU::S_CMP_LG_U64:
8937 case AMDGPU::S_CMP_LT_F32:
8938 case AMDGPU::S_CMP_EQ_F32:
8939 case AMDGPU::S_CMP_LE_F32:
8940 case AMDGPU::S_CMP_GT_F32:
8941 case AMDGPU::S_CMP_LG_F32:
8942 case AMDGPU::S_CMP_GE_F32:
8943 case AMDGPU::S_CMP_O_F32:
8944 case AMDGPU::S_CMP_U_F32:
8945 case AMDGPU::S_CMP_NGE_F32:
8946 case AMDGPU::S_CMP_NLG_F32:
8947 case AMDGPU::S_CMP_NGT_F32:
8948 case AMDGPU::S_CMP_NLE_F32:
8949 case AMDGPU::S_CMP_NEQ_F32:
8950 case AMDGPU::S_CMP_NLT_F32: {
8951 Register CondReg = MRI.createVirtualRegister(RegClass: RI.getWaveMaskRegClass());
8952 auto NewInstr =
8953 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode), DestReg: CondReg)
8954 .setMIFlags(Inst.getFlags());
8955 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src0_modifiers) >=
8956 0) {
8957 NewInstr
8958 .addImm(Val: 0) // src0_modifiers
8959 .add(MO: Inst.getOperand(i: 0)) // src0
8960 .addImm(Val: 0) // src1_modifiers
8961 .add(MO: Inst.getOperand(i: 1)) // src1
8962 .addImm(Val: 0); // clamp
8963 } else {
8964 NewInstr.add(MO: Inst.getOperand(i: 0)).add(MO: Inst.getOperand(i: 1));
8965 }
8966 legalizeOperands(MI&: *NewInstr, MDT);
8967 int SCCIdx = Inst.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
8968 const MachineOperand &SCCOp = Inst.getOperand(i: SCCIdx);
8969 addSCCDefUsersToVALUWorklist(Op: SCCOp, SCCDefInst&: Inst, Worklist, NewCond: CondReg);
8970 Inst.eraseFromParent();
8971 return;
8972 }
8973 case AMDGPU::S_CMP_LT_F16:
8974 case AMDGPU::S_CMP_EQ_F16:
8975 case AMDGPU::S_CMP_LE_F16:
8976 case AMDGPU::S_CMP_GT_F16:
8977 case AMDGPU::S_CMP_LG_F16:
8978 case AMDGPU::S_CMP_GE_F16:
8979 case AMDGPU::S_CMP_O_F16:
8980 case AMDGPU::S_CMP_U_F16:
8981 case AMDGPU::S_CMP_NGE_F16:
8982 case AMDGPU::S_CMP_NLG_F16:
8983 case AMDGPU::S_CMP_NGT_F16:
8984 case AMDGPU::S_CMP_NLE_F16:
8985 case AMDGPU::S_CMP_NEQ_F16:
8986 case AMDGPU::S_CMP_NLT_F16: {
8987 Register CondReg = MRI.createVirtualRegister(RegClass: RI.getWaveMaskRegClass());
8988 auto NewInstr =
8989 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode), DestReg: CondReg)
8990 .setMIFlags(Inst.getFlags());
8991 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::src0_modifiers)) {
8992 NewInstr
8993 .addImm(Val: 0) // src0_modifiers
8994 .add(MO: Inst.getOperand(i: 0)) // src0
8995 .addImm(Val: 0) // src1_modifiers
8996 .add(MO: Inst.getOperand(i: 1)) // src1
8997 .addImm(Val: 0); // clamp
8998 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::op_sel))
8999 NewInstr.addImm(Val: 0); // op_sel0
9000 } else {
9001 NewInstr
9002 .add(MO: Inst.getOperand(i: 0))
9003 .add(MO: Inst.getOperand(i: 1));
9004 }
9005 legalizeOperands(MI&: *NewInstr, MDT);
9006 int SCCIdx = Inst.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
9007 const MachineOperand &SCCOp = Inst.getOperand(i: SCCIdx);
9008 addSCCDefUsersToVALUWorklist(Op: SCCOp, SCCDefInst&: Inst, Worklist, NewCond: CondReg);
9009 Inst.eraseFromParent();
9010 return;
9011 }
9012 case AMDGPU::S_CVT_HI_F32_F16: {
9013 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9014 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9015 if (ST.useRealTrue16Insts()) {
9016 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: TmpReg)
9017 .add(MO: Inst.getOperand(i: 1));
9018 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
9019 .addImm(Val: 0) // src0_modifiers
9020 .addReg(RegNo: TmpReg, Flags: {}, SubReg: AMDGPU::hi16)
9021 .addImm(Val: 0) // clamp
9022 .addImm(Val: 0) // omod
9023 .addImm(Val: 0); // op_sel0
9024 } else {
9025 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
9026 .addImm(Val: 16)
9027 .add(MO: Inst.getOperand(i: 1));
9028 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
9029 .addImm(Val: 0) // src0_modifiers
9030 .addReg(RegNo: TmpReg)
9031 .addImm(Val: 0) // clamp
9032 .addImm(Val: 0); // omod
9033 }
9034
9035 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
9036 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
9037 Inst.eraseFromParent();
9038 return;
9039 }
9040 case AMDGPU::S_MINIMUM_F32:
9041 case AMDGPU::S_MAXIMUM_F32: {
9042 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9043 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
9044 .addImm(Val: 0) // src0_modifiers
9045 .add(MO: Inst.getOperand(i: 1))
9046 .addImm(Val: 0) // src1_modifiers
9047 .add(MO: Inst.getOperand(i: 2))
9048 .addImm(Val: 0) // clamp
9049 .addImm(Val: 0); // omod
9050 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
9051
9052 legalizeOperands(MI&: *NewInstr, MDT);
9053 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
9054 Inst.eraseFromParent();
9055 return;
9056 }
9057 case AMDGPU::S_MINIMUM_F16:
9058 case AMDGPU::S_MAXIMUM_F16: {
9059 Register NewDst = MRI.createVirtualRegister(RegClass: ST.useRealTrue16Insts()
9060 ? &AMDGPU::VGPR_16RegClass
9061 : &AMDGPU::VGPR_32RegClass);
9062 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
9063 .addImm(Val: 0) // src0_modifiers
9064 .add(MO: Inst.getOperand(i: 1))
9065 .addImm(Val: 0) // src1_modifiers
9066 .add(MO: Inst.getOperand(i: 2))
9067 .addImm(Val: 0) // clamp
9068 .addImm(Val: 0) // omod
9069 .addImm(Val: 0); // opsel0
9070 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
9071 legalizeOperands(MI&: *NewInstr, MDT);
9072 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
9073 Inst.eraseFromParent();
9074 return;
9075 }
9076 case AMDGPU::V_S_EXP_F16_e64:
9077 case AMDGPU::V_S_LOG_F16_e64:
9078 case AMDGPU::V_S_RCP_F16_e64:
9079 case AMDGPU::V_S_RSQ_F16_e64:
9080 case AMDGPU::V_S_SQRT_F16_e64: {
9081 Register NewDst = MRI.createVirtualRegister(RegClass: ST.useRealTrue16Insts()
9082 ? &AMDGPU::VGPR_16RegClass
9083 : &AMDGPU::VGPR_32RegClass);
9084 auto NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
9085 .add(MO: Inst.getOperand(i: 1)) // src0_modifiers
9086 .add(MO: Inst.getOperand(i: 2))
9087 .add(MO: Inst.getOperand(i: 3)) // clamp
9088 .add(MO: Inst.getOperand(i: 4)) // omod
9089 .setMIFlags(Inst.getFlags());
9090 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::op_sel))
9091 NewInstr.addImm(Val: 0); // opsel0
9092 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
9093 legalizeOperands(MI&: *NewInstr, MDT);
9094 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
9095 Inst.eraseFromParent();
9096 return;
9097 }
9098 }
9099
9100 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
9101 // We cannot move this instruction to the VALU, so we should try to
9102 // legalize its operands instead.
9103 legalizeOperands(MI&: Inst, MDT);
9104 return;
9105 }
9106 // Handle converting generic instructions like COPY-to-SGPR into
9107 // COPY-to-VGPR.
9108 if (NewOpcode == Opcode) {
9109 Register DstReg = Inst.getOperand(i: 0).getReg();
9110 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
9111
9112 if (Inst.isCopy() && DstReg.isPhysical() &&
9113 Inst.getOperand(i: 1).getReg().isVirtual()) {
9114 handleCopyToPhysHelper(Worklist, DstReg, Inst, MRI, MDT, WaterFalls,
9115 V2SPhyCopiesToErase);
9116 return;
9117 }
9118
9119 if (Inst.isCopy() && Inst.getOperand(i: 1).getReg().isVirtual()) {
9120 Register NewDstReg = Inst.getOperand(i: 1).getReg();
9121 const TargetRegisterClass *SrcRC = RI.getRegClassForReg(MRI, Reg: NewDstReg);
9122 if (const TargetRegisterClass *CommonRC =
9123 RI.getCommonSubClass(A: NewDstRC, B: SrcRC)) {
9124 // Instead of creating a copy where src and dst are the same register
9125 // class, we just replace all uses of dst with src. These kinds of
9126 // copies interfere with the heuristics MachineSink uses to decide
9127 // whether or not to split a critical edge. Since the pass assumes
9128 // that copies will end up as machine instructions and not be
9129 // eliminated.
9130 addUsersToMoveToVALUWorklist(Reg: DstReg, MRI, Worklist);
9131 unsigned SrcSubReg = Inst.getOperand(i: 1).getSubReg();
9132 bool IsUndef = Inst.getOperand(i: 1).isUndef();
9133 for (MachineOperand &UseMO :
9134 make_early_inc_range(Range: MRI.use_operands(Reg: DstReg))) {
9135 UseMO.setSubReg(
9136 RI.composeSubRegIndices(a: SrcSubReg, b: UseMO.getSubReg()));
9137 UseMO.setReg(NewDstReg);
9138 if (IsUndef)
9139 UseMO.setIsUndef();
9140 }
9141
9142 if (!MRI.constrainRegClass(Reg: NewDstReg, RC: CommonRC))
9143 llvm_unreachable("failed to constrain register");
9144
9145 Inst.eraseFromParent();
9146
9147 for (MachineOperand &UseMO :
9148 make_early_inc_range(Range: MRI.use_operands(Reg: NewDstReg))) {
9149 MachineInstr &UseMI = *UseMO.getParent();
9150
9151 // Legalize t16 operands since replaceReg is called after
9152 // addUsersToVALU.
9153 legalizeOperandsVALUt16(MI&: UseMI, MRI);
9154
9155 unsigned OpIdx = UseMI.getOperandNo(I: &UseMO);
9156 if (const TargetRegisterClass *OpRC =
9157 getRegClass(MCID: UseMI.getDesc(), OpNum: OpIdx))
9158 MRI.constrainRegClass(Reg: NewDstReg, RC: OpRC);
9159 }
9160
9161 return;
9162 }
9163 }
9164
9165 // If this is a v2s copy between 16bit and 32bit reg,
9166 // replace vgpr copy to reg_sequence/extract_subreg
9167 // This can be remove after we have sgpr16 in place
9168 if (ST.useRealTrue16Insts() && Inst.isCopy() &&
9169 Inst.getOperand(i: 1).getReg().isVirtual() &&
9170 RI.isVGPR(MRI, Reg: Inst.getOperand(i: 1).getReg())) {
9171 const TargetRegisterClass *SrcRegRC = getOpRegClass(MI: Inst, OpNo: 1);
9172 if (RI.getMatchingSuperRegClass(A: NewDstRC, B: SrcRegRC, Idx: AMDGPU::lo16)) {
9173 Register NewDstReg = buildVGPR32FromLo16(
9174 MBB&: *Inst.getParent(), I: Inst, DL: Inst.getDebugLoc(),
9175 Src: Inst.getOperand(i: 1).getReg(), SrcSubReg: AMDGPU::NoSubRegister);
9176 Inst.eraseFromParent();
9177 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
9178 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
9179 return;
9180 } else if (RI.getMatchingSuperRegClass(A: SrcRegRC, B: NewDstRC,
9181 Idx: AMDGPU::lo16)) {
9182 Inst.getOperand(i: 1).setSubReg(AMDGPU::lo16);
9183 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
9184 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
9185 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
9186 return;
9187 }
9188 }
9189
9190 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
9191 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
9192 legalizeOperands(MI&: Inst, MDT);
9193 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
9194 return;
9195 }
9196
9197 // Use the new VALU Opcode.
9198 auto NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode))
9199 .setMIFlags(Inst.getFlags());
9200 if (isVOP3(Opcode: NewOpcode) && !isVOP3(Opcode)) {
9201 // Intersperse VOP3 modifiers among the SALU operands.
9202 NewInstr->addOperand(Op: Inst.getOperand(i: 0));
9203 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
9204 Name: AMDGPU::OpName::src0_modifiers) >= 0)
9205 NewInstr.addImm(Val: 0);
9206 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::src0)) {
9207 const MachineOperand &Src = Inst.getOperand(i: 1);
9208 NewInstr->addOperand(Op: Src);
9209 }
9210
9211 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
9212 // We are converting these to a BFE, so we need to add the missing
9213 // operands for the size and offset.
9214 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
9215 NewInstr.addImm(Val: 0);
9216 NewInstr.addImm(Val: Size);
9217 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
9218 // The VALU version adds the second operand to the result, so insert an
9219 // extra 0 operand.
9220 NewInstr.addImm(Val: 0);
9221 } else if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
9222 const MachineOperand &OffsetWidthOp = Inst.getOperand(i: 2);
9223 // If we need to move this to VGPRs, we need to unpack the second
9224 // operand back into the 2 separate ones for bit offset and width.
9225 assert(OffsetWidthOp.isImm() &&
9226 "Scalar BFE is only implemented for constant width and offset");
9227 uint32_t Imm = OffsetWidthOp.getImm();
9228
9229 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
9230 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
9231 NewInstr.addImm(Val: Offset);
9232 NewInstr.addImm(Val: BitWidth);
9233 } else {
9234 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
9235 Name: AMDGPU::OpName::src1_modifiers) >= 0)
9236 NewInstr.addImm(Val: 0);
9237 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src1) >= 0)
9238 NewInstr->addOperand(Op: Inst.getOperand(i: 2));
9239 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
9240 Name: AMDGPU::OpName::src2_modifiers) >= 0)
9241 NewInstr.addImm(Val: 0);
9242 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src2) >= 0)
9243 NewInstr->addOperand(Op: Inst.getOperand(i: 3));
9244 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::clamp) >= 0)
9245 NewInstr.addImm(Val: 0);
9246 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::omod) >= 0)
9247 NewInstr.addImm(Val: 0);
9248 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::op_sel) >= 0)
9249 NewInstr.addImm(Val: 0);
9250 }
9251 } else {
9252 // Just copy the SALU operands.
9253 for (const MachineOperand &Op : Inst.explicit_operands())
9254 NewInstr->addOperand(Op);
9255 }
9256
9257 // Remove any references to SCC. Vector instructions can't read from it, and
9258 // We're just about to add the implicit use / defs of VCC, and we don't want
9259 // both.
9260 bool DeadSCCDef = false;
9261 for (MachineOperand &Op : Inst.implicit_operands()) {
9262 if (Op.getReg() == AMDGPU::SCC) {
9263 // Only propagate through live-def of SCC.
9264 if (Op.isDef()) {
9265 if (Op.isDead())
9266 DeadSCCDef = true;
9267 else
9268 addSCCDefUsersToVALUWorklist(Op, SCCDefInst&: Inst, Worklist);
9269 continue;
9270 }
9271
9272 addSCCDefsToVALUWorklist(SCCUseInst: NewInstr, Worklist);
9273 }
9274 }
9275 Inst.eraseFromParent();
9276 Register NewDstReg;
9277 if (NewInstr->getOperand(i: 0).isReg() && NewInstr->getOperand(i: 0).isDef()) {
9278 Register DstReg = NewInstr->getOperand(i: 0).getReg();
9279 assert(DstReg.isVirtual());
9280 // Update the destination register class.
9281 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst: *NewInstr);
9282 assert(NewDstRC);
9283 NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
9284 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
9285 }
9286 fixImplicitOperands(MI&: *NewInstr);
9287
9288 if (DeadSCCDef) {
9289 // A scalar op with a dead SCC def lowers to a VALU op whose VCC def will
9290 // also be dead.
9291 if (MachineOperand *VCCDef =
9292 NewInstr->findRegisterDefOperand(Reg: RI.getVCC(), TRI: &RI))
9293 VCCDef->setIsDead();
9294 }
9295
9296 // Legalize the operands
9297 legalizeOperands(MI&: *NewInstr, MDT);
9298 if (NewDstReg)
9299 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
9300}
9301
9302// Add/sub require special handling to deal with carry outs.
9303std::pair<bool, MachineBasicBlock *>
9304SIInstrInfo::moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
9305 MachineDominatorTree *MDT) const {
9306 if (ST.hasAddNoCarryInsts()) {
9307 // Assume there is no user of scc since we don't select this in that case.
9308 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
9309 // is used.
9310
9311 MachineBasicBlock &MBB = *Inst.getParent();
9312 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9313
9314 Register OldDstReg = Inst.getOperand(i: 0).getReg();
9315 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9316
9317 unsigned Opc = Inst.getOpcode();
9318 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
9319
9320 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
9321 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
9322
9323 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
9324 Inst.removeOperand(OpNo: 3);
9325
9326 Inst.setDesc(get(Opcode: NewOpc));
9327 Inst.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // clamp bit
9328 Inst.addImplicitDefUseOperands(MF&: *MBB.getParent());
9329 MRI.replaceRegWith(FromReg: OldDstReg, ToReg: ResultReg);
9330 MachineBasicBlock *NewBB = legalizeOperands(MI&: Inst, MDT);
9331
9332 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9333 return std::pair(true, NewBB);
9334 }
9335
9336 return std::pair(false, nullptr);
9337}
9338
9339void SIInstrInfo::lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
9340 MachineDominatorTree *MDT) const {
9341
9342 MachineBasicBlock &MBB = *Inst.getParent();
9343 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9344 MachineBasicBlock::iterator MII = Inst;
9345 const DebugLoc &DL = Inst.getDebugLoc();
9346
9347 MachineOperand &Dest = Inst.getOperand(i: 0);
9348 MachineOperand &Src0 = Inst.getOperand(i: 1);
9349 MachineOperand &Src1 = Inst.getOperand(i: 2);
9350 MachineOperand &Cond = Inst.getOperand(i: 3);
9351
9352 Register CondReg = Cond.getReg();
9353 bool IsSCC = (CondReg == AMDGPU::SCC);
9354
9355 // Remove S_CSELECT instructions that we previously inserted to feed the SCC
9356 // condition output from S_CMP into the SGPR condition input of V_CNDMASK. If
9357 // the S_CMP has been promoted to V_CMP then we can feed its SGPR condition
9358 // output directly into the V_CNDMASK.
9359 if (!IsSCC && Src0.isImm() && (Src0.getImm() == -1) && Src1.isImm() &&
9360 (Src1.getImm() == 0)) {
9361 for (MachineOperand &UseMO :
9362 make_early_inc_range(Range: MRI.use_nodbg_operands(Reg: Dest.getReg()))) {
9363 MachineInstr &UseMI = *UseMO.getParent();
9364 switch (UseMI.getOpcode()) {
9365 case AMDGPU::V_CNDMASK_B16_fake16_e32:
9366 case AMDGPU::V_CNDMASK_B16_fake16_e64:
9367 case AMDGPU::V_CNDMASK_B16_t16_e32:
9368 case AMDGPU::V_CNDMASK_B16_t16_e64:
9369 case AMDGPU::V_CNDMASK_B32_e32:
9370 case AMDGPU::V_CNDMASK_B32_e64:
9371 case AMDGPU::V_CNDMASK_B64_PSEUDO:
9372 if (UseMO.isImplicit() ||
9373 &UseMO == getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src2))
9374 UseMO.setReg(CondReg);
9375 }
9376 }
9377 if (MRI.use_nodbg_empty(RegNo: Dest.getReg()))
9378 return;
9379 }
9380
9381 Register NewCondReg = CondReg;
9382 if (IsSCC) {
9383 const TargetRegisterClass *TC = RI.getWaveMaskRegClass();
9384 NewCondReg = MRI.createVirtualRegister(RegClass: TC);
9385
9386 // Now look for the closest SCC def if it is a copy
9387 // replacing the CondReg with the COPY source register
9388 bool CopyFound = false;
9389 for (MachineInstr &CandI :
9390 make_range(x: std::next(x: MachineBasicBlock::reverse_iterator(Inst)),
9391 y: Inst.getParent()->rend())) {
9392 if (CandI.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isDead: false, Overlap: false) !=
9393 -1) {
9394 if (CandI.isCopy() && CandI.getOperand(i: 0).getReg() == AMDGPU::SCC) {
9395 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: NewCondReg)
9396 .addReg(RegNo: CandI.getOperand(i: 1).getReg());
9397 CopyFound = true;
9398 }
9399 break;
9400 }
9401 }
9402 if (!CopyFound) {
9403 // SCC def is not a copy
9404 // Insert a trivial select instead of creating a copy, because a copy from
9405 // SCC would semantically mean just copying a single bit, but we may need
9406 // the result to be a vector condition mask that needs preserving.
9407 unsigned Opcode =
9408 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
9409 auto NewSelect =
9410 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: NewCondReg).addImm(Val: -1).addImm(Val: 0);
9411 NewSelect->getOperand(i: 3).setIsUndef(Cond.isUndef());
9412 }
9413 }
9414
9415 Register NewDestReg = MRI.createVirtualRegister(
9416 RegClass: RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest.getReg())));
9417 MachineInstr *NewInst;
9418 if (Inst.getOpcode() == AMDGPU::S_CSELECT_B32) {
9419 NewInst = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: NewDestReg)
9420 .addImm(Val: 0)
9421 .add(MO: Src1) // False
9422 .addImm(Val: 0)
9423 .add(MO: Src0) // True
9424 .addReg(RegNo: NewCondReg);
9425 } else {
9426 NewInst =
9427 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B64_PSEUDO), DestReg: NewDestReg)
9428 .add(MO: Src1) // False
9429 .add(MO: Src0) // True
9430 .addReg(RegNo: NewCondReg);
9431 }
9432 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDestReg);
9433 legalizeOperands(MI&: *NewInst, MDT);
9434 addUsersToMoveToVALUWorklist(Reg: NewDestReg, MRI, Worklist);
9435}
9436
9437void SIInstrInfo::lowerScalarAbs(SIInstrWorklist &Worklist,
9438 MachineInstr &Inst) const {
9439 MachineBasicBlock &MBB = *Inst.getParent();
9440 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9441 MachineBasicBlock::iterator MII = Inst;
9442 const DebugLoc &DL = Inst.getDebugLoc();
9443
9444 MachineOperand &Dest = Inst.getOperand(i: 0);
9445 MachineOperand &Src = Inst.getOperand(i: 1);
9446 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9447 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9448
9449 bool HasCarryOut = !ST.hasAddNoCarryInsts();
9450 unsigned SubOp =
9451 HasCarryOut ? AMDGPU::V_SUB_CO_U32_e32 : AMDGPU::V_SUB_U32_e32;
9452
9453 MachineInstrBuilder Sub =
9454 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: TmpReg).addImm(Val: 0).addReg(RegNo: Src.getReg());
9455 if (HasCarryOut)
9456 Sub.setOperandDead(3); // Dead vcc
9457
9458 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MAX_I32_e64), DestReg: ResultReg)
9459 .addReg(RegNo: Src.getReg())
9460 .addReg(RegNo: TmpReg);
9461
9462 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9463 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9464}
9465
9466void SIInstrInfo::lowerScalarAbsDiff(SIInstrWorklist &Worklist,
9467 MachineInstr &Inst) const {
9468 MachineBasicBlock &MBB = *Inst.getParent();
9469 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9470 MachineBasicBlock::iterator MII = Inst;
9471 const DebugLoc &DL = Inst.getDebugLoc();
9472
9473 MachineOperand &Dest = Inst.getOperand(i: 0);
9474 MachineOperand &Src1 = Inst.getOperand(i: 1);
9475 MachineOperand &Src2 = Inst.getOperand(i: 2);
9476 Register SubResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9477 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9478 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9479
9480 bool HasCarryOut = !ST.hasAddNoCarryInsts();
9481 unsigned SubOp =
9482 HasCarryOut ? AMDGPU::V_SUB_CO_U32_e32 : AMDGPU::V_SUB_U32_e32;
9483
9484 MachineInstrBuilder Sub1 = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: SubResultReg)
9485 .addReg(RegNo: Src1.getReg())
9486 .addReg(RegNo: Src2.getReg());
9487
9488 MachineInstrBuilder Sub2 =
9489 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: TmpReg).addImm(Val: 0).addReg(RegNo: SubResultReg);
9490
9491 if (HasCarryOut) {
9492 Sub1.setOperandDead(3); // Dead vcc
9493 Sub2.setOperandDead(3); // Dead vcc
9494 }
9495
9496 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MAX_I32_e64), DestReg: ResultReg)
9497 .addReg(RegNo: SubResultReg)
9498 .addReg(RegNo: TmpReg);
9499
9500 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9501 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9502}
9503
9504void SIInstrInfo::lowerScalarXnor(SIInstrWorklist &Worklist,
9505 MachineInstr &Inst) const {
9506 MachineBasicBlock &MBB = *Inst.getParent();
9507 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9508 MachineBasicBlock::iterator MII = Inst;
9509 const DebugLoc &DL = Inst.getDebugLoc();
9510
9511 MachineOperand &Dest = Inst.getOperand(i: 0);
9512 MachineOperand &Src0 = Inst.getOperand(i: 1);
9513 MachineOperand &Src1 = Inst.getOperand(i: 2);
9514
9515 if (ST.hasDLInsts()) {
9516 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9517 legalizeGenericOperand(InsertMBB&: MBB, I: MII, DstRC: &AMDGPU::VGPR_32RegClass, Op&: Src0, MRI, DL);
9518 legalizeGenericOperand(InsertMBB&: MBB, I: MII, DstRC: &AMDGPU::VGPR_32RegClass, Op&: Src1, MRI, DL);
9519
9520 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_XNOR_B32_e64), DestReg: NewDest)
9521 .add(MO: Src0)
9522 .add(MO: Src1);
9523
9524 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9525 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9526 } else {
9527 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
9528 // invert either source and then perform the XOR. If either source is a
9529 // scalar register, then we can leave the inversion on the scalar unit to
9530 // achieve a better distribution of scalar and vector instructions.
9531 bool Src0IsSGPR = Src0.isReg() &&
9532 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src0.getReg()));
9533 bool Src1IsSGPR = Src1.isReg() &&
9534 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src1.getReg()));
9535 MachineInstr *Xor;
9536 Register Temp = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9537 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9538
9539 // Build a pair of scalar instructions and add them to the work list.
9540 // The next iteration over the work list will lower these to the vector
9541 // unit as necessary.
9542 if (Src0IsSGPR) {
9543 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Temp).add(MO: Src0);
9544 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: NewDest)
9545 .addReg(RegNo: Temp)
9546 .add(MO: Src1);
9547 } else if (Src1IsSGPR) {
9548 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Temp).add(MO: Src1);
9549 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: NewDest)
9550 .add(MO: Src0)
9551 .addReg(RegNo: Temp);
9552 } else {
9553 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: Temp)
9554 .add(MO: Src0)
9555 .add(MO: Src1);
9556 MachineInstr *Not =
9557 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: NewDest).addReg(RegNo: Temp);
9558 Worklist.insert(MI: Not);
9559 }
9560
9561 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9562
9563 Worklist.insert(MI: Xor);
9564
9565 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9566 }
9567}
9568
9569void SIInstrInfo::splitScalarNotBinop(SIInstrWorklist &Worklist,
9570 MachineInstr &Inst,
9571 unsigned Opcode) const {
9572 MachineBasicBlock &MBB = *Inst.getParent();
9573 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9574 MachineBasicBlock::iterator MII = Inst;
9575 const DebugLoc &DL = Inst.getDebugLoc();
9576
9577 MachineOperand &Dest = Inst.getOperand(i: 0);
9578 MachineOperand &Src0 = Inst.getOperand(i: 1);
9579 MachineOperand &Src1 = Inst.getOperand(i: 2);
9580
9581 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9582 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9583
9584 MachineInstr &Op = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: Interm)
9585 .add(MO: Src0)
9586 .add(MO: Src1);
9587
9588 MachineInstr &Not = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: NewDest)
9589 .addReg(RegNo: Interm);
9590
9591 Worklist.insert(MI: &Op);
9592 Worklist.insert(MI: &Not);
9593
9594 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9595 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9596}
9597
9598void SIInstrInfo::splitScalarBinOpN2(SIInstrWorklist &Worklist,
9599 MachineInstr &Inst,
9600 unsigned Opcode) const {
9601 MachineBasicBlock &MBB = *Inst.getParent();
9602 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9603 MachineBasicBlock::iterator MII = Inst;
9604 const DebugLoc &DL = Inst.getDebugLoc();
9605
9606 MachineOperand &Dest = Inst.getOperand(i: 0);
9607 MachineOperand &Src0 = Inst.getOperand(i: 1);
9608 MachineOperand &Src1 = Inst.getOperand(i: 2);
9609
9610 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
9611 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
9612
9613 MachineInstr &Not = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Interm)
9614 .add(MO: Src1);
9615
9616 MachineInstr &Op = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: NewDest)
9617 .add(MO: Src0)
9618 .addReg(RegNo: Interm);
9619
9620 Worklist.insert(MI: &Not);
9621 Worklist.insert(MI: &Op);
9622
9623 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9624 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9625}
9626
9627void SIInstrInfo::splitScalar64BitUnaryOp(SIInstrWorklist &Worklist,
9628 MachineInstr &Inst, unsigned Opcode,
9629 bool Swap) const {
9630 MachineBasicBlock &MBB = *Inst.getParent();
9631 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9632
9633 MachineOperand &Dest = Inst.getOperand(i: 0);
9634 MachineOperand &Src0 = Inst.getOperand(i: 1);
9635 const DebugLoc &DL = Inst.getDebugLoc();
9636
9637 MachineBasicBlock::iterator MII = Inst;
9638
9639 const MCInstrDesc &InstDesc = get(Opcode);
9640 const TargetRegisterClass *Src0RC = Src0.isReg() ?
9641 MRI.getRegClass(Reg: Src0.getReg()) :
9642 &AMDGPU::SGPR_32RegClass;
9643
9644 const TargetRegisterClass *Src0SubRC =
9645 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9646
9647 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9648 SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9649
9650 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9651 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(SRC: DestRC);
9652 const TargetRegisterClass *NewDestSubRC =
9653 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
9654
9655 Register DestSub0 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9656 MachineInstr &LoHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub0).add(MO: SrcReg0Sub0);
9657
9658 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9659 SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9660
9661 Register DestSub1 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9662 MachineInstr &HiHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub1).add(MO: SrcReg0Sub1);
9663
9664 if (Swap)
9665 std::swap(a&: DestSub0, b&: DestSub1);
9666
9667 Register FullDestReg = MRI.createVirtualRegister(RegClass: NewDestRC);
9668 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9669 .addReg(RegNo: DestSub0)
9670 .addImm(Val: AMDGPU::sub0)
9671 .addReg(RegNo: DestSub1)
9672 .addImm(Val: AMDGPU::sub1);
9673
9674 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9675
9676 Worklist.insert(MI: &LoHalf);
9677 Worklist.insert(MI: &HiHalf);
9678
9679 // We don't need to legalizeOperands here because for a single operand, src0
9680 // will support any kind of input.
9681
9682 // Move all users of this moved value.
9683 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9684}
9685
9686// There is not a vector equivalent of s_mul_u64. For this reason, we need to
9687// split the s_mul_u64 in 32-bit vector multiplications.
9688void SIInstrInfo::splitScalarSMulU64(SIInstrWorklist &Worklist,
9689 MachineInstr &Inst,
9690 MachineDominatorTree *MDT) const {
9691 MachineBasicBlock &MBB = *Inst.getParent();
9692 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9693
9694 Register FullDestReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9695 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9696 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9697
9698 MachineOperand &Dest = Inst.getOperand(i: 0);
9699 MachineOperand &Src0 = Inst.getOperand(i: 1);
9700 MachineOperand &Src1 = Inst.getOperand(i: 2);
9701 const DebugLoc &DL = Inst.getDebugLoc();
9702 MachineBasicBlock::iterator MII = Inst;
9703
9704 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0.getReg());
9705 const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: Src1.getReg());
9706 const TargetRegisterClass *Src0SubRC =
9707 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9708 if (RI.isSGPRClass(RC: Src0SubRC))
9709 Src0SubRC = RI.getEquivalentVGPRClass(SRC: Src0SubRC);
9710 const TargetRegisterClass *Src1SubRC =
9711 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9712 if (RI.isSGPRClass(RC: Src1SubRC))
9713 Src1SubRC = RI.getEquivalentVGPRClass(SRC: Src1SubRC);
9714
9715 // First, we extract the low 32-bit and high 32-bit values from each of the
9716 // operands.
9717 MachineOperand Op0L =
9718 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9719 MachineOperand Op1L =
9720 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9721 MachineOperand Op0H =
9722 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9723 MachineOperand Op1H =
9724 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
9725
9726 // The multilication is done as follows:
9727 //
9728 // Op1H Op1L
9729 // * Op0H Op0L
9730 // --------------------
9731 // Op1H*Op0L Op1L*Op0L
9732 // + Op1H*Op0H Op1L*Op0H
9733 // -----------------------------------------
9734 // (Op1H*Op0L + Op1L*Op0H + carry) Op1L*Op0L
9735 //
9736 // We drop Op1H*Op0H because the result of the multiplication is a 64-bit
9737 // value and that would overflow.
9738 // The low 32-bit value is Op1L*Op0L.
9739 // The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from Op1L*Op0L).
9740
9741 Register Op1L_Op0H_Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9742 MachineInstr *Op1L_Op0H =
9743 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: Op1L_Op0H_Reg)
9744 .add(MO: Op1L)
9745 .add(MO: Op0H);
9746
9747 Register Op1H_Op0L_Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9748 MachineInstr *Op1H_Op0L =
9749 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: Op1H_Op0L_Reg)
9750 .add(MO: Op1H)
9751 .add(MO: Op0L);
9752
9753 Register CarryReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9754 MachineInstr *Carry =
9755 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_HI_U32_e64), DestReg: CarryReg)
9756 .add(MO: Op1L)
9757 .add(MO: Op0L);
9758
9759 MachineInstr *LoHalf =
9760 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: DestSub0)
9761 .add(MO: Op1L)
9762 .add(MO: Op0L);
9763
9764 Register AddReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9765 MachineInstr *Add = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg: AddReg)
9766 .addReg(RegNo: Op1L_Op0H_Reg)
9767 .addReg(RegNo: Op1H_Op0L_Reg);
9768
9769 MachineInstr *HiHalf =
9770 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg: DestSub1)
9771 .addReg(RegNo: AddReg)
9772 .addReg(RegNo: CarryReg);
9773
9774 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9775 .addReg(RegNo: DestSub0)
9776 .addImm(Val: AMDGPU::sub0)
9777 .addReg(RegNo: DestSub1)
9778 .addImm(Val: AMDGPU::sub1);
9779
9780 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9781
9782 // Try to legalize the operands in case we need to swap the order to keep it
9783 // valid.
9784 legalizeOperands(MI&: *Op1L_Op0H, MDT);
9785 legalizeOperands(MI&: *Op1H_Op0L, MDT);
9786 legalizeOperands(MI&: *Carry, MDT);
9787 legalizeOperands(MI&: *LoHalf, MDT);
9788 legalizeOperands(MI&: *Add, MDT);
9789 legalizeOperands(MI&: *HiHalf, MDT);
9790
9791 // Move all users of this moved value.
9792 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9793}
9794
9795// Lower S_MUL_U64_U32_PSEUDO/S_MUL_I64_I32_PSEUDO in two 32-bit vector
9796// multiplications.
9797void SIInstrInfo::splitScalarSMulPseudo(SIInstrWorklist &Worklist,
9798 MachineInstr &Inst,
9799 MachineDominatorTree *MDT) const {
9800 MachineBasicBlock &MBB = *Inst.getParent();
9801 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9802
9803 Register FullDestReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9804 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9805 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9806
9807 MachineOperand &Dest = Inst.getOperand(i: 0);
9808 MachineOperand &Src0 = Inst.getOperand(i: 1);
9809 MachineOperand &Src1 = Inst.getOperand(i: 2);
9810 const DebugLoc &DL = Inst.getDebugLoc();
9811 MachineBasicBlock::iterator MII = Inst;
9812
9813 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0.getReg());
9814 const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: Src1.getReg());
9815 const TargetRegisterClass *Src0SubRC =
9816 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9817 if (RI.isSGPRClass(RC: Src0SubRC))
9818 Src0SubRC = RI.getEquivalentVGPRClass(SRC: Src0SubRC);
9819 const TargetRegisterClass *Src1SubRC =
9820 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9821 if (RI.isSGPRClass(RC: Src1SubRC))
9822 Src1SubRC = RI.getEquivalentVGPRClass(SRC: Src1SubRC);
9823
9824 // First, we extract the low 32-bit and high 32-bit values from each of the
9825 // operands.
9826 MachineOperand Op0L =
9827 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9828 MachineOperand Op1L =
9829 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9830
9831 unsigned Opc = Inst.getOpcode();
9832 unsigned NewOpc = Opc == AMDGPU::S_MUL_U64_U32_PSEUDO
9833 ? AMDGPU::V_MUL_HI_U32_e64
9834 : AMDGPU::V_MUL_HI_I32_e64;
9835 MachineInstr *HiHalf =
9836 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: NewOpc), DestReg: DestSub1).add(MO: Op1L).add(MO: Op0L);
9837
9838 MachineInstr *LoHalf =
9839 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: DestSub0)
9840 .add(MO: Op1L)
9841 .add(MO: Op0L);
9842
9843 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9844 .addReg(RegNo: DestSub0)
9845 .addImm(Val: AMDGPU::sub0)
9846 .addReg(RegNo: DestSub1)
9847 .addImm(Val: AMDGPU::sub1);
9848
9849 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9850
9851 // Try to legalize the operands in case we need to swap the order to keep it
9852 // valid.
9853 legalizeOperands(MI&: *HiHalf, MDT);
9854 legalizeOperands(MI&: *LoHalf, MDT);
9855
9856 // Move all users of this moved value.
9857 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9858}
9859
9860void SIInstrInfo::splitScalar64BitBinaryOp(SIInstrWorklist &Worklist,
9861 MachineInstr &Inst, unsigned Opcode,
9862 MachineDominatorTree *MDT) const {
9863 MachineBasicBlock &MBB = *Inst.getParent();
9864 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9865
9866 MachineOperand &Dest = Inst.getOperand(i: 0);
9867 MachineOperand &Src0 = Inst.getOperand(i: 1);
9868 MachineOperand &Src1 = Inst.getOperand(i: 2);
9869 const DebugLoc &DL = Inst.getDebugLoc();
9870
9871 MachineBasicBlock::iterator MII = Inst;
9872
9873 const MCInstrDesc &InstDesc = get(Opcode);
9874 const TargetRegisterClass *Src0RC = Src0.isReg() ?
9875 MRI.getRegClass(Reg: Src0.getReg()) :
9876 &AMDGPU::SGPR_32RegClass;
9877
9878 const TargetRegisterClass *Src0SubRC =
9879 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9880 const TargetRegisterClass *Src1RC = Src1.isReg() ?
9881 MRI.getRegClass(Reg: Src1.getReg()) :
9882 &AMDGPU::SGPR_32RegClass;
9883
9884 const TargetRegisterClass *Src1SubRC =
9885 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9886
9887 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9888 SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9889 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC,
9890 SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9891 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9892 SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9893 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC,
9894 SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
9895
9896 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9897 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(SRC: DestRC);
9898 const TargetRegisterClass *NewDestSubRC =
9899 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
9900
9901 Register DestSub0 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9902 MachineInstr &LoHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub0)
9903 .add(MO: SrcReg0Sub0)
9904 .add(MO: SrcReg1Sub0);
9905
9906 Register DestSub1 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9907 MachineInstr &HiHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub1)
9908 .add(MO: SrcReg0Sub1)
9909 .add(MO: SrcReg1Sub1);
9910
9911 Register FullDestReg = MRI.createVirtualRegister(RegClass: NewDestRC);
9912 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9913 .addReg(RegNo: DestSub0)
9914 .addImm(Val: AMDGPU::sub0)
9915 .addReg(RegNo: DestSub1)
9916 .addImm(Val: AMDGPU::sub1);
9917
9918 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9919
9920 Worklist.insert(MI: &LoHalf);
9921 Worklist.insert(MI: &HiHalf);
9922
9923 // Move all users of this moved value.
9924 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9925}
9926
9927void SIInstrInfo::splitScalar64BitXnor(SIInstrWorklist &Worklist,
9928 MachineInstr &Inst,
9929 MachineDominatorTree *MDT) const {
9930 MachineBasicBlock &MBB = *Inst.getParent();
9931 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9932
9933 MachineOperand &Dest = Inst.getOperand(i: 0);
9934 MachineOperand &Src0 = Inst.getOperand(i: 1);
9935 MachineOperand &Src1 = Inst.getOperand(i: 2);
9936 const DebugLoc &DL = Inst.getDebugLoc();
9937
9938 MachineBasicBlock::iterator MII = Inst;
9939
9940 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9941
9942 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
9943
9944 MachineOperand* Op0;
9945 MachineOperand* Op1;
9946
9947 if (Src0.isReg() && RI.isSGPRReg(MRI, Reg: Src0.getReg())) {
9948 Op0 = &Src0;
9949 Op1 = &Src1;
9950 } else {
9951 Op0 = &Src1;
9952 Op1 = &Src0;
9953 }
9954
9955 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B64), DestReg: Interm)
9956 .add(MO: *Op0);
9957
9958 Register NewDest = MRI.createVirtualRegister(RegClass: DestRC);
9959
9960 MachineInstr &Xor = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B64), DestReg: NewDest)
9961 .addReg(RegNo: Interm)
9962 .add(MO: *Op1);
9963
9964 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9965
9966 Worklist.insert(MI: &Xor);
9967}
9968
9969void SIInstrInfo::splitScalar64BitBCNT(SIInstrWorklist &Worklist,
9970 MachineInstr &Inst) const {
9971 MachineBasicBlock &MBB = *Inst.getParent();
9972 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9973
9974 MachineBasicBlock::iterator MII = Inst;
9975 const DebugLoc &DL = Inst.getDebugLoc();
9976
9977 MachineOperand &Dest = Inst.getOperand(i: 0);
9978 MachineOperand &Src = Inst.getOperand(i: 1);
9979
9980 const MCInstrDesc &InstDesc = get(Opcode: AMDGPU::V_BCNT_U32_B32_e64);
9981 const TargetRegisterClass *SrcRC = Src.isReg() ?
9982 MRI.getRegClass(Reg: Src.getReg()) :
9983 &AMDGPU::SGPR_32RegClass;
9984
9985 Register MidReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9986 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9987
9988 const TargetRegisterClass *SrcSubRC =
9989 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
9990
9991 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC,
9992 SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
9993 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC,
9994 SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
9995
9996 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg).add(MO: SrcRegSub0).addImm(Val: 0);
9997
9998 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: ResultReg).add(MO: SrcRegSub1).addReg(RegNo: MidReg);
9999
10000 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
10001
10002 // We don't need to legalize operands here. src0 for either instruction can be
10003 // an SGPR, and the second input is unused or determined here.
10004 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
10005}
10006
10007void SIInstrInfo::splitScalar64BitBFE(SIInstrWorklist &Worklist,
10008 MachineInstr &Inst) const {
10009 MachineBasicBlock &MBB = *Inst.getParent();
10010 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
10011 MachineBasicBlock::iterator MII = Inst;
10012 const DebugLoc &DL = Inst.getDebugLoc();
10013
10014 MachineOperand &Dest = Inst.getOperand(i: 0);
10015 uint32_t Imm = Inst.getOperand(i: 2).getImm();
10016 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
10017 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
10018
10019 (void) Offset;
10020
10021 // Only sext_inreg cases handled.
10022 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
10023 Offset == 0 && "Not implemented");
10024
10025 if (BitWidth < 32) {
10026 Register MidRegLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10027 Register MidRegHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10028 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
10029
10030 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_BFE_I32_e64), DestReg: MidRegLo)
10031 .addReg(RegNo: Inst.getOperand(i: 1).getReg(), Flags: {}, SubReg: AMDGPU::sub0)
10032 .addImm(Val: 0)
10033 .addImm(Val: BitWidth);
10034
10035 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ASHRREV_I32_e32), DestReg: MidRegHi)
10036 .addImm(Val: 31)
10037 .addReg(RegNo: MidRegLo);
10038
10039 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: ResultReg)
10040 .addReg(RegNo: MidRegLo)
10041 .addImm(Val: AMDGPU::sub0)
10042 .addReg(RegNo: MidRegHi)
10043 .addImm(Val: AMDGPU::sub1);
10044
10045 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
10046 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
10047 return;
10048 }
10049
10050 MachineOperand &Src = Inst.getOperand(i: 1);
10051 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10052 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
10053
10054 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ASHRREV_I32_e64), DestReg: TmpReg)
10055 .addImm(Val: 31)
10056 .addReg(RegNo: Src.getReg(), Flags: {}, SubReg: AMDGPU::sub0);
10057
10058 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: ResultReg)
10059 .addReg(RegNo: Src.getReg(), Flags: {}, SubReg: AMDGPU::sub0)
10060 .addImm(Val: AMDGPU::sub0)
10061 .addReg(RegNo: TmpReg)
10062 .addImm(Val: AMDGPU::sub1);
10063
10064 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
10065 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
10066}
10067
10068void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist,
10069 MachineInstr &Inst, unsigned Opcode,
10070 MachineDominatorTree *MDT) const {
10071 // (S_FLBIT_I32_B64 hi:lo) ->
10072 // -> (umin (V_FFBH_U32_e32 hi), (or (V_FFBH_U32_e32 lo), 32))
10073 // (S_FF1_I32_B64 hi:lo) ->
10074 // ->(umin (or (V_FFBL_B32_e32 hi), 32) (V_FFBL_B32_e32 lo))
10075
10076 MachineBasicBlock &MBB = *Inst.getParent();
10077 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
10078 MachineBasicBlock::iterator MII = Inst;
10079 const DebugLoc &DL = Inst.getDebugLoc();
10080
10081 MachineOperand &Dest = Inst.getOperand(i: 0);
10082 MachineOperand &Src = Inst.getOperand(i: 1);
10083
10084 const MCInstrDesc &InstDesc = get(Opcode);
10085
10086 bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32;
10087
10088 const TargetRegisterClass *SrcRC =
10089 Src.isReg() ? MRI.getRegClass(Reg: Src.getReg()) : &AMDGPU::SGPR_32RegClass;
10090 const TargetRegisterClass *SrcSubRC =
10091 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
10092
10093 MachineOperand SrcRegSub0 =
10094 buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC, SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
10095 MachineOperand SrcRegSub1 =
10096 buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC, SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
10097
10098 Register MidReg1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10099 Register MidReg2 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10100 Register MidReg3 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10101 Register MidReg4 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10102
10103 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg1).add(MO: SrcRegSub0);
10104
10105 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg2).add(MO: SrcRegSub1);
10106
10107 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_OR_B32_e32), DestReg: MidReg3)
10108 .addImm(Val: 32)
10109 .addReg(RegNo: IsCtlz ? MidReg1 : MidReg2);
10110
10111 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MIN_U32_e64), DestReg: MidReg4)
10112 .addReg(RegNo: MidReg3)
10113 .addReg(RegNo: IsCtlz ? MidReg2 : MidReg1);
10114
10115 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: MidReg4);
10116
10117 addUsersToMoveToVALUWorklist(Reg: MidReg4, MRI, Worklist);
10118}
10119
10120void SIInstrInfo::addUsersToMoveToVALUWorklist(
10121 Register DstReg, MachineRegisterInfo &MRI,
10122 SIInstrWorklist &Worklist) const {
10123 for (MachineOperand &MO : make_early_inc_range(Range: MRI.use_operands(Reg: DstReg))) {
10124 MachineInstr &UseMI = *MO.getParent();
10125
10126 unsigned OpNo = 0;
10127
10128 switch (UseMI.getOpcode()) {
10129 case AMDGPU::COPY:
10130 case AMDGPU::WQM:
10131 case AMDGPU::SOFT_WQM:
10132 case AMDGPU::STRICT_WWM:
10133 case AMDGPU::STRICT_WQM:
10134 case AMDGPU::REG_SEQUENCE:
10135 case AMDGPU::PHI:
10136 case AMDGPU::INSERT_SUBREG:
10137 break;
10138 default:
10139 OpNo = MO.getOperandNo();
10140 break;
10141 }
10142
10143 const TargetRegisterClass *OpRC = getOpRegClass(MI: UseMI, OpNo);
10144 MRI.constrainRegClass(Reg: DstReg, RC: OpRC);
10145
10146 if (!RI.hasVectorRegisters(RC: OpRC))
10147 Worklist.insert(MI: &UseMI);
10148 else
10149 // Legalization could change user list.
10150 legalizeOperandsVALUt16(MI&: UseMI, OpIdx: OpNo, MRI);
10151 }
10152}
10153
10154void SIInstrInfo::movePackToVALU(SIInstrWorklist &Worklist,
10155 MachineRegisterInfo &MRI,
10156 MachineInstr &Inst) const {
10157 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10158 MachineBasicBlock *MBB = Inst.getParent();
10159 MachineOperand &Src0 = Inst.getOperand(i: 1);
10160 MachineOperand &Src1 = Inst.getOperand(i: 2);
10161 const DebugLoc &DL = Inst.getDebugLoc();
10162
10163 if (ST.useRealTrue16Insts()) {
10164 Register SrcReg0, SrcReg1;
10165 if (!Src0.isReg() || !RI.isVGPR(MRI, Reg: Src0.getReg())) {
10166 SrcReg0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10167 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL,
10168 MCID: get(Opcode: Src0.isImm() ? AMDGPU::V_MOV_B32_e32 : AMDGPU::COPY), DestReg: SrcReg0)
10169 .add(MO: Src0);
10170 } else {
10171 SrcReg0 = Src0.getReg();
10172 }
10173
10174 if (!Src1.isReg() || !RI.isVGPR(MRI, Reg: Src1.getReg())) {
10175 SrcReg1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10176 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL,
10177 MCID: get(Opcode: Src1.isImm() ? AMDGPU::V_MOV_B32_e32 : AMDGPU::COPY), DestReg: SrcReg1)
10178 .add(MO: Src1);
10179 } else {
10180 SrcReg1 = Src1.getReg();
10181 }
10182
10183 bool isSrc0Reg16 = MRI.constrainRegClass(Reg: SrcReg0, RC: &AMDGPU::VGPR_16RegClass);
10184 bool isSrc1Reg16 = MRI.constrainRegClass(Reg: SrcReg1, RC: &AMDGPU::VGPR_16RegClass);
10185
10186 auto NewMI = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: ResultReg);
10187 switch (Inst.getOpcode()) {
10188 case AMDGPU::S_PACK_LL_B32_B16:
10189 NewMI
10190 .addReg(RegNo: SrcReg0, Flags: {},
10191 SubReg: isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10192 .addImm(Val: AMDGPU::lo16)
10193 .addReg(RegNo: SrcReg1, Flags: {},
10194 SubReg: isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10195 .addImm(Val: AMDGPU::hi16);
10196 break;
10197 case AMDGPU::S_PACK_LH_B32_B16:
10198 NewMI
10199 .addReg(RegNo: SrcReg0, Flags: {},
10200 SubReg: isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10201 .addImm(Val: AMDGPU::lo16)
10202 .addReg(RegNo: SrcReg1, Flags: {}, SubReg: AMDGPU::hi16)
10203 .addImm(Val: AMDGPU::hi16);
10204 break;
10205 case AMDGPU::S_PACK_HL_B32_B16:
10206 NewMI.addReg(RegNo: SrcReg0, Flags: {}, SubReg: AMDGPU::hi16)
10207 .addImm(Val: AMDGPU::lo16)
10208 .addReg(RegNo: SrcReg1, Flags: {},
10209 SubReg: isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10210 .addImm(Val: AMDGPU::hi16);
10211 break;
10212 case AMDGPU::S_PACK_HH_B32_B16:
10213 NewMI.addReg(RegNo: SrcReg0, Flags: {}, SubReg: AMDGPU::hi16)
10214 .addImm(Val: AMDGPU::lo16)
10215 .addReg(RegNo: SrcReg1, Flags: {}, SubReg: AMDGPU::hi16)
10216 .addImm(Val: AMDGPU::hi16);
10217 break;
10218 default:
10219 llvm_unreachable("unhandled s_pack_* instruction");
10220 }
10221
10222 MachineOperand &Dest = Inst.getOperand(i: 0);
10223 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
10224 addUsersToMoveToVALUWorklist(DstReg: ResultReg, MRI, Worklist);
10225 return;
10226 }
10227
10228 switch (Inst.getOpcode()) {
10229 case AMDGPU::S_PACK_LL_B32_B16: {
10230 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10231 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10232
10233 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are
10234 // 0.
10235 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
10236 .addImm(Val: 0xffff);
10237
10238 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_AND_B32_e64), DestReg: TmpReg)
10239 .addReg(RegNo: ImmReg)
10240 .add(MO: Src0);
10241
10242 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: ResultReg)
10243 .add(MO: Src1)
10244 .addImm(Val: 16)
10245 .addReg(RegNo: TmpReg);
10246 break;
10247 }
10248 case AMDGPU::S_PACK_LH_B32_B16: {
10249 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10250 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
10251 .addImm(Val: 0xffff);
10252 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_BFI_B32_e64), DestReg: ResultReg)
10253 .addReg(RegNo: ImmReg)
10254 .add(MO: Src0)
10255 .add(MO: Src1);
10256 break;
10257 }
10258 case AMDGPU::S_PACK_HL_B32_B16: {
10259 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10260 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
10261 .addImm(Val: 16)
10262 .add(MO: Src0);
10263 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: ResultReg)
10264 .add(MO: Src1)
10265 .addImm(Val: 16)
10266 .addReg(RegNo: TmpReg);
10267 break;
10268 }
10269 case AMDGPU::S_PACK_HH_B32_B16: {
10270 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10271 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10272 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
10273 .addImm(Val: 16)
10274 .add(MO: Src0);
10275 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
10276 .addImm(Val: 0xffff0000);
10277 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_AND_OR_B32_e64), DestReg: ResultReg)
10278 .add(MO: Src1)
10279 .addReg(RegNo: ImmReg)
10280 .addReg(RegNo: TmpReg);
10281 break;
10282 }
10283 default:
10284 llvm_unreachable("unhandled s_pack_* instruction");
10285 }
10286
10287 MachineOperand &Dest = Inst.getOperand(i: 0);
10288 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
10289 addUsersToMoveToVALUWorklist(DstReg: ResultReg, MRI, Worklist);
10290}
10291
10292void SIInstrInfo::addSCCDefUsersToVALUWorklist(const MachineOperand &Op,
10293 MachineInstr &SCCDefInst,
10294 SIInstrWorklist &Worklist,
10295 Register NewCond) const {
10296
10297 // Ensure that def inst defines SCC, which is still live.
10298 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() &&
10299 !Op.isDead() && Op.getParent() == &SCCDefInst);
10300 SmallVector<MachineInstr *, 4> CopyToDelete;
10301 // This assumes that all the users of SCC are in the same block
10302 // as the SCC def.
10303 for (MachineInstr &MI : // Skip the def inst itself.
10304 make_range(x: std::next(x: MachineBasicBlock::iterator(SCCDefInst)),
10305 y: SCCDefInst.getParent()->end())) {
10306 // Check if SCC is used first.
10307 int SCCIdx = MI.findRegisterUseOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isKill: false);
10308 if (SCCIdx != -1) {
10309 if (MI.isCopy()) {
10310 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
10311 Register DestReg = MI.getOperand(i: 0).getReg();
10312
10313 MRI.replaceRegWith(FromReg: DestReg, ToReg: NewCond);
10314 CopyToDelete.push_back(Elt: &MI);
10315 } else {
10316
10317 if (NewCond.isValid())
10318 MI.getOperand(i: SCCIdx).setReg(NewCond);
10319
10320 Worklist.insert(MI: &MI);
10321 }
10322 }
10323 // Exit if we find another SCC def.
10324 if (MI.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isDead: false, Overlap: false) != -1)
10325 break;
10326 }
10327 for (auto &Copy : CopyToDelete)
10328 Copy->eraseFromParent();
10329}
10330
10331// Instructions that use SCC may be converted to VALU instructions. When that
10332// happens, the SCC register is changed to VCC_LO. The instruction that defines
10333// SCC must be changed to an instruction that defines VCC. This function makes
10334// sure that the instruction that defines SCC is added to the moveToVALU
10335// worklist.
10336void SIInstrInfo::addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
10337 SIInstrWorklist &Worklist) const {
10338 // Look for a preceding instruction that either defines VCC or SCC. If VCC
10339 // then there is nothing to do because the defining instruction has been
10340 // converted to a VALU already. If SCC then that instruction needs to be
10341 // converted to a VALU.
10342 for (MachineInstr &MI :
10343 make_range(x: std::next(x: MachineBasicBlock::reverse_iterator(SCCUseInst)),
10344 y: SCCUseInst->getParent()->rend())) {
10345 if (MI.modifiesRegister(Reg: AMDGPU::VCC, TRI: &RI))
10346 break;
10347 if (MI.definesRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
10348 Worklist.insert(MI: &MI);
10349 break;
10350 }
10351 }
10352}
10353
10354const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
10355 const MachineInstr &Inst) const {
10356 const TargetRegisterClass *NewDstRC = getOpRegClass(MI: Inst, OpNo: 0);
10357
10358 switch (Inst.getOpcode()) {
10359 // For target instructions, getOpRegClass just returns the virtual register
10360 // class associated with the operand, so we need to find an equivalent VGPR
10361 // register class in order to move the instruction to the VALU.
10362 case AMDGPU::COPY:
10363 case AMDGPU::PHI:
10364 case AMDGPU::REG_SEQUENCE:
10365 case AMDGPU::INSERT_SUBREG:
10366 case AMDGPU::WQM:
10367 case AMDGPU::SOFT_WQM:
10368 case AMDGPU::STRICT_WWM:
10369 case AMDGPU::STRICT_WQM: {
10370 const TargetRegisterClass *SrcRC = getOpRegClass(MI: Inst, OpNo: 1);
10371 if (RI.isAGPRClass(RC: SrcRC)) {
10372 if (RI.isAGPRClass(RC: NewDstRC))
10373 return nullptr;
10374
10375 switch (Inst.getOpcode()) {
10376 case AMDGPU::PHI:
10377 case AMDGPU::REG_SEQUENCE:
10378 case AMDGPU::INSERT_SUBREG:
10379 NewDstRC = RI.getEquivalentAGPRClass(SRC: NewDstRC);
10380 break;
10381 default:
10382 NewDstRC = RI.getEquivalentVGPRClass(SRC: NewDstRC);
10383 }
10384
10385 if (!NewDstRC)
10386 return nullptr;
10387 } else {
10388 if (!RI.isSGPRClass(RC: NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass)
10389 return nullptr;
10390
10391 NewDstRC = RI.getEquivalentVGPRClass(SRC: NewDstRC);
10392 if (!NewDstRC)
10393 return nullptr;
10394 }
10395
10396 return NewDstRC;
10397 }
10398 default:
10399 return NewDstRC;
10400 }
10401}
10402
10403// Find the one SGPR operand we are allowed to use.
10404Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
10405 int OpIndices[3]) const {
10406 const MCInstrDesc &Desc = MI.getDesc();
10407
10408 // Find the one SGPR operand we are allowed to use.
10409 //
10410 // First we need to consider the instruction's operand requirements before
10411 // legalizing. Some operands are required to be SGPRs, such as implicit uses
10412 // of VCC, but we are still bound by the constant bus requirement to only use
10413 // one.
10414 //
10415 // If the operand's class is an SGPR, we can never move it.
10416
10417 Register SGPRReg = findImplicitSGPRRead(MI);
10418 if (SGPRReg)
10419 return SGPRReg;
10420
10421 Register UsedSGPRs[3] = {Register()};
10422 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
10423
10424 for (unsigned i = 0; i < 3; ++i) {
10425 int Idx = OpIndices[i];
10426 if (Idx == -1)
10427 break;
10428
10429 const MachineOperand &MO = MI.getOperand(i: Idx);
10430 if (!MO.isReg())
10431 continue;
10432
10433 // Is this operand statically required to be an SGPR based on the operand
10434 // constraints?
10435 const TargetRegisterClass *OpRC =
10436 RI.getRegClass(i: getOpRegClassID(OpInfo: Desc.operands()[Idx]));
10437 bool IsRequiredSGPR = RI.isSGPRClass(RC: OpRC);
10438 if (IsRequiredSGPR)
10439 return MO.getReg();
10440
10441 // If this could be a VGPR or an SGPR, Check the dynamic register class.
10442 Register Reg = MO.getReg();
10443 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
10444 if (RI.isSGPRClass(RC: RegRC))
10445 UsedSGPRs[i] = Reg;
10446 }
10447
10448 // We don't have a required SGPR operand, so we have a bit more freedom in
10449 // selecting operands to move.
10450
10451 // Try to select the most used SGPR. If an SGPR is equal to one of the
10452 // others, we choose that.
10453 //
10454 // e.g.
10455 // V_FMA_F32 v0, s0, s0, s0 -> No moves
10456 // V_FMA_F32 v0, s0, s1, s0 -> Move s1
10457
10458 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
10459 // prefer those.
10460
10461 if (UsedSGPRs[0]) {
10462 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
10463 SGPRReg = UsedSGPRs[0];
10464 }
10465
10466 if (!SGPRReg && UsedSGPRs[1]) {
10467 if (UsedSGPRs[1] == UsedSGPRs[2])
10468 SGPRReg = UsedSGPRs[1];
10469 }
10470
10471 return SGPRReg;
10472}
10473
10474MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI,
10475 AMDGPU::OpName OperandName) const {
10476 if (OperandName == AMDGPU::OpName::NUM_OPERAND_NAMES)
10477 return nullptr;
10478
10479 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OperandName);
10480 if (Idx == -1)
10481 return nullptr;
10482
10483 return &MI.getOperand(i: Idx);
10484}
10485
10486uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {
10487 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
10488 int64_t Format = ST.getGeneration() >= AMDGPUSubtarget::GFX11
10489 ? (int64_t)AMDGPU::UfmtGFX11::UFMT_32_FLOAT
10490 : (int64_t)AMDGPU::UfmtGFX10::UFMT_32_FLOAT;
10491 return (Format << 44) |
10492 (1ULL << 56) | // RESOURCE_LEVEL = 1
10493 (3ULL << 60); // OOB_SELECT = 3
10494 }
10495
10496 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
10497 if (ST.isAmdHsaOS()) {
10498 // Set ATC = 1. GFX9 doesn't have this bit.
10499 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10500 RsrcDataFormat |= (1ULL << 56);
10501
10502 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
10503 // BTW, it disables TC L2 and therefore decreases performance.
10504 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)
10505 RsrcDataFormat |= (2ULL << 59);
10506 }
10507
10508 return RsrcDataFormat;
10509}
10510
10511uint64_t SIInstrInfo::getScratchRsrcWords23() const {
10512 uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
10513 AMDGPU::RSRC_TID_ENABLE |
10514 0xffffffff; // Size;
10515
10516 // GFX9 doesn't have ELEMENT_SIZE.
10517 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
10518 uint64_t EltSizeValue = Log2_32(Value: ST.getMaxPrivateElementSize(ForBufferRSrc: true)) - 1;
10519 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
10520 }
10521
10522 // IndexStride = 64 / 32.
10523 uint64_t IndexStride = ST.isWave64() ? 3 : 2;
10524 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
10525
10526 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
10527 // Clear them unless we want a huge stride.
10528 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
10529 ST.getGeneration() <= AMDGPUSubtarget::GFX9)
10530 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
10531
10532 return Rsrc23;
10533}
10534
10535bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const {
10536 unsigned Opc = MI.getOpcode();
10537
10538 return isSMRD(Opcode: Opc);
10539}
10540
10541bool SIInstrInfo::isHighLatencyDef(int Opc) const {
10542 return get(Opcode: Opc).mayLoad() &&
10543 (isMUBUF(Opcode: Opc) || isMTBUF(Opcode: Opc) || isMIMG(Opcode: Opc) || isFLAT(Opcode: Opc));
10544}
10545
10546Register SIInstrInfo::isStackAccess(const MachineInstr &MI, int &FrameIndex,
10547 TypeSize &MemBytes) const {
10548 const MachineOperand *Addr = getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
10549 if (!Addr || !Addr->isFI())
10550 return Register();
10551
10552 assert(!MI.memoperands_empty() &&
10553 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS);
10554
10555 FrameIndex = Addr->getIndex();
10556
10557 int VDataIdx =
10558 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdata);
10559 MemBytes = TypeSize::getFixed(ExactSize: getOpSize(Opcode: MI.getOpcode(), OpNo: VDataIdx));
10560 return MI.getOperand(i: VDataIdx).getReg();
10561}
10562
10563Register SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex,
10564 TypeSize &MemBytes) const {
10565 const MachineOperand *Addr = getNamedOperand(MI, OperandName: AMDGPU::OpName::addr);
10566 assert(Addr && Addr->isFI());
10567 FrameIndex = Addr->getIndex();
10568
10569 int DataIdx =
10570 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::data);
10571 MemBytes = TypeSize::getFixed(ExactSize: getOpSize(Opcode: MI.getOpcode(), OpNo: DataIdx));
10572 return MI.getOperand(i: DataIdx).getReg();
10573}
10574
10575Register SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
10576 int &FrameIndex,
10577 TypeSize &MemBytes) const {
10578 if (!MI.mayLoad())
10579 return Register();
10580
10581 if (isMUBUF(MI) || isVGPRSpill(MI))
10582 return isStackAccess(MI, FrameIndex, MemBytes);
10583
10584 if (isSGPRSpill(MI))
10585 return isSGPRStackAccess(MI, FrameIndex, MemBytes);
10586
10587 return Register();
10588}
10589
10590Register SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
10591 int &FrameIndex,
10592 TypeSize &MemBytes) const {
10593 if (!MI.mayStore())
10594 return Register();
10595
10596 if (isMUBUF(MI) || isVGPRSpill(MI))
10597 return isStackAccess(MI, FrameIndex, MemBytes);
10598
10599 if (isSGPRSpill(MI))
10600 return isSGPRStackAccess(MI, FrameIndex, MemBytes);
10601
10602 return Register();
10603}
10604
10605unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
10606 unsigned Opc = MI.getOpcode();
10607 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: Opc);
10608 unsigned DescSize = Desc.getSize();
10609 if (Opc == AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay)
10610 return DescSize + 12; // Three scalar moves around the load.
10611
10612 // If we have a definitive size, we can use it. Otherwise we need to inspect
10613 // the operands to know the size.
10614 if (isFixedSize(MI)) {
10615 unsigned Size = DescSize;
10616
10617 // If we hit the buggy offset, an extra nop will be inserted in MC so
10618 // estimate the worst case.
10619 if (MI.isBranch() && ST.hasOffset3fBug())
10620 Size += 4;
10621
10622 return Size;
10623 }
10624
10625 // Instructions may have a 32-bit literal encoded after them. Check
10626 // operands that could ever be literals.
10627 if (isVALU(MI, /*AllowLDSDMA=*/false) || isSALU(MI)) {
10628 if (isDPP(MI))
10629 return DescSize;
10630 bool HasLiteral = false;
10631 unsigned LiteralSize = 4;
10632 for (int I = 0, E = MI.getNumExplicitOperands(); I != E; ++I) {
10633 const MachineOperand &Op = MI.getOperand(i: I);
10634 const MCOperandInfo &OpInfo = Desc.operands()[I];
10635 if (!Op.isReg() && !isInlineConstant(MO: Op, OpInfo)) {
10636 HasLiteral = true;
10637 if (ST.has64BitLiterals()) {
10638 switch (OpInfo.OperandType) {
10639 default:
10640 break;
10641 case AMDGPU::OPERAND_REG_IMM_FP64:
10642 case AMDGPU::OPERAND_REG_IMM_V2FP64:
10643 if (!AMDGPU::isValid32BitLiteral(Val: Op.getImm(), IsFP64: true))
10644 LiteralSize = 8;
10645 break;
10646 case AMDGPU::OPERAND_REG_IMM_INT64:
10647 case AMDGPU::OPERAND_REG_IMM_V2INT64:
10648 // A 32-bit literal is only valid when the value fits in BOTH signed
10649 // and unsigned 32-bit ranges [0, 2^31-1], matching the MC code
10650 // emitter's getLit64Encoding logic. This is because of the lack of
10651 // abilility to tell signedness of the literal, therefore we need to
10652 // be conservative and assume values outside this range require a
10653 // 64-bit literal encoding (8 bytes).
10654 if (!Op.isImm() || !isInt<32>(x: Op.getImm()) ||
10655 !isUInt<32>(x: Op.getImm()))
10656 LiteralSize = 8;
10657 break;
10658 }
10659 }
10660 break;
10661 }
10662 }
10663 return HasLiteral ? DescSize + LiteralSize : DescSize;
10664 }
10665
10666 // Check whether we have extra NSA words.
10667 if (isMIMG(MI)) {
10668 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0);
10669 if (VAddr0Idx < 0)
10670 return 8;
10671
10672 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::srsrc);
10673 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4);
10674 }
10675
10676 switch (Opc) {
10677 case TargetOpcode::BUNDLE:
10678 return getInstBundleSize(MI);
10679 case TargetOpcode::INLINEASM:
10680 case TargetOpcode::INLINEASM_BR: {
10681 const MachineFunction *MF = MI.getMF();
10682 const char *AsmStr = MI.getOperand(i: 0).getSymbolName();
10683 return getInlineAsmLength(Str: AsmStr, MAI: MF->getTarget().getMCAsmInfo(), STI: &ST);
10684 }
10685 default:
10686 if (MI.isMetaInstruction())
10687 return 0;
10688
10689 // If D16 Pseudo inst, get correct MC code size
10690 const auto *D16Info = AMDGPU::getT16D16Helper(T16Op: Opc);
10691 if (D16Info) {
10692 // Assume d16_lo/hi inst are always in same size
10693 unsigned LoInstOpcode = D16Info->LoOp;
10694 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: LoInstOpcode);
10695 DescSize = Desc.getSize();
10696 }
10697
10698 // If FMA Pseudo inst, get correct MC code size
10699 if (Opc == AMDGPU::V_FMA_MIX_F16_t16 || Opc == AMDGPU::V_FMA_MIX_BF16_t16) {
10700 // All potential lowerings are the same size; arbitrarily pick one.
10701 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: AMDGPU::V_FMA_MIXLO_F16);
10702 DescSize = Desc.getSize();
10703 }
10704
10705 return DescSize;
10706 }
10707}
10708
10709TargetInstrInfo::InstSizeVerifyMode
10710SIInstrInfo::getInstSizeVerifyMode(const MachineInstr &MI) const {
10711 if (MI.isBranch() && ST.hasOffset3fBug())
10712 return InstSizeVerifyMode::NoVerify;
10713 return InstSizeVerifyMode::ExactSize;
10714}
10715
10716ArrayRef<std::pair<int, const char *>>
10717SIInstrInfo::getSerializableTargetIndices() const {
10718 static const std::pair<int, const char *> TargetIndices[] = {
10719 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
10720 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
10721 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
10722 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
10723 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
10724 return ArrayRef(TargetIndices);
10725}
10726
10727/// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The
10728/// post-RA version of misched uses CreateTargetMIHazardRecognizer.
10729ScheduleHazardRecognizer *
10730SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
10731 const ScheduleDAG *DAG) const {
10732 return new GCNHazardRecognizer(DAG->MF);
10733}
10734
10735/// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer
10736/// pass.
10737ScheduleHazardRecognizer *
10738SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF,
10739 MachineLoopInfo *MLI) const {
10740 return new GCNHazardRecognizer(
10741 MF, GCNHazardRecognizer::OperatingMode::HazardRecognizerMode, MLI);
10742}
10743
10744// Called during:
10745// - pre-RA scheduling and post-RA scheduling
10746ScheduleHazardRecognizer *
10747SIInstrInfo::CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
10748 const ScheduleDAGMI *DAG) const {
10749 // Borrowed from Arm Target
10750 // We would like to restrict this hazard recognizer to only
10751 // post-RA scheduling; we can tell that we're post-RA because we don't
10752 // track VRegLiveness.
10753 if (!DAG->hasVRegLiveness())
10754 return new GCNHazardRecognizer(DAG->MF);
10755 return TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG);
10756}
10757
10758std::pair<unsigned, unsigned>
10759SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
10760 return std::pair(TF & MO_MASK, TF & ~MO_MASK);
10761}
10762
10763ArrayRef<std::pair<unsigned, const char *>>
10764SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
10765 static const std::pair<unsigned, const char *> TargetFlags[] = {
10766 {MO_GOTPCREL, "amdgpu-gotprel"},
10767 {MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo"},
10768 {MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi"},
10769 {MO_GOTPCREL64, "amdgpu-gotprel64"},
10770 {MO_REL32_LO, "amdgpu-rel32-lo"},
10771 {MO_REL32_HI, "amdgpu-rel32-hi"},
10772 {MO_REL64, "amdgpu-rel64"},
10773 {MO_ABS32_LO, "amdgpu-abs32-lo"},
10774 {MO_ABS32_HI, "amdgpu-abs32-hi"},
10775 {MO_ABS64, "amdgpu-abs64"},
10776 };
10777
10778 return ArrayRef(TargetFlags);
10779}
10780
10781ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
10782SIInstrInfo::getSerializableMachineMemOperandTargetFlags() const {
10783 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
10784 {
10785 {MONoClobber, "amdgpu-noclobber"},
10786 {MOLastUse, "amdgpu-last-use"},
10787 {MOCooperative, "amdgpu-cooperative"},
10788 {MOThreadPrivate, "amdgpu-thread-private"},
10789 {MOCFSB0, "amdgpu-cfs0"},
10790 {MOCFSB1, "amdgpu-cfs1"},
10791 };
10792
10793 return ArrayRef(TargetFlags);
10794}
10795
10796unsigned SIInstrInfo::getLiveRangeSplitOpcode(Register SrcReg,
10797 const MachineFunction &MF) const {
10798 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
10799 assert(SrcReg.isVirtual());
10800 if (MFI->checkFlag(Reg: SrcReg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
10801 return AMDGPU::WWM_COPY;
10802
10803 return AMDGPU::COPY;
10804}
10805
10806bool SIInstrInfo::canAddToBBProlog(const MachineInstr &MI) const {
10807 uint32_t Opcode = MI.getOpcode();
10808 // Check if it is SGPR spill or wwm-register spill Opcode.
10809 if (isSGPRSpill(Opcode) || isWWMRegSpillOpcode(Opcode))
10810 return true;
10811
10812 const MachineFunction *MF = MI.getMF();
10813 const MachineRegisterInfo &MRI = MF->getRegInfo();
10814 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
10815
10816 // See if this is Liverange split instruction inserted for SGPR or
10817 // wwm-register. The implicit def inserted for wwm-registers should also be
10818 // included as they can appear at the bb begin.
10819 bool IsLRSplitInst = MI.getFlag(Flag: MachineInstr::LRSplit);
10820 if (!IsLRSplitInst && Opcode != AMDGPU::IMPLICIT_DEF)
10821 return false;
10822
10823 Register Reg = MI.getOperand(i: 0).getReg();
10824 if (RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg)))
10825 return IsLRSplitInst;
10826
10827 return MFI->isWWMReg(Reg);
10828}
10829
10830bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI,
10831 Register Reg) const {
10832 // We need to handle instructions which may be inserted during register
10833 // allocation to handle the prolog. The initial prolog instruction may have
10834 // been separated from the start of the block by spills and copies inserted
10835 // needed by the prolog. However, the insertions for scalar registers can
10836 // always be placed at the BB top as they are independent of the exec mask
10837 // value.
10838 bool IsNullOrVectorRegister = true;
10839 if (Reg) {
10840 const MachineFunction *MF = MI.getMF();
10841 const MachineRegisterInfo &MRI = MF->getRegInfo();
10842 IsNullOrVectorRegister = !RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg));
10843 }
10844
10845 return IsNullOrVectorRegister &&
10846 (canAddToBBProlog(MI) ||
10847 (!MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY &&
10848 MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &RI)));
10849}
10850
10851MachineInstrBuilder
10852SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
10853 MachineBasicBlock::iterator I,
10854 const DebugLoc &DL,
10855 Register DestReg) const {
10856 if (ST.hasAddNoCarryInsts())
10857 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e64), DestReg);
10858
10859 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
10860 Register UnusedCarry = MRI.createVirtualRegister(RegClass: RI.getBoolRC());
10861 MRI.setRegAllocationHint(VReg: UnusedCarry, Type: 0, PrefReg: RI.getVCC());
10862
10863 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg)
10864 .addReg(RegNo: UnusedCarry, Flags: RegState::Define | RegState::Dead);
10865}
10866
10867MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
10868 MachineBasicBlock::iterator I,
10869 const DebugLoc &DL,
10870 Register DestReg,
10871 RegScavenger &RS) const {
10872 if (ST.hasAddNoCarryInsts())
10873 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg);
10874
10875 // If available, prefer to use vcc.
10876 Register UnusedCarry = !RS.isRegUsed(Reg: AMDGPU::VCC)
10877 ? Register(RI.getVCC())
10878 : RS.scavengeRegisterBackwards(
10879 RC: *RI.getBoolRC(), To: I, /* RestoreAfter */ false,
10880 SPAdj: 0, /* AllowSpill */ false);
10881
10882 // TODO: Users need to deal with this.
10883 if (!UnusedCarry.isValid())
10884 return MachineInstrBuilder();
10885
10886 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg)
10887 .addReg(RegNo: UnusedCarry, Flags: RegState::Define | RegState::Dead);
10888}
10889
10890bool SIInstrInfo::isKillTerminator(unsigned Opcode) {
10891 switch (Opcode) {
10892 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
10893 case AMDGPU::SI_KILL_I1_TERMINATOR:
10894 return true;
10895 default:
10896 return false;
10897 }
10898}
10899
10900const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const {
10901 switch (Opcode) {
10902 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
10903 return get(Opcode: AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);
10904 case AMDGPU::SI_KILL_I1_PSEUDO:
10905 return get(Opcode: AMDGPU::SI_KILL_I1_TERMINATOR);
10906 default:
10907 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO");
10908 }
10909}
10910
10911bool SIInstrInfo::isLegalMUBUFImmOffset(unsigned Imm) const {
10912 return Imm <= getMaxMUBUFImmOffset(ST);
10913}
10914
10915unsigned SIInstrInfo::getMaxMUBUFImmOffset(const GCNSubtarget &ST) {
10916 // GFX12 field is non-negative 24-bit signed byte offset.
10917 const unsigned OffsetBits =
10918 ST.getGeneration() >= AMDGPUSubtarget::GFX12 ? 23 : 12;
10919 return (1 << OffsetBits) - 1;
10920}
10921
10922void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const {
10923 if (!ST.isWave32())
10924 return;
10925
10926 if (MI.isInlineAsm())
10927 return;
10928
10929 if (MI.getNumOperands() < MI.getDesc().getNumOperands())
10930 return;
10931
10932 for (auto &Op : MI.implicit_operands()) {
10933 if (Op.isReg() && Op.getReg() == AMDGPU::VCC)
10934 Op.setReg(AMDGPU::VCC_LO);
10935 }
10936}
10937
10938bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const {
10939 if (!isSMRD(MI))
10940 return false;
10941
10942 // Check that it is using a buffer resource.
10943 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::sbase);
10944 if (Idx == -1) // e.g. s_memtime
10945 return false;
10946
10947 const int16_t RCID = getOpRegClassID(OpInfo: MI.getDesc().operands()[Idx]);
10948 return RI.getRegClass(i: RCID)->hasSubClassEq(RC: &AMDGPU::SGPR_128RegClass);
10949}
10950
10951// Given Imm, split it into the values to put into the SOffset and ImmOffset
10952// fields in an MUBUF instruction. Return false if it is not possible (due to a
10953// hardware bug needing a workaround).
10954//
10955// The required alignment ensures that individual address components remain
10956// aligned if they are aligned to begin with. It also ensures that additional
10957// offsets within the given alignment can be added to the resulting ImmOffset.
10958bool SIInstrInfo::splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset,
10959 uint32_t &ImmOffset, Align Alignment) const {
10960 const uint64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST);
10961 const uint32_t MaxImm = alignDown(Value: MaxOffset, Align: Alignment.value());
10962 uint32_t Overflow = 0;
10963
10964 if (Imm > MaxImm) {
10965 if (Imm <= MaxImm + 64) {
10966 // Use an SOffset inline constant for 4..64
10967 Overflow = Imm - MaxImm;
10968 Imm = MaxImm;
10969 } else {
10970 // Try to keep the same value in SOffset for adjacent loads, so that
10971 // the corresponding register contents can be re-used.
10972 //
10973 // Load values with all low-bits (except for alignment bits) set into
10974 // SOffset, so that a larger range of values can be covered using
10975 // s_movk_i32.
10976 //
10977 // Atomic operations fail to work correctly when individual address
10978 // components are unaligned, even if their sum is aligned.
10979 uint32_t High = (Imm + Alignment.value()) & ~MaxOffset;
10980 uint32_t Low = (Imm + Alignment.value()) & MaxOffset;
10981 Imm = Low;
10982 Overflow = High - Alignment.value();
10983 }
10984 }
10985
10986 if (Overflow > 0) {
10987 // There is a hardware bug in SI and CI which prevents address clamping in
10988 // MUBUF instructions from working correctly with SOffsets. The immediate
10989 // offset is unaffected.
10990 if (ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
10991 return false;
10992
10993 // It is not possible to set immediate in SOffset field on some targets.
10994 if (ST.hasRestrictedSOffset())
10995 return false;
10996 }
10997
10998 ImmOffset = Imm;
10999 SOffset = Overflow;
11000 return true;
11001}
11002
11003// Depending on the used address space and instructions, some immediate offsets
11004// are allowed and some are not.
11005// Pre-GFX12, flat instruction offsets can only be non-negative, global and
11006// scratch instruction offsets can also be negative. On GFX12, offsets can be
11007// negative for all variants.
11008//
11009// There are several bugs related to these offsets:
11010// On gfx10.1, flat instructions that go into the global address space cannot
11011// use an offset.
11012//
11013// For scratch instructions, the address can be either an SGPR or a VGPR.
11014// The following offsets can be used, depending on the architecture (x means
11015// cannot be used):
11016// +----------------------------+------+------+
11017// | Address-Mode | SGPR | VGPR |
11018// +----------------------------+------+------+
11019// | gfx9 | | |
11020// | negative, 4-aligned offset | x | ok |
11021// | negative, unaligned offset | x | ok |
11022// +----------------------------+------+------+
11023// | gfx10 | | |
11024// | negative, 4-aligned offset | ok | ok |
11025// | negative, unaligned offset | ok | x |
11026// +----------------------------+------+------+
11027// | gfx10.3 | | |
11028// | negative, 4-aligned offset | ok | ok |
11029// | negative, unaligned offset | ok | ok |
11030// +----------------------------+------+------+
11031//
11032// This function ignores the addressing mode, so if an offset cannot be used in
11033// one addressing mode, it is considered illegal.
11034bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
11035 AMDGPU::FlatAddrSpace FlatVariant) const {
11036 // TODO: Should 0 be special cased?
11037 if (!ST.hasFlatInstOffsets())
11038 return false;
11039
11040 using AMDGPU::FlatAddrSpace;
11041 if (ST.hasFlatSegmentOffsetBug() && FlatVariant == FlatAddrSpace::FLAT &&
11042 (AddrSpace == AMDGPUAS::FLAT_ADDRESS ||
11043 AMDGPU::isExtendedGlobalAddrSpace(AS: AddrSpace)))
11044 return false;
11045
11046 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
11047 FlatVariant == FlatAddrSpace::FlatScratch && Offset < 0 &&
11048 (Offset % 4) != 0) {
11049 return false;
11050 }
11051
11052 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
11053 unsigned N = AMDGPU::getNumFlatOffsetBits(ST);
11054 return isIntN(N, x: Offset) && (AllowNegative || Offset >= 0);
11055}
11056
11057// See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not.
11058std::pair<int64_t, int64_t>
11059SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,
11060 AMDGPU::FlatAddrSpace FlatVariant) const {
11061 int64_t RemainderOffset = COffsetVal;
11062 int64_t ImmField = 0;
11063
11064 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
11065 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST) - 1;
11066
11067 if (AllowNegative) {
11068 // Use signed division by a power of two to truncate towards 0.
11069 int64_t D = 1LL << NumBits;
11070 RemainderOffset = (COffsetVal / D) * D;
11071 ImmField = COffsetVal - RemainderOffset;
11072
11073 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
11074 FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch && ImmField < 0 &&
11075 (ImmField % 4) != 0) {
11076 // Make ImmField a multiple of 4
11077 RemainderOffset += ImmField % 4;
11078 ImmField -= ImmField % 4;
11079 }
11080 } else if (COffsetVal >= 0) {
11081 ImmField = COffsetVal & maskTrailingOnes<uint64_t>(N: NumBits);
11082 RemainderOffset = COffsetVal - ImmField;
11083 }
11084
11085 assert(isLegalFLATOffset(ImmField, AddrSpace, FlatVariant));
11086 assert(RemainderOffset + ImmField == COffsetVal);
11087 return {ImmField, RemainderOffset};
11088}
11089
11090bool SIInstrInfo::allowNegativeFlatOffset(
11091 AMDGPU::FlatAddrSpace FlatVariant) const {
11092 if (ST.hasNegativeScratchOffsetBug() &&
11093 FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch)
11094 return false;
11095
11096 return FlatVariant != AMDGPU::FlatAddrSpace::FLAT || AMDGPU::isGFX12Plus(STI: ST);
11097}
11098
11099static unsigned subtargetEncodingFamily(const GCNSubtarget &ST) {
11100 switch (ST.getGeneration()) {
11101 default:
11102 break;
11103 case AMDGPUSubtarget::SOUTHERN_ISLANDS:
11104 case AMDGPUSubtarget::SEA_ISLANDS:
11105 return SIEncodingFamily::SI;
11106 case AMDGPUSubtarget::VOLCANIC_ISLANDS:
11107 // The GFX80 encoding family only contains buffer instructions with unpacked
11108 // D16 data; pseudoToMCOpcode falls back on VI for everything else.
11109 // TODO: remove this when we discard GFX80 encoding.
11110 return ST.hasUnpackedD16VMem() ? SIEncodingFamily::GFX80
11111 : SIEncodingFamily::VI;
11112 case AMDGPUSubtarget::GFX9:
11113 return SIEncodingFamily::VI;
11114 case AMDGPUSubtarget::GFX10:
11115 return SIEncodingFamily::GFX10;
11116 case AMDGPUSubtarget::GFX11:
11117 return ST.hasGFX11_7Insts() ? SIEncodingFamily::GFX1170
11118 : SIEncodingFamily::GFX11;
11119 case AMDGPUSubtarget::GFX12:
11120 return ST.hasGFX1250Insts() ? SIEncodingFamily::GFX1250
11121 : SIEncodingFamily::GFX12;
11122 case AMDGPUSubtarget::GFX13:
11123 return SIEncodingFamily::GFX13;
11124 }
11125 llvm_unreachable("Unknown subtarget generation!");
11126}
11127
11128bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const {
11129 switch(MCOp) {
11130 // These opcodes use indirect register addressing so
11131 // they need special handling by codegen (currently missing).
11132 // Therefore it is too risky to allow these opcodes
11133 // to be selected by dpp combiner or sdwa peepholer.
11134 case AMDGPU::V_MOVRELS_B32_dpp_gfx10:
11135 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
11136 case AMDGPU::V_MOVRELD_B32_dpp_gfx10:
11137 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10:
11138 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10:
11139 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
11140 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10:
11141 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
11142 return true;
11143 default:
11144 return false;
11145 }
11146}
11147
11148#define GENERATE_RENAMED_GFX9_CASES(OPCODE) \
11149 case OPCODE##_dpp: \
11150 case OPCODE##_e32: \
11151 case OPCODE##_e64:
11152
11153static bool isRenamedInGFX9(int Opcode) {
11154 switch (Opcode) {
11155 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADDC_U32)
11156 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_CO_U32)
11157 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_U32)
11158 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBBREV_U32)
11159 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBB_U32)
11160 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_CO_U32)
11161 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_U32)
11162 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_CO_U32)
11163 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_U32)
11164 case AMDGPU::V_DIV_FIXUP_F16_gfx9_e64:
11165 case AMDGPU::V_FMA_F16_gfx9_e64:
11166 case AMDGPU::V_INTERP_P2_F16:
11167 case AMDGPU::V_MAD_F16_e64:
11168 case AMDGPU::V_MAD_U16_e64:
11169 case AMDGPU::V_MAD_I16_e64:
11170 case AMDGPU::V_MAD_U16_gfx9_e64:
11171 case AMDGPU::V_MAD_I16_gfx9_e64:
11172 return true;
11173 default:
11174 return false;
11175 }
11176}
11177
11178int SIInstrInfo::pseudoToMCOpcode(int Opcode) const {
11179 assert(Opcode == (int)SIInstrInfo::getNonSoftWaitcntOpcode(Opcode) &&
11180 "SIInsertWaitcnts should have promoted soft waitcnt instructions!");
11181
11182 unsigned Gen = subtargetEncodingFamily(ST);
11183
11184 if (ST.getGeneration() == AMDGPUSubtarget::GFX9 && isRenamedInGFX9(Opcode))
11185 Gen = SIEncodingFamily::GFX9;
11186
11187 if (SIInstrFlags::isSDWA(O: get(Opcode))) {
11188 switch (ST.getGeneration()) {
11189 default:
11190 Gen = SIEncodingFamily::SDWA;
11191 break;
11192 case AMDGPUSubtarget::GFX9:
11193 Gen = SIEncodingFamily::SDWA9;
11194 break;
11195 case AMDGPUSubtarget::GFX10:
11196 Gen = SIEncodingFamily::SDWA10;
11197 break;
11198 }
11199 }
11200
11201 if (isMAI(Opcode)) {
11202 int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode);
11203 if (MFMAOp != -1)
11204 Opcode = MFMAOp;
11205 }
11206
11207 int32_t MCOp = AMDGPU::getMCOpcode(Opcode, Gen);
11208
11209 // Only buffer instructions with unpacked D16 data have a GFX80 encoding.
11210 // Anything else on such a subtarget uses the plain VI encoding.
11211 // TODO: remove this when we discard GFX80 encoding.
11212 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && Gen == SIEncodingFamily::GFX80)
11213 MCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::VI);
11214
11215 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && ST.hasGFX11_7Insts())
11216 MCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX11);
11217
11218 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && ST.hasGFX1250Insts())
11219 MCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX12);
11220
11221 // -1 means that Opcode is already a native instruction.
11222 if (MCOp == -1)
11223 return Opcode;
11224
11225 if (ST.hasGFX90AInsts()) {
11226 uint32_t NMCOp = AMDGPU::INSTRUCTION_LIST_END;
11227 if (ST.hasGFX940Insts())
11228 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX940);
11229 if (NMCOp == AMDGPU::INSTRUCTION_LIST_END)
11230 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX90A);
11231 if (NMCOp == AMDGPU::INSTRUCTION_LIST_END)
11232 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX9);
11233 if (NMCOp != AMDGPU::INSTRUCTION_LIST_END)
11234 MCOp = NMCOp;
11235 }
11236
11237 // INSTRUCTION_LIST_END means that Opcode is a pseudo instruction that has no
11238 // encoding in the given subtarget generation.
11239 if (MCOp == AMDGPU::INSTRUCTION_LIST_END)
11240 return -1;
11241
11242 if (isAsmOnlyOpcode(MCOp))
11243 return -1;
11244
11245 return MCOp;
11246}
11247
11248static
11249TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) {
11250 assert(RegOpnd.isReg());
11251 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() :
11252 getRegSubRegPair(O: RegOpnd);
11253}
11254
11255TargetInstrInfo::RegSubRegPair
11256llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) {
11257 assert(MI.isRegSequence());
11258 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I)
11259 if (MI.getOperand(i: 1 + 2 * I + 1).getImm() == SubReg) {
11260 auto &RegOp = MI.getOperand(i: 1 + 2 * I);
11261 return getRegOrUndef(RegOpnd: RegOp);
11262 }
11263 return TargetInstrInfo::RegSubRegPair();
11264}
11265
11266// Try to find the definition of reg:subreg in subreg-manipulation pseudos
11267// Following a subreg of reg:subreg isn't supported
11268static bool followSubRegDef(MachineInstr &MI,
11269 TargetInstrInfo::RegSubRegPair &RSR) {
11270 if (!RSR.SubReg)
11271 return false;
11272 switch (MI.getOpcode()) {
11273 default: break;
11274 case AMDGPU::REG_SEQUENCE:
11275 RSR = getRegSequenceSubReg(MI, SubReg: RSR.SubReg);
11276 return true;
11277 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg
11278 case AMDGPU::INSERT_SUBREG:
11279 if (RSR.SubReg == (unsigned)MI.getOperand(i: 3).getImm())
11280 // inserted the subreg we're looking for
11281 RSR = getRegOrUndef(RegOpnd: MI.getOperand(i: 2));
11282 else { // the subreg in the rest of the reg
11283 auto R1 = getRegOrUndef(RegOpnd: MI.getOperand(i: 1));
11284 if (R1.SubReg) // subreg of subreg isn't supported
11285 return false;
11286 RSR.Reg = R1.Reg;
11287 }
11288 return true;
11289 }
11290 return false;
11291}
11292
11293MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
11294 const MachineRegisterInfo &MRI) {
11295 assert(MRI.isSSA());
11296 if (!P.Reg.isVirtual())
11297 return nullptr;
11298
11299 auto RSR = P;
11300 auto *DefInst = MRI.getVRegDef(Reg: RSR.Reg);
11301 while (auto *MI = DefInst) {
11302 DefInst = nullptr;
11303 switch (MI->getOpcode()) {
11304 case AMDGPU::COPY:
11305 case AMDGPU::V_MOV_B32_e32: {
11306 auto &Op1 = MI->getOperand(i: 1);
11307 if (Op1.isReg() && Op1.getReg().isVirtual()) {
11308 if (Op1.isUndef())
11309 return nullptr;
11310 RSR = getRegSubRegPair(O: Op1);
11311 DefInst = MRI.getVRegDef(Reg: RSR.Reg);
11312 }
11313 break;
11314 }
11315 default:
11316 if (followSubRegDef(MI&: *MI, RSR)) {
11317 if (!RSR.Reg)
11318 return nullptr;
11319 DefInst = MRI.getVRegDef(Reg: RSR.Reg);
11320 }
11321 }
11322 if (!DefInst)
11323 return MI;
11324 }
11325 return nullptr;
11326}
11327
11328bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
11329 Register VReg,
11330 const MachineInstr &DefMI,
11331 const MachineInstr &UseMI) {
11332 assert(MRI.isSSA() && "Must be run on SSA");
11333
11334 auto *TRI = MRI.getTargetRegisterInfo();
11335 auto *DefBB = DefMI.getParent();
11336
11337 // Don't bother searching between blocks, although it is possible this block
11338 // doesn't modify exec.
11339 if (UseMI.getParent() != DefBB)
11340 return true;
11341
11342 const int MaxInstScan = 20;
11343 int NumInst = 0;
11344
11345 // Stop scan at the use.
11346 auto E = UseMI.getIterator();
11347 for (auto I = std::next(x: DefMI.getIterator()); I != E; ++I) {
11348 if (I->isDebugInstr())
11349 continue;
11350
11351 if (++NumInst > MaxInstScan)
11352 return true;
11353
11354 if (I->modifiesRegister(Reg: AMDGPU::EXEC, TRI))
11355 return true;
11356 }
11357
11358 return false;
11359}
11360
11361bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
11362 Register VReg,
11363 const MachineInstr &DefMI) {
11364 assert(MRI.isSSA() && "Must be run on SSA");
11365
11366 auto *TRI = MRI.getTargetRegisterInfo();
11367 auto *DefBB = DefMI.getParent();
11368
11369 const int MaxUseScan = 10;
11370 int NumUse = 0;
11371
11372 for (auto &Use : MRI.use_nodbg_operands(Reg: VReg)) {
11373 auto &UseInst = *Use.getParent();
11374 // Don't bother searching between blocks, although it is possible this block
11375 // doesn't modify exec.
11376 if (UseInst.getParent() != DefBB || UseInst.isPHI())
11377 return true;
11378
11379 if (++NumUse > MaxUseScan)
11380 return true;
11381 }
11382
11383 if (NumUse == 0)
11384 return false;
11385
11386 const int MaxInstScan = 20;
11387 int NumInst = 0;
11388
11389 // Stop scan when we have seen all the uses.
11390 for (auto I = std::next(x: DefMI.getIterator()); ; ++I) {
11391 assert(I != DefBB->end());
11392
11393 if (I->isDebugInstr())
11394 continue;
11395
11396 if (++NumInst > MaxInstScan)
11397 return true;
11398
11399 for (const MachineOperand &Op : I->operands()) {
11400 // We don't check reg masks here as they're used only on calls:
11401 // 1. EXEC is only considered const within one BB
11402 // 2. Call should be a terminator instruction if present in a BB
11403
11404 if (!Op.isReg())
11405 continue;
11406
11407 Register Reg = Op.getReg();
11408 if (Op.isUse()) {
11409 if (Reg == VReg && --NumUse == 0)
11410 return false;
11411 } else if (TRI->regsOverlap(RegA: Reg, RegB: AMDGPU::EXEC))
11412 return true;
11413 }
11414 }
11415}
11416
11417MachineInstr *SIInstrInfo::createPHIDestinationCopy(
11418 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt,
11419 const DebugLoc &DL, Register Src, Register Dst) const {
11420 auto Cur = MBB.begin();
11421 if (Cur != MBB.end())
11422 do {
11423 if (!Cur->isPHI() && Cur->readsRegister(Reg: Dst, /*TRI=*/nullptr))
11424 return BuildMI(BB&: MBB, I: Cur, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Dst).addReg(RegNo: Src);
11425 ++Cur;
11426 } while (Cur != MBB.end() && Cur != LastPHIIt);
11427
11428 return TargetInstrInfo::createPHIDestinationCopy(MBB, InsPt: LastPHIIt, DL, Src,
11429 Dst);
11430}
11431
11432MachineInstr *SIInstrInfo::createPHISourceCopy(
11433 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt,
11434 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const {
11435 if (InsPt != MBB.end() &&
11436 (InsPt->getOpcode() == AMDGPU::SI_IF ||
11437 InsPt->getOpcode() == AMDGPU::SI_ELSE ||
11438 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) &&
11439 InsPt->definesRegister(Reg: Src, /*TRI=*/nullptr)) {
11440 InsPt++;
11441 return BuildMI(BB&: MBB, I: InsPt, MIMD: DL,
11442 MCID: get(Opcode: AMDGPU::LaneMaskConstants::get(ST).MovTermOpc), DestReg: Dst)
11443 .addReg(RegNo: Src, Flags: {}, SubReg: SrcSubReg)
11444 .addReg(RegNo: AMDGPU::EXEC, Flags: RegState::Implicit);
11445 }
11446 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg,
11447 Dst);
11448}
11449
11450bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); }
11451
11452bool SIInstrInfo::hasRAWDependency(const MachineInstr &FirstMI,
11453 const MachineInstr &SecondMI) const {
11454 for (const auto &Use : SecondMI.all_uses()) {
11455 if (Use.isReg() && FirstMI.modifiesRegister(Reg: Use.getReg(), TRI: &RI))
11456 return true;
11457 }
11458 return false;
11459}
11460
11461/// If OpX is multicycle, anti-dependencies are not allowed.
11462/// isDPMACCInstruction was not designed for VOPD, but it is fit for the
11463/// purpose.
11464bool llvm::SIInstrInfo::isVOPDAntidependencyAllowed(
11465 const MachineInstr &OpX) const {
11466 return !AMDGPU::isDPMACCInstruction(Opc: OpX.getOpcode());
11467}
11468
11469MachineInstr *
11470SIInstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
11471 ArrayRef<unsigned> Ops, int FrameIndex,
11472 MachineInstr *&CopyMI, LiveIntervals *LIS,
11473 VirtRegMap *VRM) const {
11474 // This is a bit of a hack (copied from AArch64). Consider this instruction:
11475 //
11476 // %0:sreg_32 = COPY $m0
11477 //
11478 // We explicitly chose SReg_32 for the virtual register so such a copy might
11479 // be eliminated by RegisterCoalescer. However, that may not be possible, and
11480 // %0 may even spill. We can't spill $m0 normally (it would require copying to
11481 // a numbered SGPR anyway), and since it is in the SReg_32 register class,
11482 // TargetInstrInfo::foldMemoryOperand() is going to try.
11483 // A similar issue also exists with spilling and reloading $exec registers.
11484 //
11485 // To prevent that, constrain the %0 register class here.
11486 if (isFullCopyInstr(MI)) {
11487 Register DstReg = MI.getOperand(i: 0).getReg();
11488 Register SrcReg = MI.getOperand(i: 1).getReg();
11489 if ((DstReg.isVirtual() || SrcReg.isVirtual()) &&
11490 (DstReg.isVirtual() != SrcReg.isVirtual())) {
11491 MachineRegisterInfo &MRI = MF.getRegInfo();
11492 Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg;
11493 const TargetRegisterClass *RC = MRI.getRegClass(Reg: VirtReg);
11494 if (RC->hasSuperClassEq(RC: &AMDGPU::SReg_32RegClass)) {
11495 MRI.constrainRegClass(Reg: VirtReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
11496 return nullptr;
11497 }
11498 if (RC->hasSuperClassEq(RC: &AMDGPU::SReg_64RegClass)) {
11499 MRI.constrainRegClass(Reg: VirtReg, RC: &AMDGPU::SReg_64_XEXECRegClass);
11500 return nullptr;
11501 }
11502 }
11503 }
11504
11505 return nullptr;
11506}
11507
11508unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
11509 const MachineInstr &MI,
11510 unsigned *PredCost) const {
11511 if (MI.isBundle()) {
11512 MachineBasicBlock::const_instr_iterator I(MI.getIterator());
11513 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end());
11514 unsigned Lat = 0, Count = 0;
11515 for (++I; I != E && I->isBundledWithPred(); ++I) {
11516 ++Count;
11517 Lat = std::max(a: Lat, b: SchedModel.computeInstrLatency(MI: &*I));
11518 }
11519 return Lat + Count - 1;
11520 }
11521
11522 return SchedModel.computeInstrLatency(MI: &MI);
11523}
11524
11525unsigned SIInstrInfo::getBlockingCycles(const MachineInstr &MI) const {
11526 if (!ST.hasGFX1250VALUBlockingCycles())
11527 return 0;
11528 return getGFX1250BlockingCyclesTable(MI);
11529}
11530
11531unsigned
11532SIInstrInfo::getGFX1250BlockingCyclesTable(const MachineInstr &MI) const {
11533 if (const auto *Entry = AMDGPU::getGFX1250BlockingCyclesInfo(Opcode: MI.getOpcode()))
11534 return Entry->GFX1250BlockingCycles;
11535 return 0;
11536}
11537
11538const MachineOperand &
11539SIInstrInfo::getCalleeOperand(const MachineInstr &MI) const {
11540 if (const MachineOperand *CallAddrOp =
11541 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0))
11542 return *CallAddrOp;
11543 return TargetInstrInfo::getCalleeOperand(MI);
11544}
11545
11546ValueUniformity
11547SIInstrInfo::getGenericValueUniformity(const MachineInstr &MI) const {
11548 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
11549 unsigned Opcode = MI.getOpcode();
11550
11551 auto HandleAddrSpaceCast = [this, &MRI](const MachineInstr &MI) {
11552 Register Dst = MI.getOperand(i: 0).getReg();
11553 Register Src = MI.getOperand(i: 1).getReg();
11554 LLT DstTy = MRI.getType(Reg: Dst);
11555 LLT SrcTy = MRI.getType(Reg: Src);
11556 unsigned DstAS = DstTy.getAddressSpace();
11557 unsigned SrcAS = SrcTy.getAddressSpace();
11558 return SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
11559 DstAS == AMDGPUAS::FLAT_ADDRESS &&
11560 ST.hasGloballyAddressableScratch()
11561 ? ValueUniformity::NeverUniform
11562 : ValueUniformity::Default;
11563 };
11564
11565 // If the target supports globally addressable scratch, the mapping from
11566 // scratch memory to the flat aperture changes therefore an address space cast
11567 // is no longer uniform.
11568 if (Opcode == TargetOpcode::G_ADDRSPACE_CAST)
11569 return HandleAddrSpaceCast(MI);
11570
11571 if (auto *GI = dyn_cast<GIntrinsic>(Val: &MI)) {
11572 auto IID = GI->getIntrinsicID();
11573 if (AMDGPU::isIntrinsicSourceOfDivergence(IntrID: IID))
11574 return ValueUniformity::NeverUniform;
11575 if (AMDGPU::isIntrinsicAlwaysUniform(IntrID: IID))
11576 return ValueUniformity::AlwaysUniform;
11577
11578 switch (IID) {
11579 case Intrinsic::amdgcn_if:
11580 case Intrinsic::amdgcn_else:
11581 // FIXME: Uniform if second result
11582 break;
11583 }
11584
11585 return ValueUniformity::Default;
11586 }
11587
11588 // Each lane reads its own registers, however uniform the index.
11589 if (Opcode == AMDGPU::G_AMDGPU_REG_LOAD)
11590 return ValueUniformity::NeverUniform;
11591
11592 // Loads from the private and flat address spaces are divergent, because
11593 // threads can execute the load instruction with the same inputs and get
11594 // different results. So are VGPR address space loads, before they become
11595 // G_AMDGPU_REG_LOAD.
11596 //
11597 // All other loads are not divergent, because if threads issue loads with the
11598 // same arguments, they will always get the same result.
11599 if (Opcode == AMDGPU::G_LOAD || Opcode == AMDGPU::G_ZEXTLOAD ||
11600 Opcode == AMDGPU::G_SEXTLOAD) {
11601 if (MI.memoperands_empty())
11602 return ValueUniformity::NeverUniform; // conservative assumption
11603
11604 if (llvm::any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *mmo) {
11605 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
11606 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS ||
11607 mmo->getAddrSpace() == AMDGPUAS::VGPR;
11608 })) {
11609 // At least one MMO in a non-global address space.
11610 return ValueUniformity::NeverUniform;
11611 }
11612 return ValueUniformity::Default;
11613 }
11614
11615 if (SIInstrInfo::isGenericAtomicRMWOpcode(Opc: Opcode) ||
11616 Opcode == AMDGPU::G_ATOMIC_CMPXCHG ||
11617 Opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS ||
11618 AMDGPU::isGenericAtomic(Opc: Opcode)) {
11619 return ValueUniformity::NeverUniform;
11620 }
11621
11622 // Result is computed from uniform SP and uniform wave-wide max size.
11623 if (Opcode == TargetOpcode::G_DYN_STACKALLOC)
11624 return ValueUniformity::AlwaysUniform;
11625
11626 if (Opcode == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
11627 return ValueUniformity::NeverUniform;
11628
11629 return ValueUniformity::Default;
11630}
11631
11632const MIRFormatter *SIInstrInfo::getMIRFormatter() const {
11633 if (!Formatter)
11634 Formatter = std::make_unique<AMDGPUMIRFormatter>(args: ST);
11635 return Formatter.get();
11636}
11637
11638ValueUniformity SIInstrInfo::getValueUniformity(const MachineInstr &MI) const {
11639
11640 if (isNeverUniform(MI))
11641 return ValueUniformity::NeverUniform;
11642
11643 unsigned opcode = MI.getOpcode();
11644 if (opcode == AMDGPU::V_READLANE_B32 ||
11645 opcode == AMDGPU::V_READFIRSTLANE_B32 ||
11646 opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR)
11647 return ValueUniformity::AlwaysUniform;
11648
11649 // If any of defs is divergent, report as NeverUniform. isUniformReg will
11650 // calculate in more detail for each def from its reg class, if available.
11651 if (MI.isInlineAsm()) {
11652 for (const MachineOperand &MO : MI.operands()) {
11653 if (!MO.isReg() || !MO.isDef())
11654 continue;
11655 const TargetRegisterClass *RC =
11656 MI.getRegClassConstraint(OpIdx: MO.getOperandNo(), TII: this, TRI: &RI);
11657 if (!RC || !RI.isSGPRClass(RC))
11658 return ValueUniformity::NeverUniform;
11659 }
11660 }
11661
11662 if (isCopyInstr(MI)) {
11663 const MachineOperand &srcOp = MI.getOperand(i: 1);
11664 if (srcOp.isReg() && srcOp.getReg().isPhysical()) {
11665 const TargetRegisterClass *regClass =
11666 RI.getPhysRegBaseClass(Reg: srcOp.getReg());
11667 return RI.isSGPRClass(RC: regClass) ? ValueUniformity::AlwaysUniform
11668 : ValueUniformity::NeverUniform;
11669 }
11670 return ValueUniformity::Default;
11671 }
11672
11673 // GMIR handling
11674 if (MI.isPreISelOpcode())
11675 return SIInstrInfo::getGenericValueUniformity(MI);
11676
11677 // Atomics are divergent because they are executed sequentially: when an
11678 // atomic operation refers to the same address in each thread, then each
11679 // thread after the first sees the value written by the previous thread as
11680 // original value.
11681
11682 if (isAtomic(MI))
11683 return ValueUniformity::NeverUniform;
11684
11685 // Loads from the private and flat address spaces are divergent, because
11686 // threads can execute the load instruction with the same inputs and get
11687 // different results.
11688 if (isFLAT(MI) && MI.mayLoad()) {
11689 if (MI.memoperands_empty())
11690 return ValueUniformity::NeverUniform; // conservative assumption
11691
11692 if (llvm::any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *mmo) {
11693 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
11694 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
11695 })) {
11696 // At least one MMO in a non-global address space.
11697 return ValueUniformity::NeverUniform;
11698 }
11699
11700 return ValueUniformity::Default;
11701 }
11702
11703 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
11704 const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo();
11705
11706 // FIXME: It's conceptually broken to report this for an instruction, and not
11707 // a specific def operand. For inline asm in particular, there could be mixed
11708 // uniform and divergent results.
11709 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
11710 const MachineOperand &SrcOp = MI.getOperand(i: I);
11711 if (!SrcOp.isReg())
11712 continue;
11713
11714 Register Reg = SrcOp.getReg();
11715 if (!Reg || !SrcOp.readsReg())
11716 continue;
11717
11718 // If RegBank is null, this is unassigned or an unallocatable special
11719 // register, which are all scalars.
11720 const RegisterBank *RegBank = RBI->getRegBank(Reg, MRI, TRI: RI);
11721 if (RegBank && RegBank->getID() != AMDGPU::SGPRRegBankID)
11722 return ValueUniformity::NeverUniform;
11723 }
11724
11725 // TODO: Uniformity check condtions above can be rearranged for more
11726 // redability
11727
11728 // TODO: amdgcn.{ballot, [if]cmp} should be AlwaysUniform, but they are
11729 // currently turned into no-op COPYs by SelectionDAG ISel and are
11730 // therefore no longer recognizable.
11731
11732 return ValueUniformity::Default;
11733}
11734
11735unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) {
11736 switch (MF.getFunction().getCallingConv()) {
11737 case CallingConv::AMDGPU_PS:
11738 return 1;
11739 case CallingConv::AMDGPU_VS:
11740 return 2;
11741 case CallingConv::AMDGPU_GS:
11742 return 3;
11743 case CallingConv::AMDGPU_HS:
11744 case CallingConv::AMDGPU_LS:
11745 case CallingConv::AMDGPU_ES: {
11746 const Function &F = MF.getFunction();
11747 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
11748 F, "ds_ordered_count unsupported for this calling conv"));
11749 [[fallthrough]];
11750 }
11751 case CallingConv::AMDGPU_CS:
11752 case CallingConv::AMDGPU_KERNEL:
11753 case CallingConv::C:
11754 case CallingConv::Fast:
11755 default:
11756 // Assume other calling conventions are various compute callable functions
11757 return 0;
11758 }
11759}
11760
11761bool SIInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
11762 Register &SrcReg2, int64_t &CmpMask,
11763 int64_t &CmpValue) const {
11764 if (!MI.getOperand(i: 0).isReg() || MI.getOperand(i: 0).getSubReg())
11765 return false;
11766
11767 switch (MI.getOpcode()) {
11768 default:
11769 break;
11770 case AMDGPU::S_CMP_EQ_U32:
11771 case AMDGPU::S_CMP_EQ_I32:
11772 case AMDGPU::S_CMP_LG_U32:
11773 case AMDGPU::S_CMP_LG_I32:
11774 case AMDGPU::S_CMP_LT_U32:
11775 case AMDGPU::S_CMP_LT_I32:
11776 case AMDGPU::S_CMP_GT_U32:
11777 case AMDGPU::S_CMP_GT_I32:
11778 case AMDGPU::S_CMP_LE_U32:
11779 case AMDGPU::S_CMP_LE_I32:
11780 case AMDGPU::S_CMP_GE_U32:
11781 case AMDGPU::S_CMP_GE_I32:
11782 case AMDGPU::S_CMP_EQ_U64:
11783 case AMDGPU::S_CMP_LG_U64:
11784 SrcReg = MI.getOperand(i: 0).getReg();
11785 if (MI.getOperand(i: 1).isReg()) {
11786 if (MI.getOperand(i: 1).getSubReg())
11787 return false;
11788 SrcReg2 = MI.getOperand(i: 1).getReg();
11789 CmpValue = 0;
11790 } else if (MI.getOperand(i: 1).isImm()) {
11791 SrcReg2 = Register();
11792 CmpValue = MI.getOperand(i: 1).getImm();
11793 } else {
11794 return false;
11795 }
11796 CmpMask = ~0;
11797 return true;
11798 case AMDGPU::S_CMPK_EQ_U32:
11799 case AMDGPU::S_CMPK_EQ_I32:
11800 case AMDGPU::S_CMPK_LG_U32:
11801 case AMDGPU::S_CMPK_LG_I32:
11802 case AMDGPU::S_CMPK_LT_U32:
11803 case AMDGPU::S_CMPK_LT_I32:
11804 case AMDGPU::S_CMPK_GT_U32:
11805 case AMDGPU::S_CMPK_GT_I32:
11806 case AMDGPU::S_CMPK_LE_U32:
11807 case AMDGPU::S_CMPK_LE_I32:
11808 case AMDGPU::S_CMPK_GE_U32:
11809 case AMDGPU::S_CMPK_GE_I32:
11810 SrcReg = MI.getOperand(i: 0).getReg();
11811 SrcReg2 = Register();
11812 CmpValue = MI.getOperand(i: 1).getImm();
11813 CmpMask = ~0;
11814 return true;
11815 }
11816
11817 return false;
11818}
11819
11820static bool isSCCDeadOnExit(MachineBasicBlock *MBB) {
11821 for (MachineBasicBlock *S : MBB->successors()) {
11822 if (S->isLiveIn(Reg: AMDGPU::SCC))
11823 return false;
11824 }
11825 return true;
11826}
11827
11828// Invert all uses of SCC following SCCDef because SCCDef may be deleted and
11829// (incoming SCC) = !(SCC defined by SCCDef).
11830// Return true if all uses can be re-written, false otherwise.
11831bool SIInstrInfo::invertSCCUse(MachineInstr *SCCDef) const {
11832 MachineBasicBlock *MBB = SCCDef->getParent();
11833 SmallVector<MachineInstr *> InvertInstr;
11834 bool SCCIsDead = false;
11835
11836 // Scan instructions for SCC uses that need to be inverted until SCC is dead.
11837 constexpr unsigned ScanLimit = 12;
11838 unsigned Count = 0;
11839 for (MachineInstr &MI : instructionsWithoutDebug(
11840 It: std::next(x: MachineBasicBlock::iterator(SCCDef)), End: MBB->end())) {
11841 if (++Count > ScanLimit)
11842 return false;
11843 if (MI.readsRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
11844 if (MI.getOpcode() == AMDGPU::S_CSELECT_B32 ||
11845 MI.getOpcode() == AMDGPU::S_CSELECT_B64 ||
11846 MI.getOpcode() == AMDGPU::S_CBRANCH_SCC0 ||
11847 MI.getOpcode() == AMDGPU::S_CBRANCH_SCC1)
11848 InvertInstr.push_back(Elt: &MI);
11849 else
11850 return false;
11851 }
11852 if (MI.definesRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
11853 SCCIsDead = true;
11854 break;
11855 }
11856 }
11857 if (!SCCIsDead && isSCCDeadOnExit(MBB))
11858 SCCIsDead = true;
11859
11860 // SCC may have more uses. Can't invert all of them.
11861 if (!SCCIsDead)
11862 return false;
11863
11864 // Invert uses
11865 for (MachineInstr *MI : InvertInstr) {
11866 if (MI->getOpcode() == AMDGPU::S_CSELECT_B32 ||
11867 MI->getOpcode() == AMDGPU::S_CSELECT_B64) {
11868 swapOperands(Inst&: *MI);
11869 } else if (MI->getOpcode() == AMDGPU::S_CBRANCH_SCC0 ||
11870 MI->getOpcode() == AMDGPU::S_CBRANCH_SCC1) {
11871 MI->setDesc(get(Opcode: MI->getOpcode() == AMDGPU::S_CBRANCH_SCC0
11872 ? AMDGPU::S_CBRANCH_SCC1
11873 : AMDGPU::S_CBRANCH_SCC0));
11874 } else {
11875 llvm_unreachable("SCC used but no inversion handling");
11876 }
11877 }
11878 return true;
11879}
11880
11881// SCC is already valid after SCCValid.
11882// SCCRedefine will redefine SCC to the same value already available after
11883// SCCValid. If there are no intervening SCC conflicts delete SCCRedefine and
11884// update kill/dead flags if necessary.
11885bool SIInstrInfo::optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,
11886 bool NeedInversion) const {
11887 MachineInstr *KillsSCC = nullptr;
11888 if (SCCValid->getParent() != SCCRedefine->getParent())
11889 return false;
11890 for (MachineInstr &MI : make_range(x: std::next(x: SCCValid->getIterator()),
11891 y: SCCRedefine->getIterator())) {
11892 if (MI.modifiesRegister(Reg: AMDGPU::SCC, TRI: &RI))
11893 return false;
11894 if (MI.killsRegister(Reg: AMDGPU::SCC, TRI: &RI))
11895 KillsSCC = &MI;
11896 }
11897 if (NeedInversion && !invertSCCUse(SCCDef: SCCRedefine))
11898 return false;
11899 if (MachineOperand *SccDef =
11900 SCCValid->findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr))
11901 SccDef->setIsDead(false);
11902 if (KillsSCC)
11903 KillsSCC->clearRegisterKills(Reg: AMDGPU::SCC, /*TRI=*/RegInfo: nullptr);
11904 SCCRedefine->eraseFromParent();
11905 return true;
11906}
11907
11908/// If \p Sel is an S_CSELECT* of two different constants A and B, return them,
11909/// truncated to the width of the select.
11910static std::optional<std::pair<int64_t, int64_t>>
11911getSelectConstants(const SIInstrInfo &TII, const MachineRegisterInfo &MRI,
11912 const MachineInstr &Sel) {
11913 unsigned Opc = Sel.getOpcode();
11914 if (Opc != AMDGPU::S_CSELECT_B32 && Opc != AMDGPU::S_CSELECT_B64)
11915 return {};
11916 std::optional<int64_t> A =
11917 TII.getImmOrMaterializedImm(MRI, Op: Sel.getOperand(i: 1));
11918 if (!A)
11919 return {};
11920 std::optional<int64_t> B =
11921 TII.getImmOrMaterializedImm(MRI, Op: Sel.getOperand(i: 2));
11922 if (!B)
11923 return {};
11924 if (Opc == AMDGPU::S_CSELECT_B32) {
11925 A = Lo_32(Value: *A);
11926 B = Lo_32(Value: *B);
11927 }
11928 if (*A == *B)
11929 return {};
11930 return std::pair(*A, *B);
11931}
11932
11933static bool setsSCCIfResultIsZero(const MachineInstr &Def, bool &NeedInversion,
11934 unsigned &NewDefOpc) {
11935 // S_ADD_U32 X, 1 sets SCC on carryout which can only happen if result==0.
11936 // S_ADD_I32 X, 1 can be converted to S_ADD_U32 X, 1 if SCC is dead.
11937 if (Def.getOpcode() != AMDGPU::S_ADD_I32 &&
11938 Def.getOpcode() != AMDGPU::S_ADD_U32)
11939 return false;
11940 const MachineOperand &AddSrc1 = Def.getOperand(i: 1);
11941 const MachineOperand &AddSrc2 = Def.getOperand(i: 2);
11942 const MachineRegisterInfo &MRI = Def.getMF()->getRegInfo();
11943 const SIInstrInfo *TII = static_cast<const SIInstrInfo *>(
11944 Def.getMF()->getSubtarget().getInstrInfo());
11945
11946 auto Imm1 = TII->getImmOrMaterializedImm(MRI, Op: AddSrc1);
11947 auto Imm2 = TII->getImmOrMaterializedImm(MRI, Op: AddSrc2);
11948 if ((!Imm1 || *Imm1 != 1) && (!Imm2 || *Imm2 != 1))
11949 return false;
11950
11951 if (Def.getOpcode() == AMDGPU::S_ADD_I32) {
11952 const MachineOperand *SccDef =
11953 Def.findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
11954 if (!SccDef->isDead())
11955 return false;
11956 NewDefOpc = AMDGPU::S_ADD_U32;
11957 }
11958 NeedInversion = !NeedInversion;
11959 return true;
11960}
11961
11962bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
11963 Register SrcReg2, int64_t CmpMask,
11964 int64_t CmpValue,
11965 const MachineRegisterInfo *MRI) const {
11966 if (!SrcReg || SrcReg.isPhysical())
11967 return false;
11968
11969 if (SrcReg2) {
11970 auto ImmOpt = getImmOrMaterializedImm(MRI: *MRI, Reg: SrcReg2);
11971 if (!ImmOpt)
11972 return false;
11973 CmpValue = *ImmOpt;
11974 }
11975
11976 const auto optimizeCmpSelect = [&CmpInstr, SrcReg, CmpValue, MRI,
11977 this](bool NeedInversion) -> bool {
11978 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
11979 if (!Def)
11980 return false;
11981
11982 unsigned NewDefOpc = Def->getOpcode();
11983 if (auto Consts = getSelectConstants(TII: *this, MRI: *MRI, Sel: *Def)) {
11984 // sX = S_CSELECT* A, B with A != B, so sX == A exactly when SCC was set.
11985 // Comparing sX with A or B recomputes SCC or its inverse:
11986 //
11987 // s_cmp_eq_* sX, A => SCC s_cmp_lg_* sX, A => !SCC
11988 // s_cmp_eq_* sX, B => !SCC s_cmp_lg_* sX, B => SCC
11989 auto [A, B] = *Consts;
11990 int64_t C = Def->getOpcode() == AMDGPU::S_CSELECT_B32 ? Lo_32(Value: CmpValue)
11991 : CmpValue;
11992 if (C == A)
11993 NeedInversion = !NeedInversion;
11994 else if (C != B)
11995 return false;
11996 } else {
11997 // For S_OP that set SCC = DST!=0, do the transformation
11998 //
11999 // s_cmp_[lg|eq]_* (S_OP ...), 0 => (S_OP ...)
12000 //
12001 // For (S_OP ...) that set SCC = DST==0, invert NeedInversion and
12002 // do the transformation:
12003 //
12004 // s_cmp_[lg|eq]_* (S_OP ...), 0 => (S_OP ...)
12005 if (CmpValue != 0 ||
12006 (!setsSCCIfResultIsNonZero(*Def) &&
12007 !setsSCCIfResultIsZero(Def: *Def, NeedInversion, NewDefOpc)))
12008 return false;
12009 }
12010
12011 if (!optimizeSCC(SCCValid: Def, SCCRedefine: &CmpInstr, NeedInversion))
12012 return false;
12013
12014 if (NewDefOpc != Def->getOpcode())
12015 Def->setDesc(get(Opcode: NewDefOpc));
12016
12017 // If s_or_b32 result, sY, is unused (i.e. it is effectively a 64-bit
12018 // s_cmp_lg of a register pair) and the inputs are the hi and lo-halves of a
12019 // 64-bit select then delete s_or_b32 in the sequence:
12020 // sX = s_cselect_b64 A, B (A != B, one of them 0)
12021 // sLo = copy sX.sub0
12022 // sHi = copy sX.sub1
12023 // sY = s_or_b32 sLo, sHi
12024 if (Def->getOpcode() == AMDGPU::S_OR_B32 &&
12025 MRI->use_nodbg_empty(RegNo: Def->getOperand(i: 0).getReg())) {
12026 const MachineOperand &OrOpnd1 = Def->getOperand(i: 1);
12027 const MachineOperand &OrOpnd2 = Def->getOperand(i: 2);
12028 if (OrOpnd1.isReg() && OrOpnd2.isReg()) {
12029 MachineInstr *Def1 = MRI->getVRegDef(Reg: OrOpnd1.getReg());
12030 MachineInstr *Def2 = MRI->getVRegDef(Reg: OrOpnd2.getReg());
12031 if (Def1 && Def1->getOpcode() == AMDGPU::COPY && Def2 &&
12032 Def2->getOpcode() == AMDGPU::COPY && Def1->getOperand(i: 1).isReg() &&
12033 Def2->getOperand(i: 1).isReg() &&
12034 Def1->getOperand(i: 1).getSubReg() == AMDGPU::sub0 &&
12035 Def2->getOperand(i: 1).getSubReg() == AMDGPU::sub1 &&
12036 Def1->getOperand(i: 1).getReg() == Def2->getOperand(i: 1).getReg()) {
12037 if (MachineInstr *Select =
12038 MRI->getVRegDef(Reg: Def1->getOperand(i: 1).getReg())) {
12039 if (auto Consts = getSelectConstants(TII: *this, MRI: *MRI, Sel: *Select)) {
12040 auto [A, B] = *Consts;
12041 if (A == 0 || B == 0)
12042 optimizeSCC(SCCValid: Select, SCCRedefine: Def, /*NeedInversion=*/A == 0);
12043 }
12044 }
12045 }
12046 }
12047 }
12048 return true;
12049 };
12050
12051 const auto optimizeCmpAnd = [&CmpInstr, SrcReg, CmpValue, MRI,
12052 this](int64_t ExpectedValue, unsigned SrcSize,
12053 bool IsReversible, bool IsSigned) -> bool {
12054 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
12055 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
12056 // s_cmp_ge_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
12057 // s_cmp_ge_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
12058 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 1 << n => s_and_b64 $src, 1 << n
12059 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
12060 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
12061 // s_cmp_gt_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
12062 // s_cmp_gt_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
12063 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 0 => s_and_b64 $src, 1 << n
12064 //
12065 // Signed ge/gt are not used for the sign bit.
12066 //
12067 // If result of the AND is unused except in the compare:
12068 // s_and_b(32|64) $src, 1 << n => s_bitcmp1_b(32|64) $src, n
12069 //
12070 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
12071 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
12072 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 0 => s_bitcmp0_b64 $src, n
12073 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
12074 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
12075 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 1 << n => s_bitcmp0_b64 $src, n
12076
12077 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
12078 if (!Def)
12079 return false;
12080
12081 if (Def->getOpcode() != AMDGPU::S_AND_B32 &&
12082 Def->getOpcode() != AMDGPU::S_AND_B64)
12083 return false;
12084
12085 int64_t Mask;
12086 const auto isMask = [&Mask, SrcSize, MRI,
12087 this](const MachineOperand *MO) -> bool {
12088 auto ImmOpt = this->getImmOrMaterializedImm(MRI: *MRI, Op: *MO);
12089 if (!ImmOpt)
12090 return false;
12091 Mask = *ImmOpt;
12092 Mask &= maxUIntN(N: SrcSize);
12093 return isPowerOf2_64(Value: Mask);
12094 };
12095
12096 MachineOperand *SrcOp = &Def->getOperand(i: 1);
12097 if (isMask(SrcOp))
12098 SrcOp = &Def->getOperand(i: 2);
12099 else if (isMask(&Def->getOperand(i: 2)))
12100 SrcOp = &Def->getOperand(i: 1);
12101 else
12102 return false;
12103
12104 // A valid Mask is required to have a single bit set, hence a non-zero and
12105 // power-of-two value. This verifies that we will not do 64-bit shift below.
12106 assert(llvm::has_single_bit<uint64_t>(Mask) && "Invalid mask.");
12107 unsigned BitNo = llvm::countr_zero(Val: (uint64_t)Mask);
12108 if (IsSigned && BitNo == SrcSize - 1)
12109 return false;
12110
12111 ExpectedValue <<= BitNo;
12112
12113 bool IsReversedCC = false;
12114 if (CmpValue != ExpectedValue) {
12115 if (!IsReversible)
12116 return false;
12117 IsReversedCC = CmpValue == (ExpectedValue ^ Mask);
12118 if (!IsReversedCC)
12119 return false;
12120 }
12121
12122 Register DefReg = Def->getOperand(i: 0).getReg();
12123 if (IsReversedCC && !MRI->hasOneNonDBGUse(RegNo: DefReg))
12124 return false;
12125
12126 if (!optimizeSCC(SCCValid: Def, SCCRedefine: &CmpInstr, /*NeedInversion=*/false))
12127 return false;
12128
12129 if (!MRI->use_nodbg_empty(RegNo: DefReg)) {
12130 assert(!IsReversedCC);
12131 return true;
12132 }
12133
12134 // Replace AND with unused result with a S_BITCMP.
12135 MachineBasicBlock *MBB = Def->getParent();
12136
12137 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32
12138 : AMDGPU::S_BITCMP1_B32
12139 : IsReversedCC ? AMDGPU::S_BITCMP0_B64
12140 : AMDGPU::S_BITCMP1_B64;
12141
12142 BuildMI(BB&: *MBB, I: Def, MIMD: Def->getDebugLoc(), MCID: get(Opcode: NewOpc))
12143 .add(MO: *SrcOp)
12144 .addImm(Val: BitNo);
12145 Def->eraseFromParent();
12146
12147 return true;
12148 };
12149
12150 switch (CmpInstr.getOpcode()) {
12151 default:
12152 break;
12153 case AMDGPU::S_CMP_EQ_U32:
12154 case AMDGPU::S_CMP_EQ_I32:
12155 case AMDGPU::S_CMPK_EQ_U32:
12156 case AMDGPU::S_CMPK_EQ_I32:
12157 return optimizeCmpAnd(1, 32, true, false) ||
12158 optimizeCmpSelect(/*NeedInversion=*/true);
12159 case AMDGPU::S_CMP_GE_U32:
12160 case AMDGPU::S_CMPK_GE_U32:
12161 return optimizeCmpAnd(1, 32, false, false);
12162 case AMDGPU::S_CMP_GE_I32:
12163 case AMDGPU::S_CMPK_GE_I32:
12164 return optimizeCmpAnd(1, 32, false, true);
12165 case AMDGPU::S_CMP_EQ_U64:
12166 return optimizeCmpAnd(1, 64, true, false) ||
12167 optimizeCmpSelect(/*NeedInversion=*/true);
12168 case AMDGPU::S_CMP_LG_U32:
12169 case AMDGPU::S_CMP_LG_I32:
12170 case AMDGPU::S_CMPK_LG_U32:
12171 case AMDGPU::S_CMPK_LG_I32:
12172 return optimizeCmpAnd(0, 32, true, false) ||
12173 optimizeCmpSelect(/*NeedInversion=*/false);
12174 case AMDGPU::S_CMP_GT_U32:
12175 case AMDGPU::S_CMPK_GT_U32:
12176 return optimizeCmpAnd(0, 32, false, false);
12177 case AMDGPU::S_CMP_GT_I32:
12178 case AMDGPU::S_CMPK_GT_I32:
12179 return optimizeCmpAnd(0, 32, false, true);
12180 case AMDGPU::S_CMP_LG_U64:
12181 return optimizeCmpAnd(0, 64, true, false) ||
12182 optimizeCmpSelect(/*NeedInversion=*/false);
12183 }
12184
12185 return false;
12186}
12187
12188void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI,
12189 AMDGPU::OpName OpName) const {
12190 if (!ST.needsAlignedVGPRs())
12191 return;
12192
12193 int OpNo = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpName);
12194 if (OpNo < 0)
12195 return;
12196 MachineOperand &Op = MI.getOperand(i: OpNo);
12197 if (getOpSize(MI, OpNo) > 4)
12198 return;
12199
12200 // Add implicit aligned super-reg to force alignment on the data operand.
12201 const DebugLoc &DL = MI.getDebugLoc();
12202 MachineBasicBlock *BB = MI.getParent();
12203 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
12204 Register DataReg = Op.getReg();
12205 bool IsAGPR = RI.isAGPR(MRI, Reg: DataReg);
12206 Register Undef = MRI.createVirtualRegister(
12207 RegClass: IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);
12208 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
12209 Register NewVR =
12210 MRI.createVirtualRegister(RegClass: IsAGPR ? &AMDGPU::AReg_64_Align2RegClass
12211 : &AMDGPU::VReg_64_Align2RegClass);
12212 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewVR)
12213 .addReg(RegNo: DataReg, Flags: {}, SubReg: Op.getSubReg())
12214 .addImm(Val: AMDGPU::sub0)
12215 .addReg(RegNo: Undef)
12216 .addImm(Val: AMDGPU::sub1);
12217 Op.setReg(NewVR);
12218 Op.setSubReg(AMDGPU::sub0);
12219 MI.addOperand(Op: MachineOperand::CreateReg(Reg: NewVR, isDef: false, isImp: true));
12220}
12221
12222unsigned SIInstrInfo::getRepeatRate(const MachineInstr &MI) const {
12223 if (!SchedModel.hasInstrSchedModel())
12224 return 0;
12225
12226 // The repeat rate is the throughput-limiting resource occupancy: the largest
12227 // number of cycles any written processor resource is held.
12228 const MCSchedClassDesc *SCDesc = SchedModel.resolveSchedClass(MI: &MI);
12229 unsigned RepeatRate = 0;
12230 for (TargetSchedModel::ProcResIter
12231 PI = SchedModel.getWriteProcResBegin(SC: SCDesc),
12232 PE = SchedModel.getWriteProcResEnd(SC: SCDesc);
12233 PI != PE; ++PI) {
12234 RepeatRate = std::max(a: RepeatRate, b: (unsigned)PI->ReleaseAtCycle);
12235 }
12236
12237 return RepeatRate;
12238}
12239
12240bool SIInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const {
12241 if (isIGLP(MI: *MI))
12242 return false;
12243
12244 return TargetInstrInfo::isGlobalMemoryObject(MI);
12245}
12246
12247bool SIInstrInfo::isXDLWMMA(const MachineInstr &MI) const {
12248 if (!isWMMA(MI) && !isSWMMAC(MI))
12249 return false;
12250
12251 if (ST.hasGFX1250Insts())
12252 return AMDGPU::getWMMAIsXDL(Opc: MI.getOpcode());
12253
12254 return true;
12255}
12256
12257bool SIInstrInfo::isXDL(const MachineInstr &MI) const {
12258 unsigned Opcode = MI.getOpcode();
12259
12260 if (AMDGPU::isGFX12Plus(STI: ST))
12261 return isDOT(MI) || isXDLWMMA(MI);
12262
12263 if (!isMAI(MI) || isDGEMM(Opcode) ||
12264 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
12265 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
12266 return false;
12267
12268 if (!ST.hasGFX940Insts())
12269 return true;
12270
12271 return AMDGPU::getMAIIsGFX940XDL(Opc: Opcode);
12272}
12273
12274bool SIInstrInfo::isVAVDSTOrderedXDL(const MachineInstr &MI) const {
12275 constexpr unsigned MinOrderedXDLVAVDSTVGPRs = 8;
12276 if (!AMDGPU::isGFX1250(STI: ST) || !isXDL(MI))
12277 return false;
12278
12279 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
12280 const TargetRegisterClass *DstRC =
12281 getRegClass(MCID: MI.getDesc(), OpNum: Dst->getOperandNo());
12282 return DstRC && RI.isVGPRClass(RC: DstRC) &&
12283 RI.getRegSizeInBits(RC: *DstRC) >= MinOrderedXDLVAVDSTVGPRs * 32;
12284}
12285