1//===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// SI Implementation of TargetInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIInstrInfo.h"
15#include "AMDGPU.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPULaneMaskUtils.h"
18#include "GCNHazardRecognizer.h"
19#include "GCNSubtarget.h"
20#include "SIMachineFunctionInfo.h"
21#include "Utils/AMDGPUBaseInfo.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
25#include "llvm/CodeGen/LiveIntervals.h"
26#include "llvm/CodeGen/LiveVariables.h"
27#include "llvm/CodeGen/MachineCycleAnalysis.h"
28#include "llvm/CodeGen/MachineDominators.h"
29#include "llvm/CodeGen/MachineFrameInfo.h"
30#include "llvm/CodeGen/MachineScheduler.h"
31#include "llvm/CodeGen/RegisterScavenging.h"
32#include "llvm/CodeGen/ScheduleDAG.h"
33#include "llvm/IR/DiagnosticInfo.h"
34#include "llvm/IR/IntrinsicsAMDGPU.h"
35#include "llvm/MC/MCContext.h"
36#include "llvm/Support/CommandLine.h"
37#include "llvm/Target/TargetMachine.h"
38#include <tuple>
39
40using namespace llvm;
41
42#define DEBUG_TYPE "si-instr-info"
43
44#define GET_INSTRINFO_CTOR_DTOR
45#include "AMDGPUGenInstrInfo.inc"
46
47namespace llvm::AMDGPU {
48#define GET_D16ImageDimIntrinsics_IMPL
49#define GET_ImageDimIntrinsicTable_IMPL
50#define GET_RsrcIntrinsics_IMPL
51#include "AMDGPUGenSearchableTables.inc"
52} // namespace llvm::AMDGPU
53
54// Must be at least 4 to be able to branch over minimum unconditional branch
55// code. This is only for making it possible to write reasonably small tests for
56// long branches.
57static cl::opt<unsigned>
58BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(Val: 16),
59 cl::desc("Restrict range of branch instructions (DEBUG)"));
60
61static cl::opt<bool> Fix16BitCopies(
62 "amdgpu-fix-16-bit-physreg-copies",
63 cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"),
64 cl::init(Val: true),
65 cl::ReallyHidden);
66
67SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
68 : AMDGPUGenInstrInfo(ST, RI, AMDGPU::ADJCALLSTACKUP,
69 AMDGPU::ADJCALLSTACKDOWN),
70 RI(ST), ST(ST) {
71 SchedModel.init(TSInfo: &ST);
72}
73
74//===----------------------------------------------------------------------===//
75// TargetInstrInfo callbacks
76//===----------------------------------------------------------------------===//
77
78static unsigned getNumOperandsNoGlue(SDNode *Node) {
79 unsigned N = Node->getNumOperands();
80 while (N && Node->getOperand(Num: N - 1).getValueType() == MVT::Glue)
81 --N;
82 return N;
83}
84
85/// Returns true if both nodes have the same value for the given
86/// operand \p Op, or if both nodes do not have this operand.
87static bool nodesHaveSameOperandValue(SDNode *N0, SDNode *N1,
88 AMDGPU::OpName OpName) {
89 unsigned Opc0 = N0->getMachineOpcode();
90 unsigned Opc1 = N1->getMachineOpcode();
91
92 int Op0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: OpName);
93 int Op1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: OpName);
94
95 if (Op0Idx == -1 && Op1Idx == -1)
96 return true;
97
98
99 if ((Op0Idx == -1 && Op1Idx != -1) ||
100 (Op1Idx == -1 && Op0Idx != -1))
101 return false;
102
103 // getNamedOperandIdx returns the index for the MachineInstr's operands,
104 // which includes the result as the first operand. We are indexing into the
105 // MachineSDNode's operands, so we need to skip the result operand to get
106 // the real index.
107 --Op0Idx;
108 --Op1Idx;
109
110 return N0->getOperand(Num: Op0Idx) == N1->getOperand(Num: Op1Idx);
111}
112
113static bool canRemat(const MachineInstr &MI) {
114
115 if (SIInstrInfo::isVOP1(MI) || SIInstrInfo::isVOP2(MI) ||
116 SIInstrInfo::isVOP3(MI) || SIInstrInfo::isSDWA(MI) ||
117 SIInstrInfo::isSALU(MI))
118 return true;
119
120 if (SIInstrInfo::isSMRD(MI)) {
121 return !MI.memoperands_empty() &&
122 llvm::all_of(Range: MI.memoperands(), P: [](const MachineMemOperand *MMO) {
123 return MMO->isLoad() && MMO->isInvariant();
124 });
125 }
126
127 return false;
128}
129
130// Split relocation flags for 64-bit global-address materialization into a
131// common base and the hi/lo relocation variants.
132static std::tuple<unsigned, unsigned, unsigned>
133splitGlobalAddressRelocFlags(const GCNSubtarget &ST,
134 const MachineOperand &SrcOp) {
135 unsigned SrcFlags = SrcOp.getTargetFlags();
136
137 // Infer the relocation type from the existing flags on the global operand.
138 // The relocation type should have been determined earlier in the pipeline.
139 unsigned LoReloc = SIInstrInfo::MO_ABS32_LO;
140 unsigned HiReloc = SIInstrInfo::MO_ABS32_HI;
141
142 if (SrcFlags & SIInstrInfo::MO_REL32) {
143 LoReloc = SIInstrInfo::MO_REL32_LO;
144 HiReloc = SIInstrInfo::MO_REL32_HI;
145 } else if (SrcFlags & SIInstrInfo::MO_GOTPCREL32_LO) {
146 LoReloc = SIInstrInfo::MO_GOTPCREL32_LO;
147 HiReloc = SIInstrInfo::MO_GOTPCREL32_HI;
148 } else if (SrcFlags & SIInstrInfo::MO_GOTPCREL64) {
149 // For 64-bit GOT-relative, use the 64-bit relocation.
150 LoReloc = SIInstrInfo::MO_GOTPCREL64;
151 HiReloc = SIInstrInfo::MO_GOTPCREL64;
152 }
153
154 unsigned BaseFlags =
155 SrcFlags & ~(SIInstrInfo::MO_ABS32_LO | SIInstrInfo::MO_ABS32_HI |
156 SIInstrInfo::MO_REL32_LO | SIInstrInfo::MO_REL32_HI |
157 SIInstrInfo::MO_GOTPCREL32_LO |
158 SIInstrInfo::MO_GOTPCREL32_HI | SIInstrInfo::MO_GOTPCREL64);
159
160 return std::make_tuple(args&: BaseFlags, args&: LoReloc, args&: HiReloc);
161}
162
163bool SIInstrInfo::isReMaterializableImpl(
164 const MachineInstr &MI) const {
165
166 if (canRemat(MI)) {
167 // Normally VALU use of exec would block the rematerialization, but that
168 // is OK in this case to have an implicit exec read as all VALU do.
169 // We really want all of the generic logic for this except for this.
170
171 // Another potential implicit use is mode register. The core logic of
172 // the RA will not attempt rematerialization if mode is set anywhere
173 // in the function, otherwise it is safe since mode is not changed.
174
175 // There is difference to generic method which does not allow
176 // rematerialization if there are virtual register uses. We allow this,
177 // therefore this method includes SOP instructions as well.
178 if (!MI.hasImplicitDef() &&
179 MI.getNumImplicitOperands() == MI.getDesc().implicit_uses().size() &&
180 !MI.mayRaiseFPException())
181 return true;
182 }
183
184 return TargetInstrInfo::isReMaterializableImpl(MI);
185}
186
187// Returns true if the result of a VALU instruction depends on exec.
188bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {
189 assert(isVALU(MI, /*AllowLDSDMA=*/true));
190
191 // If it is convergent it depends on EXEC.
192 if (MI.isConvergent())
193 return true;
194
195 // If it defines SGPR it depends on EXEC
196 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
197 for (const MachineOperand &Def : MI.defs()) {
198 if (!Def.isReg())
199 continue;
200
201 Register Reg = Def.getReg();
202 if (Reg && RI.isSGPRReg(MRI, Reg))
203 return true;
204 }
205
206 return false;
207}
208
209bool SIInstrInfo::isIgnorableUse(const MachineOperand &MO) const {
210 // Any implicit use of exec by VALU is not a real register read.
211 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&
212 isVALU(MI: *MO.getParent(), /*AllowLDSDMA=*/true) &&
213 !resultDependsOnExec(MI: *MO.getParent());
214}
215
216bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
217 MachineBasicBlock *SuccToSinkTo,
218 MachineCycleInfo *CI) const {
219 // Allow sinking if MI edits lane mask (divergent i1 in sgpr).
220 if (MI.getOpcode() == AMDGPU::SI_IF_BREAK)
221 return true;
222
223 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
224 // Check if sinking of MI would create temporal divergent use.
225 for (auto Op : MI.uses()) {
226 if (Op.isReg() && Op.getReg().isVirtual() &&
227 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Op.getReg()))) {
228 MachineInstr *SgprDef = MRI.getVRegDef(Reg: Op.getReg());
229
230 // SgprDef defined inside cycle
231 CycleRef FromCycle = CI->getCycle(Block: SgprDef->getParent());
232 if (!FromCycle)
233 continue;
234
235 CycleRef ToCycle = CI->getCycle(Block: SuccToSinkTo);
236 // Check if there is a FromCycle that contains SgprDef's basic block but
237 // does not contain SuccToSinkTo and also has divergent exit condition.
238 while (FromCycle && !(ToCycle && CI->contains(Outer: FromCycle, Inner: ToCycle))) {
239 SmallVector<MachineBasicBlock *, 1> ExitingBlocks;
240 CI->getExitingBlocks(C: FromCycle, TmpStorage&: ExitingBlocks);
241
242 // FromCycle has divergent exit condition.
243 for (MachineBasicBlock *ExitingBlock : ExitingBlocks) {
244 if (hasDivergentBranch(MBB: ExitingBlock))
245 return false;
246 }
247
248 FromCycle = CI->getParentCycle(C: FromCycle);
249 }
250 }
251 }
252
253 return true;
254}
255
256bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,
257 int64_t &Offset0,
258 int64_t &Offset1) const {
259 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
260 return false;
261
262 unsigned Opc0 = Load0->getMachineOpcode();
263 unsigned Opc1 = Load1->getMachineOpcode();
264
265 // Make sure both are actually loads.
266 if (!get(Opcode: Opc0).mayLoad() || !get(Opcode: Opc1).mayLoad())
267 return false;
268
269 // A mayLoad instruction without a def is not a load. Likely a prefetch.
270 if (!get(Opcode: Opc0).getNumDefs() || !get(Opcode: Opc1).getNumDefs())
271 return false;
272
273 if (isDS(Opcode: Opc0) && isDS(Opcode: Opc1)) {
274
275 // FIXME: Handle this case:
276 if (getNumOperandsNoGlue(Node: Load0) != getNumOperandsNoGlue(Node: Load1))
277 return false;
278
279 // Check base reg.
280 if (Load0->getOperand(Num: 0) != Load1->getOperand(Num: 0))
281 return false;
282
283 // Skip read2 / write2 variants for simplicity.
284 // TODO: We should report true if the used offsets are adjacent (excluded
285 // st64 versions).
286 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: AMDGPU::OpName::offset);
287 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: AMDGPU::OpName::offset);
288 if (Offset0Idx == -1 || Offset1Idx == -1)
289 return false;
290
291 // XXX - be careful of dataless loads
292 // getNamedOperandIdx returns the index for MachineInstrs. Since they
293 // include the output in the operand list, but SDNodes don't, we need to
294 // subtract the index by one.
295 Offset0Idx -= get(Opcode: Opc0).NumDefs;
296 Offset1Idx -= get(Opcode: Opc1).NumDefs;
297 Offset0 = Load0->getConstantOperandVal(Num: Offset0Idx);
298 Offset1 = Load1->getConstantOperandVal(Num: Offset1Idx);
299 return true;
300 }
301
302 if (isSMRD(Opcode: Opc0) && isSMRD(Opcode: Opc1)) {
303 // Skip time and cache invalidation instructions.
304 if (!AMDGPU::hasNamedOperand(Opcode: Opc0, NamedIdx: AMDGPU::OpName::sbase) ||
305 !AMDGPU::hasNamedOperand(Opcode: Opc1, NamedIdx: AMDGPU::OpName::sbase))
306 return false;
307
308 unsigned NumOps = getNumOperandsNoGlue(Node: Load0);
309 if (NumOps != getNumOperandsNoGlue(Node: Load1))
310 return false;
311
312 // Check base reg.
313 if (Load0->getOperand(Num: 0) != Load1->getOperand(Num: 0))
314 return false;
315
316 // Match register offsets, if both register and immediate offsets present.
317 assert(NumOps == 4 || NumOps == 5);
318 if (NumOps == 5 && Load0->getOperand(Num: 1) != Load1->getOperand(Num: 1))
319 return false;
320
321 const ConstantSDNode *Load0Offset =
322 dyn_cast<ConstantSDNode>(Val: Load0->getOperand(Num: NumOps - 3));
323 const ConstantSDNode *Load1Offset =
324 dyn_cast<ConstantSDNode>(Val: Load1->getOperand(Num: NumOps - 3));
325
326 if (!Load0Offset || !Load1Offset)
327 return false;
328
329 Offset0 = Load0Offset->getZExtValue();
330 Offset1 = Load1Offset->getZExtValue();
331 return true;
332 }
333
334 // MUBUF and MTBUF can access the same addresses.
335 if ((isMUBUF(Opcode: Opc0) || isMTBUF(Opcode: Opc0)) && (isMUBUF(Opcode: Opc1) || isMTBUF(Opcode: Opc1))) {
336
337 // MUBUF and MTBUF have vaddr at different indices.
338 if (!nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::soffset) ||
339 !nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::vaddr) ||
340 !nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::srsrc))
341 return false;
342
343 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: AMDGPU::OpName::offset);
344 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: AMDGPU::OpName::offset);
345
346 if (OffIdx0 == -1 || OffIdx1 == -1)
347 return false;
348
349 // getNamedOperandIdx returns the index for MachineInstrs. Since they
350 // include the output in the operand list, but SDNodes don't, we need to
351 // subtract the index by one.
352 OffIdx0 -= get(Opcode: Opc0).NumDefs;
353 OffIdx1 -= get(Opcode: Opc1).NumDefs;
354
355 SDValue Off0 = Load0->getOperand(Num: OffIdx0);
356 SDValue Off1 = Load1->getOperand(Num: OffIdx1);
357
358 // The offset might be a FrameIndexSDNode.
359 if (!isa<ConstantSDNode>(Val: Off0) || !isa<ConstantSDNode>(Val: Off1))
360 return false;
361
362 Offset0 = Off0->getAsZExtVal();
363 Offset1 = Off1->getAsZExtVal();
364 return true;
365 }
366
367 return false;
368}
369
370static bool isStride64(unsigned Opc) {
371 switch (Opc) {
372 case AMDGPU::DS_READ2ST64_B32:
373 case AMDGPU::DS_READ2ST64_B64:
374 case AMDGPU::DS_WRITE2ST64_B32:
375 case AMDGPU::DS_WRITE2ST64_B64:
376 return true;
377 default:
378 return false;
379 }
380}
381
382bool SIInstrInfo::getMemOperandsWithOffsetWidth(
383 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
384 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
385 const TargetRegisterInfo *TRI) const {
386 if (!LdSt.mayLoadOrStore())
387 return false;
388
389 unsigned Opc = LdSt.getOpcode();
390 OffsetIsScalable = false;
391 const MachineOperand *BaseOp, *OffsetOp;
392 int DataOpIdx;
393
394 if (isDS(MI: LdSt)) {
395 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::addr);
396 OffsetOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
397 if (OffsetOp) {
398 // Normal, single offset LDS instruction.
399 if (!BaseOp) {
400 // DS_CONSUME/DS_APPEND use M0 for the base address.
401 // TODO: find the implicit use operand for M0 and use that as BaseOp?
402 return false;
403 }
404 BaseOps.push_back(Elt: BaseOp);
405 Offset = OffsetOp->getImm();
406 // Get appropriate operand, and compute width accordingly.
407 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
408 if (DataOpIdx == -1)
409 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
410 if (Opc == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
411 Width = LocationSize::precise(Value: 64);
412 else
413 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
414 } else {
415 // The 2 offset instructions use offset0 and offset1 instead. We can treat
416 // these as a load with a single offset if the 2 offsets are consecutive.
417 // We will use this for some partially aligned loads.
418 const MachineOperand *Offset0Op =
419 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset0);
420 const MachineOperand *Offset1Op =
421 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset1);
422
423 unsigned Offset0 = Offset0Op->getImm() & 0xff;
424 unsigned Offset1 = Offset1Op->getImm() & 0xff;
425 if (Offset0 + 1 != Offset1)
426 return false;
427
428 // Each of these offsets is in element sized units, so we need to convert
429 // to bytes of the individual reads.
430
431 unsigned EltSize;
432 if (LdSt.mayLoad())
433 EltSize = TRI->getRegSizeInBits(RC: *getOpRegClass(MI: LdSt, OpNo: 0)) / 16;
434 else {
435 assert(LdSt.mayStore());
436 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
437 EltSize = TRI->getRegSizeInBits(RC: *getOpRegClass(MI: LdSt, OpNo: Data0Idx)) / 8;
438 }
439
440 if (isStride64(Opc))
441 EltSize *= 64;
442
443 BaseOps.push_back(Elt: BaseOp);
444 Offset = EltSize * Offset0;
445 // Get appropriate operand(s), and compute width accordingly.
446 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
447 if (DataOpIdx == -1) {
448 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
449 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
450 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data1);
451 Width = LocationSize::precise(
452 Value: Width.getValue() + TypeSize::getFixed(ExactSize: getOpSize(MI: LdSt, OpNo: DataOpIdx)));
453 } else {
454 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
455 }
456 }
457 return true;
458 }
459
460 if (isMUBUF(MI: LdSt) || isMTBUF(MI: LdSt)) {
461 const MachineOperand *RSrc = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::srsrc);
462 if (!RSrc) // e.g. BUFFER_WBINVL1_VOL
463 return false;
464 BaseOps.push_back(Elt: RSrc);
465 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr);
466 if (BaseOp && !BaseOp->isFI())
467 BaseOps.push_back(Elt: BaseOp);
468 const MachineOperand *OffsetImm =
469 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
470 Offset = OffsetImm->getImm();
471 const MachineOperand *SOffset =
472 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::soffset);
473 if (SOffset) {
474 if (SOffset->isReg())
475 BaseOps.push_back(Elt: SOffset);
476 else
477 Offset += SOffset->getImm();
478 }
479 // Get appropriate operand, and compute width accordingly.
480 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
481 if (DataOpIdx == -1)
482 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
483 if (DataOpIdx == -1) // LDS DMA
484 return false;
485 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
486 return true;
487 }
488
489 if (isImage(MI: LdSt)) {
490 auto RsrcOpName =
491 isMIMG(MI: LdSt) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
492 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: RsrcOpName);
493 BaseOps.push_back(Elt: &LdSt.getOperand(i: SRsrcIdx));
494 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0);
495 if (VAddr0Idx >= 0) {
496 // GFX10 possible NSA encoding.
497 for (int I = VAddr0Idx; I < SRsrcIdx; ++I)
498 BaseOps.push_back(Elt: &LdSt.getOperand(i: I));
499 } else {
500 BaseOps.push_back(Elt: getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr));
501 }
502 Offset = 0;
503 // Get appropriate operand, and compute width accordingly.
504 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
505 if (DataOpIdx == -1)
506 return false; // no return sampler
507 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
508 return true;
509 }
510
511 if (isSMRD(MI: LdSt)) {
512 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::sbase);
513 if (!BaseOp) // e.g. S_MEMTIME
514 return false;
515 BaseOps.push_back(Elt: BaseOp);
516 OffsetOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
517 Offset = OffsetOp ? OffsetOp->getImm() : 0;
518 // Get appropriate operand, and compute width accordingly.
519 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::sdst);
520 if (DataOpIdx == -1)
521 return false;
522 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
523 return true;
524 }
525
526 if (isFLAT(MI: LdSt)) {
527 // Instructions have either vaddr or saddr or both or none.
528 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr);
529 if (BaseOp)
530 BaseOps.push_back(Elt: BaseOp);
531 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::saddr);
532 if (BaseOp)
533 BaseOps.push_back(Elt: BaseOp);
534 Offset = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset)->getImm();
535 // Get appropriate operand, and compute width accordingly.
536 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
537 if (DataOpIdx == -1)
538 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
539 if (DataOpIdx == -1) // LDS DMA
540 return false;
541 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
542 return true;
543 }
544
545 return false;
546}
547
548static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
549 ArrayRef<const MachineOperand *> BaseOps1,
550 const MachineInstr &MI2,
551 ArrayRef<const MachineOperand *> BaseOps2) {
552 // Only examine the first "base" operand of each instruction, on the
553 // assumption that it represents the real base address of the memory access.
554 // Other operands are typically offsets or indices from this base address.
555 if (BaseOps1.front()->isIdenticalTo(Other: *BaseOps2.front()))
556 return true;
557
558 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
559 return false;
560
561 auto *MO1 = *MI1.memoperands_begin();
562 auto *MO2 = *MI2.memoperands_begin();
563 if (MO1->getAddrSpace() != MO2->getAddrSpace())
564 return false;
565
566 const auto *Base1 = MO1->getValue();
567 const auto *Base2 = MO2->getValue();
568 if (!Base1 || !Base2)
569 return false;
570 Base1 = getUnderlyingObject(V: Base1);
571 Base2 = getUnderlyingObject(V: Base2);
572
573 if (isa<UndefValue>(Val: Base1) || isa<UndefValue>(Val: Base2))
574 return false;
575
576 return Base1 == Base2;
577}
578
579bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
580 int64_t Offset1, bool OffsetIsScalable1,
581 ArrayRef<const MachineOperand *> BaseOps2,
582 int64_t Offset2, bool OffsetIsScalable2,
583 unsigned ClusterSize,
584 unsigned NumBytes) const {
585 // If the mem ops (to be clustered) do not have the same base ptr, then they
586 // should not be clustered
587 unsigned MaxMemoryClusterDWords = DefaultMemoryClusterDWordsLimit;
588 if (!BaseOps1.empty() && !BaseOps2.empty()) {
589 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
590 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
591 if (!memOpsHaveSameBasePtr(MI1: FirstLdSt, BaseOps1, MI2: SecondLdSt, BaseOps2))
592 return false;
593
594 const SIMachineFunctionInfo *MFI =
595 FirstLdSt.getMF()->getInfo<SIMachineFunctionInfo>();
596 MaxMemoryClusterDWords = MFI->getMaxMemoryClusterDWords();
597 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
598 // If only one base op is empty, they do not have the same base ptr
599 return false;
600 }
601
602 // In order to avoid register pressure, on an average, the number of DWORDS
603 // loaded together by all clustered mem ops should not exceed
604 // MaxMemoryClusterDWords. This is an empirical value based on certain
605 // observations and performance related experiments.
606 // The good thing about this heuristic is - it avoids clustering of too many
607 // sub-word loads, and also avoids clustering of wide loads. Below is the
608 // brief summary of how the heuristic behaves for various `LoadSize` when
609 // MaxMemoryClusterDWords is 8.
610 //
611 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops
612 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops
613 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops
614 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops
615 // (5) LoadSize >= 17: do not cluster
616 const unsigned LoadSize = NumBytes / ClusterSize;
617 const unsigned NumDWords = ((LoadSize + 3) / 4) * ClusterSize;
618 return NumDWords <= MaxMemoryClusterDWords;
619}
620
621// FIXME: This behaves strangely. If, for example, you have 32 load + stores,
622// the first 16 loads will be interleaved with the stores, and the next 16 will
623// be clustered as expected. It should really split into 2 16 store batches.
624//
625// Loads are clustered until this returns false, rather than trying to schedule
626// groups of stores. This also means we have to deal with saying different
627// address space loads should be clustered, and ones which might cause bank
628// conflicts.
629//
630// This might be deprecated so it might not be worth that much effort to fix.
631bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1,
632 int64_t Offset0, int64_t Offset1,
633 unsigned NumLoads) const {
634 assert(Offset1 > Offset0 &&
635 "Second offset should be larger than first offset!");
636 // If we have less than 16 loads in a row, and the offsets are within 64
637 // bytes, then schedule together.
638
639 // A cacheline is 64 bytes (for global memory).
640 return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
641}
642
643static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB,
644 MachineBasicBlock::iterator MI,
645 const DebugLoc &DL, MCRegister DestReg,
646 MCRegister SrcReg, bool KillSrc,
647 const char *Msg = "illegal VGPR to SGPR copy") {
648 MachineFunction *MF = MBB.getParent();
649
650 LLVMContext &C = MF->getFunction().getContext();
651 C.diagnose(DI: DiagnosticInfoUnsupported(MF->getFunction(), Msg, DL, DS_Error));
652
653 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_ILLEGAL_COPY), DestReg)
654 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
655}
656
657/// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not
658/// possible to have a direct copy in these cases on GFX908, so an intermediate
659/// VGPR copy is required.
660static void indirectCopyToAGPR(const SIInstrInfo &TII, MachineBasicBlock &MBB,
661 MachineBasicBlock::iterator MI,
662 const DebugLoc &DL, MCRegister DestReg,
663 MCRegister SrcReg, bool KillSrc,
664 RegScavenger &RS, bool RegsOverlap,
665 Register ImpUseSuperReg = Register()) {
666 assert((TII.getSubtarget().hasMAIInsts() &&
667 !TII.getSubtarget().hasGFX90AInsts()) &&
668 "Expected GFX908 subtarget.");
669
670 assert((AMDGPU::SReg_32RegClass.contains(SrcReg) ||
671 AMDGPU::AGPR_32RegClass.contains(SrcReg)) &&
672 "Source register of the copy should be either an SGPR or an AGPR.");
673
674 assert(AMDGPU::AGPR_32RegClass.contains(DestReg) &&
675 "Destination register of the copy should be an AGPR.");
676
677 const SIRegisterInfo &RI = TII.getRegisterInfo();
678
679 // First try to find defining accvgpr_write to avoid temporary registers.
680 // In the case of copies of overlapping AGPRs, we conservatively do not
681 // reuse previous accvgpr_writes. Otherwise, we may incorrectly pick up
682 // an accvgpr_write used for this same copy due to implicit-defs
683 if (!RegsOverlap) {
684 for (auto Def = MI, E = MBB.begin(); Def != E; ) {
685 --Def;
686
687 if (!Def->modifiesRegister(Reg: SrcReg, TRI: &RI))
688 continue;
689
690 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
691 Def->getOperand(i: 0).getReg() != SrcReg)
692 break;
693
694 MachineOperand &DefOp = Def->getOperand(i: 1);
695 assert(DefOp.isReg() || DefOp.isImm());
696
697 if (DefOp.isReg()) {
698 bool SafeToPropagate = true;
699 // Check that register source operand is not clobbered before MI.
700 // Immediate operands are always safe to propagate.
701 for (auto I = Def; I != MI && SafeToPropagate; ++I)
702 if (I->modifiesRegister(Reg: DefOp.getReg(), TRI: &RI))
703 SafeToPropagate = false;
704
705 if (!SafeToPropagate)
706 break;
707
708 for (auto I = Def; I != MI; ++I)
709 I->clearRegisterKills(Reg: DefOp.getReg(), RegInfo: &RI);
710 }
711
712 MachineInstrBuilder Builder =
713 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64),
714 DestReg)
715 .add(MO: DefOp);
716
717 if (ImpUseSuperReg) {
718 Builder.addReg(RegNo: ImpUseSuperReg,
719 Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
720 }
721
722 return;
723 }
724 }
725
726 RS.enterBasicBlockEnd(MBB);
727 RS.backward(I: std::next(x: MI));
728
729 // Ideally we want to have three registers for a long reg_sequence copy
730 // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
731 unsigned MaxVGPRs = RI.getRegPressureLimit(RC: &AMDGPU::VGPR_32RegClass,
732 MF&: *MBB.getParent());
733
734 // Registers in the sequence are allocated contiguously so we can just
735 // use register number to pick one of three round-robin temps.
736 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3;
737 Register Tmp =
738 MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getVGPRForAGPRCopy();
739 assert(MBB.getParent()->getRegInfo().isReserved(Tmp) &&
740 "VGPR used for an intermediate copy should have been reserved.");
741
742 // Only loop through if there are any free registers left. We don't want to
743 // spill.
744 while (RegNo--) {
745 Register Tmp2 = RS.scavengeRegisterBackwards(RC: AMDGPU::VGPR_32RegClass, To: MI,
746 /* RestoreAfter */ false, SPAdj: 0,
747 /* AllowSpill */ false);
748 if (!Tmp2 || RI.getHWRegIndex(Reg: Tmp2) >= MaxVGPRs)
749 break;
750 Tmp = Tmp2;
751 RS.setRegUsed(Reg: Tmp);
752 }
753
754 // Insert copy to temporary VGPR.
755 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;
756 if (AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg)) {
757 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64;
758 } else {
759 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
760 }
761
762 MachineInstrBuilder UseBuilder = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: TmpCopyOp), DestReg: Tmp)
763 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
764 if (ImpUseSuperReg) {
765 UseBuilder.addReg(RegNo: ImpUseSuperReg,
766 Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
767 }
768
769 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
770 .addReg(RegNo: Tmp, Flags: RegState::Kill);
771}
772
773static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB,
774 MachineBasicBlock::iterator MI, const DebugLoc &DL,
775 MCRegister DestReg, MCRegister SrcReg, bool KillSrc,
776 const TargetRegisterClass *RC, bool Forward) {
777 const SIRegisterInfo &RI = TII.getRegisterInfo();
778 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, EltSize: 4);
779 MachineBasicBlock::iterator I = MI;
780 MachineInstr *FirstMI = nullptr, *LastMI = nullptr;
781
782 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) {
783 int16_t SubIdx = BaseIndices[Idx];
784 Register DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
785 Register SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
786 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
787 unsigned Opcode = AMDGPU::S_MOV_B32;
788
789 // Is SGPR aligned? If so try to combine with next.
790 bool AlignedDest = ((DestSubReg - AMDGPU::SGPR0) % 2) == 0;
791 bool AlignedSrc = ((SrcSubReg - AMDGPU::SGPR0) % 2) == 0;
792 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) {
793 // Can use SGPR64 copy
794 unsigned Channel = RI.getChannelFromSubReg(SubReg: SubIdx);
795 SubIdx = RI.getSubRegFromChannel(Channel, NumRegs: 2);
796 DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
797 SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
798 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
799 Opcode = AMDGPU::S_MOV_B64;
800 Idx++;
801 }
802
803 LastMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode), DestReg: DestSubReg)
804 .addReg(RegNo: SrcSubReg)
805 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
806
807 if (!FirstMI)
808 FirstMI = LastMI;
809
810 if (!Forward)
811 I--;
812 }
813
814 assert(FirstMI && LastMI);
815 if (!Forward)
816 std::swap(a&: FirstMI, b&: LastMI);
817
818 if (KillSrc)
819 LastMI->addRegisterKilled(IncomingReg: SrcReg, RegInfo: &RI);
820}
821
822void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
823 MachineBasicBlock::iterator MI,
824 const DebugLoc &DL, Register DestReg,
825 Register SrcReg, bool KillSrc, bool RenamableDest,
826 bool RenamableSrc) const {
827 const TargetRegisterClass *RC = RI.getPhysRegBaseClass(Reg: DestReg);
828 unsigned Size = RI.getRegSizeInBits(RC: *RC);
829 const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(Reg: SrcReg);
830 unsigned SrcSize = RI.getRegSizeInBits(RC: *SrcRC);
831
832 // The rest of copyPhysReg assumes Src and Dst size are the same size.
833 // TODO-GFX11_16BIT If all true 16 bit instruction patterns are completed can
834 // we remove Fix16BitCopies and this code block?
835 if (Fix16BitCopies) {
836 if (((Size == 16) != (SrcSize == 16))) {
837 // Non-VGPR Src and Dst will later be expanded back to 32 bits.
838 assert(ST.useRealTrue16Insts());
839 Register &RegToFix = (Size == 32) ? DestReg : SrcReg;
840 MCRegister SubReg = RI.getSubReg(Reg: RegToFix, Idx: AMDGPU::lo16);
841 RegToFix = SubReg;
842
843 if (DestReg == SrcReg) {
844 // Identity copy. Insert empty bundle since ExpandPostRA expects an
845 // instruction here.
846 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::BUNDLE));
847 return;
848 }
849 RC = RI.getPhysRegBaseClass(Reg: DestReg);
850 Size = RI.getRegSizeInBits(RC: *RC);
851 SrcRC = RI.getPhysRegBaseClass(Reg: SrcReg);
852 SrcSize = RI.getRegSizeInBits(RC: *SrcRC);
853 }
854 }
855
856 if (RC == &AMDGPU::VGPR_32RegClass) {
857 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
858 AMDGPU::SReg_32RegClass.contains(SrcReg) ||
859 AMDGPU::AGPR_32RegClass.contains(SrcReg));
860 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg) ?
861 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;
862 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
863 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
864 return;
865 }
866
867 if (RC == &AMDGPU::SReg_32_XM0RegClass ||
868 RC == &AMDGPU::SReg_32RegClass) {
869 if (SrcReg == AMDGPU::SCC) {
870 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B32), DestReg)
871 .addImm(Val: 1)
872 .addImm(Val: 0);
873 return;
874 }
875
876 if (!AMDGPU::SReg_32RegClass.contains(Reg: SrcReg)) {
877 if (DestReg == AMDGPU::VCC_LO) {
878 // FIXME: Hack until VReg_1 removed.
879 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
880 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CMP_NE_U32_e32))
881 .addImm(Val: 0)
882 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
883 return;
884 }
885
886 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
887 return;
888 }
889
890 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg)
891 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
892 return;
893 }
894
895 if (RC == &AMDGPU::SReg_64RegClass) {
896 if (SrcReg == AMDGPU::SCC) {
897 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B64), DestReg)
898 .addImm(Val: 1)
899 .addImm(Val: 0);
900 return;
901 }
902
903 if (!AMDGPU::SReg_64_EncodableRegClass.contains(Reg: SrcReg)) {
904 if (DestReg == AMDGPU::VCC) {
905 // FIXME: Hack until VReg_1 removed.
906 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
907 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CMP_NE_U32_e32))
908 .addImm(Val: 0)
909 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
910 return;
911 }
912
913 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
914 return;
915 }
916
917 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B64), DestReg)
918 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
919 return;
920 }
921
922 if (DestReg == AMDGPU::SCC) {
923 // Copying 64-bit or 32-bit sources to SCC barely makes sense,
924 // but SelectionDAG emits such copies for i1 sources.
925 if (AMDGPU::SReg_64RegClass.contains(Reg: SrcReg)) {
926 // This copy can only be produced by patterns
927 // with explicit SCC, which are known to be enabled
928 // only for subtargets with S_CMP_LG_U64 present.
929 assert(ST.hasScalarCompareEq64());
930 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U64))
931 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
932 .addImm(Val: 0);
933 } else {
934 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
935 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U32))
936 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
937 .addImm(Val: 0);
938 }
939
940 return;
941 }
942
943 if (RC == &AMDGPU::AGPR_32RegClass) {
944 if (AMDGPU::VGPR_32RegClass.contains(Reg: SrcReg) ||
945 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(Reg: SrcReg))) {
946 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
947 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
948 return;
949 }
950
951 if (AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg) && ST.hasGFX90AInsts()) {
952 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_MOV_B32), DestReg)
953 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
954 return;
955 }
956
957 // FIXME: Pass should maintain scavenger to avoid scan through the block on
958 // every AGPR spill.
959 RegScavenger RS;
960 const bool Overlap = RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
961 indirectCopyToAGPR(TII: *this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS, RegsOverlap: Overlap);
962 return;
963 }
964
965 if (Size == 16) {
966 assert(AMDGPU::VGPR_16RegClass.contains(SrcReg) ||
967 AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
968 AMDGPU::AGPR_LO16RegClass.contains(SrcReg));
969
970 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(Reg: DestReg);
971 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(Reg: SrcReg);
972 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(Reg: DestReg);
973 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(Reg: SrcReg);
974 bool DstLow = !AMDGPU::isHi16Reg(Reg: DestReg, MRI: RI);
975 bool SrcLow = !AMDGPU::isHi16Reg(Reg: SrcReg, MRI: RI);
976 MCRegister NewDestReg = RI.get32BitRegister(Reg: DestReg);
977 MCRegister NewSrcReg = RI.get32BitRegister(Reg: SrcReg);
978
979 if (IsSGPRDst) {
980 if (!IsSGPRSrc) {
981 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
982 return;
983 }
984
985 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: NewDestReg)
986 .addReg(RegNo: NewSrcReg, Flags: getKillRegState(B: KillSrc));
987 return;
988 }
989
990 if (IsAGPRDst || IsAGPRSrc) {
991 if (!DstLow || !SrcLow) {
992 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
993 Msg: "Cannot use hi16 subreg with an AGPR!");
994 }
995
996 copyPhysReg(MBB, MI, DL, DestReg: NewDestReg, SrcReg: NewSrcReg, KillSrc);
997 return;
998 }
999
1000 if (ST.useRealTrue16Insts()) {
1001 if (IsSGPRSrc) {
1002 assert(SrcLow);
1003 SrcReg = NewSrcReg;
1004 }
1005 // Use the smaller instruction encoding if possible.
1006 if (AMDGPU::VGPR_16_Lo128RegClass.contains(Reg: DestReg) &&
1007 (IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.contains(Reg: SrcReg))) {
1008 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B16_t16_e32), DestReg)
1009 .addReg(RegNo: SrcReg);
1010 } else {
1011 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B16_t16_e64), DestReg)
1012 .addImm(Val: 0) // src0_modifiers
1013 .addReg(RegNo: SrcReg)
1014 .addImm(Val: 0); // op_sel
1015 }
1016 return;
1017 }
1018
1019 if (IsSGPRSrc && !ST.hasSDWAScalar()) {
1020 if (!DstLow || !SrcLow) {
1021 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
1022 Msg: "Cannot use hi16 subreg on VI!");
1023 }
1024
1025 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: NewDestReg)
1026 .addReg(RegNo: NewSrcReg, Flags: getKillRegState(B: KillSrc));
1027 return;
1028 }
1029
1030 auto MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_sdwa), DestReg: NewDestReg)
1031 .addImm(Val: 0) // src0_modifiers
1032 .addReg(RegNo: NewSrcReg)
1033 .addImm(Val: 0) // clamp
1034 .addImm(Val: DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1035 : AMDGPU::SDWA::SdwaSel::WORD_1)
1036 .addImm(Val: AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE)
1037 .addImm(Val: SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1038 : AMDGPU::SDWA::SdwaSel::WORD_1)
1039 .addReg(RegNo: NewDestReg, Flags: RegState::Implicit | RegState::Undef);
1040 // First implicit operand is $exec.
1041 MIB->tieOperands(DefIdx: 0, UseIdx: MIB->getNumOperands() - 1);
1042 return;
1043 }
1044
1045 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(RC: SrcRC))) {
1046 if (ST.hasVMovB64Inst()) {
1047 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B64_e32), DestReg)
1048 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1049 return;
1050 }
1051 if (ST.hasPkMovB32()) {
1052 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg)
1053 .addImm(Val: SISrcMods::OP_SEL_1)
1054 .addReg(RegNo: SrcReg)
1055 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
1056 .addReg(RegNo: SrcReg)
1057 .addImm(Val: 0) // op_sel_lo
1058 .addImm(Val: 0) // op_sel_hi
1059 .addImm(Val: 0) // neg_lo
1060 .addImm(Val: 0) // neg_hi
1061 .addImm(Val: 0) // clamp
1062 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
1063 return;
1064 }
1065 }
1066
1067 const bool Forward = RI.getHWRegIndex(Reg: DestReg) <= RI.getHWRegIndex(Reg: SrcReg);
1068 if (RI.isSGPRClass(RC)) {
1069 if (!RI.isSGPRClass(RC: SrcRC)) {
1070 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1071 return;
1072 }
1073 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1074 expandSGPRCopy(TII: *this, MBB, MI, DL, DestReg, SrcReg, KillSrc: CanKillSuperReg, RC,
1075 Forward);
1076 return;
1077 }
1078
1079 unsigned EltSize = 4;
1080 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1081 if (RI.isAGPRClass(RC)) {
1082 if (ST.hasGFX90AInsts() && RI.isAGPRClass(RC: SrcRC))
1083 Opcode = AMDGPU::V_ACCVGPR_MOV_B32;
1084 else if (RI.hasVGPRs(RC: SrcRC) ||
1085 (ST.hasGFX90AInsts() && RI.isSGPRClass(RC: SrcRC)))
1086 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
1087 else
1088 Opcode = AMDGPU::INSTRUCTION_LIST_END;
1089 } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(RC: SrcRC)) {
1090 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;
1091 } else if ((Size % 64 == 0) && RI.hasVGPRs(RC) &&
1092 (RI.isProperlyAlignedRC(RC: *RC) &&
1093 (SrcRC == RC || RI.isSGPRClass(RC: SrcRC)))) {
1094 // TODO: In 96-bit case, could do a 64-bit mov and then a 32-bit mov.
1095 if (ST.hasVMovB64Inst()) {
1096 Opcode = AMDGPU::V_MOV_B64_e32;
1097 EltSize = 8;
1098 } else if (ST.hasPkMovB32()) {
1099 Opcode = AMDGPU::V_PK_MOV_B32;
1100 EltSize = 8;
1101 }
1102 }
1103
1104 // For the cases where we need an intermediate instruction/temporary register
1105 // (destination is an AGPR), we need a scavenger.
1106 //
1107 // FIXME: The pass should maintain this for us so we don't have to re-scan the
1108 // whole block for every handled copy.
1109 std::unique_ptr<RegScavenger> RS;
1110 if (Opcode == AMDGPU::INSTRUCTION_LIST_END)
1111 RS = std::make_unique<RegScavenger>();
1112
1113 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
1114
1115 // If there is an overlap, we can't kill the super-register on the last
1116 // instruction, since it will also kill the components made live by this def.
1117 const bool Overlap = RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1118 const bool CanKillSuperReg = KillSrc && !Overlap;
1119
1120 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
1121 unsigned SubIdx;
1122 if (Forward)
1123 SubIdx = SubIndices[Idx];
1124 else
1125 SubIdx = SubIndices[SubIndices.size() - Idx - 1];
1126 Register DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
1127 Register SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
1128 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
1129
1130 bool UseKill = CanKillSuperReg && Idx == SubIndices.size() - 1;
1131
1132 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) {
1133 Register ImpUseSuper = SrcReg;
1134 indirectCopyToAGPR(TII: *this, MBB, MI, DL, DestReg: DestSubReg, SrcReg: SrcSubReg, KillSrc: UseKill,
1135 RS&: *RS, RegsOverlap: Overlap, ImpUseSuperReg: ImpUseSuper);
1136 } else if (Opcode == AMDGPU::V_PK_MOV_B32) {
1137 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg: DestSubReg)
1138 .addImm(Val: SISrcMods::OP_SEL_1)
1139 .addReg(RegNo: SrcSubReg)
1140 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
1141 .addReg(RegNo: SrcSubReg)
1142 .addImm(Val: 0) // op_sel_lo
1143 .addImm(Val: 0) // op_sel_hi
1144 .addImm(Val: 0) // neg_lo
1145 .addImm(Val: 0) // neg_hi
1146 .addImm(Val: 0) // clamp
1147 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: UseKill) | RegState::Implicit);
1148 } else {
1149 MachineInstrBuilder Builder =
1150 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode), DestReg: DestSubReg).addReg(RegNo: SrcSubReg);
1151
1152 Builder.addReg(RegNo: SrcReg, Flags: getKillRegState(B: UseKill) | RegState::Implicit);
1153 }
1154 }
1155}
1156
1157int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
1158 int32_t NewOpc;
1159
1160 // Try to map original to commuted opcode
1161 NewOpc = AMDGPU::getCommuteRev(Opcode);
1162 if (NewOpc != -1)
1163 // Check if the commuted (REV) opcode exists on the target.
1164 return pseudoToMCOpcode(Opcode: NewOpc) != -1 ? NewOpc : -1;
1165
1166 // Try to map commuted to original opcode
1167 NewOpc = AMDGPU::getCommuteOrig(Opcode);
1168 if (NewOpc != -1)
1169 // Check if the original (non-REV) opcode exists on the target.
1170 return pseudoToMCOpcode(Opcode: NewOpc) != -1 ? NewOpc : -1;
1171
1172 return Opcode;
1173}
1174
1175bool SIInstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
1176 const Register Reg,
1177 int64_t &ImmVal) const {
1178 switch (MI.getOpcode()) {
1179 case AMDGPU::V_MOV_B32_e32:
1180 case AMDGPU::S_MOV_B32:
1181 case AMDGPU::S_MOVK_I32:
1182 case AMDGPU::S_MOV_B64:
1183 case AMDGPU::V_MOV_B64_e32:
1184 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
1185 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
1186 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
1187 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
1188 case AMDGPU::V_MOV_B64_PSEUDO:
1189 case AMDGPU::V_MOV_B16_t16_e32: {
1190 const MachineOperand &Src0 = MI.getOperand(i: 1);
1191 if (Src0.isImm()) {
1192 ImmVal = Src0.getImm();
1193 return MI.getOperand(i: 0).getReg() == Reg;
1194 }
1195
1196 return false;
1197 }
1198 case AMDGPU::V_MOV_B16_t16_e64: {
1199 const MachineOperand &Src0 = MI.getOperand(i: 2);
1200 if (Src0.isImm() && !MI.getOperand(i: 1).getImm()) {
1201 ImmVal = Src0.getImm();
1202 return MI.getOperand(i: 0).getReg() == Reg;
1203 }
1204
1205 return false;
1206 }
1207 case AMDGPU::S_BREV_B32:
1208 case AMDGPU::V_BFREV_B32_e32:
1209 case AMDGPU::V_BFREV_B32_e64: {
1210 const MachineOperand &Src0 = MI.getOperand(i: 1);
1211 if (Src0.isImm()) {
1212 ImmVal = static_cast<int64_t>(reverseBits<int32_t>(Val: Src0.getImm()));
1213 return MI.getOperand(i: 0).getReg() == Reg;
1214 }
1215
1216 return false;
1217 }
1218 case AMDGPU::S_NOT_B32:
1219 case AMDGPU::V_NOT_B32_e32:
1220 case AMDGPU::V_NOT_B32_e64: {
1221 const MachineOperand &Src0 = MI.getOperand(i: 1);
1222 if (Src0.isImm()) {
1223 ImmVal = static_cast<int64_t>(~static_cast<int32_t>(Src0.getImm()));
1224 return MI.getOperand(i: 0).getReg() == Reg;
1225 }
1226
1227 return false;
1228 }
1229 default:
1230 return false;
1231 }
1232}
1233
1234std::optional<int64_t>
1235SIInstrInfo::getImmOrMaterializedImm(MachineOperand &Op) const {
1236 if (Op.isImm())
1237 return Op.getImm();
1238
1239 if (!Op.isReg() || !Op.getReg().isVirtual())
1240 return std::nullopt;
1241 MachineRegisterInfo &MRI = Op.getParent()->getMF()->getRegInfo();
1242 const MachineInstr *Def = MRI.getVRegDef(Reg: Op.getReg());
1243 if (Def && Def->isMoveImmediate()) {
1244 const MachineOperand &ImmSrc = Def->getOperand(i: 1);
1245 if (ImmSrc.isImm())
1246 return extractSubregFromImm(ImmVal: ImmSrc.getImm(), SubRegIndex: Op.getSubReg());
1247 }
1248
1249 return std::nullopt;
1250}
1251
1252unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
1253
1254 if (RI.isAGPRClass(RC: DstRC))
1255 return AMDGPU::COPY;
1256 if (RI.getRegSizeInBits(RC: *DstRC) == 16) {
1257 // Assume hi bits are unneeded. Only _e64 true16 instructions are legal
1258 // before RA.
1259 return RI.isSGPRClass(RC: DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64;
1260 }
1261 if (RI.getRegSizeInBits(RC: *DstRC) == 32)
1262 return RI.isSGPRClass(RC: DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1263 if (RI.getRegSizeInBits(RC: *DstRC) == 64 && RI.isSGPRClass(RC: DstRC))
1264 return AMDGPU::S_MOV_B64;
1265 if (RI.getRegSizeInBits(RC: *DstRC) == 64 && !RI.isSGPRClass(RC: DstRC))
1266 return AMDGPU::V_MOV_B64_PSEUDO;
1267 return AMDGPU::COPY;
1268}
1269
1270const MCInstrDesc &
1271SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize,
1272 bool IsIndirectSrc) const {
1273 if (IsIndirectSrc) {
1274 if (VecSize <= 32) // 4 bytes
1275 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);
1276 if (VecSize <= 64) // 8 bytes
1277 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);
1278 if (VecSize <= 96) // 12 bytes
1279 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);
1280 if (VecSize <= 128) // 16 bytes
1281 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);
1282 if (VecSize <= 160) // 20 bytes
1283 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);
1284 if (VecSize <= 192) // 24 bytes
1285 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V6);
1286 if (VecSize <= 224) // 28 bytes
1287 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V7);
1288 if (VecSize <= 256) // 32 bytes
1289 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);
1290 if (VecSize <= 288) // 36 bytes
1291 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9);
1292 if (VecSize <= 320) // 40 bytes
1293 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10);
1294 if (VecSize <= 352) // 44 bytes
1295 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11);
1296 if (VecSize <= 384) // 48 bytes
1297 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12);
1298 if (VecSize <= 512) // 64 bytes
1299 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);
1300 if (VecSize <= 1024) // 128 bytes
1301 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);
1302
1303 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos");
1304 }
1305
1306 if (VecSize <= 32) // 4 bytes
1307 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);
1308 if (VecSize <= 64) // 8 bytes
1309 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);
1310 if (VecSize <= 96) // 12 bytes
1311 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);
1312 if (VecSize <= 128) // 16 bytes
1313 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);
1314 if (VecSize <= 160) // 20 bytes
1315 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);
1316 if (VecSize <= 192) // 24 bytes
1317 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V6);
1318 if (VecSize <= 224) // 28 bytes
1319 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V7);
1320 if (VecSize <= 256) // 32 bytes
1321 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);
1322 if (VecSize <= 288) // 36 bytes
1323 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9);
1324 if (VecSize <= 320) // 40 bytes
1325 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10);
1326 if (VecSize <= 352) // 44 bytes
1327 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11);
1328 if (VecSize <= 384) // 48 bytes
1329 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12);
1330 if (VecSize <= 512) // 64 bytes
1331 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);
1332 if (VecSize <= 1024) // 128 bytes
1333 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);
1334
1335 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos");
1336}
1337
1338static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) {
1339 if (VecSize <= 32) // 4 bytes
1340 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1341 if (VecSize <= 64) // 8 bytes
1342 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1343 if (VecSize <= 96) // 12 bytes
1344 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1345 if (VecSize <= 128) // 16 bytes
1346 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1347 if (VecSize <= 160) // 20 bytes
1348 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1349 if (VecSize <= 192) // 24 bytes
1350 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V6;
1351 if (VecSize <= 224) // 28 bytes
1352 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V7;
1353 if (VecSize <= 256) // 32 bytes
1354 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1355 if (VecSize <= 288) // 36 bytes
1356 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1357 if (VecSize <= 320) // 40 bytes
1358 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1359 if (VecSize <= 352) // 44 bytes
1360 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1361 if (VecSize <= 384) // 48 bytes
1362 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1363 if (VecSize <= 512) // 64 bytes
1364 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1365 if (VecSize <= 1024) // 128 bytes
1366 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1367
1368 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1369}
1370
1371static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) {
1372 if (VecSize <= 32) // 4 bytes
1373 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1374 if (VecSize <= 64) // 8 bytes
1375 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1376 if (VecSize <= 96) // 12 bytes
1377 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1378 if (VecSize <= 128) // 16 bytes
1379 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1380 if (VecSize <= 160) // 20 bytes
1381 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1382 if (VecSize <= 192) // 24 bytes
1383 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V6;
1384 if (VecSize <= 224) // 28 bytes
1385 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V7;
1386 if (VecSize <= 256) // 32 bytes
1387 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1388 if (VecSize <= 288) // 36 bytes
1389 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1390 if (VecSize <= 320) // 40 bytes
1391 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1392 if (VecSize <= 352) // 44 bytes
1393 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1394 if (VecSize <= 384) // 48 bytes
1395 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1396 if (VecSize <= 512) // 64 bytes
1397 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1398 if (VecSize <= 1024) // 128 bytes
1399 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1400
1401 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1402}
1403
1404static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) {
1405 if (VecSize <= 64) // 8 bytes
1406 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;
1407 if (VecSize <= 128) // 16 bytes
1408 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;
1409 if (VecSize <= 256) // 32 bytes
1410 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;
1411 if (VecSize <= 512) // 64 bytes
1412 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;
1413 if (VecSize <= 1024) // 128 bytes
1414 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;
1415
1416 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1417}
1418
1419const MCInstrDesc &
1420SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize,
1421 bool IsSGPR) const {
1422 if (IsSGPR) {
1423 switch (EltSize) {
1424 case 32:
1425 return get(Opcode: getIndirectSGPRWriteMovRelPseudo32(VecSize));
1426 case 64:
1427 return get(Opcode: getIndirectSGPRWriteMovRelPseudo64(VecSize));
1428 default:
1429 llvm_unreachable("invalid reg indexing elt size");
1430 }
1431 }
1432
1433 assert(EltSize == 32 && "invalid reg indexing elt size");
1434 return get(Opcode: getIndirectVGPRWriteMovRelPseudoOpc(VecSize));
1435}
1436
1437static unsigned getSGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1438 switch (Size) {
1439 case 4:
1440 return NeedsCFI ? AMDGPU::SI_SPILL_S32_CFI_SAVE : AMDGPU::SI_SPILL_S32_SAVE;
1441 case 8:
1442 return NeedsCFI ? AMDGPU::SI_SPILL_S64_CFI_SAVE : AMDGPU::SI_SPILL_S64_SAVE;
1443 case 12:
1444 return NeedsCFI ? AMDGPU::SI_SPILL_S96_CFI_SAVE : AMDGPU::SI_SPILL_S96_SAVE;
1445 case 16:
1446 return NeedsCFI ? AMDGPU::SI_SPILL_S128_CFI_SAVE
1447 : AMDGPU::SI_SPILL_S128_SAVE;
1448 case 20:
1449 return NeedsCFI ? AMDGPU::SI_SPILL_S160_CFI_SAVE
1450 : AMDGPU::SI_SPILL_S160_SAVE;
1451 case 24:
1452 return NeedsCFI ? AMDGPU::SI_SPILL_S192_CFI_SAVE
1453 : AMDGPU::SI_SPILL_S192_SAVE;
1454 case 28:
1455 return NeedsCFI ? AMDGPU::SI_SPILL_S224_CFI_SAVE
1456 : AMDGPU::SI_SPILL_S224_SAVE;
1457 case 32:
1458 return AMDGPU::SI_SPILL_S256_SAVE;
1459 case 36:
1460 return AMDGPU::SI_SPILL_S288_SAVE;
1461 case 40:
1462 return AMDGPU::SI_SPILL_S320_SAVE;
1463 case 44:
1464 return AMDGPU::SI_SPILL_S352_SAVE;
1465 case 48:
1466 return AMDGPU::SI_SPILL_S384_SAVE;
1467 case 64:
1468 return NeedsCFI ? AMDGPU::SI_SPILL_S512_CFI_SAVE
1469 : AMDGPU::SI_SPILL_S512_SAVE;
1470 case 128:
1471 return NeedsCFI ? AMDGPU::SI_SPILL_S1024_CFI_SAVE
1472 : AMDGPU::SI_SPILL_S1024_SAVE;
1473 default:
1474 llvm_unreachable("unknown register size");
1475 }
1476}
1477
1478static unsigned getVGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1479 switch (Size) {
1480 case 2:
1481 return AMDGPU::SI_SPILL_V16_SAVE;
1482 case 4:
1483 return NeedsCFI ? AMDGPU::SI_SPILL_V32_CFI_SAVE : AMDGPU::SI_SPILL_V32_SAVE;
1484 case 8:
1485 return NeedsCFI ? AMDGPU::SI_SPILL_V64_CFI_SAVE : AMDGPU::SI_SPILL_V64_SAVE;
1486 case 12:
1487 return NeedsCFI ? AMDGPU::SI_SPILL_V96_CFI_SAVE : AMDGPU::SI_SPILL_V96_SAVE;
1488 case 16:
1489 return NeedsCFI ? AMDGPU::SI_SPILL_V128_CFI_SAVE
1490 : AMDGPU::SI_SPILL_V128_SAVE;
1491 case 20:
1492 return NeedsCFI ? AMDGPU::SI_SPILL_V160_CFI_SAVE
1493 : AMDGPU::SI_SPILL_V160_SAVE;
1494 case 24:
1495 return NeedsCFI ? AMDGPU::SI_SPILL_V192_CFI_SAVE
1496 : AMDGPU::SI_SPILL_V192_SAVE;
1497 case 28:
1498 return NeedsCFI ? AMDGPU::SI_SPILL_V224_CFI_SAVE
1499 : AMDGPU::SI_SPILL_V224_SAVE;
1500 case 32:
1501 return NeedsCFI ? AMDGPU::SI_SPILL_V256_CFI_SAVE
1502 : AMDGPU::SI_SPILL_V256_SAVE;
1503 case 36:
1504 return NeedsCFI ? AMDGPU::SI_SPILL_V288_CFI_SAVE
1505 : AMDGPU::SI_SPILL_V288_SAVE;
1506 case 40:
1507 return NeedsCFI ? AMDGPU::SI_SPILL_V320_CFI_SAVE
1508 : AMDGPU::SI_SPILL_V320_SAVE;
1509 case 44:
1510 return NeedsCFI ? AMDGPU::SI_SPILL_V352_CFI_SAVE
1511 : AMDGPU::SI_SPILL_V352_SAVE;
1512 case 48:
1513 return NeedsCFI ? AMDGPU::SI_SPILL_V384_CFI_SAVE
1514 : AMDGPU::SI_SPILL_V384_SAVE;
1515 case 64:
1516 return NeedsCFI ? AMDGPU::SI_SPILL_V512_CFI_SAVE
1517 : AMDGPU::SI_SPILL_V512_SAVE;
1518 case 128:
1519 return NeedsCFI ? AMDGPU::SI_SPILL_V1024_CFI_SAVE
1520 : AMDGPU::SI_SPILL_V1024_SAVE;
1521 default:
1522 llvm_unreachable("unknown register size");
1523 }
1524}
1525
1526static unsigned getAVSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1527 switch (Size) {
1528 case 4:
1529 return NeedsCFI ? AMDGPU::SI_SPILL_AV32_CFI_SAVE
1530 : AMDGPU::SI_SPILL_AV32_SAVE;
1531 case 8:
1532 return NeedsCFI ? AMDGPU::SI_SPILL_AV64_CFI_SAVE
1533 : AMDGPU::SI_SPILL_AV64_SAVE;
1534 case 12:
1535 return NeedsCFI ? AMDGPU::SI_SPILL_AV96_CFI_SAVE
1536 : AMDGPU::SI_SPILL_AV96_SAVE;
1537 case 16:
1538 return NeedsCFI ? AMDGPU::SI_SPILL_AV128_CFI_SAVE
1539 : AMDGPU::SI_SPILL_AV128_SAVE;
1540 case 20:
1541 return NeedsCFI ? AMDGPU::SI_SPILL_AV160_CFI_SAVE
1542 : AMDGPU::SI_SPILL_AV160_SAVE;
1543 case 24:
1544 return NeedsCFI ? AMDGPU::SI_SPILL_AV192_CFI_SAVE
1545 : AMDGPU::SI_SPILL_AV192_SAVE;
1546 case 28:
1547 return NeedsCFI ? AMDGPU::SI_SPILL_AV224_CFI_SAVE
1548 : AMDGPU::SI_SPILL_AV224_SAVE;
1549 case 32:
1550 return NeedsCFI ? AMDGPU::SI_SPILL_AV256_CFI_SAVE
1551 : AMDGPU::SI_SPILL_AV256_SAVE;
1552 case 36:
1553 return AMDGPU::SI_SPILL_AV288_SAVE;
1554 case 40:
1555 return AMDGPU::SI_SPILL_AV320_SAVE;
1556 case 44:
1557 return AMDGPU::SI_SPILL_AV352_SAVE;
1558 case 48:
1559 return AMDGPU::SI_SPILL_AV384_SAVE;
1560 case 64:
1561 return NeedsCFI ? AMDGPU::SI_SPILL_AV512_CFI_SAVE
1562 : AMDGPU::SI_SPILL_AV512_SAVE;
1563 case 128:
1564 return NeedsCFI ? AMDGPU::SI_SPILL_AV1024_CFI_SAVE
1565 : AMDGPU::SI_SPILL_AV1024_SAVE;
1566 default:
1567 llvm_unreachable("unknown register size");
1568 }
1569}
1570
1571static unsigned getWWMRegSpillSaveOpcode(unsigned Size,
1572 bool IsVectorSuperClass) {
1573 // Currently, there is only 32-bit WWM register spills needed.
1574 if (Size != 4)
1575 llvm_unreachable("unknown wwm register spill size");
1576
1577 if (IsVectorSuperClass)
1578 return AMDGPU::SI_SPILL_WWM_AV32_SAVE;
1579
1580 return AMDGPU::SI_SPILL_WWM_V32_SAVE;
1581}
1582
1583unsigned SIInstrInfo::getVectorRegSpillSaveOpcode(
1584 Register Reg, const TargetRegisterClass *RC, unsigned Size,
1585 const SIMachineFunctionInfo &MFI, bool NeedsCFI) const {
1586 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
1587
1588 // Choose the right opcode if spilling a WWM register.
1589 if (MFI.checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
1590 return getWWMRegSpillSaveOpcode(Size, IsVectorSuperClass);
1591
1592 // TODO: Check if AGPRs are available
1593 if (ST.hasMAIInsts())
1594 return getAVSpillSaveOpcode(Size, NeedsCFI);
1595
1596 return getVGPRSpillSaveOpcode(Size, NeedsCFI);
1597}
1598
1599void SIInstrInfo::storeRegToStackSlotImpl(
1600 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
1601 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
1602 MachineInstr::MIFlag Flags, bool NeedsCFI) const {
1603 MachineFunction *MF = MBB.getParent();
1604 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1605 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1606 const DebugLoc &DL = MBB.findDebugLoc(MBBI: MI);
1607
1608 MachinePointerInfo PtrInfo
1609 = MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
1610 MachineMemOperand *MMO = MF->getMachineMemOperand(
1611 PtrInfo, F: MachineMemOperand::MOStore, Size: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
1612 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
1613 unsigned SpillSize = RI.getSpillSize(RC: *RC);
1614
1615 MachineRegisterInfo &MRI = MF->getRegInfo();
1616 if (RI.isSGPRClass(RC)) {
1617 if (FrameInfo.getStackID(ObjectIdx: FrameIndex) == TargetStackID::SGPRSpill)
1618 MFI->setHasSpilledSGPRs();
1619 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled");
1620 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI &&
1621 SrcReg != AMDGPU::EXEC && "exec should not be spilled");
1622
1623 // We are only allowed to create one new instruction when spilling
1624 // registers, so we need to use pseudo instruction for spilling SGPRs.
1625 const MCInstrDesc &OpDesc =
1626 get(Opcode: getSGPRSpillSaveOpcode(Size: SpillSize, NeedsCFI));
1627
1628 // The SGPR spill/restore instructions only work on number sgprs, so we need
1629 // to make sure we are using the correct register class.
1630 if (SrcReg.isVirtual() && SpillSize == 4) {
1631 MRI.constrainRegClass(Reg: SrcReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
1632 }
1633
1634 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: OpDesc)
1635 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill)) // data
1636 .addFrameIndex(Idx: FrameIndex) // addr
1637 .addMemOperand(MMO)
1638 .addReg(RegNo: MFI->getStackPtrOffsetReg(), Flags: RegState::Implicit);
1639
1640 return;
1641 }
1642
1643 unsigned Opcode = getVectorRegSpillSaveOpcode(Reg: VReg ? VReg : SrcReg, RC,
1644 Size: SpillSize, MFI: *MFI, NeedsCFI);
1645 MFI->setHasSpilledVGPRs();
1646
1647 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode))
1648 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill)) // data
1649 .addFrameIndex(Idx: FrameIndex) // addr
1650 .addReg(RegNo: MFI->getStackPtrOffsetReg()) // scratch_offset
1651 .addImm(Val: 0) // offset
1652 .addMemOperand(MMO);
1653}
1654
1655void SIInstrInfo::storeRegToStackSlot(
1656 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
1657 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
1658 MachineInstr::MIFlag Flags) const {
1659 storeRegToStackSlotImpl(MBB, MI, SrcReg, isKill, FrameIndex, RC, VReg, Flags,
1660 NeedsCFI: false);
1661}
1662
1663void SIInstrInfo::storeRegToStackSlotCFI(MachineBasicBlock &MBB,
1664 MachineBasicBlock::iterator MI,
1665 Register SrcReg, bool isKill,
1666 int FrameIndex,
1667 const TargetRegisterClass *RC) const {
1668 storeRegToStackSlotImpl(MBB, MI, SrcReg, isKill, FrameIndex, RC, VReg: Register(),
1669 Flags: MachineInstr::NoFlags, NeedsCFI: true);
1670}
1671
1672static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1673 switch (Size) {
1674 case 4:
1675 return AMDGPU::SI_SPILL_S32_RESTORE;
1676 case 8:
1677 return AMDGPU::SI_SPILL_S64_RESTORE;
1678 case 12:
1679 return AMDGPU::SI_SPILL_S96_RESTORE;
1680 case 16:
1681 return AMDGPU::SI_SPILL_S128_RESTORE;
1682 case 20:
1683 return AMDGPU::SI_SPILL_S160_RESTORE;
1684 case 24:
1685 return AMDGPU::SI_SPILL_S192_RESTORE;
1686 case 28:
1687 return AMDGPU::SI_SPILL_S224_RESTORE;
1688 case 32:
1689 return AMDGPU::SI_SPILL_S256_RESTORE;
1690 case 36:
1691 return AMDGPU::SI_SPILL_S288_RESTORE;
1692 case 40:
1693 return AMDGPU::SI_SPILL_S320_RESTORE;
1694 case 44:
1695 return AMDGPU::SI_SPILL_S352_RESTORE;
1696 case 48:
1697 return AMDGPU::SI_SPILL_S384_RESTORE;
1698 case 64:
1699 return AMDGPU::SI_SPILL_S512_RESTORE;
1700 case 128:
1701 return AMDGPU::SI_SPILL_S1024_RESTORE;
1702 default:
1703 llvm_unreachable("unknown register size");
1704 }
1705}
1706
1707static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1708 switch (Size) {
1709 case 2:
1710 return AMDGPU::SI_SPILL_V16_RESTORE;
1711 case 4:
1712 return AMDGPU::SI_SPILL_V32_RESTORE;
1713 case 8:
1714 return AMDGPU::SI_SPILL_V64_RESTORE;
1715 case 12:
1716 return AMDGPU::SI_SPILL_V96_RESTORE;
1717 case 16:
1718 return AMDGPU::SI_SPILL_V128_RESTORE;
1719 case 20:
1720 return AMDGPU::SI_SPILL_V160_RESTORE;
1721 case 24:
1722 return AMDGPU::SI_SPILL_V192_RESTORE;
1723 case 28:
1724 return AMDGPU::SI_SPILL_V224_RESTORE;
1725 case 32:
1726 return AMDGPU::SI_SPILL_V256_RESTORE;
1727 case 36:
1728 return AMDGPU::SI_SPILL_V288_RESTORE;
1729 case 40:
1730 return AMDGPU::SI_SPILL_V320_RESTORE;
1731 case 44:
1732 return AMDGPU::SI_SPILL_V352_RESTORE;
1733 case 48:
1734 return AMDGPU::SI_SPILL_V384_RESTORE;
1735 case 64:
1736 return AMDGPU::SI_SPILL_V512_RESTORE;
1737 case 128:
1738 return AMDGPU::SI_SPILL_V1024_RESTORE;
1739 default:
1740 llvm_unreachable("unknown register size");
1741 }
1742}
1743
1744static unsigned getAVSpillRestoreOpcode(unsigned Size) {
1745 switch (Size) {
1746 case 4:
1747 return AMDGPU::SI_SPILL_AV32_RESTORE;
1748 case 8:
1749 return AMDGPU::SI_SPILL_AV64_RESTORE;
1750 case 12:
1751 return AMDGPU::SI_SPILL_AV96_RESTORE;
1752 case 16:
1753 return AMDGPU::SI_SPILL_AV128_RESTORE;
1754 case 20:
1755 return AMDGPU::SI_SPILL_AV160_RESTORE;
1756 case 24:
1757 return AMDGPU::SI_SPILL_AV192_RESTORE;
1758 case 28:
1759 return AMDGPU::SI_SPILL_AV224_RESTORE;
1760 case 32:
1761 return AMDGPU::SI_SPILL_AV256_RESTORE;
1762 case 36:
1763 return AMDGPU::SI_SPILL_AV288_RESTORE;
1764 case 40:
1765 return AMDGPU::SI_SPILL_AV320_RESTORE;
1766 case 44:
1767 return AMDGPU::SI_SPILL_AV352_RESTORE;
1768 case 48:
1769 return AMDGPU::SI_SPILL_AV384_RESTORE;
1770 case 64:
1771 return AMDGPU::SI_SPILL_AV512_RESTORE;
1772 case 128:
1773 return AMDGPU::SI_SPILL_AV1024_RESTORE;
1774 default:
1775 llvm_unreachable("unknown register size");
1776 }
1777}
1778
1779static unsigned getWWMRegSpillRestoreOpcode(unsigned Size,
1780 bool IsVectorSuperClass) {
1781 // Currently, there is only 32-bit WWM register spills needed.
1782 if (Size != 4)
1783 llvm_unreachable("unknown wwm register spill size");
1784
1785 if (IsVectorSuperClass) // TODO: Always use this if there are AGPRs
1786 return AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
1787
1788 return AMDGPU::SI_SPILL_WWM_V32_RESTORE;
1789}
1790
1791unsigned SIInstrInfo::getVectorRegSpillRestoreOpcode(
1792 Register Reg, const TargetRegisterClass *RC, unsigned Size,
1793 const SIMachineFunctionInfo &MFI) const {
1794 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
1795
1796 // Choose the right opcode if restoring a WWM register.
1797 if (MFI.checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
1798 return getWWMRegSpillRestoreOpcode(Size, IsVectorSuperClass);
1799
1800 // TODO: Check if AGPRs are available
1801 if (ST.hasMAIInsts())
1802 return getAVSpillRestoreOpcode(Size);
1803
1804 assert(!RI.isAGPRClass(RC));
1805 return getVGPRSpillRestoreOpcode(Size);
1806}
1807
1808void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1809 MachineBasicBlock::iterator MI,
1810 Register DestReg, int FrameIndex,
1811 const TargetRegisterClass *RC,
1812 Register VReg, unsigned SubReg,
1813 MachineInstr::MIFlag Flags) const {
1814 MachineFunction *MF = MBB.getParent();
1815 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1816 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1817 const DebugLoc &DL = MBB.findDebugLoc(MBBI: MI);
1818 unsigned SpillSize = RI.getSpillSize(RC: *RC);
1819
1820 MachinePointerInfo PtrInfo
1821 = MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
1822
1823 MachineMemOperand *MMO = MF->getMachineMemOperand(
1824 PtrInfo, F: MachineMemOperand::MOLoad, Size: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
1825 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
1826
1827 if (RI.isSGPRClass(RC)) {
1828 if (FrameInfo.getStackID(ObjectIdx: FrameIndex) == TargetStackID::SGPRSpill)
1829 MFI->setHasSpilledSGPRs();
1830 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into");
1831 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI &&
1832 DestReg != AMDGPU::EXEC && "exec should not be spilled");
1833
1834 // FIXME: Maybe this should not include a memoperand because it will be
1835 // lowered to non-memory instructions.
1836 const MCInstrDesc &OpDesc = get(Opcode: getSGPRSpillRestoreOpcode(Size: SpillSize));
1837 if (DestReg.isVirtual() && SpillSize == 4) {
1838 MachineRegisterInfo &MRI = MF->getRegInfo();
1839 MRI.constrainRegClass(Reg: DestReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
1840 }
1841
1842 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: OpDesc, DestReg)
1843 .addFrameIndex(Idx: FrameIndex) // addr
1844 .addMemOperand(MMO)
1845 .addReg(RegNo: MFI->getStackPtrOffsetReg(), Flags: RegState::Implicit);
1846
1847 return;
1848 }
1849
1850 unsigned Opcode = getVectorRegSpillRestoreOpcode(Reg: VReg ? VReg : DestReg, RC,
1851 Size: SpillSize, MFI: *MFI);
1852 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode), DestReg)
1853 .addFrameIndex(Idx: FrameIndex) // vaddr
1854 .addReg(RegNo: MFI->getStackPtrOffsetReg()) // scratch_offset
1855 .addImm(Val: 0) // offset
1856 .addMemOperand(MMO);
1857}
1858
1859void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
1860 MachineBasicBlock::iterator MI) const {
1861 insertNoops(MBB, MI, Quantity: 1);
1862}
1863
1864void SIInstrInfo::insertNoops(MachineBasicBlock &MBB,
1865 MachineBasicBlock::iterator MI,
1866 unsigned Quantity) const {
1867 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
1868 unsigned MaxSNopCount = 1u << ST.getSNopBits();
1869 while (Quantity > 0) {
1870 unsigned Arg = std::min(a: Quantity, b: MaxSNopCount);
1871 Quantity -= Arg;
1872 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOP)).addImm(Val: Arg - 1);
1873 }
1874}
1875
1876MachineBasicBlock *SIInstrInfo::insertSimulatedTrap(MachineRegisterInfo &MRI,
1877 MachineBasicBlock &MBB,
1878 MachineInstr &MI,
1879 const DebugLoc &DL) const {
1880 MachineFunction *MF = MBB.getParent();
1881 constexpr unsigned DoorbellIDMask = 0x3ff;
1882 constexpr unsigned ECQueueWaveAbort = 0x400;
1883
1884 MachineBasicBlock *TrapBB = &MBB;
1885 MachineBasicBlock *HaltLoopBB = MF->CreateMachineBasicBlock();
1886
1887 if (!MBB.succ_empty() || std::next(x: MI.getIterator()) != MBB.end()) {
1888 MBB.splitAt(SplitInst&: MI, /*UpdateLiveIns=*/false);
1889 TrapBB = MF->CreateMachineBasicBlock();
1890 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CBRANCH_EXECNZ)).addMBB(MBB: TrapBB);
1891 MF->push_back(MBB: TrapBB);
1892 MBB.addSuccessor(Succ: TrapBB);
1893 }
1894 // Start with a `s_trap 2`, if we're in PRIV=1 and we need the workaround this
1895 // will be a nop.
1896 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_TRAP))
1897 .addImm(Val: static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));
1898 Register DoorbellReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1899 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SENDMSG_RTN_B32),
1900 DestReg: DoorbellReg)
1901 .addImm(Val: AMDGPU::SendMsg::ID_RTN_GET_DOORBELL);
1902 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::TTMP2)
1903 .addUse(RegNo: AMDGPU::M0);
1904 Register DoorbellRegMasked =
1905 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1906 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_AND_B32), DestReg: DoorbellRegMasked)
1907 .addUse(RegNo: DoorbellReg)
1908 .addImm(Val: DoorbellIDMask);
1909 Register SetWaveAbortBit =
1910 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1911 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_OR_B32), DestReg: SetWaveAbortBit)
1912 .addUse(RegNo: DoorbellRegMasked)
1913 .addImm(Val: ECQueueWaveAbort);
1914 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
1915 .addUse(RegNo: SetWaveAbortBit);
1916 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SENDMSG))
1917 .addImm(Val: AMDGPU::SendMsg::ID_INTERRUPT);
1918 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
1919 .addUse(RegNo: AMDGPU::TTMP2);
1920 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH)).addMBB(MBB: HaltLoopBB);
1921 TrapBB->addSuccessor(Succ: HaltLoopBB);
1922
1923 BuildMI(BB&: *HaltLoopBB, I: HaltLoopBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETHALT)).addImm(Val: 5);
1924 BuildMI(BB&: *HaltLoopBB, I: HaltLoopBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
1925 .addMBB(MBB: HaltLoopBB);
1926 MF->push_back(MBB: HaltLoopBB);
1927 HaltLoopBB->addSuccessor(Succ: HaltLoopBB);
1928
1929 return MBB.getNextNode();
1930}
1931
1932unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) {
1933 switch (MI.getOpcode()) {
1934 default:
1935 if (MI.isMetaInstruction())
1936 return 0;
1937 return 1; // FIXME: Do wait states equal cycles?
1938
1939 case AMDGPU::S_NOP:
1940 return MI.getOperand(i: 0).getImm() + 1;
1941 // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The
1942 // hazard, even if one exist, won't really be visible. Should we handle it?
1943 }
1944}
1945
1946bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1947 MachineBasicBlock &MBB = *MI.getParent();
1948 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
1949 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
1950
1951 switch (MI.getOpcode()) {
1952 default: return TargetInstrInfo::expandPostRAPseudo(MI);
1953 case AMDGPU::S_MOV_B64_term:
1954 // This is only a terminator to get the correct spill code placement during
1955 // register allocation.
1956 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
1957 break;
1958
1959 case AMDGPU::S_MOV_B32_term:
1960 // This is only a terminator to get the correct spill code placement during
1961 // register allocation.
1962 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B32));
1963 break;
1964
1965 case AMDGPU::S_XOR_B64_term:
1966 // This is only a terminator to get the correct spill code placement during
1967 // register allocation.
1968 MI.setDesc(get(Opcode: AMDGPU::S_XOR_B64));
1969 break;
1970
1971 case AMDGPU::S_XOR_B32_term:
1972 // This is only a terminator to get the correct spill code placement during
1973 // register allocation.
1974 MI.setDesc(get(Opcode: AMDGPU::S_XOR_B32));
1975 break;
1976 case AMDGPU::S_OR_B64_term:
1977 // This is only a terminator to get the correct spill code placement during
1978 // register allocation.
1979 MI.setDesc(get(Opcode: AMDGPU::S_OR_B64));
1980 break;
1981 case AMDGPU::S_OR_B32_term:
1982 // This is only a terminator to get the correct spill code placement during
1983 // register allocation.
1984 MI.setDesc(get(Opcode: AMDGPU::S_OR_B32));
1985 break;
1986
1987 case AMDGPU::S_ANDN2_B64_term:
1988 // This is only a terminator to get the correct spill code placement during
1989 // register allocation.
1990 MI.setDesc(get(Opcode: AMDGPU::S_ANDN2_B64));
1991 break;
1992
1993 case AMDGPU::S_ANDN2_B32_term:
1994 // This is only a terminator to get the correct spill code placement during
1995 // register allocation.
1996 MI.setDesc(get(Opcode: AMDGPU::S_ANDN2_B32));
1997 break;
1998
1999 case AMDGPU::S_AND_B64_term:
2000 // This is only a terminator to get the correct spill code placement during
2001 // register allocation.
2002 MI.setDesc(get(Opcode: AMDGPU::S_AND_B64));
2003 break;
2004
2005 case AMDGPU::S_AND_B32_term:
2006 // This is only a terminator to get the correct spill code placement during
2007 // register allocation.
2008 MI.setDesc(get(Opcode: AMDGPU::S_AND_B32));
2009 break;
2010
2011 case AMDGPU::S_AND_SAVEEXEC_B64_term:
2012 // This is only a terminator to get the correct spill code placement during
2013 // register allocation.
2014 MI.setDesc(get(Opcode: AMDGPU::S_AND_SAVEEXEC_B64));
2015 break;
2016
2017 case AMDGPU::S_AND_SAVEEXEC_B32_term:
2018 // This is only a terminator to get the correct spill code placement during
2019 // register allocation.
2020 MI.setDesc(get(Opcode: AMDGPU::S_AND_SAVEEXEC_B32));
2021 break;
2022
2023 case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
2024 MI.setDesc(get(Opcode: AMDGPU::V_CMPX_EQ_U32_nosdst_e32));
2025 break;
2026 case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
2027 MI.setDesc(get(Opcode: AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
2028 break;
2029
2030 case AMDGPU::SI_SPILL_S32_TO_VGPR:
2031 MI.setDesc(get(Opcode: AMDGPU::V_WRITELANE_B32));
2032 break;
2033
2034 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
2035 MI.setDesc(get(Opcode: AMDGPU::V_READLANE_B32));
2036 break;
2037 case AMDGPU::AV_MOV_B32_IMM_PSEUDO: {
2038 Register Dst = MI.getOperand(i: 0).getReg();
2039 bool IsAGPR = SIRegisterInfo::isAGPRClass(RC: RI.getPhysRegBaseClass(Reg: Dst));
2040 MI.setDesc(
2041 get(Opcode: IsAGPR ? AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::V_MOV_B32_e32));
2042 break;
2043 }
2044 case AMDGPU::AV_MOV_B64_IMM_PSEUDO: {
2045 Register Dst = MI.getOperand(i: 0).getReg();
2046 if (SIRegisterInfo::isAGPRClass(RC: RI.getPhysRegBaseClass(Reg: Dst))) {
2047 int64_t Imm = MI.getOperand(i: 1).getImm();
2048
2049 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2050 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2051 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: DstLo)
2052 .addImm(Val: SignExtend64<32>(x: Imm));
2053 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: DstHi)
2054 .addImm(Val: SignExtend64<32>(x: Imm >> 32));
2055 MI.eraseFromParent();
2056 break;
2057 }
2058
2059 [[fallthrough]];
2060 }
2061 case AMDGPU::V_MOV_B64_PSEUDO: {
2062 Register Dst = MI.getOperand(i: 0).getReg();
2063 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2064 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2065
2066 const MCInstrDesc &Mov64Desc = get(Opcode: AMDGPU::V_MOV_B64_e32);
2067 const TargetRegisterClass *Mov64RC = getRegClass(MCID: Mov64Desc, /*OpNum=*/0);
2068
2069 const MachineOperand &SrcOp = MI.getOperand(i: 1);
2070 // FIXME: Will this work for 64-bit floating point immediates?
2071 assert(!SrcOp.isFPImm());
2072 if (ST.hasVMovB64Inst() && Mov64RC->contains(Reg: Dst)) {
2073 MI.setDesc(Mov64Desc);
2074 if (SrcOp.isReg() || isInlineConstant(MI, OpIdx: 1) ||
2075 (SrcOp.isImm() &&
2076 (isUInt<32>(x: SrcOp.getImm()) || ST.has64BitLiterals())) ||
2077 (SrcOp.isGlobal() && ST.has64BitLiterals()))
2078 break;
2079 }
2080 if (SrcOp.isGlobal()) {
2081 // The address is unknown until link time, so the PK_MOV inline-constant
2082 // shortcut cannot apply.
2083 const GlobalValue *GV = SrcOp.getGlobal();
2084 int64_t Offset = SrcOp.getOffset();
2085 unsigned BaseFlags, LoReloc, HiReloc;
2086 std::tie(args&: BaseFlags, args&: LoReloc, args&: HiReloc) =
2087 splitGlobalAddressRelocFlags(ST, SrcOp);
2088
2089 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2090 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | LoReloc);
2091 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2092 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | HiReloc);
2093 } else if (SrcOp.isImm()) {
2094 APInt Imm(64, SrcOp.getImm());
2095 APInt Lo(32, Imm.getLoBits(numBits: 32).getZExtValue());
2096 APInt Hi(32, Imm.getHiBits(numBits: 32).getZExtValue());
2097 const MCInstrDesc &PkMovDesc = get(Opcode: AMDGPU::V_PK_MOV_B32);
2098 const TargetRegisterClass *PkMovRC = getRegClass(MCID: PkMovDesc, /*OpNum=*/0);
2099
2100 if (ST.hasPkMovB32() && Lo == Hi && isInlineConstant(Imm: Lo) &&
2101 PkMovRC->contains(Reg: Dst)) {
2102 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: PkMovDesc, DestReg: Dst)
2103 .addImm(Val: SISrcMods::OP_SEL_1)
2104 .addImm(Val: Lo.getSExtValue())
2105 .addImm(Val: SISrcMods::OP_SEL_1)
2106 .addImm(Val: Lo.getSExtValue())
2107 .addImm(Val: 0) // op_sel_lo
2108 .addImm(Val: 0) // op_sel_hi
2109 .addImm(Val: 0) // neg_lo
2110 .addImm(Val: 0) // neg_hi
2111 .addImm(Val: 0); // clamp
2112 } else {
2113 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2114 .addImm(Val: Lo.getSExtValue());
2115 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2116 .addImm(Val: Hi.getSExtValue());
2117 }
2118 } else {
2119 assert(SrcOp.isReg());
2120 if (ST.hasPkMovB32() &&
2121 !RI.isAGPR(MRI: MBB.getParent()->getRegInfo(), Reg: SrcOp.getReg())) {
2122 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg: Dst)
2123 .addImm(Val: SISrcMods::OP_SEL_1) // src0_mod
2124 .addReg(RegNo: SrcOp.getReg())
2125 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) // src1_mod
2126 .addReg(RegNo: SrcOp.getReg())
2127 .addImm(Val: 0) // op_sel_lo
2128 .addImm(Val: 0) // op_sel_hi
2129 .addImm(Val: 0) // neg_lo
2130 .addImm(Val: 0) // neg_hi
2131 .addImm(Val: 0); // clamp
2132 } else {
2133 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2134 .addReg(RegNo: RI.getSubReg(Reg: SrcOp.getReg(), Idx: AMDGPU::sub0));
2135 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2136 .addReg(RegNo: RI.getSubReg(Reg: SrcOp.getReg(), Idx: AMDGPU::sub1));
2137 }
2138 }
2139 MI.eraseFromParent();
2140 break;
2141 }
2142 case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
2143 expandMovDPP64(MI);
2144 break;
2145 }
2146 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2147 const MachineOperand &SrcOp = MI.getOperand(i: 1);
2148 assert(!SrcOp.isFPImm());
2149
2150 if (ST.has64BitLiterals()) {
2151 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2152 break;
2153 }
2154
2155 if (SrcOp.isGlobal()) {
2156 Register Dst = MI.getOperand(i: 0).getReg();
2157 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2158 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2159 const GlobalValue *GV = SrcOp.getGlobal();
2160 int64_t Offset = SrcOp.getOffset();
2161 unsigned BaseFlags, LoReloc, HiReloc;
2162 std::tie(args&: BaseFlags, args&: LoReloc, args&: HiReloc) =
2163 splitGlobalAddressRelocFlags(ST, SrcOp);
2164
2165 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstLo)
2166 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | LoReloc);
2167 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi)
2168 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | HiReloc);
2169 MI.eraseFromParent();
2170 break;
2171 }
2172
2173 // SrcOp is immediate
2174 APInt Imm(64, SrcOp.getImm());
2175 if (Imm.isIntN(N: 32) || isInlineConstant(Imm)) {
2176 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2177 break;
2178 }
2179
2180 Register Dst = MI.getOperand(i: 0).getReg();
2181 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2182 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2183
2184 APInt Lo(32, Imm.getLoBits(numBits: 32).getZExtValue());
2185 APInt Hi(32, Imm.getHiBits(numBits: 32).getZExtValue());
2186 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstLo)
2187 .addImm(Val: Lo.getSExtValue());
2188 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi)
2189 .addImm(Val: Hi.getSExtValue());
2190 MI.eraseFromParent();
2191 break;
2192 }
2193 case AMDGPU::V_SET_INACTIVE_B32: {
2194 // Lower V_SET_INACTIVE_B32 to V_CNDMASK_B32.
2195 Register DstReg = MI.getOperand(i: 0).getReg();
2196 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
2197 .add(MO: MI.getOperand(i: 3))
2198 .add(MO: MI.getOperand(i: 4))
2199 .add(MO: MI.getOperand(i: 1))
2200 .add(MO: MI.getOperand(i: 2))
2201 .add(MO: MI.getOperand(i: 5));
2202 MI.eraseFromParent();
2203 break;
2204 }
2205 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2206 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2207 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2208 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2209 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2210 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V6:
2211 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V7:
2212 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2213 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2214 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2215 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2216 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2217 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2218 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2219 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2220 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2221 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2222 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2223 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2224 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V6:
2225 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V7:
2226 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2227 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2228 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2229 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2230 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2231 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2232 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2233 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:
2234 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:
2235 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:
2236 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:
2237 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {
2238 const TargetRegisterClass *EltRC = getOpRegClass(MI, OpNo: 2);
2239
2240 unsigned Opc;
2241 if (RI.hasVGPRs(RC: EltRC)) {
2242 Opc = AMDGPU::V_MOVRELD_B32_e32;
2243 } else {
2244 Opc = RI.getRegSizeInBits(RC: *EltRC) == 64 ? AMDGPU::S_MOVRELD_B64
2245 : AMDGPU::S_MOVRELD_B32;
2246 }
2247
2248 const MCInstrDesc &OpDesc = get(Opcode: Opc);
2249 Register VecReg = MI.getOperand(i: 0).getReg();
2250 bool IsUndef = MI.getOperand(i: 1).isUndef();
2251 unsigned SubReg = MI.getOperand(i: 3).getImm();
2252 assert(VecReg == MI.getOperand(1).getReg());
2253
2254 MachineInstrBuilder MIB =
2255 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: OpDesc)
2256 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2257 .add(MO: MI.getOperand(i: 2))
2258 .addReg(RegNo: VecReg, Flags: RegState::ImplicitDefine)
2259 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2260
2261 const int ImpDefIdx =
2262 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2263 const int ImpUseIdx = ImpDefIdx + 1;
2264 MIB->tieOperands(DefIdx: ImpDefIdx, UseIdx: ImpUseIdx);
2265 MI.eraseFromParent();
2266 break;
2267 }
2268 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:
2269 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:
2270 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:
2271 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:
2272 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:
2273 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V6:
2274 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V7:
2275 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:
2276 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9:
2277 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10:
2278 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11:
2279 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12:
2280 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:
2281 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {
2282 assert(ST.useVGPRIndexMode());
2283 Register VecReg = MI.getOperand(i: 0).getReg();
2284 bool IsUndef = MI.getOperand(i: 1).isUndef();
2285 MachineOperand &Idx = MI.getOperand(i: 3);
2286 Register SubReg = MI.getOperand(i: 4).getImm();
2287
2288 MachineInstr *SetOn = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_ON))
2289 .add(MO: Idx)
2290 .addImm(Val: AMDGPU::VGPRIndexMode::DST_ENABLE);
2291 SetOn->getOperand(i: 3).setIsUndef();
2292
2293 const MCInstrDesc &OpDesc = get(Opcode: AMDGPU::V_MOV_B32_indirect_write);
2294 MachineInstrBuilder MIB =
2295 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: OpDesc)
2296 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2297 .add(MO: MI.getOperand(i: 2))
2298 .addReg(RegNo: VecReg, Flags: RegState::ImplicitDefine)
2299 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2300
2301 const int ImpDefIdx =
2302 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2303 const int ImpUseIdx = ImpDefIdx + 1;
2304 MIB->tieOperands(DefIdx: ImpDefIdx, UseIdx: ImpUseIdx);
2305
2306 MachineInstr *SetOff = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_OFF));
2307
2308 finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2309
2310 MI.eraseFromParent();
2311 break;
2312 }
2313 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:
2314 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:
2315 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:
2316 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:
2317 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:
2318 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V6:
2319 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V7:
2320 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:
2321 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9:
2322 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10:
2323 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11:
2324 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12:
2325 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:
2326 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {
2327 assert(ST.useVGPRIndexMode());
2328 Register Dst = MI.getOperand(i: 0).getReg();
2329 Register VecReg = MI.getOperand(i: 1).getReg();
2330 bool IsUndef = MI.getOperand(i: 1).isUndef();
2331 Register SubReg = MI.getOperand(i: 3).getImm();
2332
2333 MachineInstr *SetOn = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_ON))
2334 .add(MO: MI.getOperand(i: 2))
2335 .addImm(Val: AMDGPU::VGPRIndexMode::SRC0_ENABLE);
2336 SetOn->getOperand(i: 3).setIsUndef();
2337
2338 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_indirect_read))
2339 .addDef(RegNo: Dst)
2340 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2341 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2342
2343 MachineInstr *SetOff = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_OFF));
2344
2345 finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2346
2347 MI.eraseFromParent();
2348 break;
2349 }
2350 case AMDGPU::SI_PC_ADD_REL_OFFSET: {
2351 MachineFunction &MF = *MBB.getParent();
2352 Register Reg = MI.getOperand(i: 0).getReg();
2353 Register RegLo = RI.getSubReg(Reg, Idx: AMDGPU::sub0);
2354 Register RegHi = RI.getSubReg(Reg, Idx: AMDGPU::sub1);
2355 MachineOperand OpLo = MI.getOperand(i: 1);
2356 MachineOperand OpHi = MI.getOperand(i: 2);
2357
2358 // Create a bundle so these instructions won't be re-ordered by the
2359 // post-RA scheduler.
2360 MIBundleBuilder Bundler(MBB, MI);
2361 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: Reg));
2362
2363 // What we want here is an offset from the value returned by s_getpc (which
2364 // is the address of the s_add_u32 instruction) to the global variable, but
2365 // since the encoding of $symbol starts 4 bytes after the start of the
2366 // s_add_u32 instruction, we end up with an offset that is 4 bytes too
2367 // small. This requires us to add 4 to the global variable offset in order
2368 // to compute the correct address. Similarly for the s_addc_u32 instruction,
2369 // the encoding of $symbol starts 12 bytes after the start of the s_add_u32
2370 // instruction.
2371
2372 int64_t Adjust = 0;
2373 if (ST.hasGetPCZeroExtension()) {
2374 // Fix up hardware that does not sign-extend the 48-bit PC value by
2375 // inserting: s_sext_i32_i16 reghi, reghi
2376 Bundler.append(
2377 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SEXT_I32_I16), DestReg: RegHi).addReg(RegNo: RegHi));
2378 Adjust += 4;
2379 }
2380
2381 if (OpLo.isGlobal())
2382 OpLo.setOffset(OpLo.getOffset() + Adjust + 4);
2383 Bundler.append(
2384 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U32), DestReg: RegLo).addReg(RegNo: RegLo).add(MO: OpLo));
2385
2386 if (OpHi.isGlobal())
2387 OpHi.setOffset(OpHi.getOffset() + Adjust + 12);
2388 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADDC_U32), DestReg: RegHi)
2389 .addReg(RegNo: RegHi)
2390 .add(MO: OpHi));
2391
2392 finalizeBundle(MBB, FirstMI: Bundler.begin());
2393
2394 MI.eraseFromParent();
2395 break;
2396 }
2397 case AMDGPU::SI_PC_ADD_REL_OFFSET64: {
2398 MachineFunction &MF = *MBB.getParent();
2399 Register Reg = MI.getOperand(i: 0).getReg();
2400 MachineOperand Op = MI.getOperand(i: 1);
2401
2402 // Create a bundle so these instructions won't be re-ordered by the
2403 // post-RA scheduler.
2404 MIBundleBuilder Bundler(MBB, MI);
2405 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: Reg));
2406 if (Op.isGlobal())
2407 Op.setOffset(Op.getOffset() + 4);
2408 Bundler.append(
2409 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U64), DestReg: Reg).addReg(RegNo: Reg).add(MO: Op));
2410
2411 finalizeBundle(MBB, FirstMI: Bundler.begin());
2412
2413 MI.eraseFromParent();
2414 break;
2415 }
2416 case AMDGPU::ENTER_STRICT_WWM: {
2417 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2418 // Whole Wave Mode is entered.
2419 MI.setDesc(get(Opcode: LMC.OrSaveExecOpc));
2420 break;
2421 }
2422 case AMDGPU::ENTER_STRICT_WQM: {
2423 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2424 // STRICT_WQM is entered.
2425 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: LMC.MovOpc), DestReg: MI.getOperand(i: 0).getReg())
2426 .addReg(RegNo: LMC.ExecReg);
2427 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: LMC.WQMOpc), DestReg: LMC.ExecReg).addReg(RegNo: LMC.ExecReg);
2428
2429 MI.eraseFromParent();
2430 break;
2431 }
2432 case AMDGPU::EXIT_STRICT_WWM:
2433 case AMDGPU::EXIT_STRICT_WQM: {
2434 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2435 // WWM/STICT_WQM is exited.
2436 MI.setDesc(get(Opcode: LMC.MovOpc));
2437 break;
2438 }
2439 case AMDGPU::SI_RETURN: {
2440 const MachineFunction *MF = MBB.getParent();
2441 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2442 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2443 // Hiding the return address use with SI_RETURN may lead to extra kills in
2444 // the function and missing live-ins. We are fine in practice because callee
2445 // saved register handling ensures the register value is restored before
2446 // RET, but we need the undef flag here to appease the MachineVerifier
2447 // liveness checks.
2448 MachineInstrBuilder MIB =
2449 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETPC_B64_return))
2450 .addReg(RegNo: TRI->getReturnAddressReg(MF: *MF), Flags: RegState::Undef);
2451
2452 MIB.copyImplicitOps(OtherMI: MI);
2453 MI.eraseFromParent();
2454 break;
2455 }
2456
2457 case AMDGPU::S_MUL_U64_U32_PSEUDO:
2458 case AMDGPU::S_MUL_I64_I32_PSEUDO:
2459 MI.setDesc(get(Opcode: AMDGPU::S_MUL_U64));
2460 break;
2461
2462 case AMDGPU::S_GETPC_B64_pseudo:
2463 MI.setDesc(get(Opcode: AMDGPU::S_GETPC_B64));
2464 if (ST.hasGetPCZeroExtension()) {
2465 Register Dst = MI.getOperand(i: 0).getReg();
2466 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2467 // Fix up hardware that does not sign-extend the 48-bit PC value by
2468 // inserting: s_sext_i32_i16 dsthi, dsthi
2469 BuildMI(BB&: MBB, I: std::next(x: MI.getIterator()), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SEXT_I32_I16),
2470 DestReg: DstHi)
2471 .addReg(RegNo: DstHi);
2472 }
2473 break;
2474
2475 case AMDGPU::V_MAX_BF16_PSEUDO_e64: {
2476 assert(ST.hasBF16PackedInsts());
2477 MI.setDesc(get(Opcode: AMDGPU::V_PK_MAX_NUM_BF16));
2478 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // op_sel
2479 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // neg_lo
2480 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // neg_hi
2481 auto Op0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
2482 Op0->setImm(Op0->getImm() | SISrcMods::OP_SEL_1);
2483 auto Op1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
2484 Op1->setImm(Op1->getImm() | SISrcMods::OP_SEL_1);
2485 break;
2486 }
2487
2488 case AMDGPU::GET_STACK_BASE:
2489 // The stack starts at offset 0 unless we need to reserve some space at the
2490 // bottom.
2491 if (ST.getFrameLowering()->mayReserveScratchForCWSR(MF: *MBB.getParent())) {
2492 // When CWSR is used in dynamic VGPR mode, the trap handler needs to save
2493 // some of the VGPRs. The size of the required scratch space has already
2494 // been computed by prolog epilog insertion.
2495 const SIMachineFunctionInfo *MFI =
2496 MBB.getParent()->getInfo<SIMachineFunctionInfo>();
2497 unsigned VGPRSize = MFI->getScratchReservedForDynamicVGPRs();
2498 Register DestReg = MI.getOperand(i: 0).getReg();
2499 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETREG_B32), DestReg)
2500 .addImm(Val: AMDGPU::Hwreg::HwregEncoding::encode(
2501 Values: AMDGPU::Hwreg::ID_HW_ID2, Values: AMDGPU::Hwreg::OFFSET_ME_ID, Values: 2));
2502 // The MicroEngine ID is 0 for the graphics queue, and 1 or 2 for compute
2503 // (3 is unused, so we ignore it). Unfortunately, S_GETREG doesn't set
2504 // SCC, so we need to check for 0 manually.
2505 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U32)).addImm(Val: 0).addReg(RegNo: DestReg);
2506 // Change the implicif-def of SCC to an explicit use (but first remove
2507 // the dead flag if present).
2508 MI.getOperand(i: MI.getNumExplicitOperands()).setIsDead(false);
2509 MI.getOperand(i: MI.getNumExplicitOperands()).setIsUse();
2510 MI.setDesc(get(Opcode: AMDGPU::S_CMOVK_I32));
2511 MI.addOperand(Op: MachineOperand::CreateImm(Val: VGPRSize));
2512 } else {
2513 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B32));
2514 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2515 MI.removeOperand(
2516 OpNo: MI.getNumExplicitOperands()); // Drop implicit def of SCC.
2517 }
2518 break;
2519 }
2520
2521 return true;
2522}
2523
2524void SIInstrInfo::reMaterialize(MachineBasicBlock &MBB,
2525 MachineBasicBlock::iterator I, Register DestReg,
2526 unsigned SubIdx, const MachineInstr &Orig,
2527 LaneBitmask UsedLanes) const {
2528
2529 // Try shrinking the instruction to remat only the part needed for current
2530 // context.
2531 // TODO: Handle more cases.
2532 unsigned Opcode = Orig.getOpcode();
2533 switch (Opcode) {
2534 case AMDGPU::S_MOV_B64:
2535 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2536 if (SubIdx != 0)
2537 break;
2538
2539 if (!Orig.getOperand(i: 1).isImm())
2540 break;
2541
2542 // Shrink S_MOV_B64 to S_MOV_B32 when UsedLanes indicates only a single
2543 // 32-bit lane of the 64-bit value is live at the rematerialization point.
2544 if (UsedLanes.all())
2545 break;
2546
2547 // Determine which half of the 64-bit immediate corresponds to the use.
2548 unsigned OrigSubReg = Orig.getOperand(i: 0).getSubReg();
2549 unsigned LoSubReg = RI.composeSubRegIndices(a: OrigSubReg, b: AMDGPU::sub0);
2550 unsigned HiSubReg = RI.composeSubRegIndices(a: OrigSubReg, b: AMDGPU::sub1);
2551
2552 bool NeedLo = (UsedLanes & RI.getSubRegIndexLaneMask(SubIdx: LoSubReg)).any();
2553 bool NeedHi = (UsedLanes & RI.getSubRegIndexLaneMask(SubIdx: HiSubReg)).any();
2554
2555 if (NeedLo && NeedHi)
2556 break;
2557
2558 int64_t Imm64 = Orig.getOperand(i: 1).getImm();
2559 int32_t Imm32 = NeedLo ? Lo_32(Value: Imm64) : Hi_32(Value: Imm64);
2560
2561 unsigned UseSubReg = NeedLo ? LoSubReg : HiSubReg;
2562
2563 // Emit S_MOV_B32 defining just the needed 32-bit subreg of DestReg.
2564 BuildMI(BB&: MBB, I, MIMD: Orig.getDebugLoc(), MCID: get(Opcode: AMDGPU::S_MOV_B32))
2565 .addReg(RegNo: DestReg, Flags: RegState::Define | RegState::Undef, SubReg: UseSubReg)
2566 .addImm(Val: Imm32);
2567 return;
2568 }
2569
2570 case AMDGPU::S_LOAD_DWORDX16_IMM:
2571 case AMDGPU::S_LOAD_DWORDX8_IMM: {
2572 if (SubIdx != 0)
2573 break;
2574
2575 if (I == MBB.end())
2576 break;
2577
2578 if (I->isBundled())
2579 break;
2580
2581 // Look for a single use of the register that is also a subreg.
2582 Register RegToFind = Orig.getOperand(i: 0).getReg();
2583 MachineOperand *UseMO = nullptr;
2584 for (auto &CandMO : I->operands()) {
2585 if (!CandMO.isReg() || CandMO.getReg() != RegToFind || CandMO.isDef())
2586 continue;
2587 if (UseMO) {
2588 UseMO = nullptr;
2589 break;
2590 }
2591 UseMO = &CandMO;
2592 }
2593 if (!UseMO || UseMO->getSubReg() == AMDGPU::NoSubRegister)
2594 break;
2595
2596 unsigned Offset = RI.getSubRegIdxOffset(Idx: UseMO->getSubReg());
2597 unsigned SubregSize = RI.getSubRegIdxSize(Idx: UseMO->getSubReg());
2598
2599 MachineFunction *MF = MBB.getParent();
2600 MachineRegisterInfo &MRI = MF->getRegInfo();
2601 assert(MRI.use_nodbg_empty(DestReg) && "DestReg should have no users yet.");
2602
2603 unsigned NewOpcode = -1;
2604 if (SubregSize == 256)
2605 NewOpcode = AMDGPU::S_LOAD_DWORDX8_IMM;
2606 else if (SubregSize == 128)
2607 NewOpcode = AMDGPU::S_LOAD_DWORDX4_IMM;
2608 else
2609 break;
2610
2611 const MCInstrDesc &TID = get(Opcode: NewOpcode);
2612 const TargetRegisterClass *NewRC =
2613 RI.getAllocatableClass(RC: getRegClass(MCID: TID, OpNum: 0));
2614 MRI.setRegClass(Reg: DestReg, RC: NewRC);
2615
2616 UseMO->setReg(DestReg);
2617 UseMO->setSubReg(AMDGPU::NoSubRegister);
2618
2619 // Use a smaller load with the desired size, possibly with updated offset.
2620 MachineInstr *MI = MF->CloneMachineInstr(Orig: &Orig);
2621 MI->setDesc(TID);
2622 MI->getOperand(i: 0).setReg(DestReg);
2623 MI->getOperand(i: 0).setSubReg(AMDGPU::NoSubRegister);
2624 if (Offset) {
2625 MachineOperand *OffsetMO = getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::offset);
2626 int64_t FinalOffset = OffsetMO->getImm() + Offset / 8;
2627 OffsetMO->setImm(FinalOffset);
2628 }
2629 SmallVector<MachineMemOperand *> NewMMOs;
2630 for (const MachineMemOperand *MemOp : Orig.memoperands())
2631 NewMMOs.push_back(Elt: MF->getMachineMemOperand(MMO: MemOp, PtrInfo: MemOp->getPointerInfo(),
2632 Size: SubregSize / 8));
2633 MI->setMemRefs(MF&: *MF, MemRefs: NewMMOs);
2634
2635 MBB.insert(I, MI);
2636 return;
2637 }
2638
2639 default:
2640 break;
2641 }
2642
2643 TargetInstrInfo::reMaterialize(MBB, MI: I, DestReg, SubIdx, Orig, UsedLanes);
2644}
2645
2646std::pair<MachineInstr*, MachineInstr*>
2647SIInstrInfo::expandMovDPP64(MachineInstr &MI) const {
2648 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
2649
2650 if (ST.hasVMovB64Inst() && ST.hasFeature(Feature: AMDGPU::FeatureDPALU_DPP) &&
2651 AMDGPU::isLegalDPALU_DPPControl(
2652 ST, DC: getNamedOperand(MI, OperandName: AMDGPU::OpName::dpp_ctrl)->getImm())) {
2653 MI.setDesc(get(Opcode: AMDGPU::V_MOV_B64_dpp));
2654 return std::pair(&MI, nullptr);
2655 }
2656
2657 MachineBasicBlock &MBB = *MI.getParent();
2658 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2659 MachineFunction *MF = MBB.getParent();
2660 MachineRegisterInfo &MRI = MF->getRegInfo();
2661 Register Dst = MI.getOperand(i: 0).getReg();
2662 unsigned Part = 0;
2663 MachineInstr *Split[2];
2664
2665 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
2666 auto MovDPP = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_dpp));
2667 if (Dst.isPhysical()) {
2668 MovDPP.addDef(RegNo: RI.getSubReg(Reg: Dst, Idx: Sub));
2669 } else {
2670 assert(MRI.isSSA());
2671 auto Tmp = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2672 MovDPP.addDef(RegNo: Tmp);
2673 }
2674
2675 for (unsigned I = 1; I <= 2; ++I) { // old and src operands.
2676 const MachineOperand &SrcOp = MI.getOperand(i: I);
2677 assert(!SrcOp.isFPImm());
2678 if (SrcOp.isImm()) {
2679 APInt Imm(64, SrcOp.getImm());
2680 Imm.ashrInPlace(ShiftAmt: Part * 32);
2681 MovDPP.addImm(Val: Imm.getLoBits(numBits: 32).getZExtValue());
2682 } else {
2683 assert(SrcOp.isReg());
2684 Register Src = SrcOp.getReg();
2685 if (Src.isPhysical())
2686 MovDPP.addReg(RegNo: RI.getSubReg(Reg: Src, Idx: Sub));
2687 else
2688 MovDPP.addReg(RegNo: Src, Flags: getUndefRegState(B: SrcOp.isUndef()), SubReg: Sub);
2689 }
2690 }
2691
2692 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.explicit_operands(), N: 3))
2693 MovDPP.addImm(Val: MO.getImm());
2694
2695 Split[Part] = MovDPP;
2696 ++Part;
2697 }
2698
2699 if (Dst.isVirtual())
2700 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
2701 .addReg(RegNo: Split[0]->getOperand(i: 0).getReg())
2702 .addImm(Val: AMDGPU::sub0)
2703 .addReg(RegNo: Split[1]->getOperand(i: 0).getReg())
2704 .addImm(Val: AMDGPU::sub1);
2705
2706 MI.eraseFromParent();
2707 return std::pair(Split[0], Split[1]);
2708}
2709
2710std::optional<DestSourcePair>
2711SIInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
2712 if (MI.getOpcode() == AMDGPU::WWM_COPY)
2713 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2714
2715 return std::nullopt;
2716}
2717
2718bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,
2719 AMDGPU::OpName Src0OpName,
2720 MachineOperand &Src1,
2721 AMDGPU::OpName Src1OpName) const {
2722 MachineOperand *Src0Mods = getNamedOperand(MI, OperandName: Src0OpName);
2723 if (!Src0Mods)
2724 return false;
2725
2726 MachineOperand *Src1Mods = getNamedOperand(MI, OperandName: Src1OpName);
2727 assert(Src1Mods &&
2728 "All commutable instructions have both src0 and src1 modifiers");
2729
2730 int Src0ModsVal = Src0Mods->getImm();
2731 int Src1ModsVal = Src1Mods->getImm();
2732
2733 Src1Mods->setImm(Src0ModsVal);
2734 Src0Mods->setImm(Src1ModsVal);
2735 return true;
2736}
2737
2738static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI,
2739 MachineOperand &RegOp,
2740 MachineOperand &NonRegOp) {
2741 Register Reg = RegOp.getReg();
2742 unsigned SubReg = RegOp.getSubReg();
2743 bool IsKill = RegOp.isKill();
2744 bool IsDead = RegOp.isDead();
2745 bool IsUndef = RegOp.isUndef();
2746 bool IsDebug = RegOp.isDebug();
2747
2748 if (NonRegOp.isImm())
2749 RegOp.ChangeToImmediate(ImmVal: NonRegOp.getImm());
2750 else if (NonRegOp.isFI())
2751 RegOp.ChangeToFrameIndex(Idx: NonRegOp.getIndex());
2752 else if (NonRegOp.isGlobal()) {
2753 RegOp.ChangeToGA(GV: NonRegOp.getGlobal(), Offset: NonRegOp.getOffset(),
2754 TargetFlags: NonRegOp.getTargetFlags());
2755 } else
2756 return nullptr;
2757
2758 // Make sure we don't reinterpret a subreg index in the target flags.
2759 RegOp.setTargetFlags(NonRegOp.getTargetFlags());
2760
2761 NonRegOp.ChangeToRegister(Reg, isDef: false, isImp: false, isKill: IsKill, isDead: IsDead, isUndef: IsUndef, isDebug: IsDebug);
2762 NonRegOp.setSubReg(SubReg);
2763
2764 return &MI;
2765}
2766
2767static MachineInstr *swapImmOperands(MachineInstr &MI,
2768 MachineOperand &NonRegOp1,
2769 MachineOperand &NonRegOp2) {
2770 unsigned TargetFlags = NonRegOp1.getTargetFlags();
2771 int64_t NonRegVal = NonRegOp1.getImm();
2772
2773 NonRegOp1.setImm(NonRegOp2.getImm());
2774 NonRegOp2.setImm(NonRegVal);
2775 NonRegOp1.setTargetFlags(NonRegOp2.getTargetFlags());
2776 NonRegOp2.setTargetFlags(TargetFlags);
2777 return &MI;
2778}
2779
2780bool SIInstrInfo::isLegalToSwap(const MachineInstr &MI, unsigned OpIdx0,
2781 unsigned OpIdx1) const {
2782 const MCInstrDesc &InstDesc = MI.getDesc();
2783 const MCOperandInfo &OpInfo0 = InstDesc.operands()[OpIdx0];
2784 const MCOperandInfo &OpInfo1 = InstDesc.operands()[OpIdx1];
2785
2786 unsigned Opc = MI.getOpcode();
2787 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
2788
2789 const MachineOperand &MO0 = MI.getOperand(i: OpIdx0);
2790 const MachineOperand &MO1 = MI.getOperand(i: OpIdx1);
2791
2792 // Swap doesn't breach constant bus or literal limits
2793 // It may move literal to position other than src0, this is not allowed
2794 // pre-gfx10 However, most test cases need literals in Src0 for VOP
2795 // FIXME: After gfx9, literal can be in place other than Src0
2796 if (isVALU(MI, /*AllowLDSDMA=*/true)) {
2797 if ((int)OpIdx0 == Src0Idx && !MO0.isReg() &&
2798 !isInlineConstant(MO: MO0, OpInfo: OpInfo1))
2799 return false;
2800 if ((int)OpIdx1 == Src0Idx && !MO1.isReg() &&
2801 !isInlineConstant(MO: MO1, OpInfo: OpInfo0))
2802 return false;
2803 }
2804
2805 if ((int)OpIdx1 != Src0Idx && MO0.isReg()) {
2806 if (OpInfo1.RegClass == -1)
2807 return OpInfo1.OperandType == MCOI::OPERAND_UNKNOWN;
2808 return isLegalRegOperand(MI, OpIdx: OpIdx1, MO: MO0) &&
2809 (!MO1.isReg() || isLegalRegOperand(MI, OpIdx: OpIdx0, MO: MO1));
2810 }
2811 if ((int)OpIdx0 != Src0Idx && MO1.isReg()) {
2812 if (OpInfo0.RegClass == -1)
2813 return OpInfo0.OperandType == MCOI::OPERAND_UNKNOWN;
2814 return (!MO0.isReg() || isLegalRegOperand(MI, OpIdx: OpIdx1, MO: MO0)) &&
2815 isLegalRegOperand(MI, OpIdx: OpIdx0, MO: MO1);
2816 }
2817
2818 // No need to check 64-bit literals since swapping does not bring new
2819 // 64-bit literals into current instruction to fold to 32-bit
2820
2821 return isImmOperandLegal(MI, OpNo: OpIdx1, MO: MO0);
2822}
2823
2824MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
2825 unsigned Src0Idx,
2826 unsigned Src1Idx) const {
2827 assert(!NewMI && "this should never be used");
2828
2829 unsigned Opc = MI.getOpcode();
2830 int CommutedOpcode = commuteOpcode(Opcode: Opc);
2831 if (CommutedOpcode == -1)
2832 return nullptr;
2833
2834 if (Src0Idx > Src1Idx)
2835 std::swap(a&: Src0Idx, b&: Src1Idx);
2836
2837 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
2838 static_cast<int>(Src0Idx) &&
2839 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
2840 static_cast<int>(Src1Idx) &&
2841 "inconsistency with findCommutedOpIndices");
2842
2843 if (!isLegalToSwap(MI, OpIdx0: Src0Idx, OpIdx1: Src1Idx))
2844 return nullptr;
2845
2846 MachineInstr *CommutedMI = nullptr;
2847 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
2848 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
2849 if (Src0.isReg() && Src1.isReg()) {
2850 // Be sure to copy the source modifiers to the right place.
2851 CommutedMI =
2852 TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1: Src0Idx, OpIdx2: Src1Idx);
2853 } else if (Src0.isReg() && !Src1.isReg()) {
2854 CommutedMI = swapRegAndNonRegOperand(MI, RegOp&: Src0, NonRegOp&: Src1);
2855 } else if (!Src0.isReg() && Src1.isReg()) {
2856 CommutedMI = swapRegAndNonRegOperand(MI, RegOp&: Src1, NonRegOp&: Src0);
2857 } else if (Src0.isImm() && Src1.isImm()) {
2858 CommutedMI = swapImmOperands(MI, NonRegOp1&: Src0, NonRegOp2&: Src1);
2859 } else {
2860 // FIXME: Found two non registers to commute. This does happen.
2861 return nullptr;
2862 }
2863
2864 if (CommutedMI) {
2865 swapSourceModifiers(MI, Src0, Src0OpName: AMDGPU::OpName::src0_modifiers,
2866 Src1, Src1OpName: AMDGPU::OpName::src1_modifiers);
2867
2868 swapSourceModifiers(MI, Src0, Src0OpName: AMDGPU::OpName::src0_sel, Src1,
2869 Src1OpName: AMDGPU::OpName::src1_sel);
2870
2871 CommutedMI->setDesc(get(Opcode: CommutedOpcode));
2872 }
2873
2874 return CommutedMI;
2875}
2876
2877// This needs to be implemented because the source modifiers may be inserted
2878// between the true commutable operands, and the base
2879// TargetInstrInfo::commuteInstruction uses it.
2880bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
2881 unsigned &SrcOpIdx0,
2882 unsigned &SrcOpIdx1) const {
2883 return findCommutedOpIndices(Desc: MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
2884}
2885
2886bool SIInstrInfo::findCommutedOpIndices(const MCInstrDesc &Desc,
2887 unsigned &SrcOpIdx0,
2888 unsigned &SrcOpIdx1) const {
2889 if (!Desc.isCommutable())
2890 return false;
2891
2892 unsigned Opc = Desc.getOpcode();
2893 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
2894 if (Src0Idx == -1)
2895 return false;
2896
2897 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
2898 if (Src1Idx == -1)
2899 return false;
2900
2901 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx0, ResultIdx2&: SrcOpIdx1, CommutableOpIdx1: Src0Idx, CommutableOpIdx2: Src1Idx);
2902}
2903
2904bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
2905 int64_t BrOffset) const {
2906 // BranchRelaxation should never have to check s_setpc_b64 or s_add_pc_i64
2907 // because its dest block is unanalyzable.
2908 assert(isSOPP(BranchOp) || isSOPK(BranchOp));
2909
2910 // Convert to dwords.
2911 BrOffset /= 4;
2912
2913 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
2914 // from the next instruction.
2915 BrOffset -= 1;
2916
2917 return isIntN(N: BranchOffsetBits, x: BrOffset);
2918}
2919
2920MachineBasicBlock *
2921SIInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
2922 return MI.getOperand(i: 0).getMBB();
2923}
2924
2925bool SIInstrInfo::hasDivergentBranch(const MachineBasicBlock *MBB) const {
2926 for (const MachineInstr &MI : MBB->terminators()) {
2927 if (MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE ||
2928 MI.getOpcode() == AMDGPU::SI_LOOP)
2929 return true;
2930 }
2931 return false;
2932}
2933
2934void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
2935 MachineBasicBlock &DestBB,
2936 MachineBasicBlock &RestoreBB,
2937 const DebugLoc &DL, int64_t BrOffset,
2938 RegScavenger *RS) const {
2939 assert(MBB.empty() &&
2940 "new block should be inserted for expanding unconditional branch");
2941 assert(MBB.pred_size() == 1);
2942 assert(RestoreBB.empty() &&
2943 "restore block should be inserted for restoring clobbered registers");
2944
2945 MachineFunction *MF = MBB.getParent();
2946 MachineRegisterInfo &MRI = MF->getRegInfo();
2947 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
2948 auto I = MBB.end();
2949 auto &MCCtx = MF->getContext();
2950
2951 if (ST.useAddPC64Inst()) {
2952 MCSymbol *Offset =
2953 MCCtx.createTempSymbol(Name: "offset", /*AlwaysAddSuffix=*/true);
2954 auto AddPC = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_PC_I64))
2955 .addSym(Sym: Offset, TargetFlags: MO_FAR_BRANCH_OFFSET);
2956 MCSymbol *PostAddPCLabel =
2957 MCCtx.createTempSymbol(Name: "post_addpc", /*AlwaysAddSuffix=*/true);
2958 AddPC->setPostInstrSymbol(MF&: *MF, Symbol: PostAddPCLabel);
2959 auto *OffsetExpr = MCBinaryExpr::createSub(
2960 LHS: MCSymbolRefExpr::create(Symbol: DestBB.getSymbol(), Ctx&: MCCtx),
2961 RHS: MCSymbolRefExpr::create(Symbol: PostAddPCLabel, Ctx&: MCCtx), Ctx&: MCCtx);
2962 Offset->setVariableValue(OffsetExpr);
2963 return;
2964 }
2965
2966 assert(RS && "RegScavenger required for long branching");
2967
2968 // FIXME: Virtual register workaround for RegScavenger not working with empty
2969 // blocks.
2970 Register PCReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
2971
2972 // Note: as this is used after hazard recognizer we need to apply some hazard
2973 // workarounds directly.
2974 const bool FlushSGPRWrites = (ST.isWave64() && ST.hasVALUMaskWriteHazard()) ||
2975 ST.hasVALUReadSGPRHazard();
2976 auto ApplyHazardWorkarounds = [this, &MBB, &I, &DL, FlushSGPRWrites]() {
2977 if (FlushSGPRWrites)
2978 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
2979 .addImm(Val: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST));
2980 };
2981
2982 // We need to compute the offset relative to the instruction immediately after
2983 // s_getpc_b64. Insert pc arithmetic code before last terminator.
2984 MachineInstr *GetPC = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: PCReg);
2985 ApplyHazardWorkarounds();
2986
2987 MCSymbol *PostGetPCLabel =
2988 MCCtx.createTempSymbol(Name: "post_getpc", /*AlwaysAddSuffix=*/true);
2989 GetPC->setPostInstrSymbol(MF&: *MF, Symbol: PostGetPCLabel);
2990
2991 MCSymbol *OffsetLo =
2992 MCCtx.createTempSymbol(Name: "offset_lo", /*AlwaysAddSuffix=*/true);
2993 MCSymbol *OffsetHi =
2994 MCCtx.createTempSymbol(Name: "offset_hi", /*AlwaysAddSuffix=*/true);
2995 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U32))
2996 .addReg(RegNo: PCReg, Flags: RegState::Define, SubReg: AMDGPU::sub0)
2997 .addReg(RegNo: PCReg, Flags: {}, SubReg: AMDGPU::sub0)
2998 .addSym(Sym: OffsetLo, TargetFlags: MO_FAR_BRANCH_OFFSET);
2999 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADDC_U32))
3000 .addReg(RegNo: PCReg, Flags: RegState::Define, SubReg: AMDGPU::sub1)
3001 .addReg(RegNo: PCReg, Flags: {}, SubReg: AMDGPU::sub1)
3002 .addSym(Sym: OffsetHi, TargetFlags: MO_FAR_BRANCH_OFFSET);
3003 ApplyHazardWorkarounds();
3004
3005 // Insert the indirect branch after the other terminator.
3006 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETPC_B64))
3007 .addReg(RegNo: PCReg);
3008
3009 // If a spill is needed for the pc register pair, we need to insert a spill
3010 // restore block right before the destination block, and insert a short branch
3011 // into the old destination block's fallthrough predecessor.
3012 // e.g.:
3013 //
3014 // s_cbranch_scc0 skip_long_branch:
3015 //
3016 // long_branch_bb:
3017 // spill s[8:9]
3018 // s_getpc_b64 s[8:9]
3019 // s_add_u32 s8, s8, restore_bb
3020 // s_addc_u32 s9, s9, 0
3021 // s_setpc_b64 s[8:9]
3022 //
3023 // skip_long_branch:
3024 // foo;
3025 //
3026 // .....
3027 //
3028 // dest_bb_fallthrough_predecessor:
3029 // bar;
3030 // s_branch dest_bb
3031 //
3032 // restore_bb:
3033 // restore s[8:9]
3034 // fallthrough dest_bb
3035 ///
3036 // dest_bb:
3037 // buzz;
3038
3039 Register LongBranchReservedReg = MFI->getLongBranchReservedReg();
3040 Register Scav;
3041
3042 // If we've previously reserved a register for long branches
3043 // avoid running the scavenger and just use those registers
3044 if (LongBranchReservedReg) {
3045 RS->enterBasicBlock(MBB);
3046 Scav = LongBranchReservedReg;
3047 } else {
3048 RS->enterBasicBlockEnd(MBB);
3049 Scav = RS->scavengeRegisterBackwards(
3050 RC: AMDGPU::SReg_64RegClass, To: MachineBasicBlock::iterator(GetPC),
3051 /* RestoreAfter */ false, SPAdj: 0, /* AllowSpill */ false);
3052 }
3053 if (Scav) {
3054 RS->setRegUsed(Reg: Scav);
3055 MRI.replaceRegWith(FromReg: PCReg, ToReg: Scav);
3056 MRI.clearVirtRegs();
3057 } else {
3058 // As SGPR needs VGPR to be spilled, we reuse the slot of temporary VGPR for
3059 // SGPR spill.
3060 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3061 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3062 TRI->spillEmergencySGPR(MI: GetPC, RestoreMBB&: RestoreBB, SGPR: AMDGPU::SGPR0_SGPR1, RS);
3063 MRI.replaceRegWith(FromReg: PCReg, ToReg: AMDGPU::SGPR0_SGPR1);
3064 MRI.clearVirtRegs();
3065 }
3066
3067 MCSymbol *DestLabel = Scav ? DestBB.getSymbol() : RestoreBB.getSymbol();
3068 // Now, the distance could be defined.
3069 auto *Offset = MCBinaryExpr::createSub(
3070 LHS: MCSymbolRefExpr::create(Symbol: DestLabel, Ctx&: MCCtx),
3071 RHS: MCSymbolRefExpr::create(Symbol: PostGetPCLabel, Ctx&: MCCtx), Ctx&: MCCtx);
3072 // Add offset assignments.
3073 auto *Mask = MCConstantExpr::create(Value: 0xFFFFFFFFULL, Ctx&: MCCtx);
3074 OffsetLo->setVariableValue(MCBinaryExpr::createAnd(LHS: Offset, RHS: Mask, Ctx&: MCCtx));
3075 auto *ShAmt = MCConstantExpr::create(Value: 32, Ctx&: MCCtx);
3076 OffsetHi->setVariableValue(MCBinaryExpr::createAShr(LHS: Offset, RHS: ShAmt, Ctx&: MCCtx));
3077}
3078
3079unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
3080 switch (Cond) {
3081 case SIInstrInfo::SCC_TRUE:
3082 return AMDGPU::S_CBRANCH_SCC1;
3083 case SIInstrInfo::SCC_FALSE:
3084 return AMDGPU::S_CBRANCH_SCC0;
3085 case SIInstrInfo::VCCNZ:
3086 return AMDGPU::S_CBRANCH_VCCNZ;
3087 case SIInstrInfo::VCCZ:
3088 return AMDGPU::S_CBRANCH_VCCZ;
3089 case SIInstrInfo::EXECNZ:
3090 return AMDGPU::S_CBRANCH_EXECNZ;
3091 case SIInstrInfo::EXECZ:
3092 return AMDGPU::S_CBRANCH_EXECZ;
3093 default:
3094 llvm_unreachable("invalid branch predicate");
3095 }
3096}
3097
3098SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
3099 switch (Opcode) {
3100 case AMDGPU::S_CBRANCH_SCC0:
3101 return SCC_FALSE;
3102 case AMDGPU::S_CBRANCH_SCC1:
3103 return SCC_TRUE;
3104 case AMDGPU::S_CBRANCH_VCCNZ:
3105 return VCCNZ;
3106 case AMDGPU::S_CBRANCH_VCCZ:
3107 return VCCZ;
3108 case AMDGPU::S_CBRANCH_EXECNZ:
3109 return EXECNZ;
3110 case AMDGPU::S_CBRANCH_EXECZ:
3111 return EXECZ;
3112 default:
3113 return INVALID_BR;
3114 }
3115}
3116
3117bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB,
3118 MachineBasicBlock::iterator I,
3119 MachineBasicBlock *&TBB,
3120 MachineBasicBlock *&FBB,
3121 SmallVectorImpl<MachineOperand> &Cond,
3122 bool AllowModify) const {
3123 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3124 // Unconditional Branch
3125 TBB = I->getOperand(i: 0).getMBB();
3126 return false;
3127 }
3128
3129 BranchPredicate Pred = getBranchPredicate(Opcode: I->getOpcode());
3130 if (Pred == INVALID_BR)
3131 return true;
3132
3133 MachineBasicBlock *CondBB = I->getOperand(i: 0).getMBB();
3134 Cond.push_back(Elt: MachineOperand::CreateImm(Val: Pred));
3135 Cond.push_back(Elt: I->getOperand(i: 1)); // Save the branch register.
3136
3137 ++I;
3138
3139 if (I == MBB.end()) {
3140 // Conditional branch followed by fall-through.
3141 TBB = CondBB;
3142 return false;
3143 }
3144
3145 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3146 TBB = CondBB;
3147 FBB = I->getOperand(i: 0).getMBB();
3148 return false;
3149 }
3150
3151 return true;
3152}
3153
3154bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
3155 MachineBasicBlock *&FBB,
3156 SmallVectorImpl<MachineOperand> &Cond,
3157 bool AllowModify) const {
3158 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
3159 auto E = MBB.end();
3160 if (I == E)
3161 return false;
3162
3163 // Skip over the instructions that are artificially terminators for special
3164 // exec management.
3165 while (I != E && !I->isBranch() && !I->isReturn()) {
3166 switch (I->getOpcode()) {
3167 case AMDGPU::S_MOV_B64_term:
3168 case AMDGPU::S_XOR_B64_term:
3169 case AMDGPU::S_OR_B64_term:
3170 case AMDGPU::S_ANDN2_B64_term:
3171 case AMDGPU::S_AND_B64_term:
3172 case AMDGPU::S_AND_SAVEEXEC_B64_term:
3173 case AMDGPU::S_MOV_B32_term:
3174 case AMDGPU::S_XOR_B32_term:
3175 case AMDGPU::S_OR_B32_term:
3176 case AMDGPU::S_ANDN2_B32_term:
3177 case AMDGPU::S_AND_B32_term:
3178 case AMDGPU::S_AND_SAVEEXEC_B32_term:
3179 case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
3180 case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
3181 break;
3182 case AMDGPU::SI_IF:
3183 case AMDGPU::SI_ELSE:
3184 case AMDGPU::SI_KILL_I1_TERMINATOR:
3185 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
3186 // FIXME: It's messy that these need to be considered here at all.
3187 return true;
3188 default:
3189 llvm_unreachable("unexpected non-branch terminator inst");
3190 }
3191
3192 ++I;
3193 }
3194
3195 if (I == E)
3196 return false;
3197
3198 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
3199}
3200
3201unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB,
3202 int *BytesRemoved) const {
3203 unsigned Count = 0;
3204 unsigned RemovedSize = 0;
3205 for (MachineInstr &MI : llvm::make_early_inc_range(Range: MBB.terminators())) {
3206 // Skip over artificial terminators when removing instructions.
3207 if (MI.isBranch() || MI.isReturn()) {
3208 RemovedSize += getInstSizeInBytes(MI);
3209 MI.eraseFromParent();
3210 ++Count;
3211 }
3212 }
3213
3214 if (BytesRemoved)
3215 *BytesRemoved = RemovedSize;
3216
3217 return Count;
3218}
3219
3220// Copy the flags onto the implicit condition register operand.
3221static void preserveCondRegFlags(MachineOperand &CondReg,
3222 const MachineOperand &OrigCond) {
3223 CondReg.setIsUndef(OrigCond.isUndef());
3224 CondReg.setIsKill(OrigCond.isKill());
3225}
3226
3227unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,
3228 MachineBasicBlock *TBB,
3229 MachineBasicBlock *FBB,
3230 ArrayRef<MachineOperand> Cond,
3231 const DebugLoc &DL,
3232 int *BytesAdded) const {
3233 if (!FBB && Cond.empty()) {
3234 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
3235 .addMBB(MBB: TBB);
3236 if (BytesAdded)
3237 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3238 return 1;
3239 }
3240
3241 assert(TBB && Cond[0].isImm());
3242
3243 unsigned Opcode
3244 = getBranchOpcode(Cond: static_cast<BranchPredicate>(Cond[0].getImm()));
3245
3246 if (!FBB) {
3247 MachineInstr *CondBr =
3248 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode))
3249 .addMBB(MBB: TBB);
3250
3251 // Copy the flags onto the implicit condition register operand.
3252 preserveCondRegFlags(CondReg&: CondBr->getOperand(i: 1), OrigCond: Cond[1]);
3253 fixImplicitOperands(MI&: *CondBr);
3254
3255 if (BytesAdded)
3256 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3257 return 1;
3258 }
3259
3260 assert(TBB && FBB);
3261
3262 MachineInstr *CondBr =
3263 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode))
3264 .addMBB(MBB: TBB);
3265 fixImplicitOperands(MI&: *CondBr);
3266 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
3267 .addMBB(MBB: FBB);
3268
3269 MachineOperand &CondReg = CondBr->getOperand(i: 1);
3270 CondReg.setIsUndef(Cond[1].isUndef());
3271 CondReg.setIsKill(Cond[1].isKill());
3272
3273 if (BytesAdded)
3274 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8;
3275
3276 return 2;
3277}
3278
3279bool SIInstrInfo::reverseBranchCondition(
3280 SmallVectorImpl<MachineOperand> &Cond) const {
3281 if (Cond.size() != 2) {
3282 return true;
3283 }
3284
3285 if (Cond[0].isImm()) {
3286 Cond[0].setImm(-Cond[0].getImm());
3287 return false;
3288 }
3289
3290 return true;
3291}
3292
3293bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
3294 ArrayRef<MachineOperand> Cond,
3295 Register DstReg, Register TrueReg,
3296 Register FalseReg, int &CondCycles,
3297 int &TrueCycles, int &FalseCycles) const {
3298 switch (Cond[0].getImm()) {
3299 case VCCNZ:
3300 case VCCZ: {
3301 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3302 const TargetRegisterClass *RC = MRI.getRegClass(Reg: TrueReg);
3303 if (MRI.getRegClass(Reg: FalseReg) != RC)
3304 return false;
3305
3306 int NumInsts = AMDGPU::getRegBitWidth(RC: *RC) / 32;
3307 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
3308
3309 // Limit to equal cost for branch vs. N v_cndmask_b32s.
3310 return RI.hasVGPRs(RC) && NumInsts <= 6;
3311 }
3312 case SCC_TRUE:
3313 case SCC_FALSE: {
3314 // FIXME: We could insert for VGPRs if we could replace the original compare
3315 // with a vector one.
3316 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3317 const TargetRegisterClass *RC = MRI.getRegClass(Reg: TrueReg);
3318 if (MRI.getRegClass(Reg: FalseReg) != RC)
3319 return false;
3320
3321 int NumInsts = AMDGPU::getRegBitWidth(RC: *RC) / 32;
3322
3323 // Multiples of 8 can do s_cselect_b64
3324 if (NumInsts % 2 == 0)
3325 NumInsts /= 2;
3326
3327 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
3328 return RI.isSGPRClass(RC);
3329 }
3330 default:
3331 return false;
3332 }
3333}
3334
3335void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,
3336 MachineBasicBlock::iterator I, const DebugLoc &DL,
3337 Register DstReg, ArrayRef<MachineOperand> Cond,
3338 Register TrueReg, Register FalseReg) const {
3339 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
3340 if (Pred == VCCZ || Pred == SCC_FALSE) {
3341 Pred = static_cast<BranchPredicate>(-Pred);
3342 std::swap(a&: TrueReg, b&: FalseReg);
3343 }
3344
3345 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3346 const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: DstReg);
3347 unsigned DstSize = RI.getRegSizeInBits(RC: *DstRC);
3348
3349 if (DstSize == 32) {
3350 MachineInstr *Select;
3351 if (Pred == SCC_TRUE) {
3352 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: DstReg)
3353 .addReg(RegNo: TrueReg)
3354 .addReg(RegNo: FalseReg);
3355 } else {
3356 // Instruction's operands are backwards from what is expected.
3357 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e32), DestReg: DstReg)
3358 .addReg(RegNo: FalseReg)
3359 .addReg(RegNo: TrueReg);
3360 }
3361
3362 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3363 return;
3364 }
3365
3366 if (DstSize == 64 && Pred == SCC_TRUE) {
3367 MachineInstr *Select =
3368 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B64), DestReg: DstReg)
3369 .addReg(RegNo: TrueReg)
3370 .addReg(RegNo: FalseReg);
3371
3372 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3373 return;
3374 }
3375
3376 static const int16_t Sub0_15[] = {
3377 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
3378 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
3379 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
3380 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
3381 };
3382
3383 static const int16_t Sub0_15_64[] = {
3384 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
3385 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
3386 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
3387 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
3388 };
3389
3390 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
3391 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
3392 const int16_t *SubIndices = Sub0_15;
3393 int NElts = DstSize / 32;
3394
3395 // 64-bit select is only available for SALU.
3396 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
3397 if (Pred == SCC_TRUE) {
3398 if (NElts % 2) {
3399 SelOp = AMDGPU::S_CSELECT_B32;
3400 EltRC = &AMDGPU::SGPR_32RegClass;
3401 } else {
3402 SelOp = AMDGPU::S_CSELECT_B64;
3403 EltRC = &AMDGPU::SGPR_64RegClass;
3404 SubIndices = Sub0_15_64;
3405 NElts /= 2;
3406 }
3407 }
3408
3409 MachineInstrBuilder MIB = BuildMI(
3410 BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
3411
3412 I = MIB->getIterator();
3413
3414 SmallVector<Register, 8> Regs;
3415 for (int Idx = 0; Idx != NElts; ++Idx) {
3416 Register DstElt = MRI.createVirtualRegister(RegClass: EltRC);
3417 Regs.push_back(Elt: DstElt);
3418
3419 unsigned SubIdx = SubIndices[Idx];
3420
3421 MachineInstr *Select;
3422 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) {
3423 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SelOp), DestReg: DstElt)
3424 .addReg(RegNo: FalseReg, Flags: {}, SubReg: SubIdx)
3425 .addReg(RegNo: TrueReg, Flags: {}, SubReg: SubIdx);
3426 } else {
3427 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SelOp), DestReg: DstElt)
3428 .addReg(RegNo: TrueReg, Flags: {}, SubReg: SubIdx)
3429 .addReg(RegNo: FalseReg, Flags: {}, SubReg: SubIdx);
3430 }
3431
3432 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3433 fixImplicitOperands(MI&: *Select);
3434
3435 MIB.addReg(RegNo: DstElt)
3436 .addImm(Val: SubIdx);
3437 }
3438}
3439
3440bool SIInstrInfo::isXcntDrain(const MachineInstr &MI) {
3441
3442 if (MI.isBranch() || MI.isCall() || MI.isReturn() || MI.isIndirectBranch())
3443 return true;
3444
3445 switch (MI.getOpcode()) {
3446 case AMDGPU::S_ENDPGM:
3447 case AMDGPU::S_ENDPGM_SAVED:
3448 case AMDGPU::S_TRAP:
3449 case AMDGPU::S_GETREG_B32:
3450 case AMDGPU::S_SETREG_B32:
3451 case AMDGPU::S_SETREG_B32_mode:
3452 case AMDGPU::S_SETREG_IMM32_B32:
3453 case AMDGPU::S_SETREG_IMM32_B32_mode:
3454 case AMDGPU::S_SENDMSG:
3455 case AMDGPU::S_SENDMSGHALT:
3456 case AMDGPU::S_SENDMSG_RTN_B32:
3457 case AMDGPU::S_SENDMSG_RTN_B64:
3458 case AMDGPU::S_BARRIER_WAIT:
3459 case AMDGPU::S_BARRIER_SIGNAL_M0:
3460 case AMDGPU::S_BARRIER_SIGNAL_IMM:
3461 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0:
3462 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM:
3463 return true;
3464 default:
3465 return false;
3466 }
3467}
3468
3469bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) {
3470 switch (MI.getOpcode()) {
3471 case AMDGPU::V_MOV_B16_t16_e32:
3472 case AMDGPU::V_MOV_B16_t16_e64:
3473 case AMDGPU::V_MOV_B32_e32:
3474 case AMDGPU::V_MOV_B32_e64:
3475 case AMDGPU::V_MOV_B64_PSEUDO:
3476 case AMDGPU::V_MOV_B64_e32:
3477 case AMDGPU::V_MOV_B64_e64:
3478 case AMDGPU::S_MOV_B32:
3479 case AMDGPU::S_MOV_B64:
3480 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3481 case AMDGPU::COPY:
3482 case AMDGPU::WWM_COPY:
3483 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3484 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3485 case AMDGPU::V_ACCVGPR_MOV_B32:
3486 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
3487 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
3488 return true;
3489 default:
3490 return false;
3491 }
3492}
3493
3494unsigned SIInstrInfo::getFoldableCopySrcIdx(const MachineInstr &MI) {
3495 switch (MI.getOpcode()) {
3496 case AMDGPU::V_MOV_B16_t16_e32:
3497 case AMDGPU::V_MOV_B16_t16_e64:
3498 return 2;
3499 case AMDGPU::V_MOV_B32_e32:
3500 case AMDGPU::V_MOV_B32_e64:
3501 case AMDGPU::V_MOV_B64_PSEUDO:
3502 case AMDGPU::V_MOV_B64_e32:
3503 case AMDGPU::V_MOV_B64_e64:
3504 case AMDGPU::S_MOV_B32:
3505 case AMDGPU::S_MOV_B64:
3506 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3507 case AMDGPU::COPY:
3508 case AMDGPU::WWM_COPY:
3509 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3510 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3511 case AMDGPU::V_ACCVGPR_MOV_B32:
3512 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
3513 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
3514 return 1;
3515 default:
3516 llvm_unreachable("MI is not a foldable copy");
3517 }
3518}
3519
3520static constexpr AMDGPU::OpName ModifierOpNames[] = {
3521 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
3522 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
3523 AMDGPU::OpName::omod, AMDGPU::OpName::op_sel};
3524
3525void SIInstrInfo::removeModOperands(MachineInstr &MI) const {
3526 unsigned Opc = MI.getOpcode();
3527 for (AMDGPU::OpName Name : reverse(C: ModifierOpNames)) {
3528 int Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name);
3529 if (Idx >= 0)
3530 MI.removeOperand(OpNo: Idx);
3531 }
3532}
3533
3534void SIInstrInfo::mutateAndCleanupImplicit(MachineInstr &MI,
3535 const MCInstrDesc &NewDesc) const {
3536 MI.setDesc(NewDesc);
3537
3538 // Remove any leftover implicit operands from mutating the instruction. e.g.
3539 // if we replace an s_and_b32 with a copy, we don't need the implicit scc def
3540 // anymore.
3541 const MCInstrDesc &Desc = MI.getDesc();
3542 unsigned NumOps = Desc.getNumOperands() + Desc.implicit_uses().size() +
3543 Desc.implicit_defs().size();
3544
3545 for (unsigned I = MI.getNumOperands() - 1; I >= NumOps; --I)
3546 MI.removeOperand(OpNo: I);
3547}
3548
3549std::optional<int64_t> SIInstrInfo::extractSubregFromImm(int64_t Imm,
3550 unsigned SubRegIndex) {
3551 switch (SubRegIndex) {
3552 case AMDGPU::NoSubRegister:
3553 return Imm;
3554 case AMDGPU::sub0:
3555 return SignExtend64<32>(x: Imm);
3556 case AMDGPU::sub1:
3557 return SignExtend64<32>(x: Imm >> 32);
3558 case AMDGPU::lo16:
3559 return SignExtend64<16>(x: Imm);
3560 case AMDGPU::hi16:
3561 return SignExtend64<16>(x: Imm >> 16);
3562 case AMDGPU::sub1_lo16:
3563 return SignExtend64<16>(x: Imm >> 32);
3564 case AMDGPU::sub1_hi16:
3565 return SignExtend64<16>(x: Imm >> 48);
3566 default:
3567 return std::nullopt;
3568 }
3569
3570 llvm_unreachable("covered subregister switch");
3571}
3572
3573static unsigned getNewFMAAKInst(const GCNSubtarget &ST, unsigned Opc) {
3574 switch (Opc) {
3575 case AMDGPU::V_MAC_F16_e32:
3576 case AMDGPU::V_MAC_F16_e64:
3577 case AMDGPU::V_MAD_F16_e64:
3578 return AMDGPU::V_MADAK_F16;
3579 case AMDGPU::V_MAC_F32_e32:
3580 case AMDGPU::V_MAC_F32_e64:
3581 case AMDGPU::V_MAD_F32_e64:
3582 return AMDGPU::V_MADAK_F32;
3583 case AMDGPU::V_FMAC_F32_e32:
3584 case AMDGPU::V_FMAC_F32_e64:
3585 case AMDGPU::V_FMA_F32_e64:
3586 return AMDGPU::V_FMAAK_F32;
3587 case AMDGPU::V_FMAC_F16_e32:
3588 case AMDGPU::V_FMAC_F16_e64:
3589 case AMDGPU::V_FMAC_F16_t16_e64:
3590 case AMDGPU::V_FMAC_F16_fake16_e64:
3591 case AMDGPU::V_FMAC_F16_t16_e32:
3592 case AMDGPU::V_FMAC_F16_fake16_e32:
3593 case AMDGPU::V_FMA_F16_e64:
3594 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
3595 ? AMDGPU::V_FMAAK_F16_t16
3596 : AMDGPU::V_FMAAK_F16_fake16
3597 : AMDGPU::V_FMAAK_F16;
3598 case AMDGPU::V_FMAC_F64_e32:
3599 case AMDGPU::V_FMAC_F64_e64:
3600 case AMDGPU::V_FMA_F64_e64:
3601 return AMDGPU::V_FMAAK_F64;
3602 default:
3603 llvm_unreachable("invalid instruction");
3604 }
3605}
3606
3607static unsigned getNewFMAMKInst(const GCNSubtarget &ST, unsigned Opc) {
3608 switch (Opc) {
3609 case AMDGPU::V_MAC_F16_e32:
3610 case AMDGPU::V_MAC_F16_e64:
3611 case AMDGPU::V_MAD_F16_e64:
3612 return AMDGPU::V_MADMK_F16;
3613 case AMDGPU::V_MAC_F32_e32:
3614 case AMDGPU::V_MAC_F32_e64:
3615 case AMDGPU::V_MAD_F32_e64:
3616 return AMDGPU::V_MADMK_F32;
3617 case AMDGPU::V_FMAC_F32_e32:
3618 case AMDGPU::V_FMAC_F32_e64:
3619 case AMDGPU::V_FMA_F32_e64:
3620 return AMDGPU::V_FMAMK_F32;
3621 case AMDGPU::V_FMAC_F16_e32:
3622 case AMDGPU::V_FMAC_F16_e64:
3623 case AMDGPU::V_FMAC_F16_t16_e64:
3624 case AMDGPU::V_FMAC_F16_fake16_e64:
3625 case AMDGPU::V_FMAC_F16_t16_e32:
3626 case AMDGPU::V_FMAC_F16_fake16_e32:
3627 case AMDGPU::V_FMA_F16_e64:
3628 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
3629 ? AMDGPU::V_FMAMK_F16_t16
3630 : AMDGPU::V_FMAMK_F16_fake16
3631 : AMDGPU::V_FMAMK_F16;
3632 case AMDGPU::V_FMAC_F64_e32:
3633 case AMDGPU::V_FMAC_F64_e64:
3634 case AMDGPU::V_FMA_F64_e64:
3635 return AMDGPU::V_FMAMK_F64;
3636 default:
3637 llvm_unreachable("invalid instruction");
3638 }
3639}
3640
3641bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
3642 Register Reg, MachineRegisterInfo *MRI) const {
3643 int64_t Imm;
3644 if (!getConstValDefinedInReg(MI: DefMI, Reg, ImmVal&: Imm))
3645 return false;
3646
3647 const bool HasMultipleUses = !MRI->hasOneNonDBGUse(RegNo: Reg);
3648
3649 assert(!DefMI.getOperand(0).getSubReg() && "Expected SSA form");
3650
3651 unsigned Opc = UseMI.getOpcode();
3652 if (Opc == AMDGPU::COPY) {
3653 assert(!UseMI.getOperand(0).getSubReg() && "Expected SSA form");
3654
3655 Register DstReg = UseMI.getOperand(i: 0).getReg();
3656 Register UseSubReg = UseMI.getOperand(i: 1).getSubReg();
3657
3658 const TargetRegisterClass *DstRC = RI.getRegClassForReg(MRI: *MRI, Reg: DstReg);
3659
3660 if (HasMultipleUses) {
3661 // TODO: This should fold in more cases with multiple use, but we need to
3662 // more carefully consider what those uses are.
3663 unsigned ImmDefSize = RI.getRegSizeInBits(RC: *MRI->getRegClass(Reg));
3664
3665 // Avoid breaking up a 64-bit inline immediate into a subregister extract.
3666 if (UseSubReg != AMDGPU::NoSubRegister && ImmDefSize == 64)
3667 return false;
3668
3669 // Most of the time folding a 32-bit inline constant is free (though this
3670 // might not be true if we can't later fold it into a real user).
3671 //
3672 // FIXME: This isInlineConstant check is imprecise if
3673 // getConstValDefinedInReg handled the tricky non-mov cases.
3674 if (ImmDefSize == 32 &&
3675 !isInlineConstant(ImmVal: Imm, OperandType: AMDGPU::OPERAND_REG_IMM_INT32))
3676 return false;
3677 }
3678
3679 bool Is16Bit = UseSubReg != AMDGPU::NoSubRegister &&
3680 RI.getSubRegIdxSize(Idx: UseSubReg) == 16;
3681
3682 if (Is16Bit) {
3683 if (RI.hasVGPRs(RC: DstRC))
3684 return false; // Do not clobber vgpr_hi16
3685
3686 if (DstReg.isVirtual() && UseSubReg != AMDGPU::lo16)
3687 return false;
3688 }
3689
3690 MachineFunction *MF = UseMI.getMF();
3691
3692 unsigned NewOpc = AMDGPU::INSTRUCTION_LIST_END;
3693 MCRegister MovDstPhysReg =
3694 DstReg.isPhysical() ? DstReg.asMCReg() : MCRegister();
3695
3696 std::optional<int64_t> SubRegImm = extractSubregFromImm(Imm, SubRegIndex: UseSubReg);
3697
3698 // TODO: Try to fold with AMDGPU::V_MOV_B16_t16_e64
3699 for (unsigned MovOp :
3700 {AMDGPU::S_MOV_B32, AMDGPU::V_MOV_B32_e32, AMDGPU::S_MOV_B64,
3701 AMDGPU::V_MOV_B64_PSEUDO, AMDGPU::V_ACCVGPR_WRITE_B32_e64}) {
3702 const MCInstrDesc &MovDesc = get(Opcode: MovOp);
3703
3704 const TargetRegisterClass *MovDstRC = getRegClass(MCID: MovDesc, OpNum: 0);
3705 if (Is16Bit) {
3706 // We just need to find a correctly sized register class, so the
3707 // subregister index compatibility doesn't matter since we're statically
3708 // extracting the immediate value.
3709 MovDstRC = RI.getMatchingSuperRegClass(A: MovDstRC, B: DstRC, Idx: AMDGPU::lo16);
3710 if (!MovDstRC)
3711 continue;
3712
3713 if (MovDstPhysReg) {
3714 // FIXME: We probably should not do this. If there is a live value in
3715 // the high half of the register, it will be corrupted.
3716 MovDstPhysReg =
3717 RI.getMatchingSuperReg(Reg: MovDstPhysReg, SubIdx: AMDGPU::lo16, RC: MovDstRC);
3718 if (!MovDstPhysReg)
3719 continue;
3720 }
3721 }
3722
3723 // Result class isn't the right size, try the next instruction.
3724 if (MovDstPhysReg) {
3725 if (!MovDstRC->contains(Reg: MovDstPhysReg))
3726 return false;
3727 } else if (!MRI->constrainRegClass(Reg: DstReg, RC: MovDstRC)) {
3728 // TODO: This will be overly conservative in the case of 16-bit virtual
3729 // SGPRs. We could hack up the virtual register uses to use a compatible
3730 // 32-bit class.
3731 continue;
3732 }
3733
3734 const MCOperandInfo &OpInfo = MovDesc.operands()[1];
3735
3736 // Ensure the interpreted immediate value is a valid operand in the new
3737 // mov.
3738 //
3739 // FIXME: isImmOperandLegal should have form that doesn't require existing
3740 // MachineInstr or MachineOperand
3741 if (!RI.opCanUseLiteralConstant(OpType: OpInfo.OperandType) &&
3742 !isInlineConstant(ImmVal: *SubRegImm, OperandType: OpInfo.OperandType))
3743 break;
3744
3745 NewOpc = MovOp;
3746 break;
3747 }
3748
3749 if (NewOpc == AMDGPU::INSTRUCTION_LIST_END)
3750 return false;
3751
3752 if (Is16Bit) {
3753 UseMI.getOperand(i: 0).setSubReg(AMDGPU::NoSubRegister);
3754 if (MovDstPhysReg)
3755 UseMI.getOperand(i: 0).setReg(MovDstPhysReg);
3756 assert(UseMI.getOperand(1).getReg().isVirtual());
3757 }
3758
3759 const MCInstrDesc &NewMCID = get(Opcode: NewOpc);
3760 UseMI.setDesc(NewMCID);
3761 UseMI.getOperand(i: 1).ChangeToImmediate(ImmVal: *SubRegImm);
3762 UseMI.addImplicitDefUseOperands(MF&: *MF);
3763 return true;
3764 }
3765
3766 if (HasMultipleUses)
3767 return false;
3768
3769 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
3770 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3771 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3772 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3773 Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
3774 Opc == AMDGPU::V_FMAC_F16_fake16_e64 || Opc == AMDGPU::V_FMA_F64_e64 ||
3775 Opc == AMDGPU::V_FMAC_F64_e64) {
3776 // Don't fold if we are using source or output modifiers. The new VOP2
3777 // instructions don't have them.
3778 if (hasAnyModifiersSet(MI: UseMI))
3779 return false;
3780
3781 // If this is a free constant, there's no reason to do this.
3782 // TODO: We could fold this here instead of letting SIFoldOperands do it
3783 // later.
3784 int Src0Idx = getNamedOperandIdx(Opcode: UseMI.getOpcode(), Name: AMDGPU::OpName::src0);
3785
3786 // Any src operand can be used for the legality check.
3787 if (isInlineConstant(MI: UseMI, OpIdx: Src0Idx, ImmVal: Imm))
3788 return false;
3789
3790 MachineOperand *Src0 = &UseMI.getOperand(i: Src0Idx);
3791
3792 MachineOperand *Src1 = getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src1);
3793 MachineOperand *Src2 = getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src2);
3794
3795 auto CopyRegOperandToNarrowerRC =
3796 [MRI, this](MachineInstr &MI, unsigned OpNo,
3797 const TargetRegisterClass *NewRC) -> void {
3798 if (!MI.getOperand(i: OpNo).isReg())
3799 return;
3800 Register Reg = MI.getOperand(i: OpNo).getReg();
3801 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI: *MRI, Reg);
3802 if (RI.getCommonSubClass(A: RC, B: NewRC) != NewRC)
3803 return;
3804 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
3805 BuildMI(BB&: *MI.getParent(), I: MI.getIterator(), MIMD: MI.getDebugLoc(),
3806 MCID: get(Opcode: AMDGPU::COPY), DestReg: Tmp)
3807 .addReg(RegNo: Reg);
3808 MI.getOperand(i: OpNo).setReg(Tmp);
3809 MI.getOperand(i: OpNo).setIsKill();
3810 };
3811
3812 // Multiplied part is the constant: Use v_madmk_{f16, f32}.
3813 if ((Src0->isReg() && Src0->getReg() == Reg) ||
3814 (Src1->isReg() && Src1->getReg() == Reg)) {
3815 MachineOperand *RegSrc =
3816 Src1->isReg() && Src1->getReg() == Reg ? Src0 : Src1;
3817 if (!RegSrc->isReg())
3818 return false;
3819 if (RI.isSGPRClass(RC: MRI->getRegClass(Reg: RegSrc->getReg())) &&
3820 ST.getConstantBusLimit(Opcode: Opc) < 2)
3821 return false;
3822
3823 if (!Src2->isReg() || RI.isSGPRClass(RC: MRI->getRegClass(Reg: Src2->getReg())))
3824 return false;
3825
3826 // If src2 is also a literal constant then we have to choose which one to
3827 // fold. In general it is better to choose madak so that the other literal
3828 // can be materialized in an sgpr instead of a vgpr:
3829 // s_mov_b32 s0, literal
3830 // v_madak_f32 v0, s0, v0, literal
3831 // Instead of:
3832 // v_mov_b32 v1, literal
3833 // v_madmk_f32 v0, v0, literal, v1
3834 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src2->getReg());
3835 if (Def && Def->isMoveImmediate() &&
3836 !isInlineConstant(MO: Def->getOperand(i: 1)))
3837 return false;
3838
3839 unsigned NewOpc = getNewFMAMKInst(ST, Opc);
3840 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
3841 return false;
3842
3843 const std::optional<int64_t> SubRegImm = extractSubregFromImm(
3844 Imm, SubRegIndex: RegSrc == Src1 ? Src0->getSubReg() : Src1->getSubReg());
3845
3846 // FIXME: This would be a lot easier if we could return a new instruction
3847 // instead of having to modify in place.
3848
3849 Register SrcReg = RegSrc->getReg();
3850 unsigned SrcSubReg = RegSrc->getSubReg();
3851 Src0->setReg(SrcReg);
3852 Src0->setSubReg(SrcSubReg);
3853 Src0->setIsKill(RegSrc->isKill());
3854
3855 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3856 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
3857 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
3858 Opc == AMDGPU::V_FMAC_F16_e64 || Opc == AMDGPU::V_FMAC_F64_e64)
3859 UseMI.untieRegOperand(
3860 OpIdx: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2));
3861
3862 Src1->ChangeToImmediate(ImmVal: *SubRegImm);
3863
3864 removeModOperands(MI&: UseMI);
3865 UseMI.setDesc(get(Opcode: NewOpc));
3866
3867 if (NewOpc == AMDGPU::V_FMAMK_F16_t16 ||
3868 NewOpc == AMDGPU::V_FMAMK_F16_fake16) {
3869 const TargetRegisterClass *NewRC = getRegClass(MCID: get(Opcode: NewOpc), OpNum: 0);
3870 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
3871 BuildMI(BB&: *UseMI.getParent(), I: std::next(x: UseMI.getIterator()),
3872 MIMD: UseMI.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
3873 DestReg: UseMI.getOperand(i: 0).getReg())
3874 .addReg(RegNo: Tmp, Flags: RegState::Kill);
3875 UseMI.getOperand(i: 0).setReg(Tmp);
3876 CopyRegOperandToNarrowerRC(UseMI, 1, NewRC);
3877 CopyRegOperandToNarrowerRC(UseMI, 3, NewRC);
3878 }
3879
3880 bool DeleteDef = MRI->use_nodbg_empty(RegNo: Reg);
3881 if (DeleteDef)
3882 DefMI.eraseFromParent();
3883
3884 return true;
3885 }
3886
3887 // Added part is the constant: Use v_madak_{f16, f32}.
3888 if (Src2->isReg() && Src2->getReg() == Reg) {
3889 if (ST.getConstantBusLimit(Opcode: Opc) < 2) {
3890 // Not allowed to use constant bus for another operand.
3891 // We can however allow an inline immediate as src0.
3892 bool Src0Inlined = false;
3893 if (Src0->isReg()) {
3894 // Try to inline constant if possible.
3895 // If the Def moves immediate and the use is single
3896 // We are saving VGPR here.
3897 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src0->getReg());
3898 if (Def && Def->isMoveImmediate() &&
3899 isInlineConstant(MO: Def->getOperand(i: 1)) &&
3900 MRI->hasOneNonDBGUse(RegNo: Src0->getReg())) {
3901 Src0->ChangeToImmediate(ImmVal: Def->getOperand(i: 1).getImm());
3902 Src0Inlined = true;
3903 } else if (ST.getConstantBusLimit(Opcode: Opc) <= 1 &&
3904 RI.isSGPRReg(MRI: *MRI, Reg: Src0->getReg())) {
3905 return false;
3906 }
3907 // VGPR is okay as Src0 - fallthrough
3908 }
3909
3910 if (Src1->isReg() && !Src0Inlined) {
3911 // We have one slot for inlinable constant so far - try to fill it
3912 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src1->getReg());
3913 if (Def && Def->isMoveImmediate() &&
3914 isInlineConstant(MO: Def->getOperand(i: 1)) &&
3915 MRI->hasOneNonDBGUse(RegNo: Src1->getReg()) && commuteInstruction(MI&: UseMI))
3916 Src0->ChangeToImmediate(ImmVal: Def->getOperand(i: 1).getImm());
3917 else if (RI.isSGPRReg(MRI: *MRI, Reg: Src1->getReg()))
3918 return false;
3919 // VGPR is okay as Src1 - fallthrough
3920 }
3921 }
3922
3923 unsigned NewOpc = getNewFMAAKInst(ST, Opc);
3924 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
3925 return false;
3926
3927 // FIXME: This would be a lot easier if we could return a new instruction
3928 // instead of having to modify in place.
3929
3930 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3931 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
3932 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
3933 Opc == AMDGPU::V_FMAC_F16_e64 || Opc == AMDGPU::V_FMAC_F64_e64)
3934 UseMI.untieRegOperand(
3935 OpIdx: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2));
3936
3937 const std::optional<int64_t> SubRegImm =
3938 extractSubregFromImm(Imm, SubRegIndex: Src2->getSubReg());
3939
3940 // ChangingToImmediate adds Src2 back to the instruction.
3941 Src2->ChangeToImmediate(ImmVal: *SubRegImm);
3942
3943 // These come before src2.
3944 removeModOperands(MI&: UseMI);
3945 UseMI.setDesc(get(Opcode: NewOpc));
3946
3947 if (NewOpc == AMDGPU::V_FMAAK_F16_t16 ||
3948 NewOpc == AMDGPU::V_FMAAK_F16_fake16) {
3949 const TargetRegisterClass *NewRC = getRegClass(MCID: get(Opcode: NewOpc), OpNum: 0);
3950 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
3951 BuildMI(BB&: *UseMI.getParent(), I: std::next(x: UseMI.getIterator()),
3952 MIMD: UseMI.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
3953 DestReg: UseMI.getOperand(i: 0).getReg())
3954 .addReg(RegNo: Tmp, Flags: RegState::Kill);
3955 UseMI.getOperand(i: 0).setReg(Tmp);
3956 CopyRegOperandToNarrowerRC(UseMI, 1, NewRC);
3957 CopyRegOperandToNarrowerRC(UseMI, 2, NewRC);
3958 }
3959
3960 // It might happen that UseMI was commuted
3961 // and we now have SGPR as SRC1. If so 2 inlined
3962 // constant and SGPR are illegal.
3963 legalizeOperands(MI&: UseMI);
3964
3965 bool DeleteDef = MRI->use_nodbg_empty(RegNo: Reg);
3966 if (DeleteDef)
3967 DefMI.eraseFromParent();
3968
3969 return true;
3970 }
3971 }
3972
3973 return false;
3974}
3975
3976static bool
3977memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1,
3978 ArrayRef<const MachineOperand *> BaseOps2) {
3979 if (BaseOps1.size() != BaseOps2.size())
3980 return false;
3981 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) {
3982 if (!BaseOps1[I]->isIdenticalTo(Other: *BaseOps2[I]))
3983 return false;
3984 }
3985 return true;
3986}
3987
3988static bool offsetsDoNotOverlap(LocationSize WidthA, int OffsetA,
3989 LocationSize WidthB, int OffsetB) {
3990 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
3991 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
3992 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
3993 return LowWidth.hasValue() &&
3994 LowOffset + (int)LowWidth.getValue() <= HighOffset;
3995}
3996
3997bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
3998 const MachineInstr &MIb) const {
3999 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;
4000 int64_t Offset0, Offset1;
4001 LocationSize Dummy0 = LocationSize::precise(Value: 0);
4002 LocationSize Dummy1 = LocationSize::precise(Value: 0);
4003 bool Offset0IsScalable, Offset1IsScalable;
4004 if (!getMemOperandsWithOffsetWidth(LdSt: MIa, BaseOps&: BaseOps0, Offset&: Offset0, OffsetIsScalable&: Offset0IsScalable,
4005 Width&: Dummy0, TRI: &RI) ||
4006 !getMemOperandsWithOffsetWidth(LdSt: MIb, BaseOps&: BaseOps1, Offset&: Offset1, OffsetIsScalable&: Offset1IsScalable,
4007 Width&: Dummy1, TRI: &RI))
4008 return false;
4009
4010 if (!memOpsHaveSameBaseOperands(BaseOps1: BaseOps0, BaseOps2: BaseOps1))
4011 return false;
4012
4013 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
4014 // FIXME: Handle ds_read2 / ds_write2.
4015 return false;
4016 }
4017 LocationSize Width0 = MIa.memoperands().front()->getSize();
4018 LocationSize Width1 = MIb.memoperands().front()->getSize();
4019 return offsetsDoNotOverlap(WidthA: Width0, OffsetA: Offset0, WidthB: Width1, OffsetB: Offset1);
4020}
4021
4022bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
4023 const MachineInstr &MIb) const {
4024 assert(MIa.mayLoadOrStore() &&
4025 "MIa must load from or modify a memory location");
4026 assert(MIb.mayLoadOrStore() &&
4027 "MIb must load from or modify a memory location");
4028
4029 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects())
4030 return false;
4031
4032 // XXX - Can we relax this between address spaces?
4033 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
4034 return false;
4035
4036 if (isLDSDMA(MI: MIa) || isLDSDMA(MI: MIb))
4037 return false;
4038
4039 if (MIa.isBundle() || MIb.isBundle())
4040 return false;
4041
4042 // TODO: Should we check the address space from the MachineMemOperand? That
4043 // would allow us to distinguish objects we know don't alias based on the
4044 // underlying address space, even if it was lowered to a different one,
4045 // e.g. private accesses lowered to use MUBUF instructions on a scratch
4046 // buffer.
4047 if (isDS(MI: MIa)) {
4048 if (isDS(MI: MIb))
4049 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4050
4051 return !isFLAT(MI: MIb) || isSegmentSpecificFLAT(MI: MIb);
4052 }
4053
4054 if (isMUBUF(MI: MIa) || isMTBUF(MI: MIa)) {
4055 if (isMUBUF(MI: MIb) || isMTBUF(MI: MIb))
4056 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4057
4058 if (isFLAT(MI: MIb))
4059 return isFLATScratch(MI: MIb);
4060
4061 return !isSMRD(MI: MIb);
4062 }
4063
4064 if (isSMRD(MI: MIa)) {
4065 if (isSMRD(MI: MIb))
4066 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4067
4068 if (isFLAT(MI: MIb))
4069 return isFLATScratch(MI: MIb);
4070
4071 return !isMUBUF(MI: MIb) && !isMTBUF(MI: MIb);
4072 }
4073
4074 if (isFLAT(MI: MIa)) {
4075 if (isFLAT(MI: MIb)) {
4076 if ((isFLATScratch(MI: MIa) && isFLATGlobal(MI: MIb)) ||
4077 (isFLATGlobal(MI: MIa) && isFLATScratch(MI: MIb)))
4078 return true;
4079
4080 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4081 }
4082
4083 return false;
4084 }
4085
4086 return false;
4087}
4088
4089static bool getFoldableImm(Register Reg, const MachineRegisterInfo &MRI,
4090 int64_t &Imm, MachineInstr **DefMI = nullptr) {
4091 if (Reg.isPhysical())
4092 return false;
4093 auto *Def = MRI.getUniqueVRegDef(Reg);
4094 if (Def && SIInstrInfo::isFoldableCopy(MI: *Def) && Def->getOperand(i: 1).isImm()) {
4095 Imm = Def->getOperand(i: 1).getImm();
4096 if (DefMI)
4097 *DefMI = Def;
4098 return true;
4099 }
4100 return false;
4101}
4102
4103static bool getFoldableImm(const MachineOperand *MO, int64_t &Imm,
4104 MachineInstr **DefMI = nullptr) {
4105 if (!MO->isReg())
4106 return false;
4107 const MachineFunction *MF = MO->getParent()->getMF();
4108 const MachineRegisterInfo &MRI = MF->getRegInfo();
4109 return getFoldableImm(Reg: MO->getReg(), MRI, Imm, DefMI);
4110}
4111
4112static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI,
4113 MachineInstr &NewMI) {
4114 if (LV) {
4115 unsigned NumOps = MI.getNumOperands();
4116 for (unsigned I = 1; I < NumOps; ++I) {
4117 MachineOperand &Op = MI.getOperand(i: I);
4118 if (Op.isReg() && Op.isKill())
4119 LV->replaceKillInstruction(Reg: Op.getReg(), OldMI&: MI, NewMI);
4120 }
4121 }
4122}
4123
4124static unsigned getNewFMAInst(const GCNSubtarget &ST, unsigned Opc) {
4125 switch (Opc) {
4126 case AMDGPU::V_MAC_F16_e32:
4127 case AMDGPU::V_MAC_F16_e64:
4128 return AMDGPU::V_MAD_F16_e64;
4129 case AMDGPU::V_MAC_F32_e32:
4130 case AMDGPU::V_MAC_F32_e64:
4131 return AMDGPU::V_MAD_F32_e64;
4132 case AMDGPU::V_MAC_LEGACY_F32_e32:
4133 case AMDGPU::V_MAC_LEGACY_F32_e64:
4134 return AMDGPU::V_MAD_LEGACY_F32_e64;
4135 case AMDGPU::V_FMAC_LEGACY_F32_e32:
4136 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4137 return AMDGPU::V_FMA_LEGACY_F32_e64;
4138 case AMDGPU::V_FMAC_F16_e32:
4139 case AMDGPU::V_FMAC_F16_e64:
4140 case AMDGPU::V_FMAC_F16_t16_e64:
4141 case AMDGPU::V_FMAC_F16_fake16_e64:
4142 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
4143 ? AMDGPU::V_FMA_F16_gfx9_t16_e64
4144 : AMDGPU::V_FMA_F16_gfx9_fake16_e64
4145 : AMDGPU::V_FMA_F16_gfx9_e64;
4146 case AMDGPU::V_FMAC_F32_e32:
4147 case AMDGPU::V_FMAC_F32_e64:
4148 return AMDGPU::V_FMA_F32_e64;
4149 case AMDGPU::V_FMAC_F64_e32:
4150 case AMDGPU::V_FMAC_F64_e64:
4151 return AMDGPU::V_FMA_F64_e64;
4152 default:
4153 llvm_unreachable("invalid instruction");
4154 }
4155}
4156
4157/// Helper struct for the implementation of 3-address conversion to communicate
4158/// updates made to instruction operands.
4159struct SIInstrInfo::ThreeAddressUpdates {
4160 /// Other instruction whose def is no longer used by the converted
4161 /// instruction.
4162 MachineInstr *RemoveMIUse = nullptr;
4163};
4164
4165MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI,
4166 LiveVariables *LV,
4167 LiveIntervals *LIS) const {
4168 MachineBasicBlock &MBB = *MI.getParent();
4169 MachineInstr *CandidateMI = &MI;
4170
4171 if (MI.isBundle()) {
4172 // This is a temporary placeholder for bundle handling that enables us to
4173 // exercise the relevant code paths in the two-address instruction pass.
4174 if (MI.getBundleSize() != 1)
4175 return nullptr;
4176 CandidateMI = MI.getNextNode();
4177 }
4178
4179 ThreeAddressUpdates U;
4180 MachineInstr *NewMI = convertToThreeAddressImpl(MI&: *CandidateMI, Updates&: U);
4181 if (!NewMI)
4182 return nullptr;
4183
4184 if (MI.isBundle()) {
4185 CandidateMI->eraseFromBundle();
4186
4187 for (MachineOperand &MO : MI.all_defs()) {
4188 if (MO.isTied())
4189 MI.untieRegOperand(OpIdx: MO.getOperandNo());
4190 }
4191 } else {
4192 updateLiveVariables(LV, MI, NewMI&: *NewMI);
4193 if (LIS) {
4194 LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *NewMI);
4195 // SlotIndex of defs needs to be updated when converting to early-clobber
4196 MachineOperand &Def = NewMI->getOperand(i: 0);
4197 if (Def.isEarlyClobber() && Def.isReg() &&
4198 LIS->hasInterval(Reg: Def.getReg())) {
4199 SlotIndex OldIndex = LIS->getInstructionIndex(Instr: *NewMI).getRegSlot(EC: false);
4200 SlotIndex NewIndex = LIS->getInstructionIndex(Instr: *NewMI).getRegSlot(EC: true);
4201 auto &LI = LIS->getInterval(Reg: Def.getReg());
4202 auto UpdateDefIndex = [&](LiveRange &LR) {
4203 auto *S = LR.find(Pos: OldIndex);
4204 if (S != LR.end() && S->start == OldIndex) {
4205 assert(S->valno && S->valno->def == OldIndex);
4206 S->start = NewIndex;
4207 S->valno->def = NewIndex;
4208 }
4209 };
4210 UpdateDefIndex(LI);
4211 for (auto &SR : LI.subranges())
4212 UpdateDefIndex(SR);
4213 }
4214 }
4215 }
4216
4217 if (U.RemoveMIUse) {
4218 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4219 // The only user is the instruction which will be killed.
4220 Register DefReg = U.RemoveMIUse->getOperand(i: 0).getReg();
4221
4222 if (MRI.hasOneNonDBGUse(RegNo: DefReg)) {
4223 // We cannot just remove the DefMI here, calling pass will crash.
4224 U.RemoveMIUse->setDesc(get(Opcode: AMDGPU::IMPLICIT_DEF));
4225 U.RemoveMIUse->getOperand(i: 0).setIsDead(true);
4226 for (unsigned I = U.RemoveMIUse->getNumOperands() - 1; I != 0; --I)
4227 U.RemoveMIUse->removeOperand(OpNo: I);
4228 if (LV)
4229 LV->getVarInfo(Reg: DefReg).AliveBlocks.clear();
4230 }
4231
4232 if (MI.isBundle()) {
4233 VirtRegInfo VRI = AnalyzeVirtRegInBundle(MI, Reg: DefReg);
4234 if (!VRI.Reads && !VRI.Writes) {
4235 for (MachineOperand &MO : MI.all_uses()) {
4236 if (MO.isReg() && MO.getReg() == DefReg) {
4237 assert(MO.getSubReg() == 0 &&
4238 "tied sub-registers in bundles currently not supported");
4239 MI.removeOperand(OpNo: MO.getOperandNo());
4240 break;
4241 }
4242 }
4243
4244 if (LIS)
4245 LIS->shrinkToUses(li: &LIS->getInterval(Reg: DefReg));
4246 }
4247 } else if (LIS) {
4248 LiveInterval &DefLI = LIS->getInterval(Reg: DefReg);
4249
4250 // We cannot delete the original instruction here, so hack out the use
4251 // in the original instruction with a dummy register so we can use
4252 // shrinkToUses to deal with any multi-use edge cases. Other targets do
4253 // not have the complexity of deleting a use to consider here.
4254 Register DummyReg = MRI.cloneVirtualRegister(VReg: DefReg);
4255 for (MachineOperand &MIOp : MI.uses()) {
4256 if (MIOp.isReg() && MIOp.getReg() == DefReg) {
4257 MIOp.setIsUndef(true);
4258 MIOp.setReg(DummyReg);
4259 }
4260 }
4261
4262 if (MI.isBundle()) {
4263 VirtRegInfo VRI = AnalyzeVirtRegInBundle(MI, Reg: DefReg);
4264 if (!VRI.Reads && !VRI.Writes) {
4265 for (MachineOperand &MIOp : MI.uses()) {
4266 if (MIOp.isReg() && MIOp.getReg() == DefReg) {
4267 MIOp.setIsUndef(true);
4268 MIOp.setReg(DummyReg);
4269 }
4270 }
4271 }
4272
4273 MI.addOperand(Op: MachineOperand::CreateReg(Reg: DummyReg, isDef: false, isImp: false, isKill: false,
4274 isDead: false, /*isUndef=*/true));
4275 }
4276
4277 LIS->shrinkToUses(li: &DefLI);
4278 }
4279 }
4280
4281 return MI.isBundle() ? &MI : NewMI;
4282}
4283
4284MachineInstr *
4285SIInstrInfo::convertToThreeAddressImpl(MachineInstr &MI,
4286 ThreeAddressUpdates &U) const {
4287 MachineBasicBlock &MBB = *MI.getParent();
4288 unsigned Opc = MI.getOpcode();
4289
4290 // Handle MFMA.
4291 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opcode: Opc);
4292 if (NewMFMAOpc != -1) {
4293 MachineInstrBuilder MIB =
4294 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewMFMAOpc));
4295 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I)
4296 MIB.add(MO: MI.getOperand(i: I));
4297 return MIB;
4298 }
4299
4300 if (SIInstrInfo::isWMMA(MI)) {
4301 unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(Opc: MI.getOpcode());
4302 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4303 .setMIFlags(MI.getFlags());
4304 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I)
4305 MIB->addOperand(Op: MI.getOperand(i: I));
4306 return MIB;
4307 }
4308
4309 assert(Opc != AMDGPU::V_FMAC_F16_t16_e32 &&
4310 Opc != AMDGPU::V_FMAC_F16_fake16_e32 &&
4311 "V_FMAC_F16_t16/fake16_e32 is not supported and not expected to be "
4312 "present pre-RA");
4313
4314 // Handle MAC/FMAC.
4315 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
4316 bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 ||
4317 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 ||
4318 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
4319 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64;
4320 bool Src0Literal = false;
4321
4322 switch (Opc) {
4323 default:
4324 return nullptr;
4325 case AMDGPU::V_MAC_F16_e64:
4326 case AMDGPU::V_FMAC_F16_e64:
4327 case AMDGPU::V_FMAC_F16_t16_e64:
4328 case AMDGPU::V_FMAC_F16_fake16_e64:
4329 case AMDGPU::V_MAC_F32_e64:
4330 case AMDGPU::V_MAC_LEGACY_F32_e64:
4331 case AMDGPU::V_FMAC_F32_e64:
4332 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4333 case AMDGPU::V_FMAC_F64_e64:
4334 break;
4335 case AMDGPU::V_MAC_F16_e32:
4336 case AMDGPU::V_FMAC_F16_e32:
4337 case AMDGPU::V_MAC_F32_e32:
4338 case AMDGPU::V_MAC_LEGACY_F32_e32:
4339 case AMDGPU::V_FMAC_F32_e32:
4340 case AMDGPU::V_FMAC_LEGACY_F32_e32:
4341 case AMDGPU::V_FMAC_F64_e32: {
4342 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
4343 Name: AMDGPU::OpName::src0);
4344 const MachineOperand *Src0 = &MI.getOperand(i: Src0Idx);
4345 if (!Src0->isReg() && !Src0->isImm())
4346 return nullptr;
4347
4348 if (Src0->isImm() && !isInlineConstant(MI, OpIdx: Src0Idx, MO: *Src0))
4349 Src0Literal = true;
4350
4351 break;
4352 }
4353 }
4354
4355 MachineInstrBuilder MIB;
4356 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
4357 const MachineOperand *Src0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
4358 const MachineOperand *Src0Mods =
4359 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
4360 const MachineOperand *Src1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
4361 const MachineOperand *Src1Mods =
4362 getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
4363 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
4364 const MachineOperand *Src2Mods =
4365 getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers);
4366 const MachineOperand *Clamp = getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
4367 const MachineOperand *Omod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
4368 const MachineOperand *OpSel = getNamedOperand(MI, OperandName: AMDGPU::OpName::op_sel);
4369
4370 if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsLegacy &&
4371 (!IsF64 || ST.hasFmaakFmamkF64Insts()) &&
4372 // If we have an SGPR input, we will violate the constant bus restriction.
4373 (ST.getConstantBusLimit(Opcode: Opc) > 1 || !Src0->isReg() ||
4374 !RI.isSGPRReg(MRI: MBB.getParent()->getRegInfo(), Reg: Src0->getReg()))) {
4375 MachineInstr *DefMI = nullptr;
4376
4377 int64_t Imm;
4378 if (!Src0Literal && getFoldableImm(MO: Src2, Imm, DefMI: &DefMI)) {
4379 unsigned NewOpc = getNewFMAAKInst(ST, Opc);
4380 if (pseudoToMCOpcode(Opcode: NewOpc) != -1) {
4381 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4382 .add(MO: *Dst)
4383 .add(MO: *Src0)
4384 .add(MO: *Src1)
4385 .addImm(Val: Imm)
4386 .setMIFlags(MI.getFlags());
4387 U.RemoveMIUse = DefMI;
4388 return MIB;
4389 }
4390 }
4391 unsigned NewOpc = getNewFMAMKInst(ST, Opc);
4392 if (!Src0Literal && getFoldableImm(MO: Src1, Imm, DefMI: &DefMI)) {
4393 if (pseudoToMCOpcode(Opcode: NewOpc) != -1) {
4394 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4395 .add(MO: *Dst)
4396 .add(MO: *Src0)
4397 .addImm(Val: Imm)
4398 .add(MO: *Src2)
4399 .setMIFlags(MI.getFlags());
4400 U.RemoveMIUse = DefMI;
4401 return MIB;
4402 }
4403 }
4404 if (Src0Literal || getFoldableImm(MO: Src0, Imm, DefMI: &DefMI)) {
4405 if (Src0Literal) {
4406 Imm = Src0->getImm();
4407 DefMI = nullptr;
4408 }
4409 if (pseudoToMCOpcode(Opcode: NewOpc) != -1 &&
4410 isOperandLegal(
4411 MI, OpIdx: AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::src0),
4412 MO: Src1)) {
4413 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4414 .add(MO: *Dst)
4415 .add(MO: *Src1)
4416 .addImm(Val: Imm)
4417 .add(MO: *Src2)
4418 .setMIFlags(MI.getFlags());
4419 U.RemoveMIUse = DefMI;
4420 return MIB;
4421 }
4422 }
4423 }
4424
4425 // VOP2 mac/fmac with a literal operand cannot be converted to VOP3 mad/fma
4426 // if VOP3 does not allow a literal operand.
4427 if (Src0Literal && !ST.hasVOP3Literal())
4428 return nullptr;
4429
4430 unsigned NewOpc = getNewFMAInst(ST, Opc);
4431
4432 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
4433 return nullptr;
4434
4435 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4436 .add(MO: *Dst)
4437 .addImm(Val: Src0Mods ? Src0Mods->getImm() : 0)
4438 .add(MO: *Src0)
4439 .addImm(Val: Src1Mods ? Src1Mods->getImm() : 0)
4440 .add(MO: *Src1)
4441 .addImm(Val: Src2Mods ? Src2Mods->getImm() : 0)
4442 .add(MO: *Src2)
4443 .addImm(Val: Clamp ? Clamp->getImm() : 0)
4444 .addImm(Val: Omod ? Omod->getImm() : 0)
4445 .setMIFlags(MI.getFlags());
4446 if (AMDGPU::hasNamedOperand(Opcode: NewOpc, NamedIdx: AMDGPU::OpName::op_sel))
4447 MIB.addImm(Val: OpSel ? OpSel->getImm() : 0);
4448 return MIB;
4449}
4450
4451// It's not generally safe to move VALU instructions across these since it will
4452// start using the register as a base index rather than directly.
4453// XXX - Why isn't hasSideEffects sufficient for these?
4454static bool changesVGPRIndexingMode(const MachineInstr &MI) {
4455 switch (MI.getOpcode()) {
4456 case AMDGPU::S_SET_GPR_IDX_ON:
4457 case AMDGPU::S_SET_GPR_IDX_MODE:
4458 case AMDGPU::S_SET_GPR_IDX_OFF:
4459 return true;
4460 default:
4461 return false;
4462 }
4463}
4464
4465bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
4466 const MachineBasicBlock *MBB,
4467 const MachineFunction &MF) const {
4468 // Skipping the check for SP writes in the base implementation. The reason it
4469 // was added was apparently due to compile time concerns.
4470 //
4471 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops
4472 // but is probably avoidable.
4473
4474 // Copied from base implementation.
4475 // Terminators and labels can't be scheduled around.
4476 if (MI.isTerminator() || MI.isPosition())
4477 return true;
4478
4479 // INLINEASM_BR can jump to another block
4480 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
4481 return true;
4482
4483 if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(i: 0).getImm() == 0)
4484 return true;
4485
4486 // Target-independent instructions do not have an implicit-use of EXEC, even
4487 // when they operate on VGPRs. Treating EXEC modifications as scheduling
4488 // boundaries prevents incorrect movements of such instructions.
4489 return MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &RI) ||
4490 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
4491 MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
4492 MI.getOpcode() == AMDGPU::S_SETPRIO ||
4493 MI.getOpcode() == AMDGPU::S_SETPRIO_INC_WG ||
4494 changesVGPRIndexingMode(MI);
4495}
4496
4497bool SIInstrInfo::isAlwaysGDS(uint32_t Opcode) const {
4498 return Opcode == AMDGPU::DS_ORDERED_COUNT ||
4499 Opcode == AMDGPU::DS_ADD_GS_REG_RTN ||
4500 Opcode == AMDGPU::DS_SUB_GS_REG_RTN || isGWS(Opcode);
4501}
4502
4503bool SIInstrInfo::mayAccessScratch(const MachineInstr &MI) const {
4504 // Instructions that access scratch use FLAT encoding or BUF encodings.
4505 if ((!isFLAT(MI) || isFLATGlobal(MI)) && !isBUF(MI))
4506 return false;
4507
4508 // SCRATCH instructions always access scratch.
4509 if (isFLATScratch(MI))
4510 return true;
4511
4512 // If FLAT_SCRATCH registers are not initialized, we can never access scratch
4513 // via the aperture.
4514 if (MI.getMF()->getFunction().hasFnAttribute(Kind: "amdgpu-no-flat-scratch-init"))
4515 return false;
4516
4517 // If there are no memory operands then conservatively assume the flat
4518 // operation may access scratch.
4519 if (MI.memoperands_empty())
4520 return true;
4521
4522 // See if any memory operand specifies an address space that involves scratch.
4523 return any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *Memop) {
4524 unsigned AS = Memop->getAddrSpace();
4525 if (AS == AMDGPUAS::FLAT_ADDRESS) {
4526 const MDNode *MD = Memop->getAAInfo().NoAliasAddrSpace;
4527 return !MD || !AMDGPU::hasValueInRangeLikeMetadata(
4528 MD: *MD, Val: AMDGPUAS::PRIVATE_ADDRESS);
4529 }
4530 return AS == AMDGPUAS::PRIVATE_ADDRESS;
4531 });
4532}
4533
4534bool SIInstrInfo::mayAccessVMEMThroughFlat(const MachineInstr &MI) const {
4535 assert(isFLAT(MI));
4536
4537 // All flat instructions use the VMEM counter except prefetch.
4538 if (!usesVM_CNT(MI))
4539 return false;
4540
4541 // If there are no memory operands then conservatively assume the flat
4542 // operation may access VMEM.
4543 if (MI.memoperands_empty())
4544 return true;
4545
4546 // See if any memory operand specifies an address space that involves VMEM.
4547 // Flat operations only supported FLAT, LOCAL (LDS), or address spaces
4548 // involving VMEM such as GLOBAL, CONSTANT, PRIVATE (SCRATCH), etc. The REGION
4549 // (GDS) address space is not supported by flat operations. Therefore, simply
4550 // return true unless only the LDS address space is found.
4551 for (const MachineMemOperand *Memop : MI.memoperands()) {
4552 unsigned AS = Memop->getAddrSpace();
4553 assert(AS != AMDGPUAS::REGION_ADDRESS);
4554 if (AS != AMDGPUAS::LOCAL_ADDRESS)
4555 return true;
4556 }
4557
4558 return false;
4559}
4560
4561bool SIInstrInfo::mayAccessLDSThroughFlat(const MachineInstr &MI,
4562 bool TgSplit) const {
4563 assert(isFLAT(MI));
4564
4565 // Flat instruction such as SCRATCH and GLOBAL do not use the lgkm counter.
4566 if (!usesLGKM_CNT(MI))
4567 return false;
4568
4569 // If in tgsplit mode then there can be no use of LDS.
4570 if (TgSplit)
4571 return false;
4572
4573 // If there are no memory operands then conservatively assume the flat
4574 // operation may access LDS.
4575 if (MI.memoperands_empty())
4576 return true;
4577
4578 // See if any memory operand specifies an address space that involves LDS.
4579 for (const MachineMemOperand *Memop : MI.memoperands()) {
4580 unsigned AS = Memop->getAddrSpace();
4581 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)
4582 return true;
4583 }
4584
4585 return false;
4586}
4587
4588bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) {
4589 // Skip the full operand and register alias search modifiesRegister
4590 // does. There's only a handful of instructions that touch this, it's only an
4591 // implicit def, and doesn't alias any other registers.
4592 return is_contained(Range: MI.getDesc().implicit_defs(), Element: AMDGPU::MODE);
4593}
4594
4595bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const {
4596 unsigned Opcode = MI.getOpcode();
4597
4598 if (MI.mayStore() && isSMRD(MI))
4599 return true; // scalar store or atomic
4600
4601 // This will terminate the function when other lanes may need to continue.
4602 if (MI.isReturn())
4603 return true;
4604
4605 // These instructions cause shader I/O that may cause hardware lockups
4606 // when executed with an empty EXEC mask.
4607 //
4608 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
4609 // EXEC = 0, but checking for that case here seems not worth it
4610 // given the typical code patterns.
4611 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
4612 isEXP(Opcode) || Opcode == AMDGPU::DS_ORDERED_COUNT ||
4613 Opcode == AMDGPU::S_TRAP || Opcode == AMDGPU::S_WAIT_EVENT ||
4614 Opcode == AMDGPU::S_SETHALT)
4615 return true;
4616
4617 if (MI.isCall() || MI.isInlineAsm())
4618 return true; // conservative assumption
4619
4620 // Assume that barrier interactions are only intended with active lanes.
4621 if (isBarrier(Opcode))
4622 return true;
4623
4624 // A mode change is a scalar operation that influences vector instructions.
4625 if (modifiesModeRegister(MI))
4626 return true;
4627
4628 // These are like SALU instructions in terms of effects, so it's questionable
4629 // whether we should return true for those.
4630 //
4631 // However, executing them with EXEC = 0 causes them to operate on undefined
4632 // data, which we avoid by returning true here.
4633 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 ||
4634 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32 ||
4635 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
4636 Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR)
4637 return true;
4638
4639 return false;
4640}
4641
4642bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI,
4643 const MachineInstr &MI) const {
4644 if (MI.isMetaInstruction())
4645 return false;
4646
4647 // This won't read exec if this is an SGPR->SGPR copy.
4648 if (MI.isCopyLike()) {
4649 if (!RI.isSGPRReg(MRI, Reg: MI.getOperand(i: 0).getReg()))
4650 return true;
4651
4652 // Make sure this isn't copying exec as a normal operand
4653 return MI.readsRegister(Reg: AMDGPU::EXEC, TRI: &RI);
4654 }
4655
4656 // Make a conservative assumption about the callee.
4657 if (MI.isCall())
4658 return true;
4659
4660 // Be conservative with any unhandled generic opcodes.
4661 if (!isTargetSpecificOpcode(Opcode: MI.getOpcode()))
4662 return true;
4663
4664 return !isSALU(MI) || MI.readsRegister(Reg: AMDGPU::EXEC, TRI: &RI);
4665}
4666
4667bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
4668 switch (Imm.getBitWidth()) {
4669 case 1: // This likely will be a condition code mask.
4670 return true;
4671
4672 case 32:
4673 return AMDGPU::isInlinableLiteral32(Literal: Imm.getSExtValue(),
4674 HasInv2Pi: ST.hasInv2PiInlineImm());
4675 case 64:
4676 return AMDGPU::isInlinableLiteral64(Literal: Imm.getSExtValue(),
4677 HasInv2Pi: ST.hasInv2PiInlineImm());
4678 case 16:
4679 return ST.has16BitInsts() &&
4680 AMDGPU::isInlinableLiteralI16(Literal: Imm.getSExtValue(),
4681 HasInv2Pi: ST.hasInv2PiInlineImm());
4682 default:
4683 llvm_unreachable("invalid bitwidth");
4684 }
4685}
4686
4687bool SIInstrInfo::isInlineConstant(const APFloat &Imm) const {
4688 APInt IntImm = Imm.bitcastToAPInt();
4689 int64_t IntImmVal = IntImm.getSExtValue();
4690 bool HasInv2Pi = ST.hasInv2PiInlineImm();
4691 switch (APFloat::SemanticsToEnum(Sem: Imm.getSemantics())) {
4692 default:
4693 llvm_unreachable("invalid fltSemantics");
4694 case APFloatBase::S_IEEEsingle:
4695 case APFloatBase::S_IEEEdouble:
4696 return isInlineConstant(Imm: IntImm);
4697 case APFloatBase::S_BFloat:
4698 return ST.has16BitInsts() &&
4699 AMDGPU::isInlinableLiteralBF16(Literal: IntImmVal, HasInv2Pi);
4700 case APFloatBase::S_IEEEhalf:
4701 return ST.has16BitInsts() &&
4702 AMDGPU::isInlinableLiteralFP16(Literal: IntImmVal, HasInv2Pi);
4703 }
4704}
4705
4706bool SIInstrInfo::isInlineConstant(int64_t Imm, uint8_t OperandType) const {
4707 // MachineOperand provides no way to tell the true operand size, since it only
4708 // records a 64-bit value. We need to know the size to determine if a 32-bit
4709 // floating point immediate bit pattern is legal for an integer immediate. It
4710 // would be for any 32-bit integer operand, but would not be for a 64-bit one.
4711 switch (OperandType) {
4712 case AMDGPU::OPERAND_REG_IMM_INT32:
4713 case AMDGPU::OPERAND_REG_IMM_FP32:
4714 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
4715 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
4716 case AMDGPU::OPERAND_REG_IMM_V2FP32:
4717 case AMDGPU::OPERAND_REG_IMM_V2INT32:
4718 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
4719 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
4720 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: {
4721 int32_t Trunc = static_cast<int32_t>(Imm);
4722 return AMDGPU::isInlinableLiteral32(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
4723 }
4724 case AMDGPU::OPERAND_REG_IMM_INT64:
4725 case AMDGPU::OPERAND_REG_IMM_FP64:
4726 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
4727 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
4728 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
4729 case AMDGPU::OPERAND_REG_IMM_V2FP64:
4730 case AMDGPU::OPERAND_REG_IMM_V2INT64:
4731 return AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: ST.hasInv2PiInlineImm());
4732 case AMDGPU::OPERAND_REG_IMM_INT16:
4733 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
4734 // We would expect inline immediates to not be concerned with an integer/fp
4735 // distinction. However, in the case of 16-bit integer operations, the
4736 // "floating point" values appear to not work. It seems read the low 16-bits
4737 // of 32-bit immediates, which happens to always work for the integer
4738 // values.
4739 //
4740 // See llvm bugzilla 46302.
4741 //
4742 // TODO: Theoretically we could use op-sel to use the high bits of the
4743 // 32-bit FP values.
4744 return AMDGPU::isInlinableIntLiteral(Literal: Imm);
4745 case AMDGPU::OPERAND_REG_IMM_V2INT16:
4746 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
4747 return AMDGPU::isInlinableLiteralV2I16(Literal: Imm);
4748 case AMDGPU::OPERAND_REG_IMM_V2FP16:
4749 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
4750 return AMDGPU::isInlinableLiteralV2F16(Literal: Imm);
4751 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
4752 return AMDGPU::isPKFMACF16InlineConstant(Literal: Imm, IsGFX11Plus: ST.isGFX11Plus());
4753 case AMDGPU::OPERAND_REG_IMM_V2BF16:
4754 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
4755 return AMDGPU::isInlinableLiteralV2BF16(Literal: Imm);
4756 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
4757 return false;
4758 case AMDGPU::OPERAND_REG_IMM_FP16:
4759 case AMDGPU::OPERAND_REG_INLINE_C_FP16: {
4760 if (isInt<16>(x: Imm) || isUInt<16>(x: Imm)) {
4761 // A few special case instructions have 16-bit operands on subtargets
4762 // where 16-bit instructions are not legal.
4763 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
4764 // constants in these cases
4765 int16_t Trunc = static_cast<int16_t>(Imm);
4766 return ST.has16BitInsts() &&
4767 AMDGPU::isInlinableLiteralFP16(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
4768 }
4769
4770 return false;
4771 }
4772 case AMDGPU::OPERAND_REG_IMM_BF16:
4773 case AMDGPU::OPERAND_REG_INLINE_C_BF16: {
4774 if (isInt<16>(x: Imm) || isUInt<16>(x: Imm)) {
4775 int16_t Trunc = static_cast<int16_t>(Imm);
4776 return ST.has16BitInsts() &&
4777 AMDGPU::isInlinableLiteralBF16(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
4778 }
4779 return false;
4780 }
4781 case AMDGPU::OPERAND_KIMM32:
4782 case AMDGPU::OPERAND_KIMM16:
4783 case AMDGPU::OPERAND_KIMM64:
4784 return false;
4785 case AMDGPU::OPERAND_INLINE_C_AV64_PSEUDO:
4786 return isLegalAV64PseudoImm(Imm);
4787 case AMDGPU::OPERAND_INPUT_MODS:
4788 case MCOI::OPERAND_IMMEDIATE:
4789 // Always embedded in the instruction for free.
4790 return true;
4791 case MCOI::OPERAND_UNKNOWN:
4792 case MCOI::OPERAND_REGISTER:
4793 case MCOI::OPERAND_PCREL:
4794 case MCOI::OPERAND_GENERIC_0:
4795 case MCOI::OPERAND_GENERIC_1:
4796 case MCOI::OPERAND_GENERIC_2:
4797 case MCOI::OPERAND_GENERIC_3:
4798 case MCOI::OPERAND_GENERIC_4:
4799 case MCOI::OPERAND_GENERIC_5:
4800 // Just ignore anything else.
4801 return true;
4802 default:
4803 llvm_unreachable("invalid operand type");
4804 }
4805}
4806
4807static bool compareMachineOp(const MachineOperand &Op0,
4808 const MachineOperand &Op1) {
4809 if (Op0.getType() != Op1.getType())
4810 return false;
4811
4812 switch (Op0.getType()) {
4813 case MachineOperand::MO_Register:
4814 return Op0.getReg() == Op1.getReg();
4815 case MachineOperand::MO_Immediate:
4816 return Op0.getImm() == Op1.getImm();
4817 default:
4818 llvm_unreachable("Didn't expect to be comparing these operand types");
4819 }
4820}
4821
4822bool SIInstrInfo::isLiteralOperandLegal(const MCInstrDesc &InstDesc,
4823 const MCOperandInfo &OpInfo) const {
4824 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
4825 return true;
4826
4827 if (!RI.opCanUseLiteralConstant(OpType: OpInfo.OperandType))
4828 return false;
4829
4830 if (!isVOP3(Desc: InstDesc) || !AMDGPU::isSISrcOperand(OpInfo))
4831 return true;
4832
4833 return ST.hasVOP3Literal();
4834}
4835
4836bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
4837 int64_t ImmVal) const {
4838 const unsigned Opc = InstDesc.getOpcode();
4839 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
4840 if (Src1Idx != -1 && isDPP(Opcode: Opc) && !ST.hasDPPSrc1SGPR() &&
4841 OpNo == static_cast<unsigned>(Src1Idx))
4842 return false;
4843
4844 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
4845 if (isInlineConstant(Imm: ImmVal, OperandType: OpInfo.OperandType)) {
4846 if (isMAI(Desc: InstDesc) && ST.hasMFMAInlineLiteralBug() &&
4847 OpNo == (unsigned)AMDGPU::getNamedOperandIdx(Opcode: InstDesc.getOpcode(),
4848 Name: AMDGPU::OpName::src2))
4849 return false;
4850 return RI.opCanUseInlineConstant(OpType: OpInfo.OperandType);
4851 }
4852
4853 return isLiteralOperandLegal(InstDesc, OpInfo);
4854}
4855
4856bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
4857 const MachineOperand &MO) const {
4858 if (MO.isImm())
4859 return isImmOperandLegal(InstDesc, OpNo, ImmVal: MO.getImm());
4860
4861 assert((MO.isTargetIndex() || MO.isFI() || MO.isGlobal()) &&
4862 "unexpected imm-like operand kind");
4863 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
4864 return isLiteralOperandLegal(InstDesc, OpInfo);
4865}
4866
4867bool SIInstrInfo::isLegalAV64PseudoImm(uint64_t Imm) const {
4868 // 2 32-bit inline constants packed into one.
4869 return AMDGPU::isInlinableLiteral32(Literal: Lo_32(Value: Imm), HasInv2Pi: ST.hasInv2PiInlineImm()) &&
4870 AMDGPU::isInlinableLiteral32(Literal: Hi_32(Value: Imm), HasInv2Pi: ST.hasInv2PiInlineImm());
4871}
4872
4873bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
4874 // GFX90A does not have V_MUL_LEGACY_F32_e32.
4875 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())
4876 return false;
4877
4878 int Op32 = AMDGPU::getVOPe32(Opcode);
4879 if (Op32 == -1)
4880 return false;
4881
4882 return pseudoToMCOpcode(Opcode: Op32) != -1;
4883}
4884
4885bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
4886 // The src0_modifier operand is present on all instructions
4887 // that have modifiers.
4888
4889 return AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0_modifiers);
4890}
4891
4892bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI,
4893 AMDGPU::OpName OpName) const {
4894 const MachineOperand *Mods = getNamedOperand(MI, OperandName: OpName);
4895 return Mods && Mods->getImm();
4896}
4897
4898bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const {
4899 return any_of(Range: ModifierOpNames,
4900 P: [&](AMDGPU::OpName Name) { return hasModifiersSet(MI, OpName: Name); });
4901}
4902
4903bool SIInstrInfo::canShrink(const MachineInstr &MI,
4904 const MachineRegisterInfo &MRI) const {
4905 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
4906 // Can't shrink instruction with three operands.
4907 if (Src2) {
4908 switch (MI.getOpcode()) {
4909 default: return false;
4910
4911 case AMDGPU::V_ADDC_U32_e64:
4912 case AMDGPU::V_SUBB_U32_e64:
4913 case AMDGPU::V_SUBBREV_U32_e64: {
4914 const MachineOperand *Src1
4915 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
4916 if (!Src1->isReg() || !RI.isVGPR(MRI, Reg: Src1->getReg()))
4917 return false;
4918 // Additional verification is needed for sdst/src2.
4919 return true;
4920 }
4921 case AMDGPU::V_MAC_F16_e64:
4922 case AMDGPU::V_MAC_F32_e64:
4923 case AMDGPU::V_MAC_LEGACY_F32_e64:
4924 case AMDGPU::V_FMAC_F16_e64:
4925 case AMDGPU::V_FMAC_F16_t16_e64:
4926 case AMDGPU::V_FMAC_F16_fake16_e64:
4927 case AMDGPU::V_FMAC_F32_e64:
4928 case AMDGPU::V_FMAC_F64_e64:
4929 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4930 if (!Src2->isReg() || !RI.isVGPR(MRI, Reg: Src2->getReg()) ||
4931 hasModifiersSet(MI, OpName: AMDGPU::OpName::src2_modifiers))
4932 return false;
4933 break;
4934
4935 case AMDGPU::V_CNDMASK_B32_e64:
4936 break;
4937 }
4938 }
4939
4940 const MachineOperand *Src1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
4941 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Reg: Src1->getReg()) ||
4942 hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers)))
4943 return false;
4944
4945 // We don't need to check src0, all input types are legal, so just make sure
4946 // src0 isn't using any modifiers.
4947 if (hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers))
4948 return false;
4949
4950 // Can it be shrunk to a valid 32 bit opcode?
4951 if (!hasVALU32BitEncoding(Opcode: MI.getOpcode()))
4952 return false;
4953
4954 // Check output modifiers
4955 return !hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) &&
4956 !hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) &&
4957 !hasModifiersSet(MI, OpName: AMDGPU::OpName::byte_sel) &&
4958 // TODO: Can we avoid checking bound_ctrl/fi here?
4959 // They are only used by permlane*_swap special case.
4960 !hasModifiersSet(MI, OpName: AMDGPU::OpName::bound_ctrl) &&
4961 !hasModifiersSet(MI, OpName: AMDGPU::OpName::fi);
4962}
4963
4964// Set VCC operand with all flags from \p Orig, except for setting it as
4965// implicit.
4966static void copyFlagsToImplicitVCC(MachineInstr &MI,
4967 const MachineOperand &Orig) {
4968
4969 for (MachineOperand &Use : MI.implicit_operands()) {
4970 if (Use.isUse() &&
4971 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) {
4972 Use.setIsUndef(Orig.isUndef());
4973 Use.setIsKill(Orig.isKill());
4974 return;
4975 }
4976 }
4977}
4978
4979MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI,
4980 unsigned Op32) const {
4981 MachineBasicBlock *MBB = MI.getParent();
4982
4983 const MCInstrDesc &Op32Desc = get(Opcode: Op32);
4984 MachineInstrBuilder Inst32 =
4985 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: Op32Desc)
4986 .setMIFlags(MI.getFlags());
4987
4988 // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
4989 // For VOPC instructions, this is replaced by an implicit def of vcc.
4990
4991 // We assume the defs of the shrunk opcode are in the same order, and the
4992 // shrunk opcode loses the last def (SGPR def, in the VOP3->VOPC case).
4993 for (int I = 0, E = Op32Desc.getNumDefs(); I != E; ++I)
4994 Inst32.add(MO: MI.getOperand(i: I));
4995
4996 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
4997
4998 int Idx = MI.getNumExplicitDefs();
4999 for (const MachineOperand &Use : MI.explicit_uses()) {
5000 int OpTy = MI.getDesc().operands()[Idx++].OperandType;
5001 if (OpTy == AMDGPU::OPERAND_INPUT_MODS || OpTy == MCOI::OPERAND_IMMEDIATE)
5002 continue;
5003
5004 if (&Use == Src2) {
5005 if (AMDGPU::getNamedOperandIdx(Opcode: Op32, Name: AMDGPU::OpName::src2) == -1) {
5006 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
5007 // replaced with an implicit read of vcc or vcc_lo. The implicit read
5008 // of vcc was already added during the initial BuildMI, but we
5009 // 1) may need to change vcc to vcc_lo to preserve the original register
5010 // 2) have to preserve the original flags.
5011 copyFlagsToImplicitVCC(MI&: *Inst32, Orig: *Src2);
5012 continue;
5013 }
5014 }
5015
5016 Inst32.add(MO: Use);
5017 }
5018
5019 // FIXME: Losing implicit operands
5020 fixImplicitOperands(MI&: *Inst32);
5021 return Inst32;
5022}
5023
5024bool SIInstrInfo::physRegUsesConstantBus(const MachineOperand &RegOp) const {
5025 // Null is free
5026 Register Reg = RegOp.getReg();
5027 if (Reg == AMDGPU::SGPR_NULL || Reg == AMDGPU::SGPR_NULL64)
5028 return false;
5029
5030 // SGPRs use the constant bus
5031
5032 // FIXME: implicit registers that are not part of the MCInstrDesc's implicit
5033 // physical register operands should also count, except for exec.
5034 if (RegOp.isImplicit())
5035 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::M0;
5036
5037 // SGPRs use the constant bus
5038 return AMDGPU::SReg_32RegClass.contains(Reg) ||
5039 AMDGPU::SReg_64RegClass.contains(Reg);
5040}
5041
5042bool SIInstrInfo::regUsesConstantBus(const MachineOperand &RegOp,
5043 const MachineRegisterInfo &MRI) const {
5044 Register Reg = RegOp.getReg();
5045 return Reg.isVirtual() ? RI.isSGPRClass(RC: MRI.getRegClass(Reg))
5046 : physRegUsesConstantBus(RegOp);
5047}
5048
5049bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,
5050 const MachineOperand &MO,
5051 const MCOperandInfo &OpInfo) const {
5052 // Literal constants use the constant bus.
5053 if (!MO.isReg())
5054 return !isInlineConstant(MO, OpInfo);
5055
5056 Register Reg = MO.getReg();
5057 return Reg.isVirtual() ? RI.isSGPRClass(RC: MRI.getRegClass(Reg))
5058 : physRegUsesConstantBus(RegOp: MO);
5059}
5060
5061static Register findImplicitSGPRRead(const MachineInstr &MI) {
5062 for (const MachineOperand &MO : MI.implicit_operands()) {
5063 // We only care about reads.
5064 if (MO.isDef())
5065 continue;
5066
5067 switch (MO.getReg()) {
5068 case AMDGPU::VCC:
5069 case AMDGPU::VCC_LO:
5070 case AMDGPU::VCC_HI:
5071 case AMDGPU::M0:
5072 case AMDGPU::FLAT_SCR:
5073 return MO.getReg();
5074
5075 default:
5076 break;
5077 }
5078 }
5079
5080 return Register();
5081}
5082
5083static bool shouldReadExec(const MachineInstr &MI) {
5084 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) {
5085 switch (MI.getOpcode()) {
5086 case AMDGPU::V_READLANE_B32:
5087 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
5088 case AMDGPU::V_WRITELANE_B32:
5089 case AMDGPU::SI_SPILL_S32_TO_VGPR:
5090 return false;
5091 }
5092
5093 return true;
5094 }
5095
5096 if (MI.isPreISelOpcode() ||
5097 SIInstrInfo::isGenericOpcode(Opc: MI.getOpcode()) ||
5098 SIInstrInfo::isSALU(MI) ||
5099 SIInstrInfo::isSMRD(MI))
5100 return false;
5101
5102 return true;
5103}
5104
5105static bool isRegOrFI(const MachineOperand &MO) {
5106 return MO.isReg() || MO.isFI();
5107}
5108
5109static bool isSubRegOf(const SIRegisterInfo &TRI,
5110 const MachineOperand &SuperVec,
5111 const MachineOperand &SubReg) {
5112 if (SubReg.getReg().isPhysical())
5113 return TRI.isSubRegister(RegA: SuperVec.getReg(), RegB: SubReg.getReg());
5114
5115 return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
5116 SubReg.getReg() == SuperVec.getReg();
5117}
5118
5119// Verify the illegal copy from vector register to SGPR for generic opcode COPY
5120bool SIInstrInfo::verifyCopy(const MachineInstr &MI,
5121 const MachineRegisterInfo &MRI,
5122 StringRef &ErrInfo) const {
5123 Register DstReg = MI.getOperand(i: 0).getReg();
5124 Register SrcReg = MI.getOperand(i: 1).getReg();
5125 // This is a check for copy from vector register to SGPR
5126 if (RI.isVectorRegister(MRI, Reg: SrcReg) && RI.isSGPRReg(MRI, Reg: DstReg)) {
5127 ErrInfo = "illegal copy from vector register to SGPR";
5128 return false;
5129 }
5130 return true;
5131}
5132
5133bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
5134 StringRef &ErrInfo) const {
5135 uint32_t Opcode = MI.getOpcode();
5136 const MachineFunction *MF = MI.getMF();
5137 const MachineRegisterInfo &MRI = MF->getRegInfo();
5138
5139 // FIXME: At this point the COPY verify is done only for non-ssa forms.
5140 // Find a better property to recognize the point where instruction selection
5141 // is just done.
5142 // We can only enforce this check after SIFixSGPRCopies pass so that the
5143 // illegal copies are legalized and thereafter we don't expect a pass
5144 // inserting similar copies.
5145 if (!MRI.isSSA() && MI.isCopy())
5146 return verifyCopy(MI, MRI, ErrInfo);
5147
5148 if (SIInstrInfo::isGenericOpcode(Opc: Opcode))
5149 return true;
5150
5151 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0);
5152 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src1);
5153 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src2);
5154 int Src3Idx = -1;
5155 if (Src0Idx == -1) {
5156 // VOPD V_DUAL_* instructions use different operand names.
5157 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0X);
5158 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vsrc1X);
5159 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0Y);
5160 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vsrc1Y);
5161 }
5162
5163 // Make sure the number of operands is correct.
5164 const MCInstrDesc &Desc = get(Opcode);
5165 if (!Desc.isVariadic() &&
5166 Desc.getNumOperands() != MI.getNumExplicitOperands()) {
5167 ErrInfo = "Instruction has wrong number of operands.";
5168 return false;
5169 }
5170
5171 if (MI.isInlineAsm()) {
5172 // Verify register classes for inlineasm constraints.
5173 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
5174 I != E; ++I) {
5175 const TargetRegisterClass *RC = MI.getRegClassConstraint(OpIdx: I, TII: this, TRI: &RI);
5176 if (!RC)
5177 continue;
5178
5179 const MachineOperand &Op = MI.getOperand(i: I);
5180 if (!Op.isReg())
5181 continue;
5182
5183 Register Reg = Op.getReg();
5184 if (!Reg.isVirtual() && !RC->contains(Reg)) {
5185 ErrInfo = "inlineasm operand has incorrect register class.";
5186 return false;
5187 }
5188 }
5189
5190 return true;
5191 }
5192
5193 if (isImage(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) {
5194 ErrInfo = "missing memory operand from image instruction.";
5195 return false;
5196 }
5197
5198 // Make sure the register classes are correct.
5199 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
5200 const MachineOperand &MO = MI.getOperand(i);
5201 if (MO.isFPImm()) {
5202 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
5203 "all fp values to integers.";
5204 return false;
5205 }
5206
5207 const MCOperandInfo &OpInfo = Desc.operands()[i];
5208 int16_t RegClass = getOpRegClassID(OpInfo);
5209
5210 switch (OpInfo.OperandType) {
5211 case MCOI::OPERAND_REGISTER:
5212 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
5213 ErrInfo = "Illegal immediate value for operand.";
5214 return false;
5215 }
5216 break;
5217 case AMDGPU::OPERAND_REG_IMM_INT32:
5218 case AMDGPU::OPERAND_REG_IMM_INT16:
5219 case AMDGPU::OPERAND_REG_IMM_FP32:
5220 case AMDGPU::OPERAND_REG_IMM_BF16:
5221 case AMDGPU::OPERAND_REG_IMM_FP16:
5222 case AMDGPU::OPERAND_REG_IMM_V2FP16:
5223 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
5224 case AMDGPU::OPERAND_REG_IMM_V2INT16:
5225 case AMDGPU::OPERAND_REG_IMM_V2BF16:
5226 case AMDGPU::OPERAND_REG_IMM_V2FP64:
5227 case AMDGPU::OPERAND_REG_IMM_V2INT64:
5228 break;
5229 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
5230 break;
5231 break;
5232 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
5233 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
5234 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
5235 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
5236 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
5237 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
5238 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
5239 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
5240 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
5241 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
5242 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
5243 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
5244 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: {
5245 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, OpIdx: i))) {
5246 ErrInfo = "Illegal immediate value for operand.";
5247 return false;
5248 }
5249 break;
5250 }
5251 case AMDGPU::OPERAND_REG_IMM_FP64:
5252 case AMDGPU::OPERAND_REG_IMM_INT64:
5253 case AMDGPU::OPERAND_REG_IMM_V2INT32:
5254 case AMDGPU::OPERAND_REG_IMM_V2FP32:
5255 if (ST.has64BitLiterals() && Desc.getSize() != 4 && MO.isImm() &&
5256 !isInlineConstant(MI, OpIdx: i) &&
5257 !AMDGPU::isValid32BitLiteral(Val: MO.getImm(),
5258 IsFP64: OpInfo.OperandType ==
5259 AMDGPU::OPERAND_REG_IMM_FP64)) {
5260 ErrInfo = "illegal 64-bit immediate value for operand.";
5261 return false;
5262 }
5263 break;
5264 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
5265 case AMDGPU::OPERAND_INPUT_MODS:
5266 if (!MI.getOperand(i).isImm() || !isInlineConstant(MI, OpIdx: i)) {
5267 ErrInfo = "Expected inline constant for operand.";
5268 return false;
5269 }
5270 break;
5271 case AMDGPU::OPERAND_SDWA_VOPC_DST:
5272 case AMDGPU::OPERAND_KIMM16:
5273 break;
5274 case MCOI::OPERAND_IMMEDIATE:
5275 case AMDGPU::OPERAND_KIMM32:
5276 case AMDGPU::OPERAND_KIMM64:
5277 case AMDGPU::OPERAND_INLINE_C_AV64_PSEUDO:
5278 // Check if this operand is an immediate.
5279 // FrameIndex operands will be replaced by immediates, so they are
5280 // allowed.
5281 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
5282 ErrInfo = "Expected immediate, but got non-immediate";
5283 return false;
5284 }
5285 break;
5286 case MCOI::OPERAND_UNKNOWN:
5287 case MCOI::OPERAND_MEMORY:
5288 case MCOI::OPERAND_PCREL:
5289 break;
5290 default:
5291 if (OpInfo.isGenericType())
5292 continue;
5293 break;
5294 }
5295
5296 if (!MO.isReg())
5297 continue;
5298 Register Reg = MO.getReg();
5299 if (!Reg)
5300 continue;
5301
5302 // FIXME: Ideally we would have separate instruction definitions with the
5303 // aligned register constraint.
5304 // FIXME: We do not verify inline asm operands, but custom inline asm
5305 // verification is broken anyway
5306 if (ST.needsAlignedVGPRs() && Opcode != AMDGPU::AV_MOV_B64_IMM_PSEUDO &&
5307 Opcode != AMDGPU::V_MOV_B64_PSEUDO && !isSpill(MI)) {
5308 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI, Reg);
5309 if (RI.hasVectorRegisters(RC) && MO.getSubReg()) {
5310 if (const TargetRegisterClass *SubRC =
5311 RI.getSubRegisterClass(RC, MO.getSubReg())) {
5312 RC = RI.getCompatibleSubRegClass(SuperRC: RC, SubRC, SubIdx: MO.getSubReg());
5313 if (RC)
5314 RC = SubRC;
5315 }
5316 }
5317
5318 // Check that this is the aligned version of the class.
5319 if (!RC || !RI.isProperlyAlignedRC(RC: *RC)) {
5320 ErrInfo = "Subtarget requires even aligned vector registers";
5321 return false;
5322 }
5323 }
5324
5325 if (RegClass != -1) {
5326 if (Reg.isVirtual())
5327 continue;
5328
5329 const TargetRegisterClass *RC = RI.getRegClass(i: RegClass);
5330 if (!RC->contains(Reg)) {
5331 ErrInfo = "Operand has incorrect register class.";
5332 return false;
5333 }
5334 }
5335 }
5336
5337 // Verify SDWA
5338 if (isSDWA(MI)) {
5339 if (!ST.hasSDWA()) {
5340 ErrInfo = "SDWA is not supported on this target";
5341 return false;
5342 }
5343
5344 for (auto Op : {AMDGPU::OpName::src0_sel, AMDGPU::OpName::src1_sel,
5345 AMDGPU::OpName::dst_sel}) {
5346 const MachineOperand *MO = getNamedOperand(MI, OperandName: Op);
5347 if (!MO)
5348 continue;
5349 int64_t Imm = MO->getImm();
5350 if (Imm < 0 || Imm > AMDGPU::SDWA::SdwaSel::DWORD) {
5351 ErrInfo = "Invalid SDWA selection";
5352 return false;
5353 }
5354 }
5355
5356 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdst);
5357
5358 for (int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) {
5359 if (OpIdx == -1)
5360 continue;
5361 const MachineOperand &MO = MI.getOperand(i: OpIdx);
5362
5363 if (!ST.hasSDWAScalar()) {
5364 // Only VGPRS on VI
5365 if (!MO.isReg() || !RI.hasVGPRs(RC: RI.getRegClassForReg(MRI, Reg: MO.getReg()))) {
5366 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
5367 return false;
5368 }
5369 } else {
5370 // No immediates on GFX9
5371 if (!MO.isReg()) {
5372 ErrInfo =
5373 "Only reg allowed as operands in SDWA instructions on GFX9+";
5374 return false;
5375 }
5376 }
5377 }
5378
5379 if (!ST.hasSDWAOmod()) {
5380 // No omod allowed on VI
5381 const MachineOperand *OMod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
5382 if (OMod != nullptr &&
5383 (!OMod->isImm() || OMod->getImm() != 0)) {
5384 ErrInfo = "OMod not allowed in SDWA instructions on VI";
5385 return false;
5386 }
5387 }
5388
5389 if (Opcode == AMDGPU::V_CVT_F32_FP8_sdwa ||
5390 Opcode == AMDGPU::V_CVT_F32_BF8_sdwa ||
5391 Opcode == AMDGPU::V_CVT_PK_F32_FP8_sdwa ||
5392 Opcode == AMDGPU::V_CVT_PK_F32_BF8_sdwa) {
5393 const MachineOperand *Src0ModsMO =
5394 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
5395 unsigned Mods = Src0ModsMO->getImm();
5396 if (Mods & SISrcMods::ABS || Mods & SISrcMods::NEG ||
5397 Mods & SISrcMods::SEXT) {
5398 ErrInfo = "sext, abs and neg are not allowed on this instruction";
5399 return false;
5400 }
5401 }
5402
5403 uint32_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
5404 if (isVOPC(Opcode: BasicOpcode)) {
5405 if (!ST.hasSDWASdst() && DstIdx != -1) {
5406 // Only vcc allowed as dst on VI for VOPC
5407 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5408 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
5409 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
5410 return false;
5411 }
5412 } else if (!ST.hasSDWAOutModsVOPC()) {
5413 // No clamp allowed on GFX9 for VOPC
5414 const MachineOperand *Clamp = getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
5415 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
5416 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
5417 return false;
5418 }
5419
5420 // No omod allowed on GFX9 for VOPC
5421 const MachineOperand *OMod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
5422 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
5423 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
5424 return false;
5425 }
5426 }
5427 }
5428
5429 const MachineOperand *DstUnused = getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_unused);
5430 if (DstUnused && DstUnused->isImm() &&
5431 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
5432 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5433 if (!Dst.isReg() || !Dst.isTied()) {
5434 ErrInfo = "Dst register should have tied register";
5435 return false;
5436 }
5437
5438 const MachineOperand &TiedMO =
5439 MI.getOperand(i: MI.findTiedOperandIdx(OpIdx: DstIdx));
5440 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
5441 ErrInfo =
5442 "Dst register should be tied to implicit use of preserved register";
5443 return false;
5444 }
5445 if (TiedMO.getReg().isPhysical() && Dst.getReg() != TiedMO.getReg()) {
5446 ErrInfo = "Dst register should use same physical register as preserved";
5447 return false;
5448 }
5449 }
5450 }
5451
5452 if (isDPP(MI) && !ST.hasDPPSrc1SGPR() && Src1Idx != -1) {
5453 const MachineOperand &Src1MO = MI.getOperand(i: Src1Idx);
5454 if (Src1MO.isReg() && RI.isSGPRReg(MRI, Reg: Src1MO.getReg())) {
5455 ErrInfo = "DPP src1 cannot be SGPR on this subtarget";
5456 return false;
5457 }
5458 if (Src1MO.isImm()) {
5459 ErrInfo = "DPP src1 cannot be an immediate on this subtarget";
5460 return false;
5461 }
5462 }
5463
5464 // Verify MIMG / VIMAGE / VSAMPLE
5465 if (isImage(Opcode) && !MI.mayStore()) {
5466 // Ensure that the return type used is large enough for all the options
5467 // being used TFE/LWE require an extra result register.
5468 const MachineOperand *DMask = getNamedOperand(MI, OperandName: AMDGPU::OpName::dmask);
5469 if (DMask) {
5470 uint64_t DMaskImm = DMask->getImm();
5471 uint32_t RegCount = isGather4(Opcode) ? 4 : llvm::popcount(Value: DMaskImm);
5472 const MachineOperand *TFE = getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe);
5473 const MachineOperand *LWE = getNamedOperand(MI, OperandName: AMDGPU::OpName::lwe);
5474 const MachineOperand *D16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::d16);
5475
5476 // Adjust for packed 16 bit values
5477 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
5478 RegCount = divideCeil(Numerator: RegCount, Denominator: 2);
5479
5480 // Adjust if using LWE or TFE
5481 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
5482 RegCount += 1;
5483
5484 const uint32_t DstIdx =
5485 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdata);
5486 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5487 if (Dst.isReg()) {
5488 const TargetRegisterClass *DstRC = getOpRegClass(MI, OpNo: DstIdx);
5489 uint32_t DstSize = RI.getRegSizeInBits(RC: *DstRC) / 32;
5490 if (RegCount > DstSize) {
5491 ErrInfo = "Image instruction returns too many registers for dst "
5492 "register class";
5493 return false;
5494 }
5495 }
5496 }
5497 }
5498
5499 // Verify VOP*. Ignore multiple sgpr operands on writelane.
5500 if (isVALU(MI, /*AllowLDSDMA=*/true) &&
5501 Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
5502 unsigned ConstantBusCount = 0;
5503 bool UsesLiteral = false;
5504 const MachineOperand *LiteralVal = nullptr;
5505
5506 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::imm);
5507 if (ImmIdx != -1) {
5508 ++ConstantBusCount;
5509 UsesLiteral = true;
5510 LiteralVal = &MI.getOperand(i: ImmIdx);
5511 }
5512
5513 SmallVector<Register, 2> SGPRsUsed;
5514 Register SGPRUsed;
5515
5516 // Only look at the true operands. Only a real operand can use the constant
5517 // bus, and we don't want to check pseudo-operands like the source modifier
5518 // flags.
5519 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) {
5520 if (OpIdx == -1)
5521 continue;
5522 const MachineOperand &MO = MI.getOperand(i: OpIdx);
5523 if (usesConstantBus(MRI, MO, OpInfo: MI.getDesc().operands()[OpIdx])) {
5524 if (MO.isReg()) {
5525 SGPRUsed = MO.getReg();
5526 if (!llvm::is_contained(Range&: SGPRsUsed, Element: SGPRUsed)) {
5527 ++ConstantBusCount;
5528 SGPRsUsed.push_back(Elt: SGPRUsed);
5529 }
5530 } else if (!MO.isFI()) { // Treat FI like a register.
5531 if (!UsesLiteral) {
5532 ++ConstantBusCount;
5533 UsesLiteral = true;
5534 LiteralVal = &MO;
5535 } else if (!MO.isIdenticalTo(Other: *LiteralVal)) {
5536 assert(isVOP2(MI) || isVOP3(MI));
5537 ErrInfo = "VOP2/VOP3 instruction uses more than one literal";
5538 return false;
5539 }
5540 }
5541 }
5542 }
5543
5544 SGPRUsed = findImplicitSGPRRead(MI);
5545 if (SGPRUsed) {
5546 // Implicit uses may safely overlap true operands
5547 if (llvm::all_of(Range&: SGPRsUsed, P: [this, SGPRUsed](unsigned SGPR) {
5548 return !RI.regsOverlap(RegA: SGPRUsed, RegB: SGPR);
5549 })) {
5550 ++ConstantBusCount;
5551 SGPRsUsed.push_back(Elt: SGPRUsed);
5552 }
5553 }
5554
5555 // v_writelane_b32 is an exception from constant bus restriction:
5556 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
5557 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
5558 Opcode != AMDGPU::V_WRITELANE_B32) {
5559 ErrInfo = "VOP* instruction violates constant bus restriction";
5560 return false;
5561 }
5562
5563 if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) {
5564 ErrInfo = "VOP3 instruction uses literal";
5565 return false;
5566 }
5567 }
5568
5569 // Special case for writelane - this can break the multiple constant bus rule,
5570 // but still can't use more than one SGPR register
5571 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
5572 unsigned SGPRCount = 0;
5573 Register SGPRUsed;
5574
5575 for (int OpIdx : {Src0Idx, Src1Idx}) {
5576 if (OpIdx == -1)
5577 break;
5578
5579 const MachineOperand &MO = MI.getOperand(i: OpIdx);
5580
5581 if (usesConstantBus(MRI, MO, OpInfo: MI.getDesc().operands()[OpIdx])) {
5582 if (MO.isReg() && MO.getReg() != AMDGPU::M0) {
5583 if (MO.getReg() != SGPRUsed)
5584 ++SGPRCount;
5585 SGPRUsed = MO.getReg();
5586 }
5587 }
5588 if (SGPRCount > ST.getConstantBusLimit(Opcode)) {
5589 ErrInfo = "WRITELANE instruction violates constant bus restriction";
5590 return false;
5591 }
5592 }
5593 }
5594
5595 // Verify misc. restrictions on specific instructions.
5596 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
5597 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
5598 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5599 const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
5600 const MachineOperand &Src2 = MI.getOperand(i: Src2Idx);
5601 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
5602 if (!compareMachineOp(Op0: Src0, Op1: Src1) &&
5603 !compareMachineOp(Op0: Src0, Op1: Src2)) {
5604 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
5605 return false;
5606 }
5607 }
5608 if ((getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers)->getImm() &
5609 SISrcMods::ABS) ||
5610 (getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers)->getImm() &
5611 SISrcMods::ABS) ||
5612 (getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers)->getImm() &
5613 SISrcMods::ABS)) {
5614 ErrInfo = "ABS not allowed in VOP3B instructions";
5615 return false;
5616 }
5617 }
5618
5619 if (isSOP2(MI) || isSOPC(MI)) {
5620 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5621 const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
5622
5623 if (!isRegOrFI(MO: Src0) && !isRegOrFI(MO: Src1) &&
5624 !isInlineConstant(MO: Src0, OpInfo: Desc.operands()[Src0Idx]) &&
5625 !isInlineConstant(MO: Src1, OpInfo: Desc.operands()[Src1Idx]) &&
5626 !Src0.isIdenticalTo(Other: Src1)) {
5627 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
5628 return false;
5629 }
5630 }
5631
5632 if (isSOPK(MI)) {
5633 const auto *Op = getNamedOperand(MI, OperandName: AMDGPU::OpName::simm16);
5634 if (Desc.isBranch()) {
5635 if (!Op->isMBB()) {
5636 ErrInfo = "invalid branch target for SOPK instruction";
5637 return false;
5638 }
5639 } else {
5640 uint64_t Imm = Op->getImm();
5641 if (sopkIsZext(Opcode)) {
5642 if (!isUInt<16>(x: Imm)) {
5643 ErrInfo = "invalid immediate for SOPK instruction";
5644 return false;
5645 }
5646 } else {
5647 if (!isInt<16>(x: Imm)) {
5648 ErrInfo = "invalid immediate for SOPK instruction";
5649 return false;
5650 }
5651 }
5652 }
5653 }
5654
5655 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
5656 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
5657 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
5658 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
5659 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
5660 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
5661
5662 const unsigned StaticNumOps =
5663 Desc.getNumOperands() + Desc.implicit_uses().size();
5664 const unsigned NumImplicitOps = IsDst ? 2 : 1;
5665
5666 // Require additional implicit operands. This allows a fixup done by the
5667 // post RA scheduler where the main implicit operand is killed and
5668 // implicit-defs are added for sub-registers that remain live after this
5669 // instruction.
5670 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
5671 ErrInfo = "missing implicit register operands";
5672 return false;
5673 }
5674
5675 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
5676 if (IsDst) {
5677 if (!Dst->isUse()) {
5678 ErrInfo = "v_movreld_b32 vdst should be a use operand";
5679 return false;
5680 }
5681
5682 unsigned UseOpIdx;
5683 if (!MI.isRegTiedToUseOperand(DefOpIdx: StaticNumOps, UseOpIdx: &UseOpIdx) ||
5684 UseOpIdx != StaticNumOps + 1) {
5685 ErrInfo = "movrel implicit operands should be tied";
5686 return false;
5687 }
5688 }
5689
5690 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5691 const MachineOperand &ImpUse
5692 = MI.getOperand(i: StaticNumOps + NumImplicitOps - 1);
5693 if (!ImpUse.isReg() || !ImpUse.isUse() ||
5694 !isSubRegOf(TRI: RI, SuperVec: ImpUse, SubReg: IsDst ? *Dst : Src0)) {
5695 ErrInfo = "src0 should be subreg of implicit vector use";
5696 return false;
5697 }
5698 }
5699
5700 // Make sure we aren't losing exec uses in the td files. This mostly requires
5701 // being careful when using let Uses to try to add other use registers.
5702 if (shouldReadExec(MI)) {
5703 if (!MI.hasRegisterImplicitUseOperand(Reg: AMDGPU::EXEC)) {
5704 ErrInfo = "VALU instruction does not implicitly read exec mask";
5705 return false;
5706 }
5707 }
5708
5709 if (isSMRD(MI)) {
5710 if (MI.mayStore() &&
5711 ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) {
5712 // The register offset form of scalar stores may only use m0 as the
5713 // soffset register.
5714 const MachineOperand *Soff = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
5715 if (Soff && Soff->getReg() != AMDGPU::M0) {
5716 ErrInfo = "scalar stores must use m0 as offset register";
5717 return false;
5718 }
5719 }
5720 }
5721
5722 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) {
5723 const MachineOperand *Offset = getNamedOperand(MI, OperandName: AMDGPU::OpName::offset);
5724 if (Offset->getImm() != 0) {
5725 ErrInfo = "subtarget does not support offsets in flat instructions";
5726 return false;
5727 }
5728 }
5729
5730 if (isDS(MI) && !ST.hasGDS()) {
5731 const MachineOperand *GDSOp = getNamedOperand(MI, OperandName: AMDGPU::OpName::gds);
5732 if (GDSOp && GDSOp->getImm() != 0) {
5733 ErrInfo = "GDS is not supported on this subtarget";
5734 return false;
5735 }
5736 }
5737
5738 if (isImage(MI)) {
5739 const MachineOperand *DimOp = getNamedOperand(MI, OperandName: AMDGPU::OpName::dim);
5740 if (DimOp) {
5741 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
5742 Name: AMDGPU::OpName::vaddr0);
5743 AMDGPU::OpName RSrcOpName =
5744 isMIMG(MI) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
5745 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: RSrcOpName);
5746 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: Opcode);
5747 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
5748 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
5749 const AMDGPU::MIMGDimInfo *Dim =
5750 AMDGPU::getMIMGDimInfoByEncoding(DimEnc: DimOp->getImm());
5751
5752 if (!Dim) {
5753 ErrInfo = "dim is out of range";
5754 return false;
5755 }
5756
5757 bool IsA16 = false;
5758 if (ST.hasR128A16()) {
5759 const MachineOperand *R128A16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::r128);
5760 IsA16 = R128A16->getImm() != 0;
5761 } else if (ST.hasA16()) {
5762 const MachineOperand *A16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::a16);
5763 IsA16 = A16->getImm() != 0;
5764 }
5765
5766 bool IsNSA = RsrcIdx - VAddr0Idx > 1;
5767
5768 unsigned AddrWords =
5769 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, IsG16Supported: ST.hasG16());
5770
5771 unsigned VAddrWords;
5772 if (IsNSA) {
5773 VAddrWords = RsrcIdx - VAddr0Idx;
5774 if (ST.hasPartialNSAEncoding() &&
5775 AddrWords > ST.getNSAMaxSize(HasSampler: isVSAMPLE(MI))) {
5776 unsigned LastVAddrIdx = RsrcIdx - 1;
5777 VAddrWords += getOpSize(MI, OpNo: LastVAddrIdx) / 4 - 1;
5778 }
5779 } else {
5780 VAddrWords = getOpSize(MI, OpNo: VAddr0Idx) / 4;
5781 if (AddrWords > 12)
5782 AddrWords = 16;
5783 }
5784
5785 if (VAddrWords != AddrWords) {
5786 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords
5787 << " but got " << VAddrWords << "\n");
5788 ErrInfo = "bad vaddr size";
5789 return false;
5790 }
5791 }
5792 }
5793
5794 const MachineOperand *DppCt = getNamedOperand(MI, OperandName: AMDGPU::OpName::dpp_ctrl);
5795 if (DppCt) {
5796 using namespace AMDGPU::DPP;
5797
5798 unsigned DC = DppCt->getImm();
5799 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
5800 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
5801 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) ||
5802 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) ||
5803 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) ||
5804 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) ||
5805 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) {
5806 ErrInfo = "Invalid dpp_ctrl value";
5807 return false;
5808 }
5809 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
5810 !ST.hasDPPWavefrontShifts()) {
5811 ErrInfo = "Invalid dpp_ctrl value: "
5812 "wavefront shifts are not supported on GFX10+";
5813 return false;
5814 }
5815 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
5816 !ST.hasDPPBroadcasts()) {
5817 ErrInfo = "Invalid dpp_ctrl value: "
5818 "broadcasts are not supported on GFX10+";
5819 return false;
5820 }
5821 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST &&
5822 ST.getGeneration() < AMDGPUSubtarget::GFX10) {
5823 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST &&
5824 DC <= DppCtrl::ROW_NEWBCAST_LAST &&
5825 !ST.hasGFX90AInsts()) {
5826 ErrInfo = "Invalid dpp_ctrl value: "
5827 "row_newbroadcast/row_share is not supported before "
5828 "GFX90A/GFX10";
5829 return false;
5830 }
5831 if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) {
5832 ErrInfo = "Invalid dpp_ctrl value: "
5833 "row_share and row_xmask are not supported before GFX10";
5834 return false;
5835 }
5836 }
5837
5838 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&
5839 !AMDGPU::isLegalDPALU_DPPControl(ST, DC) &&
5840 AMDGPU::isDPALU_DPP(OpDesc: Desc, MII: *this, ST)) {
5841 ErrInfo = "Invalid dpp_ctrl value: "
5842 "DP ALU dpp only support row_newbcast";
5843 return false;
5844 }
5845 }
5846
5847 if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) {
5848 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
5849 AMDGPU::OpName DataName =
5850 isDS(Opcode) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata;
5851 const MachineOperand *Data = getNamedOperand(MI, OperandName: DataName);
5852 const MachineOperand *Data2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::data1);
5853 if (Data && !Data->isReg())
5854 Data = nullptr;
5855
5856 if (ST.hasGFX90AInsts()) {
5857 if (Dst && Data && !Dst->isTied() && !Data->isTied() &&
5858 (RI.isAGPR(MRI, Reg: Dst->getReg()) != RI.isAGPR(MRI, Reg: Data->getReg()))) {
5859 ErrInfo = "Invalid register class: "
5860 "vdata and vdst should be both VGPR or AGPR";
5861 return false;
5862 }
5863 if (Data && Data2 &&
5864 (RI.isAGPR(MRI, Reg: Data->getReg()) != RI.isAGPR(MRI, Reg: Data2->getReg()))) {
5865 ErrInfo = "Invalid register class: "
5866 "both data operands should be VGPR or AGPR";
5867 return false;
5868 }
5869 } else {
5870 if ((Dst && RI.isAGPR(MRI, Reg: Dst->getReg())) ||
5871 (Data && RI.isAGPR(MRI, Reg: Data->getReg())) ||
5872 (Data2 && RI.isAGPR(MRI, Reg: Data2->getReg()))) {
5873 ErrInfo = "Invalid register class: "
5874 "agpr loads and stores not supported on this GPU";
5875 return false;
5876 }
5877 }
5878 }
5879
5880 if (ST.needsAlignedVGPRs()) {
5881 const auto isAlignedReg = [&MI, &MRI, this](AMDGPU::OpName OpName) -> bool {
5882 const MachineOperand *Op = getNamedOperand(MI, OperandName: OpName);
5883 if (!Op)
5884 return true;
5885 Register Reg = Op->getReg();
5886 if (Reg.isPhysical())
5887 return !(RI.getHWRegIndex(Reg) & 1);
5888 const TargetRegisterClass &RC = *MRI.getRegClass(Reg);
5889 return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
5890 !(RI.getChannelFromSubReg(SubReg: Op->getSubReg()) & 1);
5891 };
5892
5893 if (Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_SEMA_BR ||
5894 Opcode == AMDGPU::DS_GWS_BARRIER) {
5895
5896 if (!isAlignedReg(AMDGPU::OpName::data0)) {
5897 ErrInfo = "Subtarget requires even aligned vector registers "
5898 "for DS_GWS instructions";
5899 return false;
5900 }
5901 }
5902
5903 if (isMIMG(MI)) {
5904 if (!isAlignedReg(AMDGPU::OpName::vaddr)) {
5905 ErrInfo = "Subtarget requires even aligned vector registers "
5906 "for vaddr operand of image instructions";
5907 return false;
5908 }
5909 }
5910 }
5911
5912 if (Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts()) {
5913 const MachineOperand *Src = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
5914 if (Src->isReg() && RI.isSGPRReg(MRI, Reg: Src->getReg())) {
5915 ErrInfo = "Invalid register class: "
5916 "v_accvgpr_write with an SGPR is not supported on this GPU";
5917 return false;
5918 }
5919 }
5920
5921 if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) {
5922 const MachineOperand &SrcOp = MI.getOperand(i: 1);
5923 if (!SrcOp.isReg() || SrcOp.getReg().isVirtual()) {
5924 ErrInfo = "pseudo expects only physical SGPRs";
5925 return false;
5926 }
5927 }
5928
5929 if (const MachineOperand *CPol = getNamedOperand(MI, OperandName: AMDGPU::OpName::cpol)) {
5930 if (CPol->getImm() & AMDGPU::CPol::SCAL) {
5931 if (!ST.hasScaleOffset()) {
5932 ErrInfo = "Subtarget does not support offset scaling";
5933 return false;
5934 }
5935 if (!AMDGPU::supportsScaleOffset(MII: *this, Opcode: MI.getOpcode())) {
5936 ErrInfo = "Instruction does not support offset scaling";
5937 return false;
5938 }
5939 }
5940 }
5941
5942 // See SIInstrInfo::isLegalGFX12PlusPackedMathFP32or64BitOperand for more
5943 // information.
5944 if (AMDGPU::isPackedFP32or64BitInst(Opc: Opcode) && AMDGPU::isGFX12Plus(STI: ST)) {
5945 for (unsigned I = 0; I < 3; ++I) {
5946 if (!isLegalGFX12PlusPackedMathFP32or64BitOperand(MRI, MI, SrcN: I))
5947 return false;
5948 }
5949 }
5950
5951 if (ST.hasFlatScratchHiInB64InstHazard() && isSALU(MI) &&
5952 MI.readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, TRI: nullptr)) {
5953 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
5954 if ((Dst && RI.getRegClassForReg(MRI, Reg: Dst->getReg()) ==
5955 &AMDGPU::SReg_64RegClass) ||
5956 Opcode == AMDGPU::S_BITCMP0_B64 || Opcode == AMDGPU::S_BITCMP1_B64) {
5957 ErrInfo = "Instruction cannot read flat_scratch_base_hi";
5958 return false;
5959 }
5960 }
5961
5962 return true;
5963}
5964
5965unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
5966 if (MI.getOpcode() == AMDGPU::S_MOV_B32) {
5967 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
5968 return MI.getOperand(i: 1).isReg() || RI.isAGPR(MRI, Reg: MI.getOperand(i: 0).getReg())
5969 ? AMDGPU::COPY
5970 : AMDGPU::V_MOV_B32_e32;
5971 }
5972 return getVALUOp(Opc: MI.getOpcode());
5973}
5974
5975// It is more readable to list mapped opcodes on the same line.
5976// clang-format off
5977
5978unsigned SIInstrInfo::getVALUOp(unsigned Opc) const {
5979 switch (Opc) {
5980 default: return AMDGPU::INSTRUCTION_LIST_END;
5981 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
5982 case AMDGPU::COPY: return AMDGPU::COPY;
5983 case AMDGPU::PHI: return AMDGPU::PHI;
5984 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
5985 case AMDGPU::WQM: return AMDGPU::WQM;
5986 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
5987 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM;
5988 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM;
5989 case AMDGPU::S_ADD_I32:
5990 return ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
5991 case AMDGPU::S_ADDC_U32:
5992 return AMDGPU::V_ADDC_U32_e32;
5993 case AMDGPU::S_SUB_I32:
5994 return ST.hasAddNoCarryInsts() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32;
5995 // FIXME: These are not consistently handled, and selected when the carry is
5996 // used.
5997 case AMDGPU::S_ADD_U32:
5998 return AMDGPU::V_ADD_CO_U32_e32;
5999 case AMDGPU::S_SUB_U32:
6000 return AMDGPU::V_SUB_CO_U32_e32;
6001 case AMDGPU::S_ADD_U64_PSEUDO:
6002 return AMDGPU::V_ADD_U64_PSEUDO;
6003 case AMDGPU::S_SUB_U64_PSEUDO:
6004 return AMDGPU::V_SUB_U64_PSEUDO;
6005 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
6006 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64;
6007 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64;
6008 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64;
6009 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
6010 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
6011 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
6012 case AMDGPU::S_XNOR_B32:
6013 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
6014 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
6015 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
6016 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
6017 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
6018 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
6019 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64;
6020 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
6021 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64;
6022 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
6023 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64;
6024 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64;
6025 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64;
6026 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64;
6027 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64;
6028 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
6029 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
6030 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
6031 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
6032 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64;
6033 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64;
6034 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64;
6035 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64;
6036 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64;
6037 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64;
6038 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64;
6039 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64;
6040 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64;
6041 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64;
6042 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64;
6043 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64;
6044 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64;
6045 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64;
6046 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
6047 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
6048 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
6049 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
6050 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
6051 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
6052 case AMDGPU::S_CVT_F32_I32: return AMDGPU::V_CVT_F32_I32_e64;
6053 case AMDGPU::S_CVT_F32_U32: return AMDGPU::V_CVT_F32_U32_e64;
6054 case AMDGPU::S_CVT_I32_F32: return AMDGPU::V_CVT_I32_F32_e64;
6055 case AMDGPU::S_CVT_U32_F32: return AMDGPU::V_CVT_U32_F32_e64;
6056 case AMDGPU::S_CVT_F32_F16:
6057 case AMDGPU::S_CVT_HI_F32_F16:
6058 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F32_F16_t16_e64
6059 : AMDGPU::V_CVT_F32_F16_fake16_e64;
6060 case AMDGPU::S_CVT_F16_F32:
6061 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F16_F32_t16_e64
6062 : AMDGPU::V_CVT_F16_F32_fake16_e64;
6063 case AMDGPU::S_CEIL_F32: return AMDGPU::V_CEIL_F32_e64;
6064 case AMDGPU::S_FLOOR_F32: return AMDGPU::V_FLOOR_F32_e64;
6065 case AMDGPU::S_TRUNC_F32: return AMDGPU::V_TRUNC_F32_e64;
6066 case AMDGPU::S_RNDNE_F32: return AMDGPU::V_RNDNE_F32_e64;
6067 case AMDGPU::S_CEIL_F16:
6068 return ST.useRealTrue16Insts() ? AMDGPU::V_CEIL_F16_t16_e64
6069 : AMDGPU::V_CEIL_F16_fake16_e64;
6070 case AMDGPU::S_FLOOR_F16:
6071 return ST.useRealTrue16Insts() ? AMDGPU::V_FLOOR_F16_t16_e64
6072 : AMDGPU::V_FLOOR_F16_fake16_e64;
6073 case AMDGPU::S_TRUNC_F16:
6074 return ST.useRealTrue16Insts() ? AMDGPU::V_TRUNC_F16_t16_e64
6075 : AMDGPU::V_TRUNC_F16_fake16_e64;
6076 case AMDGPU::S_RNDNE_F16:
6077 return ST.useRealTrue16Insts() ? AMDGPU::V_RNDNE_F16_t16_e64
6078 : AMDGPU::V_RNDNE_F16_fake16_e64;
6079 case AMDGPU::S_ADD_F32: return AMDGPU::V_ADD_F32_e64;
6080 case AMDGPU::S_SUB_F32: return AMDGPU::V_SUB_F32_e64;
6081 case AMDGPU::S_MIN_F32: return AMDGPU::V_MIN_F32_e64;
6082 case AMDGPU::S_MAX_F32: return AMDGPU::V_MAX_F32_e64;
6083 case AMDGPU::S_MINIMUM_F32: return AMDGPU::V_MINIMUM_F32_e64;
6084 case AMDGPU::S_MAXIMUM_F32: return AMDGPU::V_MAXIMUM_F32_e64;
6085 case AMDGPU::S_MUL_F32: return AMDGPU::V_MUL_F32_e64;
6086 case AMDGPU::S_ADD_F16:
6087 return ST.useRealTrue16Insts() ? AMDGPU::V_ADD_F16_t16_e64
6088 : AMDGPU::V_ADD_F16_fake16_e64;
6089 case AMDGPU::S_SUB_F16:
6090 return ST.useRealTrue16Insts() ? AMDGPU::V_SUB_F16_t16_e64
6091 : AMDGPU::V_SUB_F16_fake16_e64;
6092 case AMDGPU::S_MIN_F16:
6093 return ST.useRealTrue16Insts() ? AMDGPU::V_MIN_F16_t16_e64
6094 : AMDGPU::V_MIN_F16_fake16_e64;
6095 case AMDGPU::S_MAX_F16:
6096 return ST.useRealTrue16Insts() ? AMDGPU::V_MAX_F16_t16_e64
6097 : AMDGPU::V_MAX_F16_fake16_e64;
6098 case AMDGPU::S_MINIMUM_F16:
6099 return ST.useRealTrue16Insts() ? AMDGPU::V_MINIMUM_F16_t16_e64
6100 : AMDGPU::V_MINIMUM_F16_fake16_e64;
6101 case AMDGPU::S_MAXIMUM_F16:
6102 return ST.useRealTrue16Insts() ? AMDGPU::V_MAXIMUM_F16_t16_e64
6103 : AMDGPU::V_MAXIMUM_F16_fake16_e64;
6104 case AMDGPU::S_MUL_F16:
6105 return ST.useRealTrue16Insts() ? AMDGPU::V_MUL_F16_t16_e64
6106 : AMDGPU::V_MUL_F16_fake16_e64;
6107 case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64;
6108 case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64;
6109 case AMDGPU::S_FMAC_F16:
6110 return ST.useRealTrue16Insts() ? AMDGPU::V_FMAC_F16_t16_e64
6111 : AMDGPU::V_FMAC_F16_fake16_e64;
6112 case AMDGPU::S_FMAMK_F32: return AMDGPU::V_FMAMK_F32;
6113 case AMDGPU::S_FMAAK_F32: return AMDGPU::V_FMAAK_F32;
6114 case AMDGPU::S_CMP_LT_F32: return AMDGPU::V_CMP_LT_F32_e64;
6115 case AMDGPU::S_CMP_EQ_F32: return AMDGPU::V_CMP_EQ_F32_e64;
6116 case AMDGPU::S_CMP_LE_F32: return AMDGPU::V_CMP_LE_F32_e64;
6117 case AMDGPU::S_CMP_GT_F32: return AMDGPU::V_CMP_GT_F32_e64;
6118 case AMDGPU::S_CMP_LG_F32: return AMDGPU::V_CMP_LG_F32_e64;
6119 case AMDGPU::S_CMP_GE_F32: return AMDGPU::V_CMP_GE_F32_e64;
6120 case AMDGPU::S_CMP_O_F32: return AMDGPU::V_CMP_O_F32_e64;
6121 case AMDGPU::S_CMP_U_F32: return AMDGPU::V_CMP_U_F32_e64;
6122 case AMDGPU::S_CMP_NGE_F32: return AMDGPU::V_CMP_NGE_F32_e64;
6123 case AMDGPU::S_CMP_NLG_F32: return AMDGPU::V_CMP_NLG_F32_e64;
6124 case AMDGPU::S_CMP_NGT_F32: return AMDGPU::V_CMP_NGT_F32_e64;
6125 case AMDGPU::S_CMP_NLE_F32: return AMDGPU::V_CMP_NLE_F32_e64;
6126 case AMDGPU::S_CMP_NEQ_F32: return AMDGPU::V_CMP_NEQ_F32_e64;
6127 case AMDGPU::S_CMP_NLT_F32: return AMDGPU::V_CMP_NLT_F32_e64;
6128 case AMDGPU::S_CMP_LT_F16:
6129 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LT_F16_t16_e64
6130 : AMDGPU::V_CMP_LT_F16_fake16_e64;
6131 case AMDGPU::S_CMP_EQ_F16:
6132 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_EQ_F16_t16_e64
6133 : AMDGPU::V_CMP_EQ_F16_fake16_e64;
6134 case AMDGPU::S_CMP_LE_F16:
6135 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LE_F16_t16_e64
6136 : AMDGPU::V_CMP_LE_F16_fake16_e64;
6137 case AMDGPU::S_CMP_GT_F16:
6138 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GT_F16_t16_e64
6139 : AMDGPU::V_CMP_GT_F16_fake16_e64;
6140 case AMDGPU::S_CMP_LG_F16:
6141 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LG_F16_t16_e64
6142 : AMDGPU::V_CMP_LG_F16_fake16_e64;
6143 case AMDGPU::S_CMP_GE_F16:
6144 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GE_F16_t16_e64
6145 : AMDGPU::V_CMP_GE_F16_fake16_e64;
6146 case AMDGPU::S_CMP_O_F16:
6147 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_O_F16_t16_e64
6148 : AMDGPU::V_CMP_O_F16_fake16_e64;
6149 case AMDGPU::S_CMP_U_F16:
6150 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_U_F16_t16_e64
6151 : AMDGPU::V_CMP_U_F16_fake16_e64;
6152 case AMDGPU::S_CMP_NGE_F16:
6153 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGE_F16_t16_e64
6154 : AMDGPU::V_CMP_NGE_F16_fake16_e64;
6155 case AMDGPU::S_CMP_NLG_F16:
6156 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLG_F16_t16_e64
6157 : AMDGPU::V_CMP_NLG_F16_fake16_e64;
6158 case AMDGPU::S_CMP_NGT_F16:
6159 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGT_F16_t16_e64
6160 : AMDGPU::V_CMP_NGT_F16_fake16_e64;
6161 case AMDGPU::S_CMP_NLE_F16:
6162 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLE_F16_t16_e64
6163 : AMDGPU::V_CMP_NLE_F16_fake16_e64;
6164 case AMDGPU::S_CMP_NEQ_F16:
6165 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NEQ_F16_t16_e64
6166 : AMDGPU::V_CMP_NEQ_F16_fake16_e64;
6167 case AMDGPU::S_CMP_NLT_F16:
6168 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLT_F16_t16_e64
6169 : AMDGPU::V_CMP_NLT_F16_fake16_e64;
6170 case AMDGPU::V_S_EXP_F32_e64: return AMDGPU::V_EXP_F32_e64;
6171 case AMDGPU::V_S_EXP_F16_e64:
6172 return ST.useRealTrue16Insts() ? AMDGPU::V_EXP_F16_t16_e64
6173 : AMDGPU::V_EXP_F16_fake16_e64;
6174 case AMDGPU::V_S_LOG_F32_e64: return AMDGPU::V_LOG_F32_e64;
6175 case AMDGPU::V_S_LOG_F16_e64:
6176 return ST.useRealTrue16Insts() ? AMDGPU::V_LOG_F16_t16_e64
6177 : AMDGPU::V_LOG_F16_fake16_e64;
6178 case AMDGPU::V_S_RCP_F32_e64: return AMDGPU::V_RCP_F32_e64;
6179 case AMDGPU::V_S_RCP_F16_e64:
6180 return ST.useRealTrue16Insts() ? AMDGPU::V_RCP_F16_t16_e64
6181 : AMDGPU::V_RCP_F16_fake16_e64;
6182 case AMDGPU::V_S_RSQ_F32_e64: return AMDGPU::V_RSQ_F32_e64;
6183 case AMDGPU::V_S_RSQ_F16_e64:
6184 return ST.useRealTrue16Insts() ? AMDGPU::V_RSQ_F16_t16_e64
6185 : AMDGPU::V_RSQ_F16_fake16_e64;
6186 case AMDGPU::V_S_SQRT_F32_e64: return AMDGPU::V_SQRT_F32_e64;
6187 case AMDGPU::V_S_SQRT_F16_e64:
6188 return ST.useRealTrue16Insts() ? AMDGPU::V_SQRT_F16_t16_e64
6189 : AMDGPU::V_SQRT_F16_fake16_e64;
6190 }
6191 llvm_unreachable(
6192 "Unexpected scalar opcode without corresponding vector one!");
6193}
6194
6195// clang-format on
6196
6197void SIInstrInfo::insertScratchExecCopy(MachineFunction &MF,
6198 MachineBasicBlock &MBB,
6199 MachineBasicBlock::iterator MBBI,
6200 const DebugLoc &DL, Register Reg,
6201 bool IsSCCLive,
6202 SlotIndexes *Indexes) const {
6203 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
6204 const SIInstrInfo *TII = ST.getInstrInfo();
6205 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
6206 if (IsSCCLive) {
6207 // Insert two move instructions, one to save the original value of EXEC and
6208 // the other to turn on all bits in EXEC. This is required as we can't use
6209 // the single instruction S_OR_SAVEEXEC that clobbers SCC.
6210 auto StoreExecMI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: Reg)
6211 .addReg(RegNo: LMC.ExecReg, Flags: RegState::Kill);
6212 auto FlipExecMI =
6213 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg).addImm(Val: -1);
6214 if (Indexes) {
6215 Indexes->insertMachineInstrInMaps(MI&: *StoreExecMI);
6216 Indexes->insertMachineInstrInMaps(MI&: *FlipExecMI);
6217 }
6218 } else {
6219 auto SaveExec =
6220 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.OrSaveExecOpc), DestReg: Reg).addImm(Val: -1);
6221 SaveExec->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
6222 if (Indexes)
6223 Indexes->insertMachineInstrInMaps(MI&: *SaveExec);
6224 }
6225}
6226
6227void SIInstrInfo::restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
6228 MachineBasicBlock::iterator MBBI,
6229 const DebugLoc &DL, Register Reg,
6230 SlotIndexes *Indexes) const {
6231 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
6232 auto ExecRestoreMI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
6233 .addReg(RegNo: Reg, Flags: RegState::Kill);
6234 if (Indexes)
6235 Indexes->insertMachineInstrInMaps(MI&: *ExecRestoreMI);
6236}
6237
6238MachineInstr *
6239SIInstrInfo::getWholeWaveFunctionSetup(MachineFunction &MF) const {
6240 assert(MF.getInfo<SIMachineFunctionInfo>()->isWholeWaveFunction() &&
6241 "Not a whole wave func");
6242 MachineBasicBlock &MBB = *MF.begin();
6243 for (MachineInstr &MI : MBB)
6244 if (MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_SETUP ||
6245 MI.getOpcode() == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
6246 return &MI;
6247
6248 llvm_unreachable("Couldn't find SI_SETUP_WHOLE_WAVE_FUNC instruction");
6249}
6250
6251const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
6252 unsigned OpNo) const {
6253 const MCInstrDesc &Desc = get(Opcode: MI.getOpcode());
6254 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
6255 Desc.operands()[OpNo].RegClass == -1) {
6256 Register Reg = MI.getOperand(i: OpNo).getReg();
6257
6258 if (Reg.isVirtual()) {
6259 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6260 return MRI.getRegClass(Reg);
6261 }
6262 return RI.getPhysRegBaseClass(Reg);
6263 }
6264
6265 int16_t RegClass = getOpRegClassID(OpInfo: Desc.operands()[OpNo]);
6266 return RegClass < 0 ? nullptr : RI.getRegClass(i: RegClass);
6267}
6268
6269void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
6270 MachineBasicBlock::iterator I = MI;
6271 MachineBasicBlock *MBB = MI.getParent();
6272 MachineOperand &MO = MI.getOperand(i: OpIdx);
6273 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
6274 unsigned RCID = getOpRegClassID(OpInfo: get(Opcode: MI.getOpcode()).operands()[OpIdx]);
6275 const TargetRegisterClass *RC = RI.getRegClass(i: RCID);
6276 unsigned Size = RI.getRegSizeInBits(RC: *RC);
6277 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO
6278 : Size == 16 ? AMDGPU::V_MOV_B16_t16_e64
6279 : AMDGPU::V_MOV_B32_e32;
6280 if (MO.isReg())
6281 Opcode = AMDGPU::COPY;
6282 else if (RI.isSGPRClass(RC))
6283 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
6284
6285 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(SRC: RC);
6286 Register Reg = MRI.createVirtualRegister(RegClass: VRC);
6287 DebugLoc DL = MBB->findDebugLoc(MBBI: I);
6288 BuildMI(BB&: *MI.getParent(), I, MIMD: DL, MCID: get(Opcode), DestReg: Reg).add(MO);
6289 MO.ChangeToRegister(Reg, isDef: false);
6290}
6291
6292unsigned SIInstrInfo::buildExtractSubReg(
6293 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
6294 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
6295 unsigned SubIdx, const TargetRegisterClass *SubRC) const {
6296 if (!SuperReg.getReg().isVirtual())
6297 return RI.getSubReg(Reg: SuperReg.getReg(), Idx: SubIdx);
6298
6299 MachineBasicBlock *MBB = MI->getParent();
6300 const DebugLoc &DL = MI->getDebugLoc();
6301 Register SubReg = MRI.createVirtualRegister(RegClass: SubRC);
6302
6303 unsigned NewSubIdx = RI.composeSubRegIndices(a: SuperReg.getSubReg(), b: SubIdx);
6304 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: SubReg)
6305 .addReg(RegNo: SuperReg.getReg(), Flags: {}, SubReg: NewSubIdx);
6306 return SubReg;
6307}
6308
6309MachineOperand SIInstrInfo::buildExtractSubRegOrImm(
6310 MachineBasicBlock::iterator MII, MachineRegisterInfo &MRI,
6311 const MachineOperand &Op, const TargetRegisterClass *SuperRC,
6312 unsigned SubIdx, const TargetRegisterClass *SubRC) const {
6313 if (Op.isImm()) {
6314 if (SubIdx == AMDGPU::sub0)
6315 return MachineOperand::CreateImm(Val: static_cast<int32_t>(Op.getImm()));
6316 if (SubIdx == AMDGPU::sub1)
6317 return MachineOperand::CreateImm(Val: static_cast<int32_t>(Op.getImm() >> 32));
6318
6319 llvm_unreachable("Unhandled register index for immediate");
6320 }
6321
6322 unsigned SubReg = buildExtractSubReg(MI: MII, MRI, SuperReg: Op, SuperRC,
6323 SubIdx, SubRC);
6324 return MachineOperand::CreateReg(Reg: SubReg, isDef: false);
6325}
6326
6327// Change the order of operands from (0, 1, 2) to (0, 2, 1)
6328void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
6329 assert(Inst.getNumExplicitOperands() == 3);
6330 MachineOperand Op1 = Inst.getOperand(i: 1);
6331 Inst.removeOperand(OpNo: 1);
6332 Inst.addOperand(Op: Op1);
6333}
6334
6335bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
6336 const MCOperandInfo &OpInfo,
6337 const MachineOperand &MO) const {
6338 if (!MO.isReg())
6339 return false;
6340
6341 Register Reg = MO.getReg();
6342
6343 const TargetRegisterClass *DRC = RI.getRegClass(i: getOpRegClassID(OpInfo));
6344 if (Reg.isPhysical())
6345 return DRC->contains(Reg);
6346
6347 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
6348
6349 if (MO.getSubReg()) {
6350 const MachineFunction *MF = MO.getParent()->getMF();
6351 const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, MF: *MF);
6352 if (!SuperRC)
6353 return false;
6354 return RI.getMatchingSuperRegClass(A: SuperRC, B: DRC, Idx: MO.getSubReg()) != nullptr;
6355 }
6356
6357 return RI.getCommonSubClass(A: DRC, B: RC) != nullptr;
6358}
6359
6360bool SIInstrInfo::isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
6361 const MachineOperand &MO) const {
6362 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6363 const MCOperandInfo OpInfo = MI.getDesc().operands()[OpIdx];
6364 unsigned Opc = MI.getOpcode();
6365
6366 // See SIInstrInfo::isLegalGFX12PlusPackedMathFP32or64BitOperand for more
6367 // information.
6368 if (AMDGPU::isPackedFP32or64BitInst(Opc: MI.getOpcode()) &&
6369 AMDGPU::isGFX12Plus(STI: ST) && MO.isReg() && RI.isSGPRReg(MRI, Reg: MO.getReg())) {
6370 constexpr AMDGPU::OpName OpNames[] = {
6371 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};
6372
6373 for (auto [I, OpName] : enumerate(First: OpNames)) {
6374 int SrcIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[I]);
6375 if (static_cast<unsigned>(SrcIdx) == OpIdx &&
6376 !isLegalGFX12PlusPackedMathFP32or64BitOperand(MRI, MI, SrcN: I, MO: &MO))
6377 return false;
6378 }
6379 }
6380
6381 if (!isLegalRegOperand(MRI, OpInfo, MO))
6382 return false;
6383
6384 // check Accumulate GPR operand
6385 bool IsAGPR = RI.isAGPR(MRI, Reg: MO.getReg());
6386 if (IsAGPR && !ST.hasMAIInsts())
6387 return false;
6388 if (IsAGPR && (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
6389 (MI.mayLoad() || MI.mayStore() || isDS(Opcode: Opc) || isMIMG(Opcode: Opc)))
6390 return false;
6391 // Atomics should have both vdst and vdata either vgpr or agpr.
6392 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
6393 const int DataIdx = AMDGPU::getNamedOperandIdx(
6394 Opcode: Opc, Name: isDS(Opcode: Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata);
6395 if ((int)OpIdx == VDstIdx && DataIdx != -1 &&
6396 MI.getOperand(i: DataIdx).isReg() &&
6397 RI.isAGPR(MRI, Reg: MI.getOperand(i: DataIdx).getReg()) != IsAGPR)
6398 return false;
6399 if ((int)OpIdx == DataIdx) {
6400 if (VDstIdx != -1 &&
6401 RI.isAGPR(MRI, Reg: MI.getOperand(i: VDstIdx).getReg()) != IsAGPR)
6402 return false;
6403 // DS instructions with 2 src operands also must have tied RC.
6404 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data1);
6405 if (Data1Idx != -1 && MI.getOperand(i: Data1Idx).isReg() &&
6406 RI.isAGPR(MRI, Reg: MI.getOperand(i: Data1Idx).getReg()) != IsAGPR)
6407 return false;
6408 }
6409
6410 // Check V_ACCVGPR_WRITE_B32_e64
6411 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() &&
6412 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0) &&
6413 RI.isSGPRReg(MRI, Reg: MO.getReg()))
6414 return false;
6415
6416 if (ST.hasFlatScratchHiInB64InstHazard() &&
6417 MO.getReg() == AMDGPU::SRC_FLAT_SCRATCH_BASE_HI && isSALU(MI)) {
6418 if (const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst)) {
6419 if (AMDGPU::getRegBitWidth(RC: *RI.getRegClassForReg(MRI, Reg: Dst->getReg())) ==
6420 64)
6421 return false;
6422 }
6423 if (Opc == AMDGPU::S_BITCMP0_B64 || Opc == AMDGPU::S_BITCMP1_B64)
6424 return false;
6425 }
6426 if (!ST.hasDPPSrc1SGPR() && isDPP(MI) && RI.isSGPRReg(MRI, Reg: MO.getReg()) &&
6427 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1))
6428 return false;
6429
6430 return true;
6431}
6432
6433bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI,
6434 const MCOperandInfo &OpInfo,
6435 const MachineOperand &MO) const {
6436 if (MO.isReg())
6437 return isLegalRegOperand(MRI, OpInfo, MO);
6438
6439 // Handle non-register types that are treated like immediates.
6440 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
6441 return true;
6442}
6443
6444bool SIInstrInfo::isLegalGFX12PlusPackedMathFP32or64BitOperand(
6445 const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,
6446 const MachineOperand *MO) const {
6447 constexpr unsigned NumOps = 3;
6448 constexpr AMDGPU::OpName OpNames[NumOps * 2] = {
6449 AMDGPU::OpName::src0, AMDGPU::OpName::src1,
6450 AMDGPU::OpName::src2, AMDGPU::OpName::src0_modifiers,
6451 AMDGPU::OpName::src1_modifiers, AMDGPU::OpName::src2_modifiers};
6452
6453 assert(SrcN < NumOps);
6454
6455 if (!MO) {
6456 int SrcIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[SrcN]);
6457 if (SrcIdx == -1)
6458 return true;
6459 MO = &MI.getOperand(i: SrcIdx);
6460 }
6461
6462 if (!MO->isReg() || !RI.isSGPRReg(MRI, Reg: MO->getReg()))
6463 return true;
6464
6465 int ModsIdx =
6466 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[NumOps + SrcN]);
6467 if (ModsIdx == -1)
6468 return false;
6469
6470 unsigned Mods = MI.getOperand(i: ModsIdx).getImm();
6471 bool OpSel = Mods & SISrcMods::OP_SEL_0;
6472 bool OpSelHi = Mods & SISrcMods::OP_SEL_1;
6473
6474 return !OpSel && !OpSelHi;
6475}
6476
6477bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
6478 const MachineOperand *MO) const {
6479 const MachineFunction &MF = *MI.getMF();
6480 const MachineRegisterInfo &MRI = MF.getRegInfo();
6481 const MCInstrDesc &InstDesc = MI.getDesc();
6482 const MCOperandInfo &OpInfo = InstDesc.operands()[OpIdx];
6483 int64_t RegClass = getOpRegClassID(OpInfo);
6484 const TargetRegisterClass *DefinedRC =
6485 RegClass != -1 ? RI.getRegClass(i: RegClass) : nullptr;
6486 if (!MO)
6487 MO = &MI.getOperand(i: OpIdx);
6488
6489 const bool IsInlineConst = !MO->isReg() && isInlineConstant(MO: *MO, OpInfo);
6490
6491 if (isVALU(MI, /*AllowLDSDMA=*/true) && !IsInlineConst &&
6492 usesConstantBus(MRI, MO: *MO, OpInfo)) {
6493 const MachineOperand *UsedLiteral = nullptr;
6494
6495 int ConstantBusLimit = ST.getConstantBusLimit(Opcode: MI.getOpcode());
6496 int LiteralLimit = !isVOP3(MI) || ST.hasVOP3Literal() ? 1 : 0;
6497
6498 // TODO: Be more permissive with frame indexes.
6499 if (!MO->isReg() && !isInlineConstant(MO: *MO, OpInfo)) {
6500 if (!LiteralLimit--)
6501 return false;
6502
6503 UsedLiteral = MO;
6504 }
6505
6506 SmallDenseSet<RegSubRegPair> SGPRsUsed;
6507 if (MO->isReg())
6508 SGPRsUsed.insert(V: RegSubRegPair(MO->getReg(), MO->getSubReg()));
6509
6510 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
6511 if (i == OpIdx)
6512 continue;
6513 const MachineOperand &Op = MI.getOperand(i);
6514 if (Op.isReg()) {
6515 if (Op.isUse()) {
6516 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
6517 if (regUsesConstantBus(RegOp: Op, MRI) && SGPRsUsed.insert(V: SGPR).second) {
6518 if (--ConstantBusLimit <= 0)
6519 return false;
6520 }
6521 }
6522 } else if (AMDGPU::isSISrcOperand(OpInfo: InstDesc.operands()[i]) &&
6523 !isInlineConstant(MO: Op, OpInfo: InstDesc.operands()[i])) {
6524 // The same literal may be used multiple times.
6525 if (!UsedLiteral)
6526 UsedLiteral = &Op;
6527 else if (UsedLiteral->isIdenticalTo(Other: Op))
6528 continue;
6529
6530 if (!LiteralLimit--)
6531 return false;
6532 if (--ConstantBusLimit <= 0)
6533 return false;
6534 }
6535 }
6536 } else if (!IsInlineConst && !MO->isReg() && isSALU(MI)) {
6537 // There can be at most one literal operand, but it can be repeated.
6538 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
6539 if (i == OpIdx)
6540 continue;
6541 const MachineOperand &Op = MI.getOperand(i);
6542 if (!Op.isReg() && !Op.isFI() && !Op.isRegMask() &&
6543 !isInlineConstant(MO: Op, OpInfo: InstDesc.operands()[i]) &&
6544 !Op.isIdenticalTo(Other: *MO))
6545 return false;
6546
6547 // Do not fold a non-inlineable and non-register operand into an
6548 // instruction that already has a frame index. The frame index handling
6549 // code could not handle well when a frame index co-exists with another
6550 // non-register operand, unless that operand is an inlineable immediate.
6551 if (Op.isFI())
6552 return false;
6553 }
6554 } else if (IsInlineConst && ST.hasNoF16PseudoScalarTransInlineConstants() &&
6555 isF16PseudoScalarTrans(Opcode: MI.getOpcode())) {
6556 return false;
6557 }
6558
6559 if (MO->isReg()) {
6560 if (!DefinedRC)
6561 return OpInfo.OperandType == MCOI::OPERAND_UNKNOWN;
6562 return isLegalRegOperand(MI, OpIdx, MO: *MO);
6563 }
6564
6565 if (MO->isImm()) {
6566 uint64_t Imm = MO->getImm();
6567 bool Is64BitFPOp = OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_FP64 ||
6568 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP64;
6569 bool Is64BitOp = Is64BitFPOp ||
6570 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64 ||
6571 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT32 ||
6572 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32 ||
6573 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT64;
6574 if (Is64BitOp &&
6575 !AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: ST.hasInv2PiInlineImm())) {
6576 if (!AMDGPU::isValid32BitLiteral(Val: Imm, IsFP64: Is64BitFPOp) &&
6577 (!ST.has64BitLiterals() || InstDesc.getSize() != 4))
6578 return false;
6579
6580 // FIXME: We can use sign extended 64-bit literals, but only for signed
6581 // operands. At the moment we do not know if an operand is signed.
6582 // Such operand will be encoded as its low 32 bits and then either
6583 // correctly sign extended or incorrectly zero extended by HW.
6584 // If 64-bit literals are supported and the literal will be encoded
6585 // as full 64 bit we still can use it.
6586 if (!Is64BitFPOp && (int32_t)Imm < 0 &&
6587 (!ST.has64BitLiterals() || AMDGPU::isValid32BitLiteral(Val: Imm, IsFP64: false)))
6588 return false;
6589 }
6590 }
6591
6592 // Handle non-register types that are treated like immediates.
6593 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
6594
6595 if (!DefinedRC) {
6596 // This operand expects an immediate.
6597 return true;
6598 }
6599
6600 return isImmOperandLegal(MI, OpNo: OpIdx, MO: *MO);
6601}
6602
6603bool SIInstrInfo::isNeverCoissue(MachineInstr &MI) const {
6604 bool IsGFX950Only = ST.hasGFX950Insts();
6605 bool IsGFX940Only = ST.hasGFX940Insts();
6606
6607 if (!IsGFX950Only && !IsGFX940Only)
6608 return false;
6609
6610 if (!isVALU(MI, /*AllowLDSDMA=*/true))
6611 return false;
6612
6613 // V_COS, V_EXP, V_RCP, etc.
6614 if (isTRANS(MI))
6615 return true;
6616
6617 // DOT2, DOT2C, DOT4, etc.
6618 if (isDOT(MI))
6619 return true;
6620
6621 // MFMA, SMFMA
6622 if (isMFMA(MI))
6623 return true;
6624
6625 unsigned Opcode = MI.getOpcode();
6626 switch (Opcode) {
6627 case AMDGPU::V_CVT_PK_BF8_F32_e64:
6628 case AMDGPU::V_CVT_PK_FP8_F32_e64:
6629 case AMDGPU::V_MQSAD_PK_U16_U8_e64:
6630 case AMDGPU::V_MQSAD_U32_U8_e64:
6631 case AMDGPU::V_PK_ADD_F16:
6632 case AMDGPU::V_PK_ADD_F32:
6633 case AMDGPU::V_PK_ADD_I16:
6634 case AMDGPU::V_PK_ADD_U16:
6635 case AMDGPU::V_PK_ASHRREV_I16:
6636 case AMDGPU::V_PK_FMA_F16:
6637 case AMDGPU::V_PK_FMA_F32:
6638 case AMDGPU::V_PK_FMAC_F16_e32:
6639 case AMDGPU::V_PK_FMAC_F16_e64:
6640 case AMDGPU::V_PK_LSHLREV_B16:
6641 case AMDGPU::V_PK_LSHRREV_B16:
6642 case AMDGPU::V_PK_MAD_I16:
6643 case AMDGPU::V_PK_MAD_U16:
6644 case AMDGPU::V_PK_MAX_F16:
6645 case AMDGPU::V_PK_MAX_I16:
6646 case AMDGPU::V_PK_MAX_U16:
6647 case AMDGPU::V_PK_MIN_F16:
6648 case AMDGPU::V_PK_MIN_I16:
6649 case AMDGPU::V_PK_MIN_U16:
6650 case AMDGPU::V_PK_MOV_B32:
6651 case AMDGPU::V_PK_MUL_F16:
6652 case AMDGPU::V_PK_MUL_F32:
6653 case AMDGPU::V_PK_MUL_LO_U16:
6654 case AMDGPU::V_PK_SUB_I16:
6655 case AMDGPU::V_PK_SUB_U16:
6656 case AMDGPU::V_QSAD_PK_U16_U8_e64:
6657 return true;
6658 default:
6659 return false;
6660 }
6661}
6662
6663void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
6664 MachineInstr &MI) const {
6665 unsigned Opc = MI.getOpcode();
6666 const MCInstrDesc &InstrDesc = get(Opcode: Opc);
6667
6668 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
6669 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
6670
6671 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
6672 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
6673
6674 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
6675 // we need to only have one constant bus use before GFX10.
6676 bool HasImplicitSGPR = findImplicitSGPRRead(MI);
6677 if (HasImplicitSGPR && ST.getConstantBusLimit(Opcode: Opc) <= 1 && Src0.isReg() &&
6678 RI.isSGPRReg(MRI, Reg: Src0.getReg()))
6679 legalizeOpWithMove(MI, OpIdx: Src0Idx);
6680
6681 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
6682 // both the value to write (src0) and lane select (src1). Fix up non-SGPR
6683 // src0/src1 with V_READFIRSTLANE.
6684 if (Opc == AMDGPU::V_WRITELANE_B32) {
6685 const DebugLoc &DL = MI.getDebugLoc();
6686 if (Src0.isReg() && RI.isVGPR(MRI, Reg: Src0.getReg())) {
6687 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6688 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
6689 .add(MO: Src0);
6690 Src0.ChangeToRegister(Reg, isDef: false);
6691 }
6692 if (Src1.isReg() && RI.isVGPR(MRI, Reg: Src1.getReg())) {
6693 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6694 const DebugLoc &DL = MI.getDebugLoc();
6695 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
6696 .add(MO: Src1);
6697 Src1.ChangeToRegister(Reg, isDef: false);
6698 }
6699 return;
6700 }
6701
6702 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2.
6703 if (Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F16_e32) {
6704 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
6705 if (!RI.isVGPR(MRI, Reg: MI.getOperand(i: Src2Idx).getReg()))
6706 legalizeOpWithMove(MI, OpIdx: Src2Idx);
6707 }
6708
6709 // VOP2 src0 instructions support all operand types, so we don't need to check
6710 // their legality. If src1 is already legal, we don't need to do anything.
6711 if (isLegalRegOperand(MRI, OpInfo: InstrDesc.operands()[Src1Idx], MO: Src1))
6712 return;
6713
6714 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
6715 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
6716 // select is uniform.
6717 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
6718 RI.isVGPR(MRI, Reg: Src1.getReg())) {
6719 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6720 const DebugLoc &DL = MI.getDebugLoc();
6721 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
6722 .add(MO: Src1);
6723 Src1.ChangeToRegister(Reg, isDef: false);
6724 return;
6725 }
6726
6727 // We do not use commuteInstruction here because it is too aggressive and will
6728 // commute if it is possible. We only want to commute here if it improves
6729 // legality. This can be called a fairly large number of times so don't waste
6730 // compile time pointlessly swapping and checking legality again.
6731 if (HasImplicitSGPR || !MI.isCommutable()) {
6732 legalizeOpWithMove(MI, OpIdx: Src1Idx);
6733 return;
6734 }
6735
6736 // If src0 can be used as src1, commuting will make the operands legal.
6737 // Otherwise we have to give up and insert a move.
6738 //
6739 // TODO: Other immediate-like operand kinds could be commuted if there was a
6740 // MachineOperand::ChangeTo* for them.
6741 if ((!Src1.isImm() && !Src1.isReg()) ||
6742 !isLegalRegOperand(MRI, OpInfo: InstrDesc.operands()[Src1Idx], MO: Src0)) {
6743 legalizeOpWithMove(MI, OpIdx: Src1Idx);
6744 return;
6745 }
6746
6747 int CommutedOpc = commuteOpcode(MI);
6748 if (CommutedOpc == -1) {
6749 legalizeOpWithMove(MI, OpIdx: Src1Idx);
6750 return;
6751 }
6752
6753 MI.setDesc(get(Opcode: CommutedOpc));
6754
6755 Register Src0Reg = Src0.getReg();
6756 unsigned Src0SubReg = Src0.getSubReg();
6757 bool Src0Kill = Src0.isKill();
6758
6759 if (Src1.isImm())
6760 Src0.ChangeToImmediate(ImmVal: Src1.getImm());
6761 else if (Src1.isReg()) {
6762 Src0.ChangeToRegister(Reg: Src1.getReg(), isDef: false, isImp: false, isKill: Src1.isKill());
6763 Src0.setSubReg(Src1.getSubReg());
6764 } else
6765 llvm_unreachable("Should only have register or immediate operands");
6766
6767 Src1.ChangeToRegister(Reg: Src0Reg, isDef: false, isImp: false, isKill: Src0Kill);
6768 Src1.setSubReg(Src0SubReg);
6769 fixImplicitOperands(MI);
6770}
6771
6772// Legalize VOP3 operands. All operand types are supported for any operand
6773// but only one literal constant and only starting from GFX10.
6774void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
6775 MachineInstr &MI) const {
6776 unsigned Opc = MI.getOpcode();
6777
6778 int VOP3Idx[3] = {
6779 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0),
6780 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1),
6781 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2)
6782 };
6783
6784 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 ||
6785 Opc == AMDGPU::V_PERMLANEX16_B32_e64 ||
6786 Opc == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
6787 Opc == AMDGPU::V_PERMLANE_UP_B32_e64 ||
6788 Opc == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
6789 Opc == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
6790 Opc == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64) {
6791 // src1 and src2 must be scalar
6792 MachineOperand &Src1 = MI.getOperand(i: VOP3Idx[1]);
6793 const DebugLoc &DL = MI.getDebugLoc();
6794 if (Src1.isReg() && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src1.getReg()))) {
6795 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6796 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
6797 .add(MO: Src1);
6798 Src1.ChangeToRegister(Reg, isDef: false);
6799 }
6800 if (VOP3Idx[2] != -1) {
6801 MachineOperand &Src2 = MI.getOperand(i: VOP3Idx[2]);
6802 if (Src2.isReg() && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src2.getReg()))) {
6803 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6804 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
6805 .add(MO: Src2);
6806 Src2.ChangeToRegister(Reg, isDef: false);
6807 }
6808 }
6809 }
6810
6811 // Find the one SGPR operand we are allowed to use.
6812 int ConstantBusLimit = ST.getConstantBusLimit(Opcode: Opc);
6813 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
6814 SmallDenseSet<unsigned> SGPRsUsed;
6815 Register SGPRReg = findUsedSGPR(MI, OpIndices: VOP3Idx);
6816 if (SGPRReg) {
6817 SGPRsUsed.insert(V: SGPRReg);
6818 --ConstantBusLimit;
6819 }
6820
6821 for (int Idx : VOP3Idx) {
6822 if (Idx == -1)
6823 break;
6824 MachineOperand &MO = MI.getOperand(i: Idx);
6825
6826 if (!MO.isReg()) {
6827 if (isInlineConstant(MO, OpInfo: get(Opcode: Opc).operands()[Idx]))
6828 continue;
6829
6830 if (LiteralLimit > 0 && ConstantBusLimit > 0) {
6831 --LiteralLimit;
6832 --ConstantBusLimit;
6833 continue;
6834 }
6835
6836 --LiteralLimit;
6837 --ConstantBusLimit;
6838 legalizeOpWithMove(MI, OpIdx: Idx);
6839 continue;
6840 }
6841
6842 if (!RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg: MO.getReg())))
6843 continue; // VGPRs are legal
6844
6845 // We can use one SGPR in each VOP3 instruction prior to GFX10
6846 // and two starting from GFX10.
6847 if (SGPRsUsed.count(V: MO.getReg()))
6848 continue;
6849 if (ConstantBusLimit > 0) {
6850 SGPRsUsed.insert(V: MO.getReg());
6851 --ConstantBusLimit;
6852 continue;
6853 }
6854
6855 // If we make it this far, then the operand is not legal and we must
6856 // legalize it.
6857 legalizeOpWithMove(MI, OpIdx: Idx);
6858 }
6859
6860 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2 tied to vdst.
6861 if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64) &&
6862 !RI.isVGPR(MRI, Reg: MI.getOperand(i: VOP3Idx[2]).getReg()))
6863 legalizeOpWithMove(MI, OpIdx: VOP3Idx[2]);
6864
6865 // Fix the register class of packed FP32 instructions on gfx12+. See
6866 // SIInstrInfo::isLegalGFX12PlusPackedMathFP32or64BitOperand for more
6867 // information.
6868 if (AMDGPU::isPackedFP32or64BitInst(Opc) && AMDGPU::isGFX12Plus(STI: ST)) {
6869 for (unsigned I = 0; I < 3; ++I) {
6870 if (!isLegalGFX12PlusPackedMathFP32or64BitOperand(MRI, MI, /*SrcN=*/I))
6871 legalizeOpWithMove(MI, OpIdx: VOP3Idx[I]);
6872 }
6873 }
6874}
6875
6876Register SIInstrInfo::readlaneVGPRToSGPR(
6877 Register SrcReg, MachineInstr &UseMI, MachineRegisterInfo &MRI,
6878 const TargetRegisterClass *DstRC /*=nullptr*/) const {
6879 const TargetRegisterClass *VRC = MRI.getRegClass(Reg: SrcReg);
6880 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
6881 if (DstRC)
6882 SRC = RI.getCommonSubClass(A: SRC, B: DstRC);
6883
6884 Register DstReg = MRI.createVirtualRegister(RegClass: SRC);
6885 unsigned SubRegs = RI.getRegSizeInBits(RC: *VRC) / 32;
6886
6887 if (RI.hasAGPRs(RC: VRC)) {
6888 VRC = RI.getEquivalentVGPRClass(SRC: VRC);
6889 Register NewSrcReg = MRI.createVirtualRegister(RegClass: VRC);
6890 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
6891 MCID: get(Opcode: TargetOpcode::COPY), DestReg: NewSrcReg)
6892 .addReg(RegNo: SrcReg);
6893 SrcReg = NewSrcReg;
6894 }
6895
6896 if (SubRegs == 1) {
6897 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
6898 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
6899 .addReg(RegNo: SrcReg);
6900 return DstReg;
6901 }
6902
6903 SmallVector<Register, 8> SRegs;
6904 for (unsigned i = 0; i < SubRegs; ++i) {
6905 Register SGPR = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
6906 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
6907 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: SGPR)
6908 .addReg(RegNo: SrcReg, Flags: {}, SubReg: RI.getSubRegFromChannel(Channel: i));
6909 SRegs.push_back(Elt: SGPR);
6910 }
6911
6912 MachineInstrBuilder MIB =
6913 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
6914 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
6915 for (unsigned i = 0; i < SubRegs; ++i) {
6916 MIB.addReg(RegNo: SRegs[i]);
6917 MIB.addImm(Val: RI.getSubRegFromChannel(Channel: i));
6918 }
6919 return DstReg;
6920}
6921
6922void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,
6923 MachineInstr &MI) const {
6924
6925 // If the pointer is store in VGPRs, then we need to move them to
6926 // SGPRs using v_readfirstlane. This is safe because we only select
6927 // loads with uniform pointers to SMRD instruction so we know the
6928 // pointer value is uniform.
6929 MachineOperand *SBase = getNamedOperand(MI, OperandName: AMDGPU::OpName::sbase);
6930 if (SBase && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SBase->getReg()))) {
6931 Register SGPR = readlaneVGPRToSGPR(SrcReg: SBase->getReg(), UseMI&: MI, MRI);
6932 SBase->setReg(SGPR);
6933 }
6934 MachineOperand *SOff = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
6935 if (SOff && !RI.isSGPRReg(MRI, Reg: SOff->getReg())) {
6936 Register SGPR = readlaneVGPRToSGPR(SrcReg: SOff->getReg(), UseMI&: MI, MRI);
6937 SOff->setReg(SGPR);
6938 }
6939}
6940
6941bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const {
6942 unsigned Opc = Inst.getOpcode();
6943 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::saddr);
6944 if (OldSAddrIdx < 0)
6945 return false;
6946
6947 assert(isSegmentSpecificFLAT(Inst) || (isFLAT(Inst) && ST.hasFlatGVSMode()));
6948
6949 int NewOpc = AMDGPU::getGlobalVaddrOp(Opcode: Opc);
6950 if (NewOpc < 0)
6951 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opcode: Opc);
6952 if (NewOpc < 0)
6953 return false;
6954
6955 MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
6956 MachineOperand &SAddr = Inst.getOperand(i: OldSAddrIdx);
6957 if (RI.isSGPRReg(MRI, Reg: SAddr.getReg()))
6958 return false;
6959
6960 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::vaddr);
6961 if (NewVAddrIdx < 0)
6962 return false;
6963
6964 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
6965
6966 // Check vaddr, it shall be zero or absent.
6967 MachineInstr *VAddrDef = nullptr;
6968 if (OldVAddrIdx >= 0) {
6969 MachineOperand &VAddr = Inst.getOperand(i: OldVAddrIdx);
6970 VAddrDef = MRI.getUniqueVRegDef(Reg: VAddr.getReg());
6971 if (!VAddrDef || !VAddrDef->isMoveImmediate() ||
6972 !VAddrDef->getOperand(i: 1).isImm() ||
6973 VAddrDef->getOperand(i: 1).getImm() != 0)
6974 return false;
6975 }
6976
6977 const MCInstrDesc &NewDesc = get(Opcode: NewOpc);
6978 Inst.setDesc(NewDesc);
6979
6980 // Callers expect iterator to be valid after this call, so modify the
6981 // instruction in place.
6982 if (OldVAddrIdx == NewVAddrIdx) {
6983 MachineOperand &NewVAddr = Inst.getOperand(i: NewVAddrIdx);
6984 // Clear use list from the old vaddr holding a zero register.
6985 MRI.removeRegOperandFromUseList(MO: &NewVAddr);
6986 MRI.moveOperands(Dst: &NewVAddr, Src: &SAddr, NumOps: 1);
6987 Inst.removeOperand(OpNo: OldSAddrIdx);
6988 // Update the use list with the pointer we have just moved from vaddr to
6989 // saddr position. Otherwise new vaddr will be missing from the use list.
6990 MRI.removeRegOperandFromUseList(MO: &NewVAddr);
6991 MRI.addRegOperandToUseList(MO: &NewVAddr);
6992 } else {
6993 assert(OldSAddrIdx == NewVAddrIdx);
6994
6995 if (OldVAddrIdx >= 0) {
6996 int NewVDstIn = AMDGPU::getNamedOperandIdx(Opcode: NewOpc,
6997 Name: AMDGPU::OpName::vdst_in);
6998
6999 // removeOperand doesn't try to fixup tied operand indexes at it goes, so
7000 // it asserts. Untie the operands for now and retie them afterwards.
7001 if (NewVDstIn != -1) {
7002 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst_in);
7003 Inst.untieRegOperand(OpIdx: OldVDstIn);
7004 }
7005
7006 Inst.removeOperand(OpNo: OldVAddrIdx);
7007
7008 if (NewVDstIn != -1) {
7009 int NewVDst = AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::vdst);
7010 Inst.tieOperands(DefIdx: NewVDst, UseIdx: NewVDstIn);
7011 }
7012 }
7013 }
7014
7015 if (VAddrDef && MRI.use_nodbg_empty(RegNo: VAddrDef->getOperand(i: 0).getReg()))
7016 VAddrDef->eraseFromParent();
7017
7018 return true;
7019}
7020
7021// FIXME: Remove this when SelectionDAG is obsoleted.
7022void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI,
7023 MachineInstr &MI) const {
7024 if (!isSegmentSpecificFLAT(MI) && !ST.hasFlatGVSMode())
7025 return;
7026
7027 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence
7028 // thinks they are uniform, so a readfirstlane should be valid.
7029 MachineOperand *SAddr = getNamedOperand(MI, OperandName: AMDGPU::OpName::saddr);
7030 if (!SAddr || RI.isSGPRClass(RC: MRI.getRegClass(Reg: SAddr->getReg())))
7031 return;
7032
7033 if (moveFlatAddrToVGPR(Inst&: MI))
7034 return;
7035
7036 const TargetRegisterClass *DeclaredRC =
7037 getRegClass(MCID: MI.getDesc(), OpNum: SAddr->getOperandNo());
7038
7039 Register ToSGPR = readlaneVGPRToSGPR(SrcReg: SAddr->getReg(), UseMI&: MI, MRI, DstRC: DeclaredRC);
7040 SAddr->setReg(ToSGPR);
7041}
7042
7043void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
7044 MachineBasicBlock::iterator I,
7045 const TargetRegisterClass *DstRC,
7046 MachineOperand &Op,
7047 MachineRegisterInfo &MRI,
7048 const DebugLoc &DL) const {
7049 Register OpReg = Op.getReg();
7050 unsigned OpSubReg = Op.getSubReg();
7051
7052 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
7053 RI.getRegClassForReg(MRI, Reg: OpReg), OpSubReg);
7054
7055 // Check if operand is already the correct register class.
7056 if (DstRC == OpRC)
7057 return;
7058
7059 Register DstReg = MRI.createVirtualRegister(RegClass: DstRC);
7060 auto Copy =
7061 BuildMI(BB&: InsertMBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: DstReg).addReg(RegNo: OpReg);
7062 Op.setReg(DstReg);
7063
7064 MachineInstr *Def = MRI.getVRegDef(Reg: OpReg);
7065 if (!Def)
7066 return;
7067
7068 // Try to eliminate the copy if it is copying an immediate value.
7069 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass)
7070 foldImmediate(UseMI&: *Copy, DefMI&: *Def, Reg: OpReg, MRI: &MRI);
7071
7072 bool ImpDef = Def->isImplicitDef();
7073 while (!ImpDef && Def && Def->isCopy()) {
7074 if (Def->getOperand(i: 1).getReg().isPhysical())
7075 break;
7076 Def = MRI.getUniqueVRegDef(Reg: Def->getOperand(i: 1).getReg());
7077 ImpDef = Def && Def->isImplicitDef();
7078 }
7079 if (!RI.isSGPRClass(RC: DstRC) && !Copy->readsRegister(Reg: AMDGPU::EXEC, TRI: &RI) &&
7080 !ImpDef)
7081 Copy.addReg(RegNo: AMDGPU::EXEC, Flags: RegState::Implicit);
7082}
7083
7084// Emit the actual waterfall loop, executing the wrapped instruction for each
7085// unique value of \p ScalarOps across all lanes. In the best case we execute 1
7086// iteration, in the worst case we execute 64 (once per lane).
7087static void emitLoadScalarOpsFromVGPRLoop(
7088 const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &PredBB,
7089 MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
7090 ArrayRef<MachineOperand *> ScalarOps, ArrayRef<Register> PhySGPRs = {}) {
7091 MachineFunction &MF = *LoopBB.getParent();
7092 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
7093 const SIRegisterInfo *TRI = ST.getRegisterInfo();
7094 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
7095 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
7096
7097 // Emit v_cmpx_eq and s_andn2_wrexec when both instructions are
7098 // available. Otherwise, use the previous pattern of v_cmp_eq,
7099 // s_and_saveexec, and s_xor.
7100 bool UseNewExecInstructions =
7101 ST.hasNoSdstCMPX() && TII.pseudoToMCOpcode(Opcode: LMC.AndN2WrExecOpc) != -1;
7102
7103 MachineBasicBlock::iterator I = LoopBB.begin();
7104 Register CondReg;
7105
7106 Register PhiExec;
7107 Register NewExec;
7108
7109 if (UseNewExecInstructions) {
7110 PhiExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7111 NewExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7112 Register InitExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7113 BuildMI(BB&: PredBB, I: PredBB.end(), MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: InitExec)
7114 .addReg(RegNo: LMC.ExecReg);
7115
7116 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::PHI), DestReg: PhiExec)
7117 .addReg(RegNo: InitExec)
7118 .addMBB(MBB: &PredBB)
7119 .addReg(RegNo: NewExec)
7120 .addMBB(MBB: &BodyBB);
7121 }
7122
7123 // Placement of v_cmpx instructions (when index is longer than 64 bit)
7124 // involves a trade-off between register pressure and latency:
7125 // (a) Defering all v_cmpx after all v_readfirstlane may increase
7126 // register pressure because arguments and results of all
7127 // v_readfirstlane instructions must stay live until deferred v_cmpx use them.
7128 // (b) Interleaving v_cmpx with v_readfirstlanes may reduce live ranges and
7129 // increase latency by placing v_readfirstlane instructions
7130 // immediately before v_cmpx instruction that directly depend on it.
7131 ///
7132 // Emitting interleaved v_cmpx and v_readfirstlane requires
7133 // block splitting because v_cmpx changes EXEC mask and therefore for safety
7134 // v_cmpx needs to be treated as terminator until after register allocation
7135 // (spill placement) and instruction reordering.
7136 //
7137 // Current implementation defers v_cmpx and leaves other instruction
7138 // scheduling decisions to later passes, where register pressure is known or
7139 // easier to approximate.
7140 // Non-terminators (V_READFIRSTLANE and REG_SEQUENCE) are inserted before I;
7141 // v_cmpx instructions are inserted at the end of LoopBB.
7142 // After the first v_cmpx is emitted, I is updated to point to it
7143 // so subsequent non-terminators are inserted before all v_cmpx instructions.
7144 for (auto [Idx, ScalarOp] : enumerate(First&: ScalarOps)) {
7145 unsigned RegSize = TRI->getRegSizeInBits(Reg: ScalarOp->getReg(), MRI);
7146 unsigned NumSubRegs = RegSize / 32;
7147 Register VScalarOp = ScalarOp->getReg();
7148
7149 const TargetRegisterClass *RFLSrcRC =
7150 TII.getRegClass(MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), OpNum: 1);
7151
7152 if (NumSubRegs == 1) {
7153 const TargetRegisterClass *VScalarOpRC = MRI.getRegClass(Reg: VScalarOp);
7154 if (const TargetRegisterClass *Common =
7155 TRI->getCommonSubClass(A: VScalarOpRC, B: RFLSrcRC);
7156 Common != VScalarOpRC) {
7157 Register VRReg = MRI.createVirtualRegister(RegClass: Common);
7158 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VRReg).addReg(RegNo: VScalarOp);
7159 VScalarOp = VRReg;
7160 }
7161 Register CurReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7162
7163 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurReg)
7164 .addReg(RegNo: VScalarOp);
7165
7166 if (UseNewExecInstructions) {
7167 auto CmpxMI = BuildMI(BB&: LoopBB, I: LoopBB.end(), MIMD: DL,
7168 MCID: TII.get(Opcode: AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term))
7169 .addReg(RegNo: CurReg)
7170 .addReg(RegNo: VScalarOp);
7171 if (I == LoopBB.end())
7172 I = CmpxMI.getInstr()->getIterator();
7173 } else {
7174 Register NewCondReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7175
7176 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: NewCondReg)
7177 .addReg(RegNo: CurReg)
7178 .addReg(RegNo: VScalarOp);
7179
7180 // Combine the comparison results with AND.
7181 if (!CondReg) { // First.
7182 CondReg = NewCondReg;
7183 } else { // If not the first, we create an AND.
7184 Register AndReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7185 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndOpc), DestReg: AndReg)
7186 .addReg(RegNo: CondReg)
7187 .addReg(RegNo: NewCondReg);
7188 CondReg = AndReg;
7189 }
7190 }
7191
7192 // Update ScalarOp operand to use the SGPR ScalarOp.
7193 if (PhySGPRs.empty() || !PhySGPRs[Idx].isValid())
7194 ScalarOp->setReg(CurReg);
7195 else {
7196 // Insert into the same block of use
7197 BuildMI(BB&: *ScalarOp->getParent()->getParent(), I: ScalarOp->getParent(), MIMD: DL,
7198 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: PhySGPRs[Idx])
7199 .addReg(RegNo: CurReg);
7200 ScalarOp->setReg(PhySGPRs[Idx]);
7201 }
7202 ScalarOp->setIsKill();
7203 } else {
7204 SmallVector<Register, 8> ReadlanePieces;
7205 RegState VScalarOpUndef = getUndefRegState(B: ScalarOp->isUndef());
7206 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 &&
7207 "Unhandled register size");
7208
7209 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) {
7210 Register CurRegLo =
7211 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7212 Register CurRegHi =
7213 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7214
7215 // Read the next variant <- also loop target.
7216 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurRegLo)
7217 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg: TRI->getSubRegFromChannel(Channel: Idx));
7218
7219 // Read the next variant <- also loop target.
7220 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurRegHi)
7221 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef,
7222 SubReg: TRI->getSubRegFromChannel(Channel: Idx + 1));
7223
7224 ReadlanePieces.push_back(Elt: CurRegLo);
7225 ReadlanePieces.push_back(Elt: CurRegHi);
7226
7227 // Comparison is to be done as 64-bit.
7228 Register CurReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_64RegClass);
7229 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: CurReg)
7230 .addReg(RegNo: CurRegLo)
7231 .addImm(Val: AMDGPU::sub0)
7232 .addReg(RegNo: CurRegHi)
7233 .addImm(Val: AMDGPU::sub1);
7234
7235 unsigned SubReg =
7236 NumSubRegs <= 2 ? 0 : TRI->getSubRegFromChannel(Channel: Idx, NumRegs: 2);
7237
7238 if (UseNewExecInstructions) {
7239 auto CmpxMI = BuildMI(BB&: LoopBB, I: LoopBB.end(), MIMD: DL,
7240 MCID: TII.get(Opcode: AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term))
7241 .addReg(RegNo: CurReg)
7242 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg);
7243 if (I == LoopBB.end())
7244 I = CmpxMI.getInstr()->getIterator();
7245 } else {
7246 Register NewCondReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7247 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U64_e64), DestReg: NewCondReg)
7248 .addReg(RegNo: CurReg)
7249 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg);
7250
7251 // Combine the comparison results with AND.
7252 if (!CondReg) { // First.
7253 CondReg = NewCondReg;
7254 } else { // If not the first, we create an AND.
7255 Register AndReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7256 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndOpc), DestReg: AndReg)
7257 .addReg(RegNo: CondReg)
7258 .addReg(RegNo: NewCondReg);
7259 CondReg = AndReg;
7260 }
7261 }
7262 } // End for loop.
7263
7264 const auto *SScalarOpRC =
7265 TRI->getEquivalentSGPRClass(VRC: MRI.getRegClass(Reg: VScalarOp));
7266 Register SScalarOp = MRI.createVirtualRegister(RegClass: SScalarOpRC);
7267
7268 // Build scalar ScalarOp.
7269 auto Merge =
7270 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: SScalarOp);
7271 unsigned Channel = 0;
7272 for (Register Piece : ReadlanePieces) {
7273 Merge.addReg(RegNo: Piece).addImm(Val: TRI->getSubRegFromChannel(Channel: Channel++));
7274 }
7275
7276 // Update ScalarOp operand to use the SGPR ScalarOp.
7277 if (PhySGPRs.empty() || !PhySGPRs[Idx].isValid())
7278 ScalarOp->setReg(SScalarOp);
7279 else {
7280 BuildMI(BB&: *ScalarOp->getParent()->getParent(), I: ScalarOp->getParent(), MIMD: DL,
7281 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: PhySGPRs[Idx])
7282 .addReg(RegNo: SScalarOp);
7283 ScalarOp->setReg(PhySGPRs[Idx]);
7284 }
7285 ScalarOp->setIsKill();
7286 }
7287 }
7288
7289 // Instructions AndSaveExecOpc and AndN2WrExecOpc that modify EXEC mask
7290 // should have isTerminator=1 but terminators that define
7291 // virtual registers are not supported.
7292 Register SaveExec;
7293 if (!UseNewExecInstructions) {
7294 SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7295 MRI.setSimpleHint(VReg: SaveExec, PrefReg: CondReg);
7296
7297 // Update EXEC to matching lanes, saving original to SaveExec.
7298 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndSaveExecOpc), DestReg: SaveExec)
7299 .addReg(RegNo: CondReg, Flags: RegState::Kill);
7300 }
7301
7302 // The original instruction is here; we insert the terminators after it.
7303 I = BodyBB.end();
7304
7305 if (UseNewExecInstructions) {
7306 MRI.setSimpleHint(VReg: NewExec, PrefReg: PhiExec);
7307 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndN2WrExecOpc), DestReg: NewExec)
7308 .addReg(RegNo: PhiExec);
7309 } else {
7310 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
7311 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.XorTermOpc), DestReg: LMC.ExecReg)
7312 .addReg(RegNo: LMC.ExecReg)
7313 .addReg(RegNo: SaveExec);
7314 }
7315
7316 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::SI_WATERFALL_LOOP)).addMBB(MBB: &LoopBB);
7317}
7318
7319// Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register
7320// with SGPRs by iterating over all unique values across all lanes.
7321// Returns the loop basic block that now contains \p MI.
7322static MachineBasicBlock *
7323generateWaterFallLoop(const SIInstrInfo &TII, MachineInstr &MI,
7324 ArrayRef<MachineOperand *> ScalarOps,
7325 MachineDominatorTree *MDT,
7326 MachineBasicBlock::iterator Begin = nullptr,
7327 MachineBasicBlock::iterator End = nullptr,
7328 ArrayRef<Register> PhySGPRs = {}) {
7329 assert((PhySGPRs.empty() || PhySGPRs.size() == ScalarOps.size()) &&
7330 "Physical SGPRs must be empty or match the number of scalar operands");
7331 MachineBasicBlock &MBB = *MI.getParent();
7332 MachineFunction &MF = *MBB.getParent();
7333 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
7334 const SIRegisterInfo *TRI = ST.getRegisterInfo();
7335 MachineRegisterInfo &MRI = MF.getRegInfo();
7336 if (!Begin.isValid())
7337 Begin = &MI;
7338 if (!End.isValid()) {
7339 End = &MI;
7340 ++End;
7341 }
7342 const DebugLoc &DL = MI.getDebugLoc();
7343 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
7344 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
7345
7346 // Save SCC. Waterfall Loop may overwrite SCC.
7347 Register SaveSCCReg;
7348
7349 // FIXME: We should maintain SCC liveness while doing the FixSGPRCopies walk
7350 // rather than unlimited scan everywhere
7351 bool SCCNotDead =
7352 MBB.computeRegisterLiveness(TRI, Reg: AMDGPU::SCC, Before: MI,
7353 Neighborhood: std::numeric_limits<unsigned>::max()) !=
7354 MachineBasicBlock::LQR_Dead;
7355 if (SCCNotDead) {
7356 SaveSCCReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7357 BuildMI(BB&: MBB, I: Begin, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: SaveSCCReg)
7358 .addImm(Val: 1)
7359 .addImm(Val: 0);
7360 }
7361
7362 Register SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7363
7364 // Save the EXEC mask
7365 BuildMI(BB&: MBB, I: Begin, MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: SaveExec).addReg(RegNo: LMC.ExecReg);
7366
7367 // Killed uses in the instruction we are waterfalling around will be
7368 // incorrect due to the added control-flow.
7369 MachineBasicBlock::iterator AfterMI = MI;
7370 ++AfterMI;
7371 for (auto I = Begin; I != AfterMI; I++) {
7372 for (auto &MO : I->all_uses())
7373 MRI.clearKillFlags(Reg: MO.getReg());
7374 }
7375
7376 // To insert the loop we need to split the block. Move everything after this
7377 // point to a new block, and insert a new empty block between the two.
7378 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
7379 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock();
7380 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
7381 MachineFunction::iterator MBBI(MBB);
7382 ++MBBI;
7383
7384 MF.insert(MBBI, MBB: LoopBB);
7385 MF.insert(MBBI, MBB: BodyBB);
7386 MF.insert(MBBI, MBB: RemainderBB);
7387
7388 LoopBB->addSuccessor(Succ: BodyBB);
7389 BodyBB->addSuccessor(Succ: LoopBB);
7390 BodyBB->addSuccessor(Succ: RemainderBB);
7391
7392 // Move Begin to MI to the BodyBB, and the remainder of the block to
7393 // RemainderBB.
7394 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
7395 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: End, To: MBB.end());
7396 BodyBB->splice(Where: BodyBB->begin(), Other: &MBB, From: Begin, To: MBB.end());
7397
7398 MBB.addSuccessor(Succ: LoopBB);
7399
7400 // Update dominators. We know that MBB immediately dominates LoopBB, that
7401 // LoopBB immediately dominates BodyBB, and BodyBB immediately dominates
7402 // RemainderBB. RemainderBB immediately dominates all of the successors
7403 // transferred to it from MBB that MBB used to properly dominate.
7404 if (MDT) {
7405 MDT->addNewBlock(BB: LoopBB, DomBB: &MBB);
7406 MDT->addNewBlock(BB: BodyBB, DomBB: LoopBB);
7407 MDT->addNewBlock(BB: RemainderBB, DomBB: BodyBB);
7408 for (auto &Succ : RemainderBB->successors()) {
7409 if (MDT->properlyDominates(A: &MBB, B: Succ)) {
7410 MDT->changeImmediateDominator(BB: Succ, NewBB: RemainderBB);
7411 }
7412 }
7413 }
7414
7415 emitLoadScalarOpsFromVGPRLoop(TII, MRI, PredBB&: MBB, LoopBB&: *LoopBB, BodyBB&: *BodyBB, DL, ScalarOps,
7416 PhySGPRs);
7417
7418 MachineBasicBlock::iterator First = RemainderBB->begin();
7419 // Restore SCC
7420 if (SCCNotDead) {
7421 BuildMI(BB&: *RemainderBB, I: First, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CMP_LG_U32))
7422 .addReg(RegNo: SaveSCCReg, Flags: RegState::Kill)
7423 .addImm(Val: 0);
7424 }
7425
7426 // Restore the EXEC mask
7427 BuildMI(BB&: *RemainderBB, I: First, MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
7428 .addReg(RegNo: SaveExec);
7429 return BodyBB;
7430}
7431
7432// Extract pointer from Rsrc and return a zero-value Rsrc replacement.
7433static std::tuple<unsigned, unsigned>
7434extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) {
7435 MachineBasicBlock &MBB = *MI.getParent();
7436 MachineFunction &MF = *MBB.getParent();
7437 MachineRegisterInfo &MRI = MF.getRegInfo();
7438
7439 // Extract the ptr from the resource descriptor.
7440 unsigned RsrcPtr =
7441 TII.buildExtractSubReg(MI, MRI, SuperReg: Rsrc, SuperRC: &AMDGPU::VReg_128RegClass,
7442 SubIdx: AMDGPU::sub0_sub1, SubRC: &AMDGPU::VReg_64RegClass);
7443
7444 // Create an empty resource descriptor
7445 Register Zero64 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
7446 Register SRsrcFormatLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
7447 Register SRsrcFormatHi = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
7448 Register NewSRsrc = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_128RegClass);
7449 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
7450
7451 // Zero64 = 0
7452 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B64), DestReg: Zero64)
7453 .addImm(Val: 0);
7454
7455 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
7456 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: SRsrcFormatLo)
7457 .addImm(Val: Lo_32(Value: RsrcDataFormat));
7458
7459 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
7460 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: SRsrcFormatHi)
7461 .addImm(Val: Hi_32(Value: RsrcDataFormat));
7462
7463 // NewSRsrc = {Zero64, SRsrcFormat}
7464 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewSRsrc)
7465 .addReg(RegNo: Zero64)
7466 .addImm(Val: AMDGPU::sub0_sub1)
7467 .addReg(RegNo: SRsrcFormatLo)
7468 .addImm(Val: AMDGPU::sub2)
7469 .addReg(RegNo: SRsrcFormatHi)
7470 .addImm(Val: AMDGPU::sub3);
7471
7472 return std::tuple(RsrcPtr, NewSRsrc);
7473}
7474
7475MachineBasicBlock *
7476SIInstrInfo::legalizeOperands(MachineInstr &MI,
7477 MachineDominatorTree *MDT) const {
7478 MachineFunction &MF = *MI.getMF();
7479 MachineRegisterInfo &MRI = MF.getRegInfo();
7480 MachineBasicBlock *CreatedBB = nullptr;
7481
7482 // Legalize VOP2
7483 if (isVOP2(MI) || isVOPC(MI)) {
7484 legalizeOperandsVOP2(MRI, MI);
7485 return CreatedBB;
7486 }
7487
7488 // Legalize VOP3
7489 if (isVOP3(MI)) {
7490 legalizeOperandsVOP3(MRI, MI);
7491 return CreatedBB;
7492 }
7493
7494 // Legalize SMRD
7495 if (isSMRD(MI)) {
7496 legalizeOperandsSMRD(MRI, MI);
7497 return CreatedBB;
7498 }
7499
7500 // Legalize FLAT
7501 if (isFLAT(MI)) {
7502 legalizeOperandsFLAT(MRI, MI);
7503 return CreatedBB;
7504 }
7505
7506 // Legalize PHI
7507 // The register class of the operands must be the same type as the register
7508 // class of the output.
7509 if (MI.getOpcode() == AMDGPU::PHI) {
7510 const TargetRegisterClass *VRC = getOpRegClass(MI, OpNo: 0);
7511 assert(!RI.isSGPRClass(VRC));
7512
7513 // Update all the operands so they have the same type.
7514 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
7515 MachineOperand &Op = MI.getOperand(i: I);
7516 if (!Op.isReg() || !Op.getReg().isVirtual())
7517 continue;
7518
7519 // MI is a PHI instruction.
7520 MachineBasicBlock *InsertBB = MI.getOperand(i: I + 1).getMBB();
7521 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
7522
7523 // Avoid creating no-op copies with the same src and dst reg class. These
7524 // confuse some of the machine passes.
7525 legalizeGenericOperand(InsertMBB&: *InsertBB, I: Insert, DstRC: VRC, Op, MRI, DL: MI.getDebugLoc());
7526 }
7527 }
7528
7529 // REG_SEQUENCE doesn't really require operand legalization, but if one has a
7530 // VGPR dest type and SGPR sources, insert copies so all operands are
7531 // VGPRs. This seems to help operand folding / the register coalescer.
7532 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
7533 MachineBasicBlock *MBB = MI.getParent();
7534 const TargetRegisterClass *DstRC = getOpRegClass(MI, OpNo: 0);
7535 if (RI.hasVGPRs(RC: DstRC)) {
7536 // Update all the operands so they are VGPR register classes. These may
7537 // not be the same register class because REG_SEQUENCE supports mixing
7538 // subregister index types e.g. sub0_sub1 + sub2 + sub3
7539 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
7540 MachineOperand &Op = MI.getOperand(i: I);
7541 if (!Op.isReg() || !Op.getReg().isVirtual())
7542 continue;
7543
7544 const TargetRegisterClass *OpRC = MRI.getRegClass(Reg: Op.getReg());
7545 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(SRC: OpRC);
7546 if (VRC == OpRC)
7547 continue;
7548
7549 legalizeGenericOperand(InsertMBB&: *MBB, I: MI, DstRC: VRC, Op, MRI, DL: MI.getDebugLoc());
7550 Op.setIsKill();
7551 }
7552 }
7553
7554 return CreatedBB;
7555 }
7556
7557 // Legalize INSERT_SUBREG
7558 // src0 must have the same register class as dst
7559 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
7560 Register Dst = MI.getOperand(i: 0).getReg();
7561 Register Src0 = MI.getOperand(i: 1).getReg();
7562 const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: Dst);
7563 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0);
7564 if (DstRC != Src0RC) {
7565 MachineBasicBlock *MBB = MI.getParent();
7566 MachineOperand &Op = MI.getOperand(i: 1);
7567 legalizeGenericOperand(InsertMBB&: *MBB, I: MI, DstRC, Op, MRI, DL: MI.getDebugLoc());
7568 }
7569 return CreatedBB;
7570 }
7571
7572 // Legalize SI_INIT_M0
7573 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
7574 MachineOperand &Src = MI.getOperand(i: 0);
7575 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
7576 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
7577 return CreatedBB;
7578 }
7579
7580 // Legalize S_BITREPLICATE, S_QUADMASK and S_WQM
7581 if (MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 ||
7582 MI.getOpcode() == AMDGPU::S_QUADMASK_B32 ||
7583 MI.getOpcode() == AMDGPU::S_QUADMASK_B64 ||
7584 MI.getOpcode() == AMDGPU::S_WQM_B32 ||
7585 MI.getOpcode() == AMDGPU::S_WQM_B64 ||
7586 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U32 ||
7587 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U64) {
7588 MachineOperand &Src = MI.getOperand(i: 1);
7589 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
7590 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
7591 return CreatedBB;
7592 }
7593
7594 // Legalize MIMG/VIMAGE/VSAMPLE and MUBUF/MTBUF for shaders.
7595 //
7596 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
7597 // scratch memory access. In both cases, the legalization never involves
7598 // conversion to the addr64 form.
7599 if (isImage(MI) || (AMDGPU::isGraphics(CC: MF.getFunction().getCallingConv()) &&
7600 (isMUBUF(MI) || isMTBUF(MI)))) {
7601 AMDGPU::OpName RSrcOpName = (isVIMAGE(MI) || isVSAMPLE(MI))
7602 ? AMDGPU::OpName::rsrc
7603 : AMDGPU::OpName::srsrc;
7604 MachineOperand *SRsrc = getNamedOperand(MI, OperandName: RSrcOpName);
7605 if (SRsrc && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SRsrc->getReg())))
7606 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {SRsrc}, MDT);
7607
7608 AMDGPU::OpName SampOpName =
7609 isMIMG(MI) ? AMDGPU::OpName::ssamp : AMDGPU::OpName::samp;
7610 MachineOperand *SSamp = getNamedOperand(MI, OperandName: SampOpName);
7611 if (SSamp && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SSamp->getReg())))
7612 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {SSamp}, MDT);
7613
7614 return CreatedBB;
7615 }
7616
7617 // Legalize SI_CALL
7618 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) {
7619 MachineOperand *Dest = &MI.getOperand(i: 0);
7620 if (!RI.isSGPRClass(RC: MRI.getRegClass(Reg: Dest->getReg()))) {
7621 createWaterFallForSiCall(MI: &MI, MDT, ScalarOps: {Dest});
7622 }
7623 }
7624
7625 // Legalize s_sleep_var.
7626 if (MI.getOpcode() == AMDGPU::S_SLEEP_VAR) {
7627 const DebugLoc &DL = MI.getDebugLoc();
7628 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7629 int Src0Idx =
7630 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
7631 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
7632 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7633 .add(MO: Src0);
7634 Src0.ChangeToRegister(Reg, isDef: false);
7635 return nullptr;
7636 }
7637
7638 // Legalize TENSOR_LOAD_TO_LDS_d2/_d4, TENSOR_STORE_FROM_LDS_d2/_d4. All their
7639 // operands are scalar.
7640 if (MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_d2 ||
7641 MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_d4 ||
7642 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_d2 ||
7643 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_d4) {
7644 for (MachineOperand &Src : MI.explicit_operands()) {
7645 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
7646 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
7647 }
7648 return CreatedBB;
7649 }
7650
7651 // Legalize MUBUF instructions.
7652 bool isSoffsetLegal = true;
7653 int SoffsetIdx =
7654 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::soffset);
7655 if (SoffsetIdx != -1) {
7656 MachineOperand *Soffset = &MI.getOperand(i: SoffsetIdx);
7657 if (Soffset->isReg() && Soffset->getReg().isVirtual() &&
7658 !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Soffset->getReg()))) {
7659 isSoffsetLegal = false;
7660 }
7661 }
7662
7663 bool isRsrcLegal = true;
7664 int RsrcIdx =
7665 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::srsrc);
7666 if (RsrcIdx != -1) {
7667 MachineOperand *Rsrc = &MI.getOperand(i: RsrcIdx);
7668 if (Rsrc->isReg() && !RI.isSGPRReg(MRI, Reg: Rsrc->getReg()))
7669 isRsrcLegal = false;
7670 }
7671
7672 // The operands are legal.
7673 if (isRsrcLegal && isSoffsetLegal)
7674 return CreatedBB;
7675
7676 if (!isRsrcLegal) {
7677 // Legalize a VGPR Rsrc
7678 //
7679 // If the instruction is _ADDR64, we can avoid a waterfall by extracting
7680 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
7681 // a zero-value SRsrc.
7682 //
7683 // If the instruction is _OFFSET (both idxen and offen disabled), and we
7684 // support ADDR64 instructions, we can convert to ADDR64 and do the same as
7685 // above.
7686 //
7687 // Otherwise we are on non-ADDR64 hardware, and/or we have
7688 // idxen/offen/bothen and we fall back to a waterfall loop.
7689
7690 MachineOperand *Rsrc = &MI.getOperand(i: RsrcIdx);
7691 MachineBasicBlock &MBB = *MI.getParent();
7692
7693 MachineOperand *VAddr = getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
7694 if (VAddr && AMDGPU::getIfAddr64Inst(Opcode: MI.getOpcode()) != -1) {
7695 // This is already an ADDR64 instruction so we need to add the pointer
7696 // extracted from the resource descriptor to the current value of VAddr.
7697 Register NewVAddrLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7698 Register NewVAddrHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7699 Register NewVAddr = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
7700
7701 const auto *BoolXExecRC = RI.getWaveMaskRegClass();
7702 Register CondReg0 = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7703 Register CondReg1 = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7704
7705 unsigned RsrcPtr, NewSRsrc;
7706 std::tie(args&: RsrcPtr, args&: NewSRsrc) = extractRsrcPtr(TII: *this, MI, Rsrc&: *Rsrc);
7707
7708 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
7709 const DebugLoc &DL = MI.getDebugLoc();
7710 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg: NewVAddrLo)
7711 .addDef(RegNo: CondReg0)
7712 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub0)
7713 .addReg(RegNo: VAddr->getReg(), Flags: {}, SubReg: AMDGPU::sub0)
7714 .addImm(Val: 0);
7715
7716 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
7717 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADDC_U32_e64), DestReg: NewVAddrHi)
7718 .addDef(RegNo: CondReg1, Flags: RegState::Dead)
7719 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub1)
7720 .addReg(RegNo: VAddr->getReg(), Flags: {}, SubReg: AMDGPU::sub1)
7721 .addReg(RegNo: CondReg0, Flags: RegState::Kill)
7722 .addImm(Val: 0);
7723
7724 // NewVaddr = {NewVaddrHi, NewVaddrLo}
7725 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewVAddr)
7726 .addReg(RegNo: NewVAddrLo)
7727 .addImm(Val: AMDGPU::sub0)
7728 .addReg(RegNo: NewVAddrHi)
7729 .addImm(Val: AMDGPU::sub1);
7730
7731 VAddr->setReg(NewVAddr);
7732 Rsrc->setReg(NewSRsrc);
7733 } else if (!VAddr && ST.hasAddr64()) {
7734 // This instructions is the _OFFSET variant, so we need to convert it to
7735 // ADDR64.
7736 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS &&
7737 "FIXME: Need to emit flat atomics here");
7738
7739 unsigned RsrcPtr, NewSRsrc;
7740 std::tie(args&: RsrcPtr, args&: NewSRsrc) = extractRsrcPtr(TII: *this, MI, Rsrc&: *Rsrc);
7741
7742 Register NewVAddr = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
7743 MachineOperand *VData = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata);
7744 MachineOperand *Offset = getNamedOperand(MI, OperandName: AMDGPU::OpName::offset);
7745 MachineOperand *SOffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
7746 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(Opcode: MI.getOpcode());
7747
7748 // Atomics with return have an additional tied operand and are
7749 // missing some of the special bits.
7750 MachineOperand *VDataIn = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata_in);
7751 MachineInstr *Addr64;
7752
7753 if (!VDataIn) {
7754 // Regular buffer load / store.
7755 MachineInstrBuilder MIB =
7756 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Addr64Opcode))
7757 .add(MO: *VData)
7758 .addReg(RegNo: NewVAddr)
7759 .addReg(RegNo: NewSRsrc)
7760 .add(MO: *SOffset)
7761 .add(MO: *Offset);
7762
7763 if (const MachineOperand *CPol =
7764 getNamedOperand(MI, OperandName: AMDGPU::OpName::cpol)) {
7765 MIB.addImm(Val: CPol->getImm());
7766 }
7767
7768 if (const MachineOperand *TFE =
7769 getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe)) {
7770 MIB.addImm(Val: TFE->getImm());
7771 }
7772
7773 MIB.addImm(Val: getNamedImmOperand(MI, OperandName: AMDGPU::OpName::swz));
7774
7775 MIB.cloneMemRefs(OtherMI: MI);
7776 Addr64 = MIB;
7777 } else {
7778 // Atomics with return.
7779 Addr64 = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Addr64Opcode))
7780 .add(MO: *VData)
7781 .add(MO: *VDataIn)
7782 .addReg(RegNo: NewVAddr)
7783 .addReg(RegNo: NewSRsrc)
7784 .add(MO: *SOffset)
7785 .add(MO: *Offset)
7786 .addImm(Val: getNamedImmOperand(MI, OperandName: AMDGPU::OpName::cpol))
7787 .cloneMemRefs(OtherMI: MI);
7788 }
7789
7790 MI.removeFromParent();
7791
7792 // NewVaddr = {NewVaddrHi, NewVaddrLo}
7793 BuildMI(BB&: MBB, I: Addr64, MIMD: Addr64->getDebugLoc(), MCID: get(Opcode: AMDGPU::REG_SEQUENCE),
7794 DestReg: NewVAddr)
7795 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub0)
7796 .addImm(Val: AMDGPU::sub0)
7797 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub1)
7798 .addImm(Val: AMDGPU::sub1);
7799 } else {
7800 // Legalize a VGPR Rsrc and soffset together.
7801 if (!isSoffsetLegal) {
7802 MachineOperand *Soffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
7803 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Rsrc, Soffset}, MDT);
7804 return CreatedBB;
7805 }
7806 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Rsrc}, MDT);
7807 return CreatedBB;
7808 }
7809 }
7810
7811 // Legalize a VGPR soffset.
7812 if (!isSoffsetLegal) {
7813 MachineOperand *Soffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
7814 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Soffset}, MDT);
7815 return CreatedBB;
7816 }
7817 return CreatedBB;
7818}
7819
7820void SIInstrWorklist::insert(MachineInstr *MI) {
7821 InstrList.insert(X: MI);
7822 // Add MBUF instructiosn to deferred list.
7823 int RsrcIdx =
7824 AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(), Name: AMDGPU::OpName::srsrc);
7825 if (RsrcIdx != -1) {
7826 DeferredList.insert(X: MI);
7827 }
7828}
7829
7830bool SIInstrWorklist::isDeferred(MachineInstr *MI) {
7831 return DeferredList.contains(key: MI);
7832}
7833
7834// Legalize size mismatches between 16bit and 32bit registers in v2s copy
7835// lowering (change sgpr to vgpr).
7836// This is mainly caused by 16bit SALU and 16bit VALU using reg with different
7837// size. Need to legalize the size of the operands during the vgpr lowering
7838// chain. This can be removed after we have sgpr16 in place
7839void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &MI, unsigned OpIdx,
7840 MachineRegisterInfo &MRI) const {
7841 if (!ST.useRealTrue16Insts())
7842 return;
7843
7844 unsigned Opcode = MI.getOpcode();
7845 MachineBasicBlock *MBB = MI.getParent();
7846 // Legalize operands and check for size mismatch
7847 if (!OpIdx || OpIdx >= MI.getNumExplicitOperands() ||
7848 OpIdx >= get(Opcode).getNumOperands() ||
7849 get(Opcode).operands()[OpIdx].RegClass == -1)
7850 return;
7851
7852 MachineOperand &Op = MI.getOperand(i: OpIdx);
7853 if (!Op.isReg() || !Op.getReg().isVirtual())
7854 return;
7855
7856 const TargetRegisterClass *CurrRC = MRI.getRegClass(Reg: Op.getReg());
7857 if (!RI.isVGPRClass(RC: CurrRC))
7858 return;
7859
7860 int16_t RCID = getOpRegClassID(OpInfo: get(Opcode).operands()[OpIdx]);
7861 const TargetRegisterClass *ExpectedRC = RI.getRegClass(i: RCID);
7862 if (RI.getMatchingSuperRegClass(A: CurrRC, B: ExpectedRC, Idx: AMDGPU::lo16)) {
7863 Op.setSubReg(AMDGPU::lo16);
7864 } else if (RI.getMatchingSuperRegClass(A: ExpectedRC, B: CurrRC, Idx: AMDGPU::lo16)) {
7865 const DebugLoc &DL = MI.getDebugLoc();
7866 Register NewDstReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7867 Register Undef = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
7868 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
7869 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewDstReg)
7870 .addReg(RegNo: Op.getReg())
7871 .addImm(Val: AMDGPU::lo16)
7872 .addReg(RegNo: Undef)
7873 .addImm(Val: AMDGPU::hi16);
7874 Op.setReg(NewDstReg);
7875 }
7876}
7877void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &MI,
7878 MachineRegisterInfo &MRI) const {
7879 for (unsigned OpIdx = 1; OpIdx < MI.getNumExplicitOperands(); OpIdx++)
7880 legalizeOperandsVALUt16(MI, OpIdx, MRI);
7881}
7882
7883void SIInstrInfo::createWaterFallForSiCall(MachineInstr *MI,
7884 MachineDominatorTree *MDT,
7885 ArrayRef<MachineOperand *> ScalarOps,
7886 ArrayRef<Register> PhySGPRs) const {
7887 assert(MI->getOpcode() == AMDGPU::SI_CALL_ISEL &&
7888 "This only handle waterfall for SI_CALL_ISEL");
7889 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and
7890 // following copies, we also need to move copies from and to physical
7891 // registers into the loop block.
7892 // Also move the copies to physical registers into the loop block
7893 MachineBasicBlock &MBB = *MI->getParent();
7894 MachineBasicBlock::iterator Start(MI);
7895 while (Start->getOpcode() != AMDGPU::ADJCALLSTACKUP)
7896 --Start;
7897 MachineBasicBlock::iterator End(MI);
7898 while (End->getOpcode() != AMDGPU::ADJCALLSTACKDOWN)
7899 ++End;
7900
7901 // Also include following copies of the return value
7902 ++End;
7903 while (End != MBB.end() && End->isCopy() &&
7904 MI->definesRegister(Reg: End->getOperand(i: 1).getReg(), TRI: &RI))
7905 ++End;
7906
7907 generateWaterFallLoop(TII: *this, MI&: *MI, ScalarOps, MDT, Begin: Start, End, PhySGPRs);
7908}
7909
7910void SIInstrInfo::moveToVALU(SIInstrWorklist &Worklist,
7911 MachineDominatorTree *MDT) const {
7912 DenseMap<MachineInstr *, V2PhysSCopyInfo> WaterFalls;
7913 DenseMap<MachineInstr *, bool> V2SPhyCopiesToErase;
7914 while (!Worklist.empty()) {
7915 MachineInstr &Inst = *Worklist.top();
7916 Worklist.erase_top();
7917 // Skip MachineInstr in the deferred list.
7918 if (Worklist.isDeferred(MI: &Inst))
7919 continue;
7920 moveToVALUImpl(Worklist, MDT, Inst, WaterFalls, V2SPhyCopiesToErase);
7921 }
7922
7923 // Deferred list of instructions will be processed once
7924 // all the MachineInstr in the worklist are done.
7925 for (MachineInstr *Inst : Worklist.getDeferredList()) {
7926 moveToVALUImpl(Worklist, MDT, Inst&: *Inst, WaterFalls, V2SPhyCopiesToErase);
7927 assert(Worklist.empty() &&
7928 "Deferred MachineInstr are not supposed to re-populate worklist");
7929 }
7930
7931 for (std::pair<MachineInstr *, V2PhysSCopyInfo> &Entry : WaterFalls) {
7932 if (Entry.first->getOpcode() == AMDGPU::SI_CALL_ISEL)
7933 createWaterFallForSiCall(MI: Entry.first, MDT, ScalarOps: Entry.second.MOs,
7934 PhySGPRs: Entry.second.SGPRs);
7935 }
7936
7937 for (std::pair<MachineInstr *, bool> Entry : V2SPhyCopiesToErase)
7938 if (Entry.second)
7939 Entry.first->eraseFromParent();
7940}
7941void SIInstrInfo::createReadFirstLaneFromCopyToPhysReg(
7942 MachineRegisterInfo &MRI, Register DstReg, MachineInstr &Inst) const {
7943 // If it's a copy of a VGPR to a physical SGPR, insert a V_READFIRSTLANE and
7944 // hope for the best.
7945 const TargetRegisterClass *DstRC = RI.getRegClassForReg(MRI, Reg: DstReg);
7946 ArrayRef<int16_t> SubRegIndices = RI.getRegSplitParts(RC: DstRC, EltSize: 4);
7947 if (SubRegIndices.size() <= 1) {
7948 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7949 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
7950 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: NewDst)
7951 .add(MO: Inst.getOperand(i: 1));
7952 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
7953 DestReg: DstReg)
7954 .addReg(RegNo: NewDst);
7955 } else {
7956 SmallVector<Register, 8> DstRegs;
7957 for (int16_t Indice : SubRegIndices) {
7958 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7959 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
7960 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: NewDst)
7961 .addReg(RegNo: Inst.getOperand(i: 1).getReg(), Flags: {}, SubReg: Indice);
7962
7963 DstRegs.push_back(Elt: NewDst);
7964 }
7965 MachineInstrBuilder MIB =
7966 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
7967 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
7968 for (unsigned i = 0; i < SubRegIndices.size(); ++i) {
7969 MIB.addReg(RegNo: DstRegs[i]);
7970 MIB.addImm(Val: RI.getSubRegFromChannel(Channel: i));
7971 }
7972 }
7973}
7974
7975void SIInstrInfo::handleCopyToPhysHelper(
7976 SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst,
7977 MachineRegisterInfo &MRI,
7978 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
7979 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const {
7980 if (DstReg == AMDGPU::M0) {
7981 createReadFirstLaneFromCopyToPhysReg(MRI, DstReg, Inst);
7982 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
7983 return;
7984 }
7985 Register SrcReg = Inst.getOperand(i: 1).getReg();
7986 MachineBasicBlock::iterator I = Inst.getIterator();
7987 MachineBasicBlock::iterator E = Inst.getParent()->end();
7988 // Only search current block since phyreg's def & use cannot cross
7989 // blocks when MF.NoPhi = false.
7990 while (++I != E) {
7991 // For SI_CALL_ISEL users, replace the phys SGPR with the VGPR source
7992 // and record the operand for later waterfall loop generation.
7993 if (I->getOpcode() == AMDGPU::SI_CALL_ISEL) {
7994 MachineInstr *UseMI = &*I;
7995 for (unsigned i = 0; i < UseMI->getNumOperands(); ++i) {
7996 if (UseMI->getOperand(i).isReg() &&
7997 UseMI->getOperand(i).getReg() == DstReg) {
7998 MachineOperand *MO = &UseMI->getOperand(i);
7999 MO->setReg(SrcReg);
8000 V2PhysSCopyInfo &V2SCopyInfo = WaterFalls[UseMI];
8001 V2SCopyInfo.MOs.push_back(Elt: MO);
8002 V2SCopyInfo.SGPRs.push_back(Elt: DstReg);
8003 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
8004 }
8005 }
8006 } else if (I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG &&
8007 I->getOperand(i: 0).isReg() &&
8008 I->getOperand(i: 0).getReg() == DstReg) {
8009 createReadFirstLaneFromCopyToPhysReg(MRI, DstReg, Inst);
8010 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
8011 } else if (I->readsRegister(Reg: DstReg, TRI: &RI)) {
8012 // COPY cannot be erased if other type of inst uses it.
8013 V2SPhyCopiesToErase[&Inst] = false;
8014 }
8015 if (I->findRegisterDefOperand(Reg: DstReg, TRI: &RI))
8016 break;
8017 }
8018}
8019
8020void SIInstrInfo::moveToVALUImpl(
8021 SIInstrWorklist &Worklist, MachineDominatorTree *MDT, MachineInstr &Inst,
8022 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
8023 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const {
8024
8025 MachineBasicBlock *MBB = Inst.getParent();
8026 if (!MBB)
8027 return;
8028 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
8029 unsigned Opcode = Inst.getOpcode();
8030 unsigned NewOpcode = getVALUOp(MI: Inst);
8031 const DebugLoc &DL = Inst.getDebugLoc();
8032
8033 // Handle some special cases
8034 switch (Opcode) {
8035 default:
8036 break;
8037 case AMDGPU::S_ADD_I32:
8038 case AMDGPU::S_SUB_I32: {
8039 // FIXME: The u32 versions currently selected use the carry.
8040 bool Changed;
8041 MachineBasicBlock *CreatedBBTmp = nullptr;
8042 std::tie(args&: Changed, args&: CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT);
8043 if (Changed)
8044 return;
8045
8046 // Default handling
8047 break;
8048 }
8049
8050 case AMDGPU::S_MUL_U64:
8051 if (ST.hasVMulU64Inst()) {
8052 NewOpcode = AMDGPU::V_MUL_U64_e64;
8053 break;
8054 }
8055 // Split s_mul_u64 in 32-bit vector multiplications.
8056 splitScalarSMulU64(Worklist, Inst, MDT);
8057 Inst.eraseFromParent();
8058 return;
8059
8060 case AMDGPU::S_MUL_U64_U32_PSEUDO:
8061 case AMDGPU::S_MUL_I64_I32_PSEUDO:
8062 // This is a special case of s_mul_u64 where all the operands are either
8063 // zero extended or sign extended.
8064 splitScalarSMulPseudo(Worklist, Inst, MDT);
8065 Inst.eraseFromParent();
8066 return;
8067
8068 case AMDGPU::S_AND_B64:
8069 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_AND_B32, MDT);
8070 Inst.eraseFromParent();
8071 return;
8072
8073 case AMDGPU::S_OR_B64:
8074 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_OR_B32, MDT);
8075 Inst.eraseFromParent();
8076 return;
8077
8078 case AMDGPU::S_XOR_B64:
8079 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_XOR_B32, MDT);
8080 Inst.eraseFromParent();
8081 return;
8082
8083 case AMDGPU::S_NAND_B64:
8084 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_NAND_B32, MDT);
8085 Inst.eraseFromParent();
8086 return;
8087
8088 case AMDGPU::S_NOR_B64:
8089 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_NOR_B32, MDT);
8090 Inst.eraseFromParent();
8091 return;
8092
8093 case AMDGPU::S_XNOR_B64:
8094 if (ST.hasDLInsts())
8095 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_XNOR_B32, MDT);
8096 else
8097 splitScalar64BitXnor(Worklist, Inst, MDT);
8098 Inst.eraseFromParent();
8099 return;
8100
8101 case AMDGPU::S_ANDN2_B64:
8102 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_ANDN2_B32, MDT);
8103 Inst.eraseFromParent();
8104 return;
8105
8106 case AMDGPU::S_ORN2_B64:
8107 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_ORN2_B32, MDT);
8108 Inst.eraseFromParent();
8109 return;
8110
8111 case AMDGPU::S_BREV_B64:
8112 splitScalar64BitUnaryOp(Worklist, Inst, Opcode: AMDGPU::S_BREV_B32, Swap: true);
8113 Inst.eraseFromParent();
8114 return;
8115
8116 case AMDGPU::S_NOT_B64:
8117 splitScalar64BitUnaryOp(Worklist, Inst, Opcode: AMDGPU::S_NOT_B32);
8118 Inst.eraseFromParent();
8119 return;
8120
8121 case AMDGPU::S_BCNT1_I32_B64:
8122 splitScalar64BitBCNT(Worklist, Inst);
8123 Inst.eraseFromParent();
8124 return;
8125
8126 case AMDGPU::S_BFE_I64:
8127 splitScalar64BitBFE(Worklist, Inst);
8128 Inst.eraseFromParent();
8129 return;
8130
8131 case AMDGPU::S_FLBIT_I32_B64:
8132 splitScalar64BitCountOp(Worklist, Inst, Opcode: AMDGPU::V_FFBH_U32_e32);
8133 Inst.eraseFromParent();
8134 return;
8135 case AMDGPU::S_FF1_I32_B64:
8136 splitScalar64BitCountOp(Worklist, Inst, Opcode: AMDGPU::V_FFBL_B32_e32);
8137 Inst.eraseFromParent();
8138 return;
8139
8140 case AMDGPU::S_LSHL_B32:
8141 if (ST.hasOnlyRevVALUShifts()) {
8142 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
8143 swapOperands(Inst);
8144 }
8145 break;
8146 case AMDGPU::S_ASHR_I32:
8147 if (ST.hasOnlyRevVALUShifts()) {
8148 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
8149 swapOperands(Inst);
8150 }
8151 break;
8152 case AMDGPU::S_LSHR_B32:
8153 if (ST.hasOnlyRevVALUShifts()) {
8154 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
8155 swapOperands(Inst);
8156 }
8157 break;
8158 case AMDGPU::S_LSHL_B64:
8159 if (ST.hasOnlyRevVALUShifts()) {
8160 NewOpcode = ST.getGeneration() >= AMDGPUSubtarget::GFX12
8161 ? AMDGPU::V_LSHLREV_B64_pseudo_e64
8162 : AMDGPU::V_LSHLREV_B64_e64;
8163 swapOperands(Inst);
8164 }
8165 break;
8166 case AMDGPU::S_ASHR_I64:
8167 if (ST.hasOnlyRevVALUShifts()) {
8168 NewOpcode = AMDGPU::V_ASHRREV_I64_e64;
8169 swapOperands(Inst);
8170 }
8171 break;
8172 case AMDGPU::S_LSHR_B64:
8173 if (ST.hasOnlyRevVALUShifts()) {
8174 NewOpcode = AMDGPU::V_LSHRREV_B64_e64;
8175 swapOperands(Inst);
8176 }
8177 break;
8178
8179 case AMDGPU::S_ABS_I32:
8180 lowerScalarAbs(Worklist, Inst);
8181 Inst.eraseFromParent();
8182 return;
8183
8184 case AMDGPU::S_ABSDIFF_I32:
8185 lowerScalarAbsDiff(Worklist, Inst);
8186 Inst.eraseFromParent();
8187 return;
8188
8189 case AMDGPU::S_CBRANCH_SCC0:
8190 case AMDGPU::S_CBRANCH_SCC1: {
8191 // Clear unused bits of vcc
8192 Register CondReg = Inst.getOperand(i: 1).getReg();
8193 bool IsSCC = CondReg == AMDGPU::SCC;
8194 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
8195 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: LMC.AndOpc), DestReg: LMC.VccReg)
8196 .addReg(RegNo: LMC.ExecReg)
8197 .addReg(RegNo: IsSCC ? LMC.VccReg : CondReg);
8198 Inst.removeOperand(OpNo: 1);
8199 } break;
8200
8201 case AMDGPU::S_BFE_U64:
8202 case AMDGPU::S_BFM_B64:
8203 llvm_unreachable("Moving this op to VALU not implemented");
8204
8205 case AMDGPU::S_PACK_LL_B32_B16:
8206 case AMDGPU::S_PACK_LH_B32_B16:
8207 case AMDGPU::S_PACK_HL_B32_B16:
8208 case AMDGPU::S_PACK_HH_B32_B16:
8209 movePackToVALU(Worklist, MRI, Inst);
8210 Inst.eraseFromParent();
8211 return;
8212
8213 case AMDGPU::S_XNOR_B32:
8214 lowerScalarXnor(Worklist, Inst);
8215 Inst.eraseFromParent();
8216 return;
8217
8218 case AMDGPU::S_NAND_B32:
8219 splitScalarNotBinop(Worklist, Inst, Opcode: AMDGPU::S_AND_B32);
8220 Inst.eraseFromParent();
8221 return;
8222
8223 case AMDGPU::S_NOR_B32:
8224 splitScalarNotBinop(Worklist, Inst, Opcode: AMDGPU::S_OR_B32);
8225 Inst.eraseFromParent();
8226 return;
8227
8228 case AMDGPU::S_ANDN2_B32:
8229 splitScalarBinOpN2(Worklist, Inst, Opcode: AMDGPU::S_AND_B32);
8230 Inst.eraseFromParent();
8231 return;
8232
8233 case AMDGPU::S_ORN2_B32:
8234 splitScalarBinOpN2(Worklist, Inst, Opcode: AMDGPU::S_OR_B32);
8235 Inst.eraseFromParent();
8236 return;
8237
8238 // TODO: remove as soon as everything is ready
8239 // to replace VGPR to SGPR copy with V_READFIRSTLANEs.
8240 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO
8241 // can only be selected from the uniform SDNode.
8242 case AMDGPU::S_ADD_CO_PSEUDO:
8243 case AMDGPU::S_SUB_CO_PSEUDO: {
8244 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
8245 ? AMDGPU::V_ADDC_U32_e64
8246 : AMDGPU::V_SUBB_U32_e64;
8247 const auto *CarryRC = RI.getWaveMaskRegClass();
8248
8249 Register CarryInReg = Inst.getOperand(i: 4).getReg();
8250 if (!MRI.constrainRegClass(Reg: CarryInReg, RC: CarryRC)) {
8251 Register NewCarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
8252 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY), DestReg: NewCarryReg)
8253 .addReg(RegNo: CarryInReg);
8254 }
8255
8256 Register CarryOutReg = Inst.getOperand(i: 1).getReg();
8257
8258 Register DestReg = MRI.createVirtualRegister(RegClass: RI.getEquivalentVGPRClass(
8259 SRC: MRI.getRegClass(Reg: Inst.getOperand(i: 0).getReg())));
8260 MachineInstr *CarryOp =
8261 BuildMI(BB&: *MBB, I: &Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: Opc), DestReg)
8262 .addReg(RegNo: CarryOutReg, Flags: RegState::Define)
8263 .add(MO: Inst.getOperand(i: 2))
8264 .add(MO: Inst.getOperand(i: 3))
8265 .addReg(RegNo: CarryInReg)
8266 .addImm(Val: 0);
8267 legalizeOperands(MI&: *CarryOp);
8268 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: DestReg);
8269 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8270 Inst.eraseFromParent();
8271 }
8272 return;
8273 case AMDGPU::S_UADDO_PSEUDO:
8274 case AMDGPU::S_USUBO_PSEUDO: {
8275 MachineOperand &Dest0 = Inst.getOperand(i: 0);
8276 MachineOperand &Dest1 = Inst.getOperand(i: 1);
8277 MachineOperand &Src0 = Inst.getOperand(i: 2);
8278 MachineOperand &Src1 = Inst.getOperand(i: 3);
8279
8280 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
8281 ? AMDGPU::V_ADD_CO_U32_e64
8282 : AMDGPU::V_SUB_CO_U32_e64;
8283 const TargetRegisterClass *NewRC =
8284 RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest0.getReg()));
8285 Register DestReg = MRI.createVirtualRegister(RegClass: NewRC);
8286 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I: &Inst, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
8287 .addReg(RegNo: Dest1.getReg(), Flags: RegState::Define)
8288 .add(MO: Src0)
8289 .add(MO: Src1)
8290 .addImm(Val: 0); // clamp bit
8291
8292 legalizeOperands(MI&: *NewInstr, MDT);
8293 MRI.replaceRegWith(FromReg: Dest0.getReg(), ToReg: DestReg);
8294 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8295 Inst.eraseFromParent();
8296 }
8297 return;
8298 case AMDGPU::S_LSHL1_ADD_U32:
8299 case AMDGPU::S_LSHL2_ADD_U32:
8300 case AMDGPU::S_LSHL3_ADD_U32:
8301 case AMDGPU::S_LSHL4_ADD_U32: {
8302 MachineOperand &Dest = Inst.getOperand(i: 0);
8303 MachineOperand &Src0 = Inst.getOperand(i: 1);
8304 MachineOperand &Src1 = Inst.getOperand(i: 2);
8305 unsigned ShiftAmt = (Opcode == AMDGPU::S_LSHL1_ADD_U32 ? 1
8306 : Opcode == AMDGPU::S_LSHL2_ADD_U32 ? 2
8307 : Opcode == AMDGPU::S_LSHL3_ADD_U32 ? 3
8308 : 4);
8309
8310 const TargetRegisterClass *NewRC =
8311 RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest.getReg()));
8312 Register DestReg = MRI.createVirtualRegister(RegClass: NewRC);
8313 MachineInstr *NewInstr =
8314 BuildMI(BB&: *MBB, I: &Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_ADD_U32_e64), DestReg)
8315 .add(MO: Src0)
8316 .addImm(Val: ShiftAmt)
8317 .add(MO: Src1);
8318
8319 legalizeOperands(MI&: *NewInstr, MDT);
8320 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: DestReg);
8321 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8322 Inst.eraseFromParent();
8323 }
8324 return;
8325 case AMDGPU::S_CSELECT_B32:
8326 case AMDGPU::S_CSELECT_B64:
8327 lowerSelect(Worklist, Inst, MDT);
8328 Inst.eraseFromParent();
8329 return;
8330 case AMDGPU::S_CMP_EQ_I32:
8331 case AMDGPU::S_CMP_LG_I32:
8332 case AMDGPU::S_CMP_GT_I32:
8333 case AMDGPU::S_CMP_GE_I32:
8334 case AMDGPU::S_CMP_LT_I32:
8335 case AMDGPU::S_CMP_LE_I32:
8336 case AMDGPU::S_CMP_EQ_U32:
8337 case AMDGPU::S_CMP_LG_U32:
8338 case AMDGPU::S_CMP_GT_U32:
8339 case AMDGPU::S_CMP_GE_U32:
8340 case AMDGPU::S_CMP_LT_U32:
8341 case AMDGPU::S_CMP_LE_U32:
8342 case AMDGPU::S_CMP_EQ_U64:
8343 case AMDGPU::S_CMP_LG_U64:
8344 case AMDGPU::S_CMP_LT_F32:
8345 case AMDGPU::S_CMP_EQ_F32:
8346 case AMDGPU::S_CMP_LE_F32:
8347 case AMDGPU::S_CMP_GT_F32:
8348 case AMDGPU::S_CMP_LG_F32:
8349 case AMDGPU::S_CMP_GE_F32:
8350 case AMDGPU::S_CMP_O_F32:
8351 case AMDGPU::S_CMP_U_F32:
8352 case AMDGPU::S_CMP_NGE_F32:
8353 case AMDGPU::S_CMP_NLG_F32:
8354 case AMDGPU::S_CMP_NGT_F32:
8355 case AMDGPU::S_CMP_NLE_F32:
8356 case AMDGPU::S_CMP_NEQ_F32:
8357 case AMDGPU::S_CMP_NLT_F32: {
8358 Register CondReg = MRI.createVirtualRegister(RegClass: RI.getWaveMaskRegClass());
8359 auto NewInstr =
8360 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode), DestReg: CondReg)
8361 .setMIFlags(Inst.getFlags());
8362 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src0_modifiers) >=
8363 0) {
8364 NewInstr
8365 .addImm(Val: 0) // src0_modifiers
8366 .add(MO: Inst.getOperand(i: 0)) // src0
8367 .addImm(Val: 0) // src1_modifiers
8368 .add(MO: Inst.getOperand(i: 1)) // src1
8369 .addImm(Val: 0); // clamp
8370 } else {
8371 NewInstr.add(MO: Inst.getOperand(i: 0)).add(MO: Inst.getOperand(i: 1));
8372 }
8373 legalizeOperands(MI&: *NewInstr, MDT);
8374 int SCCIdx = Inst.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
8375 const MachineOperand &SCCOp = Inst.getOperand(i: SCCIdx);
8376 addSCCDefUsersToVALUWorklist(Op: SCCOp, SCCDefInst&: Inst, Worklist, NewCond: CondReg);
8377 Inst.eraseFromParent();
8378 return;
8379 }
8380 case AMDGPU::S_CMP_LT_F16:
8381 case AMDGPU::S_CMP_EQ_F16:
8382 case AMDGPU::S_CMP_LE_F16:
8383 case AMDGPU::S_CMP_GT_F16:
8384 case AMDGPU::S_CMP_LG_F16:
8385 case AMDGPU::S_CMP_GE_F16:
8386 case AMDGPU::S_CMP_O_F16:
8387 case AMDGPU::S_CMP_U_F16:
8388 case AMDGPU::S_CMP_NGE_F16:
8389 case AMDGPU::S_CMP_NLG_F16:
8390 case AMDGPU::S_CMP_NGT_F16:
8391 case AMDGPU::S_CMP_NLE_F16:
8392 case AMDGPU::S_CMP_NEQ_F16:
8393 case AMDGPU::S_CMP_NLT_F16: {
8394 Register CondReg = MRI.createVirtualRegister(RegClass: RI.getWaveMaskRegClass());
8395 auto NewInstr =
8396 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode), DestReg: CondReg)
8397 .setMIFlags(Inst.getFlags());
8398 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::src0_modifiers)) {
8399 NewInstr
8400 .addImm(Val: 0) // src0_modifiers
8401 .add(MO: Inst.getOperand(i: 0)) // src0
8402 .addImm(Val: 0) // src1_modifiers
8403 .add(MO: Inst.getOperand(i: 1)) // src1
8404 .addImm(Val: 0); // clamp
8405 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::op_sel))
8406 NewInstr.addImm(Val: 0); // op_sel0
8407 } else {
8408 NewInstr
8409 .add(MO: Inst.getOperand(i: 0))
8410 .add(MO: Inst.getOperand(i: 1));
8411 }
8412 legalizeOperandsVALUt16(MI&: *NewInstr, MRI);
8413 legalizeOperands(MI&: *NewInstr, MDT);
8414 int SCCIdx = Inst.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
8415 const MachineOperand &SCCOp = Inst.getOperand(i: SCCIdx);
8416 addSCCDefUsersToVALUWorklist(Op: SCCOp, SCCDefInst&: Inst, Worklist, NewCond: CondReg);
8417 Inst.eraseFromParent();
8418 return;
8419 }
8420 case AMDGPU::S_CVT_HI_F32_F16: {
8421 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8422 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8423 if (ST.useRealTrue16Insts()) {
8424 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: TmpReg)
8425 .add(MO: Inst.getOperand(i: 1));
8426 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8427 .addImm(Val: 0) // src0_modifiers
8428 .addReg(RegNo: TmpReg, Flags: {}, SubReg: AMDGPU::hi16)
8429 .addImm(Val: 0) // clamp
8430 .addImm(Val: 0) // omod
8431 .addImm(Val: 0); // op_sel0
8432 } else {
8433 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
8434 .addImm(Val: 16)
8435 .add(MO: Inst.getOperand(i: 1));
8436 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8437 .addImm(Val: 0) // src0_modifiers
8438 .addReg(RegNo: TmpReg)
8439 .addImm(Val: 0) // clamp
8440 .addImm(Val: 0); // omod
8441 }
8442
8443 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8444 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8445 Inst.eraseFromParent();
8446 return;
8447 }
8448 case AMDGPU::S_MINIMUM_F32:
8449 case AMDGPU::S_MAXIMUM_F32: {
8450 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8451 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8452 .addImm(Val: 0) // src0_modifiers
8453 .add(MO: Inst.getOperand(i: 1))
8454 .addImm(Val: 0) // src1_modifiers
8455 .add(MO: Inst.getOperand(i: 2))
8456 .addImm(Val: 0) // clamp
8457 .addImm(Val: 0); // omod
8458 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8459
8460 legalizeOperands(MI&: *NewInstr, MDT);
8461 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8462 Inst.eraseFromParent();
8463 return;
8464 }
8465 case AMDGPU::S_MINIMUM_F16:
8466 case AMDGPU::S_MAXIMUM_F16: {
8467 Register NewDst = MRI.createVirtualRegister(RegClass: ST.useRealTrue16Insts()
8468 ? &AMDGPU::VGPR_16RegClass
8469 : &AMDGPU::VGPR_32RegClass);
8470 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8471 .addImm(Val: 0) // src0_modifiers
8472 .add(MO: Inst.getOperand(i: 1))
8473 .addImm(Val: 0) // src1_modifiers
8474 .add(MO: Inst.getOperand(i: 2))
8475 .addImm(Val: 0) // clamp
8476 .addImm(Val: 0) // omod
8477 .addImm(Val: 0); // opsel0
8478 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8479 legalizeOperandsVALUt16(MI&: *NewInstr, MRI);
8480 legalizeOperands(MI&: *NewInstr, MDT);
8481 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8482 Inst.eraseFromParent();
8483 return;
8484 }
8485 case AMDGPU::V_S_EXP_F16_e64:
8486 case AMDGPU::V_S_LOG_F16_e64:
8487 case AMDGPU::V_S_RCP_F16_e64:
8488 case AMDGPU::V_S_RSQ_F16_e64:
8489 case AMDGPU::V_S_SQRT_F16_e64: {
8490 Register NewDst = MRI.createVirtualRegister(RegClass: ST.useRealTrue16Insts()
8491 ? &AMDGPU::VGPR_16RegClass
8492 : &AMDGPU::VGPR_32RegClass);
8493 auto NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8494 .add(MO: Inst.getOperand(i: 1)) // src0_modifiers
8495 .add(MO: Inst.getOperand(i: 2))
8496 .add(MO: Inst.getOperand(i: 3)) // clamp
8497 .add(MO: Inst.getOperand(i: 4)) // omod
8498 .setMIFlags(Inst.getFlags());
8499 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::op_sel))
8500 NewInstr.addImm(Val: 0); // opsel0
8501 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8502 legalizeOperandsVALUt16(MI&: *NewInstr, MRI);
8503 legalizeOperands(MI&: *NewInstr, MDT);
8504 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8505 Inst.eraseFromParent();
8506 return;
8507 }
8508 }
8509
8510 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
8511 // We cannot move this instruction to the VALU, so we should try to
8512 // legalize its operands instead.
8513 legalizeOperands(MI&: Inst, MDT);
8514 return;
8515 }
8516 // Handle converting generic instructions like COPY-to-SGPR into
8517 // COPY-to-VGPR.
8518 if (NewOpcode == Opcode) {
8519 Register DstReg = Inst.getOperand(i: 0).getReg();
8520 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
8521
8522 if (Inst.isCopy() && DstReg.isPhysical() &&
8523 Inst.getOperand(i: 1).getReg().isVirtual()) {
8524 handleCopyToPhysHelper(Worklist, DstReg, Inst, MRI, WaterFalls,
8525 V2SPhyCopiesToErase);
8526 return;
8527 }
8528
8529 if (Inst.isCopy() && Inst.getOperand(i: 1).getReg().isVirtual()) {
8530 Register NewDstReg = Inst.getOperand(i: 1).getReg();
8531 const TargetRegisterClass *SrcRC = RI.getRegClassForReg(MRI, Reg: NewDstReg);
8532 if (const TargetRegisterClass *CommonRC =
8533 RI.getCommonSubClass(A: NewDstRC, B: SrcRC)) {
8534 // Instead of creating a copy where src and dst are the same register
8535 // class, we just replace all uses of dst with src. These kinds of
8536 // copies interfere with the heuristics MachineSink uses to decide
8537 // whether or not to split a critical edge. Since the pass assumes
8538 // that copies will end up as machine instructions and not be
8539 // eliminated.
8540 addUsersToMoveToVALUWorklist(Reg: DstReg, MRI, Worklist);
8541 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
8542 MRI.clearKillFlags(Reg: NewDstReg);
8543 Inst.getOperand(i: 0).setReg(DstReg);
8544
8545 if (!MRI.constrainRegClass(Reg: NewDstReg, RC: CommonRC))
8546 llvm_unreachable("failed to constrain register");
8547
8548 Inst.eraseFromParent();
8549
8550 for (MachineOperand &UseMO :
8551 make_early_inc_range(Range: MRI.use_operands(Reg: NewDstReg))) {
8552 MachineInstr &UseMI = *UseMO.getParent();
8553
8554 // Legalize t16 operands since replaceReg is called after
8555 // addUsersToVALU.
8556 legalizeOperandsVALUt16(MI&: UseMI, MRI);
8557
8558 unsigned OpIdx = UseMI.getOperandNo(I: &UseMO);
8559 if (const TargetRegisterClass *OpRC =
8560 getRegClass(MCID: UseMI.getDesc(), OpNum: OpIdx))
8561 MRI.constrainRegClass(Reg: NewDstReg, RC: OpRC);
8562 }
8563
8564 return;
8565 }
8566 }
8567
8568 // If this is a v2s copy between 16bit and 32bit reg,
8569 // replace vgpr copy to reg_sequence/extract_subreg
8570 // This can be remove after we have sgpr16 in place
8571 if (ST.useRealTrue16Insts() && Inst.isCopy() &&
8572 Inst.getOperand(i: 1).getReg().isVirtual() &&
8573 RI.isVGPR(MRI, Reg: Inst.getOperand(i: 1).getReg())) {
8574 const TargetRegisterClass *SrcRegRC = getOpRegClass(MI: Inst, OpNo: 1);
8575 if (RI.getMatchingSuperRegClass(A: NewDstRC, B: SrcRegRC, Idx: AMDGPU::lo16)) {
8576 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
8577 Register Undef = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
8578 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8579 MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
8580 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8581 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewDstReg)
8582 .addReg(RegNo: Inst.getOperand(i: 1).getReg())
8583 .addImm(Val: AMDGPU::lo16)
8584 .addReg(RegNo: Undef)
8585 .addImm(Val: AMDGPU::hi16);
8586 Inst.eraseFromParent();
8587 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
8588 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
8589 return;
8590 } else if (RI.getMatchingSuperRegClass(A: SrcRegRC, B: NewDstRC,
8591 Idx: AMDGPU::lo16)) {
8592 Inst.getOperand(i: 1).setSubReg(AMDGPU::lo16);
8593 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
8594 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
8595 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
8596 return;
8597 }
8598 }
8599
8600 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
8601 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
8602 legalizeOperands(MI&: Inst, MDT);
8603 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
8604 return;
8605 }
8606
8607 // Use the new VALU Opcode.
8608 auto NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode))
8609 .setMIFlags(Inst.getFlags());
8610 if (isVOP3(Opcode: NewOpcode) && !isVOP3(Opcode)) {
8611 // Intersperse VOP3 modifiers among the SALU operands.
8612 NewInstr->addOperand(Op: Inst.getOperand(i: 0));
8613 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
8614 Name: AMDGPU::OpName::src0_modifiers) >= 0)
8615 NewInstr.addImm(Val: 0);
8616 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::src0)) {
8617 const MachineOperand &Src = Inst.getOperand(i: 1);
8618 NewInstr->addOperand(Op: Src);
8619 }
8620
8621 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
8622 // We are converting these to a BFE, so we need to add the missing
8623 // operands for the size and offset.
8624 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
8625 NewInstr.addImm(Val: 0);
8626 NewInstr.addImm(Val: Size);
8627 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
8628 // The VALU version adds the second operand to the result, so insert an
8629 // extra 0 operand.
8630 NewInstr.addImm(Val: 0);
8631 } else if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
8632 const MachineOperand &OffsetWidthOp = Inst.getOperand(i: 2);
8633 // If we need to move this to VGPRs, we need to unpack the second
8634 // operand back into the 2 separate ones for bit offset and width.
8635 assert(OffsetWidthOp.isImm() &&
8636 "Scalar BFE is only implemented for constant width and offset");
8637 uint32_t Imm = OffsetWidthOp.getImm();
8638
8639 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
8640 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
8641 NewInstr.addImm(Val: Offset);
8642 NewInstr.addImm(Val: BitWidth);
8643 } else {
8644 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
8645 Name: AMDGPU::OpName::src1_modifiers) >= 0)
8646 NewInstr.addImm(Val: 0);
8647 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src1) >= 0)
8648 NewInstr->addOperand(Op: Inst.getOperand(i: 2));
8649 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
8650 Name: AMDGPU::OpName::src2_modifiers) >= 0)
8651 NewInstr.addImm(Val: 0);
8652 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src2) >= 0)
8653 NewInstr->addOperand(Op: Inst.getOperand(i: 3));
8654 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::clamp) >= 0)
8655 NewInstr.addImm(Val: 0);
8656 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::omod) >= 0)
8657 NewInstr.addImm(Val: 0);
8658 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::op_sel) >= 0)
8659 NewInstr.addImm(Val: 0);
8660 }
8661 } else {
8662 // Just copy the SALU operands.
8663 for (const MachineOperand &Op : Inst.explicit_operands())
8664 NewInstr->addOperand(Op);
8665 }
8666
8667 // Remove any references to SCC. Vector instructions can't read from it, and
8668 // We're just about to add the implicit use / defs of VCC, and we don't want
8669 // both.
8670 for (MachineOperand &Op : Inst.implicit_operands()) {
8671 if (Op.getReg() == AMDGPU::SCC) {
8672 // Only propagate through live-def of SCC.
8673 if (Op.isDef() && !Op.isDead())
8674 addSCCDefUsersToVALUWorklist(Op, SCCDefInst&: Inst, Worklist);
8675 if (Op.isUse())
8676 addSCCDefsToVALUWorklist(SCCUseInst: NewInstr, Worklist);
8677 }
8678 }
8679 Inst.eraseFromParent();
8680 Register NewDstReg;
8681 if (NewInstr->getOperand(i: 0).isReg() && NewInstr->getOperand(i: 0).isDef()) {
8682 Register DstReg = NewInstr->getOperand(i: 0).getReg();
8683 assert(DstReg.isVirtual());
8684 // Update the destination register class.
8685 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst: *NewInstr);
8686 assert(NewDstRC);
8687 NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
8688 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
8689 }
8690 fixImplicitOperands(MI&: *NewInstr);
8691
8692 legalizeOperandsVALUt16(MI&: *NewInstr, MRI);
8693
8694 // Legalize the operands
8695 legalizeOperands(MI&: *NewInstr, MDT);
8696 if (NewDstReg)
8697 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
8698}
8699
8700// Add/sub require special handling to deal with carry outs.
8701std::pair<bool, MachineBasicBlock *>
8702SIInstrInfo::moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
8703 MachineDominatorTree *MDT) const {
8704 if (ST.hasAddNoCarryInsts()) {
8705 // Assume there is no user of scc since we don't select this in that case.
8706 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
8707 // is used.
8708
8709 MachineBasicBlock &MBB = *Inst.getParent();
8710 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8711
8712 Register OldDstReg = Inst.getOperand(i: 0).getReg();
8713 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8714
8715 unsigned Opc = Inst.getOpcode();
8716 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
8717
8718 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
8719 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
8720
8721 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
8722 Inst.removeOperand(OpNo: 3);
8723
8724 Inst.setDesc(get(Opcode: NewOpc));
8725 Inst.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // clamp bit
8726 Inst.addImplicitDefUseOperands(MF&: *MBB.getParent());
8727 MRI.replaceRegWith(FromReg: OldDstReg, ToReg: ResultReg);
8728 MachineBasicBlock *NewBB = legalizeOperands(MI&: Inst, MDT);
8729
8730 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
8731 return std::pair(true, NewBB);
8732 }
8733
8734 return std::pair(false, nullptr);
8735}
8736
8737void SIInstrInfo::lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
8738 MachineDominatorTree *MDT) const {
8739
8740 MachineBasicBlock &MBB = *Inst.getParent();
8741 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8742 MachineBasicBlock::iterator MII = Inst;
8743 const DebugLoc &DL = Inst.getDebugLoc();
8744
8745 MachineOperand &Dest = Inst.getOperand(i: 0);
8746 MachineOperand &Src0 = Inst.getOperand(i: 1);
8747 MachineOperand &Src1 = Inst.getOperand(i: 2);
8748 MachineOperand &Cond = Inst.getOperand(i: 3);
8749
8750 Register CondReg = Cond.getReg();
8751 bool IsSCC = (CondReg == AMDGPU::SCC);
8752
8753 // Remove S_CSELECT instructions that we previously inserted to feed the SCC
8754 // condition output from S_CMP into the SGPR condition input of V_CNDMASK. If
8755 // the S_CMP has been promoted to V_CMP then we can feed its SGPR condition
8756 // output directly into the V_CNDMASK.
8757 if (!IsSCC && Src0.isImm() && (Src0.getImm() == -1) && Src1.isImm() &&
8758 (Src1.getImm() == 0)) {
8759 for (MachineOperand &UseMO :
8760 make_early_inc_range(Range: MRI.use_nodbg_operands(Reg: Dest.getReg()))) {
8761 MachineInstr &UseMI = *UseMO.getParent();
8762 switch (UseMI.getOpcode()) {
8763 case AMDGPU::V_CNDMASK_B16_fake16_e32:
8764 case AMDGPU::V_CNDMASK_B16_fake16_e64:
8765 case AMDGPU::V_CNDMASK_B16_t16_e32:
8766 case AMDGPU::V_CNDMASK_B16_t16_e64:
8767 case AMDGPU::V_CNDMASK_B32_e32:
8768 case AMDGPU::V_CNDMASK_B32_e64:
8769 case AMDGPU::V_CNDMASK_B64_PSEUDO:
8770 if (UseMO.isImplicit() ||
8771 &UseMO == getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src2))
8772 UseMO.setReg(CondReg);
8773 }
8774 }
8775 if (MRI.use_nodbg_empty(RegNo: Dest.getReg()))
8776 return;
8777 }
8778
8779 Register NewCondReg = CondReg;
8780 if (IsSCC) {
8781 const TargetRegisterClass *TC = RI.getWaveMaskRegClass();
8782 NewCondReg = MRI.createVirtualRegister(RegClass: TC);
8783
8784 // Now look for the closest SCC def if it is a copy
8785 // replacing the CondReg with the COPY source register
8786 bool CopyFound = false;
8787 for (MachineInstr &CandI :
8788 make_range(x: std::next(x: MachineBasicBlock::reverse_iterator(Inst)),
8789 y: Inst.getParent()->rend())) {
8790 if (CandI.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isDead: false, Overlap: false) !=
8791 -1) {
8792 if (CandI.isCopy() && CandI.getOperand(i: 0).getReg() == AMDGPU::SCC) {
8793 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: NewCondReg)
8794 .addReg(RegNo: CandI.getOperand(i: 1).getReg());
8795 CopyFound = true;
8796 }
8797 break;
8798 }
8799 }
8800 if (!CopyFound) {
8801 // SCC def is not a copy
8802 // Insert a trivial select instead of creating a copy, because a copy from
8803 // SCC would semantically mean just copying a single bit, but we may need
8804 // the result to be a vector condition mask that needs preserving.
8805 unsigned Opcode =
8806 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
8807 auto NewSelect =
8808 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: NewCondReg).addImm(Val: -1).addImm(Val: 0);
8809 NewSelect->getOperand(i: 3).setIsUndef(Cond.isUndef());
8810 }
8811 }
8812
8813 Register NewDestReg = MRI.createVirtualRegister(
8814 RegClass: RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest.getReg())));
8815 MachineInstr *NewInst;
8816 if (Inst.getOpcode() == AMDGPU::S_CSELECT_B32) {
8817 NewInst = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: NewDestReg)
8818 .addImm(Val: 0)
8819 .add(MO: Src1) // False
8820 .addImm(Val: 0)
8821 .add(MO: Src0) // True
8822 .addReg(RegNo: NewCondReg);
8823 } else {
8824 NewInst =
8825 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B64_PSEUDO), DestReg: NewDestReg)
8826 .add(MO: Src1) // False
8827 .add(MO: Src0) // True
8828 .addReg(RegNo: NewCondReg);
8829 }
8830 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDestReg);
8831 legalizeOperands(MI&: *NewInst, MDT);
8832 addUsersToMoveToVALUWorklist(Reg: NewDestReg, MRI, Worklist);
8833}
8834
8835void SIInstrInfo::lowerScalarAbs(SIInstrWorklist &Worklist,
8836 MachineInstr &Inst) const {
8837 MachineBasicBlock &MBB = *Inst.getParent();
8838 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8839 MachineBasicBlock::iterator MII = Inst;
8840 const DebugLoc &DL = Inst.getDebugLoc();
8841
8842 MachineOperand &Dest = Inst.getOperand(i: 0);
8843 MachineOperand &Src = Inst.getOperand(i: 1);
8844 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8845 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8846
8847 unsigned SubOp = ST.hasAddNoCarryInsts() ? AMDGPU::V_SUB_U32_e32
8848 : AMDGPU::V_SUB_CO_U32_e32;
8849
8850 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: TmpReg)
8851 .addImm(Val: 0)
8852 .addReg(RegNo: Src.getReg());
8853
8854 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MAX_I32_e64), DestReg: ResultReg)
8855 .addReg(RegNo: Src.getReg())
8856 .addReg(RegNo: TmpReg);
8857
8858 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
8859 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
8860}
8861
8862void SIInstrInfo::lowerScalarAbsDiff(SIInstrWorklist &Worklist,
8863 MachineInstr &Inst) const {
8864 MachineBasicBlock &MBB = *Inst.getParent();
8865 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8866 MachineBasicBlock::iterator MII = Inst;
8867 const DebugLoc &DL = Inst.getDebugLoc();
8868
8869 MachineOperand &Dest = Inst.getOperand(i: 0);
8870 MachineOperand &Src1 = Inst.getOperand(i: 1);
8871 MachineOperand &Src2 = Inst.getOperand(i: 2);
8872 Register SubResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8873 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8874 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8875
8876 unsigned SubOp = ST.hasAddNoCarryInsts() ? AMDGPU::V_SUB_U32_e32
8877 : AMDGPU::V_SUB_CO_U32_e32;
8878
8879 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: SubResultReg)
8880 .addReg(RegNo: Src1.getReg())
8881 .addReg(RegNo: Src2.getReg());
8882
8883 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: TmpReg).addImm(Val: 0).addReg(RegNo: SubResultReg);
8884
8885 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MAX_I32_e64), DestReg: ResultReg)
8886 .addReg(RegNo: SubResultReg)
8887 .addReg(RegNo: TmpReg);
8888
8889 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
8890 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
8891}
8892
8893void SIInstrInfo::lowerScalarXnor(SIInstrWorklist &Worklist,
8894 MachineInstr &Inst) const {
8895 MachineBasicBlock &MBB = *Inst.getParent();
8896 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8897 MachineBasicBlock::iterator MII = Inst;
8898 const DebugLoc &DL = Inst.getDebugLoc();
8899
8900 MachineOperand &Dest = Inst.getOperand(i: 0);
8901 MachineOperand &Src0 = Inst.getOperand(i: 1);
8902 MachineOperand &Src1 = Inst.getOperand(i: 2);
8903
8904 if (ST.hasDLInsts()) {
8905 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8906 legalizeGenericOperand(InsertMBB&: MBB, I: MII, DstRC: &AMDGPU::VGPR_32RegClass, Op&: Src0, MRI, DL);
8907 legalizeGenericOperand(InsertMBB&: MBB, I: MII, DstRC: &AMDGPU::VGPR_32RegClass, Op&: Src1, MRI, DL);
8908
8909 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_XNOR_B32_e64), DestReg: NewDest)
8910 .add(MO: Src0)
8911 .add(MO: Src1);
8912
8913 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
8914 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
8915 } else {
8916 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
8917 // invert either source and then perform the XOR. If either source is a
8918 // scalar register, then we can leave the inversion on the scalar unit to
8919 // achieve a better distribution of scalar and vector instructions.
8920 bool Src0IsSGPR = Src0.isReg() &&
8921 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src0.getReg()));
8922 bool Src1IsSGPR = Src1.isReg() &&
8923 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src1.getReg()));
8924 MachineInstr *Xor;
8925 Register Temp = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
8926 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
8927
8928 // Build a pair of scalar instructions and add them to the work list.
8929 // The next iteration over the work list will lower these to the vector
8930 // unit as necessary.
8931 if (Src0IsSGPR) {
8932 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Temp).add(MO: Src0);
8933 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: NewDest)
8934 .addReg(RegNo: Temp)
8935 .add(MO: Src1);
8936 } else if (Src1IsSGPR) {
8937 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Temp).add(MO: Src1);
8938 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: NewDest)
8939 .add(MO: Src0)
8940 .addReg(RegNo: Temp);
8941 } else {
8942 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: Temp)
8943 .add(MO: Src0)
8944 .add(MO: Src1);
8945 MachineInstr *Not =
8946 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: NewDest).addReg(RegNo: Temp);
8947 Worklist.insert(MI: Not);
8948 }
8949
8950 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
8951
8952 Worklist.insert(MI: Xor);
8953
8954 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
8955 }
8956}
8957
8958void SIInstrInfo::splitScalarNotBinop(SIInstrWorklist &Worklist,
8959 MachineInstr &Inst,
8960 unsigned Opcode) const {
8961 MachineBasicBlock &MBB = *Inst.getParent();
8962 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8963 MachineBasicBlock::iterator MII = Inst;
8964 const DebugLoc &DL = Inst.getDebugLoc();
8965
8966 MachineOperand &Dest = Inst.getOperand(i: 0);
8967 MachineOperand &Src0 = Inst.getOperand(i: 1);
8968 MachineOperand &Src1 = Inst.getOperand(i: 2);
8969
8970 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
8971 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
8972
8973 MachineInstr &Op = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: Interm)
8974 .add(MO: Src0)
8975 .add(MO: Src1);
8976
8977 MachineInstr &Not = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: NewDest)
8978 .addReg(RegNo: Interm);
8979
8980 Worklist.insert(MI: &Op);
8981 Worklist.insert(MI: &Not);
8982
8983 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
8984 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
8985}
8986
8987void SIInstrInfo::splitScalarBinOpN2(SIInstrWorklist &Worklist,
8988 MachineInstr &Inst,
8989 unsigned Opcode) const {
8990 MachineBasicBlock &MBB = *Inst.getParent();
8991 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8992 MachineBasicBlock::iterator MII = Inst;
8993 const DebugLoc &DL = Inst.getDebugLoc();
8994
8995 MachineOperand &Dest = Inst.getOperand(i: 0);
8996 MachineOperand &Src0 = Inst.getOperand(i: 1);
8997 MachineOperand &Src1 = Inst.getOperand(i: 2);
8998
8999 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
9000 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
9001
9002 MachineInstr &Not = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Interm)
9003 .add(MO: Src1);
9004
9005 MachineInstr &Op = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: NewDest)
9006 .add(MO: Src0)
9007 .addReg(RegNo: Interm);
9008
9009 Worklist.insert(MI: &Not);
9010 Worklist.insert(MI: &Op);
9011
9012 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9013 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9014}
9015
9016void SIInstrInfo::splitScalar64BitUnaryOp(SIInstrWorklist &Worklist,
9017 MachineInstr &Inst, unsigned Opcode,
9018 bool Swap) const {
9019 MachineBasicBlock &MBB = *Inst.getParent();
9020 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9021
9022 MachineOperand &Dest = Inst.getOperand(i: 0);
9023 MachineOperand &Src0 = Inst.getOperand(i: 1);
9024 const DebugLoc &DL = Inst.getDebugLoc();
9025
9026 MachineBasicBlock::iterator MII = Inst;
9027
9028 const MCInstrDesc &InstDesc = get(Opcode);
9029 const TargetRegisterClass *Src0RC = Src0.isReg() ?
9030 MRI.getRegClass(Reg: Src0.getReg()) :
9031 &AMDGPU::SGPR_32RegClass;
9032
9033 const TargetRegisterClass *Src0SubRC =
9034 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9035
9036 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9037 SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9038
9039 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9040 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(SRC: DestRC);
9041 const TargetRegisterClass *NewDestSubRC =
9042 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
9043
9044 Register DestSub0 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9045 MachineInstr &LoHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub0).add(MO: SrcReg0Sub0);
9046
9047 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9048 SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9049
9050 Register DestSub1 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9051 MachineInstr &HiHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub1).add(MO: SrcReg0Sub1);
9052
9053 if (Swap)
9054 std::swap(a&: DestSub0, b&: DestSub1);
9055
9056 Register FullDestReg = MRI.createVirtualRegister(RegClass: NewDestRC);
9057 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9058 .addReg(RegNo: DestSub0)
9059 .addImm(Val: AMDGPU::sub0)
9060 .addReg(RegNo: DestSub1)
9061 .addImm(Val: AMDGPU::sub1);
9062
9063 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9064
9065 Worklist.insert(MI: &LoHalf);
9066 Worklist.insert(MI: &HiHalf);
9067
9068 // We don't need to legalizeOperands here because for a single operand, src0
9069 // will support any kind of input.
9070
9071 // Move all users of this moved value.
9072 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9073}
9074
9075// There is not a vector equivalent of s_mul_u64. For this reason, we need to
9076// split the s_mul_u64 in 32-bit vector multiplications.
9077void SIInstrInfo::splitScalarSMulU64(SIInstrWorklist &Worklist,
9078 MachineInstr &Inst,
9079 MachineDominatorTree *MDT) const {
9080 MachineBasicBlock &MBB = *Inst.getParent();
9081 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9082
9083 Register FullDestReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9084 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9085 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9086
9087 MachineOperand &Dest = Inst.getOperand(i: 0);
9088 MachineOperand &Src0 = Inst.getOperand(i: 1);
9089 MachineOperand &Src1 = Inst.getOperand(i: 2);
9090 const DebugLoc &DL = Inst.getDebugLoc();
9091 MachineBasicBlock::iterator MII = Inst;
9092
9093 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0.getReg());
9094 const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: Src1.getReg());
9095 const TargetRegisterClass *Src0SubRC =
9096 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9097 if (RI.isSGPRClass(RC: Src0SubRC))
9098 Src0SubRC = RI.getEquivalentVGPRClass(SRC: Src0SubRC);
9099 const TargetRegisterClass *Src1SubRC =
9100 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9101 if (RI.isSGPRClass(RC: Src1SubRC))
9102 Src1SubRC = RI.getEquivalentVGPRClass(SRC: Src1SubRC);
9103
9104 // First, we extract the low 32-bit and high 32-bit values from each of the
9105 // operands.
9106 MachineOperand Op0L =
9107 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9108 MachineOperand Op1L =
9109 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9110 MachineOperand Op0H =
9111 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9112 MachineOperand Op1H =
9113 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
9114
9115 // The multilication is done as follows:
9116 //
9117 // Op1H Op1L
9118 // * Op0H Op0L
9119 // --------------------
9120 // Op1H*Op0L Op1L*Op0L
9121 // + Op1H*Op0H Op1L*Op0H
9122 // -----------------------------------------
9123 // (Op1H*Op0L + Op1L*Op0H + carry) Op1L*Op0L
9124 //
9125 // We drop Op1H*Op0H because the result of the multiplication is a 64-bit
9126 // value and that would overflow.
9127 // The low 32-bit value is Op1L*Op0L.
9128 // The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from Op1L*Op0L).
9129
9130 Register Op1L_Op0H_Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9131 MachineInstr *Op1L_Op0H =
9132 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: Op1L_Op0H_Reg)
9133 .add(MO: Op1L)
9134 .add(MO: Op0H);
9135
9136 Register Op1H_Op0L_Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9137 MachineInstr *Op1H_Op0L =
9138 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: Op1H_Op0L_Reg)
9139 .add(MO: Op1H)
9140 .add(MO: Op0L);
9141
9142 Register CarryReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9143 MachineInstr *Carry =
9144 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_HI_U32_e64), DestReg: CarryReg)
9145 .add(MO: Op1L)
9146 .add(MO: Op0L);
9147
9148 MachineInstr *LoHalf =
9149 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: DestSub0)
9150 .add(MO: Op1L)
9151 .add(MO: Op0L);
9152
9153 Register AddReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9154 MachineInstr *Add = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg: AddReg)
9155 .addReg(RegNo: Op1L_Op0H_Reg)
9156 .addReg(RegNo: Op1H_Op0L_Reg);
9157
9158 MachineInstr *HiHalf =
9159 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg: DestSub1)
9160 .addReg(RegNo: AddReg)
9161 .addReg(RegNo: CarryReg);
9162
9163 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9164 .addReg(RegNo: DestSub0)
9165 .addImm(Val: AMDGPU::sub0)
9166 .addReg(RegNo: DestSub1)
9167 .addImm(Val: AMDGPU::sub1);
9168
9169 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9170
9171 // Try to legalize the operands in case we need to swap the order to keep it
9172 // valid.
9173 legalizeOperands(MI&: *Op1L_Op0H, MDT);
9174 legalizeOperands(MI&: *Op1H_Op0L, MDT);
9175 legalizeOperands(MI&: *Carry, MDT);
9176 legalizeOperands(MI&: *LoHalf, MDT);
9177 legalizeOperands(MI&: *Add, MDT);
9178 legalizeOperands(MI&: *HiHalf, MDT);
9179
9180 // Move all users of this moved value.
9181 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9182}
9183
9184// Lower S_MUL_U64_U32_PSEUDO/S_MUL_I64_I32_PSEUDO in two 32-bit vector
9185// multiplications.
9186void SIInstrInfo::splitScalarSMulPseudo(SIInstrWorklist &Worklist,
9187 MachineInstr &Inst,
9188 MachineDominatorTree *MDT) const {
9189 MachineBasicBlock &MBB = *Inst.getParent();
9190 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9191
9192 Register FullDestReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9193 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9194 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9195
9196 MachineOperand &Dest = Inst.getOperand(i: 0);
9197 MachineOperand &Src0 = Inst.getOperand(i: 1);
9198 MachineOperand &Src1 = Inst.getOperand(i: 2);
9199 const DebugLoc &DL = Inst.getDebugLoc();
9200 MachineBasicBlock::iterator MII = Inst;
9201
9202 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0.getReg());
9203 const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: Src1.getReg());
9204 const TargetRegisterClass *Src0SubRC =
9205 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9206 if (RI.isSGPRClass(RC: Src0SubRC))
9207 Src0SubRC = RI.getEquivalentVGPRClass(SRC: Src0SubRC);
9208 const TargetRegisterClass *Src1SubRC =
9209 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9210 if (RI.isSGPRClass(RC: Src1SubRC))
9211 Src1SubRC = RI.getEquivalentVGPRClass(SRC: Src1SubRC);
9212
9213 // First, we extract the low 32-bit and high 32-bit values from each of the
9214 // operands.
9215 MachineOperand Op0L =
9216 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9217 MachineOperand Op1L =
9218 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9219
9220 unsigned Opc = Inst.getOpcode();
9221 unsigned NewOpc = Opc == AMDGPU::S_MUL_U64_U32_PSEUDO
9222 ? AMDGPU::V_MUL_HI_U32_e64
9223 : AMDGPU::V_MUL_HI_I32_e64;
9224 MachineInstr *HiHalf =
9225 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: NewOpc), DestReg: DestSub1).add(MO: Op1L).add(MO: Op0L);
9226
9227 MachineInstr *LoHalf =
9228 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: DestSub0)
9229 .add(MO: Op1L)
9230 .add(MO: Op0L);
9231
9232 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9233 .addReg(RegNo: DestSub0)
9234 .addImm(Val: AMDGPU::sub0)
9235 .addReg(RegNo: DestSub1)
9236 .addImm(Val: AMDGPU::sub1);
9237
9238 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9239
9240 // Try to legalize the operands in case we need to swap the order to keep it
9241 // valid.
9242 legalizeOperands(MI&: *HiHalf, MDT);
9243 legalizeOperands(MI&: *LoHalf, MDT);
9244
9245 // Move all users of this moved value.
9246 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9247}
9248
9249void SIInstrInfo::splitScalar64BitBinaryOp(SIInstrWorklist &Worklist,
9250 MachineInstr &Inst, unsigned Opcode,
9251 MachineDominatorTree *MDT) const {
9252 MachineBasicBlock &MBB = *Inst.getParent();
9253 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9254
9255 MachineOperand &Dest = Inst.getOperand(i: 0);
9256 MachineOperand &Src0 = Inst.getOperand(i: 1);
9257 MachineOperand &Src1 = Inst.getOperand(i: 2);
9258 const DebugLoc &DL = Inst.getDebugLoc();
9259
9260 MachineBasicBlock::iterator MII = Inst;
9261
9262 const MCInstrDesc &InstDesc = get(Opcode);
9263 const TargetRegisterClass *Src0RC = Src0.isReg() ?
9264 MRI.getRegClass(Reg: Src0.getReg()) :
9265 &AMDGPU::SGPR_32RegClass;
9266
9267 const TargetRegisterClass *Src0SubRC =
9268 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9269 const TargetRegisterClass *Src1RC = Src1.isReg() ?
9270 MRI.getRegClass(Reg: Src1.getReg()) :
9271 &AMDGPU::SGPR_32RegClass;
9272
9273 const TargetRegisterClass *Src1SubRC =
9274 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9275
9276 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9277 SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9278 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC,
9279 SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9280 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9281 SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9282 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC,
9283 SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
9284
9285 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9286 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(SRC: DestRC);
9287 const TargetRegisterClass *NewDestSubRC =
9288 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
9289
9290 Register DestSub0 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9291 MachineInstr &LoHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub0)
9292 .add(MO: SrcReg0Sub0)
9293 .add(MO: SrcReg1Sub0);
9294
9295 Register DestSub1 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9296 MachineInstr &HiHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub1)
9297 .add(MO: SrcReg0Sub1)
9298 .add(MO: SrcReg1Sub1);
9299
9300 Register FullDestReg = MRI.createVirtualRegister(RegClass: NewDestRC);
9301 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9302 .addReg(RegNo: DestSub0)
9303 .addImm(Val: AMDGPU::sub0)
9304 .addReg(RegNo: DestSub1)
9305 .addImm(Val: AMDGPU::sub1);
9306
9307 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9308
9309 Worklist.insert(MI: &LoHalf);
9310 Worklist.insert(MI: &HiHalf);
9311
9312 // Move all users of this moved value.
9313 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9314}
9315
9316void SIInstrInfo::splitScalar64BitXnor(SIInstrWorklist &Worklist,
9317 MachineInstr &Inst,
9318 MachineDominatorTree *MDT) const {
9319 MachineBasicBlock &MBB = *Inst.getParent();
9320 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9321
9322 MachineOperand &Dest = Inst.getOperand(i: 0);
9323 MachineOperand &Src0 = Inst.getOperand(i: 1);
9324 MachineOperand &Src1 = Inst.getOperand(i: 2);
9325 const DebugLoc &DL = Inst.getDebugLoc();
9326
9327 MachineBasicBlock::iterator MII = Inst;
9328
9329 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9330
9331 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
9332
9333 MachineOperand* Op0;
9334 MachineOperand* Op1;
9335
9336 if (Src0.isReg() && RI.isSGPRReg(MRI, Reg: Src0.getReg())) {
9337 Op0 = &Src0;
9338 Op1 = &Src1;
9339 } else {
9340 Op0 = &Src1;
9341 Op1 = &Src0;
9342 }
9343
9344 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B64), DestReg: Interm)
9345 .add(MO: *Op0);
9346
9347 Register NewDest = MRI.createVirtualRegister(RegClass: DestRC);
9348
9349 MachineInstr &Xor = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B64), DestReg: NewDest)
9350 .addReg(RegNo: Interm)
9351 .add(MO: *Op1);
9352
9353 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9354
9355 Worklist.insert(MI: &Xor);
9356}
9357
9358void SIInstrInfo::splitScalar64BitBCNT(SIInstrWorklist &Worklist,
9359 MachineInstr &Inst) const {
9360 MachineBasicBlock &MBB = *Inst.getParent();
9361 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9362
9363 MachineBasicBlock::iterator MII = Inst;
9364 const DebugLoc &DL = Inst.getDebugLoc();
9365
9366 MachineOperand &Dest = Inst.getOperand(i: 0);
9367 MachineOperand &Src = Inst.getOperand(i: 1);
9368
9369 const MCInstrDesc &InstDesc = get(Opcode: AMDGPU::V_BCNT_U32_B32_e64);
9370 const TargetRegisterClass *SrcRC = Src.isReg() ?
9371 MRI.getRegClass(Reg: Src.getReg()) :
9372 &AMDGPU::SGPR_32RegClass;
9373
9374 Register MidReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9375 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9376
9377 const TargetRegisterClass *SrcSubRC =
9378 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
9379
9380 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC,
9381 SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
9382 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC,
9383 SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
9384
9385 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg).add(MO: SrcRegSub0).addImm(Val: 0);
9386
9387 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: ResultReg).add(MO: SrcRegSub1).addReg(RegNo: MidReg);
9388
9389 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9390
9391 // We don't need to legalize operands here. src0 for either instruction can be
9392 // an SGPR, and the second input is unused or determined here.
9393 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9394}
9395
9396void SIInstrInfo::splitScalar64BitBFE(SIInstrWorklist &Worklist,
9397 MachineInstr &Inst) const {
9398 MachineBasicBlock &MBB = *Inst.getParent();
9399 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9400 MachineBasicBlock::iterator MII = Inst;
9401 const DebugLoc &DL = Inst.getDebugLoc();
9402
9403 MachineOperand &Dest = Inst.getOperand(i: 0);
9404 uint32_t Imm = Inst.getOperand(i: 2).getImm();
9405 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
9406 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
9407
9408 (void) Offset;
9409
9410 // Only sext_inreg cases handled.
9411 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
9412 Offset == 0 && "Not implemented");
9413
9414 if (BitWidth < 32) {
9415 Register MidRegLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9416 Register MidRegHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9417 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9418
9419 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_BFE_I32_e64), DestReg: MidRegLo)
9420 .addReg(RegNo: Inst.getOperand(i: 1).getReg(), Flags: {}, SubReg: AMDGPU::sub0)
9421 .addImm(Val: 0)
9422 .addImm(Val: BitWidth);
9423
9424 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ASHRREV_I32_e32), DestReg: MidRegHi)
9425 .addImm(Val: 31)
9426 .addReg(RegNo: MidRegLo);
9427
9428 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: ResultReg)
9429 .addReg(RegNo: MidRegLo)
9430 .addImm(Val: AMDGPU::sub0)
9431 .addReg(RegNo: MidRegHi)
9432 .addImm(Val: AMDGPU::sub1);
9433
9434 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9435 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9436 return;
9437 }
9438
9439 MachineOperand &Src = Inst.getOperand(i: 1);
9440 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9441 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9442
9443 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ASHRREV_I32_e64), DestReg: TmpReg)
9444 .addImm(Val: 31)
9445 .addReg(RegNo: Src.getReg(), Flags: {}, SubReg: AMDGPU::sub0);
9446
9447 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: ResultReg)
9448 .addReg(RegNo: Src.getReg(), Flags: {}, SubReg: AMDGPU::sub0)
9449 .addImm(Val: AMDGPU::sub0)
9450 .addReg(RegNo: TmpReg)
9451 .addImm(Val: AMDGPU::sub1);
9452
9453 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9454 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9455}
9456
9457void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist,
9458 MachineInstr &Inst, unsigned Opcode,
9459 MachineDominatorTree *MDT) const {
9460 // (S_FLBIT_I32_B64 hi:lo) ->
9461 // -> (umin (V_FFBH_U32_e32 hi), (uaddsat (V_FFBH_U32_e32 lo), 32))
9462 // (S_FF1_I32_B64 hi:lo) ->
9463 // ->(umin (uaddsat (V_FFBL_B32_e32 hi), 32) (V_FFBL_B32_e32 lo))
9464
9465 MachineBasicBlock &MBB = *Inst.getParent();
9466 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9467 MachineBasicBlock::iterator MII = Inst;
9468 const DebugLoc &DL = Inst.getDebugLoc();
9469
9470 MachineOperand &Dest = Inst.getOperand(i: 0);
9471 MachineOperand &Src = Inst.getOperand(i: 1);
9472
9473 const MCInstrDesc &InstDesc = get(Opcode);
9474
9475 bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32;
9476 unsigned OpcodeAdd = ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_e64
9477 : AMDGPU::V_ADD_CO_U32_e32;
9478
9479 const TargetRegisterClass *SrcRC =
9480 Src.isReg() ? MRI.getRegClass(Reg: Src.getReg()) : &AMDGPU::SGPR_32RegClass;
9481 const TargetRegisterClass *SrcSubRC =
9482 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
9483
9484 MachineOperand SrcRegSub0 =
9485 buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC, SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
9486 MachineOperand SrcRegSub1 =
9487 buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC, SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
9488
9489 Register MidReg1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9490 Register MidReg2 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9491 Register MidReg3 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9492 Register MidReg4 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9493
9494 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg1).add(MO: SrcRegSub0);
9495
9496 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg2).add(MO: SrcRegSub1);
9497
9498 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: OpcodeAdd), DestReg: MidReg3)
9499 .addReg(RegNo: IsCtlz ? MidReg1 : MidReg2)
9500 .addImm(Val: 32)
9501 .addImm(Val: 1); // enable clamp
9502
9503 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MIN_U32_e64), DestReg: MidReg4)
9504 .addReg(RegNo: MidReg3)
9505 .addReg(RegNo: IsCtlz ? MidReg2 : MidReg1);
9506
9507 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: MidReg4);
9508
9509 addUsersToMoveToVALUWorklist(Reg: MidReg4, MRI, Worklist);
9510}
9511
9512void SIInstrInfo::addUsersToMoveToVALUWorklist(
9513 Register DstReg, MachineRegisterInfo &MRI,
9514 SIInstrWorklist &Worklist) const {
9515 for (MachineOperand &MO : make_early_inc_range(Range: MRI.use_operands(Reg: DstReg))) {
9516 MachineInstr &UseMI = *MO.getParent();
9517
9518 unsigned OpNo = 0;
9519
9520 switch (UseMI.getOpcode()) {
9521 case AMDGPU::COPY:
9522 case AMDGPU::WQM:
9523 case AMDGPU::SOFT_WQM:
9524 case AMDGPU::STRICT_WWM:
9525 case AMDGPU::STRICT_WQM:
9526 case AMDGPU::REG_SEQUENCE:
9527 case AMDGPU::PHI:
9528 case AMDGPU::INSERT_SUBREG:
9529 break;
9530 default:
9531 OpNo = MO.getOperandNo();
9532 break;
9533 }
9534
9535 const TargetRegisterClass *OpRC = getOpRegClass(MI: UseMI, OpNo);
9536 MRI.constrainRegClass(Reg: DstReg, RC: OpRC);
9537
9538 if (!RI.hasVectorRegisters(RC: OpRC))
9539 Worklist.insert(MI: &UseMI);
9540 else
9541 // Legalization could change user list.
9542 legalizeOperandsVALUt16(MI&: UseMI, OpIdx: OpNo, MRI);
9543 }
9544}
9545
9546void SIInstrInfo::movePackToVALU(SIInstrWorklist &Worklist,
9547 MachineRegisterInfo &MRI,
9548 MachineInstr &Inst) const {
9549 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9550 MachineBasicBlock *MBB = Inst.getParent();
9551 MachineOperand &Src0 = Inst.getOperand(i: 1);
9552 MachineOperand &Src1 = Inst.getOperand(i: 2);
9553 const DebugLoc &DL = Inst.getDebugLoc();
9554
9555 if (ST.useRealTrue16Insts()) {
9556 Register SrcReg0, SrcReg1;
9557 if (!Src0.isReg() || !RI.isVGPR(MRI, Reg: Src0.getReg())) {
9558 SrcReg0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9559 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL,
9560 MCID: get(Opcode: Src0.isImm() ? AMDGPU::V_MOV_B32_e32 : AMDGPU::COPY), DestReg: SrcReg0)
9561 .add(MO: Src0);
9562 } else {
9563 SrcReg0 = Src0.getReg();
9564 }
9565
9566 if (!Src1.isReg() || !RI.isVGPR(MRI, Reg: Src1.getReg())) {
9567 SrcReg1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9568 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL,
9569 MCID: get(Opcode: Src1.isImm() ? AMDGPU::V_MOV_B32_e32 : AMDGPU::COPY), DestReg: SrcReg1)
9570 .add(MO: Src1);
9571 } else {
9572 SrcReg1 = Src1.getReg();
9573 }
9574
9575 bool isSrc0Reg16 = MRI.constrainRegClass(Reg: SrcReg0, RC: &AMDGPU::VGPR_16RegClass);
9576 bool isSrc1Reg16 = MRI.constrainRegClass(Reg: SrcReg1, RC: &AMDGPU::VGPR_16RegClass);
9577
9578 auto NewMI = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: ResultReg);
9579 switch (Inst.getOpcode()) {
9580 case AMDGPU::S_PACK_LL_B32_B16:
9581 NewMI
9582 .addReg(RegNo: SrcReg0, Flags: {},
9583 SubReg: isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
9584 .addImm(Val: AMDGPU::lo16)
9585 .addReg(RegNo: SrcReg1, Flags: {},
9586 SubReg: isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
9587 .addImm(Val: AMDGPU::hi16);
9588 break;
9589 case AMDGPU::S_PACK_LH_B32_B16:
9590 NewMI
9591 .addReg(RegNo: SrcReg0, Flags: {},
9592 SubReg: isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
9593 .addImm(Val: AMDGPU::lo16)
9594 .addReg(RegNo: SrcReg1, Flags: {}, SubReg: AMDGPU::hi16)
9595 .addImm(Val: AMDGPU::hi16);
9596 break;
9597 case AMDGPU::S_PACK_HL_B32_B16:
9598 NewMI.addReg(RegNo: SrcReg0, Flags: {}, SubReg: AMDGPU::hi16)
9599 .addImm(Val: AMDGPU::lo16)
9600 .addReg(RegNo: SrcReg1, Flags: {},
9601 SubReg: isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
9602 .addImm(Val: AMDGPU::hi16);
9603 break;
9604 case AMDGPU::S_PACK_HH_B32_B16:
9605 NewMI.addReg(RegNo: SrcReg0, Flags: {}, SubReg: AMDGPU::hi16)
9606 .addImm(Val: AMDGPU::lo16)
9607 .addReg(RegNo: SrcReg1, Flags: {}, SubReg: AMDGPU::hi16)
9608 .addImm(Val: AMDGPU::hi16);
9609 break;
9610 default:
9611 llvm_unreachable("unhandled s_pack_* instruction");
9612 }
9613
9614 MachineOperand &Dest = Inst.getOperand(i: 0);
9615 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9616 addUsersToMoveToVALUWorklist(DstReg: ResultReg, MRI, Worklist);
9617 return;
9618 }
9619
9620 switch (Inst.getOpcode()) {
9621 case AMDGPU::S_PACK_LL_B32_B16: {
9622 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9623 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9624
9625 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are
9626 // 0.
9627 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
9628 .addImm(Val: 0xffff);
9629
9630 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_AND_B32_e64), DestReg: TmpReg)
9631 .addReg(RegNo: ImmReg, Flags: RegState::Kill)
9632 .add(MO: Src0);
9633
9634 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: ResultReg)
9635 .add(MO: Src1)
9636 .addImm(Val: 16)
9637 .addReg(RegNo: TmpReg, Flags: RegState::Kill);
9638 break;
9639 }
9640 case AMDGPU::S_PACK_LH_B32_B16: {
9641 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9642 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
9643 .addImm(Val: 0xffff);
9644 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_BFI_B32_e64), DestReg: ResultReg)
9645 .addReg(RegNo: ImmReg, Flags: RegState::Kill)
9646 .add(MO: Src0)
9647 .add(MO: Src1);
9648 break;
9649 }
9650 case AMDGPU::S_PACK_HL_B32_B16: {
9651 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9652 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
9653 .addImm(Val: 16)
9654 .add(MO: Src0);
9655 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: ResultReg)
9656 .add(MO: Src1)
9657 .addImm(Val: 16)
9658 .addReg(RegNo: TmpReg, Flags: RegState::Kill);
9659 break;
9660 }
9661 case AMDGPU::S_PACK_HH_B32_B16: {
9662 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9663 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9664 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
9665 .addImm(Val: 16)
9666 .add(MO: Src0);
9667 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
9668 .addImm(Val: 0xffff0000);
9669 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_AND_OR_B32_e64), DestReg: ResultReg)
9670 .add(MO: Src1)
9671 .addReg(RegNo: ImmReg, Flags: RegState::Kill)
9672 .addReg(RegNo: TmpReg, Flags: RegState::Kill);
9673 break;
9674 }
9675 default:
9676 llvm_unreachable("unhandled s_pack_* instruction");
9677 }
9678
9679 MachineOperand &Dest = Inst.getOperand(i: 0);
9680 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9681 addUsersToMoveToVALUWorklist(DstReg: ResultReg, MRI, Worklist);
9682}
9683
9684void SIInstrInfo::addSCCDefUsersToVALUWorklist(const MachineOperand &Op,
9685 MachineInstr &SCCDefInst,
9686 SIInstrWorklist &Worklist,
9687 Register NewCond) const {
9688
9689 // Ensure that def inst defines SCC, which is still live.
9690 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() &&
9691 !Op.isDead() && Op.getParent() == &SCCDefInst);
9692 SmallVector<MachineInstr *, 4> CopyToDelete;
9693 // This assumes that all the users of SCC are in the same block
9694 // as the SCC def.
9695 for (MachineInstr &MI : // Skip the def inst itself.
9696 make_range(x: std::next(x: MachineBasicBlock::iterator(SCCDefInst)),
9697 y: SCCDefInst.getParent()->end())) {
9698 // Check if SCC is used first.
9699 int SCCIdx = MI.findRegisterUseOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isKill: false);
9700 if (SCCIdx != -1) {
9701 if (MI.isCopy()) {
9702 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
9703 Register DestReg = MI.getOperand(i: 0).getReg();
9704
9705 MRI.replaceRegWith(FromReg: DestReg, ToReg: NewCond);
9706 CopyToDelete.push_back(Elt: &MI);
9707 } else {
9708
9709 if (NewCond.isValid())
9710 MI.getOperand(i: SCCIdx).setReg(NewCond);
9711
9712 Worklist.insert(MI: &MI);
9713 }
9714 }
9715 // Exit if we find another SCC def.
9716 if (MI.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isDead: false, Overlap: false) != -1)
9717 break;
9718 }
9719 for (auto &Copy : CopyToDelete)
9720 Copy->eraseFromParent();
9721}
9722
9723// Instructions that use SCC may be converted to VALU instructions. When that
9724// happens, the SCC register is changed to VCC_LO. The instruction that defines
9725// SCC must be changed to an instruction that defines VCC. This function makes
9726// sure that the instruction that defines SCC is added to the moveToVALU
9727// worklist.
9728void SIInstrInfo::addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
9729 SIInstrWorklist &Worklist) const {
9730 // Look for a preceding instruction that either defines VCC or SCC. If VCC
9731 // then there is nothing to do because the defining instruction has been
9732 // converted to a VALU already. If SCC then that instruction needs to be
9733 // converted to a VALU.
9734 for (MachineInstr &MI :
9735 make_range(x: std::next(x: MachineBasicBlock::reverse_iterator(SCCUseInst)),
9736 y: SCCUseInst->getParent()->rend())) {
9737 if (MI.modifiesRegister(Reg: AMDGPU::VCC, TRI: &RI))
9738 break;
9739 if (MI.definesRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
9740 Worklist.insert(MI: &MI);
9741 break;
9742 }
9743 }
9744}
9745
9746const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
9747 const MachineInstr &Inst) const {
9748 const TargetRegisterClass *NewDstRC = getOpRegClass(MI: Inst, OpNo: 0);
9749
9750 switch (Inst.getOpcode()) {
9751 // For target instructions, getOpRegClass just returns the virtual register
9752 // class associated with the operand, so we need to find an equivalent VGPR
9753 // register class in order to move the instruction to the VALU.
9754 case AMDGPU::COPY:
9755 case AMDGPU::PHI:
9756 case AMDGPU::REG_SEQUENCE:
9757 case AMDGPU::INSERT_SUBREG:
9758 case AMDGPU::WQM:
9759 case AMDGPU::SOFT_WQM:
9760 case AMDGPU::STRICT_WWM:
9761 case AMDGPU::STRICT_WQM: {
9762 const TargetRegisterClass *SrcRC = getOpRegClass(MI: Inst, OpNo: 1);
9763 if (RI.isAGPRClass(RC: SrcRC)) {
9764 if (RI.isAGPRClass(RC: NewDstRC))
9765 return nullptr;
9766
9767 switch (Inst.getOpcode()) {
9768 case AMDGPU::PHI:
9769 case AMDGPU::REG_SEQUENCE:
9770 case AMDGPU::INSERT_SUBREG:
9771 NewDstRC = RI.getEquivalentAGPRClass(SRC: NewDstRC);
9772 break;
9773 default:
9774 NewDstRC = RI.getEquivalentVGPRClass(SRC: NewDstRC);
9775 }
9776
9777 if (!NewDstRC)
9778 return nullptr;
9779 } else {
9780 if (RI.isVGPRClass(RC: NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass)
9781 return nullptr;
9782
9783 NewDstRC = RI.getEquivalentVGPRClass(SRC: NewDstRC);
9784 if (!NewDstRC)
9785 return nullptr;
9786 }
9787
9788 return NewDstRC;
9789 }
9790 default:
9791 return NewDstRC;
9792 }
9793}
9794
9795// Find the one SGPR operand we are allowed to use.
9796Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
9797 int OpIndices[3]) const {
9798 const MCInstrDesc &Desc = MI.getDesc();
9799
9800 // Find the one SGPR operand we are allowed to use.
9801 //
9802 // First we need to consider the instruction's operand requirements before
9803 // legalizing. Some operands are required to be SGPRs, such as implicit uses
9804 // of VCC, but we are still bound by the constant bus requirement to only use
9805 // one.
9806 //
9807 // If the operand's class is an SGPR, we can never move it.
9808
9809 Register SGPRReg = findImplicitSGPRRead(MI);
9810 if (SGPRReg)
9811 return SGPRReg;
9812
9813 Register UsedSGPRs[3] = {Register()};
9814 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
9815
9816 for (unsigned i = 0; i < 3; ++i) {
9817 int Idx = OpIndices[i];
9818 if (Idx == -1)
9819 break;
9820
9821 const MachineOperand &MO = MI.getOperand(i: Idx);
9822 if (!MO.isReg())
9823 continue;
9824
9825 // Is this operand statically required to be an SGPR based on the operand
9826 // constraints?
9827 const TargetRegisterClass *OpRC =
9828 RI.getRegClass(i: getOpRegClassID(OpInfo: Desc.operands()[Idx]));
9829 bool IsRequiredSGPR = RI.isSGPRClass(RC: OpRC);
9830 if (IsRequiredSGPR)
9831 return MO.getReg();
9832
9833 // If this could be a VGPR or an SGPR, Check the dynamic register class.
9834 Register Reg = MO.getReg();
9835 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
9836 if (RI.isSGPRClass(RC: RegRC))
9837 UsedSGPRs[i] = Reg;
9838 }
9839
9840 // We don't have a required SGPR operand, so we have a bit more freedom in
9841 // selecting operands to move.
9842
9843 // Try to select the most used SGPR. If an SGPR is equal to one of the
9844 // others, we choose that.
9845 //
9846 // e.g.
9847 // V_FMA_F32 v0, s0, s0, s0 -> No moves
9848 // V_FMA_F32 v0, s0, s1, s0 -> Move s1
9849
9850 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
9851 // prefer those.
9852
9853 if (UsedSGPRs[0]) {
9854 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
9855 SGPRReg = UsedSGPRs[0];
9856 }
9857
9858 if (!SGPRReg && UsedSGPRs[1]) {
9859 if (UsedSGPRs[1] == UsedSGPRs[2])
9860 SGPRReg = UsedSGPRs[1];
9861 }
9862
9863 return SGPRReg;
9864}
9865
9866MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI,
9867 AMDGPU::OpName OperandName) const {
9868 if (OperandName == AMDGPU::OpName::NUM_OPERAND_NAMES)
9869 return nullptr;
9870
9871 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OperandName);
9872 if (Idx == -1)
9873 return nullptr;
9874
9875 return &MI.getOperand(i: Idx);
9876}
9877
9878uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {
9879 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
9880 int64_t Format = ST.getGeneration() >= AMDGPUSubtarget::GFX11
9881 ? (int64_t)AMDGPU::UfmtGFX11::UFMT_32_FLOAT
9882 : (int64_t)AMDGPU::UfmtGFX10::UFMT_32_FLOAT;
9883 return (Format << 44) |
9884 (1ULL << 56) | // RESOURCE_LEVEL = 1
9885 (3ULL << 60); // OOB_SELECT = 3
9886 }
9887
9888 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
9889 if (ST.isAmdHsaOS()) {
9890 // Set ATC = 1. GFX9 doesn't have this bit.
9891 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)
9892 RsrcDataFormat |= (1ULL << 56);
9893
9894 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
9895 // BTW, it disables TC L2 and therefore decreases performance.
9896 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)
9897 RsrcDataFormat |= (2ULL << 59);
9898 }
9899
9900 return RsrcDataFormat;
9901}
9902
9903uint64_t SIInstrInfo::getScratchRsrcWords23() const {
9904 uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
9905 AMDGPU::RSRC_TID_ENABLE |
9906 0xffffffff; // Size;
9907
9908 // GFX9 doesn't have ELEMENT_SIZE.
9909 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
9910 uint64_t EltSizeValue = Log2_32(Value: ST.getMaxPrivateElementSize(ForBufferRSrc: true)) - 1;
9911 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
9912 }
9913
9914 // IndexStride = 64 / 32.
9915 uint64_t IndexStride = ST.isWave64() ? 3 : 2;
9916 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
9917
9918 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
9919 // Clear them unless we want a huge stride.
9920 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
9921 ST.getGeneration() <= AMDGPUSubtarget::GFX9)
9922 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
9923
9924 return Rsrc23;
9925}
9926
9927bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const {
9928 unsigned Opc = MI.getOpcode();
9929
9930 return isSMRD(Opcode: Opc);
9931}
9932
9933bool SIInstrInfo::isHighLatencyDef(int Opc) const {
9934 return get(Opcode: Opc).mayLoad() &&
9935 (isMUBUF(Opcode: Opc) || isMTBUF(Opcode: Opc) || isMIMG(Opcode: Opc) || isFLAT(Opcode: Opc));
9936}
9937
9938Register SIInstrInfo::isStackAccess(const MachineInstr &MI, int &FrameIndex,
9939 TypeSize &MemBytes) const {
9940 const MachineOperand *Addr = getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
9941 if (!Addr || !Addr->isFI())
9942 return Register();
9943
9944 assert(!MI.memoperands_empty() &&
9945 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS);
9946
9947 FrameIndex = Addr->getIndex();
9948
9949 int VDataIdx =
9950 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdata);
9951 MemBytes = TypeSize::getFixed(ExactSize: getOpSize(Opcode: MI.getOpcode(), OpNo: VDataIdx));
9952 return MI.getOperand(i: VDataIdx).getReg();
9953}
9954
9955Register SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex,
9956 TypeSize &MemBytes) const {
9957 const MachineOperand *Addr = getNamedOperand(MI, OperandName: AMDGPU::OpName::addr);
9958 assert(Addr && Addr->isFI());
9959 FrameIndex = Addr->getIndex();
9960
9961 int DataIdx =
9962 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::data);
9963 MemBytes = TypeSize::getFixed(ExactSize: getOpSize(Opcode: MI.getOpcode(), OpNo: DataIdx));
9964 return MI.getOperand(i: DataIdx).getReg();
9965}
9966
9967Register SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
9968 int &FrameIndex,
9969 TypeSize &MemBytes) const {
9970 if (!MI.mayLoad())
9971 return Register();
9972
9973 if (isMUBUF(MI) || isVGPRSpill(MI))
9974 return isStackAccess(MI, FrameIndex, MemBytes);
9975
9976 if (isSGPRSpill(MI))
9977 return isSGPRStackAccess(MI, FrameIndex, MemBytes);
9978
9979 return Register();
9980}
9981
9982Register SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
9983 int &FrameIndex,
9984 TypeSize &MemBytes) const {
9985 if (!MI.mayStore())
9986 return Register();
9987
9988 if (isMUBUF(MI) || isVGPRSpill(MI))
9989 return isStackAccess(MI, FrameIndex, MemBytes);
9990
9991 if (isSGPRSpill(MI))
9992 return isSGPRStackAccess(MI, FrameIndex, MemBytes);
9993
9994 return Register();
9995}
9996
9997unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
9998 unsigned Opc = MI.getOpcode();
9999 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: Opc);
10000 unsigned DescSize = Desc.getSize();
10001
10002 // If we have a definitive size, we can use it. Otherwise we need to inspect
10003 // the operands to know the size.
10004 if (isFixedSize(MI)) {
10005 unsigned Size = DescSize;
10006
10007 // If we hit the buggy offset, an extra nop will be inserted in MC so
10008 // estimate the worst case.
10009 if (MI.isBranch() && ST.hasOffset3fBug())
10010 Size += 4;
10011
10012 return Size;
10013 }
10014
10015 // Instructions may have a 32-bit literal encoded after them. Check
10016 // operands that could ever be literals.
10017 if (isVALU(MI, /*AllowLDSDMA=*/true) || isSALU(MI)) {
10018 if (isDPP(MI))
10019 return DescSize;
10020 bool HasLiteral = false;
10021 unsigned LiteralSize = 4;
10022 for (int I = 0, E = MI.getNumExplicitOperands(); I != E; ++I) {
10023 const MachineOperand &Op = MI.getOperand(i: I);
10024 const MCOperandInfo &OpInfo = Desc.operands()[I];
10025 if (!Op.isReg() && !isInlineConstant(MO: Op, OpInfo)) {
10026 HasLiteral = true;
10027 if (ST.has64BitLiterals()) {
10028 switch (OpInfo.OperandType) {
10029 default:
10030 break;
10031 case AMDGPU::OPERAND_REG_IMM_FP64:
10032 case AMDGPU::OPERAND_REG_IMM_V2FP64:
10033 if (!AMDGPU::isValid32BitLiteral(Val: Op.getImm(), IsFP64: true))
10034 LiteralSize = 8;
10035 break;
10036 case AMDGPU::OPERAND_REG_IMM_INT64:
10037 case AMDGPU::OPERAND_REG_IMM_V2INT64:
10038 // A 32-bit literal is only valid when the value fits in BOTH signed
10039 // and unsigned 32-bit ranges [0, 2^31-1], matching the MC code
10040 // emitter's getLit64Encoding logic. This is because of the lack of
10041 // abilility to tell signedness of the literal, therefore we need to
10042 // be conservative and assume values outside this range require a
10043 // 64-bit literal encoding (8 bytes).
10044 if (!Op.isImm() || !isInt<32>(x: Op.getImm()) ||
10045 !isUInt<32>(x: Op.getImm()))
10046 LiteralSize = 8;
10047 break;
10048 }
10049 }
10050 break;
10051 }
10052 }
10053 return HasLiteral ? DescSize + LiteralSize : DescSize;
10054 }
10055
10056 // Check whether we have extra NSA words.
10057 if (isMIMG(MI)) {
10058 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0);
10059 if (VAddr0Idx < 0)
10060 return 8;
10061
10062 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::srsrc);
10063 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4);
10064 }
10065
10066 switch (Opc) {
10067 case TargetOpcode::BUNDLE:
10068 return getInstBundleSize(MI);
10069 case TargetOpcode::INLINEASM:
10070 case TargetOpcode::INLINEASM_BR: {
10071 const MachineFunction *MF = MI.getMF();
10072 const char *AsmStr = MI.getOperand(i: 0).getSymbolName();
10073 return getInlineAsmLength(Str: AsmStr, MAI: MF->getTarget().getMCAsmInfo(), STI: &ST);
10074 }
10075 default:
10076 if (MI.isMetaInstruction())
10077 return 0;
10078
10079 // If D16 Pseudo inst, get correct MC code size
10080 const auto *D16Info = AMDGPU::getT16D16Helper(T16Op: Opc);
10081 if (D16Info) {
10082 // Assume d16_lo/hi inst are always in same size
10083 unsigned LoInstOpcode = D16Info->LoOp;
10084 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: LoInstOpcode);
10085 DescSize = Desc.getSize();
10086 }
10087
10088 // If FMA Pseudo inst, get correct MC code size
10089 if (Opc == AMDGPU::V_FMA_MIX_F16_t16 || Opc == AMDGPU::V_FMA_MIX_BF16_t16) {
10090 // All potential lowerings are the same size; arbitrarily pick one.
10091 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: AMDGPU::V_FMA_MIXLO_F16);
10092 DescSize = Desc.getSize();
10093 }
10094
10095 return DescSize;
10096 }
10097}
10098
10099TargetInstrInfo::InstSizeVerifyMode
10100SIInstrInfo::getInstSizeVerifyMode(const MachineInstr &MI) const {
10101 if (MI.isBranch() && ST.hasOffset3fBug())
10102 return InstSizeVerifyMode::NoVerify;
10103 return InstSizeVerifyMode::ExactSize;
10104}
10105
10106bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const {
10107 if (!isFLAT(MI))
10108 return false;
10109
10110 if (MI.memoperands_empty())
10111 return true;
10112
10113 for (const MachineMemOperand *MMO : MI.memoperands()) {
10114 if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)
10115 return true;
10116 }
10117 return false;
10118}
10119
10120ArrayRef<std::pair<int, const char *>>
10121SIInstrInfo::getSerializableTargetIndices() const {
10122 static const std::pair<int, const char *> TargetIndices[] = {
10123 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
10124 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
10125 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
10126 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
10127 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
10128 return ArrayRef(TargetIndices);
10129}
10130
10131/// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The
10132/// post-RA version of misched uses CreateTargetMIHazardRecognizer.
10133ScheduleHazardRecognizer *
10134SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
10135 const ScheduleDAG *DAG) const {
10136 return new GCNHazardRecognizer(DAG->MF);
10137}
10138
10139/// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer
10140/// pass.
10141ScheduleHazardRecognizer *
10142SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF,
10143 MachineLoopInfo *MLI) const {
10144 return new GCNHazardRecognizer(MF, MLI);
10145}
10146
10147// Called during:
10148// - pre-RA scheduling and post-RA scheduling
10149ScheduleHazardRecognizer *
10150SIInstrInfo::CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
10151 const ScheduleDAGMI *DAG) const {
10152 // Borrowed from Arm Target
10153 // We would like to restrict this hazard recognizer to only
10154 // post-RA scheduling; we can tell that we're post-RA because we don't
10155 // track VRegLiveness.
10156 if (!DAG->hasVRegLiveness())
10157 return new GCNHazardRecognizer(DAG->MF);
10158 return TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG);
10159}
10160
10161std::pair<unsigned, unsigned>
10162SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
10163 return std::pair(TF & MO_MASK, TF & ~MO_MASK);
10164}
10165
10166ArrayRef<std::pair<unsigned, const char *>>
10167SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
10168 static const std::pair<unsigned, const char *> TargetFlags[] = {
10169 {MO_GOTPCREL, "amdgpu-gotprel"},
10170 {MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo"},
10171 {MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi"},
10172 {MO_GOTPCREL64, "amdgpu-gotprel64"},
10173 {MO_REL32_LO, "amdgpu-rel32-lo"},
10174 {MO_REL32_HI, "amdgpu-rel32-hi"},
10175 {MO_REL64, "amdgpu-rel64"},
10176 {MO_ABS32_LO, "amdgpu-abs32-lo"},
10177 {MO_ABS32_HI, "amdgpu-abs32-hi"},
10178 {MO_ABS64, "amdgpu-abs64"},
10179 };
10180
10181 return ArrayRef(TargetFlags);
10182}
10183
10184ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
10185SIInstrInfo::getSerializableMachineMemOperandTargetFlags() const {
10186 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
10187 {
10188 {MONoClobber, "amdgpu-noclobber"},
10189 {MOLastUse, "amdgpu-last-use"},
10190 {MOCooperative, "amdgpu-cooperative"},
10191 {MOThreadPrivate, "amdgpu-thread-private"},
10192 };
10193
10194 return ArrayRef(TargetFlags);
10195}
10196
10197unsigned SIInstrInfo::getLiveRangeSplitOpcode(Register SrcReg,
10198 const MachineFunction &MF) const {
10199 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
10200 assert(SrcReg.isVirtual());
10201 if (MFI->checkFlag(Reg: SrcReg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
10202 return AMDGPU::WWM_COPY;
10203
10204 return AMDGPU::COPY;
10205}
10206
10207bool SIInstrInfo::canAddToBBProlog(const MachineInstr &MI) const {
10208 uint32_t Opcode = MI.getOpcode();
10209 // Check if it is SGPR spill or wwm-register spill Opcode.
10210 if (isSGPRSpill(Opcode) || isWWMRegSpillOpcode(Opcode))
10211 return true;
10212
10213 const MachineFunction *MF = MI.getMF();
10214 const MachineRegisterInfo &MRI = MF->getRegInfo();
10215 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
10216
10217 // See if this is Liverange split instruction inserted for SGPR or
10218 // wwm-register. The implicit def inserted for wwm-registers should also be
10219 // included as they can appear at the bb begin.
10220 bool IsLRSplitInst = MI.getFlag(Flag: MachineInstr::LRSplit);
10221 if (!IsLRSplitInst && Opcode != AMDGPU::IMPLICIT_DEF)
10222 return false;
10223
10224 Register Reg = MI.getOperand(i: 0).getReg();
10225 if (RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg)))
10226 return IsLRSplitInst;
10227
10228 return MFI->isWWMReg(Reg);
10229}
10230
10231bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI,
10232 Register Reg) const {
10233 // We need to handle instructions which may be inserted during register
10234 // allocation to handle the prolog. The initial prolog instruction may have
10235 // been separated from the start of the block by spills and copies inserted
10236 // needed by the prolog. However, the insertions for scalar registers can
10237 // always be placed at the BB top as they are independent of the exec mask
10238 // value.
10239 bool IsNullOrVectorRegister = true;
10240 if (Reg) {
10241 const MachineFunction *MF = MI.getMF();
10242 const MachineRegisterInfo &MRI = MF->getRegInfo();
10243 IsNullOrVectorRegister = !RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg));
10244 }
10245
10246 return IsNullOrVectorRegister &&
10247 (canAddToBBProlog(MI) ||
10248 (!MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY &&
10249 MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &RI)));
10250}
10251
10252MachineInstrBuilder
10253SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
10254 MachineBasicBlock::iterator I,
10255 const DebugLoc &DL,
10256 Register DestReg) const {
10257 if (ST.hasAddNoCarryInsts())
10258 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e64), DestReg);
10259
10260 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
10261 Register UnusedCarry = MRI.createVirtualRegister(RegClass: RI.getBoolRC());
10262 MRI.setRegAllocationHint(VReg: UnusedCarry, Type: 0, PrefReg: RI.getVCC());
10263
10264 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg)
10265 .addReg(RegNo: UnusedCarry, Flags: RegState::Define | RegState::Dead);
10266}
10267
10268MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
10269 MachineBasicBlock::iterator I,
10270 const DebugLoc &DL,
10271 Register DestReg,
10272 RegScavenger &RS) const {
10273 if (ST.hasAddNoCarryInsts())
10274 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg);
10275
10276 // If available, prefer to use vcc.
10277 Register UnusedCarry = !RS.isRegUsed(Reg: AMDGPU::VCC)
10278 ? Register(RI.getVCC())
10279 : RS.scavengeRegisterBackwards(
10280 RC: *RI.getBoolRC(), To: I, /* RestoreAfter */ false,
10281 SPAdj: 0, /* AllowSpill */ false);
10282
10283 // TODO: Users need to deal with this.
10284 if (!UnusedCarry.isValid())
10285 return MachineInstrBuilder();
10286
10287 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg)
10288 .addReg(RegNo: UnusedCarry, Flags: RegState::Define | RegState::Dead);
10289}
10290
10291bool SIInstrInfo::isKillTerminator(unsigned Opcode) {
10292 switch (Opcode) {
10293 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
10294 case AMDGPU::SI_KILL_I1_TERMINATOR:
10295 return true;
10296 default:
10297 return false;
10298 }
10299}
10300
10301const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const {
10302 switch (Opcode) {
10303 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
10304 return get(Opcode: AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);
10305 case AMDGPU::SI_KILL_I1_PSEUDO:
10306 return get(Opcode: AMDGPU::SI_KILL_I1_TERMINATOR);
10307 default:
10308 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO");
10309 }
10310}
10311
10312bool SIInstrInfo::isLegalMUBUFImmOffset(unsigned Imm) const {
10313 return Imm <= getMaxMUBUFImmOffset(ST);
10314}
10315
10316unsigned SIInstrInfo::getMaxMUBUFImmOffset(const GCNSubtarget &ST) {
10317 // GFX12 field is non-negative 24-bit signed byte offset.
10318 const unsigned OffsetBits =
10319 ST.getGeneration() >= AMDGPUSubtarget::GFX12 ? 23 : 12;
10320 return (1 << OffsetBits) - 1;
10321}
10322
10323void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const {
10324 if (!ST.isWave32())
10325 return;
10326
10327 if (MI.isInlineAsm())
10328 return;
10329
10330 if (MI.getNumOperands() < MI.getNumExplicitOperands())
10331 return;
10332
10333 for (auto &Op : MI.implicit_operands()) {
10334 if (Op.isReg() && Op.getReg() == AMDGPU::VCC)
10335 Op.setReg(AMDGPU::VCC_LO);
10336 }
10337}
10338
10339bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const {
10340 if (!isSMRD(MI))
10341 return false;
10342
10343 // Check that it is using a buffer resource.
10344 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::sbase);
10345 if (Idx == -1) // e.g. s_memtime
10346 return false;
10347
10348 const int16_t RCID = getOpRegClassID(OpInfo: MI.getDesc().operands()[Idx]);
10349 return RI.getRegClass(i: RCID)->hasSubClassEq(RC: &AMDGPU::SGPR_128RegClass);
10350}
10351
10352// Given Imm, split it into the values to put into the SOffset and ImmOffset
10353// fields in an MUBUF instruction. Return false if it is not possible (due to a
10354// hardware bug needing a workaround).
10355//
10356// The required alignment ensures that individual address components remain
10357// aligned if they are aligned to begin with. It also ensures that additional
10358// offsets within the given alignment can be added to the resulting ImmOffset.
10359bool SIInstrInfo::splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset,
10360 uint32_t &ImmOffset, Align Alignment) const {
10361 const uint64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST);
10362 const uint32_t MaxImm = alignDown(Value: MaxOffset, Align: Alignment.value());
10363 uint32_t Overflow = 0;
10364
10365 if (Imm > MaxImm) {
10366 if (Imm <= MaxImm + 64) {
10367 // Use an SOffset inline constant for 4..64
10368 Overflow = Imm - MaxImm;
10369 Imm = MaxImm;
10370 } else {
10371 // Try to keep the same value in SOffset for adjacent loads, so that
10372 // the corresponding register contents can be re-used.
10373 //
10374 // Load values with all low-bits (except for alignment bits) set into
10375 // SOffset, so that a larger range of values can be covered using
10376 // s_movk_i32.
10377 //
10378 // Atomic operations fail to work correctly when individual address
10379 // components are unaligned, even if their sum is aligned.
10380 uint32_t High = (Imm + Alignment.value()) & ~MaxOffset;
10381 uint32_t Low = (Imm + Alignment.value()) & MaxOffset;
10382 Imm = Low;
10383 Overflow = High - Alignment.value();
10384 }
10385 }
10386
10387 if (Overflow > 0) {
10388 // There is a hardware bug in SI and CI which prevents address clamping in
10389 // MUBUF instructions from working correctly with SOffsets. The immediate
10390 // offset is unaffected.
10391 if (ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
10392 return false;
10393
10394 // It is not possible to set immediate in SOffset field on some targets.
10395 if (ST.hasRestrictedSOffset())
10396 return false;
10397 }
10398
10399 ImmOffset = Imm;
10400 SOffset = Overflow;
10401 return true;
10402}
10403
10404// Depending on the used address space and instructions, some immediate offsets
10405// are allowed and some are not.
10406// Pre-GFX12, flat instruction offsets can only be non-negative, global and
10407// scratch instruction offsets can also be negative. On GFX12, offsets can be
10408// negative for all variants.
10409//
10410// There are several bugs related to these offsets:
10411// On gfx10.1, flat instructions that go into the global address space cannot
10412// use an offset.
10413//
10414// For scratch instructions, the address can be either an SGPR or a VGPR.
10415// The following offsets can be used, depending on the architecture (x means
10416// cannot be used):
10417// +----------------------------+------+------+
10418// | Address-Mode | SGPR | VGPR |
10419// +----------------------------+------+------+
10420// | gfx9 | | |
10421// | negative, 4-aligned offset | x | ok |
10422// | negative, unaligned offset | x | ok |
10423// +----------------------------+------+------+
10424// | gfx10 | | |
10425// | negative, 4-aligned offset | ok | ok |
10426// | negative, unaligned offset | ok | x |
10427// +----------------------------+------+------+
10428// | gfx10.3 | | |
10429// | negative, 4-aligned offset | ok | ok |
10430// | negative, unaligned offset | ok | ok |
10431// +----------------------------+------+------+
10432//
10433// This function ignores the addressing mode, so if an offset cannot be used in
10434// one addressing mode, it is considered illegal.
10435bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
10436 AMDGPU::FlatAddrSpace FlatVariant) const {
10437 // TODO: Should 0 be special cased?
10438 if (!ST.hasFlatInstOffsets())
10439 return false;
10440
10441 using AMDGPU::FlatAddrSpace;
10442 if (ST.hasFlatSegmentOffsetBug() && FlatVariant == FlatAddrSpace::FLAT &&
10443 (AddrSpace == AMDGPUAS::FLAT_ADDRESS ||
10444 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS))
10445 return false;
10446
10447 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
10448 FlatVariant == FlatAddrSpace::FlatScratch && Offset < 0 &&
10449 (Offset % 4) != 0) {
10450 return false;
10451 }
10452
10453 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
10454 unsigned N = AMDGPU::getNumFlatOffsetBits(ST);
10455 return isIntN(N, x: Offset) && (AllowNegative || Offset >= 0);
10456}
10457
10458// See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not.
10459std::pair<int64_t, int64_t>
10460SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,
10461 AMDGPU::FlatAddrSpace FlatVariant) const {
10462 int64_t RemainderOffset = COffsetVal;
10463 int64_t ImmField = 0;
10464
10465 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
10466 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST) - 1;
10467
10468 if (AllowNegative) {
10469 // Use signed division by a power of two to truncate towards 0.
10470 int64_t D = 1LL << NumBits;
10471 RemainderOffset = (COffsetVal / D) * D;
10472 ImmField = COffsetVal - RemainderOffset;
10473
10474 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
10475 FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch && ImmField < 0 &&
10476 (ImmField % 4) != 0) {
10477 // Make ImmField a multiple of 4
10478 RemainderOffset += ImmField % 4;
10479 ImmField -= ImmField % 4;
10480 }
10481 } else if (COffsetVal >= 0) {
10482 ImmField = COffsetVal & maskTrailingOnes<uint64_t>(N: NumBits);
10483 RemainderOffset = COffsetVal - ImmField;
10484 }
10485
10486 assert(isLegalFLATOffset(ImmField, AddrSpace, FlatVariant));
10487 assert(RemainderOffset + ImmField == COffsetVal);
10488 return {ImmField, RemainderOffset};
10489}
10490
10491bool SIInstrInfo::allowNegativeFlatOffset(
10492 AMDGPU::FlatAddrSpace FlatVariant) const {
10493 if (ST.hasNegativeScratchOffsetBug() &&
10494 FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch)
10495 return false;
10496
10497 return FlatVariant != AMDGPU::FlatAddrSpace::FLAT || AMDGPU::isGFX12Plus(STI: ST);
10498}
10499
10500static unsigned subtargetEncodingFamily(const GCNSubtarget &ST) {
10501 switch (ST.getGeneration()) {
10502 default:
10503 break;
10504 case AMDGPUSubtarget::SOUTHERN_ISLANDS:
10505 case AMDGPUSubtarget::SEA_ISLANDS:
10506 return SIEncodingFamily::SI;
10507 case AMDGPUSubtarget::VOLCANIC_ISLANDS:
10508 case AMDGPUSubtarget::GFX9:
10509 return SIEncodingFamily::VI;
10510 case AMDGPUSubtarget::GFX10:
10511 return SIEncodingFamily::GFX10;
10512 case AMDGPUSubtarget::GFX11:
10513 return ST.hasGFX11_7Insts() ? SIEncodingFamily::GFX1170
10514 : SIEncodingFamily::GFX11;
10515 case AMDGPUSubtarget::GFX12:
10516 return ST.hasGFX1250Insts() ? SIEncodingFamily::GFX1250
10517 : SIEncodingFamily::GFX12;
10518 case AMDGPUSubtarget::GFX13:
10519 return SIEncodingFamily::GFX13;
10520 }
10521 llvm_unreachable("Unknown subtarget generation!");
10522}
10523
10524bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const {
10525 switch(MCOp) {
10526 // These opcodes use indirect register addressing so
10527 // they need special handling by codegen (currently missing).
10528 // Therefore it is too risky to allow these opcodes
10529 // to be selected by dpp combiner or sdwa peepholer.
10530 case AMDGPU::V_MOVRELS_B32_dpp_gfx10:
10531 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
10532 case AMDGPU::V_MOVRELD_B32_dpp_gfx10:
10533 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10:
10534 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10:
10535 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
10536 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10:
10537 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
10538 return true;
10539 default:
10540 return false;
10541 }
10542}
10543
10544#define GENERATE_RENAMED_GFX9_CASES(OPCODE) \
10545 case OPCODE##_dpp: \
10546 case OPCODE##_e32: \
10547 case OPCODE##_e64: \
10548 case OPCODE##_e64_dpp: \
10549 case OPCODE##_sdwa:
10550
10551static bool isRenamedInGFX9(int Opcode) {
10552 switch (Opcode) {
10553 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADDC_U32)
10554 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_CO_U32)
10555 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_U32)
10556 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBBREV_U32)
10557 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBB_U32)
10558 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_CO_U32)
10559 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_U32)
10560 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_CO_U32)
10561 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_U32)
10562 //
10563 case AMDGPU::V_DIV_FIXUP_F16_gfx9_e64:
10564 case AMDGPU::V_DIV_FIXUP_F16_gfx9_fake16_e64:
10565 case AMDGPU::V_FMA_F16_gfx9_e64:
10566 case AMDGPU::V_FMA_F16_gfx9_fake16_e64:
10567 case AMDGPU::V_INTERP_P2_F16:
10568 case AMDGPU::V_MAD_F16_e64:
10569 case AMDGPU::V_MAD_U16_e64:
10570 case AMDGPU::V_MAD_I16_e64:
10571 return true;
10572 default:
10573 return false;
10574 }
10575}
10576
10577int SIInstrInfo::pseudoToMCOpcode(int Opcode) const {
10578 assert(Opcode == (int)SIInstrInfo::getNonSoftWaitcntOpcode(Opcode) &&
10579 "SIInsertWaitcnts should have promoted soft waitcnt instructions!");
10580
10581 unsigned Gen = subtargetEncodingFamily(ST);
10582
10583 if (ST.getGeneration() == AMDGPUSubtarget::GFX9 && isRenamedInGFX9(Opcode))
10584 Gen = SIEncodingFamily::GFX9;
10585
10586 // Adjust the encoding family to GFX80 for D16 buffer instructions when the
10587 // subtarget has UnpackedD16VMem feature.
10588 // TODO: remove this when we discard GFX80 encoding.
10589 if (ST.hasUnpackedD16VMem() && SIInstrFlags::isD16Buf(O: get(Opcode)))
10590 Gen = SIEncodingFamily::GFX80;
10591
10592 if (SIInstrFlags::isSDWA(O: get(Opcode))) {
10593 switch (ST.getGeneration()) {
10594 default:
10595 Gen = SIEncodingFamily::SDWA;
10596 break;
10597 case AMDGPUSubtarget::GFX9:
10598 Gen = SIEncodingFamily::SDWA9;
10599 break;
10600 case AMDGPUSubtarget::GFX10:
10601 Gen = SIEncodingFamily::SDWA10;
10602 break;
10603 }
10604 }
10605
10606 if (isMAI(Opcode)) {
10607 int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode);
10608 if (MFMAOp != -1)
10609 Opcode = MFMAOp;
10610 }
10611
10612 int32_t MCOp = AMDGPU::getMCOpcode(Opcode, Gen);
10613
10614 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && ST.hasGFX11_7Insts())
10615 MCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX11);
10616
10617 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && ST.hasGFX1250Insts())
10618 MCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX12);
10619
10620 // -1 means that Opcode is already a native instruction.
10621 if (MCOp == -1)
10622 return Opcode;
10623
10624 if (ST.hasGFX90AInsts()) {
10625 uint32_t NMCOp = AMDGPU::INSTRUCTION_LIST_END;
10626 if (ST.hasGFX940Insts())
10627 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX940);
10628 if (NMCOp == AMDGPU::INSTRUCTION_LIST_END)
10629 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX90A);
10630 if (NMCOp == AMDGPU::INSTRUCTION_LIST_END)
10631 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX9);
10632 if (NMCOp != AMDGPU::INSTRUCTION_LIST_END)
10633 MCOp = NMCOp;
10634 }
10635
10636 // INSTRUCTION_LIST_END means that Opcode is a pseudo instruction that has no
10637 // encoding in the given subtarget generation.
10638 if (MCOp == AMDGPU::INSTRUCTION_LIST_END)
10639 return -1;
10640
10641 if (isAsmOnlyOpcode(MCOp))
10642 return -1;
10643
10644 return MCOp;
10645}
10646
10647static
10648TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) {
10649 assert(RegOpnd.isReg());
10650 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() :
10651 getRegSubRegPair(O: RegOpnd);
10652}
10653
10654TargetInstrInfo::RegSubRegPair
10655llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) {
10656 assert(MI.isRegSequence());
10657 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I)
10658 if (MI.getOperand(i: 1 + 2 * I + 1).getImm() == SubReg) {
10659 auto &RegOp = MI.getOperand(i: 1 + 2 * I);
10660 return getRegOrUndef(RegOpnd: RegOp);
10661 }
10662 return TargetInstrInfo::RegSubRegPair();
10663}
10664
10665// Try to find the definition of reg:subreg in subreg-manipulation pseudos
10666// Following a subreg of reg:subreg isn't supported
10667static bool followSubRegDef(MachineInstr &MI,
10668 TargetInstrInfo::RegSubRegPair &RSR) {
10669 if (!RSR.SubReg)
10670 return false;
10671 switch (MI.getOpcode()) {
10672 default: break;
10673 case AMDGPU::REG_SEQUENCE:
10674 RSR = getRegSequenceSubReg(MI, SubReg: RSR.SubReg);
10675 return true;
10676 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg
10677 case AMDGPU::INSERT_SUBREG:
10678 if (RSR.SubReg == (unsigned)MI.getOperand(i: 3).getImm())
10679 // inserted the subreg we're looking for
10680 RSR = getRegOrUndef(RegOpnd: MI.getOperand(i: 2));
10681 else { // the subreg in the rest of the reg
10682 auto R1 = getRegOrUndef(RegOpnd: MI.getOperand(i: 1));
10683 if (R1.SubReg) // subreg of subreg isn't supported
10684 return false;
10685 RSR.Reg = R1.Reg;
10686 }
10687 return true;
10688 }
10689 return false;
10690}
10691
10692MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
10693 const MachineRegisterInfo &MRI) {
10694 assert(MRI.isSSA());
10695 if (!P.Reg.isVirtual())
10696 return nullptr;
10697
10698 auto RSR = P;
10699 auto *DefInst = MRI.getVRegDef(Reg: RSR.Reg);
10700 while (auto *MI = DefInst) {
10701 DefInst = nullptr;
10702 switch (MI->getOpcode()) {
10703 case AMDGPU::COPY:
10704 case AMDGPU::V_MOV_B32_e32: {
10705 auto &Op1 = MI->getOperand(i: 1);
10706 if (Op1.isReg() && Op1.getReg().isVirtual()) {
10707 if (Op1.isUndef())
10708 return nullptr;
10709 RSR = getRegSubRegPair(O: Op1);
10710 DefInst = MRI.getVRegDef(Reg: RSR.Reg);
10711 }
10712 break;
10713 }
10714 default:
10715 if (followSubRegDef(MI&: *MI, RSR)) {
10716 if (!RSR.Reg)
10717 return nullptr;
10718 DefInst = MRI.getVRegDef(Reg: RSR.Reg);
10719 }
10720 }
10721 if (!DefInst)
10722 return MI;
10723 }
10724 return nullptr;
10725}
10726
10727bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
10728 Register VReg,
10729 const MachineInstr &DefMI,
10730 const MachineInstr &UseMI) {
10731 assert(MRI.isSSA() && "Must be run on SSA");
10732
10733 auto *TRI = MRI.getTargetRegisterInfo();
10734 auto *DefBB = DefMI.getParent();
10735
10736 // Don't bother searching between blocks, although it is possible this block
10737 // doesn't modify exec.
10738 if (UseMI.getParent() != DefBB)
10739 return true;
10740
10741 const int MaxInstScan = 20;
10742 int NumInst = 0;
10743
10744 // Stop scan at the use.
10745 auto E = UseMI.getIterator();
10746 for (auto I = std::next(x: DefMI.getIterator()); I != E; ++I) {
10747 if (I->isDebugInstr())
10748 continue;
10749
10750 if (++NumInst > MaxInstScan)
10751 return true;
10752
10753 if (I->modifiesRegister(Reg: AMDGPU::EXEC, TRI))
10754 return true;
10755 }
10756
10757 return false;
10758}
10759
10760bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
10761 Register VReg,
10762 const MachineInstr &DefMI) {
10763 assert(MRI.isSSA() && "Must be run on SSA");
10764
10765 auto *TRI = MRI.getTargetRegisterInfo();
10766 auto *DefBB = DefMI.getParent();
10767
10768 const int MaxUseScan = 10;
10769 int NumUse = 0;
10770
10771 for (auto &Use : MRI.use_nodbg_operands(Reg: VReg)) {
10772 auto &UseInst = *Use.getParent();
10773 // Don't bother searching between blocks, although it is possible this block
10774 // doesn't modify exec.
10775 if (UseInst.getParent() != DefBB || UseInst.isPHI())
10776 return true;
10777
10778 if (++NumUse > MaxUseScan)
10779 return true;
10780 }
10781
10782 if (NumUse == 0)
10783 return false;
10784
10785 const int MaxInstScan = 20;
10786 int NumInst = 0;
10787
10788 // Stop scan when we have seen all the uses.
10789 for (auto I = std::next(x: DefMI.getIterator()); ; ++I) {
10790 assert(I != DefBB->end());
10791
10792 if (I->isDebugInstr())
10793 continue;
10794
10795 if (++NumInst > MaxInstScan)
10796 return true;
10797
10798 for (const MachineOperand &Op : I->operands()) {
10799 // We don't check reg masks here as they're used only on calls:
10800 // 1. EXEC is only considered const within one BB
10801 // 2. Call should be a terminator instruction if present in a BB
10802
10803 if (!Op.isReg())
10804 continue;
10805
10806 Register Reg = Op.getReg();
10807 if (Op.isUse()) {
10808 if (Reg == VReg && --NumUse == 0)
10809 return false;
10810 } else if (TRI->regsOverlap(RegA: Reg, RegB: AMDGPU::EXEC))
10811 return true;
10812 }
10813 }
10814}
10815
10816MachineInstr *SIInstrInfo::createPHIDestinationCopy(
10817 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt,
10818 const DebugLoc &DL, Register Src, Register Dst) const {
10819 auto Cur = MBB.begin();
10820 if (Cur != MBB.end())
10821 do {
10822 if (!Cur->isPHI() && Cur->readsRegister(Reg: Dst, /*TRI=*/nullptr))
10823 return BuildMI(BB&: MBB, I: Cur, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Dst).addReg(RegNo: Src);
10824 ++Cur;
10825 } while (Cur != MBB.end() && Cur != LastPHIIt);
10826
10827 return TargetInstrInfo::createPHIDestinationCopy(MBB, InsPt: LastPHIIt, DL, Src,
10828 Dst);
10829}
10830
10831MachineInstr *SIInstrInfo::createPHISourceCopy(
10832 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt,
10833 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const {
10834 if (InsPt != MBB.end() &&
10835 (InsPt->getOpcode() == AMDGPU::SI_IF ||
10836 InsPt->getOpcode() == AMDGPU::SI_ELSE ||
10837 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) &&
10838 InsPt->definesRegister(Reg: Src, /*TRI=*/nullptr)) {
10839 InsPt++;
10840 return BuildMI(BB&: MBB, I: InsPt, MIMD: DL,
10841 MCID: get(Opcode: AMDGPU::LaneMaskConstants::get(ST).MovTermOpc), DestReg: Dst)
10842 .addReg(RegNo: Src, Flags: {}, SubReg: SrcSubReg)
10843 .addReg(RegNo: AMDGPU::EXEC, Flags: RegState::Implicit);
10844 }
10845 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg,
10846 Dst);
10847}
10848
10849bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); }
10850
10851bool SIInstrInfo::hasRAWDependency(const MachineInstr &FirstMI,
10852 const MachineInstr &SecondMI) const {
10853 for (const auto &Use : SecondMI.all_uses()) {
10854 if (Use.isReg() && FirstMI.modifiesRegister(Reg: Use.getReg(), TRI: &RI))
10855 return true;
10856 }
10857 return false;
10858}
10859
10860/// If OpX is multicycle, anti-dependencies are not allowed.
10861/// isDPMACCInstruction was not designed for VOPD, but it is fit for the
10862/// purpose.
10863bool llvm::SIInstrInfo::isVOPDAntidependencyAllowed(
10864 const MachineInstr &OpX) const {
10865 return !AMDGPU::isDPMACCInstruction(Opc: OpX.getOpcode());
10866}
10867
10868MachineInstr *
10869SIInstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
10870 ArrayRef<unsigned> Ops, int FrameIndex,
10871 MachineInstr *&CopyMI, LiveIntervals *LIS,
10872 VirtRegMap *VRM) const {
10873 // This is a bit of a hack (copied from AArch64). Consider this instruction:
10874 //
10875 // %0:sreg_32 = COPY $m0
10876 //
10877 // We explicitly chose SReg_32 for the virtual register so such a copy might
10878 // be eliminated by RegisterCoalescer. However, that may not be possible, and
10879 // %0 may even spill. We can't spill $m0 normally (it would require copying to
10880 // a numbered SGPR anyway), and since it is in the SReg_32 register class,
10881 // TargetInstrInfo::foldMemoryOperand() is going to try.
10882 // A similar issue also exists with spilling and reloading $exec registers.
10883 //
10884 // To prevent that, constrain the %0 register class here.
10885 if (isFullCopyInstr(MI)) {
10886 Register DstReg = MI.getOperand(i: 0).getReg();
10887 Register SrcReg = MI.getOperand(i: 1).getReg();
10888 if ((DstReg.isVirtual() || SrcReg.isVirtual()) &&
10889 (DstReg.isVirtual() != SrcReg.isVirtual())) {
10890 MachineRegisterInfo &MRI = MF.getRegInfo();
10891 Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg;
10892 const TargetRegisterClass *RC = MRI.getRegClass(Reg: VirtReg);
10893 if (RC->hasSuperClassEq(RC: &AMDGPU::SReg_32RegClass)) {
10894 MRI.constrainRegClass(Reg: VirtReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
10895 return nullptr;
10896 }
10897 if (RC->hasSuperClassEq(RC: &AMDGPU::SReg_64RegClass)) {
10898 MRI.constrainRegClass(Reg: VirtReg, RC: &AMDGPU::SReg_64_XEXECRegClass);
10899 return nullptr;
10900 }
10901 }
10902 }
10903
10904 return nullptr;
10905}
10906
10907unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
10908 const MachineInstr &MI,
10909 unsigned *PredCost) const {
10910 if (MI.isBundle()) {
10911 MachineBasicBlock::const_instr_iterator I(MI.getIterator());
10912 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end());
10913 unsigned Lat = 0, Count = 0;
10914 for (++I; I != E && I->isBundledWithPred(); ++I) {
10915 ++Count;
10916 Lat = std::max(a: Lat, b: SchedModel.computeInstrLatency(MI: &*I));
10917 }
10918 return Lat + Count - 1;
10919 }
10920
10921 return SchedModel.computeInstrLatency(MI: &MI);
10922}
10923
10924const MachineOperand &
10925SIInstrInfo::getCalleeOperand(const MachineInstr &MI) const {
10926 if (const MachineOperand *CallAddrOp =
10927 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0))
10928 return *CallAddrOp;
10929 return TargetInstrInfo::getCalleeOperand(MI);
10930}
10931
10932ValueUniformity
10933SIInstrInfo::getGenericValueUniformity(const MachineInstr &MI) const {
10934 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
10935 unsigned Opcode = MI.getOpcode();
10936
10937 auto HandleAddrSpaceCast = [this, &MRI](const MachineInstr &MI) {
10938 Register Dst = MI.getOperand(i: 0).getReg();
10939 Register Src = isa<GIntrinsic>(Val: MI) ? MI.getOperand(i: 2).getReg()
10940 : MI.getOperand(i: 1).getReg();
10941 LLT DstTy = MRI.getType(Reg: Dst);
10942 LLT SrcTy = MRI.getType(Reg: Src);
10943 unsigned DstAS = DstTy.getAddressSpace();
10944 unsigned SrcAS = SrcTy.getAddressSpace();
10945 return SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
10946 DstAS == AMDGPUAS::FLAT_ADDRESS &&
10947 ST.hasGloballyAddressableScratch()
10948 ? ValueUniformity::NeverUniform
10949 : ValueUniformity::Default;
10950 };
10951
10952 // If the target supports globally addressable scratch, the mapping from
10953 // scratch memory to the flat aperture changes therefore an address space cast
10954 // is no longer uniform.
10955 if (Opcode == TargetOpcode::G_ADDRSPACE_CAST)
10956 return HandleAddrSpaceCast(MI);
10957
10958 if (auto *GI = dyn_cast<GIntrinsic>(Val: &MI)) {
10959 auto IID = GI->getIntrinsicID();
10960 if (AMDGPU::isIntrinsicSourceOfDivergence(IntrID: IID))
10961 return ValueUniformity::NeverUniform;
10962 if (AMDGPU::isIntrinsicAlwaysUniform(IntrID: IID))
10963 return ValueUniformity::AlwaysUniform;
10964
10965 switch (IID) {
10966 case Intrinsic::amdgcn_addrspacecast_nonnull:
10967 return HandleAddrSpaceCast(MI);
10968 case Intrinsic::amdgcn_if:
10969 case Intrinsic::amdgcn_else:
10970 // FIXME: Uniform if second result
10971 break;
10972 }
10973
10974 return ValueUniformity::Default;
10975 }
10976
10977 // Loads from the private and flat address spaces are divergent, because
10978 // threads can execute the load instruction with the same inputs and get
10979 // different results.
10980 //
10981 // All other loads are not divergent, because if threads issue loads with the
10982 // same arguments, they will always get the same result.
10983 if (Opcode == AMDGPU::G_LOAD || Opcode == AMDGPU::G_ZEXTLOAD ||
10984 Opcode == AMDGPU::G_SEXTLOAD) {
10985 if (MI.memoperands_empty())
10986 return ValueUniformity::NeverUniform; // conservative assumption
10987
10988 if (llvm::any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *mmo) {
10989 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
10990 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
10991 })) {
10992 // At least one MMO in a non-global address space.
10993 return ValueUniformity::NeverUniform;
10994 }
10995 return ValueUniformity::Default;
10996 }
10997
10998 if (SIInstrInfo::isGenericAtomicRMWOpcode(Opc: Opcode) ||
10999 Opcode == AMDGPU::G_ATOMIC_CMPXCHG ||
11000 Opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS ||
11001 AMDGPU::isGenericAtomic(Opc: Opcode)) {
11002 return ValueUniformity::NeverUniform;
11003 }
11004
11005 // Result is computed from uniform SP and uniform wave-wide max size.
11006 if (Opcode == TargetOpcode::G_DYN_STACKALLOC)
11007 return ValueUniformity::AlwaysUniform;
11008
11009 if (Opcode == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
11010 return ValueUniformity::NeverUniform;
11011
11012 return ValueUniformity::Default;
11013}
11014
11015const MIRFormatter *SIInstrInfo::getMIRFormatter() const {
11016 if (!Formatter)
11017 Formatter = std::make_unique<AMDGPUMIRFormatter>(args: ST);
11018 return Formatter.get();
11019}
11020
11021ValueUniformity SIInstrInfo::getValueUniformity(const MachineInstr &MI) const {
11022
11023 if (isNeverUniform(MI))
11024 return ValueUniformity::NeverUniform;
11025
11026 unsigned opcode = MI.getOpcode();
11027 if (opcode == AMDGPU::V_READLANE_B32 ||
11028 opcode == AMDGPU::V_READFIRSTLANE_B32 ||
11029 opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR)
11030 return ValueUniformity::AlwaysUniform;
11031
11032 // If any of defs is divergent, report as NeverUniform. isUniformReg will
11033 // calculate in more detail for each def from its reg class, if available.
11034 if (MI.isInlineAsm()) {
11035 for (const MachineOperand &MO : MI.operands()) {
11036 if (!MO.isReg() || !MO.isDef())
11037 continue;
11038 const TargetRegisterClass *RC =
11039 MI.getRegClassConstraint(OpIdx: MO.getOperandNo(), TII: this, TRI: &RI);
11040 if (!RC || !RI.isSGPRClass(RC))
11041 return ValueUniformity::NeverUniform;
11042 }
11043 }
11044
11045 if (isCopyInstr(MI)) {
11046 const MachineOperand &srcOp = MI.getOperand(i: 1);
11047 if (srcOp.isReg() && srcOp.getReg().isPhysical()) {
11048 const TargetRegisterClass *regClass =
11049 RI.getPhysRegBaseClass(Reg: srcOp.getReg());
11050 return RI.isSGPRClass(RC: regClass) ? ValueUniformity::AlwaysUniform
11051 : ValueUniformity::NeverUniform;
11052 }
11053 return ValueUniformity::Default;
11054 }
11055
11056 // GMIR handling
11057 if (MI.isPreISelOpcode())
11058 return SIInstrInfo::getGenericValueUniformity(MI);
11059
11060 // Atomics are divergent because they are executed sequentially: when an
11061 // atomic operation refers to the same address in each thread, then each
11062 // thread after the first sees the value written by the previous thread as
11063 // original value.
11064
11065 if (isAtomic(MI))
11066 return ValueUniformity::NeverUniform;
11067
11068 // Loads from the private and flat address spaces are divergent, because
11069 // threads can execute the load instruction with the same inputs and get
11070 // different results.
11071 if (isFLAT(MI) && MI.mayLoad()) {
11072 if (MI.memoperands_empty())
11073 return ValueUniformity::NeverUniform; // conservative assumption
11074
11075 if (llvm::any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *mmo) {
11076 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
11077 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
11078 })) {
11079 // At least one MMO in a non-global address space.
11080 return ValueUniformity::NeverUniform;
11081 }
11082
11083 return ValueUniformity::Default;
11084 }
11085
11086 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
11087 const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo();
11088
11089 // FIXME: It's conceptually broken to report this for an instruction, and not
11090 // a specific def operand. For inline asm in particular, there could be mixed
11091 // uniform and divergent results.
11092 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
11093 const MachineOperand &SrcOp = MI.getOperand(i: I);
11094 if (!SrcOp.isReg())
11095 continue;
11096
11097 Register Reg = SrcOp.getReg();
11098 if (!Reg || !SrcOp.readsReg())
11099 continue;
11100
11101 // If RegBank is null, this is unassigned or an unallocatable special
11102 // register, which are all scalars.
11103 const RegisterBank *RegBank = RBI->getRegBank(Reg, MRI, TRI: RI);
11104 if (RegBank && RegBank->getID() != AMDGPU::SGPRRegBankID)
11105 return ValueUniformity::NeverUniform;
11106 }
11107
11108 // TODO: Uniformity check condtions above can be rearranged for more
11109 // redability
11110
11111 // TODO: amdgcn.{ballot, [if]cmp} should be AlwaysUniform, but they are
11112 // currently turned into no-op COPYs by SelectionDAG ISel and are
11113 // therefore no longer recognizable.
11114
11115 return ValueUniformity::Default;
11116}
11117
11118unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) {
11119 switch (MF.getFunction().getCallingConv()) {
11120 case CallingConv::AMDGPU_PS:
11121 return 1;
11122 case CallingConv::AMDGPU_VS:
11123 return 2;
11124 case CallingConv::AMDGPU_GS:
11125 return 3;
11126 case CallingConv::AMDGPU_HS:
11127 case CallingConv::AMDGPU_LS:
11128 case CallingConv::AMDGPU_ES: {
11129 const Function &F = MF.getFunction();
11130 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
11131 F, "ds_ordered_count unsupported for this calling conv"));
11132 [[fallthrough]];
11133 }
11134 case CallingConv::AMDGPU_CS:
11135 case CallingConv::AMDGPU_KERNEL:
11136 case CallingConv::C:
11137 case CallingConv::Fast:
11138 default:
11139 // Assume other calling conventions are various compute callable functions
11140 return 0;
11141 }
11142}
11143
11144bool SIInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
11145 Register &SrcReg2, int64_t &CmpMask,
11146 int64_t &CmpValue) const {
11147 if (!MI.getOperand(i: 0).isReg() || MI.getOperand(i: 0).getSubReg())
11148 return false;
11149
11150 switch (MI.getOpcode()) {
11151 default:
11152 break;
11153 case AMDGPU::S_CMP_EQ_U32:
11154 case AMDGPU::S_CMP_EQ_I32:
11155 case AMDGPU::S_CMP_LG_U32:
11156 case AMDGPU::S_CMP_LG_I32:
11157 case AMDGPU::S_CMP_LT_U32:
11158 case AMDGPU::S_CMP_LT_I32:
11159 case AMDGPU::S_CMP_GT_U32:
11160 case AMDGPU::S_CMP_GT_I32:
11161 case AMDGPU::S_CMP_LE_U32:
11162 case AMDGPU::S_CMP_LE_I32:
11163 case AMDGPU::S_CMP_GE_U32:
11164 case AMDGPU::S_CMP_GE_I32:
11165 case AMDGPU::S_CMP_EQ_U64:
11166 case AMDGPU::S_CMP_LG_U64:
11167 SrcReg = MI.getOperand(i: 0).getReg();
11168 if (MI.getOperand(i: 1).isReg()) {
11169 if (MI.getOperand(i: 1).getSubReg())
11170 return false;
11171 SrcReg2 = MI.getOperand(i: 1).getReg();
11172 CmpValue = 0;
11173 } else if (MI.getOperand(i: 1).isImm()) {
11174 SrcReg2 = Register();
11175 CmpValue = MI.getOperand(i: 1).getImm();
11176 } else {
11177 return false;
11178 }
11179 CmpMask = ~0;
11180 return true;
11181 case AMDGPU::S_CMPK_EQ_U32:
11182 case AMDGPU::S_CMPK_EQ_I32:
11183 case AMDGPU::S_CMPK_LG_U32:
11184 case AMDGPU::S_CMPK_LG_I32:
11185 case AMDGPU::S_CMPK_LT_U32:
11186 case AMDGPU::S_CMPK_LT_I32:
11187 case AMDGPU::S_CMPK_GT_U32:
11188 case AMDGPU::S_CMPK_GT_I32:
11189 case AMDGPU::S_CMPK_LE_U32:
11190 case AMDGPU::S_CMPK_LE_I32:
11191 case AMDGPU::S_CMPK_GE_U32:
11192 case AMDGPU::S_CMPK_GE_I32:
11193 SrcReg = MI.getOperand(i: 0).getReg();
11194 SrcReg2 = Register();
11195 CmpValue = MI.getOperand(i: 1).getImm();
11196 CmpMask = ~0;
11197 return true;
11198 }
11199
11200 return false;
11201}
11202
11203static bool isSCCDeadOnExit(MachineBasicBlock *MBB) {
11204 for (MachineBasicBlock *S : MBB->successors()) {
11205 if (S->isLiveIn(Reg: AMDGPU::SCC))
11206 return false;
11207 }
11208 return true;
11209}
11210
11211// Invert all uses of SCC following SCCDef because SCCDef may be deleted and
11212// (incoming SCC) = !(SCC defined by SCCDef).
11213// Return true if all uses can be re-written, false otherwise.
11214bool SIInstrInfo::invertSCCUse(MachineInstr *SCCDef) const {
11215 MachineBasicBlock *MBB = SCCDef->getParent();
11216 SmallVector<MachineInstr *> InvertInstr;
11217 bool SCCIsDead = false;
11218
11219 // Scan instructions for SCC uses that need to be inverted until SCC is dead.
11220 constexpr unsigned ScanLimit = 12;
11221 unsigned Count = 0;
11222 for (MachineInstr &MI :
11223 make_range(x: std::next(x: MachineBasicBlock::iterator(SCCDef)), y: MBB->end())) {
11224 if (++Count > ScanLimit)
11225 return false;
11226 if (MI.readsRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
11227 if (MI.getOpcode() == AMDGPU::S_CSELECT_B32 ||
11228 MI.getOpcode() == AMDGPU::S_CSELECT_B64 ||
11229 MI.getOpcode() == AMDGPU::S_CBRANCH_SCC0 ||
11230 MI.getOpcode() == AMDGPU::S_CBRANCH_SCC1)
11231 InvertInstr.push_back(Elt: &MI);
11232 else
11233 return false;
11234 }
11235 if (MI.definesRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
11236 SCCIsDead = true;
11237 break;
11238 }
11239 }
11240 if (!SCCIsDead && isSCCDeadOnExit(MBB))
11241 SCCIsDead = true;
11242
11243 // SCC may have more uses. Can't invert all of them.
11244 if (!SCCIsDead)
11245 return false;
11246
11247 // Invert uses
11248 for (MachineInstr *MI : InvertInstr) {
11249 if (MI->getOpcode() == AMDGPU::S_CSELECT_B32 ||
11250 MI->getOpcode() == AMDGPU::S_CSELECT_B64) {
11251 swapOperands(Inst&: *MI);
11252 } else if (MI->getOpcode() == AMDGPU::S_CBRANCH_SCC0 ||
11253 MI->getOpcode() == AMDGPU::S_CBRANCH_SCC1) {
11254 MI->setDesc(get(Opcode: MI->getOpcode() == AMDGPU::S_CBRANCH_SCC0
11255 ? AMDGPU::S_CBRANCH_SCC1
11256 : AMDGPU::S_CBRANCH_SCC0));
11257 } else {
11258 llvm_unreachable("SCC used but no inversion handling");
11259 }
11260 }
11261 return true;
11262}
11263
11264// SCC is already valid after SCCValid.
11265// SCCRedefine will redefine SCC to the same value already available after
11266// SCCValid. If there are no intervening SCC conflicts delete SCCRedefine and
11267// update kill/dead flags if necessary.
11268bool SIInstrInfo::optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,
11269 bool NeedInversion) const {
11270 MachineInstr *KillsSCC = nullptr;
11271 if (SCCValid->getParent() != SCCRedefine->getParent())
11272 return false;
11273 for (MachineInstr &MI : make_range(x: std::next(x: SCCValid->getIterator()),
11274 y: SCCRedefine->getIterator())) {
11275 if (MI.modifiesRegister(Reg: AMDGPU::SCC, TRI: &RI))
11276 return false;
11277 if (MI.killsRegister(Reg: AMDGPU::SCC, TRI: &RI))
11278 KillsSCC = &MI;
11279 }
11280 if (NeedInversion && !invertSCCUse(SCCDef: SCCRedefine))
11281 return false;
11282 if (MachineOperand *SccDef =
11283 SCCValid->findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr))
11284 SccDef->setIsDead(false);
11285 if (KillsSCC)
11286 KillsSCC->clearRegisterKills(Reg: AMDGPU::SCC, /*TRI=*/RegInfo: nullptr);
11287 SCCRedefine->eraseFromParent();
11288 return true;
11289}
11290
11291static bool foldableSelect(const MachineInstr &Def) {
11292 if (Def.getOpcode() != AMDGPU::S_CSELECT_B32 &&
11293 Def.getOpcode() != AMDGPU::S_CSELECT_B64)
11294 return false;
11295 bool Op1IsNonZeroImm =
11296 Def.getOperand(i: 1).isImm() && Def.getOperand(i: 1).getImm() != 0;
11297 bool Op2IsZeroImm =
11298 Def.getOperand(i: 2).isImm() && Def.getOperand(i: 2).getImm() == 0;
11299 if (!Op1IsNonZeroImm || !Op2IsZeroImm)
11300 return false;
11301 return true;
11302}
11303
11304static bool setsSCCIfResultIsZero(const MachineInstr &Def, bool &NeedInversion,
11305 unsigned &NewDefOpc) {
11306 // S_ADD_U32 X, 1 sets SCC on carryout which can only happen if result==0.
11307 // S_ADD_I32 X, 1 can be converted to S_ADD_U32 X, 1 if SCC is dead.
11308 if (Def.getOpcode() != AMDGPU::S_ADD_I32 &&
11309 Def.getOpcode() != AMDGPU::S_ADD_U32)
11310 return false;
11311 const MachineOperand &AddSrc1 = Def.getOperand(i: 1);
11312 const MachineOperand &AddSrc2 = Def.getOperand(i: 2);
11313 int64_t addend;
11314
11315 if ((!AddSrc1.isImm() || AddSrc1.getImm() != 1) &&
11316 (!AddSrc2.isImm() || AddSrc2.getImm() != 1) &&
11317 (!getFoldableImm(MO: &AddSrc1, Imm&: addend) || addend != 1) &&
11318 (!getFoldableImm(MO: &AddSrc2, Imm&: addend) || addend != 1))
11319 return false;
11320
11321 if (Def.getOpcode() == AMDGPU::S_ADD_I32) {
11322 const MachineOperand *SccDef =
11323 Def.findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
11324 if (!SccDef->isDead())
11325 return false;
11326 NewDefOpc = AMDGPU::S_ADD_U32;
11327 }
11328 NeedInversion = !NeedInversion;
11329 return true;
11330}
11331
11332bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
11333 Register SrcReg2, int64_t CmpMask,
11334 int64_t CmpValue,
11335 const MachineRegisterInfo *MRI) const {
11336 if (!SrcReg || SrcReg.isPhysical())
11337 return false;
11338
11339 if (SrcReg2 && !getFoldableImm(Reg: SrcReg2, MRI: *MRI, Imm&: CmpValue))
11340 return false;
11341
11342 const auto optimizeCmpSelect = [&CmpInstr, SrcReg, CmpValue, MRI,
11343 this](bool NeedInversion) -> bool {
11344 if (CmpValue != 0)
11345 return false;
11346
11347 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
11348 if (!Def)
11349 return false;
11350
11351 // For S_OP that set SCC = DST!=0, do the transformation
11352 //
11353 // s_cmp_[lg|eq]_* (S_OP ...), 0 => (S_OP ...)
11354 //
11355 // For (S_OP ...) that set SCC = DST==0, invert NeedInversion and
11356 // do the transformation:
11357 //
11358 // s_cmp_[lg|eq]_* (S_OP ...), 0 => (S_OP ...)
11359 //
11360 // If foldableSelect, s_cmp_lg_* is redundant because the SCC input value
11361 // for S_CSELECT* already has the same value that will be calculated by
11362 // s_cmp_lg_*
11363 //
11364 // s_cmp_[lg|eq]_* (S_CSELECT* (non-zero imm), 0), 0 => (S_CSELECT*
11365 // (non-zero imm), 0)
11366
11367 unsigned NewDefOpc = Def->getOpcode();
11368 if (!setsSCCIfResultIsNonZero(*Def) &&
11369 !setsSCCIfResultIsZero(Def: *Def, NeedInversion, NewDefOpc) &&
11370 !foldableSelect(Def: *Def))
11371 return false;
11372
11373 if (!optimizeSCC(SCCValid: Def, SCCRedefine: &CmpInstr, NeedInversion))
11374 return false;
11375
11376 if (NewDefOpc != Def->getOpcode())
11377 Def->setDesc(get(Opcode: NewDefOpc));
11378
11379 // If s_or_b32 result, sY, is unused (i.e. it is effectively a 64-bit
11380 // s_cmp_lg of a register pair) and the inputs are the hi and lo-halves of a
11381 // 64-bit foldableSelect then delete s_or_b32 in the sequence:
11382 // sX = s_cselect_b64 (non-zero imm), 0
11383 // sLo = copy sX.sub0
11384 // sHi = copy sX.sub1
11385 // sY = s_or_b32 sLo, sHi
11386 if (Def->getOpcode() == AMDGPU::S_OR_B32 &&
11387 MRI->use_nodbg_empty(RegNo: Def->getOperand(i: 0).getReg())) {
11388 const MachineOperand &OrOpnd1 = Def->getOperand(i: 1);
11389 const MachineOperand &OrOpnd2 = Def->getOperand(i: 2);
11390 if (OrOpnd1.isReg() && OrOpnd2.isReg()) {
11391 MachineInstr *Def1 = MRI->getVRegDef(Reg: OrOpnd1.getReg());
11392 MachineInstr *Def2 = MRI->getVRegDef(Reg: OrOpnd2.getReg());
11393 if (Def1 && Def1->getOpcode() == AMDGPU::COPY && Def2 &&
11394 Def2->getOpcode() == AMDGPU::COPY && Def1->getOperand(i: 1).isReg() &&
11395 Def2->getOperand(i: 1).isReg() &&
11396 Def1->getOperand(i: 1).getSubReg() == AMDGPU::sub0 &&
11397 Def2->getOperand(i: 1).getSubReg() == AMDGPU::sub1 &&
11398 Def1->getOperand(i: 1).getReg() == Def2->getOperand(i: 1).getReg()) {
11399 MachineInstr *Select = MRI->getVRegDef(Reg: Def1->getOperand(i: 1).getReg());
11400 if (Select && foldableSelect(Def: *Select))
11401 optimizeSCC(SCCValid: Select, SCCRedefine: Def, /*NeedInversion=*/false);
11402 }
11403 }
11404 }
11405 return true;
11406 };
11407
11408 const auto optimizeCmpAnd = [&CmpInstr, SrcReg, CmpValue, MRI,
11409 this](int64_t ExpectedValue, unsigned SrcSize,
11410 bool IsReversible, bool IsSigned) -> bool {
11411 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11412 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11413 // s_cmp_ge_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11414 // s_cmp_ge_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11415 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 1 << n => s_and_b64 $src, 1 << n
11416 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11417 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11418 // s_cmp_gt_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11419 // s_cmp_gt_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11420 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 0 => s_and_b64 $src, 1 << n
11421 //
11422 // Signed ge/gt are not used for the sign bit.
11423 //
11424 // If result of the AND is unused except in the compare:
11425 // s_and_b(32|64) $src, 1 << n => s_bitcmp1_b(32|64) $src, n
11426 //
11427 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
11428 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
11429 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 0 => s_bitcmp0_b64 $src, n
11430 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
11431 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
11432 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 1 << n => s_bitcmp0_b64 $src, n
11433
11434 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
11435 if (!Def)
11436 return false;
11437
11438 if (Def->getOpcode() != AMDGPU::S_AND_B32 &&
11439 Def->getOpcode() != AMDGPU::S_AND_B64)
11440 return false;
11441
11442 int64_t Mask;
11443 const auto isMask = [&Mask, SrcSize](const MachineOperand *MO) -> bool {
11444 if (MO->isImm())
11445 Mask = MO->getImm();
11446 else if (!getFoldableImm(MO, Imm&: Mask))
11447 return false;
11448 Mask &= maxUIntN(N: SrcSize);
11449 return isPowerOf2_64(Value: Mask);
11450 };
11451
11452 MachineOperand *SrcOp = &Def->getOperand(i: 1);
11453 if (isMask(SrcOp))
11454 SrcOp = &Def->getOperand(i: 2);
11455 else if (isMask(&Def->getOperand(i: 2)))
11456 SrcOp = &Def->getOperand(i: 1);
11457 else
11458 return false;
11459
11460 // A valid Mask is required to have a single bit set, hence a non-zero and
11461 // power-of-two value. This verifies that we will not do 64-bit shift below.
11462 assert(llvm::has_single_bit<uint64_t>(Mask) && "Invalid mask.");
11463 unsigned BitNo = llvm::countr_zero(Val: (uint64_t)Mask);
11464 if (IsSigned && BitNo == SrcSize - 1)
11465 return false;
11466
11467 ExpectedValue <<= BitNo;
11468
11469 bool IsReversedCC = false;
11470 if (CmpValue != ExpectedValue) {
11471 if (!IsReversible)
11472 return false;
11473 IsReversedCC = CmpValue == (ExpectedValue ^ Mask);
11474 if (!IsReversedCC)
11475 return false;
11476 }
11477
11478 Register DefReg = Def->getOperand(i: 0).getReg();
11479 if (IsReversedCC && !MRI->hasOneNonDBGUse(RegNo: DefReg))
11480 return false;
11481
11482 if (!optimizeSCC(SCCValid: Def, SCCRedefine: &CmpInstr, /*NeedInversion=*/false))
11483 return false;
11484
11485 if (!MRI->use_nodbg_empty(RegNo: DefReg)) {
11486 assert(!IsReversedCC);
11487 return true;
11488 }
11489
11490 // Replace AND with unused result with a S_BITCMP.
11491 MachineBasicBlock *MBB = Def->getParent();
11492
11493 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32
11494 : AMDGPU::S_BITCMP1_B32
11495 : IsReversedCC ? AMDGPU::S_BITCMP0_B64
11496 : AMDGPU::S_BITCMP1_B64;
11497
11498 BuildMI(BB&: *MBB, I: Def, MIMD: Def->getDebugLoc(), MCID: get(Opcode: NewOpc))
11499 .add(MO: *SrcOp)
11500 .addImm(Val: BitNo);
11501 Def->eraseFromParent();
11502
11503 return true;
11504 };
11505
11506 switch (CmpInstr.getOpcode()) {
11507 default:
11508 break;
11509 case AMDGPU::S_CMP_EQ_U32:
11510 case AMDGPU::S_CMP_EQ_I32:
11511 case AMDGPU::S_CMPK_EQ_U32:
11512 case AMDGPU::S_CMPK_EQ_I32:
11513 return optimizeCmpAnd(1, 32, true, false) ||
11514 optimizeCmpSelect(/*NeedInversion=*/true);
11515 case AMDGPU::S_CMP_GE_U32:
11516 case AMDGPU::S_CMPK_GE_U32:
11517 return optimizeCmpAnd(1, 32, false, false);
11518 case AMDGPU::S_CMP_GE_I32:
11519 case AMDGPU::S_CMPK_GE_I32:
11520 return optimizeCmpAnd(1, 32, false, true);
11521 case AMDGPU::S_CMP_EQ_U64:
11522 return optimizeCmpAnd(1, 64, true, false);
11523 case AMDGPU::S_CMP_LG_U32:
11524 case AMDGPU::S_CMP_LG_I32:
11525 case AMDGPU::S_CMPK_LG_U32:
11526 case AMDGPU::S_CMPK_LG_I32:
11527 return optimizeCmpAnd(0, 32, true, false) ||
11528 optimizeCmpSelect(/*NeedInversion=*/false);
11529 case AMDGPU::S_CMP_GT_U32:
11530 case AMDGPU::S_CMPK_GT_U32:
11531 return optimizeCmpAnd(0, 32, false, false);
11532 case AMDGPU::S_CMP_GT_I32:
11533 case AMDGPU::S_CMPK_GT_I32:
11534 return optimizeCmpAnd(0, 32, false, true);
11535 case AMDGPU::S_CMP_LG_U64:
11536 return optimizeCmpAnd(0, 64, true, false) ||
11537 optimizeCmpSelect(/*NeedInversion=*/false);
11538 }
11539
11540 return false;
11541}
11542
11543void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI,
11544 AMDGPU::OpName OpName) const {
11545 if (!ST.needsAlignedVGPRs())
11546 return;
11547
11548 int OpNo = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpName);
11549 if (OpNo < 0)
11550 return;
11551 MachineOperand &Op = MI.getOperand(i: OpNo);
11552 if (getOpSize(MI, OpNo) > 4)
11553 return;
11554
11555 // Add implicit aligned super-reg to force alignment on the data operand.
11556 const DebugLoc &DL = MI.getDebugLoc();
11557 MachineBasicBlock *BB = MI.getParent();
11558 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
11559 Register DataReg = Op.getReg();
11560 bool IsAGPR = RI.isAGPR(MRI, Reg: DataReg);
11561 Register Undef = MRI.createVirtualRegister(
11562 RegClass: IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);
11563 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
11564 Register NewVR =
11565 MRI.createVirtualRegister(RegClass: IsAGPR ? &AMDGPU::AReg_64_Align2RegClass
11566 : &AMDGPU::VReg_64_Align2RegClass);
11567 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewVR)
11568 .addReg(RegNo: DataReg, Flags: {}, SubReg: Op.getSubReg())
11569 .addImm(Val: AMDGPU::sub0)
11570 .addReg(RegNo: Undef)
11571 .addImm(Val: AMDGPU::sub1);
11572 Op.setReg(NewVR);
11573 Op.setSubReg(AMDGPU::sub0);
11574 MI.addOperand(Op: MachineOperand::CreateReg(Reg: NewVR, isDef: false, isImp: true));
11575}
11576
11577bool SIInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const {
11578 if (isIGLP(MI: *MI))
11579 return false;
11580
11581 return TargetInstrInfo::isGlobalMemoryObject(MI);
11582}
11583
11584bool SIInstrInfo::isXDLWMMA(const MachineInstr &MI) const {
11585 if (!isWMMA(MI) && !isSWMMAC(MI))
11586 return false;
11587
11588 if (ST.hasGFX1250Insts())
11589 return AMDGPU::getWMMAIsXDL(Opc: MI.getOpcode());
11590
11591 return true;
11592}
11593
11594bool SIInstrInfo::isXDL(const MachineInstr &MI) const {
11595 unsigned Opcode = MI.getOpcode();
11596
11597 if (AMDGPU::isGFX12Plus(STI: ST))
11598 return isDOT(MI) || isXDLWMMA(MI);
11599
11600 if (!isMAI(MI) || isDGEMM(Opcode) ||
11601 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
11602 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
11603 return false;
11604
11605 if (!ST.hasGFX940Insts())
11606 return true;
11607
11608 return AMDGPU::getMAIIsGFX940XDL(Opc: Opcode);
11609}
11610