1//===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// SI Implementation of TargetInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIInstrInfo.h"
15#include "AMDGPU.h"
16#include "AMDGPUInstrInfo.h"
17#include "GCNHazardRecognizer.h"
18#include "GCNSubtarget.h"
19#include "SIMachineFunctionInfo.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "llvm/Analysis/ValueTracking.h"
22#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
23#include "llvm/CodeGen/LiveIntervals.h"
24#include "llvm/CodeGen/LiveVariables.h"
25#include "llvm/CodeGen/MachineDominators.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/MachineScheduler.h"
28#include "llvm/CodeGen/RegisterScavenging.h"
29#include "llvm/CodeGen/ScheduleDAG.h"
30#include "llvm/IR/DiagnosticInfo.h"
31#include "llvm/IR/IntrinsicsAMDGPU.h"
32#include "llvm/MC/MCContext.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Target/TargetMachine.h"
35
36using namespace llvm;
37
38#define DEBUG_TYPE "si-instr-info"
39
40#define GET_INSTRINFO_CTOR_DTOR
41#include "AMDGPUGenInstrInfo.inc"
42
43namespace llvm::AMDGPU {
44#define GET_D16ImageDimIntrinsics_IMPL
45#define GET_ImageDimIntrinsicTable_IMPL
46#define GET_RsrcIntrinsics_IMPL
47#include "AMDGPUGenSearchableTables.inc"
48} // namespace llvm::AMDGPU
49
50// Must be at least 4 to be able to branch over minimum unconditional branch
51// code. This is only for making it possible to write reasonably small tests for
52// long branches.
53static cl::opt<unsigned>
54BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(Val: 16),
55 cl::desc("Restrict range of branch instructions (DEBUG)"));
56
57static cl::opt<bool> Fix16BitCopies(
58 "amdgpu-fix-16-bit-physreg-copies",
59 cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"),
60 cl::init(Val: true),
61 cl::ReallyHidden);
62
63SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
64 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
65 RI(ST), ST(ST) {
66 SchedModel.init(TSInfo: &ST);
67}
68
69//===----------------------------------------------------------------------===//
70// TargetInstrInfo callbacks
71//===----------------------------------------------------------------------===//
72
73static unsigned getNumOperandsNoGlue(SDNode *Node) {
74 unsigned N = Node->getNumOperands();
75 while (N && Node->getOperand(Num: N - 1).getValueType() == MVT::Glue)
76 --N;
77 return N;
78}
79
80/// Returns true if both nodes have the same value for the given
81/// operand \p Op, or if both nodes do not have this operand.
82static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
83 unsigned Opc0 = N0->getMachineOpcode();
84 unsigned Opc1 = N1->getMachineOpcode();
85
86 int Op0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc0, NamedIdx: OpName);
87 int Op1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc1, NamedIdx: OpName);
88
89 if (Op0Idx == -1 && Op1Idx == -1)
90 return true;
91
92
93 if ((Op0Idx == -1 && Op1Idx != -1) ||
94 (Op1Idx == -1 && Op0Idx != -1))
95 return false;
96
97 // getNamedOperandIdx returns the index for the MachineInstr's operands,
98 // which includes the result as the first operand. We are indexing into the
99 // MachineSDNode's operands, so we need to skip the result operand to get
100 // the real index.
101 --Op0Idx;
102 --Op1Idx;
103
104 return N0->getOperand(Num: Op0Idx) == N1->getOperand(Num: Op1Idx);
105}
106
107static bool canRemat(const MachineInstr &MI) {
108
109 if (SIInstrInfo::isVOP1(MI) || SIInstrInfo::isVOP2(MI) ||
110 SIInstrInfo::isVOP3(MI) || SIInstrInfo::isSDWA(MI) ||
111 SIInstrInfo::isSALU(MI))
112 return true;
113
114 if (SIInstrInfo::isSMRD(MI)) {
115 return !MI.memoperands_empty() &&
116 llvm::all_of(Range: MI.memoperands(), P: [](const MachineMemOperand *MMO) {
117 return MMO->isLoad() && MMO->isInvariant();
118 });
119 }
120
121 return false;
122}
123
124bool SIInstrInfo::isReallyTriviallyReMaterializable(
125 const MachineInstr &MI) const {
126
127 if (canRemat(MI)) {
128 // Normally VALU use of exec would block the rematerialization, but that
129 // is OK in this case to have an implicit exec read as all VALU do.
130 // We really want all of the generic logic for this except for this.
131
132 // Another potential implicit use is mode register. The core logic of
133 // the RA will not attempt rematerialization if mode is set anywhere
134 // in the function, otherwise it is safe since mode is not changed.
135
136 // There is difference to generic method which does not allow
137 // rematerialization if there are virtual register uses. We allow this,
138 // therefore this method includes SOP instructions as well.
139 if (!MI.hasImplicitDef() &&
140 MI.getNumImplicitOperands() == MI.getDesc().implicit_uses().size() &&
141 !MI.mayRaiseFPException())
142 return true;
143 }
144
145 return TargetInstrInfo::isReallyTriviallyReMaterializable(MI);
146}
147
148// Returns true if the scalar result of a VALU instruction depends on exec.
149static bool resultDependsOnExec(const MachineInstr &MI) {
150 // Ignore comparisons which are only used masked with exec.
151 // This allows some hoisting/sinking of VALU comparisons.
152 if (MI.isCompare()) {
153 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
154 Register DstReg = MI.getOperand(i: 0).getReg();
155 if (!DstReg.isVirtual())
156 return true;
157 for (MachineInstr &Use : MRI.use_nodbg_instructions(Reg: DstReg)) {
158 switch (Use.getOpcode()) {
159 case AMDGPU::S_AND_SAVEEXEC_B32:
160 case AMDGPU::S_AND_SAVEEXEC_B64:
161 break;
162 case AMDGPU::S_AND_B32:
163 case AMDGPU::S_AND_B64:
164 if (!Use.readsRegister(Reg: AMDGPU::EXEC, /*TRI=*/nullptr))
165 return true;
166 break;
167 default:
168 return true;
169 }
170 }
171 return false;
172 }
173
174 switch (MI.getOpcode()) {
175 default:
176 break;
177 case AMDGPU::V_READFIRSTLANE_B32:
178 return true;
179 }
180
181 return false;
182}
183
184bool SIInstrInfo::isIgnorableUse(const MachineOperand &MO) const {
185 // Any implicit use of exec by VALU is not a real register read.
186 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&
187 isVALU(MI: *MO.getParent()) && !resultDependsOnExec(MI: *MO.getParent());
188}
189
190bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
191 MachineBasicBlock *SuccToSinkTo,
192 MachineCycleInfo *CI) const {
193 // Allow sinking if MI edits lane mask (divergent i1 in sgpr).
194 if (MI.getOpcode() == AMDGPU::SI_IF_BREAK)
195 return true;
196
197 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
198 // Check if sinking of MI would create temporal divergent use.
199 for (auto Op : MI.uses()) {
200 if (Op.isReg() && Op.getReg().isVirtual() &&
201 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Op.getReg()))) {
202 MachineInstr *SgprDef = MRI.getVRegDef(Reg: Op.getReg());
203
204 // SgprDef defined inside cycle
205 MachineCycle *FromCycle = CI->getCycle(Block: SgprDef->getParent());
206 if (FromCycle == nullptr)
207 continue;
208
209 MachineCycle *ToCycle = CI->getCycle(Block: SuccToSinkTo);
210 // Check if there is a FromCycle that contains SgprDef's basic block but
211 // does not contain SuccToSinkTo and also has divergent exit condition.
212 while (FromCycle && !FromCycle->contains(C: ToCycle)) {
213 SmallVector<MachineBasicBlock *, 1> ExitingBlocks;
214 FromCycle->getExitingBlocks(TmpStorage&: ExitingBlocks);
215
216 // FromCycle has divergent exit condition.
217 for (MachineBasicBlock *ExitingBlock : ExitingBlocks) {
218 if (hasDivergentBranch(MBB: ExitingBlock))
219 return false;
220 }
221
222 FromCycle = FromCycle->getParentCycle();
223 }
224 }
225 }
226
227 return true;
228}
229
230bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,
231 int64_t &Offset0,
232 int64_t &Offset1) const {
233 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
234 return false;
235
236 unsigned Opc0 = Load0->getMachineOpcode();
237 unsigned Opc1 = Load1->getMachineOpcode();
238
239 // Make sure both are actually loads.
240 if (!get(Opcode: Opc0).mayLoad() || !get(Opcode: Opc1).mayLoad())
241 return false;
242
243 // A mayLoad instruction without a def is not a load. Likely a prefetch.
244 if (!get(Opcode: Opc0).getNumDefs() || !get(Opcode: Opc1).getNumDefs())
245 return false;
246
247 if (isDS(Opcode: Opc0) && isDS(Opcode: Opc1)) {
248
249 // FIXME: Handle this case:
250 if (getNumOperandsNoGlue(Node: Load0) != getNumOperandsNoGlue(Node: Load1))
251 return false;
252
253 // Check base reg.
254 if (Load0->getOperand(Num: 0) != Load1->getOperand(Num: 0))
255 return false;
256
257 // Skip read2 / write2 variants for simplicity.
258 // TODO: We should report true if the used offsets are adjacent (excluded
259 // st64 versions).
260 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc0, NamedIdx: AMDGPU::OpName::offset);
261 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc1, NamedIdx: AMDGPU::OpName::offset);
262 if (Offset0Idx == -1 || Offset1Idx == -1)
263 return false;
264
265 // XXX - be careful of dataless loads
266 // getNamedOperandIdx returns the index for MachineInstrs. Since they
267 // include the output in the operand list, but SDNodes don't, we need to
268 // subtract the index by one.
269 Offset0Idx -= get(Opcode: Opc0).NumDefs;
270 Offset1Idx -= get(Opcode: Opc1).NumDefs;
271 Offset0 = Load0->getConstantOperandVal(Num: Offset0Idx);
272 Offset1 = Load1->getConstantOperandVal(Num: Offset1Idx);
273 return true;
274 }
275
276 if (isSMRD(Opcode: Opc0) && isSMRD(Opcode: Opc1)) {
277 // Skip time and cache invalidation instructions.
278 if (!AMDGPU::hasNamedOperand(Opcode: Opc0, NamedIdx: AMDGPU::OpName::sbase) ||
279 !AMDGPU::hasNamedOperand(Opcode: Opc1, NamedIdx: AMDGPU::OpName::sbase))
280 return false;
281
282 unsigned NumOps = getNumOperandsNoGlue(Node: Load0);
283 if (NumOps != getNumOperandsNoGlue(Node: Load1))
284 return false;
285
286 // Check base reg.
287 if (Load0->getOperand(Num: 0) != Load1->getOperand(Num: 0))
288 return false;
289
290 // Match register offsets, if both register and immediate offsets present.
291 assert(NumOps == 4 || NumOps == 5);
292 if (NumOps == 5 && Load0->getOperand(Num: 1) != Load1->getOperand(Num: 1))
293 return false;
294
295 const ConstantSDNode *Load0Offset =
296 dyn_cast<ConstantSDNode>(Val: Load0->getOperand(Num: NumOps - 3));
297 const ConstantSDNode *Load1Offset =
298 dyn_cast<ConstantSDNode>(Val: Load1->getOperand(Num: NumOps - 3));
299
300 if (!Load0Offset || !Load1Offset)
301 return false;
302
303 Offset0 = Load0Offset->getZExtValue();
304 Offset1 = Load1Offset->getZExtValue();
305 return true;
306 }
307
308 // MUBUF and MTBUF can access the same addresses.
309 if ((isMUBUF(Opcode: Opc0) || isMTBUF(Opcode: Opc0)) && (isMUBUF(Opcode: Opc1) || isMTBUF(Opcode: Opc1))) {
310
311 // MUBUF and MTBUF have vaddr at different indices.
312 if (!nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::soffset) ||
313 !nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::vaddr) ||
314 !nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::srsrc))
315 return false;
316
317 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opcode: Opc0, NamedIdx: AMDGPU::OpName::offset);
318 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opcode: Opc1, NamedIdx: AMDGPU::OpName::offset);
319
320 if (OffIdx0 == -1 || OffIdx1 == -1)
321 return false;
322
323 // getNamedOperandIdx returns the index for MachineInstrs. Since they
324 // include the output in the operand list, but SDNodes don't, we need to
325 // subtract the index by one.
326 OffIdx0 -= get(Opcode: Opc0).NumDefs;
327 OffIdx1 -= get(Opcode: Opc1).NumDefs;
328
329 SDValue Off0 = Load0->getOperand(Num: OffIdx0);
330 SDValue Off1 = Load1->getOperand(Num: OffIdx1);
331
332 // The offset might be a FrameIndexSDNode.
333 if (!isa<ConstantSDNode>(Val: Off0) || !isa<ConstantSDNode>(Val: Off1))
334 return false;
335
336 Offset0 = Off0->getAsZExtVal();
337 Offset1 = Off1->getAsZExtVal();
338 return true;
339 }
340
341 return false;
342}
343
344static bool isStride64(unsigned Opc) {
345 switch (Opc) {
346 case AMDGPU::DS_READ2ST64_B32:
347 case AMDGPU::DS_READ2ST64_B64:
348 case AMDGPU::DS_WRITE2ST64_B32:
349 case AMDGPU::DS_WRITE2ST64_B64:
350 return true;
351 default:
352 return false;
353 }
354}
355
356bool SIInstrInfo::getMemOperandsWithOffsetWidth(
357 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
358 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
359 const TargetRegisterInfo *TRI) const {
360 if (!LdSt.mayLoadOrStore())
361 return false;
362
363 unsigned Opc = LdSt.getOpcode();
364 OffsetIsScalable = false;
365 const MachineOperand *BaseOp, *OffsetOp;
366 int DataOpIdx;
367
368 if (isDS(MI: LdSt)) {
369 BaseOp = getNamedOperand(MI: LdSt, OpName: AMDGPU::OpName::addr);
370 OffsetOp = getNamedOperand(MI: LdSt, OpName: AMDGPU::OpName::offset);
371 if (OffsetOp) {
372 // Normal, single offset LDS instruction.
373 if (!BaseOp) {
374 // DS_CONSUME/DS_APPEND use M0 for the base address.
375 // TODO: find the implicit use operand for M0 and use that as BaseOp?
376 return false;
377 }
378 BaseOps.push_back(Elt: BaseOp);
379 Offset = OffsetOp->getImm();
380 // Get appropriate operand, and compute width accordingly.
381 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::vdst);
382 if (DataOpIdx == -1)
383 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::data0);
384 Width = getOpSize(MI: LdSt, OpNo: DataOpIdx);
385 } else {
386 // The 2 offset instructions use offset0 and offset1 instead. We can treat
387 // these as a load with a single offset if the 2 offsets are consecutive.
388 // We will use this for some partially aligned loads.
389 const MachineOperand *Offset0Op =
390 getNamedOperand(MI: LdSt, OpName: AMDGPU::OpName::offset0);
391 const MachineOperand *Offset1Op =
392 getNamedOperand(MI: LdSt, OpName: AMDGPU::OpName::offset1);
393
394 unsigned Offset0 = Offset0Op->getImm() & 0xff;
395 unsigned Offset1 = Offset1Op->getImm() & 0xff;
396 if (Offset0 + 1 != Offset1)
397 return false;
398
399 // Each of these offsets is in element sized units, so we need to convert
400 // to bytes of the individual reads.
401
402 unsigned EltSize;
403 if (LdSt.mayLoad())
404 EltSize = TRI->getRegSizeInBits(RC: *getOpRegClass(MI: LdSt, OpNo: 0)) / 16;
405 else {
406 assert(LdSt.mayStore());
407 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::data0);
408 EltSize = TRI->getRegSizeInBits(RC: *getOpRegClass(MI: LdSt, OpNo: Data0Idx)) / 8;
409 }
410
411 if (isStride64(Opc))
412 EltSize *= 64;
413
414 BaseOps.push_back(Elt: BaseOp);
415 Offset = EltSize * Offset0;
416 // Get appropriate operand(s), and compute width accordingly.
417 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::vdst);
418 if (DataOpIdx == -1) {
419 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::data0);
420 Width = getOpSize(MI: LdSt, OpNo: DataOpIdx);
421 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::data1);
422 Width = Width.getValue() + getOpSize(MI: LdSt, OpNo: DataOpIdx);
423 } else {
424 Width = getOpSize(MI: LdSt, OpNo: DataOpIdx);
425 }
426 }
427 return true;
428 }
429
430 if (isMUBUF(MI: LdSt) || isMTBUF(MI: LdSt)) {
431 const MachineOperand *RSrc = getNamedOperand(MI: LdSt, OpName: AMDGPU::OpName::srsrc);
432 if (!RSrc) // e.g. BUFFER_WBINVL1_VOL
433 return false;
434 BaseOps.push_back(Elt: RSrc);
435 BaseOp = getNamedOperand(MI: LdSt, OpName: AMDGPU::OpName::vaddr);
436 if (BaseOp && !BaseOp->isFI())
437 BaseOps.push_back(Elt: BaseOp);
438 const MachineOperand *OffsetImm =
439 getNamedOperand(MI: LdSt, OpName: AMDGPU::OpName::offset);
440 Offset = OffsetImm->getImm();
441 const MachineOperand *SOffset =
442 getNamedOperand(MI: LdSt, OpName: AMDGPU::OpName::soffset);
443 if (SOffset) {
444 if (SOffset->isReg())
445 BaseOps.push_back(Elt: SOffset);
446 else
447 Offset += SOffset->getImm();
448 }
449 // Get appropriate operand, and compute width accordingly.
450 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::vdst);
451 if (DataOpIdx == -1)
452 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::vdata);
453 if (DataOpIdx == -1) // LDS DMA
454 return false;
455 Width = getOpSize(MI: LdSt, OpNo: DataOpIdx);
456 return true;
457 }
458
459 if (isImage(MI: LdSt)) {
460 auto RsrcOpName =
461 isMIMG(MI: LdSt) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
462 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: RsrcOpName);
463 BaseOps.push_back(Elt: &LdSt.getOperand(i: SRsrcIdx));
464 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::vaddr0);
465 if (VAddr0Idx >= 0) {
466 // GFX10 possible NSA encoding.
467 for (int I = VAddr0Idx; I < SRsrcIdx; ++I)
468 BaseOps.push_back(Elt: &LdSt.getOperand(i: I));
469 } else {
470 BaseOps.push_back(Elt: getNamedOperand(MI: LdSt, OpName: AMDGPU::OpName::vaddr));
471 }
472 Offset = 0;
473 // Get appropriate operand, and compute width accordingly.
474 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::vdata);
475 if (DataOpIdx == -1)
476 return false; // no return sampler
477 Width = getOpSize(MI: LdSt, OpNo: DataOpIdx);
478 return true;
479 }
480
481 if (isSMRD(MI: LdSt)) {
482 BaseOp = getNamedOperand(MI: LdSt, OpName: AMDGPU::OpName::sbase);
483 if (!BaseOp) // e.g. S_MEMTIME
484 return false;
485 BaseOps.push_back(Elt: BaseOp);
486 OffsetOp = getNamedOperand(MI: LdSt, OpName: AMDGPU::OpName::offset);
487 Offset = OffsetOp ? OffsetOp->getImm() : 0;
488 // Get appropriate operand, and compute width accordingly.
489 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::sdst);
490 if (DataOpIdx == -1)
491 return false;
492 Width = getOpSize(MI: LdSt, OpNo: DataOpIdx);
493 return true;
494 }
495
496 if (isFLAT(MI: LdSt)) {
497 // Instructions have either vaddr or saddr or both or none.
498 BaseOp = getNamedOperand(MI: LdSt, OpName: AMDGPU::OpName::vaddr);
499 if (BaseOp)
500 BaseOps.push_back(Elt: BaseOp);
501 BaseOp = getNamedOperand(MI: LdSt, OpName: AMDGPU::OpName::saddr);
502 if (BaseOp)
503 BaseOps.push_back(Elt: BaseOp);
504 Offset = getNamedOperand(MI: LdSt, OpName: AMDGPU::OpName::offset)->getImm();
505 // Get appropriate operand, and compute width accordingly.
506 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::vdst);
507 if (DataOpIdx == -1)
508 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::vdata);
509 if (DataOpIdx == -1) // LDS DMA
510 return false;
511 Width = getOpSize(MI: LdSt, OpNo: DataOpIdx);
512 return true;
513 }
514
515 return false;
516}
517
518static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
519 ArrayRef<const MachineOperand *> BaseOps1,
520 const MachineInstr &MI2,
521 ArrayRef<const MachineOperand *> BaseOps2) {
522 // Only examine the first "base" operand of each instruction, on the
523 // assumption that it represents the real base address of the memory access.
524 // Other operands are typically offsets or indices from this base address.
525 if (BaseOps1.front()->isIdenticalTo(Other: *BaseOps2.front()))
526 return true;
527
528 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
529 return false;
530
531 auto MO1 = *MI1.memoperands_begin();
532 auto MO2 = *MI2.memoperands_begin();
533 if (MO1->getAddrSpace() != MO2->getAddrSpace())
534 return false;
535
536 auto Base1 = MO1->getValue();
537 auto Base2 = MO2->getValue();
538 if (!Base1 || !Base2)
539 return false;
540 Base1 = getUnderlyingObject(V: Base1);
541 Base2 = getUnderlyingObject(V: Base2);
542
543 if (isa<UndefValue>(Val: Base1) || isa<UndefValue>(Val: Base2))
544 return false;
545
546 return Base1 == Base2;
547}
548
549bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
550 int64_t Offset1, bool OffsetIsScalable1,
551 ArrayRef<const MachineOperand *> BaseOps2,
552 int64_t Offset2, bool OffsetIsScalable2,
553 unsigned ClusterSize,
554 unsigned NumBytes) const {
555 // If the mem ops (to be clustered) do not have the same base ptr, then they
556 // should not be clustered
557 if (!BaseOps1.empty() && !BaseOps2.empty()) {
558 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
559 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
560 if (!memOpsHaveSameBasePtr(MI1: FirstLdSt, BaseOps1, MI2: SecondLdSt, BaseOps2))
561 return false;
562 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
563 // If only one base op is empty, they do not have the same base ptr
564 return false;
565 }
566
567 // In order to avoid register pressure, on an average, the number of DWORDS
568 // loaded together by all clustered mem ops should not exceed 8. This is an
569 // empirical value based on certain observations and performance related
570 // experiments.
571 // The good thing about this heuristic is - it avoids clustering of too many
572 // sub-word loads, and also avoids clustering of wide loads. Below is the
573 // brief summary of how the heuristic behaves for various `LoadSize`.
574 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops
575 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops
576 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops
577 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops
578 // (5) LoadSize >= 17: do not cluster
579 const unsigned LoadSize = NumBytes / ClusterSize;
580 const unsigned NumDWORDs = ((LoadSize + 3) / 4) * ClusterSize;
581 return NumDWORDs <= 8;
582}
583
584// FIXME: This behaves strangely. If, for example, you have 32 load + stores,
585// the first 16 loads will be interleaved with the stores, and the next 16 will
586// be clustered as expected. It should really split into 2 16 store batches.
587//
588// Loads are clustered until this returns false, rather than trying to schedule
589// groups of stores. This also means we have to deal with saying different
590// address space loads should be clustered, and ones which might cause bank
591// conflicts.
592//
593// This might be deprecated so it might not be worth that much effort to fix.
594bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1,
595 int64_t Offset0, int64_t Offset1,
596 unsigned NumLoads) const {
597 assert(Offset1 > Offset0 &&
598 "Second offset should be larger than first offset!");
599 // If we have less than 16 loads in a row, and the offsets are within 64
600 // bytes, then schedule together.
601
602 // A cacheline is 64 bytes (for global memory).
603 return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
604}
605
606static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB,
607 MachineBasicBlock::iterator MI,
608 const DebugLoc &DL, MCRegister DestReg,
609 MCRegister SrcReg, bool KillSrc,
610 const char *Msg = "illegal VGPR to SGPR copy") {
611 MachineFunction *MF = MBB.getParent();
612 DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(), Msg, DL, DS_Error);
613 LLVMContext &C = MF->getFunction().getContext();
614 C.diagnose(DI: IllegalCopy);
615
616 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_ILLEGAL_COPY), DestReg)
617 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc));
618}
619
620/// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not
621/// possible to have a direct copy in these cases on GFX908, so an intermediate
622/// VGPR copy is required.
623static void indirectCopyToAGPR(const SIInstrInfo &TII,
624 MachineBasicBlock &MBB,
625 MachineBasicBlock::iterator MI,
626 const DebugLoc &DL, MCRegister DestReg,
627 MCRegister SrcReg, bool KillSrc,
628 RegScavenger &RS, bool RegsOverlap,
629 Register ImpDefSuperReg = Register(),
630 Register ImpUseSuperReg = Register()) {
631 assert((TII.getSubtarget().hasMAIInsts() &&
632 !TII.getSubtarget().hasGFX90AInsts()) &&
633 "Expected GFX908 subtarget.");
634
635 assert((AMDGPU::SReg_32RegClass.contains(SrcReg) ||
636 AMDGPU::AGPR_32RegClass.contains(SrcReg)) &&
637 "Source register of the copy should be either an SGPR or an AGPR.");
638
639 assert(AMDGPU::AGPR_32RegClass.contains(DestReg) &&
640 "Destination register of the copy should be an AGPR.");
641
642 const SIRegisterInfo &RI = TII.getRegisterInfo();
643
644 // First try to find defining accvgpr_write to avoid temporary registers.
645 // In the case of copies of overlapping AGPRs, we conservatively do not
646 // reuse previous accvgpr_writes. Otherwise, we may incorrectly pick up
647 // an accvgpr_write used for this same copy due to implicit-defs
648 if (!RegsOverlap) {
649 for (auto Def = MI, E = MBB.begin(); Def != E; ) {
650 --Def;
651
652 if (!Def->modifiesRegister(Reg: SrcReg, TRI: &RI))
653 continue;
654
655 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
656 Def->getOperand(i: 0).getReg() != SrcReg)
657 break;
658
659 MachineOperand &DefOp = Def->getOperand(i: 1);
660 assert(DefOp.isReg() || DefOp.isImm());
661
662 if (DefOp.isReg()) {
663 bool SafeToPropagate = true;
664 // Check that register source operand is not clobbered before MI.
665 // Immediate operands are always safe to propagate.
666 for (auto I = Def; I != MI && SafeToPropagate; ++I)
667 if (I->modifiesRegister(Reg: DefOp.getReg(), TRI: &RI))
668 SafeToPropagate = false;
669
670 if (!SafeToPropagate)
671 break;
672
673 DefOp.setIsKill(false);
674 }
675
676 MachineInstrBuilder Builder =
677 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
678 .add(MO: DefOp);
679 if (ImpDefSuperReg)
680 Builder.addReg(RegNo: ImpDefSuperReg, flags: RegState::Define | RegState::Implicit);
681
682 if (ImpUseSuperReg) {
683 Builder.addReg(RegNo: ImpUseSuperReg,
684 flags: getKillRegState(B: KillSrc) | RegState::Implicit);
685 }
686
687 return;
688 }
689 }
690
691 RS.enterBasicBlockEnd(MBB);
692 RS.backward(I: std::next(x: MI));
693
694 // Ideally we want to have three registers for a long reg_sequence copy
695 // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
696 unsigned MaxVGPRs = RI.getRegPressureLimit(RC: &AMDGPU::VGPR_32RegClass,
697 MF&: *MBB.getParent());
698
699 // Registers in the sequence are allocated contiguously so we can just
700 // use register number to pick one of three round-robin temps.
701 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3;
702 Register Tmp =
703 MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getVGPRForAGPRCopy();
704 assert(MBB.getParent()->getRegInfo().isReserved(Tmp) &&
705 "VGPR used for an intermediate copy should have been reserved.");
706
707 // Only loop through if there are any free registers left. We don't want to
708 // spill.
709 while (RegNo--) {
710 Register Tmp2 = RS.scavengeRegisterBackwards(RC: AMDGPU::VGPR_32RegClass, To: MI,
711 /* RestoreAfter */ false, SPAdj: 0,
712 /* AllowSpill */ false);
713 if (!Tmp2 || RI.getHWRegIndex(Reg: Tmp2) >= MaxVGPRs)
714 break;
715 Tmp = Tmp2;
716 RS.setRegUsed(Reg: Tmp);
717 }
718
719 // Insert copy to temporary VGPR.
720 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;
721 if (AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg)) {
722 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64;
723 } else {
724 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
725 }
726
727 MachineInstrBuilder UseBuilder = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: TmpCopyOp), DestReg: Tmp)
728 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc));
729 if (ImpUseSuperReg) {
730 UseBuilder.addReg(RegNo: ImpUseSuperReg,
731 flags: getKillRegState(B: KillSrc) | RegState::Implicit);
732 }
733
734 MachineInstrBuilder DefBuilder
735 = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
736 .addReg(RegNo: Tmp, flags: RegState::Kill);
737
738 if (ImpDefSuperReg)
739 DefBuilder.addReg(RegNo: ImpDefSuperReg, flags: RegState::Define | RegState::Implicit);
740}
741
742static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB,
743 MachineBasicBlock::iterator MI, const DebugLoc &DL,
744 MCRegister DestReg, MCRegister SrcReg, bool KillSrc,
745 const TargetRegisterClass *RC, bool Forward) {
746 const SIRegisterInfo &RI = TII.getRegisterInfo();
747 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, EltSize: 4);
748 MachineBasicBlock::iterator I = MI;
749 MachineInstr *FirstMI = nullptr, *LastMI = nullptr;
750
751 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) {
752 int16_t SubIdx = BaseIndices[Idx];
753 Register DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
754 Register SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
755 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
756 unsigned Opcode = AMDGPU::S_MOV_B32;
757
758 // Is SGPR aligned? If so try to combine with next.
759 bool AlignedDest = ((DestSubReg - AMDGPU::SGPR0) % 2) == 0;
760 bool AlignedSrc = ((SrcSubReg - AMDGPU::SGPR0) % 2) == 0;
761 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) {
762 // Can use SGPR64 copy
763 unsigned Channel = RI.getChannelFromSubReg(SubReg: SubIdx);
764 SubIdx = RI.getSubRegFromChannel(Channel, NumRegs: 2);
765 DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
766 SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
767 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
768 Opcode = AMDGPU::S_MOV_B64;
769 Idx++;
770 }
771
772 LastMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode), DestReg: DestSubReg)
773 .addReg(RegNo: SrcSubReg)
774 .addReg(RegNo: SrcReg, flags: RegState::Implicit);
775
776 if (!FirstMI)
777 FirstMI = LastMI;
778
779 if (!Forward)
780 I--;
781 }
782
783 assert(FirstMI && LastMI);
784 if (!Forward)
785 std::swap(a&: FirstMI, b&: LastMI);
786
787 FirstMI->addOperand(
788 Op: MachineOperand::CreateReg(Reg: DestReg, isDef: true /*IsDef*/, isImp: true /*IsImp*/));
789
790 if (KillSrc)
791 LastMI->addRegisterKilled(IncomingReg: SrcReg, RegInfo: &RI);
792}
793
794void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
795 MachineBasicBlock::iterator MI,
796 const DebugLoc &DL, MCRegister DestReg,
797 MCRegister SrcReg, bool KillSrc) const {
798 const TargetRegisterClass *RC = RI.getPhysRegBaseClass(Reg: DestReg);
799 unsigned Size = RI.getRegSizeInBits(RC: *RC);
800 const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(Reg: SrcReg);
801 unsigned SrcSize = RI.getRegSizeInBits(RC: *SrcRC);
802
803 // The rest of copyPhysReg assumes Src and Dst size are the same size.
804 // TODO-GFX11_16BIT If all true 16 bit instruction patterns are completed can
805 // we remove Fix16BitCopies and this code block?
806 if (Fix16BitCopies) {
807 if (((Size == 16) != (SrcSize == 16))) {
808 // Non-VGPR Src and Dst will later be expanded back to 32 bits.
809 assert(ST.hasTrue16BitInsts());
810 MCRegister &RegToFix = (Size == 32) ? DestReg : SrcReg;
811 MCRegister SubReg = RI.getSubReg(Reg: RegToFix, Idx: AMDGPU::lo16);
812 RegToFix = SubReg;
813
814 if (DestReg == SrcReg) {
815 // Identity copy. Insert empty bundle since ExpandPostRA expects an
816 // instruction here.
817 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::BUNDLE));
818 return;
819 }
820 RC = RI.getPhysRegBaseClass(Reg: DestReg);
821 Size = RI.getRegSizeInBits(RC: *RC);
822 SrcRC = RI.getPhysRegBaseClass(Reg: SrcReg);
823 SrcSize = RI.getRegSizeInBits(RC: *SrcRC);
824 }
825 }
826
827 if (RC == &AMDGPU::VGPR_32RegClass) {
828 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
829 AMDGPU::SReg_32RegClass.contains(SrcReg) ||
830 AMDGPU::AGPR_32RegClass.contains(SrcReg));
831 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg) ?
832 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;
833 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
834 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc));
835 return;
836 }
837
838 if (RC == &AMDGPU::SReg_32_XM0RegClass ||
839 RC == &AMDGPU::SReg_32RegClass) {
840 if (SrcReg == AMDGPU::SCC) {
841 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B32), DestReg)
842 .addImm(Val: 1)
843 .addImm(Val: 0);
844 return;
845 }
846
847 if (DestReg == AMDGPU::VCC_LO) {
848 if (AMDGPU::SReg_32RegClass.contains(Reg: SrcReg)) {
849 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::VCC_LO)
850 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc));
851 } else {
852 // FIXME: Hack until VReg_1 removed.
853 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
854 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CMP_NE_U32_e32))
855 .addImm(Val: 0)
856 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc));
857 }
858
859 return;
860 }
861
862 if (!AMDGPU::SReg_32RegClass.contains(Reg: SrcReg)) {
863 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
864 return;
865 }
866
867 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg)
868 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc));
869 return;
870 }
871
872 if (RC == &AMDGPU::SReg_64RegClass) {
873 if (SrcReg == AMDGPU::SCC) {
874 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B64), DestReg)
875 .addImm(Val: 1)
876 .addImm(Val: 0);
877 return;
878 }
879
880 if (DestReg == AMDGPU::VCC) {
881 if (AMDGPU::SReg_64RegClass.contains(Reg: SrcReg)) {
882 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B64), DestReg: AMDGPU::VCC)
883 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc));
884 } else {
885 // FIXME: Hack until VReg_1 removed.
886 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
887 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CMP_NE_U32_e32))
888 .addImm(Val: 0)
889 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc));
890 }
891
892 return;
893 }
894
895 if (!AMDGPU::SReg_64RegClass.contains(Reg: SrcReg)) {
896 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
897 return;
898 }
899
900 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B64), DestReg)
901 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc));
902 return;
903 }
904
905 if (DestReg == AMDGPU::SCC) {
906 // Copying 64-bit or 32-bit sources to SCC barely makes sense,
907 // but SelectionDAG emits such copies for i1 sources.
908 if (AMDGPU::SReg_64RegClass.contains(Reg: SrcReg)) {
909 // This copy can only be produced by patterns
910 // with explicit SCC, which are known to be enabled
911 // only for subtargets with S_CMP_LG_U64 present.
912 assert(ST.hasScalarCompareEq64());
913 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U64))
914 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc))
915 .addImm(Val: 0);
916 } else {
917 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
918 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U32))
919 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc))
920 .addImm(Val: 0);
921 }
922
923 return;
924 }
925
926 if (RC == &AMDGPU::AGPR_32RegClass) {
927 if (AMDGPU::VGPR_32RegClass.contains(Reg: SrcReg) ||
928 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(Reg: SrcReg))) {
929 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
930 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc));
931 return;
932 }
933
934 if (AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg) && ST.hasGFX90AInsts()) {
935 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_MOV_B32), DestReg)
936 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc));
937 return;
938 }
939
940 // FIXME: Pass should maintain scavenger to avoid scan through the block on
941 // every AGPR spill.
942 RegScavenger RS;
943 const bool Overlap = RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
944 indirectCopyToAGPR(TII: *this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS, RegsOverlap: Overlap);
945 return;
946 }
947
948 if (Size == 16) {
949 assert(AMDGPU::VGPR_16RegClass.contains(SrcReg) ||
950 AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
951 AMDGPU::AGPR_LO16RegClass.contains(SrcReg));
952
953 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(Reg: DestReg);
954 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(Reg: SrcReg);
955 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(Reg: DestReg);
956 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(Reg: SrcReg);
957 bool DstLow = !AMDGPU::isHi(Reg: DestReg, MRI: RI);
958 bool SrcLow = !AMDGPU::isHi(Reg: SrcReg, MRI: RI);
959 MCRegister NewDestReg = RI.get32BitRegister(Reg: DestReg);
960 MCRegister NewSrcReg = RI.get32BitRegister(Reg: SrcReg);
961
962 if (IsSGPRDst) {
963 if (!IsSGPRSrc) {
964 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
965 return;
966 }
967
968 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: NewDestReg)
969 .addReg(RegNo: NewSrcReg, flags: getKillRegState(B: KillSrc));
970 return;
971 }
972
973 if (IsAGPRDst || IsAGPRSrc) {
974 if (!DstLow || !SrcLow) {
975 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
976 Msg: "Cannot use hi16 subreg with an AGPR!");
977 }
978
979 copyPhysReg(MBB, MI, DL, DestReg: NewDestReg, SrcReg: NewSrcReg, KillSrc);
980 return;
981 }
982
983 if (ST.hasTrue16BitInsts()) {
984 if (IsSGPRSrc) {
985 assert(SrcLow);
986 SrcReg = NewSrcReg;
987 }
988 // Use the smaller instruction encoding if possible.
989 if (AMDGPU::VGPR_16_Lo128RegClass.contains(Reg: DestReg) &&
990 (IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.contains(Reg: SrcReg))) {
991 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B16_t16_e32), DestReg)
992 .addReg(RegNo: SrcReg);
993 } else {
994 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B16_t16_e64), DestReg)
995 .addImm(Val: 0) // src0_modifiers
996 .addReg(RegNo: SrcReg)
997 .addImm(Val: 0); // op_sel
998 }
999 return;
1000 }
1001
1002 if (IsSGPRSrc && !ST.hasSDWAScalar()) {
1003 if (!DstLow || !SrcLow) {
1004 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
1005 Msg: "Cannot use hi16 subreg on VI!");
1006 }
1007
1008 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: NewDestReg)
1009 .addReg(RegNo: NewSrcReg, flags: getKillRegState(B: KillSrc));
1010 return;
1011 }
1012
1013 auto MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_sdwa), DestReg: NewDestReg)
1014 .addImm(Val: 0) // src0_modifiers
1015 .addReg(RegNo: NewSrcReg)
1016 .addImm(Val: 0) // clamp
1017 .addImm(Val: DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1018 : AMDGPU::SDWA::SdwaSel::WORD_1)
1019 .addImm(Val: AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE)
1020 .addImm(Val: SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1021 : AMDGPU::SDWA::SdwaSel::WORD_1)
1022 .addReg(RegNo: NewDestReg, flags: RegState::Implicit | RegState::Undef);
1023 // First implicit operand is $exec.
1024 MIB->tieOperands(DefIdx: 0, UseIdx: MIB->getNumOperands() - 1);
1025 return;
1026 }
1027
1028 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(RC: SrcRC))) {
1029 if (ST.hasMovB64()) {
1030 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B64_e32), DestReg)
1031 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc));
1032 return;
1033 }
1034 if (ST.hasPkMovB32()) {
1035 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg)
1036 .addImm(Val: SISrcMods::OP_SEL_1)
1037 .addReg(RegNo: SrcReg)
1038 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
1039 .addReg(RegNo: SrcReg)
1040 .addImm(Val: 0) // op_sel_lo
1041 .addImm(Val: 0) // op_sel_hi
1042 .addImm(Val: 0) // neg_lo
1043 .addImm(Val: 0) // neg_hi
1044 .addImm(Val: 0) // clamp
1045 .addReg(RegNo: SrcReg, flags: getKillRegState(B: KillSrc) | RegState::Implicit);
1046 return;
1047 }
1048 }
1049
1050 const bool Forward = RI.getHWRegIndex(Reg: DestReg) <= RI.getHWRegIndex(Reg: SrcReg);
1051 if (RI.isSGPRClass(RC)) {
1052 if (!RI.isSGPRClass(RC: SrcRC)) {
1053 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1054 return;
1055 }
1056 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1057 expandSGPRCopy(TII: *this, MBB, MI, DL, DestReg, SrcReg, KillSrc: CanKillSuperReg, RC,
1058 Forward);
1059 return;
1060 }
1061
1062 unsigned EltSize = 4;
1063 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1064 if (RI.isAGPRClass(RC)) {
1065 if (ST.hasGFX90AInsts() && RI.isAGPRClass(RC: SrcRC))
1066 Opcode = AMDGPU::V_ACCVGPR_MOV_B32;
1067 else if (RI.hasVGPRs(RC: SrcRC) ||
1068 (ST.hasGFX90AInsts() && RI.isSGPRClass(RC: SrcRC)))
1069 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
1070 else
1071 Opcode = AMDGPU::INSTRUCTION_LIST_END;
1072 } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(RC: SrcRC)) {
1073 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;
1074 } else if ((Size % 64 == 0) && RI.hasVGPRs(RC) &&
1075 (RI.isProperlyAlignedRC(RC: *RC) &&
1076 (SrcRC == RC || RI.isSGPRClass(RC: SrcRC)))) {
1077 // TODO: In 96-bit case, could do a 64-bit mov and then a 32-bit mov.
1078 if (ST.hasMovB64()) {
1079 Opcode = AMDGPU::V_MOV_B64_e32;
1080 EltSize = 8;
1081 } else if (ST.hasPkMovB32()) {
1082 Opcode = AMDGPU::V_PK_MOV_B32;
1083 EltSize = 8;
1084 }
1085 }
1086
1087 // For the cases where we need an intermediate instruction/temporary register
1088 // (destination is an AGPR), we need a scavenger.
1089 //
1090 // FIXME: The pass should maintain this for us so we don't have to re-scan the
1091 // whole block for every handled copy.
1092 std::unique_ptr<RegScavenger> RS;
1093 if (Opcode == AMDGPU::INSTRUCTION_LIST_END)
1094 RS = std::make_unique<RegScavenger>();
1095
1096 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
1097
1098 // If there is an overlap, we can't kill the super-register on the last
1099 // instruction, since it will also kill the components made live by this def.
1100 const bool Overlap = RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1101 const bool CanKillSuperReg = KillSrc && !Overlap;
1102
1103 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
1104 unsigned SubIdx;
1105 if (Forward)
1106 SubIdx = SubIndices[Idx];
1107 else
1108 SubIdx = SubIndices[SubIndices.size() - Idx - 1];
1109 Register DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
1110 Register SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
1111 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
1112
1113 bool IsFirstSubreg = Idx == 0;
1114 bool UseKill = CanKillSuperReg && Idx == SubIndices.size() - 1;
1115
1116 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) {
1117 Register ImpDefSuper = IsFirstSubreg ? Register(DestReg) : Register();
1118 Register ImpUseSuper = SrcReg;
1119 indirectCopyToAGPR(TII: *this, MBB, MI, DL, DestReg: DestSubReg, SrcReg: SrcSubReg, KillSrc: UseKill,
1120 RS&: *RS, RegsOverlap: Overlap, ImpDefSuperReg: ImpDefSuper, ImpUseSuperReg: ImpUseSuper);
1121 } else if (Opcode == AMDGPU::V_PK_MOV_B32) {
1122 MachineInstrBuilder MIB =
1123 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg: DestSubReg)
1124 .addImm(Val: SISrcMods::OP_SEL_1)
1125 .addReg(RegNo: SrcSubReg)
1126 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
1127 .addReg(RegNo: SrcSubReg)
1128 .addImm(Val: 0) // op_sel_lo
1129 .addImm(Val: 0) // op_sel_hi
1130 .addImm(Val: 0) // neg_lo
1131 .addImm(Val: 0) // neg_hi
1132 .addImm(Val: 0) // clamp
1133 .addReg(RegNo: SrcReg, flags: getKillRegState(B: UseKill) | RegState::Implicit);
1134 if (IsFirstSubreg)
1135 MIB.addReg(RegNo: DestReg, flags: RegState::Define | RegState::Implicit);
1136 } else {
1137 MachineInstrBuilder Builder =
1138 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode), DestReg: DestSubReg).addReg(RegNo: SrcSubReg);
1139 if (IsFirstSubreg)
1140 Builder.addReg(RegNo: DestReg, flags: RegState::Define | RegState::Implicit);
1141
1142 Builder.addReg(RegNo: SrcReg, flags: getKillRegState(B: UseKill) | RegState::Implicit);
1143 }
1144 }
1145}
1146
1147int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
1148 int NewOpc;
1149
1150 // Try to map original to commuted opcode
1151 NewOpc = AMDGPU::getCommuteRev(Opcode);
1152 if (NewOpc != -1)
1153 // Check if the commuted (REV) opcode exists on the target.
1154 return pseudoToMCOpcode(Opcode: NewOpc) != -1 ? NewOpc : -1;
1155
1156 // Try to map commuted to original opcode
1157 NewOpc = AMDGPU::getCommuteOrig(Opcode);
1158 if (NewOpc != -1)
1159 // Check if the original (non-REV) opcode exists on the target.
1160 return pseudoToMCOpcode(Opcode: NewOpc) != -1 ? NewOpc : -1;
1161
1162 return Opcode;
1163}
1164
1165void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB,
1166 MachineBasicBlock::iterator MI,
1167 const DebugLoc &DL, Register DestReg,
1168 int64_t Value) const {
1169 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1170 const TargetRegisterClass *RegClass = MRI.getRegClass(Reg: DestReg);
1171 if (RegClass == &AMDGPU::SReg_32RegClass ||
1172 RegClass == &AMDGPU::SGPR_32RegClass ||
1173 RegClass == &AMDGPU::SReg_32_XM0RegClass ||
1174 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
1175 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg)
1176 .addImm(Val: Value);
1177 return;
1178 }
1179
1180 if (RegClass == &AMDGPU::SReg_64RegClass ||
1181 RegClass == &AMDGPU::SGPR_64RegClass ||
1182 RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
1183 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B64), DestReg)
1184 .addImm(Val: Value);
1185 return;
1186 }
1187
1188 if (RegClass == &AMDGPU::VGPR_32RegClass) {
1189 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg)
1190 .addImm(Val: Value);
1191 return;
1192 }
1193 if (RegClass->hasSuperClassEq(RC: &AMDGPU::VReg_64RegClass)) {
1194 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B64_PSEUDO), DestReg)
1195 .addImm(Val: Value);
1196 return;
1197 }
1198
1199 unsigned EltSize = 4;
1200 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1201 if (RI.isSGPRClass(RC: RegClass)) {
1202 if (RI.getRegSizeInBits(RC: *RegClass) > 32) {
1203 Opcode = AMDGPU::S_MOV_B64;
1204 EltSize = 8;
1205 } else {
1206 Opcode = AMDGPU::S_MOV_B32;
1207 EltSize = 4;
1208 }
1209 }
1210
1211 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC: RegClass, EltSize);
1212 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
1213 int64_t IdxValue = Idx == 0 ? Value : 0;
1214
1215 MachineInstrBuilder Builder = BuildMI(BB&: MBB, I: MI, MIMD: DL,
1216 MCID: get(Opcode), DestReg: RI.getSubReg(Reg: DestReg, Idx: SubIndices[Idx]));
1217 Builder.addImm(Val: IdxValue);
1218 }
1219}
1220
1221const TargetRegisterClass *
1222SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const {
1223 return &AMDGPU::VGPR_32RegClass;
1224}
1225
1226void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB,
1227 MachineBasicBlock::iterator I,
1228 const DebugLoc &DL, Register DstReg,
1229 ArrayRef<MachineOperand> Cond,
1230 Register TrueReg,
1231 Register FalseReg) const {
1232 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1233 const TargetRegisterClass *BoolXExecRC =
1234 RI.getRegClass(RCID: AMDGPU::SReg_1_XEXECRegClassID);
1235 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
1236 "Not a VGPR32 reg");
1237
1238 if (Cond.size() == 1) {
1239 Register SReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
1240 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: SReg)
1241 .add(MO: Cond[0]);
1242 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
1243 .addImm(Val: 0)
1244 .addReg(RegNo: FalseReg)
1245 .addImm(Val: 0)
1246 .addReg(RegNo: TrueReg)
1247 .addReg(RegNo: SReg);
1248 } else if (Cond.size() == 2) {
1249 assert(Cond[0].isImm() && "Cond[0] is not an immediate");
1250 switch (Cond[0].getImm()) {
1251 case SIInstrInfo::SCC_TRUE: {
1252 Register SReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
1253 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ST.isWave32() ? AMDGPU::S_CSELECT_B32
1254 : AMDGPU::S_CSELECT_B64), DestReg: SReg)
1255 .addImm(Val: 1)
1256 .addImm(Val: 0);
1257 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
1258 .addImm(Val: 0)
1259 .addReg(RegNo: FalseReg)
1260 .addImm(Val: 0)
1261 .addReg(RegNo: TrueReg)
1262 .addReg(RegNo: SReg);
1263 break;
1264 }
1265 case SIInstrInfo::SCC_FALSE: {
1266 Register SReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
1267 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ST.isWave32() ? AMDGPU::S_CSELECT_B32
1268 : AMDGPU::S_CSELECT_B64), DestReg: SReg)
1269 .addImm(Val: 0)
1270 .addImm(Val: 1);
1271 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
1272 .addImm(Val: 0)
1273 .addReg(RegNo: FalseReg)
1274 .addImm(Val: 0)
1275 .addReg(RegNo: TrueReg)
1276 .addReg(RegNo: SReg);
1277 break;
1278 }
1279 case SIInstrInfo::VCCNZ: {
1280 MachineOperand RegOp = Cond[1];
1281 RegOp.setImplicit(false);
1282 Register SReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
1283 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: SReg)
1284 .add(MO: RegOp);
1285 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
1286 .addImm(Val: 0)
1287 .addReg(RegNo: FalseReg)
1288 .addImm(Val: 0)
1289 .addReg(RegNo: TrueReg)
1290 .addReg(RegNo: SReg);
1291 break;
1292 }
1293 case SIInstrInfo::VCCZ: {
1294 MachineOperand RegOp = Cond[1];
1295 RegOp.setImplicit(false);
1296 Register SReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
1297 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: SReg)
1298 .add(MO: RegOp);
1299 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
1300 .addImm(Val: 0)
1301 .addReg(RegNo: TrueReg)
1302 .addImm(Val: 0)
1303 .addReg(RegNo: FalseReg)
1304 .addReg(RegNo: SReg);
1305 break;
1306 }
1307 case SIInstrInfo::EXECNZ: {
1308 Register SReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
1309 Register SReg2 = MRI.createVirtualRegister(RegClass: RI.getBoolRC());
1310 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1311 : AMDGPU::S_OR_SAVEEXEC_B64), DestReg: SReg2)
1312 .addImm(Val: 0);
1313 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ST.isWave32() ? AMDGPU::S_CSELECT_B32
1314 : AMDGPU::S_CSELECT_B64), DestReg: SReg)
1315 .addImm(Val: 1)
1316 .addImm(Val: 0);
1317 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
1318 .addImm(Val: 0)
1319 .addReg(RegNo: FalseReg)
1320 .addImm(Val: 0)
1321 .addReg(RegNo: TrueReg)
1322 .addReg(RegNo: SReg);
1323 break;
1324 }
1325 case SIInstrInfo::EXECZ: {
1326 Register SReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
1327 Register SReg2 = MRI.createVirtualRegister(RegClass: RI.getBoolRC());
1328 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1329 : AMDGPU::S_OR_SAVEEXEC_B64), DestReg: SReg2)
1330 .addImm(Val: 0);
1331 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ST.isWave32() ? AMDGPU::S_CSELECT_B32
1332 : AMDGPU::S_CSELECT_B64), DestReg: SReg)
1333 .addImm(Val: 0)
1334 .addImm(Val: 1);
1335 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
1336 .addImm(Val: 0)
1337 .addReg(RegNo: FalseReg)
1338 .addImm(Val: 0)
1339 .addReg(RegNo: TrueReg)
1340 .addReg(RegNo: SReg);
1341 llvm_unreachable("Unhandled branch predicate EXECZ");
1342 break;
1343 }
1344 default:
1345 llvm_unreachable("invalid branch predicate");
1346 }
1347 } else {
1348 llvm_unreachable("Can only handle Cond size 1 or 2");
1349 }
1350}
1351
1352Register SIInstrInfo::insertEQ(MachineBasicBlock *MBB,
1353 MachineBasicBlock::iterator I,
1354 const DebugLoc &DL,
1355 Register SrcReg, int Value) const {
1356 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1357 Register Reg = MRI.createVirtualRegister(RegClass: RI.getBoolRC());
1358 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CMP_EQ_I32_e64), DestReg: Reg)
1359 .addImm(Val: Value)
1360 .addReg(RegNo: SrcReg);
1361
1362 return Reg;
1363}
1364
1365Register SIInstrInfo::insertNE(MachineBasicBlock *MBB,
1366 MachineBasicBlock::iterator I,
1367 const DebugLoc &DL,
1368 Register SrcReg, int Value) const {
1369 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1370 Register Reg = MRI.createVirtualRegister(RegClass: RI.getBoolRC());
1371 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CMP_NE_I32_e64), DestReg: Reg)
1372 .addImm(Val: Value)
1373 .addReg(RegNo: SrcReg);
1374
1375 return Reg;
1376}
1377
1378unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
1379
1380 if (RI.isAGPRClass(RC: DstRC))
1381 return AMDGPU::COPY;
1382 if (RI.getRegSizeInBits(RC: *DstRC) == 16) {
1383 // Assume hi bits are unneeded. Only _e64 true16 instructions are legal
1384 // before RA.
1385 return RI.isSGPRClass(RC: DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64;
1386 }
1387 if (RI.getRegSizeInBits(RC: *DstRC) == 32)
1388 return RI.isSGPRClass(RC: DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1389 if (RI.getRegSizeInBits(RC: *DstRC) == 64 && RI.isSGPRClass(RC: DstRC))
1390 return AMDGPU::S_MOV_B64;
1391 if (RI.getRegSizeInBits(RC: *DstRC) == 64 && !RI.isSGPRClass(RC: DstRC))
1392 return AMDGPU::V_MOV_B64_PSEUDO;
1393 return AMDGPU::COPY;
1394}
1395
1396const MCInstrDesc &
1397SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize,
1398 bool IsIndirectSrc) const {
1399 if (IsIndirectSrc) {
1400 if (VecSize <= 32) // 4 bytes
1401 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);
1402 if (VecSize <= 64) // 8 bytes
1403 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);
1404 if (VecSize <= 96) // 12 bytes
1405 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);
1406 if (VecSize <= 128) // 16 bytes
1407 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);
1408 if (VecSize <= 160) // 20 bytes
1409 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);
1410 if (VecSize <= 256) // 32 bytes
1411 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);
1412 if (VecSize <= 288) // 36 bytes
1413 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9);
1414 if (VecSize <= 320) // 40 bytes
1415 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10);
1416 if (VecSize <= 352) // 44 bytes
1417 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11);
1418 if (VecSize <= 384) // 48 bytes
1419 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12);
1420 if (VecSize <= 512) // 64 bytes
1421 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);
1422 if (VecSize <= 1024) // 128 bytes
1423 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);
1424
1425 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos");
1426 }
1427
1428 if (VecSize <= 32) // 4 bytes
1429 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);
1430 if (VecSize <= 64) // 8 bytes
1431 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);
1432 if (VecSize <= 96) // 12 bytes
1433 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);
1434 if (VecSize <= 128) // 16 bytes
1435 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);
1436 if (VecSize <= 160) // 20 bytes
1437 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);
1438 if (VecSize <= 256) // 32 bytes
1439 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);
1440 if (VecSize <= 288) // 36 bytes
1441 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9);
1442 if (VecSize <= 320) // 40 bytes
1443 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10);
1444 if (VecSize <= 352) // 44 bytes
1445 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11);
1446 if (VecSize <= 384) // 48 bytes
1447 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12);
1448 if (VecSize <= 512) // 64 bytes
1449 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);
1450 if (VecSize <= 1024) // 128 bytes
1451 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);
1452
1453 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos");
1454}
1455
1456static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) {
1457 if (VecSize <= 32) // 4 bytes
1458 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1459 if (VecSize <= 64) // 8 bytes
1460 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1461 if (VecSize <= 96) // 12 bytes
1462 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1463 if (VecSize <= 128) // 16 bytes
1464 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1465 if (VecSize <= 160) // 20 bytes
1466 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1467 if (VecSize <= 256) // 32 bytes
1468 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1469 if (VecSize <= 288) // 36 bytes
1470 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1471 if (VecSize <= 320) // 40 bytes
1472 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1473 if (VecSize <= 352) // 44 bytes
1474 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1475 if (VecSize <= 384) // 48 bytes
1476 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1477 if (VecSize <= 512) // 64 bytes
1478 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1479 if (VecSize <= 1024) // 128 bytes
1480 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1481
1482 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1483}
1484
1485static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) {
1486 if (VecSize <= 32) // 4 bytes
1487 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1488 if (VecSize <= 64) // 8 bytes
1489 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1490 if (VecSize <= 96) // 12 bytes
1491 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1492 if (VecSize <= 128) // 16 bytes
1493 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1494 if (VecSize <= 160) // 20 bytes
1495 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1496 if (VecSize <= 256) // 32 bytes
1497 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1498 if (VecSize <= 288) // 36 bytes
1499 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1500 if (VecSize <= 320) // 40 bytes
1501 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1502 if (VecSize <= 352) // 44 bytes
1503 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1504 if (VecSize <= 384) // 48 bytes
1505 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1506 if (VecSize <= 512) // 64 bytes
1507 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1508 if (VecSize <= 1024) // 128 bytes
1509 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1510
1511 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1512}
1513
1514static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) {
1515 if (VecSize <= 64) // 8 bytes
1516 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;
1517 if (VecSize <= 128) // 16 bytes
1518 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;
1519 if (VecSize <= 256) // 32 bytes
1520 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;
1521 if (VecSize <= 512) // 64 bytes
1522 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;
1523 if (VecSize <= 1024) // 128 bytes
1524 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;
1525
1526 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1527}
1528
1529const MCInstrDesc &
1530SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize,
1531 bool IsSGPR) const {
1532 if (IsSGPR) {
1533 switch (EltSize) {
1534 case 32:
1535 return get(Opcode: getIndirectSGPRWriteMovRelPseudo32(VecSize));
1536 case 64:
1537 return get(Opcode: getIndirectSGPRWriteMovRelPseudo64(VecSize));
1538 default:
1539 llvm_unreachable("invalid reg indexing elt size");
1540 }
1541 }
1542
1543 assert(EltSize == 32 && "invalid reg indexing elt size");
1544 return get(Opcode: getIndirectVGPRWriteMovRelPseudoOpc(VecSize));
1545}
1546
1547static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
1548 switch (Size) {
1549 case 4:
1550 return AMDGPU::SI_SPILL_S32_SAVE;
1551 case 8:
1552 return AMDGPU::SI_SPILL_S64_SAVE;
1553 case 12:
1554 return AMDGPU::SI_SPILL_S96_SAVE;
1555 case 16:
1556 return AMDGPU::SI_SPILL_S128_SAVE;
1557 case 20:
1558 return AMDGPU::SI_SPILL_S160_SAVE;
1559 case 24:
1560 return AMDGPU::SI_SPILL_S192_SAVE;
1561 case 28:
1562 return AMDGPU::SI_SPILL_S224_SAVE;
1563 case 32:
1564 return AMDGPU::SI_SPILL_S256_SAVE;
1565 case 36:
1566 return AMDGPU::SI_SPILL_S288_SAVE;
1567 case 40:
1568 return AMDGPU::SI_SPILL_S320_SAVE;
1569 case 44:
1570 return AMDGPU::SI_SPILL_S352_SAVE;
1571 case 48:
1572 return AMDGPU::SI_SPILL_S384_SAVE;
1573 case 64:
1574 return AMDGPU::SI_SPILL_S512_SAVE;
1575 case 128:
1576 return AMDGPU::SI_SPILL_S1024_SAVE;
1577 default:
1578 llvm_unreachable("unknown register size");
1579 }
1580}
1581
1582static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
1583 switch (Size) {
1584 case 4:
1585 return AMDGPU::SI_SPILL_V32_SAVE;
1586 case 8:
1587 return AMDGPU::SI_SPILL_V64_SAVE;
1588 case 12:
1589 return AMDGPU::SI_SPILL_V96_SAVE;
1590 case 16:
1591 return AMDGPU::SI_SPILL_V128_SAVE;
1592 case 20:
1593 return AMDGPU::SI_SPILL_V160_SAVE;
1594 case 24:
1595 return AMDGPU::SI_SPILL_V192_SAVE;
1596 case 28:
1597 return AMDGPU::SI_SPILL_V224_SAVE;
1598 case 32:
1599 return AMDGPU::SI_SPILL_V256_SAVE;
1600 case 36:
1601 return AMDGPU::SI_SPILL_V288_SAVE;
1602 case 40:
1603 return AMDGPU::SI_SPILL_V320_SAVE;
1604 case 44:
1605 return AMDGPU::SI_SPILL_V352_SAVE;
1606 case 48:
1607 return AMDGPU::SI_SPILL_V384_SAVE;
1608 case 64:
1609 return AMDGPU::SI_SPILL_V512_SAVE;
1610 case 128:
1611 return AMDGPU::SI_SPILL_V1024_SAVE;
1612 default:
1613 llvm_unreachable("unknown register size");
1614 }
1615}
1616
1617static unsigned getAGPRSpillSaveOpcode(unsigned Size) {
1618 switch (Size) {
1619 case 4:
1620 return AMDGPU::SI_SPILL_A32_SAVE;
1621 case 8:
1622 return AMDGPU::SI_SPILL_A64_SAVE;
1623 case 12:
1624 return AMDGPU::SI_SPILL_A96_SAVE;
1625 case 16:
1626 return AMDGPU::SI_SPILL_A128_SAVE;
1627 case 20:
1628 return AMDGPU::SI_SPILL_A160_SAVE;
1629 case 24:
1630 return AMDGPU::SI_SPILL_A192_SAVE;
1631 case 28:
1632 return AMDGPU::SI_SPILL_A224_SAVE;
1633 case 32:
1634 return AMDGPU::SI_SPILL_A256_SAVE;
1635 case 36:
1636 return AMDGPU::SI_SPILL_A288_SAVE;
1637 case 40:
1638 return AMDGPU::SI_SPILL_A320_SAVE;
1639 case 44:
1640 return AMDGPU::SI_SPILL_A352_SAVE;
1641 case 48:
1642 return AMDGPU::SI_SPILL_A384_SAVE;
1643 case 64:
1644 return AMDGPU::SI_SPILL_A512_SAVE;
1645 case 128:
1646 return AMDGPU::SI_SPILL_A1024_SAVE;
1647 default:
1648 llvm_unreachable("unknown register size");
1649 }
1650}
1651
1652static unsigned getAVSpillSaveOpcode(unsigned Size) {
1653 switch (Size) {
1654 case 4:
1655 return AMDGPU::SI_SPILL_AV32_SAVE;
1656 case 8:
1657 return AMDGPU::SI_SPILL_AV64_SAVE;
1658 case 12:
1659 return AMDGPU::SI_SPILL_AV96_SAVE;
1660 case 16:
1661 return AMDGPU::SI_SPILL_AV128_SAVE;
1662 case 20:
1663 return AMDGPU::SI_SPILL_AV160_SAVE;
1664 case 24:
1665 return AMDGPU::SI_SPILL_AV192_SAVE;
1666 case 28:
1667 return AMDGPU::SI_SPILL_AV224_SAVE;
1668 case 32:
1669 return AMDGPU::SI_SPILL_AV256_SAVE;
1670 case 36:
1671 return AMDGPU::SI_SPILL_AV288_SAVE;
1672 case 40:
1673 return AMDGPU::SI_SPILL_AV320_SAVE;
1674 case 44:
1675 return AMDGPU::SI_SPILL_AV352_SAVE;
1676 case 48:
1677 return AMDGPU::SI_SPILL_AV384_SAVE;
1678 case 64:
1679 return AMDGPU::SI_SPILL_AV512_SAVE;
1680 case 128:
1681 return AMDGPU::SI_SPILL_AV1024_SAVE;
1682 default:
1683 llvm_unreachable("unknown register size");
1684 }
1685}
1686
1687static unsigned getWWMRegSpillSaveOpcode(unsigned Size,
1688 bool IsVectorSuperClass) {
1689 // Currently, there is only 32-bit WWM register spills needed.
1690 if (Size != 4)
1691 llvm_unreachable("unknown wwm register spill size");
1692
1693 if (IsVectorSuperClass)
1694 return AMDGPU::SI_SPILL_WWM_AV32_SAVE;
1695
1696 return AMDGPU::SI_SPILL_WWM_V32_SAVE;
1697}
1698
1699static unsigned getVectorRegSpillSaveOpcode(Register Reg,
1700 const TargetRegisterClass *RC,
1701 unsigned Size,
1702 const SIRegisterInfo &TRI,
1703 const SIMachineFunctionInfo &MFI) {
1704 bool IsVectorSuperClass = TRI.isVectorSuperClass(RC);
1705
1706 // Choose the right opcode if spilling a WWM register.
1707 if (MFI.checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
1708 return getWWMRegSpillSaveOpcode(Size, IsVectorSuperClass);
1709
1710 if (IsVectorSuperClass)
1711 return getAVSpillSaveOpcode(Size);
1712
1713 return TRI.isAGPRClass(RC) ? getAGPRSpillSaveOpcode(Size)
1714 : getVGPRSpillSaveOpcode(Size);
1715}
1716
1717void SIInstrInfo::storeRegToStackSlot(
1718 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
1719 bool isKill, int FrameIndex, const TargetRegisterClass *RC,
1720 const TargetRegisterInfo *TRI, Register VReg) const {
1721 MachineFunction *MF = MBB.getParent();
1722 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1723 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1724 const DebugLoc &DL = MBB.findDebugLoc(MBBI: MI);
1725
1726 MachinePointerInfo PtrInfo
1727 = MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
1728 MachineMemOperand *MMO = MF->getMachineMemOperand(
1729 PtrInfo, F: MachineMemOperand::MOStore, Size: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
1730 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
1731 unsigned SpillSize = TRI->getSpillSize(RC: *RC);
1732
1733 MachineRegisterInfo &MRI = MF->getRegInfo();
1734 if (RI.isSGPRClass(RC)) {
1735 MFI->setHasSpilledSGPRs();
1736 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled");
1737 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI &&
1738 SrcReg != AMDGPU::EXEC && "exec should not be spilled");
1739
1740 // We are only allowed to create one new instruction when spilling
1741 // registers, so we need to use pseudo instruction for spilling SGPRs.
1742 const MCInstrDesc &OpDesc = get(Opcode: getSGPRSpillSaveOpcode(Size: SpillSize));
1743
1744 // The SGPR spill/restore instructions only work on number sgprs, so we need
1745 // to make sure we are using the correct register class.
1746 if (SrcReg.isVirtual() && SpillSize == 4) {
1747 MRI.constrainRegClass(Reg: SrcReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
1748 }
1749
1750 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: OpDesc)
1751 .addReg(RegNo: SrcReg, flags: getKillRegState(B: isKill)) // data
1752 .addFrameIndex(Idx: FrameIndex) // addr
1753 .addMemOperand(MMO)
1754 .addReg(RegNo: MFI->getStackPtrOffsetReg(), flags: RegState::Implicit);
1755
1756 if (RI.spillSGPRToVGPR())
1757 FrameInfo.setStackID(ObjectIdx: FrameIndex, ID: TargetStackID::SGPRSpill);
1758 return;
1759 }
1760
1761 unsigned Opcode = getVectorRegSpillSaveOpcode(Reg: VReg ? VReg : SrcReg, RC,
1762 Size: SpillSize, TRI: RI, MFI: *MFI);
1763 MFI->setHasSpilledVGPRs();
1764
1765 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode))
1766 .addReg(RegNo: SrcReg, flags: getKillRegState(B: isKill)) // data
1767 .addFrameIndex(Idx: FrameIndex) // addr
1768 .addReg(RegNo: MFI->getStackPtrOffsetReg()) // scratch_offset
1769 .addImm(Val: 0) // offset
1770 .addMemOperand(MMO);
1771}
1772
1773static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1774 switch (Size) {
1775 case 4:
1776 return AMDGPU::SI_SPILL_S32_RESTORE;
1777 case 8:
1778 return AMDGPU::SI_SPILL_S64_RESTORE;
1779 case 12:
1780 return AMDGPU::SI_SPILL_S96_RESTORE;
1781 case 16:
1782 return AMDGPU::SI_SPILL_S128_RESTORE;
1783 case 20:
1784 return AMDGPU::SI_SPILL_S160_RESTORE;
1785 case 24:
1786 return AMDGPU::SI_SPILL_S192_RESTORE;
1787 case 28:
1788 return AMDGPU::SI_SPILL_S224_RESTORE;
1789 case 32:
1790 return AMDGPU::SI_SPILL_S256_RESTORE;
1791 case 36:
1792 return AMDGPU::SI_SPILL_S288_RESTORE;
1793 case 40:
1794 return AMDGPU::SI_SPILL_S320_RESTORE;
1795 case 44:
1796 return AMDGPU::SI_SPILL_S352_RESTORE;
1797 case 48:
1798 return AMDGPU::SI_SPILL_S384_RESTORE;
1799 case 64:
1800 return AMDGPU::SI_SPILL_S512_RESTORE;
1801 case 128:
1802 return AMDGPU::SI_SPILL_S1024_RESTORE;
1803 default:
1804 llvm_unreachable("unknown register size");
1805 }
1806}
1807
1808static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1809 switch (Size) {
1810 case 4:
1811 return AMDGPU::SI_SPILL_V32_RESTORE;
1812 case 8:
1813 return AMDGPU::SI_SPILL_V64_RESTORE;
1814 case 12:
1815 return AMDGPU::SI_SPILL_V96_RESTORE;
1816 case 16:
1817 return AMDGPU::SI_SPILL_V128_RESTORE;
1818 case 20:
1819 return AMDGPU::SI_SPILL_V160_RESTORE;
1820 case 24:
1821 return AMDGPU::SI_SPILL_V192_RESTORE;
1822 case 28:
1823 return AMDGPU::SI_SPILL_V224_RESTORE;
1824 case 32:
1825 return AMDGPU::SI_SPILL_V256_RESTORE;
1826 case 36:
1827 return AMDGPU::SI_SPILL_V288_RESTORE;
1828 case 40:
1829 return AMDGPU::SI_SPILL_V320_RESTORE;
1830 case 44:
1831 return AMDGPU::SI_SPILL_V352_RESTORE;
1832 case 48:
1833 return AMDGPU::SI_SPILL_V384_RESTORE;
1834 case 64:
1835 return AMDGPU::SI_SPILL_V512_RESTORE;
1836 case 128:
1837 return AMDGPU::SI_SPILL_V1024_RESTORE;
1838 default:
1839 llvm_unreachable("unknown register size");
1840 }
1841}
1842
1843static unsigned getAGPRSpillRestoreOpcode(unsigned Size) {
1844 switch (Size) {
1845 case 4:
1846 return AMDGPU::SI_SPILL_A32_RESTORE;
1847 case 8:
1848 return AMDGPU::SI_SPILL_A64_RESTORE;
1849 case 12:
1850 return AMDGPU::SI_SPILL_A96_RESTORE;
1851 case 16:
1852 return AMDGPU::SI_SPILL_A128_RESTORE;
1853 case 20:
1854 return AMDGPU::SI_SPILL_A160_RESTORE;
1855 case 24:
1856 return AMDGPU::SI_SPILL_A192_RESTORE;
1857 case 28:
1858 return AMDGPU::SI_SPILL_A224_RESTORE;
1859 case 32:
1860 return AMDGPU::SI_SPILL_A256_RESTORE;
1861 case 36:
1862 return AMDGPU::SI_SPILL_A288_RESTORE;
1863 case 40:
1864 return AMDGPU::SI_SPILL_A320_RESTORE;
1865 case 44:
1866 return AMDGPU::SI_SPILL_A352_RESTORE;
1867 case 48:
1868 return AMDGPU::SI_SPILL_A384_RESTORE;
1869 case 64:
1870 return AMDGPU::SI_SPILL_A512_RESTORE;
1871 case 128:
1872 return AMDGPU::SI_SPILL_A1024_RESTORE;
1873 default:
1874 llvm_unreachable("unknown register size");
1875 }
1876}
1877
1878static unsigned getAVSpillRestoreOpcode(unsigned Size) {
1879 switch (Size) {
1880 case 4:
1881 return AMDGPU::SI_SPILL_AV32_RESTORE;
1882 case 8:
1883 return AMDGPU::SI_SPILL_AV64_RESTORE;
1884 case 12:
1885 return AMDGPU::SI_SPILL_AV96_RESTORE;
1886 case 16:
1887 return AMDGPU::SI_SPILL_AV128_RESTORE;
1888 case 20:
1889 return AMDGPU::SI_SPILL_AV160_RESTORE;
1890 case 24:
1891 return AMDGPU::SI_SPILL_AV192_RESTORE;
1892 case 28:
1893 return AMDGPU::SI_SPILL_AV224_RESTORE;
1894 case 32:
1895 return AMDGPU::SI_SPILL_AV256_RESTORE;
1896 case 36:
1897 return AMDGPU::SI_SPILL_AV288_RESTORE;
1898 case 40:
1899 return AMDGPU::SI_SPILL_AV320_RESTORE;
1900 case 44:
1901 return AMDGPU::SI_SPILL_AV352_RESTORE;
1902 case 48:
1903 return AMDGPU::SI_SPILL_AV384_RESTORE;
1904 case 64:
1905 return AMDGPU::SI_SPILL_AV512_RESTORE;
1906 case 128:
1907 return AMDGPU::SI_SPILL_AV1024_RESTORE;
1908 default:
1909 llvm_unreachable("unknown register size");
1910 }
1911}
1912
1913static unsigned getWWMRegSpillRestoreOpcode(unsigned Size,
1914 bool IsVectorSuperClass) {
1915 // Currently, there is only 32-bit WWM register spills needed.
1916 if (Size != 4)
1917 llvm_unreachable("unknown wwm register spill size");
1918
1919 if (IsVectorSuperClass)
1920 return AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
1921
1922 return AMDGPU::SI_SPILL_WWM_V32_RESTORE;
1923}
1924
1925static unsigned
1926getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC,
1927 unsigned Size, const SIRegisterInfo &TRI,
1928 const SIMachineFunctionInfo &MFI) {
1929 bool IsVectorSuperClass = TRI.isVectorSuperClass(RC);
1930
1931 // Choose the right opcode if restoring a WWM register.
1932 if (MFI.checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
1933 return getWWMRegSpillRestoreOpcode(Size, IsVectorSuperClass);
1934
1935 if (IsVectorSuperClass)
1936 return getAVSpillRestoreOpcode(Size);
1937
1938 return TRI.isAGPRClass(RC) ? getAGPRSpillRestoreOpcode(Size)
1939 : getVGPRSpillRestoreOpcode(Size);
1940}
1941
1942void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1943 MachineBasicBlock::iterator MI,
1944 Register DestReg, int FrameIndex,
1945 const TargetRegisterClass *RC,
1946 const TargetRegisterInfo *TRI,
1947 Register VReg) const {
1948 MachineFunction *MF = MBB.getParent();
1949 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1950 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1951 const DebugLoc &DL = MBB.findDebugLoc(MBBI: MI);
1952 unsigned SpillSize = TRI->getSpillSize(RC: *RC);
1953
1954 MachinePointerInfo PtrInfo
1955 = MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
1956
1957 MachineMemOperand *MMO = MF->getMachineMemOperand(
1958 PtrInfo, F: MachineMemOperand::MOLoad, Size: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
1959 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
1960
1961 if (RI.isSGPRClass(RC)) {
1962 MFI->setHasSpilledSGPRs();
1963 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into");
1964 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI &&
1965 DestReg != AMDGPU::EXEC && "exec should not be spilled");
1966
1967 // FIXME: Maybe this should not include a memoperand because it will be
1968 // lowered to non-memory instructions.
1969 const MCInstrDesc &OpDesc = get(Opcode: getSGPRSpillRestoreOpcode(Size: SpillSize));
1970 if (DestReg.isVirtual() && SpillSize == 4) {
1971 MachineRegisterInfo &MRI = MF->getRegInfo();
1972 MRI.constrainRegClass(Reg: DestReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
1973 }
1974
1975 if (RI.spillSGPRToVGPR())
1976 FrameInfo.setStackID(ObjectIdx: FrameIndex, ID: TargetStackID::SGPRSpill);
1977 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: OpDesc, DestReg)
1978 .addFrameIndex(Idx: FrameIndex) // addr
1979 .addMemOperand(MMO)
1980 .addReg(RegNo: MFI->getStackPtrOffsetReg(), flags: RegState::Implicit);
1981
1982 return;
1983 }
1984
1985 unsigned Opcode = getVectorRegSpillRestoreOpcode(Reg: VReg ? VReg : DestReg, RC,
1986 Size: SpillSize, TRI: RI, MFI: *MFI);
1987 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode), DestReg)
1988 .addFrameIndex(Idx: FrameIndex) // vaddr
1989 .addReg(RegNo: MFI->getStackPtrOffsetReg()) // scratch_offset
1990 .addImm(Val: 0) // offset
1991 .addMemOperand(MMO);
1992}
1993
1994void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
1995 MachineBasicBlock::iterator MI) const {
1996 insertNoops(MBB, MI, Quantity: 1);
1997}
1998
1999void SIInstrInfo::insertNoops(MachineBasicBlock &MBB,
2000 MachineBasicBlock::iterator MI,
2001 unsigned Quantity) const {
2002 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2003 while (Quantity > 0) {
2004 unsigned Arg = std::min(a: Quantity, b: 8u);
2005 Quantity -= Arg;
2006 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOP)).addImm(Val: Arg - 1);
2007 }
2008}
2009
2010void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const {
2011 auto MF = MBB.getParent();
2012 SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
2013
2014 assert(Info->isEntryFunction());
2015
2016 if (MBB.succ_empty()) {
2017 bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
2018 if (HasNoTerminator) {
2019 if (Info->returnsVoid()) {
2020 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DebugLoc(), MCID: get(Opcode: AMDGPU::S_ENDPGM)).addImm(Val: 0);
2021 } else {
2022 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DebugLoc(), MCID: get(Opcode: AMDGPU::SI_RETURN_TO_EPILOG));
2023 }
2024 }
2025 }
2026}
2027
2028MachineBasicBlock *SIInstrInfo::insertSimulatedTrap(MachineRegisterInfo &MRI,
2029 MachineBasicBlock &MBB,
2030 MachineInstr &MI,
2031 const DebugLoc &DL) const {
2032 MachineFunction *MF = MBB.getParent();
2033 constexpr unsigned DoorbellIDMask = 0x3ff;
2034 constexpr unsigned ECQueueWaveAbort = 0x400;
2035
2036 MachineBasicBlock *TrapBB = &MBB;
2037 MachineBasicBlock *ContBB = &MBB;
2038 MachineBasicBlock *HaltLoopBB = MF->CreateMachineBasicBlock();
2039
2040 if (!MBB.succ_empty() || std::next(x: MI.getIterator()) != MBB.end()) {
2041 ContBB = MBB.splitAt(SplitInst&: MI, /*UpdateLiveIns=*/false);
2042 TrapBB = MF->CreateMachineBasicBlock();
2043 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CBRANCH_EXECNZ)).addMBB(MBB: TrapBB);
2044 MF->push_back(MBB: TrapBB);
2045 MBB.addSuccessor(Succ: TrapBB);
2046 }
2047
2048 // Start with a `s_trap 2`, if we're in PRIV=1 and we need the workaround this
2049 // will be a nop.
2050 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_TRAP))
2051 .addImm(Val: static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));
2052 Register DoorbellReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2053 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SENDMSG_RTN_B32),
2054 DestReg: DoorbellReg)
2055 .addImm(Val: AMDGPU::SendMsg::ID_RTN_GET_DOORBELL);
2056 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::TTMP2)
2057 .addUse(RegNo: AMDGPU::M0);
2058 Register DoorbellRegMasked =
2059 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2060 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_AND_B32), DestReg: DoorbellRegMasked)
2061 .addUse(RegNo: DoorbellReg)
2062 .addImm(Val: DoorbellIDMask);
2063 Register SetWaveAbortBit =
2064 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2065 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_OR_B32), DestReg: SetWaveAbortBit)
2066 .addUse(RegNo: DoorbellRegMasked)
2067 .addImm(Val: ECQueueWaveAbort);
2068 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
2069 .addUse(RegNo: SetWaveAbortBit);
2070 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SENDMSG))
2071 .addImm(Val: AMDGPU::SendMsg::ID_INTERRUPT);
2072 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
2073 .addUse(RegNo: AMDGPU::TTMP2);
2074 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH)).addMBB(MBB: HaltLoopBB);
2075 TrapBB->addSuccessor(Succ: HaltLoopBB);
2076
2077 BuildMI(BB&: *HaltLoopBB, I: HaltLoopBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETHALT)).addImm(Val: 5);
2078 BuildMI(BB&: *HaltLoopBB, I: HaltLoopBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
2079 .addMBB(MBB: HaltLoopBB);
2080 MF->push_back(MBB: HaltLoopBB);
2081 HaltLoopBB->addSuccessor(Succ: HaltLoopBB);
2082
2083 return ContBB;
2084}
2085
2086unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) {
2087 switch (MI.getOpcode()) {
2088 default:
2089 if (MI.isMetaInstruction())
2090 return 0;
2091 return 1; // FIXME: Do wait states equal cycles?
2092
2093 case AMDGPU::S_NOP:
2094 return MI.getOperand(i: 0).getImm() + 1;
2095 // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The
2096 // hazard, even if one exist, won't really be visible. Should we handle it?
2097 }
2098}
2099
2100bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2101 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2102 MachineBasicBlock &MBB = *MI.getParent();
2103 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2104 switch (MI.getOpcode()) {
2105 default: return TargetInstrInfo::expandPostRAPseudo(MI);
2106 case AMDGPU::S_MOV_B64_term:
2107 // This is only a terminator to get the correct spill code placement during
2108 // register allocation.
2109 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2110 break;
2111
2112 case AMDGPU::S_MOV_B32_term:
2113 // This is only a terminator to get the correct spill code placement during
2114 // register allocation.
2115 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B32));
2116 break;
2117
2118 case AMDGPU::S_XOR_B64_term:
2119 // This is only a terminator to get the correct spill code placement during
2120 // register allocation.
2121 MI.setDesc(get(Opcode: AMDGPU::S_XOR_B64));
2122 break;
2123
2124 case AMDGPU::S_XOR_B32_term:
2125 // This is only a terminator to get the correct spill code placement during
2126 // register allocation.
2127 MI.setDesc(get(Opcode: AMDGPU::S_XOR_B32));
2128 break;
2129 case AMDGPU::S_OR_B64_term:
2130 // This is only a terminator to get the correct spill code placement during
2131 // register allocation.
2132 MI.setDesc(get(Opcode: AMDGPU::S_OR_B64));
2133 break;
2134 case AMDGPU::S_OR_B32_term:
2135 // This is only a terminator to get the correct spill code placement during
2136 // register allocation.
2137 MI.setDesc(get(Opcode: AMDGPU::S_OR_B32));
2138 break;
2139
2140 case AMDGPU::S_ANDN2_B64_term:
2141 // This is only a terminator to get the correct spill code placement during
2142 // register allocation.
2143 MI.setDesc(get(Opcode: AMDGPU::S_ANDN2_B64));
2144 break;
2145
2146 case AMDGPU::S_ANDN2_B32_term:
2147 // This is only a terminator to get the correct spill code placement during
2148 // register allocation.
2149 MI.setDesc(get(Opcode: AMDGPU::S_ANDN2_B32));
2150 break;
2151
2152 case AMDGPU::S_AND_B64_term:
2153 // This is only a terminator to get the correct spill code placement during
2154 // register allocation.
2155 MI.setDesc(get(Opcode: AMDGPU::S_AND_B64));
2156 break;
2157
2158 case AMDGPU::S_AND_B32_term:
2159 // This is only a terminator to get the correct spill code placement during
2160 // register allocation.
2161 MI.setDesc(get(Opcode: AMDGPU::S_AND_B32));
2162 break;
2163
2164 case AMDGPU::S_AND_SAVEEXEC_B64_term:
2165 // This is only a terminator to get the correct spill code placement during
2166 // register allocation.
2167 MI.setDesc(get(Opcode: AMDGPU::S_AND_SAVEEXEC_B64));
2168 break;
2169
2170 case AMDGPU::S_AND_SAVEEXEC_B32_term:
2171 // This is only a terminator to get the correct spill code placement during
2172 // register allocation.
2173 MI.setDesc(get(Opcode: AMDGPU::S_AND_SAVEEXEC_B32));
2174 break;
2175
2176 case AMDGPU::SI_SPILL_S32_TO_VGPR:
2177 MI.setDesc(get(Opcode: AMDGPU::V_WRITELANE_B32));
2178 break;
2179
2180 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
2181 MI.setDesc(get(Opcode: AMDGPU::V_READLANE_B32));
2182 break;
2183
2184 case AMDGPU::V_MOV_B64_PSEUDO: {
2185 Register Dst = MI.getOperand(i: 0).getReg();
2186 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2187 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2188
2189 const MachineOperand &SrcOp = MI.getOperand(i: 1);
2190 // FIXME: Will this work for 64-bit floating point immediates?
2191 assert(!SrcOp.isFPImm());
2192 if (ST.hasMovB64()) {
2193 MI.setDesc(get(Opcode: AMDGPU::V_MOV_B64_e32));
2194 if (SrcOp.isReg() || isInlineConstant(MI, OpIdx: 1) ||
2195 isUInt<32>(x: SrcOp.getImm()))
2196 break;
2197 }
2198 if (SrcOp.isImm()) {
2199 APInt Imm(64, SrcOp.getImm());
2200 APInt Lo(32, Imm.getLoBits(numBits: 32).getZExtValue());
2201 APInt Hi(32, Imm.getHiBits(numBits: 32).getZExtValue());
2202 if (ST.hasPkMovB32() && Lo == Hi && isInlineConstant(Imm: Lo)) {
2203 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg: Dst)
2204 .addImm(Val: SISrcMods::OP_SEL_1)
2205 .addImm(Val: Lo.getSExtValue())
2206 .addImm(Val: SISrcMods::OP_SEL_1)
2207 .addImm(Val: Lo.getSExtValue())
2208 .addImm(Val: 0) // op_sel_lo
2209 .addImm(Val: 0) // op_sel_hi
2210 .addImm(Val: 0) // neg_lo
2211 .addImm(Val: 0) // neg_hi
2212 .addImm(Val: 0); // clamp
2213 } else {
2214 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2215 .addImm(Val: Lo.getSExtValue())
2216 .addReg(RegNo: Dst, flags: RegState::Implicit | RegState::Define);
2217 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2218 .addImm(Val: Hi.getSExtValue())
2219 .addReg(RegNo: Dst, flags: RegState::Implicit | RegState::Define);
2220 }
2221 } else {
2222 assert(SrcOp.isReg());
2223 if (ST.hasPkMovB32() &&
2224 !RI.isAGPR(MRI: MBB.getParent()->getRegInfo(), Reg: SrcOp.getReg())) {
2225 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg: Dst)
2226 .addImm(Val: SISrcMods::OP_SEL_1) // src0_mod
2227 .addReg(RegNo: SrcOp.getReg())
2228 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) // src1_mod
2229 .addReg(RegNo: SrcOp.getReg())
2230 .addImm(Val: 0) // op_sel_lo
2231 .addImm(Val: 0) // op_sel_hi
2232 .addImm(Val: 0) // neg_lo
2233 .addImm(Val: 0) // neg_hi
2234 .addImm(Val: 0); // clamp
2235 } else {
2236 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2237 .addReg(RegNo: RI.getSubReg(Reg: SrcOp.getReg(), Idx: AMDGPU::sub0))
2238 .addReg(RegNo: Dst, flags: RegState::Implicit | RegState::Define);
2239 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2240 .addReg(RegNo: RI.getSubReg(Reg: SrcOp.getReg(), Idx: AMDGPU::sub1))
2241 .addReg(RegNo: Dst, flags: RegState::Implicit | RegState::Define);
2242 }
2243 }
2244 MI.eraseFromParent();
2245 break;
2246 }
2247 case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
2248 expandMovDPP64(MI);
2249 break;
2250 }
2251 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2252 const MachineOperand &SrcOp = MI.getOperand(i: 1);
2253 assert(!SrcOp.isFPImm());
2254 APInt Imm(64, SrcOp.getImm());
2255 if (Imm.isIntN(N: 32) || isInlineConstant(Imm)) {
2256 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2257 break;
2258 }
2259
2260 Register Dst = MI.getOperand(i: 0).getReg();
2261 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2262 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2263
2264 APInt Lo(32, Imm.getLoBits(numBits: 32).getZExtValue());
2265 APInt Hi(32, Imm.getHiBits(numBits: 32).getZExtValue());
2266 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstLo)
2267 .addImm(Val: Lo.getSExtValue())
2268 .addReg(RegNo: Dst, flags: RegState::Implicit | RegState::Define);
2269 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi)
2270 .addImm(Val: Hi.getSExtValue())
2271 .addReg(RegNo: Dst, flags: RegState::Implicit | RegState::Define);
2272 MI.eraseFromParent();
2273 break;
2274 }
2275 case AMDGPU::V_SET_INACTIVE_B32: {
2276 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
2277 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
2278 // FIXME: We may possibly optimize the COPY once we find ways to make LLVM
2279 // optimizations (mainly Register Coalescer) aware of WWM register liveness.
2280 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: MI.getOperand(i: 0).getReg())
2281 .add(MO: MI.getOperand(i: 1));
2282 auto FirstNot = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: NotOpc), DestReg: Exec).addReg(RegNo: Exec);
2283 FirstNot->addRegisterDead(Reg: AMDGPU::SCC, RegInfo: TRI); // SCC is overwritten
2284 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: MI.getOperand(i: 0).getReg())
2285 .add(MO: MI.getOperand(i: 2));
2286 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: NotOpc), DestReg: Exec)
2287 .addReg(RegNo: Exec);
2288 MI.eraseFromParent();
2289 break;
2290 }
2291 case AMDGPU::V_SET_INACTIVE_B64: {
2292 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
2293 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
2294 MachineInstr *Copy = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B64_PSEUDO),
2295 DestReg: MI.getOperand(i: 0).getReg())
2296 .add(MO: MI.getOperand(i: 1));
2297 expandPostRAPseudo(MI&: *Copy);
2298 auto FirstNot = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: NotOpc), DestReg: Exec).addReg(RegNo: Exec);
2299 FirstNot->addRegisterDead(Reg: AMDGPU::SCC, RegInfo: TRI); // SCC is overwritten
2300 Copy = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B64_PSEUDO),
2301 DestReg: MI.getOperand(i: 0).getReg())
2302 .add(MO: MI.getOperand(i: 2));
2303 expandPostRAPseudo(MI&: *Copy);
2304 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: NotOpc), DestReg: Exec)
2305 .addReg(RegNo: Exec);
2306 MI.eraseFromParent();
2307 break;
2308 }
2309 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2310 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2311 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2312 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2313 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2314 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2315 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2316 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2317 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2318 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2319 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2320 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2321 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2322 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2323 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2324 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2325 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2326 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2327 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2328 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2329 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2330 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2331 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2332 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2333 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:
2334 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:
2335 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:
2336 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:
2337 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {
2338 const TargetRegisterClass *EltRC = getOpRegClass(MI, OpNo: 2);
2339
2340 unsigned Opc;
2341 if (RI.hasVGPRs(RC: EltRC)) {
2342 Opc = AMDGPU::V_MOVRELD_B32_e32;
2343 } else {
2344 Opc = RI.getRegSizeInBits(RC: *EltRC) == 64 ? AMDGPU::S_MOVRELD_B64
2345 : AMDGPU::S_MOVRELD_B32;
2346 }
2347
2348 const MCInstrDesc &OpDesc = get(Opcode: Opc);
2349 Register VecReg = MI.getOperand(i: 0).getReg();
2350 bool IsUndef = MI.getOperand(i: 1).isUndef();
2351 unsigned SubReg = MI.getOperand(i: 3).getImm();
2352 assert(VecReg == MI.getOperand(1).getReg());
2353
2354 MachineInstrBuilder MIB =
2355 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: OpDesc)
2356 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), flags: RegState::Undef)
2357 .add(MO: MI.getOperand(i: 2))
2358 .addReg(RegNo: VecReg, flags: RegState::ImplicitDefine)
2359 .addReg(RegNo: VecReg, flags: RegState::Implicit | (IsUndef ? RegState::Undef : 0));
2360
2361 const int ImpDefIdx =
2362 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2363 const int ImpUseIdx = ImpDefIdx + 1;
2364 MIB->tieOperands(DefIdx: ImpDefIdx, UseIdx: ImpUseIdx);
2365 MI.eraseFromParent();
2366 break;
2367 }
2368 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:
2369 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:
2370 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:
2371 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:
2372 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:
2373 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:
2374 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9:
2375 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10:
2376 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11:
2377 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12:
2378 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:
2379 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {
2380 assert(ST.useVGPRIndexMode());
2381 Register VecReg = MI.getOperand(i: 0).getReg();
2382 bool IsUndef = MI.getOperand(i: 1).isUndef();
2383 Register Idx = MI.getOperand(i: 3).getReg();
2384 Register SubReg = MI.getOperand(i: 4).getImm();
2385
2386 MachineInstr *SetOn = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_ON))
2387 .addReg(RegNo: Idx)
2388 .addImm(Val: AMDGPU::VGPRIndexMode::DST_ENABLE);
2389 SetOn->getOperand(i: 3).setIsUndef();
2390
2391 const MCInstrDesc &OpDesc = get(Opcode: AMDGPU::V_MOV_B32_indirect_write);
2392 MachineInstrBuilder MIB =
2393 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: OpDesc)
2394 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), flags: RegState::Undef)
2395 .add(MO: MI.getOperand(i: 2))
2396 .addReg(RegNo: VecReg, flags: RegState::ImplicitDefine)
2397 .addReg(RegNo: VecReg,
2398 flags: RegState::Implicit | (IsUndef ? RegState::Undef : 0));
2399
2400 const int ImpDefIdx =
2401 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2402 const int ImpUseIdx = ImpDefIdx + 1;
2403 MIB->tieOperands(DefIdx: ImpDefIdx, UseIdx: ImpUseIdx);
2404
2405 MachineInstr *SetOff = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_OFF));
2406
2407 finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2408
2409 MI.eraseFromParent();
2410 break;
2411 }
2412 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:
2413 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:
2414 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:
2415 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:
2416 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:
2417 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:
2418 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9:
2419 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10:
2420 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11:
2421 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12:
2422 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:
2423 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {
2424 assert(ST.useVGPRIndexMode());
2425 Register Dst = MI.getOperand(i: 0).getReg();
2426 Register VecReg = MI.getOperand(i: 1).getReg();
2427 bool IsUndef = MI.getOperand(i: 1).isUndef();
2428 Register Idx = MI.getOperand(i: 2).getReg();
2429 Register SubReg = MI.getOperand(i: 3).getImm();
2430
2431 MachineInstr *SetOn = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_ON))
2432 .addReg(RegNo: Idx)
2433 .addImm(Val: AMDGPU::VGPRIndexMode::SRC0_ENABLE);
2434 SetOn->getOperand(i: 3).setIsUndef();
2435
2436 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_indirect_read))
2437 .addDef(RegNo: Dst)
2438 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), flags: RegState::Undef)
2439 .addReg(RegNo: VecReg, flags: RegState::Implicit | (IsUndef ? RegState::Undef : 0));
2440
2441 MachineInstr *SetOff = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_OFF));
2442
2443 finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2444
2445 MI.eraseFromParent();
2446 break;
2447 }
2448 case AMDGPU::SI_PC_ADD_REL_OFFSET: {
2449 MachineFunction &MF = *MBB.getParent();
2450 Register Reg = MI.getOperand(i: 0).getReg();
2451 Register RegLo = RI.getSubReg(Reg, Idx: AMDGPU::sub0);
2452 Register RegHi = RI.getSubReg(Reg, Idx: AMDGPU::sub1);
2453 MachineOperand OpLo = MI.getOperand(i: 1);
2454 MachineOperand OpHi = MI.getOperand(i: 2);
2455
2456 // Create a bundle so these instructions won't be re-ordered by the
2457 // post-RA scheduler.
2458 MIBundleBuilder Bundler(MBB, MI);
2459 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: Reg));
2460
2461 // What we want here is an offset from the value returned by s_getpc (which
2462 // is the address of the s_add_u32 instruction) to the global variable, but
2463 // since the encoding of $symbol starts 4 bytes after the start of the
2464 // s_add_u32 instruction, we end up with an offset that is 4 bytes too
2465 // small. This requires us to add 4 to the global variable offset in order
2466 // to compute the correct address. Similarly for the s_addc_u32 instruction,
2467 // the encoding of $symbol starts 12 bytes after the start of the s_add_u32
2468 // instruction.
2469
2470 int64_t Adjust = 0;
2471 if (ST.hasGetPCZeroExtension()) {
2472 // Fix up hardware that does not sign-extend the 48-bit PC value by
2473 // inserting: s_sext_i32_i16 reghi, reghi
2474 Bundler.append(
2475 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SEXT_I32_I16), DestReg: RegHi).addReg(RegNo: RegHi));
2476 Adjust += 4;
2477 }
2478
2479 if (OpLo.isGlobal())
2480 OpLo.setOffset(OpLo.getOffset() + Adjust + 4);
2481 Bundler.append(
2482 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U32), DestReg: RegLo).addReg(RegNo: RegLo).add(MO: OpLo));
2483
2484 if (OpHi.isGlobal())
2485 OpHi.setOffset(OpHi.getOffset() + Adjust + 12);
2486 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADDC_U32), DestReg: RegHi)
2487 .addReg(RegNo: RegHi)
2488 .add(MO: OpHi));
2489
2490 finalizeBundle(MBB, FirstMI: Bundler.begin());
2491
2492 MI.eraseFromParent();
2493 break;
2494 }
2495 case AMDGPU::ENTER_STRICT_WWM: {
2496 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2497 // Whole Wave Mode is entered.
2498 MI.setDesc(get(Opcode: ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
2499 : AMDGPU::S_OR_SAVEEXEC_B64));
2500 break;
2501 }
2502 case AMDGPU::ENTER_STRICT_WQM: {
2503 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2504 // STRICT_WQM is entered.
2505 const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
2506 const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64;
2507 const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
2508 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: MovOp), DestReg: MI.getOperand(i: 0).getReg()).addReg(RegNo: Exec);
2509 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: WQMOp), DestReg: Exec).addReg(RegNo: Exec);
2510
2511 MI.eraseFromParent();
2512 break;
2513 }
2514 case AMDGPU::EXIT_STRICT_WWM:
2515 case AMDGPU::EXIT_STRICT_WQM: {
2516 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2517 // WWM/STICT_WQM is exited.
2518 MI.setDesc(get(Opcode: ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64));
2519 break;
2520 }
2521 case AMDGPU::SI_RETURN: {
2522 const MachineFunction *MF = MBB.getParent();
2523 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2524 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2525 // Hiding the return address use with SI_RETURN may lead to extra kills in
2526 // the function and missing live-ins. We are fine in practice because callee
2527 // saved register handling ensures the register value is restored before
2528 // RET, but we need the undef flag here to appease the MachineVerifier
2529 // liveness checks.
2530 MachineInstrBuilder MIB =
2531 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETPC_B64_return))
2532 .addReg(RegNo: TRI->getReturnAddressReg(MF: *MF), flags: RegState::Undef);
2533
2534 MIB.copyImplicitOps(OtherMI: MI);
2535 MI.eraseFromParent();
2536 break;
2537 }
2538
2539 case AMDGPU::S_MUL_U64_U32_PSEUDO:
2540 case AMDGPU::S_MUL_I64_I32_PSEUDO:
2541 MI.setDesc(get(Opcode: AMDGPU::S_MUL_U64));
2542 break;
2543
2544 case AMDGPU::S_GETPC_B64_pseudo:
2545 MI.setDesc(get(Opcode: AMDGPU::S_GETPC_B64));
2546 if (ST.hasGetPCZeroExtension()) {
2547 Register Dst = MI.getOperand(i: 0).getReg();
2548 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2549 // Fix up hardware that does not sign-extend the 48-bit PC value by
2550 // inserting: s_sext_i32_i16 dsthi, dsthi
2551 BuildMI(BB&: MBB, I: std::next(x: MI.getIterator()), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SEXT_I32_I16),
2552 DestReg: DstHi)
2553 .addReg(RegNo: DstHi);
2554 }
2555 break;
2556 }
2557 return true;
2558}
2559
2560void SIInstrInfo::reMaterialize(MachineBasicBlock &MBB,
2561 MachineBasicBlock::iterator I, Register DestReg,
2562 unsigned SubIdx, const MachineInstr &Orig,
2563 const TargetRegisterInfo &RI) const {
2564
2565 // Try shrinking the instruction to remat only the part needed for current
2566 // context.
2567 // TODO: Handle more cases.
2568 unsigned Opcode = Orig.getOpcode();
2569 switch (Opcode) {
2570 case AMDGPU::S_LOAD_DWORDX16_IMM:
2571 case AMDGPU::S_LOAD_DWORDX8_IMM: {
2572 if (SubIdx != 0)
2573 break;
2574
2575 if (I == MBB.end())
2576 break;
2577
2578 if (I->isBundled())
2579 break;
2580
2581 // Look for a single use of the register that is also a subreg.
2582 Register RegToFind = Orig.getOperand(i: 0).getReg();
2583 MachineOperand *UseMO = nullptr;
2584 for (auto &CandMO : I->operands()) {
2585 if (!CandMO.isReg() || CandMO.getReg() != RegToFind || CandMO.isDef())
2586 continue;
2587 if (UseMO) {
2588 UseMO = nullptr;
2589 break;
2590 }
2591 UseMO = &CandMO;
2592 }
2593 if (!UseMO || UseMO->getSubReg() == AMDGPU::NoSubRegister)
2594 break;
2595
2596 unsigned Offset = RI.getSubRegIdxOffset(Idx: UseMO->getSubReg());
2597 unsigned SubregSize = RI.getSubRegIdxSize(Idx: UseMO->getSubReg());
2598
2599 MachineFunction *MF = MBB.getParent();
2600 MachineRegisterInfo &MRI = MF->getRegInfo();
2601 assert(MRI.use_nodbg_empty(DestReg) && "DestReg should have no users yet.");
2602
2603 unsigned NewOpcode = -1;
2604 if (SubregSize == 256)
2605 NewOpcode = AMDGPU::S_LOAD_DWORDX8_IMM;
2606 else if (SubregSize == 128)
2607 NewOpcode = AMDGPU::S_LOAD_DWORDX4_IMM;
2608 else
2609 break;
2610
2611 const MCInstrDesc &TID = get(Opcode: NewOpcode);
2612 const TargetRegisterClass *NewRC =
2613 RI.getAllocatableClass(RC: getRegClass(TID, OpNum: 0, TRI: &RI, MF: *MF));
2614 MRI.setRegClass(Reg: DestReg, RC: NewRC);
2615
2616 UseMO->setReg(DestReg);
2617 UseMO->setSubReg(AMDGPU::NoSubRegister);
2618
2619 // Use a smaller load with the desired size, possibly with updated offset.
2620 MachineInstr *MI = MF->CloneMachineInstr(Orig: &Orig);
2621 MI->setDesc(TID);
2622 MI->getOperand(i: 0).setReg(DestReg);
2623 MI->getOperand(i: 0).setSubReg(AMDGPU::NoSubRegister);
2624 if (Offset) {
2625 MachineOperand *OffsetMO = getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::offset);
2626 int64_t FinalOffset = OffsetMO->getImm() + Offset / 8;
2627 OffsetMO->setImm(FinalOffset);
2628 }
2629 SmallVector<MachineMemOperand *> NewMMOs;
2630 for (const MachineMemOperand *MemOp : Orig.memoperands())
2631 NewMMOs.push_back(Elt: MF->getMachineMemOperand(MMO: MemOp, PtrInfo: MemOp->getPointerInfo(),
2632 Size: SubregSize / 8));
2633 MI->setMemRefs(MF&: *MF, MemRefs: NewMMOs);
2634
2635 MBB.insert(I, MI);
2636 return;
2637 }
2638
2639 default:
2640 break;
2641 }
2642
2643 TargetInstrInfo::reMaterialize(MBB, MI: I, DestReg, SubIdx, Orig, TRI: RI);
2644}
2645
2646std::pair<MachineInstr*, MachineInstr*>
2647SIInstrInfo::expandMovDPP64(MachineInstr &MI) const {
2648 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
2649
2650 if (ST.hasMovB64() &&
2651 AMDGPU::isLegalDPALU_DPPControl(
2652 DC: getNamedOperand(MI, OperandName: AMDGPU::OpName::dpp_ctrl)->getImm())) {
2653 MI.setDesc(get(Opcode: AMDGPU::V_MOV_B64_dpp));
2654 return std::pair(&MI, nullptr);
2655 }
2656
2657 MachineBasicBlock &MBB = *MI.getParent();
2658 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2659 MachineFunction *MF = MBB.getParent();
2660 MachineRegisterInfo &MRI = MF->getRegInfo();
2661 Register Dst = MI.getOperand(i: 0).getReg();
2662 unsigned Part = 0;
2663 MachineInstr *Split[2];
2664
2665 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
2666 auto MovDPP = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_dpp));
2667 if (Dst.isPhysical()) {
2668 MovDPP.addDef(RegNo: RI.getSubReg(Reg: Dst, Idx: Sub));
2669 } else {
2670 assert(MRI.isSSA());
2671 auto Tmp = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2672 MovDPP.addDef(RegNo: Tmp);
2673 }
2674
2675 for (unsigned I = 1; I <= 2; ++I) { // old and src operands.
2676 const MachineOperand &SrcOp = MI.getOperand(i: I);
2677 assert(!SrcOp.isFPImm());
2678 if (SrcOp.isImm()) {
2679 APInt Imm(64, SrcOp.getImm());
2680 Imm.ashrInPlace(ShiftAmt: Part * 32);
2681 MovDPP.addImm(Val: Imm.getLoBits(numBits: 32).getZExtValue());
2682 } else {
2683 assert(SrcOp.isReg());
2684 Register Src = SrcOp.getReg();
2685 if (Src.isPhysical())
2686 MovDPP.addReg(RegNo: RI.getSubReg(Reg: Src, Idx: Sub));
2687 else
2688 MovDPP.addReg(RegNo: Src, flags: SrcOp.isUndef() ? RegState::Undef : 0, SubReg: Sub);
2689 }
2690 }
2691
2692 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.explicit_operands(), N: 3))
2693 MovDPP.addImm(Val: MO.getImm());
2694
2695 Split[Part] = MovDPP;
2696 ++Part;
2697 }
2698
2699 if (Dst.isVirtual())
2700 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
2701 .addReg(RegNo: Split[0]->getOperand(i: 0).getReg())
2702 .addImm(Val: AMDGPU::sub0)
2703 .addReg(RegNo: Split[1]->getOperand(i: 0).getReg())
2704 .addImm(Val: AMDGPU::sub1);
2705
2706 MI.eraseFromParent();
2707 return std::pair(Split[0], Split[1]);
2708}
2709
2710std::optional<DestSourcePair>
2711SIInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
2712 if (MI.getOpcode() == AMDGPU::WWM_COPY)
2713 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2714
2715 return std::nullopt;
2716}
2717
2718bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI,
2719 MachineOperand &Src0,
2720 unsigned Src0OpName,
2721 MachineOperand &Src1,
2722 unsigned Src1OpName) const {
2723 MachineOperand *Src0Mods = getNamedOperand(MI, OperandName: Src0OpName);
2724 if (!Src0Mods)
2725 return false;
2726
2727 MachineOperand *Src1Mods = getNamedOperand(MI, OperandName: Src1OpName);
2728 assert(Src1Mods &&
2729 "All commutable instructions have both src0 and src1 modifiers");
2730
2731 int Src0ModsVal = Src0Mods->getImm();
2732 int Src1ModsVal = Src1Mods->getImm();
2733
2734 Src1Mods->setImm(Src0ModsVal);
2735 Src0Mods->setImm(Src1ModsVal);
2736 return true;
2737}
2738
2739static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI,
2740 MachineOperand &RegOp,
2741 MachineOperand &NonRegOp) {
2742 Register Reg = RegOp.getReg();
2743 unsigned SubReg = RegOp.getSubReg();
2744 bool IsKill = RegOp.isKill();
2745 bool IsDead = RegOp.isDead();
2746 bool IsUndef = RegOp.isUndef();
2747 bool IsDebug = RegOp.isDebug();
2748
2749 if (NonRegOp.isImm())
2750 RegOp.ChangeToImmediate(ImmVal: NonRegOp.getImm());
2751 else if (NonRegOp.isFI())
2752 RegOp.ChangeToFrameIndex(Idx: NonRegOp.getIndex());
2753 else if (NonRegOp.isGlobal()) {
2754 RegOp.ChangeToGA(GV: NonRegOp.getGlobal(), Offset: NonRegOp.getOffset(),
2755 TargetFlags: NonRegOp.getTargetFlags());
2756 } else
2757 return nullptr;
2758
2759 // Make sure we don't reinterpret a subreg index in the target flags.
2760 RegOp.setTargetFlags(NonRegOp.getTargetFlags());
2761
2762 NonRegOp.ChangeToRegister(Reg, isDef: false, isImp: false, isKill: IsKill, isDead: IsDead, isUndef: IsUndef, isDebug: IsDebug);
2763 NonRegOp.setSubReg(SubReg);
2764
2765 return &MI;
2766}
2767
2768MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
2769 unsigned Src0Idx,
2770 unsigned Src1Idx) const {
2771 assert(!NewMI && "this should never be used");
2772
2773 unsigned Opc = MI.getOpcode();
2774 int CommutedOpcode = commuteOpcode(Opcode: Opc);
2775 if (CommutedOpcode == -1)
2776 return nullptr;
2777
2778 if (Src0Idx > Src1Idx)
2779 std::swap(a&: Src0Idx, b&: Src1Idx);
2780
2781 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
2782 static_cast<int>(Src0Idx) &&
2783 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
2784 static_cast<int>(Src1Idx) &&
2785 "inconsistency with findCommutedOpIndices");
2786
2787 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
2788 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
2789
2790 MachineInstr *CommutedMI = nullptr;
2791 if (Src0.isReg() && Src1.isReg()) {
2792 if (isOperandLegal(MI, OpIdx: Src1Idx, MO: &Src0)) {
2793 // Be sure to copy the source modifiers to the right place.
2794 CommutedMI
2795 = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1: Src0Idx, OpIdx2: Src1Idx);
2796 }
2797
2798 } else if (Src0.isReg() && !Src1.isReg()) {
2799 // src0 should always be able to support any operand type, so no need to
2800 // check operand legality.
2801 CommutedMI = swapRegAndNonRegOperand(MI, RegOp&: Src0, NonRegOp&: Src1);
2802 } else if (!Src0.isReg() && Src1.isReg()) {
2803 if (isOperandLegal(MI, OpIdx: Src1Idx, MO: &Src0))
2804 CommutedMI = swapRegAndNonRegOperand(MI, RegOp&: Src1, NonRegOp&: Src0);
2805 } else {
2806 // FIXME: Found two non registers to commute. This does happen.
2807 return nullptr;
2808 }
2809
2810 if (CommutedMI) {
2811 swapSourceModifiers(MI, Src0, Src0OpName: AMDGPU::OpName::src0_modifiers,
2812 Src1, Src1OpName: AMDGPU::OpName::src1_modifiers);
2813
2814 CommutedMI->setDesc(get(Opcode: CommutedOpcode));
2815 }
2816
2817 return CommutedMI;
2818}
2819
2820// This needs to be implemented because the source modifiers may be inserted
2821// between the true commutable operands, and the base
2822// TargetInstrInfo::commuteInstruction uses it.
2823bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
2824 unsigned &SrcOpIdx0,
2825 unsigned &SrcOpIdx1) const {
2826 return findCommutedOpIndices(Desc: MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
2827}
2828
2829bool SIInstrInfo::findCommutedOpIndices(const MCInstrDesc &Desc,
2830 unsigned &SrcOpIdx0,
2831 unsigned &SrcOpIdx1) const {
2832 if (!Desc.isCommutable())
2833 return false;
2834
2835 unsigned Opc = Desc.getOpcode();
2836 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::src0);
2837 if (Src0Idx == -1)
2838 return false;
2839
2840 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::src1);
2841 if (Src1Idx == -1)
2842 return false;
2843
2844 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx0, ResultIdx2&: SrcOpIdx1, CommutableOpIdx1: Src0Idx, CommutableOpIdx2: Src1Idx);
2845}
2846
2847bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
2848 int64_t BrOffset) const {
2849 // BranchRelaxation should never have to check s_setpc_b64 because its dest
2850 // block is unanalyzable.
2851 assert(BranchOp != AMDGPU::S_SETPC_B64);
2852
2853 // Convert to dwords.
2854 BrOffset /= 4;
2855
2856 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
2857 // from the next instruction.
2858 BrOffset -= 1;
2859
2860 return isIntN(N: BranchOffsetBits, x: BrOffset);
2861}
2862
2863MachineBasicBlock *
2864SIInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
2865 return MI.getOperand(i: 0).getMBB();
2866}
2867
2868bool SIInstrInfo::hasDivergentBranch(const MachineBasicBlock *MBB) const {
2869 for (const MachineInstr &MI : MBB->terminators()) {
2870 if (MI.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO ||
2871 MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE ||
2872 MI.getOpcode() == AMDGPU::SI_LOOP)
2873 return true;
2874 }
2875 return false;
2876}
2877
2878void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
2879 MachineBasicBlock &DestBB,
2880 MachineBasicBlock &RestoreBB,
2881 const DebugLoc &DL, int64_t BrOffset,
2882 RegScavenger *RS) const {
2883 assert(RS && "RegScavenger required for long branching");
2884 assert(MBB.empty() &&
2885 "new block should be inserted for expanding unconditional branch");
2886 assert(MBB.pred_size() == 1);
2887 assert(RestoreBB.empty() &&
2888 "restore block should be inserted for restoring clobbered registers");
2889
2890 MachineFunction *MF = MBB.getParent();
2891 MachineRegisterInfo &MRI = MF->getRegInfo();
2892 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
2893
2894 // FIXME: Virtual register workaround for RegScavenger not working with empty
2895 // blocks.
2896 Register PCReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
2897
2898 auto I = MBB.end();
2899
2900 // We need to compute the offset relative to the instruction immediately after
2901 // s_getpc_b64. Insert pc arithmetic code before last terminator.
2902 MachineInstr *GetPC = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: PCReg);
2903
2904 auto &MCCtx = MF->getContext();
2905 MCSymbol *PostGetPCLabel =
2906 MCCtx.createTempSymbol(Name: "post_getpc", /*AlwaysAddSuffix=*/true);
2907 GetPC->setPostInstrSymbol(MF&: *MF, Symbol: PostGetPCLabel);
2908
2909 MCSymbol *OffsetLo =
2910 MCCtx.createTempSymbol(Name: "offset_lo", /*AlwaysAddSuffix=*/true);
2911 MCSymbol *OffsetHi =
2912 MCCtx.createTempSymbol(Name: "offset_hi", /*AlwaysAddSuffix=*/true);
2913 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U32))
2914 .addReg(RegNo: PCReg, flags: RegState::Define, SubReg: AMDGPU::sub0)
2915 .addReg(RegNo: PCReg, flags: 0, SubReg: AMDGPU::sub0)
2916 .addSym(Sym: OffsetLo, TargetFlags: MO_FAR_BRANCH_OFFSET);
2917 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADDC_U32))
2918 .addReg(RegNo: PCReg, flags: RegState::Define, SubReg: AMDGPU::sub1)
2919 .addReg(RegNo: PCReg, flags: 0, SubReg: AMDGPU::sub1)
2920 .addSym(Sym: OffsetHi, TargetFlags: MO_FAR_BRANCH_OFFSET);
2921
2922 // Insert the indirect branch after the other terminator.
2923 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETPC_B64))
2924 .addReg(RegNo: PCReg);
2925
2926 // If a spill is needed for the pc register pair, we need to insert a spill
2927 // restore block right before the destination block, and insert a short branch
2928 // into the old destination block's fallthrough predecessor.
2929 // e.g.:
2930 //
2931 // s_cbranch_scc0 skip_long_branch:
2932 //
2933 // long_branch_bb:
2934 // spill s[8:9]
2935 // s_getpc_b64 s[8:9]
2936 // s_add_u32 s8, s8, restore_bb
2937 // s_addc_u32 s9, s9, 0
2938 // s_setpc_b64 s[8:9]
2939 //
2940 // skip_long_branch:
2941 // foo;
2942 //
2943 // .....
2944 //
2945 // dest_bb_fallthrough_predecessor:
2946 // bar;
2947 // s_branch dest_bb
2948 //
2949 // restore_bb:
2950 // restore s[8:9]
2951 // fallthrough dest_bb
2952 ///
2953 // dest_bb:
2954 // buzz;
2955
2956 Register LongBranchReservedReg = MFI->getLongBranchReservedReg();
2957 Register Scav;
2958
2959 // If we've previously reserved a register for long branches
2960 // avoid running the scavenger and just use those registers
2961 if (LongBranchReservedReg) {
2962 RS->enterBasicBlock(MBB);
2963 Scav = LongBranchReservedReg;
2964 } else {
2965 RS->enterBasicBlockEnd(MBB);
2966 Scav = RS->scavengeRegisterBackwards(
2967 RC: AMDGPU::SReg_64RegClass, To: MachineBasicBlock::iterator(GetPC),
2968 /* RestoreAfter */ false, SPAdj: 0, /* AllowSpill */ false);
2969 }
2970 if (Scav) {
2971 RS->setRegUsed(Reg: Scav);
2972 MRI.replaceRegWith(FromReg: PCReg, ToReg: Scav);
2973 MRI.clearVirtRegs();
2974 } else {
2975 // As SGPR needs VGPR to be spilled, we reuse the slot of temporary VGPR for
2976 // SGPR spill.
2977 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2978 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2979 TRI->spillEmergencySGPR(MI: GetPC, RestoreMBB&: RestoreBB, SGPR: AMDGPU::SGPR0_SGPR1, RS);
2980 MRI.replaceRegWith(FromReg: PCReg, ToReg: AMDGPU::SGPR0_SGPR1);
2981 MRI.clearVirtRegs();
2982 }
2983
2984 MCSymbol *DestLabel = Scav ? DestBB.getSymbol() : RestoreBB.getSymbol();
2985 // Now, the distance could be defined.
2986 auto *Offset = MCBinaryExpr::createSub(
2987 LHS: MCSymbolRefExpr::create(Symbol: DestLabel, Ctx&: MCCtx),
2988 RHS: MCSymbolRefExpr::create(Symbol: PostGetPCLabel, Ctx&: MCCtx), Ctx&: MCCtx);
2989 // Add offset assignments.
2990 auto *Mask = MCConstantExpr::create(Value: 0xFFFFFFFFULL, Ctx&: MCCtx);
2991 OffsetLo->setVariableValue(MCBinaryExpr::createAnd(LHS: Offset, RHS: Mask, Ctx&: MCCtx));
2992 auto *ShAmt = MCConstantExpr::create(Value: 32, Ctx&: MCCtx);
2993 OffsetHi->setVariableValue(MCBinaryExpr::createAShr(LHS: Offset, RHS: ShAmt, Ctx&: MCCtx));
2994}
2995
2996unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
2997 switch (Cond) {
2998 case SIInstrInfo::SCC_TRUE:
2999 return AMDGPU::S_CBRANCH_SCC1;
3000 case SIInstrInfo::SCC_FALSE:
3001 return AMDGPU::S_CBRANCH_SCC0;
3002 case SIInstrInfo::VCCNZ:
3003 return AMDGPU::S_CBRANCH_VCCNZ;
3004 case SIInstrInfo::VCCZ:
3005 return AMDGPU::S_CBRANCH_VCCZ;
3006 case SIInstrInfo::EXECNZ:
3007 return AMDGPU::S_CBRANCH_EXECNZ;
3008 case SIInstrInfo::EXECZ:
3009 return AMDGPU::S_CBRANCH_EXECZ;
3010 default:
3011 llvm_unreachable("invalid branch predicate");
3012 }
3013}
3014
3015SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
3016 switch (Opcode) {
3017 case AMDGPU::S_CBRANCH_SCC0:
3018 return SCC_FALSE;
3019 case AMDGPU::S_CBRANCH_SCC1:
3020 return SCC_TRUE;
3021 case AMDGPU::S_CBRANCH_VCCNZ:
3022 return VCCNZ;
3023 case AMDGPU::S_CBRANCH_VCCZ:
3024 return VCCZ;
3025 case AMDGPU::S_CBRANCH_EXECNZ:
3026 return EXECNZ;
3027 case AMDGPU::S_CBRANCH_EXECZ:
3028 return EXECZ;
3029 default:
3030 return INVALID_BR;
3031 }
3032}
3033
3034bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB,
3035 MachineBasicBlock::iterator I,
3036 MachineBasicBlock *&TBB,
3037 MachineBasicBlock *&FBB,
3038 SmallVectorImpl<MachineOperand> &Cond,
3039 bool AllowModify) const {
3040 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3041 // Unconditional Branch
3042 TBB = I->getOperand(i: 0).getMBB();
3043 return false;
3044 }
3045
3046 MachineBasicBlock *CondBB = nullptr;
3047
3048 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
3049 CondBB = I->getOperand(i: 1).getMBB();
3050 Cond.push_back(Elt: I->getOperand(i: 0));
3051 } else {
3052 BranchPredicate Pred = getBranchPredicate(Opcode: I->getOpcode());
3053 if (Pred == INVALID_BR)
3054 return true;
3055
3056 CondBB = I->getOperand(i: 0).getMBB();
3057 Cond.push_back(Elt: MachineOperand::CreateImm(Val: Pred));
3058 Cond.push_back(Elt: I->getOperand(i: 1)); // Save the branch register.
3059 }
3060 ++I;
3061
3062 if (I == MBB.end()) {
3063 // Conditional branch followed by fall-through.
3064 TBB = CondBB;
3065 return false;
3066 }
3067
3068 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3069 TBB = CondBB;
3070 FBB = I->getOperand(i: 0).getMBB();
3071 return false;
3072 }
3073
3074 return true;
3075}
3076
3077bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
3078 MachineBasicBlock *&FBB,
3079 SmallVectorImpl<MachineOperand> &Cond,
3080 bool AllowModify) const {
3081 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
3082 auto E = MBB.end();
3083 if (I == E)
3084 return false;
3085
3086 // Skip over the instructions that are artificially terminators for special
3087 // exec management.
3088 while (I != E && !I->isBranch() && !I->isReturn()) {
3089 switch (I->getOpcode()) {
3090 case AMDGPU::S_MOV_B64_term:
3091 case AMDGPU::S_XOR_B64_term:
3092 case AMDGPU::S_OR_B64_term:
3093 case AMDGPU::S_ANDN2_B64_term:
3094 case AMDGPU::S_AND_B64_term:
3095 case AMDGPU::S_AND_SAVEEXEC_B64_term:
3096 case AMDGPU::S_MOV_B32_term:
3097 case AMDGPU::S_XOR_B32_term:
3098 case AMDGPU::S_OR_B32_term:
3099 case AMDGPU::S_ANDN2_B32_term:
3100 case AMDGPU::S_AND_B32_term:
3101 case AMDGPU::S_AND_SAVEEXEC_B32_term:
3102 break;
3103 case AMDGPU::SI_IF:
3104 case AMDGPU::SI_ELSE:
3105 case AMDGPU::SI_KILL_I1_TERMINATOR:
3106 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
3107 // FIXME: It's messy that these need to be considered here at all.
3108 return true;
3109 default:
3110 llvm_unreachable("unexpected non-branch terminator inst");
3111 }
3112
3113 ++I;
3114 }
3115
3116 if (I == E)
3117 return false;
3118
3119 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
3120}
3121
3122unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB,
3123 int *BytesRemoved) const {
3124 unsigned Count = 0;
3125 unsigned RemovedSize = 0;
3126 for (MachineInstr &MI : llvm::make_early_inc_range(Range: MBB.terminators())) {
3127 // Skip over artificial terminators when removing instructions.
3128 if (MI.isBranch() || MI.isReturn()) {
3129 RemovedSize += getInstSizeInBytes(MI);
3130 MI.eraseFromParent();
3131 ++Count;
3132 }
3133 }
3134
3135 if (BytesRemoved)
3136 *BytesRemoved = RemovedSize;
3137
3138 return Count;
3139}
3140
3141// Copy the flags onto the implicit condition register operand.
3142static void preserveCondRegFlags(MachineOperand &CondReg,
3143 const MachineOperand &OrigCond) {
3144 CondReg.setIsUndef(OrigCond.isUndef());
3145 CondReg.setIsKill(OrigCond.isKill());
3146}
3147
3148unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,
3149 MachineBasicBlock *TBB,
3150 MachineBasicBlock *FBB,
3151 ArrayRef<MachineOperand> Cond,
3152 const DebugLoc &DL,
3153 int *BytesAdded) const {
3154 if (!FBB && Cond.empty()) {
3155 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
3156 .addMBB(MBB: TBB);
3157 if (BytesAdded)
3158 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3159 return 1;
3160 }
3161
3162 if(Cond.size() == 1 && Cond[0].isReg()) {
3163 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
3164 .add(MO: Cond[0])
3165 .addMBB(MBB: TBB);
3166 return 1;
3167 }
3168
3169 assert(TBB && Cond[0].isImm());
3170
3171 unsigned Opcode
3172 = getBranchOpcode(Cond: static_cast<BranchPredicate>(Cond[0].getImm()));
3173
3174 if (!FBB) {
3175 MachineInstr *CondBr =
3176 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode))
3177 .addMBB(MBB: TBB);
3178
3179 // Copy the flags onto the implicit condition register operand.
3180 preserveCondRegFlags(CondReg&: CondBr->getOperand(i: 1), OrigCond: Cond[1]);
3181 fixImplicitOperands(MI&: *CondBr);
3182
3183 if (BytesAdded)
3184 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3185 return 1;
3186 }
3187
3188 assert(TBB && FBB);
3189
3190 MachineInstr *CondBr =
3191 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode))
3192 .addMBB(MBB: TBB);
3193 fixImplicitOperands(MI&: *CondBr);
3194 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
3195 .addMBB(MBB: FBB);
3196
3197 MachineOperand &CondReg = CondBr->getOperand(i: 1);
3198 CondReg.setIsUndef(Cond[1].isUndef());
3199 CondReg.setIsKill(Cond[1].isKill());
3200
3201 if (BytesAdded)
3202 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8;
3203
3204 return 2;
3205}
3206
3207bool SIInstrInfo::reverseBranchCondition(
3208 SmallVectorImpl<MachineOperand> &Cond) const {
3209 if (Cond.size() != 2) {
3210 return true;
3211 }
3212
3213 if (Cond[0].isImm()) {
3214 Cond[0].setImm(-Cond[0].getImm());
3215 return false;
3216 }
3217
3218 return true;
3219}
3220
3221bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
3222 ArrayRef<MachineOperand> Cond,
3223 Register DstReg, Register TrueReg,
3224 Register FalseReg, int &CondCycles,
3225 int &TrueCycles, int &FalseCycles) const {
3226 switch (Cond[0].getImm()) {
3227 case VCCNZ:
3228 case VCCZ: {
3229 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3230 const TargetRegisterClass *RC = MRI.getRegClass(Reg: TrueReg);
3231 if (MRI.getRegClass(Reg: FalseReg) != RC)
3232 return false;
3233
3234 int NumInsts = AMDGPU::getRegBitWidth(RC: *RC) / 32;
3235 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
3236
3237 // Limit to equal cost for branch vs. N v_cndmask_b32s.
3238 return RI.hasVGPRs(RC) && NumInsts <= 6;
3239 }
3240 case SCC_TRUE:
3241 case SCC_FALSE: {
3242 // FIXME: We could insert for VGPRs if we could replace the original compare
3243 // with a vector one.
3244 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3245 const TargetRegisterClass *RC = MRI.getRegClass(Reg: TrueReg);
3246 if (MRI.getRegClass(Reg: FalseReg) != RC)
3247 return false;
3248
3249 int NumInsts = AMDGPU::getRegBitWidth(RC: *RC) / 32;
3250
3251 // Multiples of 8 can do s_cselect_b64
3252 if (NumInsts % 2 == 0)
3253 NumInsts /= 2;
3254
3255 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
3256 return RI.isSGPRClass(RC);
3257 }
3258 default:
3259 return false;
3260 }
3261}
3262
3263void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,
3264 MachineBasicBlock::iterator I, const DebugLoc &DL,
3265 Register DstReg, ArrayRef<MachineOperand> Cond,
3266 Register TrueReg, Register FalseReg) const {
3267 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
3268 if (Pred == VCCZ || Pred == SCC_FALSE) {
3269 Pred = static_cast<BranchPredicate>(-Pred);
3270 std::swap(a&: TrueReg, b&: FalseReg);
3271 }
3272
3273 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3274 const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: DstReg);
3275 unsigned DstSize = RI.getRegSizeInBits(RC: *DstRC);
3276
3277 if (DstSize == 32) {
3278 MachineInstr *Select;
3279 if (Pred == SCC_TRUE) {
3280 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: DstReg)
3281 .addReg(RegNo: TrueReg)
3282 .addReg(RegNo: FalseReg);
3283 } else {
3284 // Instruction's operands are backwards from what is expected.
3285 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e32), DestReg: DstReg)
3286 .addReg(RegNo: FalseReg)
3287 .addReg(RegNo: TrueReg);
3288 }
3289
3290 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3291 return;
3292 }
3293
3294 if (DstSize == 64 && Pred == SCC_TRUE) {
3295 MachineInstr *Select =
3296 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B64), DestReg: DstReg)
3297 .addReg(RegNo: TrueReg)
3298 .addReg(RegNo: FalseReg);
3299
3300 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3301 return;
3302 }
3303
3304 static const int16_t Sub0_15[] = {
3305 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
3306 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
3307 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
3308 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
3309 };
3310
3311 static const int16_t Sub0_15_64[] = {
3312 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
3313 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
3314 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
3315 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
3316 };
3317
3318 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
3319 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
3320 const int16_t *SubIndices = Sub0_15;
3321 int NElts = DstSize / 32;
3322
3323 // 64-bit select is only available for SALU.
3324 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
3325 if (Pred == SCC_TRUE) {
3326 if (NElts % 2) {
3327 SelOp = AMDGPU::S_CSELECT_B32;
3328 EltRC = &AMDGPU::SGPR_32RegClass;
3329 } else {
3330 SelOp = AMDGPU::S_CSELECT_B64;
3331 EltRC = &AMDGPU::SGPR_64RegClass;
3332 SubIndices = Sub0_15_64;
3333 NElts /= 2;
3334 }
3335 }
3336
3337 MachineInstrBuilder MIB = BuildMI(
3338 BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
3339
3340 I = MIB->getIterator();
3341
3342 SmallVector<Register, 8> Regs;
3343 for (int Idx = 0; Idx != NElts; ++Idx) {
3344 Register DstElt = MRI.createVirtualRegister(RegClass: EltRC);
3345 Regs.push_back(Elt: DstElt);
3346
3347 unsigned SubIdx = SubIndices[Idx];
3348
3349 MachineInstr *Select;
3350 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) {
3351 Select =
3352 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SelOp), DestReg: DstElt)
3353 .addReg(RegNo: FalseReg, flags: 0, SubReg: SubIdx)
3354 .addReg(RegNo: TrueReg, flags: 0, SubReg: SubIdx);
3355 } else {
3356 Select =
3357 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SelOp), DestReg: DstElt)
3358 .addReg(RegNo: TrueReg, flags: 0, SubReg: SubIdx)
3359 .addReg(RegNo: FalseReg, flags: 0, SubReg: SubIdx);
3360 }
3361
3362 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3363 fixImplicitOperands(MI&: *Select);
3364
3365 MIB.addReg(RegNo: DstElt)
3366 .addImm(Val: SubIdx);
3367 }
3368}
3369
3370bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) {
3371 switch (MI.getOpcode()) {
3372 case AMDGPU::V_MOV_B32_e32:
3373 case AMDGPU::V_MOV_B32_e64:
3374 case AMDGPU::V_MOV_B64_PSEUDO:
3375 case AMDGPU::V_MOV_B64_e32:
3376 case AMDGPU::V_MOV_B64_e64:
3377 case AMDGPU::S_MOV_B32:
3378 case AMDGPU::S_MOV_B64:
3379 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3380 case AMDGPU::COPY:
3381 case AMDGPU::WWM_COPY:
3382 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3383 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3384 case AMDGPU::V_ACCVGPR_MOV_B32:
3385 return true;
3386 default:
3387 return false;
3388 }
3389}
3390
3391static constexpr unsigned ModifierOpNames[] = {
3392 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
3393 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
3394 AMDGPU::OpName::omod, AMDGPU::OpName::op_sel};
3395
3396void SIInstrInfo::removeModOperands(MachineInstr &MI) const {
3397 unsigned Opc = MI.getOpcode();
3398 for (unsigned Name : reverse(C: ModifierOpNames)) {
3399 int Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: Name);
3400 if (Idx >= 0)
3401 MI.removeOperand(OpNo: Idx);
3402 }
3403}
3404
3405bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
3406 Register Reg, MachineRegisterInfo *MRI) const {
3407 if (!MRI->hasOneNonDBGUse(RegNo: Reg))
3408 return false;
3409
3410 switch (DefMI.getOpcode()) {
3411 default:
3412 return false;
3413 case AMDGPU::V_MOV_B64_e32:
3414 case AMDGPU::S_MOV_B64:
3415 case AMDGPU::V_MOV_B64_PSEUDO:
3416 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3417 case AMDGPU::V_MOV_B32_e32:
3418 case AMDGPU::S_MOV_B32:
3419 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3420 break;
3421 }
3422
3423 const MachineOperand *ImmOp = getNamedOperand(MI&: DefMI, OperandName: AMDGPU::OpName::src0);
3424 assert(ImmOp);
3425 // FIXME: We could handle FrameIndex values here.
3426 if (!ImmOp->isImm())
3427 return false;
3428
3429 auto getImmFor = [ImmOp](const MachineOperand &UseOp) -> int64_t {
3430 int64_t Imm = ImmOp->getImm();
3431 switch (UseOp.getSubReg()) {
3432 default:
3433 return Imm;
3434 case AMDGPU::sub0:
3435 return Lo_32(Value: Imm);
3436 case AMDGPU::sub1:
3437 return Hi_32(Value: Imm);
3438 case AMDGPU::lo16:
3439 return APInt(16, Imm).getSExtValue();
3440 case AMDGPU::hi16:
3441 return APInt(32, Imm).ashr(ShiftAmt: 16).getSExtValue();
3442 case AMDGPU::sub1_lo16:
3443 return APInt(16, Hi_32(Value: Imm)).getSExtValue();
3444 case AMDGPU::sub1_hi16:
3445 return APInt(32, Hi_32(Value: Imm)).ashr(ShiftAmt: 16).getSExtValue();
3446 }
3447 };
3448
3449 assert(!DefMI.getOperand(0).getSubReg() && "Expected SSA form");
3450
3451 unsigned Opc = UseMI.getOpcode();
3452 if (Opc == AMDGPU::COPY) {
3453 assert(!UseMI.getOperand(0).getSubReg() && "Expected SSA form");
3454
3455 Register DstReg = UseMI.getOperand(i: 0).getReg();
3456 unsigned OpSize = getOpSize(MI: UseMI, OpNo: 0);
3457 bool Is16Bit = OpSize == 2;
3458 bool Is64Bit = OpSize == 8;
3459 bool isVGPRCopy = RI.isVGPR(MRI: *MRI, Reg: DstReg);
3460 unsigned NewOpc = isVGPRCopy ? Is64Bit ? AMDGPU::V_MOV_B64_PSEUDO
3461 : AMDGPU::V_MOV_B32_e32
3462 : Is64Bit ? AMDGPU::S_MOV_B64_IMM_PSEUDO
3463 : AMDGPU::S_MOV_B32;
3464 APInt Imm(Is64Bit ? 64 : 32, getImmFor(UseMI.getOperand(i: 1)));
3465
3466 if (RI.isAGPR(MRI: *MRI, Reg: DstReg)) {
3467 if (Is64Bit || !isInlineConstant(Imm))
3468 return false;
3469 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
3470 }
3471
3472 if (Is16Bit) {
3473 if (isVGPRCopy)
3474 return false; // Do not clobber vgpr_hi16
3475
3476 if (DstReg.isVirtual() && UseMI.getOperand(i: 0).getSubReg() != AMDGPU::lo16)
3477 return false;
3478
3479 UseMI.getOperand(i: 0).setSubReg(0);
3480 if (DstReg.isPhysical()) {
3481 DstReg = RI.get32BitRegister(Reg: DstReg);
3482 UseMI.getOperand(i: 0).setReg(DstReg);
3483 }
3484 assert(UseMI.getOperand(1).getReg().isVirtual());
3485 }
3486
3487 const MCInstrDesc &NewMCID = get(Opcode: NewOpc);
3488 if (DstReg.isPhysical() &&
3489 !RI.getRegClass(RCID: NewMCID.operands()[0].RegClass)->contains(Reg: DstReg))
3490 return false;
3491
3492 UseMI.setDesc(NewMCID);
3493 UseMI.getOperand(i: 1).ChangeToImmediate(ImmVal: Imm.getSExtValue());
3494 UseMI.addImplicitDefUseOperands(MF&: *UseMI.getParent()->getParent());
3495 return true;
3496 }
3497
3498 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
3499 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3500 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3501 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3502 Opc == AMDGPU::V_FMAC_F16_t16_e64) {
3503 // Don't fold if we are using source or output modifiers. The new VOP2
3504 // instructions don't have them.
3505 if (hasAnyModifiersSet(MI: UseMI))
3506 return false;
3507
3508 // If this is a free constant, there's no reason to do this.
3509 // TODO: We could fold this here instead of letting SIFoldOperands do it
3510 // later.
3511 MachineOperand *Src0 = getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src0);
3512
3513 // Any src operand can be used for the legality check.
3514 if (isInlineConstant(MI: UseMI, UseMO: *Src0, DefMO: *ImmOp))
3515 return false;
3516
3517 bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
3518 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64;
3519 bool IsFMA =
3520 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3521 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3522 Opc == AMDGPU::V_FMAC_F16_t16_e64;
3523 MachineOperand *Src1 = getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src1);
3524 MachineOperand *Src2 = getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src2);
3525
3526 // Multiplied part is the constant: Use v_madmk_{f16, f32}.
3527 if ((Src0->isReg() && Src0->getReg() == Reg) ||
3528 (Src1->isReg() && Src1->getReg() == Reg)) {
3529 MachineOperand *RegSrc =
3530 Src1->isReg() && Src1->getReg() == Reg ? Src0 : Src1;
3531 if (!RegSrc->isReg())
3532 return false;
3533 if (RI.isSGPRClass(RC: MRI->getRegClass(Reg: RegSrc->getReg())) &&
3534 ST.getConstantBusLimit(Opcode: Opc) < 2)
3535 return false;
3536
3537 if (!Src2->isReg() || RI.isSGPRClass(RC: MRI->getRegClass(Reg: Src2->getReg())))
3538 return false;
3539
3540 // If src2 is also a literal constant then we have to choose which one to
3541 // fold. In general it is better to choose madak so that the other literal
3542 // can be materialized in an sgpr instead of a vgpr:
3543 // s_mov_b32 s0, literal
3544 // v_madak_f32 v0, s0, v0, literal
3545 // Instead of:
3546 // v_mov_b32 v1, literal
3547 // v_madmk_f32 v0, v0, literal, v1
3548 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src2->getReg());
3549 if (Def && Def->isMoveImmediate() &&
3550 !isInlineConstant(MO: Def->getOperand(i: 1)))
3551 return false;
3552
3553 unsigned NewOpc =
3554 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32
3555 : ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_t16
3556 : AMDGPU::V_FMAMK_F16)
3557 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16);
3558 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
3559 return false;
3560
3561 // V_FMAMK_F16_t16 takes VGPR_32_Lo128 operands, so the rewrite
3562 // would also require restricting their register classes. For now
3563 // just bail out.
3564 if (NewOpc == AMDGPU::V_FMAMK_F16_t16)
3565 return false;
3566
3567 const int64_t Imm = getImmFor(RegSrc == Src1 ? *Src0 : *Src1);
3568
3569 // FIXME: This would be a lot easier if we could return a new instruction
3570 // instead of having to modify in place.
3571
3572 Register SrcReg = RegSrc->getReg();
3573 unsigned SrcSubReg = RegSrc->getSubReg();
3574 Src0->setReg(SrcReg);
3575 Src0->setSubReg(SrcSubReg);
3576 Src0->setIsKill(RegSrc->isKill());
3577
3578 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3579 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
3580 Opc == AMDGPU::V_FMAC_F16_e64)
3581 UseMI.untieRegOperand(
3582 OpIdx: AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::src2));
3583
3584 Src1->ChangeToImmediate(ImmVal: Imm);
3585
3586 removeModOperands(MI&: UseMI);
3587 UseMI.setDesc(get(Opcode: NewOpc));
3588
3589 bool DeleteDef = MRI->use_nodbg_empty(RegNo: Reg);
3590 if (DeleteDef)
3591 DefMI.eraseFromParent();
3592
3593 return true;
3594 }
3595
3596 // Added part is the constant: Use v_madak_{f16, f32}.
3597 if (Src2->isReg() && Src2->getReg() == Reg) {
3598 if (ST.getConstantBusLimit(Opcode: Opc) < 2) {
3599 // Not allowed to use constant bus for another operand.
3600 // We can however allow an inline immediate as src0.
3601 bool Src0Inlined = false;
3602 if (Src0->isReg()) {
3603 // Try to inline constant if possible.
3604 // If the Def moves immediate and the use is single
3605 // We are saving VGPR here.
3606 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src0->getReg());
3607 if (Def && Def->isMoveImmediate() &&
3608 isInlineConstant(MO: Def->getOperand(i: 1)) &&
3609 MRI->hasOneUse(RegNo: Src0->getReg())) {
3610 Src0->ChangeToImmediate(ImmVal: Def->getOperand(i: 1).getImm());
3611 Src0Inlined = true;
3612 } else if (ST.getConstantBusLimit(Opcode: Opc) <= 1 &&
3613 RI.isSGPRReg(MRI: *MRI, Reg: Src0->getReg())) {
3614 return false;
3615 }
3616 // VGPR is okay as Src0 - fallthrough
3617 }
3618
3619 if (Src1->isReg() && !Src0Inlined) {
3620 // We have one slot for inlinable constant so far - try to fill it
3621 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src1->getReg());
3622 if (Def && Def->isMoveImmediate() &&
3623 isInlineConstant(MO: Def->getOperand(i: 1)) &&
3624 MRI->hasOneUse(RegNo: Src1->getReg()) && commuteInstruction(MI&: UseMI))
3625 Src0->ChangeToImmediate(ImmVal: Def->getOperand(i: 1).getImm());
3626 else if (RI.isSGPRReg(MRI: *MRI, Reg: Src1->getReg()))
3627 return false;
3628 // VGPR is okay as Src1 - fallthrough
3629 }
3630 }
3631
3632 unsigned NewOpc =
3633 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32
3634 : ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_t16
3635 : AMDGPU::V_FMAAK_F16)
3636 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16);
3637 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
3638 return false;
3639
3640 // V_FMAAK_F16_t16 takes VGPR_32_Lo128 operands, so the rewrite
3641 // would also require restricting their register classes. For now
3642 // just bail out.
3643 if (NewOpc == AMDGPU::V_FMAAK_F16_t16)
3644 return false;
3645
3646 // FIXME: This would be a lot easier if we could return a new instruction
3647 // instead of having to modify in place.
3648
3649 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3650 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
3651 Opc == AMDGPU::V_FMAC_F16_e64)
3652 UseMI.untieRegOperand(
3653 OpIdx: AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::src2));
3654
3655 // ChangingToImmediate adds Src2 back to the instruction.
3656 Src2->ChangeToImmediate(ImmVal: getImmFor(*Src2));
3657
3658 // These come before src2.
3659 removeModOperands(MI&: UseMI);
3660 UseMI.setDesc(get(Opcode: NewOpc));
3661 // It might happen that UseMI was commuted
3662 // and we now have SGPR as SRC1. If so 2 inlined
3663 // constant and SGPR are illegal.
3664 legalizeOperands(MI&: UseMI);
3665
3666 bool DeleteDef = MRI->use_nodbg_empty(RegNo: Reg);
3667 if (DeleteDef)
3668 DefMI.eraseFromParent();
3669
3670 return true;
3671 }
3672 }
3673
3674 return false;
3675}
3676
3677static bool
3678memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1,
3679 ArrayRef<const MachineOperand *> BaseOps2) {
3680 if (BaseOps1.size() != BaseOps2.size())
3681 return false;
3682 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) {
3683 if (!BaseOps1[I]->isIdenticalTo(Other: *BaseOps2[I]))
3684 return false;
3685 }
3686 return true;
3687}
3688
3689static bool offsetsDoNotOverlap(LocationSize WidthA, int OffsetA,
3690 LocationSize WidthB, int OffsetB) {
3691 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
3692 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
3693 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
3694 return LowWidth.hasValue() &&
3695 LowOffset + (int)LowWidth.getValue() <= HighOffset;
3696}
3697
3698bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
3699 const MachineInstr &MIb) const {
3700 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;
3701 int64_t Offset0, Offset1;
3702 LocationSize Dummy0 = 0, Dummy1 = 0;
3703 bool Offset0IsScalable, Offset1IsScalable;
3704 if (!getMemOperandsWithOffsetWidth(LdSt: MIa, BaseOps&: BaseOps0, Offset&: Offset0, OffsetIsScalable&: Offset0IsScalable,
3705 Width&: Dummy0, TRI: &RI) ||
3706 !getMemOperandsWithOffsetWidth(LdSt: MIb, BaseOps&: BaseOps1, Offset&: Offset1, OffsetIsScalable&: Offset1IsScalable,
3707 Width&: Dummy1, TRI: &RI))
3708 return false;
3709
3710 if (!memOpsHaveSameBaseOperands(BaseOps1: BaseOps0, BaseOps2: BaseOps1))
3711 return false;
3712
3713 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
3714 // FIXME: Handle ds_read2 / ds_write2.
3715 return false;
3716 }
3717 LocationSize Width0 = MIa.memoperands().front()->getSize();
3718 LocationSize Width1 = MIb.memoperands().front()->getSize();
3719 return offsetsDoNotOverlap(WidthA: Width0, OffsetA: Offset0, WidthB: Width1, OffsetB: Offset1);
3720}
3721
3722bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
3723 const MachineInstr &MIb) const {
3724 assert(MIa.mayLoadOrStore() &&
3725 "MIa must load from or modify a memory location");
3726 assert(MIb.mayLoadOrStore() &&
3727 "MIb must load from or modify a memory location");
3728
3729 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects())
3730 return false;
3731
3732 // XXX - Can we relax this between address spaces?
3733 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
3734 return false;
3735
3736 if (isLDSDMA(MI: MIa) || isLDSDMA(MI: MIb))
3737 return false;
3738
3739 // TODO: Should we check the address space from the MachineMemOperand? That
3740 // would allow us to distinguish objects we know don't alias based on the
3741 // underlying address space, even if it was lowered to a different one,
3742 // e.g. private accesses lowered to use MUBUF instructions on a scratch
3743 // buffer.
3744 if (isDS(MI: MIa)) {
3745 if (isDS(MI: MIb))
3746 return checkInstOffsetsDoNotOverlap(MIa, MIb);
3747
3748 return !isFLAT(MI: MIb) || isSegmentSpecificFLAT(MI: MIb);
3749 }
3750
3751 if (isMUBUF(MI: MIa) || isMTBUF(MI: MIa)) {
3752 if (isMUBUF(MI: MIb) || isMTBUF(MI: MIb))
3753 return checkInstOffsetsDoNotOverlap(MIa, MIb);
3754
3755 if (isFLAT(MI: MIb))
3756 return isFLATScratch(MI: MIb);
3757
3758 return !isSMRD(MI: MIb);
3759 }
3760
3761 if (isSMRD(MI: MIa)) {
3762 if (isSMRD(MI: MIb))
3763 return checkInstOffsetsDoNotOverlap(MIa, MIb);
3764
3765 if (isFLAT(MI: MIb))
3766 return isFLATScratch(MI: MIb);
3767
3768 return !isMUBUF(MI: MIb) && !isMTBUF(MI: MIb);
3769 }
3770
3771 if (isFLAT(MI: MIa)) {
3772 if (isFLAT(MI: MIb)) {
3773 if ((isFLATScratch(MI: MIa) && isFLATGlobal(MI: MIb)) ||
3774 (isFLATGlobal(MI: MIa) && isFLATScratch(MI: MIb)))
3775 return true;
3776
3777 return checkInstOffsetsDoNotOverlap(MIa, MIb);
3778 }
3779
3780 return false;
3781 }
3782
3783 return false;
3784}
3785
3786static bool getFoldableImm(Register Reg, const MachineRegisterInfo &MRI,
3787 int64_t &Imm, MachineInstr **DefMI = nullptr) {
3788 if (Reg.isPhysical())
3789 return false;
3790 auto *Def = MRI.getUniqueVRegDef(Reg);
3791 if (Def && SIInstrInfo::isFoldableCopy(MI: *Def) && Def->getOperand(i: 1).isImm()) {
3792 Imm = Def->getOperand(i: 1).getImm();
3793 if (DefMI)
3794 *DefMI = Def;
3795 return true;
3796 }
3797 return false;
3798}
3799
3800static bool getFoldableImm(const MachineOperand *MO, int64_t &Imm,
3801 MachineInstr **DefMI = nullptr) {
3802 if (!MO->isReg())
3803 return false;
3804 const MachineFunction *MF = MO->getParent()->getParent()->getParent();
3805 const MachineRegisterInfo &MRI = MF->getRegInfo();
3806 return getFoldableImm(Reg: MO->getReg(), MRI, Imm, DefMI);
3807}
3808
3809static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI,
3810 MachineInstr &NewMI) {
3811 if (LV) {
3812 unsigned NumOps = MI.getNumOperands();
3813 for (unsigned I = 1; I < NumOps; ++I) {
3814 MachineOperand &Op = MI.getOperand(i: I);
3815 if (Op.isReg() && Op.isKill())
3816 LV->replaceKillInstruction(Reg: Op.getReg(), OldMI&: MI, NewMI);
3817 }
3818 }
3819}
3820
3821MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI,
3822 LiveVariables *LV,
3823 LiveIntervals *LIS) const {
3824 MachineBasicBlock &MBB = *MI.getParent();
3825 unsigned Opc = MI.getOpcode();
3826
3827 // Handle MFMA.
3828 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opcode: Opc);
3829 if (NewMFMAOpc != -1) {
3830 MachineInstrBuilder MIB =
3831 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewMFMAOpc));
3832 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I)
3833 MIB.add(MO: MI.getOperand(i: I));
3834 updateLiveVariables(LV, MI, NewMI&: *MIB);
3835 if (LIS) {
3836 LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
3837 // SlotIndex of defs needs to be updated when converting to early-clobber
3838 MachineOperand &Def = MIB->getOperand(i: 0);
3839 if (Def.isEarlyClobber() && Def.isReg() &&
3840 LIS->hasInterval(Reg: Def.getReg())) {
3841 SlotIndex OldIndex = LIS->getInstructionIndex(Instr: *MIB).getRegSlot(EC: false);
3842 SlotIndex NewIndex = LIS->getInstructionIndex(Instr: *MIB).getRegSlot(EC: true);
3843 auto &LI = LIS->getInterval(Reg: Def.getReg());
3844 auto UpdateDefIndex = [&](LiveRange &LR) {
3845 auto S = LR.find(Pos: OldIndex);
3846 if (S != LR.end() && S->start == OldIndex) {
3847 assert(S->valno && S->valno->def == OldIndex);
3848 S->start = NewIndex;
3849 S->valno->def = NewIndex;
3850 }
3851 };
3852 UpdateDefIndex(LI);
3853 for (auto &SR : LI.subranges())
3854 UpdateDefIndex(SR);
3855 }
3856 }
3857 return MIB;
3858 }
3859
3860 if (SIInstrInfo::isWMMA(MI)) {
3861 unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(Opc: MI.getOpcode());
3862 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
3863 .setMIFlags(MI.getFlags());
3864 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I)
3865 MIB->addOperand(Op: MI.getOperand(i: I));
3866
3867 updateLiveVariables(LV, MI, NewMI&: *MIB);
3868 if (LIS)
3869 LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
3870
3871 return MIB;
3872 }
3873
3874 assert(Opc != AMDGPU::V_FMAC_F16_t16_e32 &&
3875 "V_FMAC_F16_t16_e32 is not supported and not expected to be present "
3876 "pre-RA");
3877
3878 // Handle MAC/FMAC.
3879 bool IsF16 = Opc == AMDGPU::V_MAC_F16_e32 || Opc == AMDGPU::V_MAC_F16_e64 ||
3880 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3881 Opc == AMDGPU::V_FMAC_F16_t16_e64;
3882 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3883 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
3884 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64 ||
3885 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3886 Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
3887 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
3888 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
3889 bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 ||
3890 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 ||
3891 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
3892 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64;
3893 bool Src0Literal = false;
3894
3895 switch (Opc) {
3896 default:
3897 return nullptr;
3898 case AMDGPU::V_MAC_F16_e64:
3899 case AMDGPU::V_FMAC_F16_e64:
3900 case AMDGPU::V_FMAC_F16_t16_e64:
3901 case AMDGPU::V_MAC_F32_e64:
3902 case AMDGPU::V_MAC_LEGACY_F32_e64:
3903 case AMDGPU::V_FMAC_F32_e64:
3904 case AMDGPU::V_FMAC_LEGACY_F32_e64:
3905 case AMDGPU::V_FMAC_F64_e64:
3906 break;
3907 case AMDGPU::V_MAC_F16_e32:
3908 case AMDGPU::V_FMAC_F16_e32:
3909 case AMDGPU::V_MAC_F32_e32:
3910 case AMDGPU::V_MAC_LEGACY_F32_e32:
3911 case AMDGPU::V_FMAC_F32_e32:
3912 case AMDGPU::V_FMAC_LEGACY_F32_e32:
3913 case AMDGPU::V_FMAC_F64_e32: {
3914 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
3915 NamedIdx: AMDGPU::OpName::src0);
3916 const MachineOperand *Src0 = &MI.getOperand(i: Src0Idx);
3917 if (!Src0->isReg() && !Src0->isImm())
3918 return nullptr;
3919
3920 if (Src0->isImm() && !isInlineConstant(MI, OpIdx: Src0Idx, MO: *Src0))
3921 Src0Literal = true;
3922
3923 break;
3924 }
3925 }
3926
3927 MachineInstrBuilder MIB;
3928 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
3929 const MachineOperand *Src0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
3930 const MachineOperand *Src0Mods =
3931 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
3932 const MachineOperand *Src1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
3933 const MachineOperand *Src1Mods =
3934 getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
3935 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
3936 const MachineOperand *Src2Mods =
3937 getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers);
3938 const MachineOperand *Clamp = getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
3939 const MachineOperand *Omod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
3940 const MachineOperand *OpSel = getNamedOperand(MI, OperandName: AMDGPU::OpName::op_sel);
3941
3942 if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsF64 &&
3943 !IsLegacy &&
3944 // If we have an SGPR input, we will violate the constant bus restriction.
3945 (ST.getConstantBusLimit(Opcode: Opc) > 1 || !Src0->isReg() ||
3946 !RI.isSGPRReg(MRI: MBB.getParent()->getRegInfo(), Reg: Src0->getReg()))) {
3947 MachineInstr *DefMI;
3948 const auto killDef = [&]() -> void {
3949 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3950 // The only user is the instruction which will be killed.
3951 Register DefReg = DefMI->getOperand(i: 0).getReg();
3952 if (!MRI.hasOneNonDBGUse(RegNo: DefReg))
3953 return;
3954 // We cannot just remove the DefMI here, calling pass will crash.
3955 DefMI->setDesc(get(Opcode: AMDGPU::IMPLICIT_DEF));
3956 for (unsigned I = DefMI->getNumOperands() - 1; I != 0; --I)
3957 DefMI->removeOperand(OpNo: I);
3958 if (LV)
3959 LV->getVarInfo(Reg: DefReg).AliveBlocks.clear();
3960 };
3961
3962 int64_t Imm;
3963 if (!Src0Literal && getFoldableImm(MO: Src2, Imm, DefMI: &DefMI)) {
3964 unsigned NewOpc =
3965 IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_t16
3966 : AMDGPU::V_FMAAK_F16)
3967 : AMDGPU::V_FMAAK_F32)
3968 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32);
3969 if (pseudoToMCOpcode(Opcode: NewOpc) != -1) {
3970 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
3971 .add(MO: *Dst)
3972 .add(MO: *Src0)
3973 .add(MO: *Src1)
3974 .addImm(Val: Imm)
3975 .setMIFlags(MI.getFlags());
3976 updateLiveVariables(LV, MI, NewMI&: *MIB);
3977 if (LIS)
3978 LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
3979 killDef();
3980 return MIB;
3981 }
3982 }
3983 unsigned NewOpc =
3984 IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_t16
3985 : AMDGPU::V_FMAMK_F16)
3986 : AMDGPU::V_FMAMK_F32)
3987 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32);
3988 if (!Src0Literal && getFoldableImm(MO: Src1, Imm, DefMI: &DefMI)) {
3989 if (pseudoToMCOpcode(Opcode: NewOpc) != -1) {
3990 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
3991 .add(MO: *Dst)
3992 .add(MO: *Src0)
3993 .addImm(Val: Imm)
3994 .add(MO: *Src2)
3995 .setMIFlags(MI.getFlags());
3996 updateLiveVariables(LV, MI, NewMI&: *MIB);
3997 if (LIS)
3998 LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
3999 killDef();
4000 return MIB;
4001 }
4002 }
4003 if (Src0Literal || getFoldableImm(MO: Src0, Imm, DefMI: &DefMI)) {
4004 if (Src0Literal) {
4005 Imm = Src0->getImm();
4006 DefMI = nullptr;
4007 }
4008 if (pseudoToMCOpcode(Opcode: NewOpc) != -1 &&
4009 isOperandLegal(
4010 MI, OpIdx: AMDGPU::getNamedOperandIdx(Opcode: NewOpc, NamedIdx: AMDGPU::OpName::src0),
4011 MO: Src1)) {
4012 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4013 .add(MO: *Dst)
4014 .add(MO: *Src1)
4015 .addImm(Val: Imm)
4016 .add(MO: *Src2)
4017 .setMIFlags(MI.getFlags());
4018 updateLiveVariables(LV, MI, NewMI&: *MIB);
4019 if (LIS)
4020 LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
4021 if (DefMI)
4022 killDef();
4023 return MIB;
4024 }
4025 }
4026 }
4027
4028 // VOP2 mac/fmac with a literal operand cannot be converted to VOP3 mad/fma
4029 // if VOP3 does not allow a literal operand.
4030 if (Src0Literal && !ST.hasVOP3Literal())
4031 return nullptr;
4032
4033 unsigned NewOpc = IsFMA ? IsF16 ? AMDGPU::V_FMA_F16_gfx9_e64
4034 : IsF64 ? AMDGPU::V_FMA_F64_e64
4035 : IsLegacy
4036 ? AMDGPU::V_FMA_LEGACY_F32_e64
4037 : AMDGPU::V_FMA_F32_e64
4038 : IsF16 ? AMDGPU::V_MAD_F16_e64
4039 : IsLegacy ? AMDGPU::V_MAD_LEGACY_F32_e64
4040 : AMDGPU::V_MAD_F32_e64;
4041 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
4042 return nullptr;
4043
4044 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4045 .add(MO: *Dst)
4046 .addImm(Val: Src0Mods ? Src0Mods->getImm() : 0)
4047 .add(MO: *Src0)
4048 .addImm(Val: Src1Mods ? Src1Mods->getImm() : 0)
4049 .add(MO: *Src1)
4050 .addImm(Val: Src2Mods ? Src2Mods->getImm() : 0)
4051 .add(MO: *Src2)
4052 .addImm(Val: Clamp ? Clamp->getImm() : 0)
4053 .addImm(Val: Omod ? Omod->getImm() : 0)
4054 .setMIFlags(MI.getFlags());
4055 if (AMDGPU::hasNamedOperand(Opcode: NewOpc, NamedIdx: AMDGPU::OpName::op_sel))
4056 MIB.addImm(Val: OpSel ? OpSel->getImm() : 0);
4057 updateLiveVariables(LV, MI, NewMI&: *MIB);
4058 if (LIS)
4059 LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
4060 return MIB;
4061}
4062
4063// It's not generally safe to move VALU instructions across these since it will
4064// start using the register as a base index rather than directly.
4065// XXX - Why isn't hasSideEffects sufficient for these?
4066static bool changesVGPRIndexingMode(const MachineInstr &MI) {
4067 switch (MI.getOpcode()) {
4068 case AMDGPU::S_SET_GPR_IDX_ON:
4069 case AMDGPU::S_SET_GPR_IDX_MODE:
4070 case AMDGPU::S_SET_GPR_IDX_OFF:
4071 return true;
4072 default:
4073 return false;
4074 }
4075}
4076
4077bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
4078 const MachineBasicBlock *MBB,
4079 const MachineFunction &MF) const {
4080 // Skipping the check for SP writes in the base implementation. The reason it
4081 // was added was apparently due to compile time concerns.
4082 //
4083 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops
4084 // but is probably avoidable.
4085
4086 // Copied from base implementation.
4087 // Terminators and labels can't be scheduled around.
4088 if (MI.isTerminator() || MI.isPosition())
4089 return true;
4090
4091 // INLINEASM_BR can jump to another block
4092 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
4093 return true;
4094
4095 if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(i: 0).getImm() == 0)
4096 return true;
4097
4098 // Target-independent instructions do not have an implicit-use of EXEC, even
4099 // when they operate on VGPRs. Treating EXEC modifications as scheduling
4100 // boundaries prevents incorrect movements of such instructions.
4101 return MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &RI) ||
4102 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
4103 MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
4104 MI.getOpcode() == AMDGPU::S_SETPRIO ||
4105 changesVGPRIndexingMode(MI);
4106}
4107
4108bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {
4109 return Opcode == AMDGPU::DS_ORDERED_COUNT || isGWS(Opcode);
4110}
4111
4112bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) {
4113 // Skip the full operand and register alias search modifiesRegister
4114 // does. There's only a handful of instructions that touch this, it's only an
4115 // implicit def, and doesn't alias any other registers.
4116 return is_contained(Range: MI.getDesc().implicit_defs(), Element: AMDGPU::MODE);
4117}
4118
4119bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const {
4120 unsigned Opcode = MI.getOpcode();
4121
4122 if (MI.mayStore() && isSMRD(MI))
4123 return true; // scalar store or atomic
4124
4125 // This will terminate the function when other lanes may need to continue.
4126 if (MI.isReturn())
4127 return true;
4128
4129 // These instructions cause shader I/O that may cause hardware lockups
4130 // when executed with an empty EXEC mask.
4131 //
4132 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
4133 // EXEC = 0, but checking for that case here seems not worth it
4134 // given the typical code patterns.
4135 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
4136 isEXP(Opcode) || Opcode == AMDGPU::DS_ORDERED_COUNT ||
4137 Opcode == AMDGPU::S_TRAP || Opcode == AMDGPU::S_WAIT_EVENT)
4138 return true;
4139
4140 if (MI.isCall() || MI.isInlineAsm())
4141 return true; // conservative assumption
4142
4143 // Assume that barrier interactions are only intended with active lanes.
4144 if (isBarrier(Opcode))
4145 return true;
4146
4147 // A mode change is a scalar operation that influences vector instructions.
4148 if (modifiesModeRegister(MI))
4149 return true;
4150
4151 // These are like SALU instructions in terms of effects, so it's questionable
4152 // whether we should return true for those.
4153 //
4154 // However, executing them with EXEC = 0 causes them to operate on undefined
4155 // data, which we avoid by returning true here.
4156 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 ||
4157 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32 ||
4158 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
4159 Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR)
4160 return true;
4161
4162 return false;
4163}
4164
4165bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI,
4166 const MachineInstr &MI) const {
4167 if (MI.isMetaInstruction())
4168 return false;
4169
4170 // This won't read exec if this is an SGPR->SGPR copy.
4171 if (MI.isCopyLike()) {
4172 if (!RI.isSGPRReg(MRI, Reg: MI.getOperand(i: 0).getReg()))
4173 return true;
4174
4175 // Make sure this isn't copying exec as a normal operand
4176 return MI.readsRegister(Reg: AMDGPU::EXEC, TRI: &RI);
4177 }
4178
4179 // Make a conservative assumption about the callee.
4180 if (MI.isCall())
4181 return true;
4182
4183 // Be conservative with any unhandled generic opcodes.
4184 if (!isTargetSpecificOpcode(Opcode: MI.getOpcode()))
4185 return true;
4186
4187 return !isSALU(MI) || MI.readsRegister(Reg: AMDGPU::EXEC, TRI: &RI);
4188}
4189
4190bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
4191 switch (Imm.getBitWidth()) {
4192 case 1: // This likely will be a condition code mask.
4193 return true;
4194
4195 case 32:
4196 return AMDGPU::isInlinableLiteral32(Literal: Imm.getSExtValue(),
4197 HasInv2Pi: ST.hasInv2PiInlineImm());
4198 case 64:
4199 return AMDGPU::isInlinableLiteral64(Literal: Imm.getSExtValue(),
4200 HasInv2Pi: ST.hasInv2PiInlineImm());
4201 case 16:
4202 return ST.has16BitInsts() &&
4203 AMDGPU::isInlinableLiteralI16(Literal: Imm.getSExtValue(),
4204 HasInv2Pi: ST.hasInv2PiInlineImm());
4205 default:
4206 llvm_unreachable("invalid bitwidth");
4207 }
4208}
4209
4210bool SIInstrInfo::isInlineConstant(const APFloat &Imm) const {
4211 APInt IntImm = Imm.bitcastToAPInt();
4212 int64_t IntImmVal = IntImm.getSExtValue();
4213 bool HasInv2Pi = ST.hasInv2PiInlineImm();
4214 switch (APFloat::SemanticsToEnum(Sem: Imm.getSemantics())) {
4215 default:
4216 llvm_unreachable("invalid fltSemantics");
4217 case APFloatBase::S_IEEEsingle:
4218 case APFloatBase::S_IEEEdouble:
4219 return isInlineConstant(Imm: IntImm);
4220 case APFloatBase::S_BFloat:
4221 return ST.has16BitInsts() &&
4222 AMDGPU::isInlinableLiteralBF16(Literal: IntImmVal, HasInv2Pi);
4223 case APFloatBase::S_IEEEhalf:
4224 return ST.has16BitInsts() &&
4225 AMDGPU::isInlinableLiteralFP16(Literal: IntImmVal, HasInv2Pi);
4226 }
4227}
4228
4229bool SIInstrInfo::isInlineConstant(const MachineOperand &MO,
4230 uint8_t OperandType) const {
4231 assert(!MO.isReg() && "isInlineConstant called on register operand!");
4232 if (!MO.isImm())
4233 return false;
4234
4235 // MachineOperand provides no way to tell the true operand size, since it only
4236 // records a 64-bit value. We need to know the size to determine if a 32-bit
4237 // floating point immediate bit pattern is legal for an integer immediate. It
4238 // would be for any 32-bit integer operand, but would not be for a 64-bit one.
4239
4240 int64_t Imm = MO.getImm();
4241 switch (OperandType) {
4242 case AMDGPU::OPERAND_REG_IMM_INT32:
4243 case AMDGPU::OPERAND_REG_IMM_FP32:
4244 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED:
4245 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
4246 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
4247 case AMDGPU::OPERAND_REG_IMM_V2FP32:
4248 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
4249 case AMDGPU::OPERAND_REG_IMM_V2INT32:
4250 case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
4251 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
4252 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
4253 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: {
4254 int32_t Trunc = static_cast<int32_t>(Imm);
4255 return AMDGPU::isInlinableLiteral32(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
4256 }
4257 case AMDGPU::OPERAND_REG_IMM_INT64:
4258 case AMDGPU::OPERAND_REG_IMM_FP64:
4259 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
4260 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
4261 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
4262 return AMDGPU::isInlinableLiteral64(Literal: MO.getImm(),
4263 HasInv2Pi: ST.hasInv2PiInlineImm());
4264 case AMDGPU::OPERAND_REG_IMM_INT16:
4265 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
4266 case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
4267 // We would expect inline immediates to not be concerned with an integer/fp
4268 // distinction. However, in the case of 16-bit integer operations, the
4269 // "floating point" values appear to not work. It seems read the low 16-bits
4270 // of 32-bit immediates, which happens to always work for the integer
4271 // values.
4272 //
4273 // See llvm bugzilla 46302.
4274 //
4275 // TODO: Theoretically we could use op-sel to use the high bits of the
4276 // 32-bit FP values.
4277 return AMDGPU::isInlinableIntLiteral(Literal: Imm);
4278 case AMDGPU::OPERAND_REG_IMM_V2INT16:
4279 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
4280 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
4281 return AMDGPU::isInlinableLiteralV2I16(Literal: Imm);
4282 case AMDGPU::OPERAND_REG_IMM_V2FP16:
4283 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
4284 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
4285 return AMDGPU::isInlinableLiteralV2F16(Literal: Imm);
4286 case AMDGPU::OPERAND_REG_IMM_V2BF16:
4287 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
4288 case AMDGPU::OPERAND_REG_INLINE_AC_V2BF16:
4289 return AMDGPU::isInlinableLiteralV2BF16(Literal: Imm);
4290 case AMDGPU::OPERAND_REG_IMM_FP16:
4291 case AMDGPU::OPERAND_REG_IMM_FP16_DEFERRED:
4292 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
4293 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: {
4294 if (isInt<16>(x: Imm) || isUInt<16>(x: Imm)) {
4295 // A few special case instructions have 16-bit operands on subtargets
4296 // where 16-bit instructions are not legal.
4297 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
4298 // constants in these cases
4299 int16_t Trunc = static_cast<int16_t>(Imm);
4300 return ST.has16BitInsts() &&
4301 AMDGPU::isInlinableLiteralFP16(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
4302 }
4303
4304 return false;
4305 }
4306 case AMDGPU::OPERAND_REG_IMM_BF16:
4307 case AMDGPU::OPERAND_REG_IMM_BF16_DEFERRED:
4308 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
4309 case AMDGPU::OPERAND_REG_INLINE_AC_BF16: {
4310 if (isInt<16>(x: Imm) || isUInt<16>(x: Imm)) {
4311 int16_t Trunc = static_cast<int16_t>(Imm);
4312 return ST.has16BitInsts() &&
4313 AMDGPU::isInlinableLiteralBF16(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
4314 }
4315 return false;
4316 }
4317 case AMDGPU::OPERAND_KIMM32:
4318 case AMDGPU::OPERAND_KIMM16:
4319 return false;
4320 case AMDGPU::OPERAND_INPUT_MODS:
4321 case MCOI::OPERAND_IMMEDIATE:
4322 // Always embedded in the instruction for free.
4323 return true;
4324 case MCOI::OPERAND_UNKNOWN:
4325 case MCOI::OPERAND_REGISTER:
4326 case MCOI::OPERAND_PCREL:
4327 case MCOI::OPERAND_GENERIC_0:
4328 case MCOI::OPERAND_GENERIC_1:
4329 case MCOI::OPERAND_GENERIC_2:
4330 case MCOI::OPERAND_GENERIC_3:
4331 case MCOI::OPERAND_GENERIC_4:
4332 case MCOI::OPERAND_GENERIC_5:
4333 // Just ignore anything else.
4334 return true;
4335 default:
4336 llvm_unreachable("invalid operand type");
4337 }
4338}
4339
4340static bool compareMachineOp(const MachineOperand &Op0,
4341 const MachineOperand &Op1) {
4342 if (Op0.getType() != Op1.getType())
4343 return false;
4344
4345 switch (Op0.getType()) {
4346 case MachineOperand::MO_Register:
4347 return Op0.getReg() == Op1.getReg();
4348 case MachineOperand::MO_Immediate:
4349 return Op0.getImm() == Op1.getImm();
4350 default:
4351 llvm_unreachable("Didn't expect to be comparing these operand types");
4352 }
4353}
4354
4355bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
4356 const MachineOperand &MO) const {
4357 const MCInstrDesc &InstDesc = MI.getDesc();
4358 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
4359
4360 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
4361
4362 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
4363 return true;
4364
4365 if (OpInfo.RegClass < 0)
4366 return false;
4367
4368 if (MO.isImm() && isInlineConstant(MO, OpInfo)) {
4369 if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() &&
4370 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
4371 NamedIdx: AMDGPU::OpName::src2))
4372 return false;
4373 return RI.opCanUseInlineConstant(OpType: OpInfo.OperandType);
4374 }
4375
4376 if (!RI.opCanUseLiteralConstant(OpType: OpInfo.OperandType))
4377 return false;
4378
4379 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(Desc: InstDesc, OpNo))
4380 return true;
4381
4382 return ST.hasVOP3Literal();
4383}
4384
4385bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
4386 // GFX90A does not have V_MUL_LEGACY_F32_e32.
4387 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())
4388 return false;
4389
4390 int Op32 = AMDGPU::getVOPe32(Opcode);
4391 if (Op32 == -1)
4392 return false;
4393
4394 return pseudoToMCOpcode(Opcode: Op32) != -1;
4395}
4396
4397bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
4398 // The src0_modifier operand is present on all instructions
4399 // that have modifiers.
4400
4401 return AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0_modifiers);
4402}
4403
4404bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI,
4405 unsigned OpName) const {
4406 const MachineOperand *Mods = getNamedOperand(MI, OpName);
4407 return Mods && Mods->getImm();
4408}
4409
4410bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const {
4411 return any_of(Range: ModifierOpNames,
4412 P: [&](unsigned Name) { return hasModifiersSet(MI, OpName: Name); });
4413}
4414
4415bool SIInstrInfo::canShrink(const MachineInstr &MI,
4416 const MachineRegisterInfo &MRI) const {
4417 const MachineOperand *Src2 = getNamedOperand(MI, OpName: AMDGPU::OpName::src2);
4418 // Can't shrink instruction with three operands.
4419 if (Src2) {
4420 switch (MI.getOpcode()) {
4421 default: return false;
4422
4423 case AMDGPU::V_ADDC_U32_e64:
4424 case AMDGPU::V_SUBB_U32_e64:
4425 case AMDGPU::V_SUBBREV_U32_e64: {
4426 const MachineOperand *Src1
4427 = getNamedOperand(MI, OpName: AMDGPU::OpName::src1);
4428 if (!Src1->isReg() || !RI.isVGPR(MRI, Reg: Src1->getReg()))
4429 return false;
4430 // Additional verification is needed for sdst/src2.
4431 return true;
4432 }
4433 case AMDGPU::V_MAC_F16_e64:
4434 case AMDGPU::V_MAC_F32_e64:
4435 case AMDGPU::V_MAC_LEGACY_F32_e64:
4436 case AMDGPU::V_FMAC_F16_e64:
4437 case AMDGPU::V_FMAC_F16_t16_e64:
4438 case AMDGPU::V_FMAC_F32_e64:
4439 case AMDGPU::V_FMAC_F64_e64:
4440 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4441 if (!Src2->isReg() || !RI.isVGPR(MRI, Reg: Src2->getReg()) ||
4442 hasModifiersSet(MI, OpName: AMDGPU::OpName::src2_modifiers))
4443 return false;
4444 break;
4445
4446 case AMDGPU::V_CNDMASK_B32_e64:
4447 break;
4448 }
4449 }
4450
4451 const MachineOperand *Src1 = getNamedOperand(MI, OpName: AMDGPU::OpName::src1);
4452 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Reg: Src1->getReg()) ||
4453 hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers)))
4454 return false;
4455
4456 // We don't need to check src0, all input types are legal, so just make sure
4457 // src0 isn't using any modifiers.
4458 if (hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers))
4459 return false;
4460
4461 // Can it be shrunk to a valid 32 bit opcode?
4462 if (!hasVALU32BitEncoding(Opcode: MI.getOpcode()))
4463 return false;
4464
4465 // Check output modifiers
4466 return !hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) &&
4467 !hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) &&
4468 !hasModifiersSet(MI, OpName: AMDGPU::OpName::byte_sel);
4469}
4470
4471// Set VCC operand with all flags from \p Orig, except for setting it as
4472// implicit.
4473static void copyFlagsToImplicitVCC(MachineInstr &MI,
4474 const MachineOperand &Orig) {
4475
4476 for (MachineOperand &Use : MI.implicit_operands()) {
4477 if (Use.isUse() &&
4478 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) {
4479 Use.setIsUndef(Orig.isUndef());
4480 Use.setIsKill(Orig.isKill());
4481 return;
4482 }
4483 }
4484}
4485
4486MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI,
4487 unsigned Op32) const {
4488 MachineBasicBlock *MBB = MI.getParent();
4489
4490 const MCInstrDesc &Op32Desc = get(Opcode: Op32);
4491 MachineInstrBuilder Inst32 =
4492 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: Op32Desc)
4493 .setMIFlags(MI.getFlags());
4494
4495 // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
4496 // For VOPC instructions, this is replaced by an implicit def of vcc.
4497
4498 // We assume the defs of the shrunk opcode are in the same order, and the
4499 // shrunk opcode loses the last def (SGPR def, in the VOP3->VOPC case).
4500 for (int I = 0, E = Op32Desc.getNumDefs(); I != E; ++I)
4501 Inst32.add(MO: MI.getOperand(i: I));
4502
4503 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
4504
4505 int Idx = MI.getNumExplicitDefs();
4506 for (const MachineOperand &Use : MI.explicit_uses()) {
4507 int OpTy = MI.getDesc().operands()[Idx++].OperandType;
4508 if (OpTy == AMDGPU::OPERAND_INPUT_MODS || OpTy == MCOI::OPERAND_IMMEDIATE)
4509 continue;
4510
4511 if (&Use == Src2) {
4512 if (AMDGPU::getNamedOperandIdx(Opcode: Op32, NamedIdx: AMDGPU::OpName::src2) == -1) {
4513 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
4514 // replaced with an implicit read of vcc or vcc_lo. The implicit read
4515 // of vcc was already added during the initial BuildMI, but we
4516 // 1) may need to change vcc to vcc_lo to preserve the original register
4517 // 2) have to preserve the original flags.
4518 fixImplicitOperands(MI&: *Inst32);
4519 copyFlagsToImplicitVCC(MI&: *Inst32, Orig: *Src2);
4520 continue;
4521 }
4522 }
4523
4524 Inst32.add(MO: Use);
4525 }
4526
4527 // FIXME: Losing implicit operands
4528
4529 return Inst32;
4530}
4531
4532bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,
4533 const MachineOperand &MO,
4534 const MCOperandInfo &OpInfo) const {
4535 // Literal constants use the constant bus.
4536 if (!MO.isReg())
4537 return !isInlineConstant(MO, OpInfo);
4538
4539 if (!MO.isUse())
4540 return false;
4541
4542 if (MO.getReg().isVirtual())
4543 return RI.isSGPRClass(RC: MRI.getRegClass(Reg: MO.getReg()));
4544
4545 // Null is free
4546 if (MO.getReg() == AMDGPU::SGPR_NULL || MO.getReg() == AMDGPU::SGPR_NULL64)
4547 return false;
4548
4549 // SGPRs use the constant bus
4550 if (MO.isImplicit()) {
4551 return MO.getReg() == AMDGPU::M0 || MO.getReg() == AMDGPU::VCC ||
4552 MO.getReg() == AMDGPU::VCC_LO;
4553 }
4554 return AMDGPU::SReg_32RegClass.contains(Reg: MO.getReg()) ||
4555 AMDGPU::SReg_64RegClass.contains(Reg: MO.getReg());
4556}
4557
4558static Register findImplicitSGPRRead(const MachineInstr &MI) {
4559 for (const MachineOperand &MO : MI.implicit_operands()) {
4560 // We only care about reads.
4561 if (MO.isDef())
4562 continue;
4563
4564 switch (MO.getReg()) {
4565 case AMDGPU::VCC:
4566 case AMDGPU::VCC_LO:
4567 case AMDGPU::VCC_HI:
4568 case AMDGPU::M0:
4569 case AMDGPU::FLAT_SCR:
4570 return MO.getReg();
4571
4572 default:
4573 break;
4574 }
4575 }
4576
4577 return Register();
4578}
4579
4580static bool shouldReadExec(const MachineInstr &MI) {
4581 if (SIInstrInfo::isVALU(MI)) {
4582 switch (MI.getOpcode()) {
4583 case AMDGPU::V_READLANE_B32:
4584 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
4585 case AMDGPU::V_WRITELANE_B32:
4586 case AMDGPU::SI_SPILL_S32_TO_VGPR:
4587 return false;
4588 }
4589
4590 return true;
4591 }
4592
4593 if (MI.isPreISelOpcode() ||
4594 SIInstrInfo::isGenericOpcode(Opc: MI.getOpcode()) ||
4595 SIInstrInfo::isSALU(MI) ||
4596 SIInstrInfo::isSMRD(MI))
4597 return false;
4598
4599 return true;
4600}
4601
4602static bool isSubRegOf(const SIRegisterInfo &TRI,
4603 const MachineOperand &SuperVec,
4604 const MachineOperand &SubReg) {
4605 if (SubReg.getReg().isPhysical())
4606 return TRI.isSubRegister(RegA: SuperVec.getReg(), RegB: SubReg.getReg());
4607
4608 return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
4609 SubReg.getReg() == SuperVec.getReg();
4610}
4611
4612bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
4613 StringRef &ErrInfo) const {
4614 uint16_t Opcode = MI.getOpcode();
4615 if (SIInstrInfo::isGenericOpcode(Opc: MI.getOpcode()))
4616 return true;
4617
4618 const MachineFunction *MF = MI.getParent()->getParent();
4619 const MachineRegisterInfo &MRI = MF->getRegInfo();
4620
4621 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, NamedIdx: AMDGPU::OpName::src0);
4622 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, NamedIdx: AMDGPU::OpName::src1);
4623 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, NamedIdx: AMDGPU::OpName::src2);
4624 int Src3Idx = -1;
4625 if (Src0Idx == -1) {
4626 // VOPD V_DUAL_* instructions use different operand names.
4627 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, NamedIdx: AMDGPU::OpName::src0X);
4628 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, NamedIdx: AMDGPU::OpName::vsrc1X);
4629 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, NamedIdx: AMDGPU::OpName::src0Y);
4630 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, NamedIdx: AMDGPU::OpName::vsrc1Y);
4631 }
4632
4633 // Make sure the number of operands is correct.
4634 const MCInstrDesc &Desc = get(Opcode);
4635 if (!Desc.isVariadic() &&
4636 Desc.getNumOperands() != MI.getNumExplicitOperands()) {
4637 ErrInfo = "Instruction has wrong number of operands.";
4638 return false;
4639 }
4640
4641 if (MI.isInlineAsm()) {
4642 // Verify register classes for inlineasm constraints.
4643 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
4644 I != E; ++I) {
4645 const TargetRegisterClass *RC = MI.getRegClassConstraint(OpIdx: I, TII: this, TRI: &RI);
4646 if (!RC)
4647 continue;
4648
4649 const MachineOperand &Op = MI.getOperand(i: I);
4650 if (!Op.isReg())
4651 continue;
4652
4653 Register Reg = Op.getReg();
4654 if (!Reg.isVirtual() && !RC->contains(Reg)) {
4655 ErrInfo = "inlineasm operand has incorrect register class.";
4656 return false;
4657 }
4658 }
4659
4660 return true;
4661 }
4662
4663 if (isImage(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) {
4664 ErrInfo = "missing memory operand from image instruction.";
4665 return false;
4666 }
4667
4668 // Make sure the register classes are correct.
4669 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
4670 const MachineOperand &MO = MI.getOperand(i);
4671 if (MO.isFPImm()) {
4672 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
4673 "all fp values to integers.";
4674 return false;
4675 }
4676
4677 int RegClass = Desc.operands()[i].RegClass;
4678
4679 switch (Desc.operands()[i].OperandType) {
4680 case MCOI::OPERAND_REGISTER:
4681 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
4682 ErrInfo = "Illegal immediate value for operand.";
4683 return false;
4684 }
4685 break;
4686 case AMDGPU::OPERAND_REG_IMM_INT32:
4687 case AMDGPU::OPERAND_REG_IMM_FP32:
4688 case AMDGPU::OPERAND_REG_IMM_FP32_DEFERRED:
4689 case AMDGPU::OPERAND_REG_IMM_V2FP32:
4690 break;
4691 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
4692 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
4693 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
4694 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
4695 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
4696 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
4697 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
4698 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
4699 case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
4700 case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
4701 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: {
4702 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, OpIdx: i))) {
4703 ErrInfo = "Illegal immediate value for operand.";
4704 return false;
4705 }
4706 break;
4707 }
4708 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
4709 if (!MI.getOperand(i).isImm() || !isInlineConstant(MI, OpIdx: i)) {
4710 ErrInfo = "Expected inline constant for operand.";
4711 return false;
4712 }
4713 break;
4714 case MCOI::OPERAND_IMMEDIATE:
4715 case AMDGPU::OPERAND_KIMM32:
4716 // Check if this operand is an immediate.
4717 // FrameIndex operands will be replaced by immediates, so they are
4718 // allowed.
4719 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
4720 ErrInfo = "Expected immediate, but got non-immediate";
4721 return false;
4722 }
4723 [[fallthrough]];
4724 default:
4725 continue;
4726 }
4727
4728 if (!MO.isReg())
4729 continue;
4730 Register Reg = MO.getReg();
4731 if (!Reg)
4732 continue;
4733
4734 // FIXME: Ideally we would have separate instruction definitions with the
4735 // aligned register constraint.
4736 // FIXME: We do not verify inline asm operands, but custom inline asm
4737 // verification is broken anyway
4738 if (ST.needsAlignedVGPRs()) {
4739 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI, Reg);
4740 if (RI.hasVectorRegisters(RC) && MO.getSubReg()) {
4741 const TargetRegisterClass *SubRC =
4742 RI.getSubRegisterClass(RC, MO.getSubReg());
4743 RC = RI.getCompatibleSubRegClass(SuperRC: RC, SubRC, SubIdx: MO.getSubReg());
4744 if (RC)
4745 RC = SubRC;
4746 }
4747
4748 // Check that this is the aligned version of the class.
4749 if (!RC || !RI.isProperlyAlignedRC(RC: *RC)) {
4750 ErrInfo = "Subtarget requires even aligned vector registers";
4751 return false;
4752 }
4753 }
4754
4755 if (RegClass != -1) {
4756 if (Reg.isVirtual())
4757 continue;
4758
4759 const TargetRegisterClass *RC = RI.getRegClass(RCID: RegClass);
4760 if (!RC->contains(Reg)) {
4761 ErrInfo = "Operand has incorrect register class.";
4762 return false;
4763 }
4764 }
4765 }
4766
4767 // Verify SDWA
4768 if (isSDWA(MI)) {
4769 if (!ST.hasSDWA()) {
4770 ErrInfo = "SDWA is not supported on this target";
4771 return false;
4772 }
4773
4774 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, NamedIdx: AMDGPU::OpName::vdst);
4775
4776 for (int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) {
4777 if (OpIdx == -1)
4778 continue;
4779 const MachineOperand &MO = MI.getOperand(i: OpIdx);
4780
4781 if (!ST.hasSDWAScalar()) {
4782 // Only VGPRS on VI
4783 if (!MO.isReg() || !RI.hasVGPRs(RC: RI.getRegClassForReg(MRI, Reg: MO.getReg()))) {
4784 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
4785 return false;
4786 }
4787 } else {
4788 // No immediates on GFX9
4789 if (!MO.isReg()) {
4790 ErrInfo =
4791 "Only reg allowed as operands in SDWA instructions on GFX9+";
4792 return false;
4793 }
4794 }
4795 }
4796
4797 if (!ST.hasSDWAOmod()) {
4798 // No omod allowed on VI
4799 const MachineOperand *OMod = getNamedOperand(MI, OpName: AMDGPU::OpName::omod);
4800 if (OMod != nullptr &&
4801 (!OMod->isImm() || OMod->getImm() != 0)) {
4802 ErrInfo = "OMod not allowed in SDWA instructions on VI";
4803 return false;
4804 }
4805 }
4806
4807 if (Opcode == AMDGPU::V_CVT_F32_FP8_sdwa ||
4808 Opcode == AMDGPU::V_CVT_F32_BF8_sdwa ||
4809 Opcode == AMDGPU::V_CVT_PK_F32_FP8_sdwa ||
4810 Opcode == AMDGPU::V_CVT_PK_F32_BF8_sdwa) {
4811 const MachineOperand *Src0ModsMO =
4812 getNamedOperand(MI, OpName: AMDGPU::OpName::src0_modifiers);
4813 unsigned Mods = Src0ModsMO->getImm();
4814 if (Mods & SISrcMods::ABS || Mods & SISrcMods::NEG ||
4815 Mods & SISrcMods::SEXT) {
4816 ErrInfo = "sext, abs and neg are not allowed on this instruction";
4817 return false;
4818 }
4819 }
4820
4821 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
4822 if (isVOPC(Opcode: BasicOpcode)) {
4823 if (!ST.hasSDWASdst() && DstIdx != -1) {
4824 // Only vcc allowed as dst on VI for VOPC
4825 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
4826 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
4827 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
4828 return false;
4829 }
4830 } else if (!ST.hasSDWAOutModsVOPC()) {
4831 // No clamp allowed on GFX9 for VOPC
4832 const MachineOperand *Clamp = getNamedOperand(MI, OpName: AMDGPU::OpName::clamp);
4833 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
4834 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
4835 return false;
4836 }
4837
4838 // No omod allowed on GFX9 for VOPC
4839 const MachineOperand *OMod = getNamedOperand(MI, OpName: AMDGPU::OpName::omod);
4840 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
4841 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
4842 return false;
4843 }
4844 }
4845 }
4846
4847 const MachineOperand *DstUnused = getNamedOperand(MI, OpName: AMDGPU::OpName::dst_unused);
4848 if (DstUnused && DstUnused->isImm() &&
4849 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
4850 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
4851 if (!Dst.isReg() || !Dst.isTied()) {
4852 ErrInfo = "Dst register should have tied register";
4853 return false;
4854 }
4855
4856 const MachineOperand &TiedMO =
4857 MI.getOperand(i: MI.findTiedOperandIdx(OpIdx: DstIdx));
4858 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
4859 ErrInfo =
4860 "Dst register should be tied to implicit use of preserved register";
4861 return false;
4862 }
4863 if (TiedMO.getReg().isPhysical() && Dst.getReg() != TiedMO.getReg()) {
4864 ErrInfo = "Dst register should use same physical register as preserved";
4865 return false;
4866 }
4867 }
4868 }
4869
4870 // Verify MIMG / VIMAGE / VSAMPLE
4871 if (isImage(Opcode: MI.getOpcode()) && !MI.mayStore()) {
4872 // Ensure that the return type used is large enough for all the options
4873 // being used TFE/LWE require an extra result register.
4874 const MachineOperand *DMask = getNamedOperand(MI, OpName: AMDGPU::OpName::dmask);
4875 if (DMask) {
4876 uint64_t DMaskImm = DMask->getImm();
4877 uint32_t RegCount =
4878 isGather4(Opcode: MI.getOpcode()) ? 4 : llvm::popcount(Value: DMaskImm);
4879 const MachineOperand *TFE = getNamedOperand(MI, OpName: AMDGPU::OpName::tfe);
4880 const MachineOperand *LWE = getNamedOperand(MI, OpName: AMDGPU::OpName::lwe);
4881 const MachineOperand *D16 = getNamedOperand(MI, OpName: AMDGPU::OpName::d16);
4882
4883 // Adjust for packed 16 bit values
4884 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
4885 RegCount = divideCeil(Numerator: RegCount, Denominator: 2);
4886
4887 // Adjust if using LWE or TFE
4888 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
4889 RegCount += 1;
4890
4891 const uint32_t DstIdx =
4892 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::vdata);
4893 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
4894 if (Dst.isReg()) {
4895 const TargetRegisterClass *DstRC = getOpRegClass(MI, OpNo: DstIdx);
4896 uint32_t DstSize = RI.getRegSizeInBits(RC: *DstRC) / 32;
4897 if (RegCount > DstSize) {
4898 ErrInfo = "Image instruction returns too many registers for dst "
4899 "register class";
4900 return false;
4901 }
4902 }
4903 }
4904 }
4905
4906 // Verify VOP*. Ignore multiple sgpr operands on writelane.
4907 if (isVALU(MI) && Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
4908 unsigned ConstantBusCount = 0;
4909 bool UsesLiteral = false;
4910 const MachineOperand *LiteralVal = nullptr;
4911
4912 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, NamedIdx: AMDGPU::OpName::imm);
4913 if (ImmIdx != -1) {
4914 ++ConstantBusCount;
4915 UsesLiteral = true;
4916 LiteralVal = &MI.getOperand(i: ImmIdx);
4917 }
4918
4919 SmallVector<Register, 2> SGPRsUsed;
4920 Register SGPRUsed;
4921
4922 // Only look at the true operands. Only a real operand can use the constant
4923 // bus, and we don't want to check pseudo-operands like the source modifier
4924 // flags.
4925 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) {
4926 if (OpIdx == -1)
4927 continue;
4928 const MachineOperand &MO = MI.getOperand(i: OpIdx);
4929 if (usesConstantBus(MRI, MO, OpInfo: MI.getDesc().operands()[OpIdx])) {
4930 if (MO.isReg()) {
4931 SGPRUsed = MO.getReg();
4932 if (!llvm::is_contained(Range&: SGPRsUsed, Element: SGPRUsed)) {
4933 ++ConstantBusCount;
4934 SGPRsUsed.push_back(Elt: SGPRUsed);
4935 }
4936 } else {
4937 if (!UsesLiteral) {
4938 ++ConstantBusCount;
4939 UsesLiteral = true;
4940 LiteralVal = &MO;
4941 } else if (!MO.isIdenticalTo(Other: *LiteralVal)) {
4942 assert(isVOP2(MI) || isVOP3(MI));
4943 ErrInfo = "VOP2/VOP3 instruction uses more than one literal";
4944 return false;
4945 }
4946 }
4947 }
4948 }
4949
4950 SGPRUsed = findImplicitSGPRRead(MI);
4951 if (SGPRUsed) {
4952 // Implicit uses may safely overlap true operands
4953 if (llvm::all_of(Range&: SGPRsUsed, P: [this, SGPRUsed](unsigned SGPR) {
4954 return !RI.regsOverlap(RegA: SGPRUsed, RegB: SGPR);
4955 })) {
4956 ++ConstantBusCount;
4957 SGPRsUsed.push_back(Elt: SGPRUsed);
4958 }
4959 }
4960
4961 // v_writelane_b32 is an exception from constant bus restriction:
4962 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
4963 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
4964 Opcode != AMDGPU::V_WRITELANE_B32) {
4965 ErrInfo = "VOP* instruction violates constant bus restriction";
4966 return false;
4967 }
4968
4969 if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) {
4970 ErrInfo = "VOP3 instruction uses literal";
4971 return false;
4972 }
4973 }
4974
4975 // Special case for writelane - this can break the multiple constant bus rule,
4976 // but still can't use more than one SGPR register
4977 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
4978 unsigned SGPRCount = 0;
4979 Register SGPRUsed;
4980
4981 for (int OpIdx : {Src0Idx, Src1Idx}) {
4982 if (OpIdx == -1)
4983 break;
4984
4985 const MachineOperand &MO = MI.getOperand(i: OpIdx);
4986
4987 if (usesConstantBus(MRI, MO, OpInfo: MI.getDesc().operands()[OpIdx])) {
4988 if (MO.isReg() && MO.getReg() != AMDGPU::M0) {
4989 if (MO.getReg() != SGPRUsed)
4990 ++SGPRCount;
4991 SGPRUsed = MO.getReg();
4992 }
4993 }
4994 if (SGPRCount > ST.getConstantBusLimit(Opcode)) {
4995 ErrInfo = "WRITELANE instruction violates constant bus restriction";
4996 return false;
4997 }
4998 }
4999 }
5000
5001 // Verify misc. restrictions on specific instructions.
5002 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
5003 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
5004 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5005 const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
5006 const MachineOperand &Src2 = MI.getOperand(i: Src2Idx);
5007 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
5008 if (!compareMachineOp(Op0: Src0, Op1: Src1) &&
5009 !compareMachineOp(Op0: Src0, Op1: Src2)) {
5010 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
5011 return false;
5012 }
5013 }
5014 if ((getNamedOperand(MI, OpName: AMDGPU::OpName::src0_modifiers)->getImm() &
5015 SISrcMods::ABS) ||
5016 (getNamedOperand(MI, OpName: AMDGPU::OpName::src1_modifiers)->getImm() &
5017 SISrcMods::ABS) ||
5018 (getNamedOperand(MI, OpName: AMDGPU::OpName::src2_modifiers)->getImm() &
5019 SISrcMods::ABS)) {
5020 ErrInfo = "ABS not allowed in VOP3B instructions";
5021 return false;
5022 }
5023 }
5024
5025 if (isSOP2(MI) || isSOPC(MI)) {
5026 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5027 const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
5028
5029 if (!Src0.isReg() && !Src1.isReg() &&
5030 !isInlineConstant(MO: Src0, OpInfo: Desc.operands()[Src0Idx]) &&
5031 !isInlineConstant(MO: Src1, OpInfo: Desc.operands()[Src1Idx]) &&
5032 !Src0.isIdenticalTo(Other: Src1)) {
5033 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
5034 return false;
5035 }
5036 }
5037
5038 if (isSOPK(MI)) {
5039 auto Op = getNamedOperand(MI, OpName: AMDGPU::OpName::simm16);
5040 if (Desc.isBranch()) {
5041 if (!Op->isMBB()) {
5042 ErrInfo = "invalid branch target for SOPK instruction";
5043 return false;
5044 }
5045 } else {
5046 uint64_t Imm = Op->getImm();
5047 if (sopkIsZext(Opcode)) {
5048 if (!isUInt<16>(x: Imm)) {
5049 ErrInfo = "invalid immediate for SOPK instruction";
5050 return false;
5051 }
5052 } else {
5053 if (!isInt<16>(x: Imm)) {
5054 ErrInfo = "invalid immediate for SOPK instruction";
5055 return false;
5056 }
5057 }
5058 }
5059 }
5060
5061 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
5062 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
5063 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
5064 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
5065 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
5066 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
5067
5068 const unsigned StaticNumOps =
5069 Desc.getNumOperands() + Desc.implicit_uses().size();
5070 const unsigned NumImplicitOps = IsDst ? 2 : 1;
5071
5072 // Allow additional implicit operands. This allows a fixup done by the post
5073 // RA scheduler where the main implicit operand is killed and implicit-defs
5074 // are added for sub-registers that remain live after this instruction.
5075 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
5076 ErrInfo = "missing implicit register operands";
5077 return false;
5078 }
5079
5080 const MachineOperand *Dst = getNamedOperand(MI, OpName: AMDGPU::OpName::vdst);
5081 if (IsDst) {
5082 if (!Dst->isUse()) {
5083 ErrInfo = "v_movreld_b32 vdst should be a use operand";
5084 return false;
5085 }
5086
5087 unsigned UseOpIdx;
5088 if (!MI.isRegTiedToUseOperand(DefOpIdx: StaticNumOps, UseOpIdx: &UseOpIdx) ||
5089 UseOpIdx != StaticNumOps + 1) {
5090 ErrInfo = "movrel implicit operands should be tied";
5091 return false;
5092 }
5093 }
5094
5095 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5096 const MachineOperand &ImpUse
5097 = MI.getOperand(i: StaticNumOps + NumImplicitOps - 1);
5098 if (!ImpUse.isReg() || !ImpUse.isUse() ||
5099 !isSubRegOf(TRI: RI, SuperVec: ImpUse, SubReg: IsDst ? *Dst : Src0)) {
5100 ErrInfo = "src0 should be subreg of implicit vector use";
5101 return false;
5102 }
5103 }
5104
5105 // Make sure we aren't losing exec uses in the td files. This mostly requires
5106 // being careful when using let Uses to try to add other use registers.
5107 if (shouldReadExec(MI)) {
5108 if (!MI.hasRegisterImplicitUseOperand(Reg: AMDGPU::EXEC)) {
5109 ErrInfo = "VALU instruction does not implicitly read exec mask";
5110 return false;
5111 }
5112 }
5113
5114 if (isSMRD(MI)) {
5115 if (MI.mayStore() &&
5116 ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) {
5117 // The register offset form of scalar stores may only use m0 as the
5118 // soffset register.
5119 const MachineOperand *Soff = getNamedOperand(MI, OpName: AMDGPU::OpName::soffset);
5120 if (Soff && Soff->getReg() != AMDGPU::M0) {
5121 ErrInfo = "scalar stores must use m0 as offset register";
5122 return false;
5123 }
5124 }
5125 }
5126
5127 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) {
5128 const MachineOperand *Offset = getNamedOperand(MI, OpName: AMDGPU::OpName::offset);
5129 if (Offset->getImm() != 0) {
5130 ErrInfo = "subtarget does not support offsets in flat instructions";
5131 return false;
5132 }
5133 }
5134
5135 if (isDS(MI) && !ST.hasGDS()) {
5136 const MachineOperand *GDSOp = getNamedOperand(MI, OpName: AMDGPU::OpName::gds);
5137 if (GDSOp && GDSOp->getImm() != 0) {
5138 ErrInfo = "GDS is not supported on this subtarget";
5139 return false;
5140 }
5141 }
5142
5143 if (isImage(MI)) {
5144 const MachineOperand *DimOp = getNamedOperand(MI, OpName: AMDGPU::OpName::dim);
5145 if (DimOp) {
5146 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
5147 NamedIdx: AMDGPU::OpName::vaddr0);
5148 int RSrcOpName =
5149 isMIMG(MI) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
5150 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, NamedIdx: RSrcOpName);
5151 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: Opcode);
5152 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
5153 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
5154 const AMDGPU::MIMGDimInfo *Dim =
5155 AMDGPU::getMIMGDimInfoByEncoding(DimEnc: DimOp->getImm());
5156
5157 if (!Dim) {
5158 ErrInfo = "dim is out of range";
5159 return false;
5160 }
5161
5162 bool IsA16 = false;
5163 if (ST.hasR128A16()) {
5164 const MachineOperand *R128A16 = getNamedOperand(MI, OpName: AMDGPU::OpName::r128);
5165 IsA16 = R128A16->getImm() != 0;
5166 } else if (ST.hasA16()) {
5167 const MachineOperand *A16 = getNamedOperand(MI, OpName: AMDGPU::OpName::a16);
5168 IsA16 = A16->getImm() != 0;
5169 }
5170
5171 bool IsNSA = RsrcIdx - VAddr0Idx > 1;
5172
5173 unsigned AddrWords =
5174 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, IsG16Supported: ST.hasG16());
5175
5176 unsigned VAddrWords;
5177 if (IsNSA) {
5178 VAddrWords = RsrcIdx - VAddr0Idx;
5179 if (ST.hasPartialNSAEncoding() &&
5180 AddrWords > ST.getNSAMaxSize(HasSampler: isVSAMPLE(MI))) {
5181 unsigned LastVAddrIdx = RsrcIdx - 1;
5182 VAddrWords += getOpSize(MI, OpNo: LastVAddrIdx) / 4 - 1;
5183 }
5184 } else {
5185 VAddrWords = getOpSize(MI, OpNo: VAddr0Idx) / 4;
5186 if (AddrWords > 12)
5187 AddrWords = 16;
5188 }
5189
5190 if (VAddrWords != AddrWords) {
5191 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords
5192 << " but got " << VAddrWords << "\n");
5193 ErrInfo = "bad vaddr size";
5194 return false;
5195 }
5196 }
5197 }
5198
5199 const MachineOperand *DppCt = getNamedOperand(MI, OpName: AMDGPU::OpName::dpp_ctrl);
5200 if (DppCt) {
5201 using namespace AMDGPU::DPP;
5202
5203 unsigned DC = DppCt->getImm();
5204 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
5205 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
5206 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) ||
5207 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) ||
5208 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) ||
5209 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) ||
5210 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) {
5211 ErrInfo = "Invalid dpp_ctrl value";
5212 return false;
5213 }
5214 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
5215 ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
5216 ErrInfo = "Invalid dpp_ctrl value: "
5217 "wavefront shifts are not supported on GFX10+";
5218 return false;
5219 }
5220 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
5221 ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
5222 ErrInfo = "Invalid dpp_ctrl value: "
5223 "broadcasts are not supported on GFX10+";
5224 return false;
5225 }
5226 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST &&
5227 ST.getGeneration() < AMDGPUSubtarget::GFX10) {
5228 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST &&
5229 DC <= DppCtrl::ROW_NEWBCAST_LAST &&
5230 !ST.hasGFX90AInsts()) {
5231 ErrInfo = "Invalid dpp_ctrl value: "
5232 "row_newbroadcast/row_share is not supported before "
5233 "GFX90A/GFX10";
5234 return false;
5235 }
5236 if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) {
5237 ErrInfo = "Invalid dpp_ctrl value: "
5238 "row_share and row_xmask are not supported before GFX10";
5239 return false;
5240 }
5241 }
5242
5243 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&
5244 !AMDGPU::isLegalDPALU_DPPControl(DC) && AMDGPU::isDPALU_DPP(OpDesc: Desc)) {
5245 ErrInfo = "Invalid dpp_ctrl value: "
5246 "DP ALU dpp only support row_newbcast";
5247 return false;
5248 }
5249 }
5250
5251 if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) {
5252 const MachineOperand *Dst = getNamedOperand(MI, OpName: AMDGPU::OpName::vdst);
5253 uint16_t DataNameIdx = isDS(Opcode) ? AMDGPU::OpName::data0
5254 : AMDGPU::OpName::vdata;
5255 const MachineOperand *Data = getNamedOperand(MI, OpName: DataNameIdx);
5256 const MachineOperand *Data2 = getNamedOperand(MI, OpName: AMDGPU::OpName::data1);
5257 if (Data && !Data->isReg())
5258 Data = nullptr;
5259
5260 if (ST.hasGFX90AInsts()) {
5261 if (Dst && Data &&
5262 (RI.isAGPR(MRI, Reg: Dst->getReg()) != RI.isAGPR(MRI, Reg: Data->getReg()))) {
5263 ErrInfo = "Invalid register class: "
5264 "vdata and vdst should be both VGPR or AGPR";
5265 return false;
5266 }
5267 if (Data && Data2 &&
5268 (RI.isAGPR(MRI, Reg: Data->getReg()) != RI.isAGPR(MRI, Reg: Data2->getReg()))) {
5269 ErrInfo = "Invalid register class: "
5270 "both data operands should be VGPR or AGPR";
5271 return false;
5272 }
5273 } else {
5274 if ((Dst && RI.isAGPR(MRI, Reg: Dst->getReg())) ||
5275 (Data && RI.isAGPR(MRI, Reg: Data->getReg())) ||
5276 (Data2 && RI.isAGPR(MRI, Reg: Data2->getReg()))) {
5277 ErrInfo = "Invalid register class: "
5278 "agpr loads and stores not supported on this GPU";
5279 return false;
5280 }
5281 }
5282 }
5283
5284 if (ST.needsAlignedVGPRs()) {
5285 const auto isAlignedReg = [&MI, &MRI, this](unsigned OpName) -> bool {
5286 const MachineOperand *Op = getNamedOperand(MI, OpName);
5287 if (!Op)
5288 return true;
5289 Register Reg = Op->getReg();
5290 if (Reg.isPhysical())
5291 return !(RI.getHWRegIndex(Reg) & 1);
5292 const TargetRegisterClass &RC = *MRI.getRegClass(Reg);
5293 return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
5294 !(RI.getChannelFromSubReg(SubReg: Op->getSubReg()) & 1);
5295 };
5296
5297 if (MI.getOpcode() == AMDGPU::DS_GWS_INIT ||
5298 MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR ||
5299 MI.getOpcode() == AMDGPU::DS_GWS_BARRIER) {
5300
5301 if (!isAlignedReg(AMDGPU::OpName::data0)) {
5302 ErrInfo = "Subtarget requires even aligned vector registers "
5303 "for DS_GWS instructions";
5304 return false;
5305 }
5306 }
5307
5308 if (isMIMG(MI)) {
5309 if (!isAlignedReg(AMDGPU::OpName::vaddr)) {
5310 ErrInfo = "Subtarget requires even aligned vector registers "
5311 "for vaddr operand of image instructions";
5312 return false;
5313 }
5314 }
5315 }
5316
5317 if (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
5318 !ST.hasGFX90AInsts()) {
5319 const MachineOperand *Src = getNamedOperand(MI, OpName: AMDGPU::OpName::src0);
5320 if (Src->isReg() && RI.isSGPRReg(MRI, Reg: Src->getReg())) {
5321 ErrInfo = "Invalid register class: "
5322 "v_accvgpr_write with an SGPR is not supported on this GPU";
5323 return false;
5324 }
5325 }
5326
5327 if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) {
5328 const MachineOperand &SrcOp = MI.getOperand(i: 1);
5329 if (!SrcOp.isReg() || SrcOp.getReg().isVirtual()) {
5330 ErrInfo = "pseudo expects only physical SGPRs";
5331 return false;
5332 }
5333 }
5334
5335 return true;
5336}
5337
5338// It is more readable to list mapped opcodes on the same line.
5339// clang-format off
5340
5341unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
5342 switch (MI.getOpcode()) {
5343 default: return AMDGPU::INSTRUCTION_LIST_END;
5344 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
5345 case AMDGPU::COPY: return AMDGPU::COPY;
5346 case AMDGPU::PHI: return AMDGPU::PHI;
5347 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
5348 case AMDGPU::WQM: return AMDGPU::WQM;
5349 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
5350 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM;
5351 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM;
5352 case AMDGPU::S_MOV_B32: {
5353 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
5354 return MI.getOperand(i: 1).isReg() ||
5355 RI.isAGPR(MRI, Reg: MI.getOperand(i: 0).getReg()) ?
5356 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
5357 }
5358 case AMDGPU::S_ADD_I32:
5359 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
5360 case AMDGPU::S_ADDC_U32:
5361 return AMDGPU::V_ADDC_U32_e32;
5362 case AMDGPU::S_SUB_I32:
5363 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32;
5364 // FIXME: These are not consistently handled, and selected when the carry is
5365 // used.
5366 case AMDGPU::S_ADD_U32:
5367 return AMDGPU::V_ADD_CO_U32_e32;
5368 case AMDGPU::S_SUB_U32:
5369 return AMDGPU::V_SUB_CO_U32_e32;
5370 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
5371 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64;
5372 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64;
5373 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64;
5374 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
5375 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
5376 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
5377 case AMDGPU::S_XNOR_B32:
5378 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
5379 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
5380 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
5381 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
5382 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
5383 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
5384 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64;
5385 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
5386 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64;
5387 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
5388 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64;
5389 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64;
5390 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64;
5391 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64;
5392 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64;
5393 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
5394 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
5395 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
5396 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
5397 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64;
5398 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64;
5399 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64;
5400 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64;
5401 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64;
5402 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64;
5403 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64;
5404 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64;
5405 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64;
5406 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64;
5407 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64;
5408 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64;
5409 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64;
5410 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64;
5411 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
5412 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
5413 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
5414 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
5415 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
5416 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
5417 case AMDGPU::S_CVT_F32_I32: return AMDGPU::V_CVT_F32_I32_e64;
5418 case AMDGPU::S_CVT_F32_U32: return AMDGPU::V_CVT_F32_U32_e64;
5419 case AMDGPU::S_CVT_I32_F32: return AMDGPU::V_CVT_I32_F32_e64;
5420 case AMDGPU::S_CVT_U32_F32: return AMDGPU::V_CVT_U32_F32_e64;
5421 case AMDGPU::S_CVT_F32_F16: return AMDGPU::V_CVT_F32_F16_t16_e64;
5422 case AMDGPU::S_CVT_HI_F32_F16: return AMDGPU::V_CVT_F32_F16_t16_e64;
5423 case AMDGPU::S_CVT_F16_F32: return AMDGPU::V_CVT_F16_F32_t16_e64;
5424 case AMDGPU::S_CEIL_F32: return AMDGPU::V_CEIL_F32_e64;
5425 case AMDGPU::S_FLOOR_F32: return AMDGPU::V_FLOOR_F32_e64;
5426 case AMDGPU::S_TRUNC_F32: return AMDGPU::V_TRUNC_F32_e64;
5427 case AMDGPU::S_RNDNE_F32: return AMDGPU::V_RNDNE_F32_e64;
5428 case AMDGPU::S_CEIL_F16:
5429 return ST.useRealTrue16Insts() ? AMDGPU::V_CEIL_F16_t16_e64
5430 : AMDGPU::V_CEIL_F16_fake16_e64;
5431 case AMDGPU::S_FLOOR_F16:
5432 return ST.useRealTrue16Insts() ? AMDGPU::V_FLOOR_F16_t16_e64
5433 : AMDGPU::V_FLOOR_F16_fake16_e64;
5434 case AMDGPU::S_TRUNC_F16:
5435 return AMDGPU::V_TRUNC_F16_fake16_e64;
5436 case AMDGPU::S_RNDNE_F16:
5437 return AMDGPU::V_RNDNE_F16_fake16_e64;
5438 case AMDGPU::S_ADD_F32: return AMDGPU::V_ADD_F32_e64;
5439 case AMDGPU::S_SUB_F32: return AMDGPU::V_SUB_F32_e64;
5440 case AMDGPU::S_MIN_F32: return AMDGPU::V_MIN_F32_e64;
5441 case AMDGPU::S_MAX_F32: return AMDGPU::V_MAX_F32_e64;
5442 case AMDGPU::S_MINIMUM_F32: return AMDGPU::V_MINIMUM_F32_e64;
5443 case AMDGPU::S_MAXIMUM_F32: return AMDGPU::V_MAXIMUM_F32_e64;
5444 case AMDGPU::S_MUL_F32: return AMDGPU::V_MUL_F32_e64;
5445 case AMDGPU::S_ADD_F16: return AMDGPU::V_ADD_F16_fake16_e64;
5446 case AMDGPU::S_SUB_F16: return AMDGPU::V_SUB_F16_fake16_e64;
5447 case AMDGPU::S_MIN_F16: return AMDGPU::V_MIN_F16_fake16_e64;
5448 case AMDGPU::S_MAX_F16: return AMDGPU::V_MAX_F16_fake16_e64;
5449 case AMDGPU::S_MINIMUM_F16: return AMDGPU::V_MINIMUM_F16_e64;
5450 case AMDGPU::S_MAXIMUM_F16: return AMDGPU::V_MAXIMUM_F16_e64;
5451 case AMDGPU::S_MUL_F16: return AMDGPU::V_MUL_F16_fake16_e64;
5452 case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64;
5453 case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64;
5454 case AMDGPU::S_FMAC_F16: return AMDGPU::V_FMAC_F16_t16_e64;
5455 case AMDGPU::S_FMAMK_F32: return AMDGPU::V_FMAMK_F32;
5456 case AMDGPU::S_FMAAK_F32: return AMDGPU::V_FMAAK_F32;
5457 case AMDGPU::S_CMP_LT_F32: return AMDGPU::V_CMP_LT_F32_e64;
5458 case AMDGPU::S_CMP_EQ_F32: return AMDGPU::V_CMP_EQ_F32_e64;
5459 case AMDGPU::S_CMP_LE_F32: return AMDGPU::V_CMP_LE_F32_e64;
5460 case AMDGPU::S_CMP_GT_F32: return AMDGPU::V_CMP_GT_F32_e64;
5461 case AMDGPU::S_CMP_LG_F32: return AMDGPU::V_CMP_LG_F32_e64;
5462 case AMDGPU::S_CMP_GE_F32: return AMDGPU::V_CMP_GE_F32_e64;
5463 case AMDGPU::S_CMP_O_F32: return AMDGPU::V_CMP_O_F32_e64;
5464 case AMDGPU::S_CMP_U_F32: return AMDGPU::V_CMP_U_F32_e64;
5465 case AMDGPU::S_CMP_NGE_F32: return AMDGPU::V_CMP_NGE_F32_e64;
5466 case AMDGPU::S_CMP_NLG_F32: return AMDGPU::V_CMP_NLG_F32_e64;
5467 case AMDGPU::S_CMP_NGT_F32: return AMDGPU::V_CMP_NGT_F32_e64;
5468 case AMDGPU::S_CMP_NLE_F32: return AMDGPU::V_CMP_NLE_F32_e64;
5469 case AMDGPU::S_CMP_NEQ_F32: return AMDGPU::V_CMP_NEQ_F32_e64;
5470 case AMDGPU::S_CMP_NLT_F32: return AMDGPU::V_CMP_NLT_F32_e64;
5471 case AMDGPU::S_CMP_LT_F16: return AMDGPU::V_CMP_LT_F16_t16_e64;
5472 case AMDGPU::S_CMP_EQ_F16: return AMDGPU::V_CMP_EQ_F16_t16_e64;
5473 case AMDGPU::S_CMP_LE_F16: return AMDGPU::V_CMP_LE_F16_t16_e64;
5474 case AMDGPU::S_CMP_GT_F16: return AMDGPU::V_CMP_GT_F16_t16_e64;
5475 case AMDGPU::S_CMP_LG_F16: return AMDGPU::V_CMP_LG_F16_t16_e64;
5476 case AMDGPU::S_CMP_GE_F16: return AMDGPU::V_CMP_GE_F16_t16_e64;
5477 case AMDGPU::S_CMP_O_F16: return AMDGPU::V_CMP_O_F16_t16_e64;
5478 case AMDGPU::S_CMP_U_F16: return AMDGPU::V_CMP_U_F16_t16_e64;
5479 case AMDGPU::S_CMP_NGE_F16: return AMDGPU::V_CMP_NGE_F16_t16_e64;
5480 case AMDGPU::S_CMP_NLG_F16: return AMDGPU::V_CMP_NLG_F16_t16_e64;
5481 case AMDGPU::S_CMP_NGT_F16: return AMDGPU::V_CMP_NGT_F16_t16_e64;
5482 case AMDGPU::S_CMP_NLE_F16: return AMDGPU::V_CMP_NLE_F16_t16_e64;
5483 case AMDGPU::S_CMP_NEQ_F16: return AMDGPU::V_CMP_NEQ_F16_t16_e64;
5484 case AMDGPU::S_CMP_NLT_F16: return AMDGPU::V_CMP_NLT_F16_t16_e64;
5485 case AMDGPU::V_S_EXP_F32_e64: return AMDGPU::V_EXP_F32_e64;
5486 case AMDGPU::V_S_EXP_F16_e64: return AMDGPU::V_EXP_F16_fake16_e64;
5487 case AMDGPU::V_S_LOG_F32_e64: return AMDGPU::V_LOG_F32_e64;
5488 case AMDGPU::V_S_LOG_F16_e64: return AMDGPU::V_LOG_F16_fake16_e64;
5489 case AMDGPU::V_S_RCP_F32_e64: return AMDGPU::V_RCP_F32_e64;
5490 case AMDGPU::V_S_RCP_F16_e64: return AMDGPU::V_RCP_F16_fake16_e64;
5491 case AMDGPU::V_S_RSQ_F32_e64: return AMDGPU::V_RSQ_F32_e64;
5492 case AMDGPU::V_S_RSQ_F16_e64: return AMDGPU::V_RSQ_F16_fake16_e64;
5493 case AMDGPU::V_S_SQRT_F32_e64: return AMDGPU::V_SQRT_F32_e64;
5494 case AMDGPU::V_S_SQRT_F16_e64: return AMDGPU::V_SQRT_F16_fake16_e64;
5495 }
5496 llvm_unreachable(
5497 "Unexpected scalar opcode without corresponding vector one!");
5498}
5499
5500// clang-format on
5501
5502void SIInstrInfo::insertScratchExecCopy(MachineFunction &MF,
5503 MachineBasicBlock &MBB,
5504 MachineBasicBlock::iterator MBBI,
5505 const DebugLoc &DL, Register Reg,
5506 bool IsSCCLive,
5507 SlotIndexes *Indexes) const {
5508 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
5509 const SIInstrInfo *TII = ST.getInstrInfo();
5510 bool IsWave32 = ST.isWave32();
5511 if (IsSCCLive) {
5512 // Insert two move instructions, one to save the original value of EXEC and
5513 // the other to turn on all bits in EXEC. This is required as we can't use
5514 // the single instruction S_OR_SAVEEXEC that clobbers SCC.
5515 unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
5516 MCRegister Exec = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
5517 auto StoreExecMI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: MovOpc), DestReg: Reg)
5518 .addReg(RegNo: Exec, flags: RegState::Kill);
5519 auto FlipExecMI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: MovOpc), DestReg: Exec).addImm(Val: -1);
5520 if (Indexes) {
5521 Indexes->insertMachineInstrInMaps(MI&: *StoreExecMI);
5522 Indexes->insertMachineInstrInMaps(MI&: *FlipExecMI);
5523 }
5524 } else {
5525 const unsigned OrSaveExec =
5526 IsWave32 ? AMDGPU::S_OR_SAVEEXEC_B32 : AMDGPU::S_OR_SAVEEXEC_B64;
5527 auto SaveExec =
5528 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: OrSaveExec), DestReg: Reg).addImm(Val: -1);
5529 SaveExec->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
5530 if (Indexes)
5531 Indexes->insertMachineInstrInMaps(MI&: *SaveExec);
5532 }
5533}
5534
5535void SIInstrInfo::restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
5536 MachineBasicBlock::iterator MBBI,
5537 const DebugLoc &DL, Register Reg,
5538 SlotIndexes *Indexes) const {
5539 unsigned ExecMov = isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
5540 MCRegister Exec = isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
5541 auto ExecRestoreMI =
5542 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: ExecMov), DestReg: Exec).addReg(RegNo: Reg, flags: RegState::Kill);
5543 if (Indexes)
5544 Indexes->insertMachineInstrInMaps(MI&: *ExecRestoreMI);
5545}
5546
5547static const TargetRegisterClass *
5548adjustAllocatableRegClass(const GCNSubtarget &ST, const SIRegisterInfo &RI,
5549 const MachineRegisterInfo &MRI,
5550 const MCInstrDesc &TID, unsigned RCID,
5551 bool IsAllocatable) {
5552 if ((IsAllocatable || !ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
5553 (((TID.mayLoad() || TID.mayStore()) &&
5554 !(TID.TSFlags & SIInstrFlags::Spill)) ||
5555 (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::MIMG)))) {
5556 switch (RCID) {
5557 case AMDGPU::AV_32RegClassID:
5558 RCID = AMDGPU::VGPR_32RegClassID;
5559 break;
5560 case AMDGPU::AV_64RegClassID:
5561 RCID = AMDGPU::VReg_64RegClassID;
5562 break;
5563 case AMDGPU::AV_96RegClassID:
5564 RCID = AMDGPU::VReg_96RegClassID;
5565 break;
5566 case AMDGPU::AV_128RegClassID:
5567 RCID = AMDGPU::VReg_128RegClassID;
5568 break;
5569 case AMDGPU::AV_160RegClassID:
5570 RCID = AMDGPU::VReg_160RegClassID;
5571 break;
5572 case AMDGPU::AV_512RegClassID:
5573 RCID = AMDGPU::VReg_512RegClassID;
5574 break;
5575 default:
5576 break;
5577 }
5578 }
5579
5580 return RI.getProperlyAlignedRC(RC: RI.getRegClass(RCID));
5581}
5582
5583const TargetRegisterClass *SIInstrInfo::getRegClass(const MCInstrDesc &TID,
5584 unsigned OpNum, const TargetRegisterInfo *TRI,
5585 const MachineFunction &MF)
5586 const {
5587 if (OpNum >= TID.getNumOperands())
5588 return nullptr;
5589 auto RegClass = TID.operands()[OpNum].RegClass;
5590 bool IsAllocatable = false;
5591 if (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::FLAT)) {
5592 // vdst and vdata should be both VGPR or AGPR, same for the DS instructions
5593 // with two data operands. Request register class constrained to VGPR only
5594 // of both operands present as Machine Copy Propagation can not check this
5595 // constraint and possibly other passes too.
5596 //
5597 // The check is limited to FLAT and DS because atomics in non-flat encoding
5598 // have their vdst and vdata tied to be the same register.
5599 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opcode: TID.Opcode,
5600 NamedIdx: AMDGPU::OpName::vdst);
5601 const int DataIdx = AMDGPU::getNamedOperandIdx(Opcode: TID.Opcode,
5602 NamedIdx: (TID.TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0
5603 : AMDGPU::OpName::vdata);
5604 if (DataIdx != -1) {
5605 IsAllocatable = VDstIdx != -1 || AMDGPU::hasNamedOperand(
5606 Opcode: TID.Opcode, NamedIdx: AMDGPU::OpName::data1);
5607 }
5608 }
5609 return adjustAllocatableRegClass(ST, RI, MRI: MF.getRegInfo(), TID, RCID: RegClass,
5610 IsAllocatable);
5611}
5612
5613const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
5614 unsigned OpNo) const {
5615 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
5616 const MCInstrDesc &Desc = get(Opcode: MI.getOpcode());
5617 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
5618 Desc.operands()[OpNo].RegClass == -1) {
5619 Register Reg = MI.getOperand(i: OpNo).getReg();
5620
5621 if (Reg.isVirtual())
5622 return MRI.getRegClass(Reg);
5623 return RI.getPhysRegBaseClass(Reg);
5624 }
5625
5626 unsigned RCID = Desc.operands()[OpNo].RegClass;
5627 return adjustAllocatableRegClass(ST, RI, MRI, TID: Desc, RCID, IsAllocatable: true);
5628}
5629
5630void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
5631 MachineBasicBlock::iterator I = MI;
5632 MachineBasicBlock *MBB = MI.getParent();
5633 MachineOperand &MO = MI.getOperand(i: OpIdx);
5634 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
5635 unsigned RCID = get(Opcode: MI.getOpcode()).operands()[OpIdx].RegClass;
5636 const TargetRegisterClass *RC = RI.getRegClass(RCID);
5637 unsigned Size = RI.getRegSizeInBits(RC: *RC);
5638 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32;
5639 if (MO.isReg())
5640 Opcode = AMDGPU::COPY;
5641 else if (RI.isSGPRClass(RC))
5642 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
5643
5644 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(SRC: RC);
5645 Register Reg = MRI.createVirtualRegister(RegClass: VRC);
5646 DebugLoc DL = MBB->findDebugLoc(MBBI: I);
5647 BuildMI(BB&: *MI.getParent(), I, MIMD: DL, MCID: get(Opcode), DestReg: Reg).add(MO);
5648 MO.ChangeToRegister(Reg, isDef: false);
5649}
5650
5651unsigned SIInstrInfo::buildExtractSubReg(
5652 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
5653 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
5654 unsigned SubIdx, const TargetRegisterClass *SubRC) const {
5655 MachineBasicBlock *MBB = MI->getParent();
5656 DebugLoc DL = MI->getDebugLoc();
5657 Register SubReg = MRI.createVirtualRegister(RegClass: SubRC);
5658
5659 unsigned NewSubIdx = RI.composeSubRegIndices(a: SuperReg.getSubReg(), b: SubIdx);
5660 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: SubReg)
5661 .addReg(RegNo: SuperReg.getReg(), flags: 0, SubReg: NewSubIdx);
5662 return SubReg;
5663}
5664
5665MachineOperand SIInstrInfo::buildExtractSubRegOrImm(
5666 MachineBasicBlock::iterator MII, MachineRegisterInfo &MRI,
5667 const MachineOperand &Op, const TargetRegisterClass *SuperRC,
5668 unsigned SubIdx, const TargetRegisterClass *SubRC) const {
5669 if (Op.isImm()) {
5670 if (SubIdx == AMDGPU::sub0)
5671 return MachineOperand::CreateImm(Val: static_cast<int32_t>(Op.getImm()));
5672 if (SubIdx == AMDGPU::sub1)
5673 return MachineOperand::CreateImm(Val: static_cast<int32_t>(Op.getImm() >> 32));
5674
5675 llvm_unreachable("Unhandled register index for immediate");
5676 }
5677
5678 unsigned SubReg = buildExtractSubReg(MI: MII, MRI, SuperReg: Op, SuperRC,
5679 SubIdx, SubRC);
5680 return MachineOperand::CreateReg(Reg: SubReg, isDef: false);
5681}
5682
5683// Change the order of operands from (0, 1, 2) to (0, 2, 1)
5684void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
5685 assert(Inst.getNumExplicitOperands() == 3);
5686 MachineOperand Op1 = Inst.getOperand(i: 1);
5687 Inst.removeOperand(OpNo: 1);
5688 Inst.addOperand(Op: Op1);
5689}
5690
5691bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
5692 const MCOperandInfo &OpInfo,
5693 const MachineOperand &MO) const {
5694 if (!MO.isReg())
5695 return false;
5696
5697 Register Reg = MO.getReg();
5698
5699 const TargetRegisterClass *DRC = RI.getRegClass(RCID: OpInfo.RegClass);
5700 if (Reg.isPhysical())
5701 return DRC->contains(Reg);
5702
5703 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
5704
5705 if (MO.getSubReg()) {
5706 const MachineFunction *MF = MO.getParent()->getParent()->getParent();
5707 const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, MF: *MF);
5708 if (!SuperRC)
5709 return false;
5710
5711 DRC = RI.getMatchingSuperRegClass(A: SuperRC, B: DRC, Idx: MO.getSubReg());
5712 if (!DRC)
5713 return false;
5714 }
5715 return RC->hasSuperClassEq(RC: DRC);
5716}
5717
5718bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI,
5719 const MCOperandInfo &OpInfo,
5720 const MachineOperand &MO) const {
5721 if (MO.isReg())
5722 return isLegalRegOperand(MRI, OpInfo, MO);
5723
5724 // Handle non-register types that are treated like immediates.
5725 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
5726 return true;
5727}
5728
5729bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
5730 const MachineOperand *MO) const {
5731 const MachineFunction &MF = *MI.getParent()->getParent();
5732 const MachineRegisterInfo &MRI = MF.getRegInfo();
5733 const MCInstrDesc &InstDesc = MI.getDesc();
5734 const MCOperandInfo &OpInfo = InstDesc.operands()[OpIdx];
5735 const TargetRegisterClass *DefinedRC =
5736 OpInfo.RegClass != -1 ? RI.getRegClass(RCID: OpInfo.RegClass) : nullptr;
5737 if (!MO)
5738 MO = &MI.getOperand(i: OpIdx);
5739
5740 int ConstantBusLimit = ST.getConstantBusLimit(Opcode: MI.getOpcode());
5741 int LiteralLimit = !isVOP3(MI) || ST.hasVOP3Literal() ? 1 : 0;
5742 if (isVALU(MI) && usesConstantBus(MRI, MO: *MO, OpInfo)) {
5743 if (!MO->isReg() && !isInlineConstant(MO: *MO, OpInfo) && !LiteralLimit--)
5744 return false;
5745
5746 SmallDenseSet<RegSubRegPair> SGPRsUsed;
5747 if (MO->isReg())
5748 SGPRsUsed.insert(V: RegSubRegPair(MO->getReg(), MO->getSubReg()));
5749
5750 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
5751 if (i == OpIdx)
5752 continue;
5753 const MachineOperand &Op = MI.getOperand(i);
5754 if (Op.isReg()) {
5755 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
5756 if (!SGPRsUsed.count(V: SGPR) &&
5757 // FIXME: This can access off the end of the operands() array.
5758 usesConstantBus(MRI, MO: Op, OpInfo: InstDesc.operands().begin()[i])) {
5759 if (--ConstantBusLimit <= 0)
5760 return false;
5761 SGPRsUsed.insert(V: SGPR);
5762 }
5763 } else if (AMDGPU::isSISrcOperand(Desc: InstDesc, OpNo: i) &&
5764 !isInlineConstant(MO: Op, OpInfo: InstDesc.operands()[i])) {
5765 if (!LiteralLimit--)
5766 return false;
5767 if (--ConstantBusLimit <= 0)
5768 return false;
5769 }
5770 }
5771 } else if (ST.hasNoF16PseudoScalarTransInlineConstants() && !MO->isReg() &&
5772 isF16PseudoScalarTrans(Opcode: MI.getOpcode()) &&
5773 isInlineConstant(MO: *MO, OpInfo)) {
5774 return false;
5775 }
5776
5777 if (MO->isReg()) {
5778 if (!DefinedRC)
5779 return OpInfo.OperandType == MCOI::OPERAND_UNKNOWN;
5780 if (!isLegalRegOperand(MRI, OpInfo, MO: *MO))
5781 return false;
5782 bool IsAGPR = RI.isAGPR(MRI, Reg: MO->getReg());
5783 if (IsAGPR && !ST.hasMAIInsts())
5784 return false;
5785 unsigned Opc = MI.getOpcode();
5786 if (IsAGPR &&
5787 (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
5788 (MI.mayLoad() || MI.mayStore() || isDS(Opcode: Opc) || isMIMG(Opcode: Opc)))
5789 return false;
5790 // Atomics should have both vdst and vdata either vgpr or agpr.
5791 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::vdst);
5792 const int DataIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc,
5793 NamedIdx: isDS(Opcode: Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata);
5794 if ((int)OpIdx == VDstIdx && DataIdx != -1 &&
5795 MI.getOperand(i: DataIdx).isReg() &&
5796 RI.isAGPR(MRI, Reg: MI.getOperand(i: DataIdx).getReg()) != IsAGPR)
5797 return false;
5798 if ((int)OpIdx == DataIdx) {
5799 if (VDstIdx != -1 &&
5800 RI.isAGPR(MRI, Reg: MI.getOperand(i: VDstIdx).getReg()) != IsAGPR)
5801 return false;
5802 // DS instructions with 2 src operands also must have tied RC.
5803 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc,
5804 NamedIdx: AMDGPU::OpName::data1);
5805 if (Data1Idx != -1 && MI.getOperand(i: Data1Idx).isReg() &&
5806 RI.isAGPR(MRI, Reg: MI.getOperand(i: Data1Idx).getReg()) != IsAGPR)
5807 return false;
5808 }
5809 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() &&
5810 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::src0) &&
5811 RI.isSGPRReg(MRI, Reg: MO->getReg()))
5812 return false;
5813 return true;
5814 }
5815
5816 if (MO->isImm()) {
5817 uint64_t Imm = MO->getImm();
5818 bool Is64BitFPOp = OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_FP64;
5819 bool Is64BitOp = Is64BitFPOp ||
5820 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64 ||
5821 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT32 ||
5822 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32;
5823 if (Is64BitOp &&
5824 !AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: ST.hasInv2PiInlineImm())) {
5825 if (!AMDGPU::isValid32BitLiteral(Val: Imm, IsFP64: Is64BitFPOp))
5826 return false;
5827
5828 // FIXME: We can use sign extended 64-bit literals, but only for signed
5829 // operands. At the moment we do not know if an operand is signed.
5830 // Such operand will be encoded as its low 32 bits and then either
5831 // correctly sign extended or incorrectly zero extended by HW.
5832 if (!Is64BitFPOp && (int32_t)Imm < 0)
5833 return false;
5834 }
5835 }
5836
5837 // Handle non-register types that are treated like immediates.
5838 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
5839
5840 if (!DefinedRC) {
5841 // This operand expects an immediate.
5842 return true;
5843 }
5844
5845 return isImmOperandLegal(MI, OpNo: OpIdx, MO: *MO);
5846}
5847
5848void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
5849 MachineInstr &MI) const {
5850 unsigned Opc = MI.getOpcode();
5851 const MCInstrDesc &InstrDesc = get(Opcode: Opc);
5852
5853 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::src0);
5854 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5855
5856 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::src1);
5857 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
5858
5859 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
5860 // we need to only have one constant bus use before GFX10.
5861 bool HasImplicitSGPR = findImplicitSGPRRead(MI);
5862 if (HasImplicitSGPR && ST.getConstantBusLimit(Opcode: Opc) <= 1 && Src0.isReg() &&
5863 RI.isSGPRReg(MRI, Reg: Src0.getReg()))
5864 legalizeOpWithMove(MI, OpIdx: Src0Idx);
5865
5866 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
5867 // both the value to write (src0) and lane select (src1). Fix up non-SGPR
5868 // src0/src1 with V_READFIRSTLANE.
5869 if (Opc == AMDGPU::V_WRITELANE_B32) {
5870 const DebugLoc &DL = MI.getDebugLoc();
5871 if (Src0.isReg() && RI.isVGPR(MRI, Reg: Src0.getReg())) {
5872 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5873 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
5874 .add(MO: Src0);
5875 Src0.ChangeToRegister(Reg, isDef: false);
5876 }
5877 if (Src1.isReg() && RI.isVGPR(MRI, Reg: Src1.getReg())) {
5878 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5879 const DebugLoc &DL = MI.getDebugLoc();
5880 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
5881 .add(MO: Src1);
5882 Src1.ChangeToRegister(Reg, isDef: false);
5883 }
5884 return;
5885 }
5886
5887 // No VOP2 instructions support AGPRs.
5888 if (Src0.isReg() && RI.isAGPR(MRI, Reg: Src0.getReg()))
5889 legalizeOpWithMove(MI, OpIdx: Src0Idx);
5890
5891 if (Src1.isReg() && RI.isAGPR(MRI, Reg: Src1.getReg()))
5892 legalizeOpWithMove(MI, OpIdx: Src1Idx);
5893
5894 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2.
5895 if (Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F16_e32) {
5896 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::src2);
5897 if (!RI.isVGPR(MRI, Reg: MI.getOperand(i: Src2Idx).getReg()))
5898 legalizeOpWithMove(MI, OpIdx: Src2Idx);
5899 }
5900
5901 // VOP2 src0 instructions support all operand types, so we don't need to check
5902 // their legality. If src1 is already legal, we don't need to do anything.
5903 if (isLegalRegOperand(MRI, OpInfo: InstrDesc.operands()[Src1Idx], MO: Src1))
5904 return;
5905
5906 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
5907 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
5908 // select is uniform.
5909 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
5910 RI.isVGPR(MRI, Reg: Src1.getReg())) {
5911 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5912 const DebugLoc &DL = MI.getDebugLoc();
5913 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
5914 .add(MO: Src1);
5915 Src1.ChangeToRegister(Reg, isDef: false);
5916 return;
5917 }
5918
5919 // We do not use commuteInstruction here because it is too aggressive and will
5920 // commute if it is possible. We only want to commute here if it improves
5921 // legality. This can be called a fairly large number of times so don't waste
5922 // compile time pointlessly swapping and checking legality again.
5923 if (HasImplicitSGPR || !MI.isCommutable()) {
5924 legalizeOpWithMove(MI, OpIdx: Src1Idx);
5925 return;
5926 }
5927
5928 // If src0 can be used as src1, commuting will make the operands legal.
5929 // Otherwise we have to give up and insert a move.
5930 //
5931 // TODO: Other immediate-like operand kinds could be commuted if there was a
5932 // MachineOperand::ChangeTo* for them.
5933 if ((!Src1.isImm() && !Src1.isReg()) ||
5934 !isLegalRegOperand(MRI, OpInfo: InstrDesc.operands()[Src1Idx], MO: Src0)) {
5935 legalizeOpWithMove(MI, OpIdx: Src1Idx);
5936 return;
5937 }
5938
5939 int CommutedOpc = commuteOpcode(MI);
5940 if (CommutedOpc == -1) {
5941 legalizeOpWithMove(MI, OpIdx: Src1Idx);
5942 return;
5943 }
5944
5945 MI.setDesc(get(Opcode: CommutedOpc));
5946
5947 Register Src0Reg = Src0.getReg();
5948 unsigned Src0SubReg = Src0.getSubReg();
5949 bool Src0Kill = Src0.isKill();
5950
5951 if (Src1.isImm())
5952 Src0.ChangeToImmediate(ImmVal: Src1.getImm());
5953 else if (Src1.isReg()) {
5954 Src0.ChangeToRegister(Reg: Src1.getReg(), isDef: false, isImp: false, isKill: Src1.isKill());
5955 Src0.setSubReg(Src1.getSubReg());
5956 } else
5957 llvm_unreachable("Should only have register or immediate operands");
5958
5959 Src1.ChangeToRegister(Reg: Src0Reg, isDef: false, isImp: false, isKill: Src0Kill);
5960 Src1.setSubReg(Src0SubReg);
5961 fixImplicitOperands(MI);
5962}
5963
5964// Legalize VOP3 operands. All operand types are supported for any operand
5965// but only one literal constant and only starting from GFX10.
5966void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
5967 MachineInstr &MI) const {
5968 unsigned Opc = MI.getOpcode();
5969
5970 int VOP3Idx[3] = {
5971 AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::src0),
5972 AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::src1),
5973 AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::src2)
5974 };
5975
5976 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 ||
5977 Opc == AMDGPU::V_PERMLANEX16_B32_e64) {
5978 // src1 and src2 must be scalar
5979 MachineOperand &Src1 = MI.getOperand(i: VOP3Idx[1]);
5980 MachineOperand &Src2 = MI.getOperand(i: VOP3Idx[2]);
5981 const DebugLoc &DL = MI.getDebugLoc();
5982 if (Src1.isReg() && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src1.getReg()))) {
5983 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5984 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
5985 .add(MO: Src1);
5986 Src1.ChangeToRegister(Reg, isDef: false);
5987 }
5988 if (Src2.isReg() && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src2.getReg()))) {
5989 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5990 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
5991 .add(MO: Src2);
5992 Src2.ChangeToRegister(Reg, isDef: false);
5993 }
5994 }
5995
5996 // Find the one SGPR operand we are allowed to use.
5997 int ConstantBusLimit = ST.getConstantBusLimit(Opcode: Opc);
5998 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
5999 SmallDenseSet<unsigned> SGPRsUsed;
6000 Register SGPRReg = findUsedSGPR(MI, OpIndices: VOP3Idx);
6001 if (SGPRReg) {
6002 SGPRsUsed.insert(V: SGPRReg);
6003 --ConstantBusLimit;
6004 }
6005
6006 for (int Idx : VOP3Idx) {
6007 if (Idx == -1)
6008 break;
6009 MachineOperand &MO = MI.getOperand(i: Idx);
6010
6011 if (!MO.isReg()) {
6012 if (isInlineConstant(MO, OpInfo: get(Opcode: Opc).operands()[Idx]))
6013 continue;
6014
6015 if (LiteralLimit > 0 && ConstantBusLimit > 0) {
6016 --LiteralLimit;
6017 --ConstantBusLimit;
6018 continue;
6019 }
6020
6021 --LiteralLimit;
6022 --ConstantBusLimit;
6023 legalizeOpWithMove(MI, OpIdx: Idx);
6024 continue;
6025 }
6026
6027 if (RI.hasAGPRs(RC: RI.getRegClassForReg(MRI, Reg: MO.getReg())) &&
6028 !isOperandLegal(MI, OpIdx: Idx, MO: &MO)) {
6029 legalizeOpWithMove(MI, OpIdx: Idx);
6030 continue;
6031 }
6032
6033 if (!RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg: MO.getReg())))
6034 continue; // VGPRs are legal
6035
6036 // We can use one SGPR in each VOP3 instruction prior to GFX10
6037 // and two starting from GFX10.
6038 if (SGPRsUsed.count(V: MO.getReg()))
6039 continue;
6040 if (ConstantBusLimit > 0) {
6041 SGPRsUsed.insert(V: MO.getReg());
6042 --ConstantBusLimit;
6043 continue;
6044 }
6045
6046 // If we make it this far, then the operand is not legal and we must
6047 // legalize it.
6048 legalizeOpWithMove(MI, OpIdx: Idx);
6049 }
6050
6051 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2 tied to vdst.
6052 if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64) &&
6053 !RI.isVGPR(MRI, Reg: MI.getOperand(i: VOP3Idx[2]).getReg()))
6054 legalizeOpWithMove(MI, OpIdx: VOP3Idx[2]);
6055}
6056
6057Register SIInstrInfo::readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,
6058 MachineRegisterInfo &MRI) const {
6059 const TargetRegisterClass *VRC = MRI.getRegClass(Reg: SrcReg);
6060 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
6061 Register DstReg = MRI.createVirtualRegister(RegClass: SRC);
6062 unsigned SubRegs = RI.getRegSizeInBits(RC: *VRC) / 32;
6063
6064 if (RI.hasAGPRs(RC: VRC)) {
6065 VRC = RI.getEquivalentVGPRClass(SRC: VRC);
6066 Register NewSrcReg = MRI.createVirtualRegister(RegClass: VRC);
6067 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
6068 MCID: get(Opcode: TargetOpcode::COPY), DestReg: NewSrcReg)
6069 .addReg(RegNo: SrcReg);
6070 SrcReg = NewSrcReg;
6071 }
6072
6073 if (SubRegs == 1) {
6074 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
6075 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
6076 .addReg(RegNo: SrcReg);
6077 return DstReg;
6078 }
6079
6080 SmallVector<Register, 8> SRegs;
6081 for (unsigned i = 0; i < SubRegs; ++i) {
6082 Register SGPR = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
6083 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
6084 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: SGPR)
6085 .addReg(RegNo: SrcReg, flags: 0, SubReg: RI.getSubRegFromChannel(Channel: i));
6086 SRegs.push_back(Elt: SGPR);
6087 }
6088
6089 MachineInstrBuilder MIB =
6090 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
6091 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
6092 for (unsigned i = 0; i < SubRegs; ++i) {
6093 MIB.addReg(RegNo: SRegs[i]);
6094 MIB.addImm(Val: RI.getSubRegFromChannel(Channel: i));
6095 }
6096 return DstReg;
6097}
6098
6099void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,
6100 MachineInstr &MI) const {
6101
6102 // If the pointer is store in VGPRs, then we need to move them to
6103 // SGPRs using v_readfirstlane. This is safe because we only select
6104 // loads with uniform pointers to SMRD instruction so we know the
6105 // pointer value is uniform.
6106 MachineOperand *SBase = getNamedOperand(MI, OperandName: AMDGPU::OpName::sbase);
6107 if (SBase && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SBase->getReg()))) {
6108 Register SGPR = readlaneVGPRToSGPR(SrcReg: SBase->getReg(), UseMI&: MI, MRI);
6109 SBase->setReg(SGPR);
6110 }
6111 MachineOperand *SOff = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
6112 if (SOff && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SOff->getReg()))) {
6113 Register SGPR = readlaneVGPRToSGPR(SrcReg: SOff->getReg(), UseMI&: MI, MRI);
6114 SOff->setReg(SGPR);
6115 }
6116}
6117
6118bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const {
6119 unsigned Opc = Inst.getOpcode();
6120 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::saddr);
6121 if (OldSAddrIdx < 0)
6122 return false;
6123
6124 assert(isSegmentSpecificFLAT(Inst));
6125
6126 int NewOpc = AMDGPU::getGlobalVaddrOp(Opcode: Opc);
6127 if (NewOpc < 0)
6128 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opcode: Opc);
6129 if (NewOpc < 0)
6130 return false;
6131
6132 MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
6133 MachineOperand &SAddr = Inst.getOperand(i: OldSAddrIdx);
6134 if (RI.isSGPRReg(MRI, Reg: SAddr.getReg()))
6135 return false;
6136
6137 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: NewOpc, NamedIdx: AMDGPU::OpName::vaddr);
6138 if (NewVAddrIdx < 0)
6139 return false;
6140
6141 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::vaddr);
6142
6143 // Check vaddr, it shall be zero or absent.
6144 MachineInstr *VAddrDef = nullptr;
6145 if (OldVAddrIdx >= 0) {
6146 MachineOperand &VAddr = Inst.getOperand(i: OldVAddrIdx);
6147 VAddrDef = MRI.getUniqueVRegDef(Reg: VAddr.getReg());
6148 if (!VAddrDef || VAddrDef->getOpcode() != AMDGPU::V_MOV_B32_e32 ||
6149 !VAddrDef->getOperand(i: 1).isImm() ||
6150 VAddrDef->getOperand(i: 1).getImm() != 0)
6151 return false;
6152 }
6153
6154 const MCInstrDesc &NewDesc = get(Opcode: NewOpc);
6155 Inst.setDesc(NewDesc);
6156
6157 // Callers expect iterator to be valid after this call, so modify the
6158 // instruction in place.
6159 if (OldVAddrIdx == NewVAddrIdx) {
6160 MachineOperand &NewVAddr = Inst.getOperand(i: NewVAddrIdx);
6161 // Clear use list from the old vaddr holding a zero register.
6162 MRI.removeRegOperandFromUseList(MO: &NewVAddr);
6163 MRI.moveOperands(Dst: &NewVAddr, Src: &SAddr, NumOps: 1);
6164 Inst.removeOperand(OpNo: OldSAddrIdx);
6165 // Update the use list with the pointer we have just moved from vaddr to
6166 // saddr position. Otherwise new vaddr will be missing from the use list.
6167 MRI.removeRegOperandFromUseList(MO: &NewVAddr);
6168 MRI.addRegOperandToUseList(MO: &NewVAddr);
6169 } else {
6170 assert(OldSAddrIdx == NewVAddrIdx);
6171
6172 if (OldVAddrIdx >= 0) {
6173 int NewVDstIn = AMDGPU::getNamedOperandIdx(Opcode: NewOpc,
6174 NamedIdx: AMDGPU::OpName::vdst_in);
6175
6176 // removeOperand doesn't try to fixup tied operand indexes at it goes, so
6177 // it asserts. Untie the operands for now and retie them afterwards.
6178 if (NewVDstIn != -1) {
6179 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::vdst_in);
6180 Inst.untieRegOperand(OpIdx: OldVDstIn);
6181 }
6182
6183 Inst.removeOperand(OpNo: OldVAddrIdx);
6184
6185 if (NewVDstIn != -1) {
6186 int NewVDst = AMDGPU::getNamedOperandIdx(Opcode: NewOpc, NamedIdx: AMDGPU::OpName::vdst);
6187 Inst.tieOperands(DefIdx: NewVDst, UseIdx: NewVDstIn);
6188 }
6189 }
6190 }
6191
6192 if (VAddrDef && MRI.use_nodbg_empty(RegNo: VAddrDef->getOperand(i: 0).getReg()))
6193 VAddrDef->eraseFromParent();
6194
6195 return true;
6196}
6197
6198// FIXME: Remove this when SelectionDAG is obsoleted.
6199void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI,
6200 MachineInstr &MI) const {
6201 if (!isSegmentSpecificFLAT(MI))
6202 return;
6203
6204 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence
6205 // thinks they are uniform, so a readfirstlane should be valid.
6206 MachineOperand *SAddr = getNamedOperand(MI, OperandName: AMDGPU::OpName::saddr);
6207 if (!SAddr || RI.isSGPRClass(RC: MRI.getRegClass(Reg: SAddr->getReg())))
6208 return;
6209
6210 if (moveFlatAddrToVGPR(Inst&: MI))
6211 return;
6212
6213 Register ToSGPR = readlaneVGPRToSGPR(SrcReg: SAddr->getReg(), UseMI&: MI, MRI);
6214 SAddr->setReg(ToSGPR);
6215}
6216
6217void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
6218 MachineBasicBlock::iterator I,
6219 const TargetRegisterClass *DstRC,
6220 MachineOperand &Op,
6221 MachineRegisterInfo &MRI,
6222 const DebugLoc &DL) const {
6223 Register OpReg = Op.getReg();
6224 unsigned OpSubReg = Op.getSubReg();
6225
6226 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
6227 RI.getRegClassForReg(MRI, Reg: OpReg), OpSubReg);
6228
6229 // Check if operand is already the correct register class.
6230 if (DstRC == OpRC)
6231 return;
6232
6233 Register DstReg = MRI.createVirtualRegister(RegClass: DstRC);
6234 auto Copy = BuildMI(BB&: InsertMBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: DstReg).add(MO: Op);
6235
6236 Op.setReg(DstReg);
6237 Op.setSubReg(0);
6238
6239 MachineInstr *Def = MRI.getVRegDef(Reg: OpReg);
6240 if (!Def)
6241 return;
6242
6243 // Try to eliminate the copy if it is copying an immediate value.
6244 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass)
6245 foldImmediate(UseMI&: *Copy, DefMI&: *Def, Reg: OpReg, MRI: &MRI);
6246
6247 bool ImpDef = Def->isImplicitDef();
6248 while (!ImpDef && Def && Def->isCopy()) {
6249 if (Def->getOperand(i: 1).getReg().isPhysical())
6250 break;
6251 Def = MRI.getUniqueVRegDef(Reg: Def->getOperand(i: 1).getReg());
6252 ImpDef = Def && Def->isImplicitDef();
6253 }
6254 if (!RI.isSGPRClass(RC: DstRC) && !Copy->readsRegister(Reg: AMDGPU::EXEC, TRI: &RI) &&
6255 !ImpDef)
6256 Copy.addReg(RegNo: AMDGPU::EXEC, flags: RegState::Implicit);
6257}
6258
6259// Emit the actual waterfall loop, executing the wrapped instruction for each
6260// unique value of \p ScalarOps across all lanes. In the best case we execute 1
6261// iteration, in the worst case we execute 64 (once per lane).
6262static void emitLoadScalarOpsFromVGPRLoop(
6263 const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB,
6264 MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
6265 ArrayRef<MachineOperand *> ScalarOps) {
6266 MachineFunction &MF = *OrigBB.getParent();
6267 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
6268 const SIRegisterInfo *TRI = ST.getRegisterInfo();
6269 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6270 unsigned SaveExecOpc =
6271 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
6272 unsigned XorTermOpc =
6273 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term;
6274 unsigned AndOpc =
6275 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
6276 const auto *BoolXExecRC = TRI->getRegClass(RCID: AMDGPU::SReg_1_XEXECRegClassID);
6277
6278 MachineBasicBlock::iterator I = LoopBB.begin();
6279
6280 SmallVector<Register, 8> ReadlanePieces;
6281 Register CondReg;
6282
6283 for (MachineOperand *ScalarOp : ScalarOps) {
6284 unsigned RegSize = TRI->getRegSizeInBits(Reg: ScalarOp->getReg(), MRI);
6285 unsigned NumSubRegs = RegSize / 32;
6286 Register VScalarOp = ScalarOp->getReg();
6287
6288 if (NumSubRegs == 1) {
6289 Register CurReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
6290
6291 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurReg)
6292 .addReg(RegNo: VScalarOp);
6293
6294 Register NewCondReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
6295
6296 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: NewCondReg)
6297 .addReg(RegNo: CurReg)
6298 .addReg(RegNo: VScalarOp);
6299
6300 // Combine the comparison results with AND.
6301 if (!CondReg) // First.
6302 CondReg = NewCondReg;
6303 else { // If not the first, we create an AND.
6304 Register AndReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
6305 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: AndReg)
6306 .addReg(RegNo: CondReg)
6307 .addReg(RegNo: NewCondReg);
6308 CondReg = AndReg;
6309 }
6310
6311 // Update ScalarOp operand to use the SGPR ScalarOp.
6312 ScalarOp->setReg(CurReg);
6313 ScalarOp->setIsKill();
6314 } else {
6315 unsigned VScalarOpUndef = getUndefRegState(B: ScalarOp->isUndef());
6316 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 &&
6317 "Unhandled register size");
6318
6319 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) {
6320 Register CurRegLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
6321 Register CurRegHi = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
6322
6323 // Read the next variant <- also loop target.
6324 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurRegLo)
6325 .addReg(RegNo: VScalarOp, flags: VScalarOpUndef, SubReg: TRI->getSubRegFromChannel(Channel: Idx));
6326
6327 // Read the next variant <- also loop target.
6328 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurRegHi)
6329 .addReg(RegNo: VScalarOp, flags: VScalarOpUndef,
6330 SubReg: TRI->getSubRegFromChannel(Channel: Idx + 1));
6331
6332 ReadlanePieces.push_back(Elt: CurRegLo);
6333 ReadlanePieces.push_back(Elt: CurRegHi);
6334
6335 // Comparison is to be done as 64-bit.
6336 Register CurReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_64RegClass);
6337 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: CurReg)
6338 .addReg(RegNo: CurRegLo)
6339 .addImm(Val: AMDGPU::sub0)
6340 .addReg(RegNo: CurRegHi)
6341 .addImm(Val: AMDGPU::sub1);
6342
6343 Register NewCondReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
6344 auto Cmp = BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U64_e64),
6345 DestReg: NewCondReg)
6346 .addReg(RegNo: CurReg);
6347 if (NumSubRegs <= 2)
6348 Cmp.addReg(RegNo: VScalarOp);
6349 else
6350 Cmp.addReg(RegNo: VScalarOp, flags: VScalarOpUndef,
6351 SubReg: TRI->getSubRegFromChannel(Channel: Idx, NumRegs: 2));
6352
6353 // Combine the comparison results with AND.
6354 if (!CondReg) // First.
6355 CondReg = NewCondReg;
6356 else { // If not the first, we create an AND.
6357 Register AndReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
6358 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: AndReg)
6359 .addReg(RegNo: CondReg)
6360 .addReg(RegNo: NewCondReg);
6361 CondReg = AndReg;
6362 }
6363 } // End for loop.
6364
6365 auto SScalarOpRC =
6366 TRI->getEquivalentSGPRClass(VRC: MRI.getRegClass(Reg: VScalarOp));
6367 Register SScalarOp = MRI.createVirtualRegister(RegClass: SScalarOpRC);
6368
6369 // Build scalar ScalarOp.
6370 auto Merge =
6371 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: SScalarOp);
6372 unsigned Channel = 0;
6373 for (Register Piece : ReadlanePieces) {
6374 Merge.addReg(RegNo: Piece).addImm(Val: TRI->getSubRegFromChannel(Channel: Channel++));
6375 }
6376
6377 // Update ScalarOp operand to use the SGPR ScalarOp.
6378 ScalarOp->setReg(SScalarOp);
6379 ScalarOp->setIsKill();
6380 }
6381 }
6382
6383 Register SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
6384 MRI.setSimpleHint(VReg: SaveExec, PrefReg: CondReg);
6385
6386 // Update EXEC to matching lanes, saving original to SaveExec.
6387 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: SaveExecOpc), DestReg: SaveExec)
6388 .addReg(RegNo: CondReg, flags: RegState::Kill);
6389
6390 // The original instruction is here; we insert the terminators after it.
6391 I = BodyBB.end();
6392
6393 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
6394 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: XorTermOpc), DestReg: Exec)
6395 .addReg(RegNo: Exec)
6396 .addReg(RegNo: SaveExec);
6397
6398 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::SI_WATERFALL_LOOP)).addMBB(MBB: &LoopBB);
6399}
6400
6401// Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register
6402// with SGPRs by iterating over all unique values across all lanes.
6403// Returns the loop basic block that now contains \p MI.
6404static MachineBasicBlock *
6405loadMBUFScalarOperandsFromVGPR(const SIInstrInfo &TII, MachineInstr &MI,
6406 ArrayRef<MachineOperand *> ScalarOps,
6407 MachineDominatorTree *MDT,
6408 MachineBasicBlock::iterator Begin = nullptr,
6409 MachineBasicBlock::iterator End = nullptr) {
6410 MachineBasicBlock &MBB = *MI.getParent();
6411 MachineFunction &MF = *MBB.getParent();
6412 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
6413 const SIRegisterInfo *TRI = ST.getRegisterInfo();
6414 MachineRegisterInfo &MRI = MF.getRegInfo();
6415 if (!Begin.isValid())
6416 Begin = &MI;
6417 if (!End.isValid()) {
6418 End = &MI;
6419 ++End;
6420 }
6421 const DebugLoc &DL = MI.getDebugLoc();
6422 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6423 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6424 const auto *BoolXExecRC = TRI->getRegClass(RCID: AMDGPU::SReg_1_XEXECRegClassID);
6425
6426 // Save SCC. Waterfall Loop may overwrite SCC.
6427 Register SaveSCCReg;
6428
6429 // FIXME: We should maintain SCC liveness while doing the FixSGPRCopies walk
6430 // rather than unlimited scan everywhere
6431 bool SCCNotDead =
6432 MBB.computeRegisterLiveness(TRI, Reg: AMDGPU::SCC, Before: MI,
6433 Neighborhood: std::numeric_limits<unsigned>::max()) !=
6434 MachineBasicBlock::LQR_Dead;
6435 if (SCCNotDead) {
6436 SaveSCCReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6437 BuildMI(BB&: MBB, I: Begin, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: SaveSCCReg)
6438 .addImm(Val: 1)
6439 .addImm(Val: 0);
6440 }
6441
6442 Register SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
6443
6444 // Save the EXEC mask
6445 BuildMI(BB&: MBB, I: Begin, MIMD: DL, MCID: TII.get(Opcode: MovExecOpc), DestReg: SaveExec).addReg(RegNo: Exec);
6446
6447 // Killed uses in the instruction we are waterfalling around will be
6448 // incorrect due to the added control-flow.
6449 MachineBasicBlock::iterator AfterMI = MI;
6450 ++AfterMI;
6451 for (auto I = Begin; I != AfterMI; I++) {
6452 for (auto &MO : I->all_uses())
6453 MRI.clearKillFlags(Reg: MO.getReg());
6454 }
6455
6456 // To insert the loop we need to split the block. Move everything after this
6457 // point to a new block, and insert a new empty block between the two.
6458 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
6459 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock();
6460 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
6461 MachineFunction::iterator MBBI(MBB);
6462 ++MBBI;
6463
6464 MF.insert(MBBI, MBB: LoopBB);
6465 MF.insert(MBBI, MBB: BodyBB);
6466 MF.insert(MBBI, MBB: RemainderBB);
6467
6468 LoopBB->addSuccessor(Succ: BodyBB);
6469 BodyBB->addSuccessor(Succ: LoopBB);
6470 BodyBB->addSuccessor(Succ: RemainderBB);
6471
6472 // Move Begin to MI to the BodyBB, and the remainder of the block to
6473 // RemainderBB.
6474 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
6475 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: End, To: MBB.end());
6476 BodyBB->splice(Where: BodyBB->begin(), Other: &MBB, From: Begin, To: MBB.end());
6477
6478 MBB.addSuccessor(Succ: LoopBB);
6479
6480 // Update dominators. We know that MBB immediately dominates LoopBB, that
6481 // LoopBB immediately dominates BodyBB, and BodyBB immediately dominates
6482 // RemainderBB. RemainderBB immediately dominates all of the successors
6483 // transferred to it from MBB that MBB used to properly dominate.
6484 if (MDT) {
6485 MDT->addNewBlock(BB: LoopBB, DomBB: &MBB);
6486 MDT->addNewBlock(BB: BodyBB, DomBB: LoopBB);
6487 MDT->addNewBlock(BB: RemainderBB, DomBB: BodyBB);
6488 for (auto &Succ : RemainderBB->successors()) {
6489 if (MDT->properlyDominates(A: &MBB, B: Succ)) {
6490 MDT->changeImmediateDominator(N: Succ, NewIDom: RemainderBB);
6491 }
6492 }
6493 }
6494
6495 emitLoadScalarOpsFromVGPRLoop(TII, MRI, OrigBB&: MBB, LoopBB&: *LoopBB, BodyBB&: *BodyBB, DL, ScalarOps);
6496
6497 MachineBasicBlock::iterator First = RemainderBB->begin();
6498 // Restore SCC
6499 if (SCCNotDead) {
6500 BuildMI(BB&: *RemainderBB, I: First, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CMP_LG_U32))
6501 .addReg(RegNo: SaveSCCReg, flags: RegState::Kill)
6502 .addImm(Val: 0);
6503 }
6504
6505 // Restore the EXEC mask
6506 BuildMI(BB&: *RemainderBB, I: First, MIMD: DL, MCID: TII.get(Opcode: MovExecOpc), DestReg: Exec).addReg(RegNo: SaveExec);
6507 return BodyBB;
6508}
6509
6510// Extract pointer from Rsrc and return a zero-value Rsrc replacement.
6511static std::tuple<unsigned, unsigned>
6512extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) {
6513 MachineBasicBlock &MBB = *MI.getParent();
6514 MachineFunction &MF = *MBB.getParent();
6515 MachineRegisterInfo &MRI = MF.getRegInfo();
6516
6517 // Extract the ptr from the resource descriptor.
6518 unsigned RsrcPtr =
6519 TII.buildExtractSubReg(MI, MRI, SuperReg: Rsrc, SuperRC: &AMDGPU::VReg_128RegClass,
6520 SubIdx: AMDGPU::sub0_sub1, SubRC: &AMDGPU::VReg_64RegClass);
6521
6522 // Create an empty resource descriptor
6523 Register Zero64 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6524 Register SRsrcFormatLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
6525 Register SRsrcFormatHi = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
6526 Register NewSRsrc = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_128RegClass);
6527 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
6528
6529 // Zero64 = 0
6530 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B64), DestReg: Zero64)
6531 .addImm(Val: 0);
6532
6533 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
6534 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: SRsrcFormatLo)
6535 .addImm(Val: RsrcDataFormat & 0xFFFFFFFF);
6536
6537 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
6538 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: SRsrcFormatHi)
6539 .addImm(Val: RsrcDataFormat >> 32);
6540
6541 // NewSRsrc = {Zero64, SRsrcFormat}
6542 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewSRsrc)
6543 .addReg(RegNo: Zero64)
6544 .addImm(Val: AMDGPU::sub0_sub1)
6545 .addReg(RegNo: SRsrcFormatLo)
6546 .addImm(Val: AMDGPU::sub2)
6547 .addReg(RegNo: SRsrcFormatHi)
6548 .addImm(Val: AMDGPU::sub3);
6549
6550 return std::tuple(RsrcPtr, NewSRsrc);
6551}
6552
6553MachineBasicBlock *
6554SIInstrInfo::legalizeOperands(MachineInstr &MI,
6555 MachineDominatorTree *MDT) const {
6556 MachineFunction &MF = *MI.getParent()->getParent();
6557 MachineRegisterInfo &MRI = MF.getRegInfo();
6558 MachineBasicBlock *CreatedBB = nullptr;
6559
6560 // Legalize VOP2
6561 if (isVOP2(MI) || isVOPC(MI)) {
6562 legalizeOperandsVOP2(MRI, MI);
6563 return CreatedBB;
6564 }
6565
6566 // Legalize VOP3
6567 if (isVOP3(MI)) {
6568 legalizeOperandsVOP3(MRI, MI);
6569 return CreatedBB;
6570 }
6571
6572 // Legalize SMRD
6573 if (isSMRD(MI)) {
6574 legalizeOperandsSMRD(MRI, MI);
6575 return CreatedBB;
6576 }
6577
6578 // Legalize FLAT
6579 if (isFLAT(MI)) {
6580 legalizeOperandsFLAT(MRI, MI);
6581 return CreatedBB;
6582 }
6583
6584 // Legalize REG_SEQUENCE and PHI
6585 // The register class of the operands much be the same type as the register
6586 // class of the output.
6587 if (MI.getOpcode() == AMDGPU::PHI) {
6588 const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
6589 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
6590 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).getReg().isVirtual())
6591 continue;
6592 const TargetRegisterClass *OpRC =
6593 MRI.getRegClass(Reg: MI.getOperand(i).getReg());
6594 if (RI.hasVectorRegisters(RC: OpRC)) {
6595 VRC = OpRC;
6596 } else {
6597 SRC = OpRC;
6598 }
6599 }
6600
6601 // If any of the operands are VGPR registers, then they all most be
6602 // otherwise we will create illegal VGPR->SGPR copies when legalizing
6603 // them.
6604 if (VRC || !RI.isSGPRClass(RC: getOpRegClass(MI, OpNo: 0))) {
6605 if (!VRC) {
6606 assert(SRC);
6607 if (getOpRegClass(MI, OpNo: 0) == &AMDGPU::VReg_1RegClass) {
6608 VRC = &AMDGPU::VReg_1RegClass;
6609 } else
6610 VRC = RI.isAGPRClass(RC: getOpRegClass(MI, OpNo: 0))
6611 ? RI.getEquivalentAGPRClass(SRC)
6612 : RI.getEquivalentVGPRClass(SRC);
6613 } else {
6614 VRC = RI.isAGPRClass(RC: getOpRegClass(MI, OpNo: 0))
6615 ? RI.getEquivalentAGPRClass(SRC: VRC)
6616 : RI.getEquivalentVGPRClass(SRC: VRC);
6617 }
6618 RC = VRC;
6619 } else {
6620 RC = SRC;
6621 }
6622
6623 // Update all the operands so they have the same type.
6624 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
6625 MachineOperand &Op = MI.getOperand(i: I);
6626 if (!Op.isReg() || !Op.getReg().isVirtual())
6627 continue;
6628
6629 // MI is a PHI instruction.
6630 MachineBasicBlock *InsertBB = MI.getOperand(i: I + 1).getMBB();
6631 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
6632
6633 // Avoid creating no-op copies with the same src and dst reg class. These
6634 // confuse some of the machine passes.
6635 legalizeGenericOperand(InsertMBB&: *InsertBB, I: Insert, DstRC: RC, Op, MRI, DL: MI.getDebugLoc());
6636 }
6637 }
6638
6639 // REG_SEQUENCE doesn't really require operand legalization, but if one has a
6640 // VGPR dest type and SGPR sources, insert copies so all operands are
6641 // VGPRs. This seems to help operand folding / the register coalescer.
6642 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
6643 MachineBasicBlock *MBB = MI.getParent();
6644 const TargetRegisterClass *DstRC = getOpRegClass(MI, OpNo: 0);
6645 if (RI.hasVGPRs(RC: DstRC)) {
6646 // Update all the operands so they are VGPR register classes. These may
6647 // not be the same register class because REG_SEQUENCE supports mixing
6648 // subregister index types e.g. sub0_sub1 + sub2 + sub3
6649 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
6650 MachineOperand &Op = MI.getOperand(i: I);
6651 if (!Op.isReg() || !Op.getReg().isVirtual())
6652 continue;
6653
6654 const TargetRegisterClass *OpRC = MRI.getRegClass(Reg: Op.getReg());
6655 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(SRC: OpRC);
6656 if (VRC == OpRC)
6657 continue;
6658
6659 legalizeGenericOperand(InsertMBB&: *MBB, I: MI, DstRC: VRC, Op, MRI, DL: MI.getDebugLoc());
6660 Op.setIsKill();
6661 }
6662 }
6663
6664 return CreatedBB;
6665 }
6666
6667 // Legalize INSERT_SUBREG
6668 // src0 must have the same register class as dst
6669 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
6670 Register Dst = MI.getOperand(i: 0).getReg();
6671 Register Src0 = MI.getOperand(i: 1).getReg();
6672 const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: Dst);
6673 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0);
6674 if (DstRC != Src0RC) {
6675 MachineBasicBlock *MBB = MI.getParent();
6676 MachineOperand &Op = MI.getOperand(i: 1);
6677 legalizeGenericOperand(InsertMBB&: *MBB, I: MI, DstRC, Op, MRI, DL: MI.getDebugLoc());
6678 }
6679 return CreatedBB;
6680 }
6681
6682 // Legalize SI_INIT_M0
6683 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
6684 MachineOperand &Src = MI.getOperand(i: 0);
6685 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
6686 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
6687 return CreatedBB;
6688 }
6689
6690 // Legalize S_BITREPLICATE, S_QUADMASK and S_WQM
6691 if (MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 ||
6692 MI.getOpcode() == AMDGPU::S_QUADMASK_B32 ||
6693 MI.getOpcode() == AMDGPU::S_QUADMASK_B64 ||
6694 MI.getOpcode() == AMDGPU::S_WQM_B32 ||
6695 MI.getOpcode() == AMDGPU::S_WQM_B64 ||
6696 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U32 ||
6697 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U64) {
6698 MachineOperand &Src = MI.getOperand(i: 1);
6699 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
6700 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
6701 return CreatedBB;
6702 }
6703
6704 // Legalize MIMG/VIMAGE/VSAMPLE and MUBUF/MTBUF for shaders.
6705 //
6706 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
6707 // scratch memory access. In both cases, the legalization never involves
6708 // conversion to the addr64 form.
6709 if (isImage(MI) || (AMDGPU::isGraphics(CC: MF.getFunction().getCallingConv()) &&
6710 (isMUBUF(MI) || isMTBUF(MI)))) {
6711 int RSrcOpName = (isVIMAGE(MI) || isVSAMPLE(MI)) ? AMDGPU::OpName::rsrc
6712 : AMDGPU::OpName::srsrc;
6713 MachineOperand *SRsrc = getNamedOperand(MI, OperandName: RSrcOpName);
6714 if (SRsrc && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SRsrc->getReg())))
6715 CreatedBB = loadMBUFScalarOperandsFromVGPR(TII: *this, MI, ScalarOps: {SRsrc}, MDT);
6716
6717 int SampOpName = isMIMG(MI) ? AMDGPU::OpName::ssamp : AMDGPU::OpName::samp;
6718 MachineOperand *SSamp = getNamedOperand(MI, OperandName: SampOpName);
6719 if (SSamp && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SSamp->getReg())))
6720 CreatedBB = loadMBUFScalarOperandsFromVGPR(TII: *this, MI, ScalarOps: {SSamp}, MDT);
6721
6722 return CreatedBB;
6723 }
6724
6725 // Legalize SI_CALL
6726 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) {
6727 MachineOperand *Dest = &MI.getOperand(i: 0);
6728 if (!RI.isSGPRClass(RC: MRI.getRegClass(Reg: Dest->getReg()))) {
6729 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and
6730 // following copies, we also need to move copies from and to physical
6731 // registers into the loop block.
6732 unsigned FrameSetupOpcode = getCallFrameSetupOpcode();
6733 unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode();
6734
6735 // Also move the copies to physical registers into the loop block
6736 MachineBasicBlock &MBB = *MI.getParent();
6737 MachineBasicBlock::iterator Start(&MI);
6738 while (Start->getOpcode() != FrameSetupOpcode)
6739 --Start;
6740 MachineBasicBlock::iterator End(&MI);
6741 while (End->getOpcode() != FrameDestroyOpcode)
6742 ++End;
6743 // Also include following copies of the return value
6744 ++End;
6745 while (End != MBB.end() && End->isCopy() && End->getOperand(i: 1).isReg() &&
6746 MI.definesRegister(Reg: End->getOperand(i: 1).getReg(), /*TRI=*/nullptr))
6747 ++End;
6748 CreatedBB =
6749 loadMBUFScalarOperandsFromVGPR(TII: *this, MI, ScalarOps: {Dest}, MDT, Begin: Start, End);
6750 }
6751 }
6752
6753 // Legalize s_sleep_var.
6754 if (MI.getOpcode() == AMDGPU::S_SLEEP_VAR) {
6755 const DebugLoc &DL = MI.getDebugLoc();
6756 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6757 int Src0Idx =
6758 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::src0);
6759 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
6760 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
6761 .add(MO: Src0);
6762 Src0.ChangeToRegister(Reg, isDef: false);
6763 return nullptr;
6764 }
6765
6766 // Legalize MUBUF instructions.
6767 bool isSoffsetLegal = true;
6768 int SoffsetIdx =
6769 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::soffset);
6770 if (SoffsetIdx != -1) {
6771 MachineOperand *Soffset = &MI.getOperand(i: SoffsetIdx);
6772 if (Soffset->isReg() && Soffset->getReg().isVirtual() &&
6773 !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Soffset->getReg()))) {
6774 isSoffsetLegal = false;
6775 }
6776 }
6777
6778 bool isRsrcLegal = true;
6779 int RsrcIdx =
6780 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::srsrc);
6781 if (RsrcIdx != -1) {
6782 MachineOperand *Rsrc = &MI.getOperand(i: RsrcIdx);
6783 if (Rsrc->isReg() && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Rsrc->getReg()))) {
6784 isRsrcLegal = false;
6785 }
6786 }
6787
6788 // The operands are legal.
6789 if (isRsrcLegal && isSoffsetLegal)
6790 return CreatedBB;
6791
6792 if (!isRsrcLegal) {
6793 // Legalize a VGPR Rsrc
6794 //
6795 // If the instruction is _ADDR64, we can avoid a waterfall by extracting
6796 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
6797 // a zero-value SRsrc.
6798 //
6799 // If the instruction is _OFFSET (both idxen and offen disabled), and we
6800 // support ADDR64 instructions, we can convert to ADDR64 and do the same as
6801 // above.
6802 //
6803 // Otherwise we are on non-ADDR64 hardware, and/or we have
6804 // idxen/offen/bothen and we fall back to a waterfall loop.
6805
6806 MachineOperand *Rsrc = &MI.getOperand(i: RsrcIdx);
6807 MachineBasicBlock &MBB = *MI.getParent();
6808
6809 MachineOperand *VAddr = getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
6810 if (VAddr && AMDGPU::getIfAddr64Inst(Opcode: MI.getOpcode()) != -1) {
6811 // This is already an ADDR64 instruction so we need to add the pointer
6812 // extracted from the resource descriptor to the current value of VAddr.
6813 Register NewVAddrLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6814 Register NewVAddrHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6815 Register NewVAddr = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
6816
6817 const auto *BoolXExecRC = RI.getRegClass(RCID: AMDGPU::SReg_1_XEXECRegClassID);
6818 Register CondReg0 = MRI.createVirtualRegister(RegClass: BoolXExecRC);
6819 Register CondReg1 = MRI.createVirtualRegister(RegClass: BoolXExecRC);
6820
6821 unsigned RsrcPtr, NewSRsrc;
6822 std::tie(args&: RsrcPtr, args&: NewSRsrc) = extractRsrcPtr(TII: *this, MI, Rsrc&: *Rsrc);
6823
6824 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
6825 const DebugLoc &DL = MI.getDebugLoc();
6826 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg: NewVAddrLo)
6827 .addDef(RegNo: CondReg0)
6828 .addReg(RegNo: RsrcPtr, flags: 0, SubReg: AMDGPU::sub0)
6829 .addReg(RegNo: VAddr->getReg(), flags: 0, SubReg: AMDGPU::sub0)
6830 .addImm(Val: 0);
6831
6832 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
6833 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADDC_U32_e64), DestReg: NewVAddrHi)
6834 .addDef(RegNo: CondReg1, Flags: RegState::Dead)
6835 .addReg(RegNo: RsrcPtr, flags: 0, SubReg: AMDGPU::sub1)
6836 .addReg(RegNo: VAddr->getReg(), flags: 0, SubReg: AMDGPU::sub1)
6837 .addReg(RegNo: CondReg0, flags: RegState::Kill)
6838 .addImm(Val: 0);
6839
6840 // NewVaddr = {NewVaddrHi, NewVaddrLo}
6841 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewVAddr)
6842 .addReg(RegNo: NewVAddrLo)
6843 .addImm(Val: AMDGPU::sub0)
6844 .addReg(RegNo: NewVAddrHi)
6845 .addImm(Val: AMDGPU::sub1);
6846
6847 VAddr->setReg(NewVAddr);
6848 Rsrc->setReg(NewSRsrc);
6849 } else if (!VAddr && ST.hasAddr64()) {
6850 // This instructions is the _OFFSET variant, so we need to convert it to
6851 // ADDR64.
6852 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS &&
6853 "FIXME: Need to emit flat atomics here");
6854
6855 unsigned RsrcPtr, NewSRsrc;
6856 std::tie(args&: RsrcPtr, args&: NewSRsrc) = extractRsrcPtr(TII: *this, MI, Rsrc&: *Rsrc);
6857
6858 Register NewVAddr = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
6859 MachineOperand *VData = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata);
6860 MachineOperand *Offset = getNamedOperand(MI, OperandName: AMDGPU::OpName::offset);
6861 MachineOperand *SOffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
6862 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(Opcode: MI.getOpcode());
6863
6864 // Atomics with return have an additional tied operand and are
6865 // missing some of the special bits.
6866 MachineOperand *VDataIn = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata_in);
6867 MachineInstr *Addr64;
6868
6869 if (!VDataIn) {
6870 // Regular buffer load / store.
6871 MachineInstrBuilder MIB =
6872 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Addr64Opcode))
6873 .add(MO: *VData)
6874 .addReg(RegNo: NewVAddr)
6875 .addReg(RegNo: NewSRsrc)
6876 .add(MO: *SOffset)
6877 .add(MO: *Offset);
6878
6879 if (const MachineOperand *CPol =
6880 getNamedOperand(MI, OperandName: AMDGPU::OpName::cpol)) {
6881 MIB.addImm(Val: CPol->getImm());
6882 }
6883
6884 if (const MachineOperand *TFE =
6885 getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe)) {
6886 MIB.addImm(Val: TFE->getImm());
6887 }
6888
6889 MIB.addImm(Val: getNamedImmOperand(MI, OpName: AMDGPU::OpName::swz));
6890
6891 MIB.cloneMemRefs(OtherMI: MI);
6892 Addr64 = MIB;
6893 } else {
6894 // Atomics with return.
6895 Addr64 = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Addr64Opcode))
6896 .add(MO: *VData)
6897 .add(MO: *VDataIn)
6898 .addReg(RegNo: NewVAddr)
6899 .addReg(RegNo: NewSRsrc)
6900 .add(MO: *SOffset)
6901 .add(MO: *Offset)
6902 .addImm(Val: getNamedImmOperand(MI, OpName: AMDGPU::OpName::cpol))
6903 .cloneMemRefs(OtherMI: MI);
6904 }
6905
6906 MI.removeFromParent();
6907
6908 // NewVaddr = {NewVaddrHi, NewVaddrLo}
6909 BuildMI(BB&: MBB, I: Addr64, MIMD: Addr64->getDebugLoc(), MCID: get(Opcode: AMDGPU::REG_SEQUENCE),
6910 DestReg: NewVAddr)
6911 .addReg(RegNo: RsrcPtr, flags: 0, SubReg: AMDGPU::sub0)
6912 .addImm(Val: AMDGPU::sub0)
6913 .addReg(RegNo: RsrcPtr, flags: 0, SubReg: AMDGPU::sub1)
6914 .addImm(Val: AMDGPU::sub1);
6915 } else {
6916 // Legalize a VGPR Rsrc and soffset together.
6917 if (!isSoffsetLegal) {
6918 MachineOperand *Soffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
6919 CreatedBB =
6920 loadMBUFScalarOperandsFromVGPR(TII: *this, MI, ScalarOps: {Rsrc, Soffset}, MDT);
6921 return CreatedBB;
6922 }
6923 CreatedBB = loadMBUFScalarOperandsFromVGPR(TII: *this, MI, ScalarOps: {Rsrc}, MDT);
6924 return CreatedBB;
6925 }
6926 }
6927
6928 // Legalize a VGPR soffset.
6929 if (!isSoffsetLegal) {
6930 MachineOperand *Soffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
6931 CreatedBB = loadMBUFScalarOperandsFromVGPR(TII: *this, MI, ScalarOps: {Soffset}, MDT);
6932 return CreatedBB;
6933 }
6934 return CreatedBB;
6935}
6936
6937void SIInstrWorklist::insert(MachineInstr *MI) {
6938 InstrList.insert(X: MI);
6939 // Add MBUF instructiosn to deferred list.
6940 int RsrcIdx =
6941 AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(), NamedIdx: AMDGPU::OpName::srsrc);
6942 if (RsrcIdx != -1) {
6943 DeferredList.insert(X: MI);
6944 }
6945}
6946
6947bool SIInstrWorklist::isDeferred(MachineInstr *MI) {
6948 return DeferredList.contains(key: MI);
6949}
6950
6951void SIInstrInfo::moveToVALU(SIInstrWorklist &Worklist,
6952 MachineDominatorTree *MDT) const {
6953
6954 while (!Worklist.empty()) {
6955 MachineInstr &Inst = *Worklist.top();
6956 Worklist.erase_top();
6957 // Skip MachineInstr in the deferred list.
6958 if (Worklist.isDeferred(MI: &Inst))
6959 continue;
6960 moveToVALUImpl(Worklist, MDT, Inst);
6961 }
6962
6963 // Deferred list of instructions will be processed once
6964 // all the MachineInstr in the worklist are done.
6965 for (MachineInstr *Inst : Worklist.getDeferredList()) {
6966 moveToVALUImpl(Worklist, MDT, Inst&: *Inst);
6967 assert(Worklist.empty() &&
6968 "Deferred MachineInstr are not supposed to re-populate worklist");
6969 }
6970}
6971
6972void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
6973 MachineDominatorTree *MDT,
6974 MachineInstr &Inst) const {
6975
6976 MachineBasicBlock *MBB = Inst.getParent();
6977 if (!MBB)
6978 return;
6979 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
6980 unsigned Opcode = Inst.getOpcode();
6981 unsigned NewOpcode = getVALUOp(MI: Inst);
6982 // Handle some special cases
6983 switch (Opcode) {
6984 default:
6985 break;
6986 case AMDGPU::S_ADD_U64_PSEUDO:
6987 NewOpcode = AMDGPU::V_ADD_U64_PSEUDO;
6988 break;
6989 case AMDGPU::S_SUB_U64_PSEUDO:
6990 NewOpcode = AMDGPU::V_SUB_U64_PSEUDO;
6991 break;
6992 case AMDGPU::S_ADD_I32:
6993 case AMDGPU::S_SUB_I32: {
6994 // FIXME: The u32 versions currently selected use the carry.
6995 bool Changed;
6996 MachineBasicBlock *CreatedBBTmp = nullptr;
6997 std::tie(args&: Changed, args&: CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT);
6998 if (Changed)
6999 return;
7000
7001 // Default handling
7002 break;
7003 }
7004
7005 case AMDGPU::S_MUL_U64:
7006 // Split s_mul_u64 in 32-bit vector multiplications.
7007 splitScalarSMulU64(Worklist, Inst, MDT);
7008 Inst.eraseFromParent();
7009 return;
7010
7011 case AMDGPU::S_MUL_U64_U32_PSEUDO:
7012 case AMDGPU::S_MUL_I64_I32_PSEUDO:
7013 // This is a special case of s_mul_u64 where all the operands are either
7014 // zero extended or sign extended.
7015 splitScalarSMulPseudo(Worklist, Inst, MDT);
7016 Inst.eraseFromParent();
7017 return;
7018
7019 case AMDGPU::S_AND_B64:
7020 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_AND_B32, MDT);
7021 Inst.eraseFromParent();
7022 return;
7023
7024 case AMDGPU::S_OR_B64:
7025 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_OR_B32, MDT);
7026 Inst.eraseFromParent();
7027 return;
7028
7029 case AMDGPU::S_XOR_B64:
7030 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_XOR_B32, MDT);
7031 Inst.eraseFromParent();
7032 return;
7033
7034 case AMDGPU::S_NAND_B64:
7035 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_NAND_B32, MDT);
7036 Inst.eraseFromParent();
7037 return;
7038
7039 case AMDGPU::S_NOR_B64:
7040 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_NOR_B32, MDT);
7041 Inst.eraseFromParent();
7042 return;
7043
7044 case AMDGPU::S_XNOR_B64:
7045 if (ST.hasDLInsts())
7046 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_XNOR_B32, MDT);
7047 else
7048 splitScalar64BitXnor(Worklist, Inst, MDT);
7049 Inst.eraseFromParent();
7050 return;
7051
7052 case AMDGPU::S_ANDN2_B64:
7053 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_ANDN2_B32, MDT);
7054 Inst.eraseFromParent();
7055 return;
7056
7057 case AMDGPU::S_ORN2_B64:
7058 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_ORN2_B32, MDT);
7059 Inst.eraseFromParent();
7060 return;
7061
7062 case AMDGPU::S_BREV_B64:
7063 splitScalar64BitUnaryOp(Worklist, Inst, Opcode: AMDGPU::S_BREV_B32, Swap: true);
7064 Inst.eraseFromParent();
7065 return;
7066
7067 case AMDGPU::S_NOT_B64:
7068 splitScalar64BitUnaryOp(Worklist, Inst, Opcode: AMDGPU::S_NOT_B32);
7069 Inst.eraseFromParent();
7070 return;
7071
7072 case AMDGPU::S_BCNT1_I32_B64:
7073 splitScalar64BitBCNT(Worklist, Inst);
7074 Inst.eraseFromParent();
7075 return;
7076
7077 case AMDGPU::S_BFE_I64:
7078 splitScalar64BitBFE(Worklist, Inst);
7079 Inst.eraseFromParent();
7080 return;
7081
7082 case AMDGPU::S_FLBIT_I32_B64:
7083 splitScalar64BitCountOp(Worklist, Inst, Opcode: AMDGPU::V_FFBH_U32_e32);
7084 Inst.eraseFromParent();
7085 return;
7086 case AMDGPU::S_FF1_I32_B64:
7087 splitScalar64BitCountOp(Worklist, Inst, Opcode: AMDGPU::V_FFBL_B32_e32);
7088 Inst.eraseFromParent();
7089 return;
7090
7091 case AMDGPU::S_LSHL_B32:
7092 if (ST.hasOnlyRevVALUShifts()) {
7093 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
7094 swapOperands(Inst);
7095 }
7096 break;
7097 case AMDGPU::S_ASHR_I32:
7098 if (ST.hasOnlyRevVALUShifts()) {
7099 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
7100 swapOperands(Inst);
7101 }
7102 break;
7103 case AMDGPU::S_LSHR_B32:
7104 if (ST.hasOnlyRevVALUShifts()) {
7105 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
7106 swapOperands(Inst);
7107 }
7108 break;
7109 case AMDGPU::S_LSHL_B64:
7110 if (ST.hasOnlyRevVALUShifts()) {
7111 NewOpcode = ST.getGeneration() >= AMDGPUSubtarget::GFX12
7112 ? AMDGPU::V_LSHLREV_B64_pseudo_e64
7113 : AMDGPU::V_LSHLREV_B64_e64;
7114 swapOperands(Inst);
7115 }
7116 break;
7117 case AMDGPU::S_ASHR_I64:
7118 if (ST.hasOnlyRevVALUShifts()) {
7119 NewOpcode = AMDGPU::V_ASHRREV_I64_e64;
7120 swapOperands(Inst);
7121 }
7122 break;
7123 case AMDGPU::S_LSHR_B64:
7124 if (ST.hasOnlyRevVALUShifts()) {
7125 NewOpcode = AMDGPU::V_LSHRREV_B64_e64;
7126 swapOperands(Inst);
7127 }
7128 break;
7129
7130 case AMDGPU::S_ABS_I32:
7131 lowerScalarAbs(Worklist, Inst);
7132 Inst.eraseFromParent();
7133 return;
7134
7135 case AMDGPU::S_CBRANCH_SCC0:
7136 case AMDGPU::S_CBRANCH_SCC1: {
7137 // Clear unused bits of vcc
7138 Register CondReg = Inst.getOperand(i: 1).getReg();
7139 bool IsSCC = CondReg == AMDGPU::SCC;
7140 Register VCC = RI.getVCC();
7141 Register EXEC = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
7142 unsigned Opc = ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
7143 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: Opc), DestReg: VCC)
7144 .addReg(RegNo: EXEC)
7145 .addReg(RegNo: IsSCC ? VCC : CondReg);
7146 Inst.removeOperand(OpNo: 1);
7147 } break;
7148
7149 case AMDGPU::S_BFE_U64:
7150 case AMDGPU::S_BFM_B64:
7151 llvm_unreachable("Moving this op to VALU not implemented");
7152
7153 case AMDGPU::S_PACK_LL_B32_B16:
7154 case AMDGPU::S_PACK_LH_B32_B16:
7155 case AMDGPU::S_PACK_HL_B32_B16:
7156 case AMDGPU::S_PACK_HH_B32_B16:
7157 movePackToVALU(Worklist, MRI, Inst);
7158 Inst.eraseFromParent();
7159 return;
7160
7161 case AMDGPU::S_XNOR_B32:
7162 lowerScalarXnor(Worklist, Inst);
7163 Inst.eraseFromParent();
7164 return;
7165
7166 case AMDGPU::S_NAND_B32:
7167 splitScalarNotBinop(Worklist, Inst, Opcode: AMDGPU::S_AND_B32);
7168 Inst.eraseFromParent();
7169 return;
7170
7171 case AMDGPU::S_NOR_B32:
7172 splitScalarNotBinop(Worklist, Inst, Opcode: AMDGPU::S_OR_B32);
7173 Inst.eraseFromParent();
7174 return;
7175
7176 case AMDGPU::S_ANDN2_B32:
7177 splitScalarBinOpN2(Worklist, Inst, Opcode: AMDGPU::S_AND_B32);
7178 Inst.eraseFromParent();
7179 return;
7180
7181 case AMDGPU::S_ORN2_B32:
7182 splitScalarBinOpN2(Worklist, Inst, Opcode: AMDGPU::S_OR_B32);
7183 Inst.eraseFromParent();
7184 return;
7185
7186 // TODO: remove as soon as everything is ready
7187 // to replace VGPR to SGPR copy with V_READFIRSTLANEs.
7188 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO
7189 // can only be selected from the uniform SDNode.
7190 case AMDGPU::S_ADD_CO_PSEUDO:
7191 case AMDGPU::S_SUB_CO_PSEUDO: {
7192 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
7193 ? AMDGPU::V_ADDC_U32_e64
7194 : AMDGPU::V_SUBB_U32_e64;
7195 const auto *CarryRC = RI.getRegClass(RCID: AMDGPU::SReg_1_XEXECRegClassID);
7196
7197 Register CarryInReg = Inst.getOperand(i: 4).getReg();
7198 if (!MRI.constrainRegClass(Reg: CarryInReg, RC: CarryRC)) {
7199 Register NewCarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
7200 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY), DestReg: NewCarryReg)
7201 .addReg(RegNo: CarryInReg);
7202 }
7203
7204 Register CarryOutReg = Inst.getOperand(i: 1).getReg();
7205
7206 Register DestReg = MRI.createVirtualRegister(RegClass: RI.getEquivalentVGPRClass(
7207 SRC: MRI.getRegClass(Reg: Inst.getOperand(i: 0).getReg())));
7208 MachineInstr *CarryOp =
7209 BuildMI(BB&: *MBB, I: &Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: Opc), DestReg)
7210 .addReg(RegNo: CarryOutReg, flags: RegState::Define)
7211 .add(MO: Inst.getOperand(i: 2))
7212 .add(MO: Inst.getOperand(i: 3))
7213 .addReg(RegNo: CarryInReg)
7214 .addImm(Val: 0);
7215 legalizeOperands(MI&: *CarryOp);
7216 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: DestReg);
7217 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
7218 Inst.eraseFromParent();
7219 }
7220 return;
7221 case AMDGPU::S_UADDO_PSEUDO:
7222 case AMDGPU::S_USUBO_PSEUDO: {
7223 const DebugLoc &DL = Inst.getDebugLoc();
7224 MachineOperand &Dest0 = Inst.getOperand(i: 0);
7225 MachineOperand &Dest1 = Inst.getOperand(i: 1);
7226 MachineOperand &Src0 = Inst.getOperand(i: 2);
7227 MachineOperand &Src1 = Inst.getOperand(i: 3);
7228
7229 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
7230 ? AMDGPU::V_ADD_CO_U32_e64
7231 : AMDGPU::V_SUB_CO_U32_e64;
7232 const TargetRegisterClass *NewRC =
7233 RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest0.getReg()));
7234 Register DestReg = MRI.createVirtualRegister(RegClass: NewRC);
7235 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I: &Inst, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
7236 .addReg(RegNo: Dest1.getReg(), flags: RegState::Define)
7237 .add(MO: Src0)
7238 .add(MO: Src1)
7239 .addImm(Val: 0); // clamp bit
7240
7241 legalizeOperands(MI&: *NewInstr, MDT);
7242 MRI.replaceRegWith(FromReg: Dest0.getReg(), ToReg: DestReg);
7243 addUsersToMoveToVALUWorklist(Reg: NewInstr->getOperand(i: 0).getReg(), MRI,
7244 Worklist);
7245 Inst.eraseFromParent();
7246 }
7247 return;
7248
7249 case AMDGPU::S_CSELECT_B32:
7250 case AMDGPU::S_CSELECT_B64:
7251 lowerSelect(Worklist, Inst, MDT);
7252 Inst.eraseFromParent();
7253 return;
7254 case AMDGPU::S_CMP_EQ_I32:
7255 case AMDGPU::S_CMP_LG_I32:
7256 case AMDGPU::S_CMP_GT_I32:
7257 case AMDGPU::S_CMP_GE_I32:
7258 case AMDGPU::S_CMP_LT_I32:
7259 case AMDGPU::S_CMP_LE_I32:
7260 case AMDGPU::S_CMP_EQ_U32:
7261 case AMDGPU::S_CMP_LG_U32:
7262 case AMDGPU::S_CMP_GT_U32:
7263 case AMDGPU::S_CMP_GE_U32:
7264 case AMDGPU::S_CMP_LT_U32:
7265 case AMDGPU::S_CMP_LE_U32:
7266 case AMDGPU::S_CMP_EQ_U64:
7267 case AMDGPU::S_CMP_LG_U64:
7268 case AMDGPU::S_CMP_LT_F32:
7269 case AMDGPU::S_CMP_EQ_F32:
7270 case AMDGPU::S_CMP_LE_F32:
7271 case AMDGPU::S_CMP_GT_F32:
7272 case AMDGPU::S_CMP_LG_F32:
7273 case AMDGPU::S_CMP_GE_F32:
7274 case AMDGPU::S_CMP_O_F32:
7275 case AMDGPU::S_CMP_U_F32:
7276 case AMDGPU::S_CMP_NGE_F32:
7277 case AMDGPU::S_CMP_NLG_F32:
7278 case AMDGPU::S_CMP_NGT_F32:
7279 case AMDGPU::S_CMP_NLE_F32:
7280 case AMDGPU::S_CMP_NEQ_F32:
7281 case AMDGPU::S_CMP_NLT_F32:
7282 case AMDGPU::S_CMP_LT_F16:
7283 case AMDGPU::S_CMP_EQ_F16:
7284 case AMDGPU::S_CMP_LE_F16:
7285 case AMDGPU::S_CMP_GT_F16:
7286 case AMDGPU::S_CMP_LG_F16:
7287 case AMDGPU::S_CMP_GE_F16:
7288 case AMDGPU::S_CMP_O_F16:
7289 case AMDGPU::S_CMP_U_F16:
7290 case AMDGPU::S_CMP_NGE_F16:
7291 case AMDGPU::S_CMP_NLG_F16:
7292 case AMDGPU::S_CMP_NGT_F16:
7293 case AMDGPU::S_CMP_NLE_F16:
7294 case AMDGPU::S_CMP_NEQ_F16:
7295 case AMDGPU::S_CMP_NLT_F16: {
7296 Register CondReg = MRI.createVirtualRegister(RegClass: RI.getWaveMaskRegClass());
7297 auto NewInstr =
7298 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode), DestReg: CondReg)
7299 .setMIFlags(Inst.getFlags());
7300 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
7301 NamedIdx: AMDGPU::OpName::src0_modifiers) >= 0) {
7302 NewInstr
7303 .addImm(Val: 0) // src0_modifiers
7304 .add(MO: Inst.getOperand(i: 0)) // src0
7305 .addImm(Val: 0) // src1_modifiers
7306 .add(MO: Inst.getOperand(i: 1)) // src1
7307 .addImm(Val: 0); // clamp
7308 } else {
7309 NewInstr
7310 .add(MO: Inst.getOperand(i: 0))
7311 .add(MO: Inst.getOperand(i: 1));
7312 }
7313 legalizeOperands(MI&: *NewInstr, MDT);
7314 int SCCIdx = Inst.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
7315 MachineOperand SCCOp = Inst.getOperand(i: SCCIdx);
7316 addSCCDefUsersToVALUWorklist(Op&: SCCOp, SCCDefInst&: Inst, Worklist, NewCond: CondReg);
7317 Inst.eraseFromParent();
7318 return;
7319 }
7320 case AMDGPU::S_CVT_HI_F32_F16: {
7321 const DebugLoc &DL = Inst.getDebugLoc();
7322 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7323 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7324 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
7325 .addImm(Val: 16)
7326 .add(MO: Inst.getOperand(i: 1));
7327 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
7328 .addImm(Val: 0) // src0_modifiers
7329 .addReg(RegNo: TmpReg)
7330 .addImm(Val: 0) // clamp
7331 .addImm(Val: 0); // omod
7332
7333 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
7334 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
7335 Inst.eraseFromParent();
7336 return;
7337 }
7338 case AMDGPU::S_MINIMUM_F32:
7339 case AMDGPU::S_MAXIMUM_F32:
7340 case AMDGPU::S_MINIMUM_F16:
7341 case AMDGPU::S_MAXIMUM_F16: {
7342 const DebugLoc &DL = Inst.getDebugLoc();
7343 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7344 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
7345 .addImm(Val: 0) // src0_modifiers
7346 .add(MO: Inst.getOperand(i: 1))
7347 .addImm(Val: 0) // src1_modifiers
7348 .add(MO: Inst.getOperand(i: 2))
7349 .addImm(Val: 0) // clamp
7350 .addImm(Val: 0); // omod
7351 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
7352
7353 legalizeOperands(MI&: *NewInstr, MDT);
7354 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
7355 Inst.eraseFromParent();
7356 return;
7357 }
7358 }
7359
7360 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
7361 // We cannot move this instruction to the VALU, so we should try to
7362 // legalize its operands instead.
7363 legalizeOperands(MI&: Inst, MDT);
7364 return;
7365 }
7366 // Handle converting generic instructions like COPY-to-SGPR into
7367 // COPY-to-VGPR.
7368 if (NewOpcode == Opcode) {
7369 Register DstReg = Inst.getOperand(i: 0).getReg();
7370 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
7371
7372 // If it's a copy of a VGPR to a physical SGPR, insert a V_READFIRSTLANE and
7373 // hope for the best.
7374 if (Inst.isCopy() && DstReg.isPhysical() &&
7375 RI.isVGPR(MRI, Reg: Inst.getOperand(i: 1).getReg())) {
7376 // TODO: Only works for 32 bit registers.
7377 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
7378 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Inst.getOperand(i: 0).getReg())
7379 .add(MO: Inst.getOperand(i: 1));
7380 Inst.eraseFromParent();
7381 return;
7382 }
7383
7384 if (Inst.isCopy() && Inst.getOperand(i: 1).getReg().isVirtual() &&
7385 NewDstRC == RI.getRegClassForReg(MRI, Reg: Inst.getOperand(i: 1).getReg())) {
7386 // Instead of creating a copy where src and dst are the same register
7387 // class, we just replace all uses of dst with src. These kinds of
7388 // copies interfere with the heuristics MachineSink uses to decide
7389 // whether or not to split a critical edge. Since the pass assumes
7390 // that copies will end up as machine instructions and not be
7391 // eliminated.
7392 addUsersToMoveToVALUWorklist(Reg: DstReg, MRI, Worklist);
7393 MRI.replaceRegWith(FromReg: DstReg, ToReg: Inst.getOperand(i: 1).getReg());
7394 MRI.clearKillFlags(Reg: Inst.getOperand(i: 1).getReg());
7395 Inst.getOperand(i: 0).setReg(DstReg);
7396 // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
7397 // these are deleted later, but at -O0 it would leave a suspicious
7398 // looking illegal copy of an undef register.
7399 for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I)
7400 Inst.removeOperand(OpNo: I);
7401 Inst.setDesc(get(Opcode: AMDGPU::IMPLICIT_DEF));
7402 return;
7403 }
7404 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
7405 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
7406 legalizeOperands(MI&: Inst, MDT);
7407 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
7408 return;
7409 }
7410
7411 // Use the new VALU Opcode.
7412 auto NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode))
7413 .setMIFlags(Inst.getFlags());
7414 if (isVOP3(Opcode: NewOpcode) && !isVOP3(Opcode)) {
7415 // Intersperse VOP3 modifiers among the SALU operands.
7416 NewInstr->addOperand(Op: Inst.getOperand(i: 0));
7417 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
7418 NamedIdx: AMDGPU::OpName::src0_modifiers) >= 0)
7419 NewInstr.addImm(Val: 0);
7420 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::src0)) {
7421 MachineOperand Src = Inst.getOperand(i: 1);
7422 if (AMDGPU::isTrue16Inst(Opc: NewOpcode) && ST.useRealTrue16Insts() &&
7423 Src.isReg() && RI.isVGPR(MRI, Reg: Src.getReg()))
7424 NewInstr.addReg(RegNo: Src.getReg(), flags: 0, SubReg: AMDGPU::lo16);
7425 else
7426 NewInstr->addOperand(Op: Src);
7427 }
7428
7429 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
7430 // We are converting these to a BFE, so we need to add the missing
7431 // operands for the size and offset.
7432 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
7433 NewInstr.addImm(Val: 0);
7434 NewInstr.addImm(Val: Size);
7435 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
7436 // The VALU version adds the second operand to the result, so insert an
7437 // extra 0 operand.
7438 NewInstr.addImm(Val: 0);
7439 } else if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
7440 const MachineOperand &OffsetWidthOp = Inst.getOperand(i: 2);
7441 // If we need to move this to VGPRs, we need to unpack the second
7442 // operand back into the 2 separate ones for bit offset and width.
7443 assert(OffsetWidthOp.isImm() &&
7444 "Scalar BFE is only implemented for constant width and offset");
7445 uint32_t Imm = OffsetWidthOp.getImm();
7446
7447 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
7448 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
7449 NewInstr.addImm(Val: Offset);
7450 NewInstr.addImm(Val: BitWidth);
7451 } else {
7452 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
7453 NamedIdx: AMDGPU::OpName::src1_modifiers) >= 0)
7454 NewInstr.addImm(Val: 0);
7455 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::src1) >= 0)
7456 NewInstr->addOperand(Op: Inst.getOperand(i: 2));
7457 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
7458 NamedIdx: AMDGPU::OpName::src2_modifiers) >= 0)
7459 NewInstr.addImm(Val: 0);
7460 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::src2) >= 0)
7461 NewInstr->addOperand(Op: Inst.getOperand(i: 3));
7462 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::clamp) >= 0)
7463 NewInstr.addImm(Val: 0);
7464 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::omod) >= 0)
7465 NewInstr.addImm(Val: 0);
7466 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::op_sel) >= 0)
7467 NewInstr.addImm(Val: 0);
7468 }
7469 } else {
7470 // Just copy the SALU operands.
7471 for (const MachineOperand &Op : Inst.explicit_operands())
7472 NewInstr->addOperand(Op);
7473 }
7474
7475 // Remove any references to SCC. Vector instructions can't read from it, and
7476 // We're just about to add the implicit use / defs of VCC, and we don't want
7477 // both.
7478 for (MachineOperand &Op : Inst.implicit_operands()) {
7479 if (Op.getReg() == AMDGPU::SCC) {
7480 // Only propagate through live-def of SCC.
7481 if (Op.isDef() && !Op.isDead())
7482 addSCCDefUsersToVALUWorklist(Op, SCCDefInst&: Inst, Worklist);
7483 if (Op.isUse())
7484 addSCCDefsToVALUWorklist(SCCUseInst: NewInstr, Worklist);
7485 }
7486 }
7487 Inst.eraseFromParent();
7488 Register NewDstReg;
7489 if (NewInstr->getOperand(i: 0).isReg() && NewInstr->getOperand(i: 0).isDef()) {
7490 Register DstReg = NewInstr->getOperand(i: 0).getReg();
7491 assert(DstReg.isVirtual());
7492 // Update the destination register class.
7493 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst: *NewInstr);
7494 assert(NewDstRC);
7495 NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
7496 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
7497 }
7498 fixImplicitOperands(MI&: *NewInstr);
7499 // Legalize the operands
7500 legalizeOperands(MI&: *NewInstr, MDT);
7501 if (NewDstReg)
7502 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
7503}
7504
7505// Add/sub require special handling to deal with carry outs.
7506std::pair<bool, MachineBasicBlock *>
7507SIInstrInfo::moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
7508 MachineDominatorTree *MDT) const {
7509 if (ST.hasAddNoCarry()) {
7510 // Assume there is no user of scc since we don't select this in that case.
7511 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
7512 // is used.
7513
7514 MachineBasicBlock &MBB = *Inst.getParent();
7515 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7516
7517 Register OldDstReg = Inst.getOperand(i: 0).getReg();
7518 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7519
7520 unsigned Opc = Inst.getOpcode();
7521 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
7522
7523 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
7524 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
7525
7526 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
7527 Inst.removeOperand(OpNo: 3);
7528
7529 Inst.setDesc(get(Opcode: NewOpc));
7530 Inst.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // clamp bit
7531 Inst.addImplicitDefUseOperands(MF&: *MBB.getParent());
7532 MRI.replaceRegWith(FromReg: OldDstReg, ToReg: ResultReg);
7533 MachineBasicBlock *NewBB = legalizeOperands(MI&: Inst, MDT);
7534
7535 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
7536 return std::pair(true, NewBB);
7537 }
7538
7539 return std::pair(false, nullptr);
7540}
7541
7542void SIInstrInfo::lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
7543 MachineDominatorTree *MDT) const {
7544
7545 MachineBasicBlock &MBB = *Inst.getParent();
7546 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7547 MachineBasicBlock::iterator MII = Inst;
7548 DebugLoc DL = Inst.getDebugLoc();
7549
7550 MachineOperand &Dest = Inst.getOperand(i: 0);
7551 MachineOperand &Src0 = Inst.getOperand(i: 1);
7552 MachineOperand &Src1 = Inst.getOperand(i: 2);
7553 MachineOperand &Cond = Inst.getOperand(i: 3);
7554
7555 Register CondReg = Cond.getReg();
7556 bool IsSCC = (CondReg == AMDGPU::SCC);
7557
7558 // If this is a trivial select where the condition is effectively not SCC
7559 // (CondReg is a source of copy to SCC), then the select is semantically
7560 // equivalent to copying CondReg. Hence, there is no need to create
7561 // V_CNDMASK, we can just use that and bail out.
7562 if (!IsSCC && Src0.isImm() && (Src0.getImm() == -1) && Src1.isImm() &&
7563 (Src1.getImm() == 0)) {
7564 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: CondReg);
7565 return;
7566 }
7567
7568 Register NewCondReg = CondReg;
7569 if (IsSCC) {
7570 const TargetRegisterClass *TC =
7571 RI.getRegClass(RCID: AMDGPU::SReg_1_XEXECRegClassID);
7572 NewCondReg = MRI.createVirtualRegister(RegClass: TC);
7573
7574 // Now look for the closest SCC def if it is a copy
7575 // replacing the CondReg with the COPY source register
7576 bool CopyFound = false;
7577 for (MachineInstr &CandI :
7578 make_range(x: std::next(x: MachineBasicBlock::reverse_iterator(Inst)),
7579 y: Inst.getParent()->rend())) {
7580 if (CandI.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isDead: false, Overlap: false) !=
7581 -1) {
7582 if (CandI.isCopy() && CandI.getOperand(i: 0).getReg() == AMDGPU::SCC) {
7583 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: NewCondReg)
7584 .addReg(RegNo: CandI.getOperand(i: 1).getReg());
7585 CopyFound = true;
7586 }
7587 break;
7588 }
7589 }
7590 if (!CopyFound) {
7591 // SCC def is not a copy
7592 // Insert a trivial select instead of creating a copy, because a copy from
7593 // SCC would semantically mean just copying a single bit, but we may need
7594 // the result to be a vector condition mask that needs preserving.
7595 unsigned Opcode = (ST.getWavefrontSize() == 64) ? AMDGPU::S_CSELECT_B64
7596 : AMDGPU::S_CSELECT_B32;
7597 auto NewSelect =
7598 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: NewCondReg).addImm(Val: -1).addImm(Val: 0);
7599 NewSelect->getOperand(i: 3).setIsUndef(Cond.isUndef());
7600 }
7601 }
7602
7603 Register NewDestReg = MRI.createVirtualRegister(
7604 RegClass: RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest.getReg())));
7605 MachineInstr *NewInst;
7606 if (Inst.getOpcode() == AMDGPU::S_CSELECT_B32) {
7607 NewInst = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: NewDestReg)
7608 .addImm(Val: 0)
7609 .add(MO: Src1) // False
7610 .addImm(Val: 0)
7611 .add(MO: Src0) // True
7612 .addReg(RegNo: NewCondReg);
7613 } else {
7614 NewInst =
7615 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B64_PSEUDO), DestReg: NewDestReg)
7616 .add(MO: Src1) // False
7617 .add(MO: Src0) // True
7618 .addReg(RegNo: NewCondReg);
7619 }
7620 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDestReg);
7621 legalizeOperands(MI&: *NewInst, MDT);
7622 addUsersToMoveToVALUWorklist(Reg: NewDestReg, MRI, Worklist);
7623}
7624
7625void SIInstrInfo::lowerScalarAbs(SIInstrWorklist &Worklist,
7626 MachineInstr &Inst) const {
7627 MachineBasicBlock &MBB = *Inst.getParent();
7628 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7629 MachineBasicBlock::iterator MII = Inst;
7630 DebugLoc DL = Inst.getDebugLoc();
7631
7632 MachineOperand &Dest = Inst.getOperand(i: 0);
7633 MachineOperand &Src = Inst.getOperand(i: 1);
7634 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7635 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7636
7637 unsigned SubOp = ST.hasAddNoCarry() ?
7638 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32;
7639
7640 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: TmpReg)
7641 .addImm(Val: 0)
7642 .addReg(RegNo: Src.getReg());
7643
7644 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MAX_I32_e64), DestReg: ResultReg)
7645 .addReg(RegNo: Src.getReg())
7646 .addReg(RegNo: TmpReg);
7647
7648 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
7649 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
7650}
7651
7652void SIInstrInfo::lowerScalarXnor(SIInstrWorklist &Worklist,
7653 MachineInstr &Inst) const {
7654 MachineBasicBlock &MBB = *Inst.getParent();
7655 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7656 MachineBasicBlock::iterator MII = Inst;
7657 const DebugLoc &DL = Inst.getDebugLoc();
7658
7659 MachineOperand &Dest = Inst.getOperand(i: 0);
7660 MachineOperand &Src0 = Inst.getOperand(i: 1);
7661 MachineOperand &Src1 = Inst.getOperand(i: 2);
7662
7663 if (ST.hasDLInsts()) {
7664 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7665 legalizeGenericOperand(InsertMBB&: MBB, I: MII, DstRC: &AMDGPU::VGPR_32RegClass, Op&: Src0, MRI, DL);
7666 legalizeGenericOperand(InsertMBB&: MBB, I: MII, DstRC: &AMDGPU::VGPR_32RegClass, Op&: Src1, MRI, DL);
7667
7668 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_XNOR_B32_e64), DestReg: NewDest)
7669 .add(MO: Src0)
7670 .add(MO: Src1);
7671
7672 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
7673 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
7674 } else {
7675 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
7676 // invert either source and then perform the XOR. If either source is a
7677 // scalar register, then we can leave the inversion on the scalar unit to
7678 // achieve a better distribution of scalar and vector instructions.
7679 bool Src0IsSGPR = Src0.isReg() &&
7680 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src0.getReg()));
7681 bool Src1IsSGPR = Src1.isReg() &&
7682 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src1.getReg()));
7683 MachineInstr *Xor;
7684 Register Temp = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7685 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7686
7687 // Build a pair of scalar instructions and add them to the work list.
7688 // The next iteration over the work list will lower these to the vector
7689 // unit as necessary.
7690 if (Src0IsSGPR) {
7691 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Temp).add(MO: Src0);
7692 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: NewDest)
7693 .addReg(RegNo: Temp)
7694 .add(MO: Src1);
7695 } else if (Src1IsSGPR) {
7696 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Temp).add(MO: Src1);
7697 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: NewDest)
7698 .add(MO: Src0)
7699 .addReg(RegNo: Temp);
7700 } else {
7701 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: Temp)
7702 .add(MO: Src0)
7703 .add(MO: Src1);
7704 MachineInstr *Not =
7705 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: NewDest).addReg(RegNo: Temp);
7706 Worklist.insert(MI: Not);
7707 }
7708
7709 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
7710
7711 Worklist.insert(MI: Xor);
7712
7713 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
7714 }
7715}
7716
7717void SIInstrInfo::splitScalarNotBinop(SIInstrWorklist &Worklist,
7718 MachineInstr &Inst,
7719 unsigned Opcode) const {
7720 MachineBasicBlock &MBB = *Inst.getParent();
7721 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7722 MachineBasicBlock::iterator MII = Inst;
7723 const DebugLoc &DL = Inst.getDebugLoc();
7724
7725 MachineOperand &Dest = Inst.getOperand(i: 0);
7726 MachineOperand &Src0 = Inst.getOperand(i: 1);
7727 MachineOperand &Src1 = Inst.getOperand(i: 2);
7728
7729 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7730 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7731
7732 MachineInstr &Op = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: Interm)
7733 .add(MO: Src0)
7734 .add(MO: Src1);
7735
7736 MachineInstr &Not = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: NewDest)
7737 .addReg(RegNo: Interm);
7738
7739 Worklist.insert(MI: &Op);
7740 Worklist.insert(MI: &Not);
7741
7742 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
7743 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
7744}
7745
7746void SIInstrInfo::splitScalarBinOpN2(SIInstrWorklist &Worklist,
7747 MachineInstr &Inst,
7748 unsigned Opcode) const {
7749 MachineBasicBlock &MBB = *Inst.getParent();
7750 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7751 MachineBasicBlock::iterator MII = Inst;
7752 const DebugLoc &DL = Inst.getDebugLoc();
7753
7754 MachineOperand &Dest = Inst.getOperand(i: 0);
7755 MachineOperand &Src0 = Inst.getOperand(i: 1);
7756 MachineOperand &Src1 = Inst.getOperand(i: 2);
7757
7758 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7759 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7760
7761 MachineInstr &Not = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Interm)
7762 .add(MO: Src1);
7763
7764 MachineInstr &Op = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: NewDest)
7765 .add(MO: Src0)
7766 .addReg(RegNo: Interm);
7767
7768 Worklist.insert(MI: &Not);
7769 Worklist.insert(MI: &Op);
7770
7771 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
7772 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
7773}
7774
7775void SIInstrInfo::splitScalar64BitUnaryOp(SIInstrWorklist &Worklist,
7776 MachineInstr &Inst, unsigned Opcode,
7777 bool Swap) const {
7778 MachineBasicBlock &MBB = *Inst.getParent();
7779 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7780
7781 MachineOperand &Dest = Inst.getOperand(i: 0);
7782 MachineOperand &Src0 = Inst.getOperand(i: 1);
7783 DebugLoc DL = Inst.getDebugLoc();
7784
7785 MachineBasicBlock::iterator MII = Inst;
7786
7787 const MCInstrDesc &InstDesc = get(Opcode);
7788 const TargetRegisterClass *Src0RC = Src0.isReg() ?
7789 MRI.getRegClass(Reg: Src0.getReg()) :
7790 &AMDGPU::SGPR_32RegClass;
7791
7792 const TargetRegisterClass *Src0SubRC =
7793 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
7794
7795 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
7796 SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
7797
7798 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
7799 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(SRC: DestRC);
7800 const TargetRegisterClass *NewDestSubRC =
7801 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
7802
7803 Register DestSub0 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
7804 MachineInstr &LoHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub0).add(MO: SrcReg0Sub0);
7805
7806 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
7807 SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
7808
7809 Register DestSub1 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
7810 MachineInstr &HiHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub1).add(MO: SrcReg0Sub1);
7811
7812 if (Swap)
7813 std::swap(a&: DestSub0, b&: DestSub1);
7814
7815 Register FullDestReg = MRI.createVirtualRegister(RegClass: NewDestRC);
7816 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
7817 .addReg(RegNo: DestSub0)
7818 .addImm(Val: AMDGPU::sub0)
7819 .addReg(RegNo: DestSub1)
7820 .addImm(Val: AMDGPU::sub1);
7821
7822 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
7823
7824 Worklist.insert(MI: &LoHalf);
7825 Worklist.insert(MI: &HiHalf);
7826
7827 // We don't need to legalizeOperands here because for a single operand, src0
7828 // will support any kind of input.
7829
7830 // Move all users of this moved value.
7831 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
7832}
7833
7834// There is not a vector equivalent of s_mul_u64. For this reason, we need to
7835// split the s_mul_u64 in 32-bit vector multiplications.
7836void SIInstrInfo::splitScalarSMulU64(SIInstrWorklist &Worklist,
7837 MachineInstr &Inst,
7838 MachineDominatorTree *MDT) const {
7839 MachineBasicBlock &MBB = *Inst.getParent();
7840 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7841
7842 Register FullDestReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
7843 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7844 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7845
7846 MachineOperand &Dest = Inst.getOperand(i: 0);
7847 MachineOperand &Src0 = Inst.getOperand(i: 1);
7848 MachineOperand &Src1 = Inst.getOperand(i: 2);
7849 const DebugLoc &DL = Inst.getDebugLoc();
7850 MachineBasicBlock::iterator MII = Inst;
7851
7852 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0.getReg());
7853 const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: Src1.getReg());
7854 const TargetRegisterClass *Src0SubRC =
7855 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
7856 if (RI.isSGPRClass(RC: Src0SubRC))
7857 Src0SubRC = RI.getEquivalentVGPRClass(SRC: Src0SubRC);
7858 const TargetRegisterClass *Src1SubRC =
7859 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
7860 if (RI.isSGPRClass(RC: Src1SubRC))
7861 Src1SubRC = RI.getEquivalentVGPRClass(SRC: Src1SubRC);
7862
7863 // First, we extract the low 32-bit and high 32-bit values from each of the
7864 // operands.
7865 MachineOperand Op0L =
7866 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
7867 MachineOperand Op1L =
7868 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
7869 MachineOperand Op0H =
7870 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
7871 MachineOperand Op1H =
7872 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
7873
7874 // The multilication is done as follows:
7875 //
7876 // Op1H Op1L
7877 // * Op0H Op0L
7878 // --------------------
7879 // Op1H*Op0L Op1L*Op0L
7880 // + Op1H*Op0H Op1L*Op0H
7881 // -----------------------------------------
7882 // (Op1H*Op0L + Op1L*Op0H + carry) Op1L*Op0L
7883 //
7884 // We drop Op1H*Op0H because the result of the multiplication is a 64-bit
7885 // value and that would overflow.
7886 // The low 32-bit value is Op1L*Op0L.
7887 // The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from Op1L*Op0L).
7888
7889 Register Op1L_Op0H_Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7890 MachineInstr *Op1L_Op0H =
7891 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: Op1L_Op0H_Reg)
7892 .add(MO: Op1L)
7893 .add(MO: Op0H);
7894
7895 Register Op1H_Op0L_Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7896 MachineInstr *Op1H_Op0L =
7897 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: Op1H_Op0L_Reg)
7898 .add(MO: Op1H)
7899 .add(MO: Op0L);
7900
7901 Register CarryReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7902 MachineInstr *Carry =
7903 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_HI_U32_e64), DestReg: CarryReg)
7904 .add(MO: Op1L)
7905 .add(MO: Op0L);
7906
7907 MachineInstr *LoHalf =
7908 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: DestSub0)
7909 .add(MO: Op1L)
7910 .add(MO: Op0L);
7911
7912 Register AddReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7913 MachineInstr *Add = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg: AddReg)
7914 .addReg(RegNo: Op1L_Op0H_Reg)
7915 .addReg(RegNo: Op1H_Op0L_Reg);
7916
7917 MachineInstr *HiHalf =
7918 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg: DestSub1)
7919 .addReg(RegNo: AddReg)
7920 .addReg(RegNo: CarryReg);
7921
7922 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
7923 .addReg(RegNo: DestSub0)
7924 .addImm(Val: AMDGPU::sub0)
7925 .addReg(RegNo: DestSub1)
7926 .addImm(Val: AMDGPU::sub1);
7927
7928 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
7929
7930 // Try to legalize the operands in case we need to swap the order to keep it
7931 // valid.
7932 legalizeOperands(MI&: *Op1L_Op0H, MDT);
7933 legalizeOperands(MI&: *Op1H_Op0L, MDT);
7934 legalizeOperands(MI&: *Carry, MDT);
7935 legalizeOperands(MI&: *LoHalf, MDT);
7936 legalizeOperands(MI&: *Add, MDT);
7937 legalizeOperands(MI&: *HiHalf, MDT);
7938
7939 // Move all users of this moved value.
7940 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
7941}
7942
7943// Lower S_MUL_U64_U32_PSEUDO/S_MUL_I64_I32_PSEUDO in two 32-bit vector
7944// multiplications.
7945void SIInstrInfo::splitScalarSMulPseudo(SIInstrWorklist &Worklist,
7946 MachineInstr &Inst,
7947 MachineDominatorTree *MDT) const {
7948 MachineBasicBlock &MBB = *Inst.getParent();
7949 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7950
7951 Register FullDestReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
7952 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7953 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7954
7955 MachineOperand &Dest = Inst.getOperand(i: 0);
7956 MachineOperand &Src0 = Inst.getOperand(i: 1);
7957 MachineOperand &Src1 = Inst.getOperand(i: 2);
7958 const DebugLoc &DL = Inst.getDebugLoc();
7959 MachineBasicBlock::iterator MII = Inst;
7960
7961 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0.getReg());
7962 const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: Src1.getReg());
7963 const TargetRegisterClass *Src0SubRC =
7964 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
7965 if (RI.isSGPRClass(RC: Src0SubRC))
7966 Src0SubRC = RI.getEquivalentVGPRClass(SRC: Src0SubRC);
7967 const TargetRegisterClass *Src1SubRC =
7968 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
7969 if (RI.isSGPRClass(RC: Src1SubRC))
7970 Src1SubRC = RI.getEquivalentVGPRClass(SRC: Src1SubRC);
7971
7972 // First, we extract the low 32-bit and high 32-bit values from each of the
7973 // operands.
7974 MachineOperand Op0L =
7975 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
7976 MachineOperand Op1L =
7977 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
7978
7979 unsigned Opc = Inst.getOpcode();
7980 unsigned NewOpc = Opc == AMDGPU::S_MUL_U64_U32_PSEUDO
7981 ? AMDGPU::V_MUL_HI_U32_e64
7982 : AMDGPU::V_MUL_HI_I32_e64;
7983 MachineInstr *HiHalf =
7984 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: NewOpc), DestReg: DestSub1).add(MO: Op1L).add(MO: Op0L);
7985
7986 MachineInstr *LoHalf =
7987 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: DestSub0)
7988 .add(MO: Op1L)
7989 .add(MO: Op0L);
7990
7991 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
7992 .addReg(RegNo: DestSub0)
7993 .addImm(Val: AMDGPU::sub0)
7994 .addReg(RegNo: DestSub1)
7995 .addImm(Val: AMDGPU::sub1);
7996
7997 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
7998
7999 // Try to legalize the operands in case we need to swap the order to keep it
8000 // valid.
8001 legalizeOperands(MI&: *HiHalf, MDT);
8002 legalizeOperands(MI&: *LoHalf, MDT);
8003
8004 // Move all users of this moved value.
8005 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
8006}
8007
8008void SIInstrInfo::splitScalar64BitBinaryOp(SIInstrWorklist &Worklist,
8009 MachineInstr &Inst, unsigned Opcode,
8010 MachineDominatorTree *MDT) const {
8011 MachineBasicBlock &MBB = *Inst.getParent();
8012 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8013
8014 MachineOperand &Dest = Inst.getOperand(i: 0);
8015 MachineOperand &Src0 = Inst.getOperand(i: 1);
8016 MachineOperand &Src1 = Inst.getOperand(i: 2);
8017 DebugLoc DL = Inst.getDebugLoc();
8018
8019 MachineBasicBlock::iterator MII = Inst;
8020
8021 const MCInstrDesc &InstDesc = get(Opcode);
8022 const TargetRegisterClass *Src0RC = Src0.isReg() ?
8023 MRI.getRegClass(Reg: Src0.getReg()) :
8024 &AMDGPU::SGPR_32RegClass;
8025
8026 const TargetRegisterClass *Src0SubRC =
8027 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
8028 const TargetRegisterClass *Src1RC = Src1.isReg() ?
8029 MRI.getRegClass(Reg: Src1.getReg()) :
8030 &AMDGPU::SGPR_32RegClass;
8031
8032 const TargetRegisterClass *Src1SubRC =
8033 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
8034
8035 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
8036 SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
8037 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC,
8038 SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
8039 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
8040 SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
8041 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC,
8042 SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
8043
8044 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
8045 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(SRC: DestRC);
8046 const TargetRegisterClass *NewDestSubRC =
8047 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
8048
8049 Register DestSub0 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
8050 MachineInstr &LoHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub0)
8051 .add(MO: SrcReg0Sub0)
8052 .add(MO: SrcReg1Sub0);
8053
8054 Register DestSub1 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
8055 MachineInstr &HiHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub1)
8056 .add(MO: SrcReg0Sub1)
8057 .add(MO: SrcReg1Sub1);
8058
8059 Register FullDestReg = MRI.createVirtualRegister(RegClass: NewDestRC);
8060 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
8061 .addReg(RegNo: DestSub0)
8062 .addImm(Val: AMDGPU::sub0)
8063 .addReg(RegNo: DestSub1)
8064 .addImm(Val: AMDGPU::sub1);
8065
8066 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
8067
8068 Worklist.insert(MI: &LoHalf);
8069 Worklist.insert(MI: &HiHalf);
8070
8071 // Move all users of this moved value.
8072 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
8073}
8074
8075void SIInstrInfo::splitScalar64BitXnor(SIInstrWorklist &Worklist,
8076 MachineInstr &Inst,
8077 MachineDominatorTree *MDT) const {
8078 MachineBasicBlock &MBB = *Inst.getParent();
8079 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8080
8081 MachineOperand &Dest = Inst.getOperand(i: 0);
8082 MachineOperand &Src0 = Inst.getOperand(i: 1);
8083 MachineOperand &Src1 = Inst.getOperand(i: 2);
8084 const DebugLoc &DL = Inst.getDebugLoc();
8085
8086 MachineBasicBlock::iterator MII = Inst;
8087
8088 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
8089
8090 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
8091
8092 MachineOperand* Op0;
8093 MachineOperand* Op1;
8094
8095 if (Src0.isReg() && RI.isSGPRReg(MRI, Reg: Src0.getReg())) {
8096 Op0 = &Src0;
8097 Op1 = &Src1;
8098 } else {
8099 Op0 = &Src1;
8100 Op1 = &Src0;
8101 }
8102
8103 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B64), DestReg: Interm)
8104 .add(MO: *Op0);
8105
8106 Register NewDest = MRI.createVirtualRegister(RegClass: DestRC);
8107
8108 MachineInstr &Xor = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B64), DestReg: NewDest)
8109 .addReg(RegNo: Interm)
8110 .add(MO: *Op1);
8111
8112 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
8113
8114 Worklist.insert(MI: &Xor);
8115}
8116
8117void SIInstrInfo::splitScalar64BitBCNT(SIInstrWorklist &Worklist,
8118 MachineInstr &Inst) const {
8119 MachineBasicBlock &MBB = *Inst.getParent();
8120 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8121
8122 MachineBasicBlock::iterator MII = Inst;
8123 const DebugLoc &DL = Inst.getDebugLoc();
8124
8125 MachineOperand &Dest = Inst.getOperand(i: 0);
8126 MachineOperand &Src = Inst.getOperand(i: 1);
8127
8128 const MCInstrDesc &InstDesc = get(Opcode: AMDGPU::V_BCNT_U32_B32_e64);
8129 const TargetRegisterClass *SrcRC = Src.isReg() ?
8130 MRI.getRegClass(Reg: Src.getReg()) :
8131 &AMDGPU::SGPR_32RegClass;
8132
8133 Register MidReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8134 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8135
8136 const TargetRegisterClass *SrcSubRC =
8137 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
8138
8139 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC,
8140 SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
8141 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC,
8142 SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
8143
8144 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg).add(MO: SrcRegSub0).addImm(Val: 0);
8145
8146 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: ResultReg).add(MO: SrcRegSub1).addReg(RegNo: MidReg);
8147
8148 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
8149
8150 // We don't need to legalize operands here. src0 for either instruction can be
8151 // an SGPR, and the second input is unused or determined here.
8152 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
8153}
8154
8155void SIInstrInfo::splitScalar64BitBFE(SIInstrWorklist &Worklist,
8156 MachineInstr &Inst) const {
8157 MachineBasicBlock &MBB = *Inst.getParent();
8158 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8159 MachineBasicBlock::iterator MII = Inst;
8160 const DebugLoc &DL = Inst.getDebugLoc();
8161
8162 MachineOperand &Dest = Inst.getOperand(i: 0);
8163 uint32_t Imm = Inst.getOperand(i: 2).getImm();
8164 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
8165 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
8166
8167 (void) Offset;
8168
8169 // Only sext_inreg cases handled.
8170 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
8171 Offset == 0 && "Not implemented");
8172
8173 if (BitWidth < 32) {
8174 Register MidRegLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8175 Register MidRegHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8176 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
8177
8178 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_BFE_I32_e64), DestReg: MidRegLo)
8179 .addReg(RegNo: Inst.getOperand(i: 1).getReg(), flags: 0, SubReg: AMDGPU::sub0)
8180 .addImm(Val: 0)
8181 .addImm(Val: BitWidth);
8182
8183 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ASHRREV_I32_e32), DestReg: MidRegHi)
8184 .addImm(Val: 31)
8185 .addReg(RegNo: MidRegLo);
8186
8187 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: ResultReg)
8188 .addReg(RegNo: MidRegLo)
8189 .addImm(Val: AMDGPU::sub0)
8190 .addReg(RegNo: MidRegHi)
8191 .addImm(Val: AMDGPU::sub1);
8192
8193 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
8194 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
8195 return;
8196 }
8197
8198 MachineOperand &Src = Inst.getOperand(i: 1);
8199 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8200 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
8201
8202 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ASHRREV_I32_e64), DestReg: TmpReg)
8203 .addImm(Val: 31)
8204 .addReg(RegNo: Src.getReg(), flags: 0, SubReg: AMDGPU::sub0);
8205
8206 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: ResultReg)
8207 .addReg(RegNo: Src.getReg(), flags: 0, SubReg: AMDGPU::sub0)
8208 .addImm(Val: AMDGPU::sub0)
8209 .addReg(RegNo: TmpReg)
8210 .addImm(Val: AMDGPU::sub1);
8211
8212 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
8213 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
8214}
8215
8216void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist,
8217 MachineInstr &Inst, unsigned Opcode,
8218 MachineDominatorTree *MDT) const {
8219 // (S_FLBIT_I32_B64 hi:lo) ->
8220 // -> (umin (V_FFBH_U32_e32 hi), (uaddsat (V_FFBH_U32_e32 lo), 32))
8221 // (S_FF1_I32_B64 hi:lo) ->
8222 // ->(umin (uaddsat (V_FFBL_B32_e32 hi), 32) (V_FFBL_B32_e32 lo))
8223
8224 MachineBasicBlock &MBB = *Inst.getParent();
8225 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8226 MachineBasicBlock::iterator MII = Inst;
8227 const DebugLoc &DL = Inst.getDebugLoc();
8228
8229 MachineOperand &Dest = Inst.getOperand(i: 0);
8230 MachineOperand &Src = Inst.getOperand(i: 1);
8231
8232 const MCInstrDesc &InstDesc = get(Opcode);
8233
8234 bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32;
8235 unsigned OpcodeAdd =
8236 ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
8237
8238 const TargetRegisterClass *SrcRC =
8239 Src.isReg() ? MRI.getRegClass(Reg: Src.getReg()) : &AMDGPU::SGPR_32RegClass;
8240 const TargetRegisterClass *SrcSubRC =
8241 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
8242
8243 MachineOperand SrcRegSub0 =
8244 buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC, SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
8245 MachineOperand SrcRegSub1 =
8246 buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC, SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
8247
8248 Register MidReg1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8249 Register MidReg2 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8250 Register MidReg3 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8251 Register MidReg4 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8252
8253 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg1).add(MO: SrcRegSub0);
8254
8255 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg2).add(MO: SrcRegSub1);
8256
8257 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: OpcodeAdd), DestReg: MidReg3)
8258 .addReg(RegNo: IsCtlz ? MidReg1 : MidReg2)
8259 .addImm(Val: 32)
8260 .addImm(Val: 1); // enable clamp
8261
8262 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MIN_U32_e64), DestReg: MidReg4)
8263 .addReg(RegNo: MidReg3)
8264 .addReg(RegNo: IsCtlz ? MidReg2 : MidReg1);
8265
8266 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: MidReg4);
8267
8268 addUsersToMoveToVALUWorklist(Reg: MidReg4, MRI, Worklist);
8269}
8270
8271void SIInstrInfo::addUsersToMoveToVALUWorklist(
8272 Register DstReg, MachineRegisterInfo &MRI,
8273 SIInstrWorklist &Worklist) const {
8274 for (MachineRegisterInfo::use_iterator I = MRI.use_begin(RegNo: DstReg),
8275 E = MRI.use_end(); I != E;) {
8276 MachineInstr &UseMI = *I->getParent();
8277
8278 unsigned OpNo = 0;
8279
8280 switch (UseMI.getOpcode()) {
8281 case AMDGPU::COPY:
8282 case AMDGPU::WQM:
8283 case AMDGPU::SOFT_WQM:
8284 case AMDGPU::STRICT_WWM:
8285 case AMDGPU::STRICT_WQM:
8286 case AMDGPU::REG_SEQUENCE:
8287 case AMDGPU::PHI:
8288 case AMDGPU::INSERT_SUBREG:
8289 break;
8290 default:
8291 OpNo = I.getOperandNo();
8292 break;
8293 }
8294
8295 if (!RI.hasVectorRegisters(RC: getOpRegClass(MI: UseMI, OpNo))) {
8296 Worklist.insert(MI: &UseMI);
8297
8298 do {
8299 ++I;
8300 } while (I != E && I->getParent() == &UseMI);
8301 } else {
8302 ++I;
8303 }
8304 }
8305}
8306
8307void SIInstrInfo::movePackToVALU(SIInstrWorklist &Worklist,
8308 MachineRegisterInfo &MRI,
8309 MachineInstr &Inst) const {
8310 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8311 MachineBasicBlock *MBB = Inst.getParent();
8312 MachineOperand &Src0 = Inst.getOperand(i: 1);
8313 MachineOperand &Src1 = Inst.getOperand(i: 2);
8314 const DebugLoc &DL = Inst.getDebugLoc();
8315
8316 switch (Inst.getOpcode()) {
8317 case AMDGPU::S_PACK_LL_B32_B16: {
8318 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8319 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8320
8321 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are
8322 // 0.
8323 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
8324 .addImm(Val: 0xffff);
8325
8326 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_AND_B32_e64), DestReg: TmpReg)
8327 .addReg(RegNo: ImmReg, flags: RegState::Kill)
8328 .add(MO: Src0);
8329
8330 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: ResultReg)
8331 .add(MO: Src1)
8332 .addImm(Val: 16)
8333 .addReg(RegNo: TmpReg, flags: RegState::Kill);
8334 break;
8335 }
8336 case AMDGPU::S_PACK_LH_B32_B16: {
8337 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8338 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
8339 .addImm(Val: 0xffff);
8340 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_BFI_B32_e64), DestReg: ResultReg)
8341 .addReg(RegNo: ImmReg, flags: RegState::Kill)
8342 .add(MO: Src0)
8343 .add(MO: Src1);
8344 break;
8345 }
8346 case AMDGPU::S_PACK_HL_B32_B16: {
8347 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8348 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
8349 .addImm(Val: 16)
8350 .add(MO: Src0);
8351 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: ResultReg)
8352 .add(MO: Src1)
8353 .addImm(Val: 16)
8354 .addReg(RegNo: TmpReg, flags: RegState::Kill);
8355 break;
8356 }
8357 case AMDGPU::S_PACK_HH_B32_B16: {
8358 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8359 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8360 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
8361 .addImm(Val: 16)
8362 .add(MO: Src0);
8363 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
8364 .addImm(Val: 0xffff0000);
8365 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_AND_OR_B32_e64), DestReg: ResultReg)
8366 .add(MO: Src1)
8367 .addReg(RegNo: ImmReg, flags: RegState::Kill)
8368 .addReg(RegNo: TmpReg, flags: RegState::Kill);
8369 break;
8370 }
8371 default:
8372 llvm_unreachable("unhandled s_pack_* instruction");
8373 }
8374
8375 MachineOperand &Dest = Inst.getOperand(i: 0);
8376 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
8377 addUsersToMoveToVALUWorklist(DstReg: ResultReg, MRI, Worklist);
8378}
8379
8380void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op,
8381 MachineInstr &SCCDefInst,
8382 SIInstrWorklist &Worklist,
8383 Register NewCond) const {
8384
8385 // Ensure that def inst defines SCC, which is still live.
8386 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() &&
8387 !Op.isDead() && Op.getParent() == &SCCDefInst);
8388 SmallVector<MachineInstr *, 4> CopyToDelete;
8389 // This assumes that all the users of SCC are in the same block
8390 // as the SCC def.
8391 for (MachineInstr &MI : // Skip the def inst itself.
8392 make_range(x: std::next(x: MachineBasicBlock::iterator(SCCDefInst)),
8393 y: SCCDefInst.getParent()->end())) {
8394 // Check if SCC is used first.
8395 int SCCIdx = MI.findRegisterUseOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isKill: false);
8396 if (SCCIdx != -1) {
8397 if (MI.isCopy()) {
8398 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8399 Register DestReg = MI.getOperand(i: 0).getReg();
8400
8401 MRI.replaceRegWith(FromReg: DestReg, ToReg: NewCond);
8402 CopyToDelete.push_back(Elt: &MI);
8403 } else {
8404
8405 if (NewCond.isValid())
8406 MI.getOperand(i: SCCIdx).setReg(NewCond);
8407
8408 Worklist.insert(MI: &MI);
8409 }
8410 }
8411 // Exit if we find another SCC def.
8412 if (MI.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isDead: false, Overlap: false) != -1)
8413 break;
8414 }
8415 for (auto &Copy : CopyToDelete)
8416 Copy->eraseFromParent();
8417}
8418
8419// Instructions that use SCC may be converted to VALU instructions. When that
8420// happens, the SCC register is changed to VCC_LO. The instruction that defines
8421// SCC must be changed to an instruction that defines VCC. This function makes
8422// sure that the instruction that defines SCC is added to the moveToVALU
8423// worklist.
8424void SIInstrInfo::addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
8425 SIInstrWorklist &Worklist) const {
8426 // Look for a preceding instruction that either defines VCC or SCC. If VCC
8427 // then there is nothing to do because the defining instruction has been
8428 // converted to a VALU already. If SCC then that instruction needs to be
8429 // converted to a VALU.
8430 for (MachineInstr &MI :
8431 make_range(x: std::next(x: MachineBasicBlock::reverse_iterator(SCCUseInst)),
8432 y: SCCUseInst->getParent()->rend())) {
8433 if (MI.modifiesRegister(Reg: AMDGPU::VCC, TRI: &RI))
8434 break;
8435 if (MI.definesRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
8436 Worklist.insert(MI: &MI);
8437 break;
8438 }
8439 }
8440}
8441
8442const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
8443 const MachineInstr &Inst) const {
8444 const TargetRegisterClass *NewDstRC = getOpRegClass(MI: Inst, OpNo: 0);
8445
8446 switch (Inst.getOpcode()) {
8447 // For target instructions, getOpRegClass just returns the virtual register
8448 // class associated with the operand, so we need to find an equivalent VGPR
8449 // register class in order to move the instruction to the VALU.
8450 case AMDGPU::COPY:
8451 case AMDGPU::PHI:
8452 case AMDGPU::REG_SEQUENCE:
8453 case AMDGPU::INSERT_SUBREG:
8454 case AMDGPU::WQM:
8455 case AMDGPU::SOFT_WQM:
8456 case AMDGPU::STRICT_WWM:
8457 case AMDGPU::STRICT_WQM: {
8458 const TargetRegisterClass *SrcRC = getOpRegClass(MI: Inst, OpNo: 1);
8459 if (RI.isAGPRClass(RC: SrcRC)) {
8460 if (RI.isAGPRClass(RC: NewDstRC))
8461 return nullptr;
8462
8463 switch (Inst.getOpcode()) {
8464 case AMDGPU::PHI:
8465 case AMDGPU::REG_SEQUENCE:
8466 case AMDGPU::INSERT_SUBREG:
8467 NewDstRC = RI.getEquivalentAGPRClass(SRC: NewDstRC);
8468 break;
8469 default:
8470 NewDstRC = RI.getEquivalentVGPRClass(SRC: NewDstRC);
8471 }
8472
8473 if (!NewDstRC)
8474 return nullptr;
8475 } else {
8476 if (RI.isVGPRClass(RC: NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass)
8477 return nullptr;
8478
8479 NewDstRC = RI.getEquivalentVGPRClass(SRC: NewDstRC);
8480 if (!NewDstRC)
8481 return nullptr;
8482 }
8483
8484 return NewDstRC;
8485 }
8486 default:
8487 return NewDstRC;
8488 }
8489}
8490
8491// Find the one SGPR operand we are allowed to use.
8492Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
8493 int OpIndices[3]) const {
8494 const MCInstrDesc &Desc = MI.getDesc();
8495
8496 // Find the one SGPR operand we are allowed to use.
8497 //
8498 // First we need to consider the instruction's operand requirements before
8499 // legalizing. Some operands are required to be SGPRs, such as implicit uses
8500 // of VCC, but we are still bound by the constant bus requirement to only use
8501 // one.
8502 //
8503 // If the operand's class is an SGPR, we can never move it.
8504
8505 Register SGPRReg = findImplicitSGPRRead(MI);
8506 if (SGPRReg)
8507 return SGPRReg;
8508
8509 Register UsedSGPRs[3] = {Register()};
8510 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8511
8512 for (unsigned i = 0; i < 3; ++i) {
8513 int Idx = OpIndices[i];
8514 if (Idx == -1)
8515 break;
8516
8517 const MachineOperand &MO = MI.getOperand(i: Idx);
8518 if (!MO.isReg())
8519 continue;
8520
8521 // Is this operand statically required to be an SGPR based on the operand
8522 // constraints?
8523 const TargetRegisterClass *OpRC =
8524 RI.getRegClass(RCID: Desc.operands()[Idx].RegClass);
8525 bool IsRequiredSGPR = RI.isSGPRClass(RC: OpRC);
8526 if (IsRequiredSGPR)
8527 return MO.getReg();
8528
8529 // If this could be a VGPR or an SGPR, Check the dynamic register class.
8530 Register Reg = MO.getReg();
8531 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
8532 if (RI.isSGPRClass(RC: RegRC))
8533 UsedSGPRs[i] = Reg;
8534 }
8535
8536 // We don't have a required SGPR operand, so we have a bit more freedom in
8537 // selecting operands to move.
8538
8539 // Try to select the most used SGPR. If an SGPR is equal to one of the
8540 // others, we choose that.
8541 //
8542 // e.g.
8543 // V_FMA_F32 v0, s0, s0, s0 -> No moves
8544 // V_FMA_F32 v0, s0, s1, s0 -> Move s1
8545
8546 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
8547 // prefer those.
8548
8549 if (UsedSGPRs[0]) {
8550 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
8551 SGPRReg = UsedSGPRs[0];
8552 }
8553
8554 if (!SGPRReg && UsedSGPRs[1]) {
8555 if (UsedSGPRs[1] == UsedSGPRs[2])
8556 SGPRReg = UsedSGPRs[1];
8557 }
8558
8559 return SGPRReg;
8560}
8561
8562MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI,
8563 unsigned OperandName) const {
8564 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), NamedIdx: OperandName);
8565 if (Idx == -1)
8566 return nullptr;
8567
8568 return &MI.getOperand(i: Idx);
8569}
8570
8571uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {
8572 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
8573 int64_t Format = ST.getGeneration() >= AMDGPUSubtarget::GFX11
8574 ? (int64_t)AMDGPU::UfmtGFX11::UFMT_32_FLOAT
8575 : (int64_t)AMDGPU::UfmtGFX10::UFMT_32_FLOAT;
8576 return (Format << 44) |
8577 (1ULL << 56) | // RESOURCE_LEVEL = 1
8578 (3ULL << 60); // OOB_SELECT = 3
8579 }
8580
8581 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
8582 if (ST.isAmdHsaOS()) {
8583 // Set ATC = 1. GFX9 doesn't have this bit.
8584 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)
8585 RsrcDataFormat |= (1ULL << 56);
8586
8587 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
8588 // BTW, it disables TC L2 and therefore decreases performance.
8589 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)
8590 RsrcDataFormat |= (2ULL << 59);
8591 }
8592
8593 return RsrcDataFormat;
8594}
8595
8596uint64_t SIInstrInfo::getScratchRsrcWords23() const {
8597 uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
8598 AMDGPU::RSRC_TID_ENABLE |
8599 0xffffffff; // Size;
8600
8601 // GFX9 doesn't have ELEMENT_SIZE.
8602 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
8603 uint64_t EltSizeValue = Log2_32(Value: ST.getMaxPrivateElementSize(ForBufferRSrc: true)) - 1;
8604 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
8605 }
8606
8607 // IndexStride = 64 / 32.
8608 uint64_t IndexStride = ST.getWavefrontSize() == 64 ? 3 : 2;
8609 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
8610
8611 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
8612 // Clear them unless we want a huge stride.
8613 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
8614 ST.getGeneration() <= AMDGPUSubtarget::GFX9)
8615 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
8616
8617 return Rsrc23;
8618}
8619
8620bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const {
8621 unsigned Opc = MI.getOpcode();
8622
8623 return isSMRD(Opcode: Opc);
8624}
8625
8626bool SIInstrInfo::isHighLatencyDef(int Opc) const {
8627 return get(Opcode: Opc).mayLoad() &&
8628 (isMUBUF(Opcode: Opc) || isMTBUF(Opcode: Opc) || isMIMG(Opcode: Opc) || isFLAT(Opcode: Opc));
8629}
8630
8631unsigned SIInstrInfo::isStackAccess(const MachineInstr &MI,
8632 int &FrameIndex) const {
8633 const MachineOperand *Addr = getNamedOperand(MI, OpName: AMDGPU::OpName::vaddr);
8634 if (!Addr || !Addr->isFI())
8635 return Register();
8636
8637 assert(!MI.memoperands_empty() &&
8638 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS);
8639
8640 FrameIndex = Addr->getIndex();
8641 return getNamedOperand(MI, OpName: AMDGPU::OpName::vdata)->getReg();
8642}
8643
8644unsigned SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI,
8645 int &FrameIndex) const {
8646 const MachineOperand *Addr = getNamedOperand(MI, OpName: AMDGPU::OpName::addr);
8647 assert(Addr && Addr->isFI());
8648 FrameIndex = Addr->getIndex();
8649 return getNamedOperand(MI, OpName: AMDGPU::OpName::data)->getReg();
8650}
8651
8652Register SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
8653 int &FrameIndex) const {
8654 if (!MI.mayLoad())
8655 return Register();
8656
8657 if (isMUBUF(MI) || isVGPRSpill(MI))
8658 return isStackAccess(MI, FrameIndex);
8659
8660 if (isSGPRSpill(MI))
8661 return isSGPRStackAccess(MI, FrameIndex);
8662
8663 return Register();
8664}
8665
8666Register SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
8667 int &FrameIndex) const {
8668 if (!MI.mayStore())
8669 return Register();
8670
8671 if (isMUBUF(MI) || isVGPRSpill(MI))
8672 return isStackAccess(MI, FrameIndex);
8673
8674 if (isSGPRSpill(MI))
8675 return isSGPRStackAccess(MI, FrameIndex);
8676
8677 return Register();
8678}
8679
8680unsigned SIInstrInfo::getInstBundleSize(const MachineInstr &MI) const {
8681 unsigned Size = 0;
8682 MachineBasicBlock::const_instr_iterator I = MI.getIterator();
8683 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
8684 while (++I != E && I->isInsideBundle()) {
8685 assert(!I->isBundle() && "No nested bundle!");
8686 Size += getInstSizeInBytes(MI: *I);
8687 }
8688
8689 return Size;
8690}
8691
8692unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
8693 unsigned Opc = MI.getOpcode();
8694 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: Opc);
8695 unsigned DescSize = Desc.getSize();
8696
8697 // If we have a definitive size, we can use it. Otherwise we need to inspect
8698 // the operands to know the size.
8699 if (isFixedSize(MI)) {
8700 unsigned Size = DescSize;
8701
8702 // If we hit the buggy offset, an extra nop will be inserted in MC so
8703 // estimate the worst case.
8704 if (MI.isBranch() && ST.hasOffset3fBug())
8705 Size += 4;
8706
8707 return Size;
8708 }
8709
8710 // Instructions may have a 32-bit literal encoded after them. Check
8711 // operands that could ever be literals.
8712 if (isVALU(MI) || isSALU(MI)) {
8713 if (isDPP(MI))
8714 return DescSize;
8715 bool HasLiteral = false;
8716 for (int I = 0, E = MI.getNumExplicitOperands(); I != E; ++I) {
8717 const MachineOperand &Op = MI.getOperand(i: I);
8718 const MCOperandInfo &OpInfo = Desc.operands()[I];
8719 if (!Op.isReg() && !isInlineConstant(MO: Op, OpInfo)) {
8720 HasLiteral = true;
8721 break;
8722 }
8723 }
8724 return HasLiteral ? DescSize + 4 : DescSize;
8725 }
8726
8727 // Check whether we have extra NSA words.
8728 if (isMIMG(MI)) {
8729 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::vaddr0);
8730 if (VAddr0Idx < 0)
8731 return 8;
8732
8733 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, NamedIdx: AMDGPU::OpName::srsrc);
8734 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4);
8735 }
8736
8737 switch (Opc) {
8738 case TargetOpcode::BUNDLE:
8739 return getInstBundleSize(MI);
8740 case TargetOpcode::INLINEASM:
8741 case TargetOpcode::INLINEASM_BR: {
8742 const MachineFunction *MF = MI.getParent()->getParent();
8743 const char *AsmStr = MI.getOperand(i: 0).getSymbolName();
8744 return getInlineAsmLength(Str: AsmStr, MAI: *MF->getTarget().getMCAsmInfo(), STI: &ST);
8745 }
8746 default:
8747 if (MI.isMetaInstruction())
8748 return 0;
8749 return DescSize;
8750 }
8751}
8752
8753bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const {
8754 if (!isFLAT(MI))
8755 return false;
8756
8757 if (MI.memoperands_empty())
8758 return true;
8759
8760 for (const MachineMemOperand *MMO : MI.memoperands()) {
8761 if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)
8762 return true;
8763 }
8764 return false;
8765}
8766
8767bool SIInstrInfo::isNonUniformBranchInstr(MachineInstr &Branch) const {
8768 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO;
8769}
8770
8771void SIInstrInfo::convertNonUniformIfRegion(MachineBasicBlock *IfEntry,
8772 MachineBasicBlock *IfEnd) const {
8773 MachineBasicBlock::iterator TI = IfEntry->getFirstTerminator();
8774 assert(TI != IfEntry->end());
8775
8776 MachineInstr *Branch = &(*TI);
8777 MachineFunction *MF = IfEntry->getParent();
8778 MachineRegisterInfo &MRI = IfEntry->getParent()->getRegInfo();
8779
8780 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
8781 Register DstReg = MRI.createVirtualRegister(RegClass: RI.getBoolRC());
8782 MachineInstr *SIIF =
8783 BuildMI(MF&: *MF, MIMD: Branch->getDebugLoc(), MCID: get(Opcode: AMDGPU::SI_IF), DestReg: DstReg)
8784 .add(MO: Branch->getOperand(i: 0))
8785 .add(MO: Branch->getOperand(i: 1));
8786 MachineInstr *SIEND =
8787 BuildMI(MF&: *MF, MIMD: Branch->getDebugLoc(), MCID: get(Opcode: AMDGPU::SI_END_CF))
8788 .addReg(RegNo: DstReg);
8789
8790 IfEntry->erase(I: TI);
8791 IfEntry->insert(I: IfEntry->end(), MI: SIIF);
8792 IfEnd->insert(I: IfEnd->getFirstNonPHI(), MI: SIEND);
8793 }
8794}
8795
8796void SIInstrInfo::convertNonUniformLoopRegion(
8797 MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const {
8798 MachineBasicBlock::iterator TI = LoopEnd->getFirstTerminator();
8799 // We expect 2 terminators, one conditional and one unconditional.
8800 assert(TI != LoopEnd->end());
8801
8802 MachineInstr *Branch = &(*TI);
8803 MachineFunction *MF = LoopEnd->getParent();
8804 MachineRegisterInfo &MRI = LoopEnd->getParent()->getRegInfo();
8805
8806 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
8807
8808 Register DstReg = MRI.createVirtualRegister(RegClass: RI.getBoolRC());
8809 Register BackEdgeReg = MRI.createVirtualRegister(RegClass: RI.getBoolRC());
8810 MachineInstrBuilder HeaderPHIBuilder =
8811 BuildMI(MF&: *(MF), MIMD: Branch->getDebugLoc(), MCID: get(Opcode: TargetOpcode::PHI), DestReg: DstReg);
8812 for (MachineBasicBlock *PMBB : LoopEntry->predecessors()) {
8813 if (PMBB == LoopEnd) {
8814 HeaderPHIBuilder.addReg(RegNo: BackEdgeReg);
8815 } else {
8816 Register ZeroReg = MRI.createVirtualRegister(RegClass: RI.getBoolRC());
8817 materializeImmediate(MBB&: *PMBB, MI: PMBB->getFirstTerminator(), DL: DebugLoc(),
8818 DestReg: ZeroReg, Value: 0);
8819 HeaderPHIBuilder.addReg(RegNo: ZeroReg);
8820 }
8821 HeaderPHIBuilder.addMBB(MBB: PMBB);
8822 }
8823 MachineInstr *HeaderPhi = HeaderPHIBuilder;
8824 MachineInstr *SIIFBREAK = BuildMI(MF&: *(MF), MIMD: Branch->getDebugLoc(),
8825 MCID: get(Opcode: AMDGPU::SI_IF_BREAK), DestReg: BackEdgeReg)
8826 .addReg(RegNo: DstReg)
8827 .add(MO: Branch->getOperand(i: 0));
8828 MachineInstr *SILOOP =
8829 BuildMI(MF&: *(MF), MIMD: Branch->getDebugLoc(), MCID: get(Opcode: AMDGPU::SI_LOOP))
8830 .addReg(RegNo: BackEdgeReg)
8831 .addMBB(MBB: LoopEntry);
8832
8833 LoopEntry->insert(I: LoopEntry->begin(), MI: HeaderPhi);
8834 LoopEnd->erase(I: TI);
8835 LoopEnd->insert(I: LoopEnd->end(), MI: SIIFBREAK);
8836 LoopEnd->insert(I: LoopEnd->end(), MI: SILOOP);
8837 }
8838}
8839
8840ArrayRef<std::pair<int, const char *>>
8841SIInstrInfo::getSerializableTargetIndices() const {
8842 static const std::pair<int, const char *> TargetIndices[] = {
8843 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
8844 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
8845 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
8846 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
8847 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
8848 return ArrayRef(TargetIndices);
8849}
8850
8851/// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The
8852/// post-RA version of misched uses CreateTargetMIHazardRecognizer.
8853ScheduleHazardRecognizer *
8854SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
8855 const ScheduleDAG *DAG) const {
8856 return new GCNHazardRecognizer(DAG->MF);
8857}
8858
8859/// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer
8860/// pass.
8861ScheduleHazardRecognizer *
8862SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const {
8863 return new GCNHazardRecognizer(MF);
8864}
8865
8866// Called during:
8867// - pre-RA scheduling and post-RA scheduling
8868ScheduleHazardRecognizer *
8869SIInstrInfo::CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
8870 const ScheduleDAGMI *DAG) const {
8871 // Borrowed from Arm Target
8872 // We would like to restrict this hazard recognizer to only
8873 // post-RA scheduling; we can tell that we're post-RA because we don't
8874 // track VRegLiveness.
8875 if (!DAG->hasVRegLiveness())
8876 return new GCNHazardRecognizer(DAG->MF);
8877 return TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG);
8878}
8879
8880std::pair<unsigned, unsigned>
8881SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
8882 return std::pair(TF & MO_MASK, TF & ~MO_MASK);
8883}
8884
8885ArrayRef<std::pair<unsigned, const char *>>
8886SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
8887 static const std::pair<unsigned, const char *> TargetFlags[] = {
8888 { MO_GOTPCREL, "amdgpu-gotprel" },
8889 { MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo" },
8890 { MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi" },
8891 { MO_REL32_LO, "amdgpu-rel32-lo" },
8892 { MO_REL32_HI, "amdgpu-rel32-hi" },
8893 { MO_ABS32_LO, "amdgpu-abs32-lo" },
8894 { MO_ABS32_HI, "amdgpu-abs32-hi" },
8895 };
8896
8897 return ArrayRef(TargetFlags);
8898}
8899
8900ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
8901SIInstrInfo::getSerializableMachineMemOperandTargetFlags() const {
8902 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
8903 {
8904 {MONoClobber, "amdgpu-noclobber"},
8905 {MOLastUse, "amdgpu-last-use"},
8906 };
8907
8908 return ArrayRef(TargetFlags);
8909}
8910
8911unsigned SIInstrInfo::getLiveRangeSplitOpcode(Register SrcReg,
8912 const MachineFunction &MF) const {
8913 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
8914 assert(SrcReg.isVirtual());
8915 if (MFI->checkFlag(Reg: SrcReg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
8916 return AMDGPU::WWM_COPY;
8917
8918 return AMDGPU::COPY;
8919}
8920
8921bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI,
8922 Register Reg) const {
8923 // We need to handle instructions which may be inserted during register
8924 // allocation to handle the prolog. The initial prolog instruction may have
8925 // been separated from the start of the block by spills and copies inserted
8926 // needed by the prolog. However, the insertions for scalar registers can
8927 // always be placed at the BB top as they are independent of the exec mask
8928 // value.
8929 bool IsNullOrVectorRegister = true;
8930 if (Reg) {
8931 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
8932 IsNullOrVectorRegister = !RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg));
8933 }
8934
8935 uint16_t Opcode = MI.getOpcode();
8936 // FIXME: Copies inserted in the block prolog for live-range split should also
8937 // be included.
8938 return IsNullOrVectorRegister &&
8939 (isSpill(Opcode) || (!MI.isTerminator() && Opcode != AMDGPU::COPY &&
8940 MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &RI)));
8941}
8942
8943MachineInstrBuilder
8944SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
8945 MachineBasicBlock::iterator I,
8946 const DebugLoc &DL,
8947 Register DestReg) const {
8948 if (ST.hasAddNoCarry())
8949 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e64), DestReg);
8950
8951 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8952 Register UnusedCarry = MRI.createVirtualRegister(RegClass: RI.getBoolRC());
8953 MRI.setRegAllocationHint(VReg: UnusedCarry, Type: 0, PrefReg: RI.getVCC());
8954
8955 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg)
8956 .addReg(RegNo: UnusedCarry, flags: RegState::Define | RegState::Dead);
8957}
8958
8959MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
8960 MachineBasicBlock::iterator I,
8961 const DebugLoc &DL,
8962 Register DestReg,
8963 RegScavenger &RS) const {
8964 if (ST.hasAddNoCarry())
8965 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg);
8966
8967 // If available, prefer to use vcc.
8968 Register UnusedCarry = !RS.isRegUsed(Reg: AMDGPU::VCC)
8969 ? Register(RI.getVCC())
8970 : RS.scavengeRegisterBackwards(
8971 RC: *RI.getBoolRC(), To: I, /* RestoreAfter */ false,
8972 SPAdj: 0, /* AllowSpill */ false);
8973
8974 // TODO: Users need to deal with this.
8975 if (!UnusedCarry.isValid())
8976 return MachineInstrBuilder();
8977
8978 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg)
8979 .addReg(RegNo: UnusedCarry, flags: RegState::Define | RegState::Dead);
8980}
8981
8982bool SIInstrInfo::isKillTerminator(unsigned Opcode) {
8983 switch (Opcode) {
8984 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
8985 case AMDGPU::SI_KILL_I1_TERMINATOR:
8986 return true;
8987 default:
8988 return false;
8989 }
8990}
8991
8992const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const {
8993 switch (Opcode) {
8994 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
8995 return get(Opcode: AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);
8996 case AMDGPU::SI_KILL_I1_PSEUDO:
8997 return get(Opcode: AMDGPU::SI_KILL_I1_TERMINATOR);
8998 default:
8999 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO");
9000 }
9001}
9002
9003bool SIInstrInfo::isLegalMUBUFImmOffset(unsigned Imm) const {
9004 return Imm <= getMaxMUBUFImmOffset(ST);
9005}
9006
9007unsigned SIInstrInfo::getMaxMUBUFImmOffset(const GCNSubtarget &ST) {
9008 // GFX12 field is non-negative 24-bit signed byte offset.
9009 const unsigned OffsetBits =
9010 ST.getGeneration() >= AMDGPUSubtarget::GFX12 ? 23 : 12;
9011 return (1 << OffsetBits) - 1;
9012}
9013
9014void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const {
9015 if (!ST.isWave32())
9016 return;
9017
9018 if (MI.isInlineAsm())
9019 return;
9020
9021 for (auto &Op : MI.implicit_operands()) {
9022 if (Op.isReg() && Op.getReg() == AMDGPU::VCC)
9023 Op.setReg(AMDGPU::VCC_LO);
9024 }
9025}
9026
9027bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const {
9028 if (!isSMRD(MI))
9029 return false;
9030
9031 // Check that it is using a buffer resource.
9032 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::sbase);
9033 if (Idx == -1) // e.g. s_memtime
9034 return false;
9035
9036 const auto RCID = MI.getDesc().operands()[Idx].RegClass;
9037 return RI.getRegClass(RCID)->hasSubClassEq(RC: &AMDGPU::SGPR_128RegClass);
9038}
9039
9040// Given Imm, split it into the values to put into the SOffset and ImmOffset
9041// fields in an MUBUF instruction. Return false if it is not possible (due to a
9042// hardware bug needing a workaround).
9043//
9044// The required alignment ensures that individual address components remain
9045// aligned if they are aligned to begin with. It also ensures that additional
9046// offsets within the given alignment can be added to the resulting ImmOffset.
9047bool SIInstrInfo::splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset,
9048 uint32_t &ImmOffset, Align Alignment) const {
9049 const uint32_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST);
9050 const uint32_t MaxImm = alignDown(Value: MaxOffset, Align: Alignment.value());
9051 uint32_t Overflow = 0;
9052
9053 if (Imm > MaxImm) {
9054 if (Imm <= MaxImm + 64) {
9055 // Use an SOffset inline constant for 4..64
9056 Overflow = Imm - MaxImm;
9057 Imm = MaxImm;
9058 } else {
9059 // Try to keep the same value in SOffset for adjacent loads, so that
9060 // the corresponding register contents can be re-used.
9061 //
9062 // Load values with all low-bits (except for alignment bits) set into
9063 // SOffset, so that a larger range of values can be covered using
9064 // s_movk_i32.
9065 //
9066 // Atomic operations fail to work correctly when individual address
9067 // components are unaligned, even if their sum is aligned.
9068 uint32_t High = (Imm + Alignment.value()) & ~MaxOffset;
9069 uint32_t Low = (Imm + Alignment.value()) & MaxOffset;
9070 Imm = Low;
9071 Overflow = High - Alignment.value();
9072 }
9073 }
9074
9075 if (Overflow > 0) {
9076 // There is a hardware bug in SI and CI which prevents address clamping in
9077 // MUBUF instructions from working correctly with SOffsets. The immediate
9078 // offset is unaffected.
9079 if (ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
9080 return false;
9081
9082 // It is not possible to set immediate in SOffset field on some targets.
9083 if (ST.hasRestrictedSOffset())
9084 return false;
9085 }
9086
9087 ImmOffset = Imm;
9088 SOffset = Overflow;
9089 return true;
9090}
9091
9092// Depending on the used address space and instructions, some immediate offsets
9093// are allowed and some are not.
9094// Pre-GFX12, flat instruction offsets can only be non-negative, global and
9095// scratch instruction offsets can also be negative. On GFX12, offsets can be
9096// negative for all variants.
9097//
9098// There are several bugs related to these offsets:
9099// On gfx10.1, flat instructions that go into the global address space cannot
9100// use an offset.
9101//
9102// For scratch instructions, the address can be either an SGPR or a VGPR.
9103// The following offsets can be used, depending on the architecture (x means
9104// cannot be used):
9105// +----------------------------+------+------+
9106// | Address-Mode | SGPR | VGPR |
9107// +----------------------------+------+------+
9108// | gfx9 | | |
9109// | negative, 4-aligned offset | x | ok |
9110// | negative, unaligned offset | x | ok |
9111// +----------------------------+------+------+
9112// | gfx10 | | |
9113// | negative, 4-aligned offset | ok | ok |
9114// | negative, unaligned offset | ok | x |
9115// +----------------------------+------+------+
9116// | gfx10.3 | | |
9117// | negative, 4-aligned offset | ok | ok |
9118// | negative, unaligned offset | ok | ok |
9119// +----------------------------+------+------+
9120//
9121// This function ignores the addressing mode, so if an offset cannot be used in
9122// one addressing mode, it is considered illegal.
9123bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
9124 uint64_t FlatVariant) const {
9125 // TODO: Should 0 be special cased?
9126 if (!ST.hasFlatInstOffsets())
9127 return false;
9128
9129 if (ST.hasFlatSegmentOffsetBug() && FlatVariant == SIInstrFlags::FLAT &&
9130 (AddrSpace == AMDGPUAS::FLAT_ADDRESS ||
9131 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS))
9132 return false;
9133
9134 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
9135 FlatVariant == SIInstrFlags::FlatScratch && Offset < 0 &&
9136 (Offset % 4) != 0) {
9137 return false;
9138 }
9139
9140 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
9141 unsigned N = AMDGPU::getNumFlatOffsetBits(ST);
9142 return isIntN(N, x: Offset) && (AllowNegative || Offset >= 0);
9143}
9144
9145// See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not.
9146std::pair<int64_t, int64_t>
9147SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,
9148 uint64_t FlatVariant) const {
9149 int64_t RemainderOffset = COffsetVal;
9150 int64_t ImmField = 0;
9151
9152 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
9153 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST) - 1;
9154
9155 if (AllowNegative) {
9156 // Use signed division by a power of two to truncate towards 0.
9157 int64_t D = 1LL << NumBits;
9158 RemainderOffset = (COffsetVal / D) * D;
9159 ImmField = COffsetVal - RemainderOffset;
9160
9161 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
9162 FlatVariant == SIInstrFlags::FlatScratch && ImmField < 0 &&
9163 (ImmField % 4) != 0) {
9164 // Make ImmField a multiple of 4
9165 RemainderOffset += ImmField % 4;
9166 ImmField -= ImmField % 4;
9167 }
9168 } else if (COffsetVal >= 0) {
9169 ImmField = COffsetVal & maskTrailingOnes<uint64_t>(N: NumBits);
9170 RemainderOffset = COffsetVal - ImmField;
9171 }
9172
9173 assert(isLegalFLATOffset(ImmField, AddrSpace, FlatVariant));
9174 assert(RemainderOffset + ImmField == COffsetVal);
9175 return {ImmField, RemainderOffset};
9176}
9177
9178bool SIInstrInfo::allowNegativeFlatOffset(uint64_t FlatVariant) const {
9179 if (ST.hasNegativeScratchOffsetBug() &&
9180 FlatVariant == SIInstrFlags::FlatScratch)
9181 return false;
9182
9183 return FlatVariant != SIInstrFlags::FLAT || AMDGPU::isGFX12Plus(STI: ST);
9184}
9185
9186static unsigned subtargetEncodingFamily(const GCNSubtarget &ST) {
9187 switch (ST.getGeneration()) {
9188 default:
9189 break;
9190 case AMDGPUSubtarget::SOUTHERN_ISLANDS:
9191 case AMDGPUSubtarget::SEA_ISLANDS:
9192 return SIEncodingFamily::SI;
9193 case AMDGPUSubtarget::VOLCANIC_ISLANDS:
9194 case AMDGPUSubtarget::GFX9:
9195 return SIEncodingFamily::VI;
9196 case AMDGPUSubtarget::GFX10:
9197 return SIEncodingFamily::GFX10;
9198 case AMDGPUSubtarget::GFX11:
9199 return SIEncodingFamily::GFX11;
9200 case AMDGPUSubtarget::GFX12:
9201 return SIEncodingFamily::GFX12;
9202 }
9203 llvm_unreachable("Unknown subtarget generation!");
9204}
9205
9206bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const {
9207 switch(MCOp) {
9208 // These opcodes use indirect register addressing so
9209 // they need special handling by codegen (currently missing).
9210 // Therefore it is too risky to allow these opcodes
9211 // to be selected by dpp combiner or sdwa peepholer.
9212 case AMDGPU::V_MOVRELS_B32_dpp_gfx10:
9213 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
9214 case AMDGPU::V_MOVRELD_B32_dpp_gfx10:
9215 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10:
9216 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10:
9217 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
9218 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10:
9219 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
9220 return true;
9221 default:
9222 return false;
9223 }
9224}
9225
9226int SIInstrInfo::pseudoToMCOpcode(int Opcode) const {
9227 Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode);
9228
9229 unsigned Gen = subtargetEncodingFamily(ST);
9230
9231 if ((get(Opcode).TSFlags & SIInstrFlags::renamedInGFX9) != 0 &&
9232 ST.getGeneration() == AMDGPUSubtarget::GFX9)
9233 Gen = SIEncodingFamily::GFX9;
9234
9235 // Adjust the encoding family to GFX80 for D16 buffer instructions when the
9236 // subtarget has UnpackedD16VMem feature.
9237 // TODO: remove this when we discard GFX80 encoding.
9238 if (ST.hasUnpackedD16VMem() && (get(Opcode).TSFlags & SIInstrFlags::D16Buf))
9239 Gen = SIEncodingFamily::GFX80;
9240
9241 if (get(Opcode).TSFlags & SIInstrFlags::SDWA) {
9242 switch (ST.getGeneration()) {
9243 default:
9244 Gen = SIEncodingFamily::SDWA;
9245 break;
9246 case AMDGPUSubtarget::GFX9:
9247 Gen = SIEncodingFamily::SDWA9;
9248 break;
9249 case AMDGPUSubtarget::GFX10:
9250 Gen = SIEncodingFamily::SDWA10;
9251 break;
9252 }
9253 }
9254
9255 if (isMAI(Opcode)) {
9256 int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode);
9257 if (MFMAOp != -1)
9258 Opcode = MFMAOp;
9259 }
9260
9261 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen);
9262
9263 // -1 means that Opcode is already a native instruction.
9264 if (MCOp == -1)
9265 return Opcode;
9266
9267 if (ST.hasGFX90AInsts()) {
9268 uint16_t NMCOp = (uint16_t)-1;
9269 if (ST.hasGFX940Insts())
9270 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX940);
9271 if (NMCOp == (uint16_t)-1)
9272 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX90A);
9273 if (NMCOp == (uint16_t)-1)
9274 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX9);
9275 if (NMCOp != (uint16_t)-1)
9276 MCOp = NMCOp;
9277 }
9278
9279 // (uint16_t)-1 means that Opcode is a pseudo instruction that has
9280 // no encoding in the given subtarget generation.
9281 if (MCOp == (uint16_t)-1)
9282 return -1;
9283
9284 if (isAsmOnlyOpcode(MCOp))
9285 return -1;
9286
9287 return MCOp;
9288}
9289
9290static
9291TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) {
9292 assert(RegOpnd.isReg());
9293 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() :
9294 getRegSubRegPair(O: RegOpnd);
9295}
9296
9297TargetInstrInfo::RegSubRegPair
9298llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) {
9299 assert(MI.isRegSequence());
9300 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I)
9301 if (MI.getOperand(i: 1 + 2 * I + 1).getImm() == SubReg) {
9302 auto &RegOp = MI.getOperand(i: 1 + 2 * I);
9303 return getRegOrUndef(RegOpnd: RegOp);
9304 }
9305 return TargetInstrInfo::RegSubRegPair();
9306}
9307
9308// Try to find the definition of reg:subreg in subreg-manipulation pseudos
9309// Following a subreg of reg:subreg isn't supported
9310static bool followSubRegDef(MachineInstr &MI,
9311 TargetInstrInfo::RegSubRegPair &RSR) {
9312 if (!RSR.SubReg)
9313 return false;
9314 switch (MI.getOpcode()) {
9315 default: break;
9316 case AMDGPU::REG_SEQUENCE:
9317 RSR = getRegSequenceSubReg(MI, SubReg: RSR.SubReg);
9318 return true;
9319 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg
9320 case AMDGPU::INSERT_SUBREG:
9321 if (RSR.SubReg == (unsigned)MI.getOperand(i: 3).getImm())
9322 // inserted the subreg we're looking for
9323 RSR = getRegOrUndef(RegOpnd: MI.getOperand(i: 2));
9324 else { // the subreg in the rest of the reg
9325 auto R1 = getRegOrUndef(RegOpnd: MI.getOperand(i: 1));
9326 if (R1.SubReg) // subreg of subreg isn't supported
9327 return false;
9328 RSR.Reg = R1.Reg;
9329 }
9330 return true;
9331 }
9332 return false;
9333}
9334
9335MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
9336 MachineRegisterInfo &MRI) {
9337 assert(MRI.isSSA());
9338 if (!P.Reg.isVirtual())
9339 return nullptr;
9340
9341 auto RSR = P;
9342 auto *DefInst = MRI.getVRegDef(Reg: RSR.Reg);
9343 while (auto *MI = DefInst) {
9344 DefInst = nullptr;
9345 switch (MI->getOpcode()) {
9346 case AMDGPU::COPY:
9347 case AMDGPU::V_MOV_B32_e32: {
9348 auto &Op1 = MI->getOperand(i: 1);
9349 if (Op1.isReg() && Op1.getReg().isVirtual()) {
9350 if (Op1.isUndef())
9351 return nullptr;
9352 RSR = getRegSubRegPair(O: Op1);
9353 DefInst = MRI.getVRegDef(Reg: RSR.Reg);
9354 }
9355 break;
9356 }
9357 default:
9358 if (followSubRegDef(MI&: *MI, RSR)) {
9359 if (!RSR.Reg)
9360 return nullptr;
9361 DefInst = MRI.getVRegDef(Reg: RSR.Reg);
9362 }
9363 }
9364 if (!DefInst)
9365 return MI;
9366 }
9367 return nullptr;
9368}
9369
9370bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
9371 Register VReg,
9372 const MachineInstr &DefMI,
9373 const MachineInstr &UseMI) {
9374 assert(MRI.isSSA() && "Must be run on SSA");
9375
9376 auto *TRI = MRI.getTargetRegisterInfo();
9377 auto *DefBB = DefMI.getParent();
9378
9379 // Don't bother searching between blocks, although it is possible this block
9380 // doesn't modify exec.
9381 if (UseMI.getParent() != DefBB)
9382 return true;
9383
9384 const int MaxInstScan = 20;
9385 int NumInst = 0;
9386
9387 // Stop scan at the use.
9388 auto E = UseMI.getIterator();
9389 for (auto I = std::next(x: DefMI.getIterator()); I != E; ++I) {
9390 if (I->isDebugInstr())
9391 continue;
9392
9393 if (++NumInst > MaxInstScan)
9394 return true;
9395
9396 if (I->modifiesRegister(Reg: AMDGPU::EXEC, TRI))
9397 return true;
9398 }
9399
9400 return false;
9401}
9402
9403bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
9404 Register VReg,
9405 const MachineInstr &DefMI) {
9406 assert(MRI.isSSA() && "Must be run on SSA");
9407
9408 auto *TRI = MRI.getTargetRegisterInfo();
9409 auto *DefBB = DefMI.getParent();
9410
9411 const int MaxUseScan = 10;
9412 int NumUse = 0;
9413
9414 for (auto &Use : MRI.use_nodbg_operands(Reg: VReg)) {
9415 auto &UseInst = *Use.getParent();
9416 // Don't bother searching between blocks, although it is possible this block
9417 // doesn't modify exec.
9418 if (UseInst.getParent() != DefBB || UseInst.isPHI())
9419 return true;
9420
9421 if (++NumUse > MaxUseScan)
9422 return true;
9423 }
9424
9425 if (NumUse == 0)
9426 return false;
9427
9428 const int MaxInstScan = 20;
9429 int NumInst = 0;
9430
9431 // Stop scan when we have seen all the uses.
9432 for (auto I = std::next(x: DefMI.getIterator()); ; ++I) {
9433 assert(I != DefBB->end());
9434
9435 if (I->isDebugInstr())
9436 continue;
9437
9438 if (++NumInst > MaxInstScan)
9439 return true;
9440
9441 for (const MachineOperand &Op : I->operands()) {
9442 // We don't check reg masks here as they're used only on calls:
9443 // 1. EXEC is only considered const within one BB
9444 // 2. Call should be a terminator instruction if present in a BB
9445
9446 if (!Op.isReg())
9447 continue;
9448
9449 Register Reg = Op.getReg();
9450 if (Op.isUse()) {
9451 if (Reg == VReg && --NumUse == 0)
9452 return false;
9453 } else if (TRI->regsOverlap(RegA: Reg, RegB: AMDGPU::EXEC))
9454 return true;
9455 }
9456 }
9457}
9458
9459MachineInstr *SIInstrInfo::createPHIDestinationCopy(
9460 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt,
9461 const DebugLoc &DL, Register Src, Register Dst) const {
9462 auto Cur = MBB.begin();
9463 if (Cur != MBB.end())
9464 do {
9465 if (!Cur->isPHI() && Cur->readsRegister(Reg: Dst, /*TRI=*/nullptr))
9466 return BuildMI(BB&: MBB, I: Cur, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Dst).addReg(RegNo: Src);
9467 ++Cur;
9468 } while (Cur != MBB.end() && Cur != LastPHIIt);
9469
9470 return TargetInstrInfo::createPHIDestinationCopy(MBB, InsPt: LastPHIIt, DL, Src,
9471 Dst);
9472}
9473
9474MachineInstr *SIInstrInfo::createPHISourceCopy(
9475 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt,
9476 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const {
9477 if (InsPt != MBB.end() &&
9478 (InsPt->getOpcode() == AMDGPU::SI_IF ||
9479 InsPt->getOpcode() == AMDGPU::SI_ELSE ||
9480 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) &&
9481 InsPt->definesRegister(Reg: Src, /*TRI=*/nullptr)) {
9482 InsPt++;
9483 return BuildMI(BB&: MBB, I: InsPt, MIMD: DL,
9484 MCID: get(Opcode: ST.isWave32() ? AMDGPU::S_MOV_B32_term
9485 : AMDGPU::S_MOV_B64_term),
9486 DestReg: Dst)
9487 .addReg(RegNo: Src, flags: 0, SubReg: SrcSubReg)
9488 .addReg(RegNo: AMDGPU::EXEC, flags: RegState::Implicit);
9489 }
9490 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg,
9491 Dst);
9492}
9493
9494bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); }
9495
9496MachineInstr *SIInstrInfo::foldMemoryOperandImpl(
9497 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
9498 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS,
9499 VirtRegMap *VRM) const {
9500 // This is a bit of a hack (copied from AArch64). Consider this instruction:
9501 //
9502 // %0:sreg_32 = COPY $m0
9503 //
9504 // We explicitly chose SReg_32 for the virtual register so such a copy might
9505 // be eliminated by RegisterCoalescer. However, that may not be possible, and
9506 // %0 may even spill. We can't spill $m0 normally (it would require copying to
9507 // a numbered SGPR anyway), and since it is in the SReg_32 register class,
9508 // TargetInstrInfo::foldMemoryOperand() is going to try.
9509 // A similar issue also exists with spilling and reloading $exec registers.
9510 //
9511 // To prevent that, constrain the %0 register class here.
9512 if (isFullCopyInstr(MI)) {
9513 Register DstReg = MI.getOperand(i: 0).getReg();
9514 Register SrcReg = MI.getOperand(i: 1).getReg();
9515 if ((DstReg.isVirtual() || SrcReg.isVirtual()) &&
9516 (DstReg.isVirtual() != SrcReg.isVirtual())) {
9517 MachineRegisterInfo &MRI = MF.getRegInfo();
9518 Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg;
9519 const TargetRegisterClass *RC = MRI.getRegClass(Reg: VirtReg);
9520 if (RC->hasSuperClassEq(RC: &AMDGPU::SReg_32RegClass)) {
9521 MRI.constrainRegClass(Reg: VirtReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
9522 return nullptr;
9523 }
9524 if (RC->hasSuperClassEq(RC: &AMDGPU::SReg_64RegClass)) {
9525 MRI.constrainRegClass(Reg: VirtReg, RC: &AMDGPU::SReg_64_XEXECRegClass);
9526 return nullptr;
9527 }
9528 }
9529 }
9530
9531 return nullptr;
9532}
9533
9534unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
9535 const MachineInstr &MI,
9536 unsigned *PredCost) const {
9537 if (MI.isBundle()) {
9538 MachineBasicBlock::const_instr_iterator I(MI.getIterator());
9539 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end());
9540 unsigned Lat = 0, Count = 0;
9541 for (++I; I != E && I->isBundledWithPred(); ++I) {
9542 ++Count;
9543 Lat = std::max(a: Lat, b: SchedModel.computeInstrLatency(MI: &*I));
9544 }
9545 return Lat + Count - 1;
9546 }
9547
9548 return SchedModel.computeInstrLatency(MI: &MI);
9549}
9550
9551InstructionUniformity
9552SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const {
9553 unsigned opcode = MI.getOpcode();
9554 if (auto *GI = dyn_cast<GIntrinsic>(Val: &MI)) {
9555 auto IID = GI->getIntrinsicID();
9556 if (AMDGPU::isIntrinsicSourceOfDivergence(IntrID: IID))
9557 return InstructionUniformity::NeverUniform;
9558 if (AMDGPU::isIntrinsicAlwaysUniform(IntrID: IID))
9559 return InstructionUniformity::AlwaysUniform;
9560
9561 switch (IID) {
9562 case Intrinsic::amdgcn_if:
9563 case Intrinsic::amdgcn_else:
9564 // FIXME: Uniform if second result
9565 break;
9566 }
9567
9568 return InstructionUniformity::Default;
9569 }
9570
9571 // Loads from the private and flat address spaces are divergent, because
9572 // threads can execute the load instruction with the same inputs and get
9573 // different results.
9574 //
9575 // All other loads are not divergent, because if threads issue loads with the
9576 // same arguments, they will always get the same result.
9577 if (opcode == AMDGPU::G_LOAD) {
9578 if (MI.memoperands_empty())
9579 return InstructionUniformity::NeverUniform; // conservative assumption
9580
9581 if (llvm::any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *mmo) {
9582 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
9583 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
9584 })) {
9585 // At least one MMO in a non-global address space.
9586 return InstructionUniformity::NeverUniform;
9587 }
9588 return InstructionUniformity::Default;
9589 }
9590
9591 if (SIInstrInfo::isGenericAtomicRMWOpcode(Opc: opcode) ||
9592 opcode == AMDGPU::G_ATOMIC_CMPXCHG ||
9593 opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS ||
9594 AMDGPU::isGenericAtomic(Opc: opcode)) {
9595 return InstructionUniformity::NeverUniform;
9596 }
9597 return InstructionUniformity::Default;
9598}
9599
9600InstructionUniformity
9601SIInstrInfo::getInstructionUniformity(const MachineInstr &MI) const {
9602
9603 if (isNeverUniform(MI))
9604 return InstructionUniformity::NeverUniform;
9605
9606 unsigned opcode = MI.getOpcode();
9607 if (opcode == AMDGPU::V_READLANE_B32 ||
9608 opcode == AMDGPU::V_READFIRSTLANE_B32 ||
9609 opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR)
9610 return InstructionUniformity::AlwaysUniform;
9611
9612 if (isCopyInstr(MI)) {
9613 const MachineOperand &srcOp = MI.getOperand(i: 1);
9614 if (srcOp.isReg() && srcOp.getReg().isPhysical()) {
9615 const TargetRegisterClass *regClass =
9616 RI.getPhysRegBaseClass(Reg: srcOp.getReg());
9617 return RI.isSGPRClass(RC: regClass) ? InstructionUniformity::AlwaysUniform
9618 : InstructionUniformity::NeverUniform;
9619 }
9620 return InstructionUniformity::Default;
9621 }
9622
9623 // GMIR handling
9624 if (MI.isPreISelOpcode())
9625 return SIInstrInfo::getGenericInstructionUniformity(MI);
9626
9627 // Atomics are divergent because they are executed sequentially: when an
9628 // atomic operation refers to the same address in each thread, then each
9629 // thread after the first sees the value written by the previous thread as
9630 // original value.
9631
9632 if (isAtomic(MI))
9633 return InstructionUniformity::NeverUniform;
9634
9635 // Loads from the private and flat address spaces are divergent, because
9636 // threads can execute the load instruction with the same inputs and get
9637 // different results.
9638 if (isFLAT(MI) && MI.mayLoad()) {
9639 if (MI.memoperands_empty())
9640 return InstructionUniformity::NeverUniform; // conservative assumption
9641
9642 if (llvm::any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *mmo) {
9643 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
9644 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
9645 })) {
9646 // At least one MMO in a non-global address space.
9647 return InstructionUniformity::NeverUniform;
9648 }
9649
9650 return InstructionUniformity::Default;
9651 }
9652
9653 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
9654 const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo();
9655
9656 // FIXME: It's conceptually broken to report this for an instruction, and not
9657 // a specific def operand. For inline asm in particular, there could be mixed
9658 // uniform and divergent results.
9659 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
9660 const MachineOperand &SrcOp = MI.getOperand(i: I);
9661 if (!SrcOp.isReg())
9662 continue;
9663
9664 Register Reg = SrcOp.getReg();
9665 if (!Reg || !SrcOp.readsReg())
9666 continue;
9667
9668 // If RegBank is null, this is unassigned or an unallocatable special
9669 // register, which are all scalars.
9670 const RegisterBank *RegBank = RBI->getRegBank(Reg, MRI, TRI: RI);
9671 if (RegBank && RegBank->getID() != AMDGPU::SGPRRegBankID)
9672 return InstructionUniformity::NeverUniform;
9673 }
9674
9675 // TODO: Uniformity check condtions above can be rearranged for more
9676 // redability
9677
9678 // TODO: amdgcn.{ballot, [if]cmp} should be AlwaysUniform, but they are
9679 // currently turned into no-op COPYs by SelectionDAG ISel and are
9680 // therefore no longer recognizable.
9681
9682 return InstructionUniformity::Default;
9683}
9684
9685unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) {
9686 switch (MF.getFunction().getCallingConv()) {
9687 case CallingConv::AMDGPU_PS:
9688 return 1;
9689 case CallingConv::AMDGPU_VS:
9690 return 2;
9691 case CallingConv::AMDGPU_GS:
9692 return 3;
9693 case CallingConv::AMDGPU_HS:
9694 case CallingConv::AMDGPU_LS:
9695 case CallingConv::AMDGPU_ES:
9696 report_fatal_error(reason: "ds_ordered_count unsupported for this calling conv");
9697 case CallingConv::AMDGPU_CS:
9698 case CallingConv::AMDGPU_KERNEL:
9699 case CallingConv::C:
9700 case CallingConv::Fast:
9701 default:
9702 // Assume other calling conventions are various compute callable functions
9703 return 0;
9704 }
9705}
9706
9707bool SIInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
9708 Register &SrcReg2, int64_t &CmpMask,
9709 int64_t &CmpValue) const {
9710 if (!MI.getOperand(i: 0).isReg() || MI.getOperand(i: 0).getSubReg())
9711 return false;
9712
9713 switch (MI.getOpcode()) {
9714 default:
9715 break;
9716 case AMDGPU::S_CMP_EQ_U32:
9717 case AMDGPU::S_CMP_EQ_I32:
9718 case AMDGPU::S_CMP_LG_U32:
9719 case AMDGPU::S_CMP_LG_I32:
9720 case AMDGPU::S_CMP_LT_U32:
9721 case AMDGPU::S_CMP_LT_I32:
9722 case AMDGPU::S_CMP_GT_U32:
9723 case AMDGPU::S_CMP_GT_I32:
9724 case AMDGPU::S_CMP_LE_U32:
9725 case AMDGPU::S_CMP_LE_I32:
9726 case AMDGPU::S_CMP_GE_U32:
9727 case AMDGPU::S_CMP_GE_I32:
9728 case AMDGPU::S_CMP_EQ_U64:
9729 case AMDGPU::S_CMP_LG_U64:
9730 SrcReg = MI.getOperand(i: 0).getReg();
9731 if (MI.getOperand(i: 1).isReg()) {
9732 if (MI.getOperand(i: 1).getSubReg())
9733 return false;
9734 SrcReg2 = MI.getOperand(i: 1).getReg();
9735 CmpValue = 0;
9736 } else if (MI.getOperand(i: 1).isImm()) {
9737 SrcReg2 = Register();
9738 CmpValue = MI.getOperand(i: 1).getImm();
9739 } else {
9740 return false;
9741 }
9742 CmpMask = ~0;
9743 return true;
9744 case AMDGPU::S_CMPK_EQ_U32:
9745 case AMDGPU::S_CMPK_EQ_I32:
9746 case AMDGPU::S_CMPK_LG_U32:
9747 case AMDGPU::S_CMPK_LG_I32:
9748 case AMDGPU::S_CMPK_LT_U32:
9749 case AMDGPU::S_CMPK_LT_I32:
9750 case AMDGPU::S_CMPK_GT_U32:
9751 case AMDGPU::S_CMPK_GT_I32:
9752 case AMDGPU::S_CMPK_LE_U32:
9753 case AMDGPU::S_CMPK_LE_I32:
9754 case AMDGPU::S_CMPK_GE_U32:
9755 case AMDGPU::S_CMPK_GE_I32:
9756 SrcReg = MI.getOperand(i: 0).getReg();
9757 SrcReg2 = Register();
9758 CmpValue = MI.getOperand(i: 1).getImm();
9759 CmpMask = ~0;
9760 return true;
9761 }
9762
9763 return false;
9764}
9765
9766bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
9767 Register SrcReg2, int64_t CmpMask,
9768 int64_t CmpValue,
9769 const MachineRegisterInfo *MRI) const {
9770 if (!SrcReg || SrcReg.isPhysical())
9771 return false;
9772
9773 if (SrcReg2 && !getFoldableImm(Reg: SrcReg2, MRI: *MRI, Imm&: CmpValue))
9774 return false;
9775
9776 const auto optimizeCmpAnd = [&CmpInstr, SrcReg, CmpValue, MRI,
9777 this](int64_t ExpectedValue, unsigned SrcSize,
9778 bool IsReversible, bool IsSigned) -> bool {
9779 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
9780 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
9781 // s_cmp_ge_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
9782 // s_cmp_ge_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
9783 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 1 << n => s_and_b64 $src, 1 << n
9784 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
9785 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
9786 // s_cmp_gt_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
9787 // s_cmp_gt_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
9788 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 0 => s_and_b64 $src, 1 << n
9789 //
9790 // Signed ge/gt are not used for the sign bit.
9791 //
9792 // If result of the AND is unused except in the compare:
9793 // s_and_b(32|64) $src, 1 << n => s_bitcmp1_b(32|64) $src, n
9794 //
9795 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
9796 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
9797 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 0 => s_bitcmp0_b64 $src, n
9798 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
9799 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
9800 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 1 << n => s_bitcmp0_b64 $src, n
9801
9802 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: SrcReg);
9803 if (!Def || Def->getParent() != CmpInstr.getParent())
9804 return false;
9805
9806 if (Def->getOpcode() != AMDGPU::S_AND_B32 &&
9807 Def->getOpcode() != AMDGPU::S_AND_B64)
9808 return false;
9809
9810 int64_t Mask;
9811 const auto isMask = [&Mask, SrcSize](const MachineOperand *MO) -> bool {
9812 if (MO->isImm())
9813 Mask = MO->getImm();
9814 else if (!getFoldableImm(MO, Imm&: Mask))
9815 return false;
9816 Mask &= maxUIntN(N: SrcSize);
9817 return isPowerOf2_64(Value: Mask);
9818 };
9819
9820 MachineOperand *SrcOp = &Def->getOperand(i: 1);
9821 if (isMask(SrcOp))
9822 SrcOp = &Def->getOperand(i: 2);
9823 else if (isMask(&Def->getOperand(i: 2)))
9824 SrcOp = &Def->getOperand(i: 1);
9825 else
9826 return false;
9827
9828 unsigned BitNo = llvm::countr_zero(Val: (uint64_t)Mask);
9829 if (IsSigned && BitNo == SrcSize - 1)
9830 return false;
9831
9832 ExpectedValue <<= BitNo;
9833
9834 bool IsReversedCC = false;
9835 if (CmpValue != ExpectedValue) {
9836 if (!IsReversible)
9837 return false;
9838 IsReversedCC = CmpValue == (ExpectedValue ^ Mask);
9839 if (!IsReversedCC)
9840 return false;
9841 }
9842
9843 Register DefReg = Def->getOperand(i: 0).getReg();
9844 if (IsReversedCC && !MRI->hasOneNonDBGUse(RegNo: DefReg))
9845 return false;
9846
9847 for (auto I = std::next(x: Def->getIterator()), E = CmpInstr.getIterator();
9848 I != E; ++I) {
9849 if (I->modifiesRegister(Reg: AMDGPU::SCC, TRI: &RI) ||
9850 I->killsRegister(Reg: AMDGPU::SCC, TRI: &RI))
9851 return false;
9852 }
9853
9854 MachineOperand *SccDef =
9855 Def->findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
9856 SccDef->setIsDead(false);
9857 CmpInstr.eraseFromParent();
9858
9859 if (!MRI->use_nodbg_empty(RegNo: DefReg)) {
9860 assert(!IsReversedCC);
9861 return true;
9862 }
9863
9864 // Replace AND with unused result with a S_BITCMP.
9865 MachineBasicBlock *MBB = Def->getParent();
9866
9867 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32
9868 : AMDGPU::S_BITCMP1_B32
9869 : IsReversedCC ? AMDGPU::S_BITCMP0_B64
9870 : AMDGPU::S_BITCMP1_B64;
9871
9872 BuildMI(BB&: *MBB, I: Def, MIMD: Def->getDebugLoc(), MCID: get(Opcode: NewOpc))
9873 .add(MO: *SrcOp)
9874 .addImm(Val: BitNo);
9875 Def->eraseFromParent();
9876
9877 return true;
9878 };
9879
9880 switch (CmpInstr.getOpcode()) {
9881 default:
9882 break;
9883 case AMDGPU::S_CMP_EQ_U32:
9884 case AMDGPU::S_CMP_EQ_I32:
9885 case AMDGPU::S_CMPK_EQ_U32:
9886 case AMDGPU::S_CMPK_EQ_I32:
9887 return optimizeCmpAnd(1, 32, true, false);
9888 case AMDGPU::S_CMP_GE_U32:
9889 case AMDGPU::S_CMPK_GE_U32:
9890 return optimizeCmpAnd(1, 32, false, false);
9891 case AMDGPU::S_CMP_GE_I32:
9892 case AMDGPU::S_CMPK_GE_I32:
9893 return optimizeCmpAnd(1, 32, false, true);
9894 case AMDGPU::S_CMP_EQ_U64:
9895 return optimizeCmpAnd(1, 64, true, false);
9896 case AMDGPU::S_CMP_LG_U32:
9897 case AMDGPU::S_CMP_LG_I32:
9898 case AMDGPU::S_CMPK_LG_U32:
9899 case AMDGPU::S_CMPK_LG_I32:
9900 return optimizeCmpAnd(0, 32, true, false);
9901 case AMDGPU::S_CMP_GT_U32:
9902 case AMDGPU::S_CMPK_GT_U32:
9903 return optimizeCmpAnd(0, 32, false, false);
9904 case AMDGPU::S_CMP_GT_I32:
9905 case AMDGPU::S_CMPK_GT_I32:
9906 return optimizeCmpAnd(0, 32, false, true);
9907 case AMDGPU::S_CMP_LG_U64:
9908 return optimizeCmpAnd(0, 64, true, false);
9909 }
9910
9911 return false;
9912}
9913
9914void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI,
9915 unsigned OpName) const {
9916 if (!ST.needsAlignedVGPRs())
9917 return;
9918
9919 int OpNo = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), NamedIdx: OpName);
9920 if (OpNo < 0)
9921 return;
9922 MachineOperand &Op = MI.getOperand(i: OpNo);
9923 if (getOpSize(MI, OpNo) > 4)
9924 return;
9925
9926 // Add implicit aligned super-reg to force alignment on the data operand.
9927 const DebugLoc &DL = MI.getDebugLoc();
9928 MachineBasicBlock *BB = MI.getParent();
9929 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
9930 Register DataReg = Op.getReg();
9931 bool IsAGPR = RI.isAGPR(MRI, Reg: DataReg);
9932 Register Undef = MRI.createVirtualRegister(
9933 RegClass: IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);
9934 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
9935 Register NewVR =
9936 MRI.createVirtualRegister(RegClass: IsAGPR ? &AMDGPU::AReg_64_Align2RegClass
9937 : &AMDGPU::VReg_64_Align2RegClass);
9938 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewVR)
9939 .addReg(RegNo: DataReg, flags: 0, SubReg: Op.getSubReg())
9940 .addImm(Val: AMDGPU::sub0)
9941 .addReg(RegNo: Undef)
9942 .addImm(Val: AMDGPU::sub1);
9943 Op.setReg(NewVR);
9944 Op.setSubReg(AMDGPU::sub0);
9945 MI.addOperand(Op: MachineOperand::CreateReg(Reg: NewVR, isDef: false, isImp: true));
9946}
9947