1//===- SIInstrInfo.h - SI Instruction Info Interface ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Interface definition for SIInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
15#define LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
16
17#include "AMDGPUMIRFormatter.h"
18#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
19#include "SIRegisterInfo.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/CodeGen/TargetInstrInfo.h"
23#include "llvm/CodeGen/TargetSchedule.h"
24
25#define GET_INSTRINFO_HEADER
26#include "AMDGPUGenInstrInfo.inc"
27
28namespace llvm {
29
30class APInt;
31class GCNSubtarget;
32class LiveVariables;
33class MachineDominatorTree;
34class MachineRegisterInfo;
35class RegScavenger;
36class SIMachineFunctionInfo;
37class TargetRegisterClass;
38class ScheduleHazardRecognizer;
39
40constexpr unsigned DefaultMemoryClusterDWordsLimit = 8;
41
42/// Mark the MMO of a uniform load if there are no potentially clobbering stores
43/// on any path from the start of an entry function to this load.
44static const MachineMemOperand::Flags MONoClobber =
45 MachineMemOperand::MOTargetFlag1;
46
47/// Mark the MMO of a load as the last use.
48static const MachineMemOperand::Flags MOLastUse =
49 MachineMemOperand::MOTargetFlag2;
50
51/// Mark the MMO of cooperative load/store atomics.
52static const MachineMemOperand::Flags MOCooperative =
53 MachineMemOperand::MOTargetFlag3;
54
55/// Mark the MMO of accesses to memory locations that are
56/// never written to by other threads.
57static const MachineMemOperand::Flags MOThreadPrivate =
58 MachineMemOperand::MOTargetFlag4;
59
60/// Utility to store machine instructions worklist.
61struct SIInstrWorklist {
62 SIInstrWorklist() = default;
63
64 void insert(MachineInstr *MI);
65
66 MachineInstr *top() const {
67 const auto *iter = InstrList.begin();
68 return *iter;
69 }
70
71 void erase_top() {
72 const auto *iter = InstrList.begin();
73 InstrList.erase(I: iter);
74 }
75
76 bool empty() const { return InstrList.empty(); }
77
78 void clear() {
79 InstrList.clear();
80 DeferredList.clear();
81 }
82
83 bool isDeferred(MachineInstr *MI);
84
85 SetVector<MachineInstr *> &getDeferredList() { return DeferredList; }
86
87private:
88 /// InstrList contains the MachineInstrs.
89 SetVector<MachineInstr *> InstrList;
90 /// Deferred instructions are specific MachineInstr
91 /// that will be added by insert method.
92 SetVector<MachineInstr *> DeferredList;
93};
94
95class SIInstrInfo final : public AMDGPUGenInstrInfo {
96 struct ThreeAddressUpdates;
97
98private:
99 const SIRegisterInfo RI;
100 const GCNSubtarget &ST;
101 TargetSchedModel SchedModel;
102 mutable std::unique_ptr<AMDGPUMIRFormatter> Formatter;
103
104 // The inverse predicate should have the negative value.
105 enum BranchPredicate {
106 INVALID_BR = 0,
107 SCC_TRUE = 1,
108 SCC_FALSE = -1,
109 VCCNZ = 2,
110 VCCZ = -2,
111 EXECNZ = -3,
112 EXECZ = 3
113 };
114
115 using SetVectorType = SmallSetVector<MachineInstr *, 32>;
116
117 static unsigned getBranchOpcode(BranchPredicate Cond);
118 static BranchPredicate getBranchPredicate(unsigned Opcode);
119
120public:
121 unsigned buildExtractSubReg(MachineBasicBlock::iterator MI,
122 MachineRegisterInfo &MRI,
123 const MachineOperand &SuperReg,
124 const TargetRegisterClass *SuperRC,
125 unsigned SubIdx,
126 const TargetRegisterClass *SubRC) const;
127 MachineOperand buildExtractSubRegOrImm(
128 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
129 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
130 unsigned SubIdx, const TargetRegisterClass *SubRC) const;
131
132private:
133 bool optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,
134 bool NeedInversion) const;
135
136 bool invertSCCUse(MachineInstr *SCCDef) const;
137
138 void swapOperands(MachineInstr &Inst) const;
139
140 std::pair<bool, MachineBasicBlock *>
141 moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
142 MachineDominatorTree *MDT = nullptr) const;
143
144 void lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
145 MachineDominatorTree *MDT = nullptr) const;
146
147 void lowerScalarAbs(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
148
149 void lowerScalarAbsDiff(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
150
151 void lowerScalarXnor(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
152
153 void splitScalarNotBinop(SIInstrWorklist &Worklist, MachineInstr &Inst,
154 unsigned Opcode) const;
155
156 void splitScalarBinOpN2(SIInstrWorklist &Worklist, MachineInstr &Inst,
157 unsigned Opcode) const;
158
159 void splitScalar64BitUnaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
160 unsigned Opcode, bool Swap = false) const;
161
162 void splitScalar64BitBinaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
163 unsigned Opcode,
164 MachineDominatorTree *MDT = nullptr) const;
165
166 void splitScalarSMulU64(SIInstrWorklist &Worklist, MachineInstr &Inst,
167 MachineDominatorTree *MDT) const;
168
169 void splitScalarSMulPseudo(SIInstrWorklist &Worklist, MachineInstr &Inst,
170 MachineDominatorTree *MDT) const;
171
172 void splitScalar64BitXnor(SIInstrWorklist &Worklist, MachineInstr &Inst,
173 MachineDominatorTree *MDT = nullptr) const;
174
175 void splitScalar64BitBCNT(SIInstrWorklist &Worklist,
176 MachineInstr &Inst) const;
177 void splitScalar64BitBFE(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
178 void splitScalar64BitCountOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
179 unsigned Opcode,
180 MachineDominatorTree *MDT = nullptr) const;
181 void movePackToVALU(SIInstrWorklist &Worklist, MachineRegisterInfo &MRI,
182 MachineInstr &Inst) const;
183
184 void addUsersToMoveToVALUWorklist(Register Reg, MachineRegisterInfo &MRI,
185 SIInstrWorklist &Worklist) const;
186
187 void addSCCDefUsersToVALUWorklist(const MachineOperand &Op,
188 MachineInstr &SCCDefInst,
189 SIInstrWorklist &Worklist,
190 Register NewCond = Register()) const;
191 void addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
192 SIInstrWorklist &Worklist) const;
193
194 const TargetRegisterClass *
195 getDestEquivalentVGPRClass(const MachineInstr &Inst) const;
196
197 bool checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
198 const MachineInstr &MIb) const;
199
200 Register findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const;
201
202 bool verifyCopy(const MachineInstr &MI, const MachineRegisterInfo &MRI,
203 StringRef &ErrInfo) const;
204
205 bool resultDependsOnExec(const MachineInstr &MI) const;
206
207 MachineInstr *convertToThreeAddressImpl(MachineInstr &MI,
208 ThreeAddressUpdates &Updates) const;
209
210protected:
211 /// If the specific machine instruction is a instruction that moves/copies
212 /// value from one register to another register return destination and source
213 /// registers as machine operands.
214 std::optional<DestSourcePair>
215 isCopyInstrImpl(const MachineInstr &MI) const override;
216
217 bool swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,
218 AMDGPU::OpName Src0OpName, MachineOperand &Src1,
219 AMDGPU::OpName Src1OpName) const;
220 bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx,
221 unsigned toIdx) const;
222 MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
223 unsigned OpIdx0,
224 unsigned OpIdx1) const override;
225
226public:
227 enum TargetOperandFlags {
228 MO_MASK = 0xf,
229
230 MO_NONE = 0,
231 // MO_GOTPCREL -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
232 MO_GOTPCREL = 1,
233 // MO_GOTPCREL32_LO -> symbol@gotpcrel32@lo -> R_AMDGPU_GOTPCREL32_LO.
234 MO_GOTPCREL32 = 2,
235 MO_GOTPCREL32_LO = 2,
236 // MO_GOTPCREL32_HI -> symbol@gotpcrel32@hi -> R_AMDGPU_GOTPCREL32_HI.
237 MO_GOTPCREL32_HI = 3,
238 // MO_GOTPCREL64 -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
239 MO_GOTPCREL64 = 4,
240 // MO_REL32_LO -> symbol@rel32@lo -> R_AMDGPU_REL32_LO.
241 MO_REL32 = 5,
242 MO_REL32_LO = 5,
243 // MO_REL32_HI -> symbol@rel32@hi -> R_AMDGPU_REL32_HI.
244 MO_REL32_HI = 6,
245 MO_REL64 = 7,
246
247 MO_FAR_BRANCH_OFFSET = 8,
248
249 MO_ABS32_LO = 9,
250 MO_ABS32_HI = 10,
251 MO_ABS64 = 11,
252 };
253
254 explicit SIInstrInfo(const GCNSubtarget &ST);
255
256 const SIRegisterInfo &getRegisterInfo() const {
257 return RI;
258 }
259
260 const GCNSubtarget &getSubtarget() const {
261 return ST;
262 }
263
264 bool isReMaterializableImpl(const MachineInstr &MI) const override;
265
266 bool isIgnorableUse(const MachineOperand &MO) const override;
267
268 bool isSafeToSink(MachineInstr &MI, MachineBasicBlock *SuccToSinkTo,
269 MachineCycleInfo *CI) const override;
270
271 bool areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, int64_t &Offset0,
272 int64_t &Offset1) const override;
273
274 bool isGlobalMemoryObject(const MachineInstr *MI) const override;
275
276 bool getMemOperandsWithOffsetWidth(
277 const MachineInstr &LdSt,
278 SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
279 bool &OffsetIsScalable, LocationSize &Width,
280 const TargetRegisterInfo *TRI) const final;
281
282 bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
283 int64_t Offset1, bool OffsetIsScalable1,
284 ArrayRef<const MachineOperand *> BaseOps2,
285 int64_t Offset2, bool OffsetIsScalable2,
286 unsigned ClusterSize,
287 unsigned NumBytes) const override;
288
289 bool shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0,
290 int64_t Offset1, unsigned NumLoads) const override;
291
292 void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
293 const DebugLoc &DL, Register DestReg, Register SrcReg,
294 bool KillSrc, bool RenamableDest = false,
295 bool RenamableSrc = false) const override;
296
297 const TargetRegisterClass *getPreferredSelectRegClass(
298 unsigned Size) const;
299
300 Register insertNE(MachineBasicBlock *MBB,
301 MachineBasicBlock::iterator I, const DebugLoc &DL,
302 Register SrcReg, int Value) const;
303
304 Register insertEQ(MachineBasicBlock *MBB,
305 MachineBasicBlock::iterator I, const DebugLoc &DL,
306 Register SrcReg, int Value) const;
307
308 bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg,
309 int64_t &ImmVal) const override;
310
311 std::optional<int64_t> getImmOrMaterializedImm(MachineOperand &Op) const;
312
313 unsigned getVectorRegSpillSaveOpcode(Register Reg,
314 const TargetRegisterClass *RC,
315 unsigned Size,
316 const SIMachineFunctionInfo &MFI) const;
317 unsigned
318 getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC,
319 unsigned Size,
320 const SIMachineFunctionInfo &MFI) const;
321
322 void storeRegToStackSlot(
323 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
324 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
325 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
326
327 void loadRegFromStackSlot(
328 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
329 int FrameIndex, const TargetRegisterClass *RC, Register VReg,
330 unsigned SubReg = 0,
331 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
332
333 bool expandPostRAPseudo(MachineInstr &MI) const override;
334
335 void
336 reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
337 Register DestReg, unsigned SubIdx, const MachineInstr &Orig,
338 LaneBitmask UsedLanes = LaneBitmask::getAll()) const override;
339
340 // Splits a V_MOV_B64_DPP_PSEUDO opcode into a pair of v_mov_b32_dpp
341 // instructions. Returns a pair of generated instructions.
342 // Can split either post-RA with physical registers or pre-RA with
343 // virtual registers. In latter case IR needs to be in SSA form and
344 // and a REG_SEQUENCE is produced to define original register.
345 std::pair<MachineInstr*, MachineInstr*>
346 expandMovDPP64(MachineInstr &MI) const;
347
348 // Returns an opcode that can be used to move a value to a \p DstRC
349 // register. If there is no hardware instruction that can store to \p
350 // DstRC, then AMDGPU::COPY is returned.
351 unsigned getMovOpcode(const TargetRegisterClass *DstRC) const;
352
353 const MCInstrDesc &getIndirectRegWriteMovRelPseudo(unsigned VecSize,
354 unsigned EltSize,
355 bool IsSGPR) const;
356
357 const MCInstrDesc &getIndirectGPRIDXPseudo(unsigned VecSize,
358 bool IsIndirectSrc) const;
359 LLVM_READONLY
360 int commuteOpcode(unsigned Opc) const;
361
362 LLVM_READONLY
363 inline int commuteOpcode(const MachineInstr &MI) const {
364 return commuteOpcode(Opc: MI.getOpcode());
365 }
366
367 bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0,
368 unsigned &SrcOpIdx1) const override;
369
370 bool findCommutedOpIndices(const MCInstrDesc &Desc, unsigned &SrcOpIdx0,
371 unsigned &SrcOpIdx1) const;
372
373 bool isBranchOffsetInRange(unsigned BranchOpc,
374 int64_t BrOffset) const override;
375
376 MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;
377
378 /// Return whether the block terminate with divergent branch.
379 /// Note this only work before lowering the pseudo control flow instructions.
380 bool hasDivergentBranch(const MachineBasicBlock *MBB) const;
381
382 void insertIndirectBranch(MachineBasicBlock &MBB,
383 MachineBasicBlock &NewDestBB,
384 MachineBasicBlock &RestoreBB, const DebugLoc &DL,
385 int64_t BrOffset, RegScavenger *RS) const override;
386
387 bool analyzeBranchImpl(MachineBasicBlock &MBB,
388 MachineBasicBlock::iterator I,
389 MachineBasicBlock *&TBB,
390 MachineBasicBlock *&FBB,
391 SmallVectorImpl<MachineOperand> &Cond,
392 bool AllowModify) const;
393
394 bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
395 MachineBasicBlock *&FBB,
396 SmallVectorImpl<MachineOperand> &Cond,
397 bool AllowModify = false) const override;
398
399 unsigned removeBranch(MachineBasicBlock &MBB,
400 int *BytesRemoved = nullptr) const override;
401
402 unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
403 MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
404 const DebugLoc &DL,
405 int *BytesAdded = nullptr) const override;
406
407 bool reverseBranchCondition(
408 SmallVectorImpl<MachineOperand> &Cond) const override;
409
410 bool canInsertSelect(const MachineBasicBlock &MBB,
411 ArrayRef<MachineOperand> Cond, Register DstReg,
412 Register TrueReg, Register FalseReg, int &CondCycles,
413 int &TrueCycles, int &FalseCycles) const override;
414
415 void insertSelect(MachineBasicBlock &MBB,
416 MachineBasicBlock::iterator I, const DebugLoc &DL,
417 Register DstReg, ArrayRef<MachineOperand> Cond,
418 Register TrueReg, Register FalseReg) const override;
419
420 void insertVectorSelect(MachineBasicBlock &MBB,
421 MachineBasicBlock::iterator I, const DebugLoc &DL,
422 Register DstReg, ArrayRef<MachineOperand> Cond,
423 Register TrueReg, Register FalseReg) const;
424
425 bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
426 Register &SrcReg2, int64_t &CmpMask,
427 int64_t &CmpValue) const override;
428
429 bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
430 Register SrcReg2, int64_t CmpMask, int64_t CmpValue,
431 const MachineRegisterInfo *MRI) const override;
432
433 bool
434 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
435 const MachineInstr &MIb) const override;
436
437 static bool isFoldableCopy(const MachineInstr &MI);
438 static unsigned getFoldableCopySrcIdx(const MachineInstr &MI);
439
440 void removeModOperands(MachineInstr &MI) const;
441
442 void mutateAndCleanupImplicit(MachineInstr &MI,
443 const MCInstrDesc &NewDesc) const;
444
445 /// Return the extracted immediate value in a subregister use from a constant
446 /// materialized in a super register.
447 ///
448 /// e.g. %imm = S_MOV_B64 K[0:63]
449 /// USE %imm.sub1
450 /// This will return K[32:63]
451 static std::optional<int64_t> extractSubregFromImm(int64_t ImmVal,
452 unsigned SubRegIndex);
453
454 bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
455 MachineRegisterInfo *MRI) const final;
456
457 unsigned getMachineCSELookAheadLimit() const override { return 500; }
458
459 MachineInstr *convertToThreeAddress(MachineInstr &MI, LiveVariables *LV,
460 LiveIntervals *LIS) const override;
461
462 bool isSchedulingBoundary(const MachineInstr &MI,
463 const MachineBasicBlock *MBB,
464 const MachineFunction &MF) const override;
465
466 static bool isSALU(const MachineInstr &MI) {
467 return MI.getDesc().TSFlags & SIInstrFlags::SALU;
468 }
469
470 bool isSALU(uint32_t Opcode) const {
471 return get(Opcode).TSFlags & SIInstrFlags::SALU;
472 }
473
474 static bool isVALU(const MachineInstr &MI) {
475 return MI.getDesc().TSFlags & SIInstrFlags::VALU;
476 }
477
478 bool isVALU(uint32_t Opcode) const {
479 return get(Opcode).TSFlags & SIInstrFlags::VALU;
480 }
481
482 static bool isImage(const MachineInstr &MI) {
483 return isMIMG(MI) || isVSAMPLE(MI) || isVIMAGE(MI);
484 }
485
486 bool isImage(uint32_t Opcode) const {
487 return isMIMG(Opcode) || isVSAMPLE(Opcode) || isVIMAGE(Opcode);
488 }
489
490 static bool isVMEM(const MachineInstr &MI) {
491 return isMUBUF(MI) || isMTBUF(MI) || isImage(MI) || isFLAT(MI);
492 }
493
494 bool isVMEM(uint32_t Opcode) const {
495 return isMUBUF(Opcode) || isMTBUF(Opcode) || isImage(Opcode) ||
496 isFLAT(Opcode);
497 }
498
499 static bool isSOP1(const MachineInstr &MI) {
500 return MI.getDesc().TSFlags & SIInstrFlags::SOP1;
501 }
502
503 bool isSOP1(uint32_t Opcode) const {
504 return get(Opcode).TSFlags & SIInstrFlags::SOP1;
505 }
506
507 static bool isSOP2(const MachineInstr &MI) {
508 return MI.getDesc().TSFlags & SIInstrFlags::SOP2;
509 }
510
511 bool isSOP2(uint32_t Opcode) const {
512 return get(Opcode).TSFlags & SIInstrFlags::SOP2;
513 }
514
515 static bool isSOPC(const MachineInstr &MI) {
516 return MI.getDesc().TSFlags & SIInstrFlags::SOPC;
517 }
518
519 bool isSOPC(uint32_t Opcode) const {
520 return get(Opcode).TSFlags & SIInstrFlags::SOPC;
521 }
522
523 static bool isSOPK(const MachineInstr &MI) {
524 return MI.getDesc().TSFlags & SIInstrFlags::SOPK;
525 }
526
527 bool isSOPK(uint32_t Opcode) const {
528 return get(Opcode).TSFlags & SIInstrFlags::SOPK;
529 }
530
531 static bool isSOPP(const MachineInstr &MI) {
532 return MI.getDesc().TSFlags & SIInstrFlags::SOPP;
533 }
534
535 bool isSOPP(uint32_t Opcode) const {
536 return get(Opcode).TSFlags & SIInstrFlags::SOPP;
537 }
538
539 static bool isPacked(const MachineInstr &MI) {
540 return MI.getDesc().TSFlags & SIInstrFlags::IsPacked;
541 }
542
543 bool isPacked(uint32_t Opcode) const {
544 return get(Opcode).TSFlags & SIInstrFlags::IsPacked;
545 }
546
547 static bool isVOP1(const MachineInstr &MI) {
548 return MI.getDesc().TSFlags & SIInstrFlags::VOP1;
549 }
550
551 bool isVOP1(uint32_t Opcode) const {
552 return get(Opcode).TSFlags & SIInstrFlags::VOP1;
553 }
554
555 static bool isVOP2(const MachineInstr &MI) {
556 return MI.getDesc().TSFlags & SIInstrFlags::VOP2;
557 }
558
559 bool isVOP2(uint32_t Opcode) const {
560 return get(Opcode).TSFlags & SIInstrFlags::VOP2;
561 }
562
563 static bool isVOP3(const MCInstrDesc &Desc) {
564 return Desc.TSFlags & SIInstrFlags::VOP3;
565 }
566
567 static bool isVOP3(const MachineInstr &MI) { return isVOP3(Desc: MI.getDesc()); }
568
569 bool isVOP3(uint32_t Opcode) const { return isVOP3(Desc: get(Opcode)); }
570
571 static bool isSDWA(const MachineInstr &MI) {
572 return MI.getDesc().TSFlags & SIInstrFlags::SDWA;
573 }
574
575 bool isSDWA(uint32_t Opcode) const {
576 return get(Opcode).TSFlags & SIInstrFlags::SDWA;
577 }
578
579 static bool isVOPC(const MachineInstr &MI) {
580 return MI.getDesc().TSFlags & SIInstrFlags::VOPC;
581 }
582
583 bool isVOPC(uint32_t Opcode) const {
584 return get(Opcode).TSFlags & SIInstrFlags::VOPC;
585 }
586
587 static bool isMUBUF(const MachineInstr &MI) {
588 return MI.getDesc().TSFlags & SIInstrFlags::MUBUF;
589 }
590
591 bool isMUBUF(uint32_t Opcode) const {
592 return get(Opcode).TSFlags & SIInstrFlags::MUBUF;
593 }
594
595 static bool isMTBUF(const MachineInstr &MI) {
596 return MI.getDesc().TSFlags & SIInstrFlags::MTBUF;
597 }
598
599 bool isMTBUF(uint32_t Opcode) const {
600 return get(Opcode).TSFlags & SIInstrFlags::MTBUF;
601 }
602
603 static bool isBUF(const MachineInstr &MI) {
604 return isMUBUF(MI) || isMTBUF(MI);
605 }
606
607 static bool isSMRD(const MachineInstr &MI) {
608 return MI.getDesc().TSFlags & SIInstrFlags::SMRD;
609 }
610
611 bool isSMRD(uint32_t Opcode) const {
612 return get(Opcode).TSFlags & SIInstrFlags::SMRD;
613 }
614
615 bool isBufferSMRD(const MachineInstr &MI) const;
616
617 static bool isDS(const MachineInstr &MI) {
618 return MI.getDesc().TSFlags & SIInstrFlags::DS;
619 }
620
621 bool isDS(uint32_t Opcode) const {
622 return get(Opcode).TSFlags & SIInstrFlags::DS;
623 }
624
625 static bool isLDSDMA(const MachineInstr &MI) {
626 return (isVALU(MI) && (isMUBUF(MI) || isFLAT(MI))) ||
627 (MI.getDesc().TSFlags & SIInstrFlags::TENSOR_CNT);
628 }
629
630 bool isLDSDMA(uint32_t Opcode) {
631 return (isVALU(Opcode) && (isMUBUF(Opcode) || isFLAT(Opcode))) ||
632 (get(Opcode).TSFlags & SIInstrFlags::TENSOR_CNT);
633 }
634
635 static bool isGWS(const MachineInstr &MI) {
636 return MI.getDesc().TSFlags & SIInstrFlags::GWS;
637 }
638
639 bool isGWS(uint32_t Opcode) const {
640 return get(Opcode).TSFlags & SIInstrFlags::GWS;
641 }
642
643 bool isAlwaysGDS(uint32_t Opcode) const;
644
645 static bool isMIMG(const MachineInstr &MI) {
646 return MI.getDesc().TSFlags & SIInstrFlags::MIMG;
647 }
648
649 bool isMIMG(uint32_t Opcode) const {
650 return get(Opcode).TSFlags & SIInstrFlags::MIMG;
651 }
652
653 static bool isVIMAGE(const MachineInstr &MI) {
654 return MI.getDesc().TSFlags & SIInstrFlags::VIMAGE;
655 }
656
657 bool isVIMAGE(uint32_t Opcode) const {
658 return get(Opcode).TSFlags & SIInstrFlags::VIMAGE;
659 }
660
661 static bool isVSAMPLE(const MachineInstr &MI) {
662 return MI.getDesc().TSFlags & SIInstrFlags::VSAMPLE;
663 }
664
665 bool isVSAMPLE(uint32_t Opcode) const {
666 return get(Opcode).TSFlags & SIInstrFlags::VSAMPLE;
667 }
668
669 static bool isGather4(const MachineInstr &MI) {
670 return MI.getDesc().TSFlags & SIInstrFlags::Gather4;
671 }
672
673 bool isGather4(uint32_t Opcode) const {
674 return get(Opcode).TSFlags & SIInstrFlags::Gather4;
675 }
676
677 static bool isFLAT(const MachineInstr &MI) {
678 return MI.getDesc().TSFlags & SIInstrFlags::FLAT;
679 }
680
681 // Is a FLAT encoded instruction which accesses a specific segment,
682 // i.e. global_* or scratch_*.
683 static bool isSegmentSpecificFLAT(const MachineInstr &MI) {
684 auto Flags = MI.getDesc().TSFlags;
685 return Flags & (SIInstrFlags::FlatGlobal | SIInstrFlags::FlatScratch);
686 }
687
688 bool isSegmentSpecificFLAT(uint32_t Opcode) const {
689 auto Flags = get(Opcode).TSFlags;
690 return Flags & (SIInstrFlags::FlatGlobal | SIInstrFlags::FlatScratch);
691 }
692
693 static bool isFLATGlobal(const MachineInstr &MI) {
694 return MI.getDesc().TSFlags & SIInstrFlags::FlatGlobal;
695 }
696
697 bool isFLATGlobal(uint32_t Opcode) const {
698 return get(Opcode).TSFlags & SIInstrFlags::FlatGlobal;
699 }
700
701 static bool isFLATScratch(const MachineInstr &MI) {
702 return MI.getDesc().TSFlags & SIInstrFlags::FlatScratch;
703 }
704
705 bool isFLATScratch(uint32_t Opcode) const {
706 return get(Opcode).TSFlags & SIInstrFlags::FlatScratch;
707 }
708
709 // Any FLAT encoded instruction, including global_* and scratch_*.
710 bool isFLAT(uint32_t Opcode) const {
711 return get(Opcode).TSFlags & SIInstrFlags::FLAT;
712 }
713
714 /// \returns true for SCRATCH_ instructions, or FLAT/BUF instructions unless
715 /// the MMOs do not include scratch.
716 /// Conservatively correct; will return true if \p MI cannot be proven
717 /// to not hit scratch.
718 bool mayAccessScratch(const MachineInstr &MI) const;
719
720 /// \returns true for FLAT instructions that can access VMEM.
721 bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const;
722
723 /// \returns true for FLAT instructions that can access LDS.
724 bool mayAccessLDSThroughFlat(const MachineInstr &MI) const;
725
726 static bool isBlockLoadStore(uint32_t Opcode) {
727 switch (Opcode) {
728 case AMDGPU::SI_BLOCK_SPILL_V1024_SAVE:
729 case AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE:
730 case AMDGPU::SCRATCH_STORE_BLOCK_SADDR:
731 case AMDGPU::SCRATCH_LOAD_BLOCK_SADDR:
732 case AMDGPU::SCRATCH_STORE_BLOCK_SVS:
733 case AMDGPU::SCRATCH_LOAD_BLOCK_SVS:
734 return true;
735 default:
736 return false;
737 }
738 }
739
740 static bool setsSCCIfResultIsNonZero(const MachineInstr &MI) {
741 switch (MI.getOpcode()) {
742 case AMDGPU::S_ABSDIFF_I32:
743 case AMDGPU::S_ABS_I32:
744 case AMDGPU::S_AND_B32:
745 case AMDGPU::S_AND_B64:
746 case AMDGPU::S_ANDN2_B32:
747 case AMDGPU::S_ANDN2_B64:
748 case AMDGPU::S_ASHR_I32:
749 case AMDGPU::S_ASHR_I64:
750 case AMDGPU::S_BCNT0_I32_B32:
751 case AMDGPU::S_BCNT0_I32_B64:
752 case AMDGPU::S_BCNT1_I32_B32:
753 case AMDGPU::S_BCNT1_I32_B64:
754 case AMDGPU::S_BFE_I32:
755 case AMDGPU::S_BFE_I64:
756 case AMDGPU::S_BFE_U32:
757 case AMDGPU::S_BFE_U64:
758 case AMDGPU::S_LSHL_B32:
759 case AMDGPU::S_LSHL_B64:
760 case AMDGPU::S_LSHR_B32:
761 case AMDGPU::S_LSHR_B64:
762 case AMDGPU::S_NAND_B32:
763 case AMDGPU::S_NAND_B64:
764 case AMDGPU::S_NOR_B32:
765 case AMDGPU::S_NOR_B64:
766 case AMDGPU::S_NOT_B32:
767 case AMDGPU::S_NOT_B64:
768 case AMDGPU::S_OR_B32:
769 case AMDGPU::S_OR_B64:
770 case AMDGPU::S_ORN2_B32:
771 case AMDGPU::S_ORN2_B64:
772 case AMDGPU::S_QUADMASK_B32:
773 case AMDGPU::S_QUADMASK_B64:
774 case AMDGPU::S_WQM_B32:
775 case AMDGPU::S_WQM_B64:
776 case AMDGPU::S_XNOR_B32:
777 case AMDGPU::S_XNOR_B64:
778 case AMDGPU::S_XOR_B32:
779 case AMDGPU::S_XOR_B64:
780 return true;
781 default:
782 return false;
783 }
784 }
785
786 static bool isEXP(const MachineInstr &MI) {
787 return MI.getDesc().TSFlags & SIInstrFlags::EXP;
788 }
789
790 static bool isDualSourceBlendEXP(const MachineInstr &MI) {
791 if (!isEXP(MI))
792 return false;
793 unsigned Target = MI.getOperand(i: 0).getImm();
794 return Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND0 ||
795 Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND1;
796 }
797
798 bool isEXP(uint32_t Opcode) const {
799 return get(Opcode).TSFlags & SIInstrFlags::EXP;
800 }
801
802 static bool isAtomicNoRet(const MachineInstr &MI) {
803 return MI.getDesc().TSFlags & SIInstrFlags::IsAtomicNoRet;
804 }
805
806 bool isAtomicNoRet(uint32_t Opcode) const {
807 return get(Opcode).TSFlags & SIInstrFlags::IsAtomicNoRet;
808 }
809
810 static bool isAtomicRet(const MachineInstr &MI) {
811 return MI.getDesc().TSFlags & SIInstrFlags::IsAtomicRet;
812 }
813
814 bool isAtomicRet(uint32_t Opcode) const {
815 return get(Opcode).TSFlags & SIInstrFlags::IsAtomicRet;
816 }
817
818 static bool isAtomic(const MachineInstr &MI) {
819 return MI.getDesc().TSFlags & (SIInstrFlags::IsAtomicRet |
820 SIInstrFlags::IsAtomicNoRet);
821 }
822
823 bool isAtomic(uint32_t Opcode) const {
824 return get(Opcode).TSFlags & (SIInstrFlags::IsAtomicRet |
825 SIInstrFlags::IsAtomicNoRet);
826 }
827
828 static bool mayWriteLDSThroughDMA(const MachineInstr &MI) {
829 unsigned Opc = MI.getOpcode();
830 // Exclude instructions that read FROM LDS (not write to it)
831 return isLDSDMA(MI) && Opc != AMDGPU::BUFFER_STORE_LDS_DWORD &&
832 Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d2 &&
833 Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d4;
834 }
835
836 static bool isSBarrierSCCWrite(unsigned Opcode) {
837 return Opcode == AMDGPU::S_BARRIER_LEAVE ||
838 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM ||
839 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0;
840 }
841
842 static bool isCBranchVCCZRead(const MachineInstr &MI) {
843 unsigned Opc = MI.getOpcode();
844 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
845 !MI.getOperand(i: 1).isUndef();
846 }
847
848 static bool isWQM(const MachineInstr &MI) {
849 return MI.getDesc().TSFlags & SIInstrFlags::WQM;
850 }
851
852 bool isWQM(uint32_t Opcode) const {
853 return get(Opcode).TSFlags & SIInstrFlags::WQM;
854 }
855
856 static bool isDisableWQM(const MachineInstr &MI) {
857 return MI.getDesc().TSFlags & SIInstrFlags::DisableWQM;
858 }
859
860 bool isDisableWQM(uint32_t Opcode) const {
861 return get(Opcode).TSFlags & SIInstrFlags::DisableWQM;
862 }
863
864 // SI_SPILL_S32_TO_VGPR and SI_RESTORE_S32_FROM_VGPR form a special case of
865 // SGPRs spilling to VGPRs which are SGPR spills but from VALU instructions
866 // therefore we need an explicit check for them since just checking if the
867 // Spill bit is set and what instruction type it came from misclassifies
868 // them.
869 static bool isVGPRSpill(const MachineInstr &MI) {
870 return MI.getOpcode() != AMDGPU::SI_SPILL_S32_TO_VGPR &&
871 MI.getOpcode() != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
872 (isSpill(MI) && isVALU(MI));
873 }
874
875 bool isVGPRSpill(uint32_t Opcode) const {
876 return Opcode != AMDGPU::SI_SPILL_S32_TO_VGPR &&
877 Opcode != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
878 (isSpill(Opcode) && isVALU(Opcode));
879 }
880
881 static bool isSGPRSpill(const MachineInstr &MI) {
882 return MI.getOpcode() == AMDGPU::SI_SPILL_S32_TO_VGPR ||
883 MI.getOpcode() == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
884 (isSpill(MI) && isSALU(MI));
885 }
886
887 bool isSGPRSpill(uint32_t Opcode) const {
888 return Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR ||
889 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
890 (isSpill(Opcode) && isSALU(Opcode));
891 }
892
893 bool isSpill(uint32_t Opcode) const {
894 return get(Opcode).TSFlags & SIInstrFlags::Spill;
895 }
896
897 static bool isSpill(const MCInstrDesc &Desc) {
898 return Desc.TSFlags & SIInstrFlags::Spill;
899 }
900
901 static bool isSpill(const MachineInstr &MI) { return isSpill(Desc: MI.getDesc()); }
902
903 static bool isWWMRegSpillOpcode(uint32_t Opcode) {
904 return Opcode == AMDGPU::SI_SPILL_WWM_V32_SAVE ||
905 Opcode == AMDGPU::SI_SPILL_WWM_AV32_SAVE ||
906 Opcode == AMDGPU::SI_SPILL_WWM_V32_RESTORE ||
907 Opcode == AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
908 }
909
910 static bool isChainCallOpcode(uint64_t Opcode) {
911 return Opcode == AMDGPU::SI_CS_CHAIN_TC_W32 ||
912 Opcode == AMDGPU::SI_CS_CHAIN_TC_W64;
913 }
914
915 static bool isDPP(const MachineInstr &MI) {
916 return MI.getDesc().TSFlags & SIInstrFlags::DPP;
917 }
918
919 bool isDPP(uint32_t Opcode) const {
920 return get(Opcode).TSFlags & SIInstrFlags::DPP;
921 }
922
923 static bool isTRANS(const MachineInstr &MI) {
924 return MI.getDesc().TSFlags & SIInstrFlags::TRANS;
925 }
926
927 bool isTRANS(uint32_t Opcode) const {
928 return get(Opcode).TSFlags & SIInstrFlags::TRANS;
929 }
930
931 static bool isVOP3P(const MachineInstr &MI) {
932 return MI.getDesc().TSFlags & SIInstrFlags::VOP3P;
933 }
934
935 bool isVOP3P(uint32_t Opcode) const {
936 return get(Opcode).TSFlags & SIInstrFlags::VOP3P;
937 }
938
939 static bool isVINTRP(const MachineInstr &MI) {
940 return MI.getDesc().TSFlags & SIInstrFlags::VINTRP;
941 }
942
943 bool isVINTRP(uint32_t Opcode) const {
944 return get(Opcode).TSFlags & SIInstrFlags::VINTRP;
945 }
946
947 static bool isMAI(const MCInstrDesc &Desc) {
948 return Desc.TSFlags & SIInstrFlags::IsMAI;
949 }
950
951 static bool isMAI(const MachineInstr &MI) { return isMAI(Desc: MI.getDesc()); }
952
953 bool isMAI(uint32_t Opcode) const { return isMAI(Desc: get(Opcode)); }
954
955 static bool isMFMA(const MachineInstr &MI) {
956 return isMAI(MI) && MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
957 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
958 }
959
960 bool isMFMA(uint32_t Opcode) const {
961 return isMAI(Opcode) && Opcode != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
962 Opcode != AMDGPU::V_ACCVGPR_READ_B32_e64;
963 }
964
965 static bool isDOT(const MachineInstr &MI) {
966 return MI.getDesc().TSFlags & SIInstrFlags::IsDOT;
967 }
968
969 static bool isWMMA(const MachineInstr &MI) {
970 return MI.getDesc().TSFlags & SIInstrFlags::IsWMMA;
971 }
972
973 bool isWMMA(uint32_t Opcode) const {
974 return get(Opcode).TSFlags & SIInstrFlags::IsWMMA;
975 }
976
977 static bool isMFMAorWMMA(const MachineInstr &MI) {
978 return isMFMA(MI) || isWMMA(MI) || isSWMMAC(MI);
979 }
980
981 bool isMFMAorWMMA(uint32_t Opcode) const {
982 return isMFMA(Opcode) || isWMMA(Opcode) || isSWMMAC(Opcode);
983 }
984
985 static bool isSWMMAC(const MachineInstr &MI) {
986 return MI.getDesc().TSFlags & SIInstrFlags::IsSWMMAC;
987 }
988
989 bool isSWMMAC(uint32_t Opcode) const {
990 return get(Opcode).TSFlags & SIInstrFlags::IsSWMMAC;
991 }
992
993 bool isDOT(uint32_t Opcode) const {
994 return get(Opcode).TSFlags & SIInstrFlags::IsDOT;
995 }
996
997 bool isXDLWMMA(const MachineInstr &MI) const;
998
999 bool isXDL(const MachineInstr &MI) const;
1000
1001 static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opc: Opcode); }
1002
1003 static bool isLDSDIR(const MachineInstr &MI) {
1004 return MI.getDesc().TSFlags & SIInstrFlags::LDSDIR;
1005 }
1006
1007 bool isLDSDIR(uint32_t Opcode) const {
1008 return get(Opcode).TSFlags & SIInstrFlags::LDSDIR;
1009 }
1010
1011 static bool isVINTERP(const MachineInstr &MI) {
1012 return MI.getDesc().TSFlags & SIInstrFlags::VINTERP;
1013 }
1014
1015 bool isVINTERP(uint32_t Opcode) const {
1016 return get(Opcode).TSFlags & SIInstrFlags::VINTERP;
1017 }
1018
1019 static bool isScalarUnit(const MachineInstr &MI) {
1020 return MI.getDesc().TSFlags & (SIInstrFlags::SALU | SIInstrFlags::SMRD);
1021 }
1022
1023 static bool usesVM_CNT(const MachineInstr &MI) {
1024 return MI.getDesc().TSFlags & SIInstrFlags::VM_CNT;
1025 }
1026
1027 static bool usesLGKM_CNT(const MachineInstr &MI) {
1028 return MI.getDesc().TSFlags & SIInstrFlags::LGKM_CNT;
1029 }
1030
1031 static bool usesASYNC_CNT(const MachineInstr &MI) {
1032 return MI.getDesc().TSFlags & SIInstrFlags::ASYNC_CNT;
1033 }
1034
1035 bool usesASYNC_CNT(uint32_t Opcode) const {
1036 return get(Opcode).TSFlags & SIInstrFlags::ASYNC_CNT;
1037 }
1038
1039 // Most sopk treat the immediate as a signed 16-bit, however some
1040 // use it as unsigned.
1041 static bool sopkIsZext(unsigned Opcode) {
1042 return Opcode == AMDGPU::S_CMPK_EQ_U32 || Opcode == AMDGPU::S_CMPK_LG_U32 ||
1043 Opcode == AMDGPU::S_CMPK_GT_U32 || Opcode == AMDGPU::S_CMPK_GE_U32 ||
1044 Opcode == AMDGPU::S_CMPK_LT_U32 || Opcode == AMDGPU::S_CMPK_LE_U32 ||
1045 Opcode == AMDGPU::S_GETREG_B32 ||
1046 Opcode == AMDGPU::S_GETREG_B32_const;
1047 }
1048
1049 /// \returns true if this is an s_store_dword* instruction. This is more
1050 /// specific than isSMEM && mayStore.
1051 static bool isScalarStore(const MachineInstr &MI) {
1052 return MI.getDesc().TSFlags & SIInstrFlags::SCALAR_STORE;
1053 }
1054
1055 bool isScalarStore(uint32_t Opcode) const {
1056 return get(Opcode).TSFlags & SIInstrFlags::SCALAR_STORE;
1057 }
1058
1059 static bool isFixedSize(const MachineInstr &MI) {
1060 return MI.getDesc().TSFlags & SIInstrFlags::FIXED_SIZE;
1061 }
1062
1063 bool isFixedSize(uint32_t Opcode) const {
1064 return get(Opcode).TSFlags & SIInstrFlags::FIXED_SIZE;
1065 }
1066
1067 static bool hasFPClamp(const MachineInstr &MI) {
1068 return MI.getDesc().TSFlags & SIInstrFlags::FPClamp;
1069 }
1070
1071 bool hasFPClamp(uint32_t Opcode) const {
1072 return get(Opcode).TSFlags & SIInstrFlags::FPClamp;
1073 }
1074
1075 static bool hasIntClamp(const MachineInstr &MI) {
1076 return MI.getDesc().TSFlags & SIInstrFlags::IntClamp;
1077 }
1078
1079 uint64_t getClampMask(const MachineInstr &MI) const {
1080 const uint64_t ClampFlags = SIInstrFlags::FPClamp |
1081 SIInstrFlags::IntClamp |
1082 SIInstrFlags::ClampLo |
1083 SIInstrFlags::ClampHi;
1084 return MI.getDesc().TSFlags & ClampFlags;
1085 }
1086
1087 static bool usesFPDPRounding(const MachineInstr &MI) {
1088 return MI.getDesc().TSFlags & SIInstrFlags::FPDPRounding;
1089 }
1090
1091 bool usesFPDPRounding(uint32_t Opcode) const {
1092 return get(Opcode).TSFlags & SIInstrFlags::FPDPRounding;
1093 }
1094
1095 static bool isFPAtomic(const MachineInstr &MI) {
1096 return MI.getDesc().TSFlags & SIInstrFlags::FPAtomic;
1097 }
1098
1099 bool isFPAtomic(uint32_t Opcode) const {
1100 return get(Opcode).TSFlags & SIInstrFlags::FPAtomic;
1101 }
1102
1103 static bool isNeverUniform(const MachineInstr &MI) {
1104 return MI.getDesc().TSFlags & SIInstrFlags::IsNeverUniform;
1105 }
1106
1107 // Check to see if opcode is for a barrier start. Pre gfx12 this is just the
1108 // S_BARRIER, but after support for S_BARRIER_SIGNAL* / S_BARRIER_WAIT we want
1109 // to check for the barrier start (S_BARRIER_SIGNAL*)
1110 bool isBarrierStart(unsigned Opcode) const {
1111 return Opcode == AMDGPU::S_BARRIER ||
1112 Opcode == AMDGPU::S_BARRIER_SIGNAL_M0 ||
1113 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0 ||
1114 Opcode == AMDGPU::S_BARRIER_SIGNAL_IMM ||
1115 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM;
1116 }
1117
1118 bool isBarrier(unsigned Opcode) const {
1119 return isBarrierStart(Opcode) || Opcode == AMDGPU::S_BARRIER_WAIT ||
1120 Opcode == AMDGPU::S_BARRIER_INIT_M0 ||
1121 Opcode == AMDGPU::S_BARRIER_INIT_IMM ||
1122 Opcode == AMDGPU::S_BARRIER_JOIN_IMM ||
1123 Opcode == AMDGPU::S_BARRIER_LEAVE || Opcode == AMDGPU::DS_GWS_INIT ||
1124 Opcode == AMDGPU::DS_GWS_BARRIER;
1125 }
1126
1127 static bool isGFX12CacheInvOrWBInst(unsigned Opc) {
1128 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||
1129 Opc == AMDGPU::GLOBAL_WBINV;
1130 }
1131
1132 static bool isF16PseudoScalarTrans(unsigned Opcode) {
1133 return Opcode == AMDGPU::V_S_EXP_F16_e64 ||
1134 Opcode == AMDGPU::V_S_LOG_F16_e64 ||
1135 Opcode == AMDGPU::V_S_RCP_F16_e64 ||
1136 Opcode == AMDGPU::V_S_RSQ_F16_e64 ||
1137 Opcode == AMDGPU::V_S_SQRT_F16_e64;
1138 }
1139
1140 static bool doesNotReadTiedSource(const MachineInstr &MI) {
1141 return MI.getDesc().TSFlags & SIInstrFlags::TiedSourceNotRead;
1142 }
1143
1144 bool doesNotReadTiedSource(uint32_t Opcode) const {
1145 return get(Opcode).TSFlags & SIInstrFlags::TiedSourceNotRead;
1146 }
1147
1148 bool isIGLP(unsigned Opcode) const {
1149 return Opcode == AMDGPU::SCHED_BARRIER ||
1150 Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
1151 }
1152
1153 bool isIGLP(const MachineInstr &MI) const { return isIGLP(Opcode: MI.getOpcode()); }
1154
1155 // Return true if the instruction is mutually exclusive with all non-IGLP DAG
1156 // mutations, requiring all other mutations to be disabled.
1157 bool isIGLPMutationOnly(unsigned Opcode) const {
1158 return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
1159 }
1160
1161 static unsigned getNonSoftWaitcntOpcode(unsigned Opcode) {
1162 switch (Opcode) {
1163 case AMDGPU::S_WAITCNT_soft:
1164 return AMDGPU::S_WAITCNT;
1165 case AMDGPU::S_WAITCNT_VSCNT_soft:
1166 return AMDGPU::S_WAITCNT_VSCNT;
1167 case AMDGPU::S_WAIT_LOADCNT_soft:
1168 return AMDGPU::S_WAIT_LOADCNT;
1169 case AMDGPU::S_WAIT_STORECNT_soft:
1170 return AMDGPU::S_WAIT_STORECNT;
1171 case AMDGPU::S_WAIT_SAMPLECNT_soft:
1172 return AMDGPU::S_WAIT_SAMPLECNT;
1173 case AMDGPU::S_WAIT_BVHCNT_soft:
1174 return AMDGPU::S_WAIT_BVHCNT;
1175 case AMDGPU::S_WAIT_DSCNT_soft:
1176 return AMDGPU::S_WAIT_DSCNT;
1177 case AMDGPU::S_WAIT_KMCNT_soft:
1178 return AMDGPU::S_WAIT_KMCNT;
1179 case AMDGPU::S_WAIT_XCNT_soft:
1180 return AMDGPU::S_WAIT_XCNT;
1181 default:
1182 return Opcode;
1183 }
1184 }
1185
1186 static bool isWaitcnt(unsigned Opcode) {
1187 switch (getNonSoftWaitcntOpcode(Opcode)) {
1188 case AMDGPU::S_WAITCNT:
1189 case AMDGPU::S_WAITCNT_VSCNT:
1190 case AMDGPU::S_WAITCNT_VMCNT:
1191 case AMDGPU::S_WAITCNT_EXPCNT:
1192 case AMDGPU::S_WAITCNT_LGKMCNT:
1193 case AMDGPU::S_WAIT_LOADCNT:
1194 case AMDGPU::S_WAIT_LOADCNT_DSCNT:
1195 case AMDGPU::S_WAIT_STORECNT:
1196 case AMDGPU::S_WAIT_STORECNT_DSCNT:
1197 case AMDGPU::S_WAIT_SAMPLECNT:
1198 case AMDGPU::S_WAIT_BVHCNT:
1199 case AMDGPU::S_WAIT_EXPCNT:
1200 case AMDGPU::S_WAIT_DSCNT:
1201 case AMDGPU::S_WAIT_KMCNT:
1202 case AMDGPU::S_WAIT_XCNT:
1203 case AMDGPU::S_WAIT_IDLE:
1204 return true;
1205 default:
1206 return false;
1207 }
1208 }
1209
1210 bool isVGPRCopy(const MachineInstr &MI) const {
1211 assert(isCopyInstr(MI));
1212 Register Dest = MI.getOperand(i: 0).getReg();
1213 const MachineFunction &MF = *MI.getMF();
1214 const MachineRegisterInfo &MRI = MF.getRegInfo();
1215 return !RI.isSGPRReg(MRI, Reg: Dest);
1216 }
1217
1218 bool hasVGPRUses(const MachineInstr &MI) const {
1219 const MachineFunction &MF = *MI.getMF();
1220 const MachineRegisterInfo &MRI = MF.getRegInfo();
1221 return llvm::any_of(Range: MI.explicit_uses(),
1222 P: [&MRI, this](const MachineOperand &MO) {
1223 return MO.isReg() && RI.isVGPR(MRI, Reg: MO.getReg());});
1224 }
1225
1226 /// Return true if the instruction modifies the mode register.q
1227 static bool modifiesModeRegister(const MachineInstr &MI);
1228
1229 /// This function is used to determine if an instruction can be safely
1230 /// executed under EXEC = 0 without hardware error, indeterminate results,
1231 /// and/or visible effects on future vector execution or outside the shader.
1232 /// Note: as of 2024 the only use of this is SIPreEmitPeephole where it is
1233 /// used in removing branches over short EXEC = 0 sequences.
1234 /// As such it embeds certain assumptions which may not apply to every case
1235 /// of EXEC = 0 execution.
1236 bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const;
1237
1238 /// Returns true if the instruction could potentially depend on the value of
1239 /// exec. If false, exec dependencies may safely be ignored.
1240 bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const;
1241
1242 bool isInlineConstant(const APInt &Imm) const;
1243
1244 bool isInlineConstant(const APFloat &Imm) const;
1245
1246 // Returns true if this non-register operand definitely does not need to be
1247 // encoded as a 32-bit literal. Note that this function handles all kinds of
1248 // operands, not just immediates.
1249 //
1250 // Some operands like FrameIndexes could resolve to an inline immediate value
1251 // that will not require an additional 4-bytes; this function assumes that it
1252 // will.
1253 bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const {
1254 if (!MO.isImm())
1255 return false;
1256 return isInlineConstant(ImmVal: MO.getImm(), OperandType);
1257 }
1258 bool isInlineConstant(int64_t ImmVal, uint8_t OperandType) const;
1259
1260 bool isInlineConstant(const MachineOperand &MO,
1261 const MCOperandInfo &OpInfo) const {
1262 return isInlineConstant(MO, OperandType: OpInfo.OperandType);
1263 }
1264
1265 /// \p returns true if \p UseMO is substituted with \p DefMO in \p MI it would
1266 /// be an inline immediate.
1267 bool isInlineConstant(const MachineInstr &MI,
1268 const MachineOperand &UseMO,
1269 const MachineOperand &DefMO) const {
1270 assert(UseMO.getParent() == &MI);
1271 int OpIdx = UseMO.getOperandNo();
1272 if (OpIdx >= MI.getDesc().NumOperands)
1273 return false;
1274
1275 return isInlineConstant(MO: DefMO, OpInfo: MI.getDesc().operands()[OpIdx]);
1276 }
1277
1278 /// \p returns true if the operand \p OpIdx in \p MI is a valid inline
1279 /// immediate.
1280 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx) const {
1281 const MachineOperand &MO = MI.getOperand(i: OpIdx);
1282 return isInlineConstant(MO, OperandType: MI.getDesc().operands()[OpIdx].OperandType);
1283 }
1284
1285 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
1286 int64_t ImmVal) const {
1287 if (OpIdx >= MI.getDesc().NumOperands)
1288 return false;
1289
1290 if (isCopyInstr(MI)) {
1291 unsigned Size = getOpSize(MI, OpNo: OpIdx);
1292 assert(Size == 8 || Size == 4);
1293
1294 uint8_t OpType = (Size == 8) ?
1295 AMDGPU::OPERAND_REG_IMM_INT64 : AMDGPU::OPERAND_REG_IMM_INT32;
1296 return isInlineConstant(ImmVal, OperandType: OpType);
1297 }
1298
1299 return isInlineConstant(ImmVal, OperandType: MI.getDesc().operands()[OpIdx].OperandType);
1300 }
1301
1302 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
1303 const MachineOperand &MO) const {
1304 return isInlineConstant(MI, OpIdx, ImmVal: MO.getImm());
1305 }
1306
1307 bool isInlineConstant(const MachineOperand &MO) const {
1308 return isInlineConstant(MI: *MO.getParent(), OpIdx: MO.getOperandNo());
1309 }
1310
1311 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
1312 const MachineOperand &MO) const;
1313
1314 bool isLiteralOperandLegal(const MCInstrDesc &InstDesc,
1315 const MCOperandInfo &OpInfo) const;
1316
1317 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
1318 int64_t ImmVal) const;
1319
1320 bool isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
1321 const MachineOperand &MO) const {
1322 return isImmOperandLegal(InstDesc: MI.getDesc(), OpNo, MO);
1323 }
1324
1325 bool isNeverCoissue(MachineInstr &MI) const;
1326
1327 /// Check if this immediate value can be used for AV_MOV_B64_IMM_PSEUDO.
1328 bool isLegalAV64PseudoImm(uint64_t Imm) const;
1329
1330 /// Return true if this 64-bit VALU instruction has a 32-bit encoding.
1331 /// This function will return false if you pass it a 32-bit instruction.
1332 bool hasVALU32BitEncoding(unsigned Opcode) const;
1333
1334 bool physRegUsesConstantBus(const MachineOperand &Reg) const;
1335 bool regUsesConstantBus(const MachineOperand &Reg,
1336 const MachineRegisterInfo &MRI) const;
1337
1338 /// Returns true if this operand uses the constant bus.
1339 bool usesConstantBus(const MachineRegisterInfo &MRI,
1340 const MachineOperand &MO,
1341 const MCOperandInfo &OpInfo) const;
1342
1343 bool usesConstantBus(const MachineRegisterInfo &MRI, const MachineInstr &MI,
1344 int OpIdx) const {
1345 return usesConstantBus(MRI, MO: MI.getOperand(i: OpIdx),
1346 OpInfo: MI.getDesc().operands()[OpIdx]);
1347 }
1348
1349 /// Return true if this instruction has any modifiers.
1350 /// e.g. src[012]_mod, omod, clamp.
1351 bool hasModifiers(unsigned Opcode) const;
1352
1353 bool hasModifiersSet(const MachineInstr &MI, AMDGPU::OpName OpName) const;
1354 bool hasAnyModifiersSet(const MachineInstr &MI) const;
1355
1356 bool canShrink(const MachineInstr &MI,
1357 const MachineRegisterInfo &MRI) const;
1358
1359 MachineInstr *buildShrunkInst(MachineInstr &MI,
1360 unsigned NewOpcode) const;
1361
1362 bool verifyInstruction(const MachineInstr &MI,
1363 StringRef &ErrInfo) const override;
1364
1365 unsigned getVALUOp(const MachineInstr &MI) const;
1366 unsigned getVALUOp(unsigned Opc) const;
1367
1368 void insertScratchExecCopy(MachineFunction &MF, MachineBasicBlock &MBB,
1369 MachineBasicBlock::iterator MBBI,
1370 const DebugLoc &DL, Register Reg, bool IsSCCLive,
1371 SlotIndexes *Indexes = nullptr) const;
1372
1373 void restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
1374 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
1375 Register Reg, SlotIndexes *Indexes = nullptr) const;
1376
1377 MachineInstr *getWholeWaveFunctionSetup(MachineFunction &MF) const;
1378
1379 /// Return the correct register class for \p OpNo. For target-specific
1380 /// instructions, this will return the register class that has been defined
1381 /// in tablegen. For generic instructions, like REG_SEQUENCE it will return
1382 /// the register class of its machine operand.
1383 /// to infer the correct register class base on the other operands.
1384 const TargetRegisterClass *getOpRegClass(const MachineInstr &MI,
1385 unsigned OpNo) const;
1386
1387 /// Return the size in bytes of the operand OpNo on the given
1388 // instruction opcode.
1389 unsigned getOpSize(uint32_t Opcode, unsigned OpNo) const {
1390 const MCOperandInfo &OpInfo = get(Opcode).operands()[OpNo];
1391
1392 if (OpInfo.RegClass == -1) {
1393 // If this is an immediate operand, this must be a 32-bit literal.
1394 assert(OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE);
1395 return 4;
1396 }
1397
1398 return RI.getRegSizeInBits(RC: *RI.getRegClass(i: getOpRegClassID(OpInfo))) / 8;
1399 }
1400
1401 /// This form should usually be preferred since it handles operands
1402 /// with unknown register classes.
1403 unsigned getOpSize(const MachineInstr &MI, unsigned OpNo) const {
1404 const MachineOperand &MO = MI.getOperand(i: OpNo);
1405 if (MO.isReg()) {
1406 if (unsigned SubReg = MO.getSubReg()) {
1407 return RI.getSubRegIdxSize(Idx: SubReg) / 8;
1408 }
1409 }
1410 return RI.getRegSizeInBits(RC: *getOpRegClass(MI, OpNo)) / 8;
1411 }
1412
1413 /// Legalize the \p OpIndex operand of this instruction by inserting
1414 /// a MOV. For example:
1415 /// ADD_I32_e32 VGPR0, 15
1416 /// to
1417 /// MOV VGPR1, 15
1418 /// ADD_I32_e32 VGPR0, VGPR1
1419 ///
1420 /// If the operand being legalized is a register, then a COPY will be used
1421 /// instead of MOV.
1422 void legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const;
1423
1424 /// Check if \p MO is a legal operand if it was the \p OpIdx Operand
1425 /// for \p MI.
1426 bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
1427 const MachineOperand *MO = nullptr) const;
1428
1429 /// Check if \p MO would be a valid operand for the given operand
1430 /// definition \p OpInfo. Note this does not attempt to validate constant bus
1431 /// restrictions (e.g. literal constant usage).
1432 bool isLegalVSrcOperand(const MachineRegisterInfo &MRI,
1433 const MCOperandInfo &OpInfo,
1434 const MachineOperand &MO) const;
1435
1436 /// Check if \p MO (a register operand) is a legal register for the
1437 /// given operand description or operand index.
1438 /// The operand index version provide more legality checks
1439 bool isLegalRegOperand(const MachineRegisterInfo &MRI,
1440 const MCOperandInfo &OpInfo,
1441 const MachineOperand &MO) const;
1442 bool isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
1443 const MachineOperand &MO) const;
1444
1445 /// Check if \p MO would be a legal operand for gfx12+ packed math FP32
1446 /// instructions. Packed math FP32 instructions typically accept SGPRs or
1447 /// VGPRs as source operands. On gfx12+, if a source operand uses SGPRs, the
1448 /// HW can only read the first SGPR and use it for both the low and high
1449 /// operations.
1450 /// \p SrcN can be 0, 1, or 2, representing src0, src1, and src2,
1451 /// respectively. If \p MO is nullptr, the operand corresponding to SrcN will
1452 /// be used.
1453 bool isLegalGFX12PlusPackedMathFP32Operand(
1454 const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,
1455 const MachineOperand *MO = nullptr) const;
1456
1457 /// Legalize operands in \p MI by either commuting it or inserting a
1458 /// copy of src1.
1459 void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1460
1461 /// Fix operands in \p MI to satisfy constant bus requirements.
1462 void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1463
1464 /// Copy a value from a VGPR (\p SrcReg) to SGPR. The desired register class
1465 /// for the dst register (\p DstRC) can be optionally supplied. This function
1466 /// can only be used when it is know that the value in SrcReg is same across
1467 /// all threads in the wave.
1468 /// \returns The SGPR register that \p SrcReg was copied to.
1469 Register readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,
1470 MachineRegisterInfo &MRI,
1471 const TargetRegisterClass *DstRC = nullptr) const;
1472
1473 void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1474 void legalizeOperandsFLAT(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1475
1476 void legalizeGenericOperand(MachineBasicBlock &InsertMBB,
1477 MachineBasicBlock::iterator I,
1478 const TargetRegisterClass *DstRC,
1479 MachineOperand &Op, MachineRegisterInfo &MRI,
1480 const DebugLoc &DL) const;
1481
1482 /// Legalize all operands in this instruction. This function may create new
1483 /// instructions and control-flow around \p MI. If present, \p MDT is
1484 /// updated.
1485 /// \returns A new basic block that contains \p MI if new blocks were created.
1486 MachineBasicBlock *
1487 legalizeOperands(MachineInstr &MI, MachineDominatorTree *MDT = nullptr) const;
1488
1489 /// Change SADDR form of a FLAT \p Inst to its VADDR form if saddr operand
1490 /// was moved to VGPR. \returns true if succeeded.
1491 bool moveFlatAddrToVGPR(MachineInstr &Inst) const;
1492
1493 /// Fix operands in Inst to fix 16bit SALU to VALU lowering.
1494 void legalizeOperandsVALUt16(MachineInstr &Inst,
1495 MachineRegisterInfo &MRI) const;
1496 void legalizeOperandsVALUt16(MachineInstr &Inst, unsigned OpIdx,
1497 MachineRegisterInfo &MRI) const;
1498
1499 /// Replace the instructions opcode with the equivalent VALU
1500 /// opcode. This function will also move the users of MachineInstruntions
1501 /// in the \p WorkList to the VALU if necessary. If present, \p MDT is
1502 /// updated.
1503 void moveToVALU(SIInstrWorklist &Worklist, MachineDominatorTree *MDT) const;
1504
1505 void moveToVALUImpl(SIInstrWorklist &Worklist, MachineDominatorTree *MDT,
1506 MachineInstr &Inst) const;
1507
1508 void insertNoop(MachineBasicBlock &MBB,
1509 MachineBasicBlock::iterator MI) const override;
1510
1511 void insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1512 unsigned Quantity) const override;
1513
1514 void insertReturn(MachineBasicBlock &MBB) const;
1515
1516 /// Build instructions that simulate the behavior of a `s_trap 2` instructions
1517 /// for hardware (namely, gfx11) that runs in PRIV=1 mode. There, s_trap is
1518 /// interpreted as a nop.
1519 MachineBasicBlock *insertSimulatedTrap(MachineRegisterInfo &MRI,
1520 MachineBasicBlock &MBB,
1521 MachineInstr &MI,
1522 const DebugLoc &DL) const;
1523
1524 /// Return the number of wait states that result from executing this
1525 /// instruction.
1526 static unsigned getNumWaitStates(const MachineInstr &MI);
1527
1528 /// Returns the operand named \p Op. If \p MI does not have an
1529 /// operand named \c Op, this function returns nullptr.
1530 LLVM_READONLY
1531 MachineOperand *getNamedOperand(MachineInstr &MI,
1532 AMDGPU::OpName OperandName) const;
1533
1534 LLVM_READONLY
1535 const MachineOperand *getNamedOperand(const MachineInstr &MI,
1536 AMDGPU::OpName OperandName) const {
1537 return getNamedOperand(MI&: const_cast<MachineInstr &>(MI), OperandName);
1538 }
1539
1540 /// Get required immediate operand
1541 int64_t getNamedImmOperand(const MachineInstr &MI,
1542 AMDGPU::OpName OperandName) const {
1543 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OperandName);
1544 return MI.getOperand(i: Idx).getImm();
1545 }
1546
1547 uint64_t getDefaultRsrcDataFormat() const;
1548 uint64_t getScratchRsrcWords23() const;
1549
1550 bool isLowLatencyInstruction(const MachineInstr &MI) const;
1551 bool isHighLatencyDef(int Opc) const override;
1552
1553 /// Return the descriptor of the target-specific machine instruction
1554 /// that corresponds to the specified pseudo or native opcode.
1555 const MCInstrDesc &getMCOpcodeFromPseudo(unsigned Opcode) const {
1556 return get(Opcode: pseudoToMCOpcode(Opcode));
1557 }
1558
1559 Register isStackAccess(const MachineInstr &MI, int &FrameIndex,
1560 TypeSize &MemBytes) const;
1561 Register isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex,
1562 TypeSize &MemBytes) const;
1563
1564 Register isLoadFromStackSlot(const MachineInstr &MI,
1565 int &FrameIndex) const override {
1566 TypeSize MemBytes = TypeSize::getZero();
1567 return isLoadFromStackSlot(MI, FrameIndex, MemBytes);
1568 }
1569
1570 Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex,
1571 TypeSize &MemBytes) const override;
1572
1573 Register isStoreToStackSlot(const MachineInstr &MI,
1574 int &FrameIndex) const override {
1575 TypeSize MemBytes = TypeSize::getZero();
1576 return isStoreToStackSlot(MI, FrameIndex, MemBytes);
1577 }
1578
1579 Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex,
1580 TypeSize &MemBytes) const override;
1581
1582 unsigned getInstBundleSize(const MachineInstr &MI) const;
1583 unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
1584
1585 bool mayAccessFlatAddressSpace(const MachineInstr &MI) const;
1586
1587 std::pair<unsigned, unsigned>
1588 decomposeMachineOperandsTargetFlags(unsigned TF) const override;
1589
1590 ArrayRef<std::pair<int, const char *>>
1591 getSerializableTargetIndices() const override;
1592
1593 ArrayRef<std::pair<unsigned, const char *>>
1594 getSerializableDirectMachineOperandTargetFlags() const override;
1595
1596 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
1597 getSerializableMachineMemOperandTargetFlags() const override;
1598
1599 ScheduleHazardRecognizer *
1600 CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
1601 const ScheduleDAG *DAG) const override;
1602
1603 ScheduleHazardRecognizer *
1604 CreateTargetPostRAHazardRecognizer(const MachineFunction &MF,
1605 MachineLoopInfo *MLI) const override;
1606
1607 ScheduleHazardRecognizer *
1608 CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
1609 const ScheduleDAGMI *DAG) const override;
1610
1611 unsigned getLiveRangeSplitOpcode(Register Reg,
1612 const MachineFunction &MF) const override;
1613
1614 bool isBasicBlockPrologue(const MachineInstr &MI,
1615 Register Reg = Register()) const override;
1616
1617 bool canAddToBBProlog(const MachineInstr &MI) const;
1618
1619 MachineInstr *createPHIDestinationCopy(MachineBasicBlock &MBB,
1620 MachineBasicBlock::iterator InsPt,
1621 const DebugLoc &DL, Register Src,
1622 Register Dst) const override;
1623
1624 MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB,
1625 MachineBasicBlock::iterator InsPt,
1626 const DebugLoc &DL, Register Src,
1627 unsigned SrcSubReg,
1628 Register Dst) const override;
1629
1630 bool isWave32() const;
1631
1632 /// Return a partially built integer add instruction without carry.
1633 /// Caller must add source operands.
1634 /// For pre-GFX9 it will generate unused carry destination operand.
1635 /// TODO: After GFX9 it should return a no-carry operation.
1636 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1637 MachineBasicBlock::iterator I,
1638 const DebugLoc &DL,
1639 Register DestReg) const;
1640
1641 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1642 MachineBasicBlock::iterator I,
1643 const DebugLoc &DL,
1644 Register DestReg,
1645 RegScavenger &RS) const;
1646
1647 static bool isKillTerminator(unsigned Opcode);
1648 const MCInstrDesc &getKillTerminatorFromPseudo(unsigned Opcode) const;
1649
1650 bool isLegalMUBUFImmOffset(unsigned Imm) const;
1651
1652 static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST);
1653
1654 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
1655 Align Alignment = Align(4)) const;
1656
1657 /// Returns if \p Offset is legal for the subtarget as the offset to a FLAT
1658 /// encoded instruction with the given \p FlatVariant.
1659 bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
1660 uint64_t FlatVariant) const;
1661
1662 /// Split \p COffsetVal into {immediate offset field, remainder offset}
1663 /// values.
1664 std::pair<int64_t, int64_t> splitFlatOffset(int64_t COffsetVal,
1665 unsigned AddrSpace,
1666 uint64_t FlatVariant) const;
1667
1668 /// Returns true if negative offsets are allowed for the given \p FlatVariant.
1669 bool allowNegativeFlatOffset(uint64_t FlatVariant) const;
1670
1671 /// \brief Return a target-specific opcode if Opcode is a pseudo instruction.
1672 /// Return -1 if the target-specific opcode for the pseudo instruction does
1673 /// not exist. If Opcode is not a pseudo instruction, this is identity.
1674 int pseudoToMCOpcode(int Opcode) const;
1675
1676 /// \brief Check if this instruction should only be used by assembler.
1677 /// Return true if this opcode should not be used by codegen.
1678 bool isAsmOnlyOpcode(int MCOp) const;
1679
1680 void fixImplicitOperands(MachineInstr &MI) const;
1681
1682 MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
1683 ArrayRef<unsigned> Ops,
1684 MachineBasicBlock::iterator InsertPt,
1685 int FrameIndex,
1686 LiveIntervals *LIS = nullptr,
1687 VirtRegMap *VRM = nullptr) const override;
1688
1689 unsigned getInstrLatency(const InstrItineraryData *ItinData,
1690 const MachineInstr &MI,
1691 unsigned *PredCost = nullptr) const override;
1692
1693 const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override;
1694
1695 ValueUniformity getValueUniformity(const MachineInstr &MI) const final;
1696
1697 ValueUniformity getGenericValueUniformity(const MachineInstr &MI) const;
1698
1699 const MIRFormatter *getMIRFormatter() const override;
1700
1701 static unsigned getDSShaderTypeValue(const MachineFunction &MF);
1702
1703 const TargetSchedModel &getSchedModel() const { return SchedModel; }
1704
1705 // FIXME: This should be removed
1706 // Enforce operand's \p OpName even alignment if required by target.
1707 // This is used if an operand is a 32 bit register but needs to be aligned
1708 // regardless.
1709 void enforceOperandRCAlignment(MachineInstr &MI, AMDGPU::OpName OpName) const;
1710};
1711
1712/// \brief Returns true if a reg:subreg pair P has a TRC class
1713inline bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P,
1714 const TargetRegisterClass &TRC,
1715 MachineRegisterInfo &MRI) {
1716 auto *RC = MRI.getRegClass(Reg: P.Reg);
1717 if (!P.SubReg)
1718 return RC == &TRC;
1719 auto *TRI = MRI.getTargetRegisterInfo();
1720 return RC == TRI->getMatchingSuperRegClass(A: RC, B: &TRC, Idx: P.SubReg);
1721}
1722
1723/// \brief Create RegSubRegPair from a register MachineOperand
1724inline
1725TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O) {
1726 assert(O.isReg());
1727 return TargetInstrInfo::RegSubRegPair(O.getReg(), O.getSubReg());
1728}
1729
1730/// \brief Return the SubReg component from REG_SEQUENCE
1731TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI,
1732 unsigned SubReg);
1733
1734/// \brief Return the defining instruction for a given reg:subreg pair
1735/// skipping copy like instructions and subreg-manipulation pseudos.
1736/// Following another subreg of a reg:subreg isn't supported.
1737MachineInstr *getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
1738 const MachineRegisterInfo &MRI);
1739
1740/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1741/// DefMI and the use at \p UseMI. Should be run on SSA. Currently does not
1742/// attempt to track between blocks.
1743bool execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
1744 Register VReg,
1745 const MachineInstr &DefMI,
1746 const MachineInstr &UseMI);
1747
1748/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1749/// DefMI and all its uses. Should be run on SSA. Currently does not attempt to
1750/// track between blocks.
1751bool execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
1752 Register VReg,
1753 const MachineInstr &DefMI);
1754
1755namespace AMDGPU {
1756
1757 LLVM_READONLY
1758 int32_t getVOPe64(uint32_t Opcode);
1759
1760 LLVM_READONLY
1761 int32_t getVOPe32(uint32_t Opcode);
1762
1763 LLVM_READONLY
1764 int32_t getSDWAOp(uint32_t Opcode);
1765
1766 LLVM_READONLY
1767 int32_t getDPPOp32(uint32_t Opcode);
1768
1769 LLVM_READONLY
1770 int32_t getDPPOp64(uint32_t Opcode);
1771
1772 LLVM_READONLY
1773 int32_t getBasicFromSDWAOp(uint32_t Opcode);
1774
1775 LLVM_READONLY
1776 int32_t getCommuteRev(uint32_t Opcode);
1777
1778 LLVM_READONLY
1779 int32_t getCommuteOrig(uint32_t Opcode);
1780
1781 LLVM_READONLY
1782 int32_t getAddr64Inst(uint32_t Opcode);
1783
1784 /// Check if \p Opcode is an Addr64 opcode.
1785 ///
1786 /// \returns \p Opcode if it is an Addr64 opcode, otherwise -1.
1787 LLVM_READONLY
1788 int32_t getIfAddr64Inst(uint32_t Opcode);
1789
1790 LLVM_READONLY
1791 int32_t getSOPKOp(uint32_t Opcode);
1792
1793 /// \returns SADDR form of a FLAT Global instruction given an \p Opcode
1794 /// of a VADDR form.
1795 LLVM_READONLY
1796 int32_t getGlobalSaddrOp(uint32_t Opcode);
1797
1798 /// \returns VADDR form of a FLAT Global instruction given an \p Opcode
1799 /// of a SADDR form.
1800 LLVM_READONLY
1801 int32_t getGlobalVaddrOp(uint32_t Opcode);
1802
1803 LLVM_READONLY
1804 int32_t getVCMPXNoSDstOp(uint32_t Opcode);
1805
1806 /// \returns ST form with only immediate offset of a FLAT Scratch instruction
1807 /// given an \p Opcode of an SS (SADDR) form.
1808 LLVM_READONLY
1809 int32_t getFlatScratchInstSTfromSS(uint32_t Opcode);
1810
1811 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
1812 /// of an SVS (SADDR + VADDR) form.
1813 LLVM_READONLY
1814 int32_t getFlatScratchInstSVfromSVS(uint32_t Opcode);
1815
1816 /// \returns SS (SADDR) form of a FLAT Scratch instruction given an \p Opcode
1817 /// of an SV (VADDR) form.
1818 LLVM_READONLY
1819 int32_t getFlatScratchInstSSfromSV(uint32_t Opcode);
1820
1821 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
1822 /// of an SS (SADDR) form.
1823 LLVM_READONLY
1824 int32_t getFlatScratchInstSVfromSS(uint32_t Opcode);
1825
1826 /// \returns earlyclobber version of a MAC MFMA is exists.
1827 LLVM_READONLY
1828 int32_t getMFMAEarlyClobberOp(uint32_t Opcode);
1829
1830 /// \returns Version of an MFMA instruction which uses AGPRs for srcC and
1831 /// vdst, given an \p Opcode of an MFMA which uses VGPRs for srcC/vdst.
1832 LLVM_READONLY
1833 int32_t getMFMASrcCVDstAGPROp(uint32_t Opcode);
1834
1835 /// \returns v_cmpx version of a v_cmp instruction.
1836 LLVM_READONLY
1837 int32_t getVCMPXOpFromVCMP(uint32_t Opcode);
1838
1839 const uint64_t RSRC_DATA_FORMAT = 0xf00000000000LL;
1840 const uint64_t RSRC_ELEMENT_SIZE_SHIFT = (32 + 19);
1841 const uint64_t RSRC_INDEX_STRIDE_SHIFT = (32 + 21);
1842 const uint64_t RSRC_TID_ENABLE = UINT64_C(1) << (32 + 23);
1843
1844} // end namespace AMDGPU
1845
1846namespace AMDGPU {
1847enum AsmComments : MachineInstr::AsmPrinterFlagTy {
1848 // For sgpr to vgpr spill instructions
1849 SGPR_SPILL = MachineInstr::TAsmComments
1850};
1851} // namespace AMDGPU
1852
1853namespace SI {
1854namespace KernelInputOffsets {
1855
1856/// Offsets in bytes from the start of the input buffer
1857enum Offsets {
1858 NGROUPS_X = 0,
1859 NGROUPS_Y = 4,
1860 NGROUPS_Z = 8,
1861 GLOBAL_SIZE_X = 12,
1862 GLOBAL_SIZE_Y = 16,
1863 GLOBAL_SIZE_Z = 20,
1864 LOCAL_SIZE_X = 24,
1865 LOCAL_SIZE_Y = 28,
1866 LOCAL_SIZE_Z = 32
1867};
1868
1869} // end namespace KernelInputOffsets
1870} // end namespace SI
1871
1872} // end namespace llvm
1873
1874#endif // LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
1875