1//===- SIInstrInfo.h - SI Instruction Info Interface ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Interface definition for SIInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
15#define LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
16
17#include "AMDGPUMIRFormatter.h"
18#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
19#include "SIRegisterInfo.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/CodeGen/TargetInstrInfo.h"
23#include "llvm/CodeGen/TargetSchedule.h"
24
25#define GET_INSTRINFO_HEADER
26#include "AMDGPUGenInstrInfo.inc"
27
28namespace llvm {
29
30class APInt;
31class GCNSubtarget;
32class LiveVariables;
33class MachineDominatorTree;
34class MachineRegisterInfo;
35class RegScavenger;
36class SIMachineFunctionInfo;
37class MCRegisterClass;
38using TargetRegisterClass = MCRegisterClass;
39class ScheduleHazardRecognizer;
40
41constexpr unsigned DefaultMemoryClusterDWordsLimit = 8;
42
43/// Mark the MMO of a uniform load if there are no potentially clobbering stores
44/// on any path from the start of an entry function to this load.
45static const MachineMemOperand::Flags MONoClobber =
46 MachineMemOperand::MOTargetFlag1;
47
48/// Mark the MMO of a load as the last use.
49static const MachineMemOperand::Flags MOLastUse =
50 MachineMemOperand::MOTargetFlag2;
51
52/// Mark the MMO of cooperative load/store atomics.
53static const MachineMemOperand::Flags MOCooperative =
54 MachineMemOperand::MOTargetFlag3;
55
56struct V2PhysSCopyInfo {
57 // Operands that need to replaced by waterfall
58 SmallVector<MachineOperand *> MOs;
59 // Target physical registers replacing the MOs
60 SmallVector<Register> SGPRs;
61};
62/// Mark the MMO of accesses to memory locations that are
63/// never written to by other threads.
64static const MachineMemOperand::Flags MOThreadPrivate =
65 MachineMemOperand::MOTargetFlag4;
66
67/// Utility to store machine instructions worklist.
68struct SIInstrWorklist {
69 SIInstrWorklist() = default;
70
71 void insert(MachineInstr *MI);
72
73 MachineInstr *top() const {
74 const auto *iter = InstrList.begin();
75 return *iter;
76 }
77
78 void erase_top() {
79 const auto *iter = InstrList.begin();
80 InstrList.erase(I: iter);
81 }
82
83 bool empty() const { return InstrList.empty(); }
84
85 void clear() {
86 InstrList.clear();
87 DeferredList.clear();
88 }
89
90 bool isDeferred(MachineInstr *MI);
91
92 SetVector<MachineInstr *> &getDeferredList() { return DeferredList; }
93
94private:
95 /// InstrList contains the MachineInstrs.
96 SetVector<MachineInstr *> InstrList;
97 /// Deferred instructions are specific MachineInstr
98 /// that will be added by insert method.
99 SetVector<MachineInstr *> DeferredList;
100};
101
102// In namespace llvm so ADL finds it when SIInstrFlags predicates are
103// instantiated with MachineInstr (MachineInstr is in namespace llvm).
104inline uint64_t getTSFlags(const MachineInstr &MI) {
105 return MI.getDesc().TSFlags;
106}
107
108class SIInstrInfo final : public AMDGPUGenInstrInfo {
109 struct ThreeAddressUpdates;
110
111private:
112 const SIRegisterInfo RI;
113 const GCNSubtarget &ST;
114 TargetSchedModel SchedModel;
115 mutable std::unique_ptr<AMDGPUMIRFormatter> Formatter;
116
117 // The inverse predicate should have the negative value.
118 enum BranchPredicate {
119 INVALID_BR = 0,
120 SCC_TRUE = 1,
121 SCC_FALSE = -1,
122 VCCNZ = 2,
123 VCCZ = -2,
124 EXECNZ = -3,
125 EXECZ = 3
126 };
127
128 using SetVectorType = SmallSetVector<MachineInstr *, 32>;
129
130 static unsigned getBranchOpcode(BranchPredicate Cond);
131 static BranchPredicate getBranchPredicate(unsigned Opcode);
132
133public:
134 unsigned buildExtractSubReg(MachineBasicBlock::iterator MI,
135 MachineRegisterInfo &MRI,
136 const MachineOperand &SuperReg,
137 const TargetRegisterClass *SuperRC,
138 unsigned SubIdx,
139 const TargetRegisterClass *SubRC) const;
140 MachineOperand buildExtractSubRegOrImm(
141 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
142 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
143 unsigned SubIdx, const TargetRegisterClass *SubRC) const;
144
145private:
146 bool optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,
147 bool NeedInversion) const;
148
149 bool invertSCCUse(MachineInstr *SCCDef) const;
150
151 void swapOperands(MachineInstr &Inst) const;
152
153 std::pair<bool, MachineBasicBlock *>
154 moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
155 MachineDominatorTree *MDT = nullptr) const;
156
157 void lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
158 MachineDominatorTree *MDT = nullptr) const;
159
160 void lowerScalarAbs(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
161
162 void lowerScalarAbsDiff(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
163
164 void lowerScalarXnor(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
165
166 void splitScalarNotBinop(SIInstrWorklist &Worklist, MachineInstr &Inst,
167 unsigned Opcode) const;
168
169 void splitScalarBinOpN2(SIInstrWorklist &Worklist, MachineInstr &Inst,
170 unsigned Opcode) const;
171
172 void splitScalar64BitUnaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
173 unsigned Opcode, bool Swap = false) const;
174
175 void splitScalar64BitBinaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
176 unsigned Opcode,
177 MachineDominatorTree *MDT = nullptr) const;
178
179 void splitScalarSMulU64(SIInstrWorklist &Worklist, MachineInstr &Inst,
180 MachineDominatorTree *MDT) const;
181
182 void splitScalarSMulPseudo(SIInstrWorklist &Worklist, MachineInstr &Inst,
183 MachineDominatorTree *MDT) const;
184
185 void splitScalar64BitXnor(SIInstrWorklist &Worklist, MachineInstr &Inst,
186 MachineDominatorTree *MDT = nullptr) const;
187
188 void splitScalar64BitBCNT(SIInstrWorklist &Worklist,
189 MachineInstr &Inst) const;
190 void splitScalar64BitBFE(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
191 void splitScalar64BitCountOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
192 unsigned Opcode,
193 MachineDominatorTree *MDT = nullptr) const;
194 void movePackToVALU(SIInstrWorklist &Worklist, MachineRegisterInfo &MRI,
195 MachineInstr &Inst) const;
196
197 void addUsersToMoveToVALUWorklist(Register Reg, MachineRegisterInfo &MRI,
198 SIInstrWorklist &Worklist) const;
199
200 void addSCCDefUsersToVALUWorklist(const MachineOperand &Op,
201 MachineInstr &SCCDefInst,
202 SIInstrWorklist &Worklist,
203 Register NewCond = Register()) const;
204 void addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
205 SIInstrWorklist &Worklist) const;
206
207 const TargetRegisterClass *
208 getDestEquivalentVGPRClass(const MachineInstr &Inst) const;
209
210 bool checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
211 const MachineInstr &MIb) const;
212
213 Register findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const;
214
215 bool verifyCopy(const MachineInstr &MI, const MachineRegisterInfo &MRI,
216 StringRef &ErrInfo) const;
217
218 bool resultDependsOnExec(const MachineInstr &MI) const;
219
220 MachineInstr *convertToThreeAddressImpl(MachineInstr &MI,
221 ThreeAddressUpdates &Updates) const;
222
223protected:
224 /// If the specific machine instruction is a instruction that moves/copies
225 /// value from one register to another register return destination and source
226 /// registers as machine operands.
227 std::optional<DestSourcePair>
228 isCopyInstrImpl(const MachineInstr &MI) const override;
229
230 bool swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,
231 AMDGPU::OpName Src0OpName, MachineOperand &Src1,
232 AMDGPU::OpName Src1OpName) const;
233 bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx,
234 unsigned toIdx) const;
235 MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
236 unsigned OpIdx0,
237 unsigned OpIdx1) const override;
238
239public:
240 enum TargetOperandFlags {
241 MO_MASK = 0xf,
242
243 MO_NONE = 0,
244 // MO_GOTPCREL -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
245 MO_GOTPCREL = 1,
246 // MO_GOTPCREL32_LO -> symbol@gotpcrel32@lo -> R_AMDGPU_GOTPCREL32_LO.
247 MO_GOTPCREL32 = 2,
248 MO_GOTPCREL32_LO = 2,
249 // MO_GOTPCREL32_HI -> symbol@gotpcrel32@hi -> R_AMDGPU_GOTPCREL32_HI.
250 MO_GOTPCREL32_HI = 3,
251 // MO_GOTPCREL64 -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
252 MO_GOTPCREL64 = 4,
253 // MO_REL32_LO -> symbol@rel32@lo -> R_AMDGPU_REL32_LO.
254 MO_REL32 = 5,
255 MO_REL32_LO = 5,
256 // MO_REL32_HI -> symbol@rel32@hi -> R_AMDGPU_REL32_HI.
257 MO_REL32_HI = 6,
258 MO_REL64 = 7,
259
260 MO_FAR_BRANCH_OFFSET = 8,
261
262 MO_ABS32_LO = 9,
263 MO_ABS32_HI = 10,
264 MO_ABS64 = 11,
265 };
266
267 explicit SIInstrInfo(const GCNSubtarget &ST);
268
269 const SIRegisterInfo &getRegisterInfo() const {
270 return RI;
271 }
272
273 const GCNSubtarget &getSubtarget() const {
274 return ST;
275 }
276
277 bool isReMaterializableImpl(const MachineInstr &MI) const override;
278
279 bool isIgnorableUse(const MachineOperand &MO) const override;
280
281 bool isSafeToSink(MachineInstr &MI, MachineBasicBlock *SuccToSinkTo,
282 MachineCycleInfo *CI) const override;
283
284 bool areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, int64_t &Offset0,
285 int64_t &Offset1) const override;
286
287 bool isGlobalMemoryObject(const MachineInstr *MI) const override;
288
289 bool getMemOperandsWithOffsetWidth(
290 const MachineInstr &LdSt,
291 SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
292 bool &OffsetIsScalable, LocationSize &Width,
293 const TargetRegisterInfo *TRI) const final;
294
295 bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
296 int64_t Offset1, bool OffsetIsScalable1,
297 ArrayRef<const MachineOperand *> BaseOps2,
298 int64_t Offset2, bool OffsetIsScalable2,
299 unsigned ClusterSize,
300 unsigned NumBytes) const override;
301
302 bool shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0,
303 int64_t Offset1, unsigned NumLoads) const override;
304
305 void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
306 const DebugLoc &DL, Register DestReg, Register SrcReg,
307 bool KillSrc, bool RenamableDest = false,
308 bool RenamableSrc = false) const override;
309
310private:
311 void storeRegToStackSlotImpl(MachineBasicBlock &MBB,
312 MachineBasicBlock::iterator MI, Register SrcReg,
313 bool isKill, int FrameIndex,
314 const TargetRegisterClass *RC, Register VReg,
315 MachineInstr::MIFlag Flags, bool NeedsCFI) const;
316
317public:
318 void storeRegToStackSlotCFI(MachineBasicBlock &MBB,
319 MachineBasicBlock::iterator MI, Register SrcReg,
320 bool isKill, int FrameIndex,
321 const TargetRegisterClass *RC) const;
322
323 bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg,
324 int64_t &ImmVal) const override;
325
326 std::optional<int64_t> getImmOrMaterializedImm(MachineOperand &Op) const;
327
328 unsigned getVectorRegSpillSaveOpcode(Register Reg,
329 const TargetRegisterClass *RC,
330 unsigned Size,
331 const SIMachineFunctionInfo &MFI,
332 bool NeedsCFI) const;
333 unsigned
334 getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC,
335 unsigned Size,
336 const SIMachineFunctionInfo &MFI) const;
337
338 void storeRegToStackSlot(
339 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
340 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
341 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
342
343 void loadRegFromStackSlot(
344 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
345 int FrameIndex, const TargetRegisterClass *RC, Register VReg,
346 unsigned SubReg = 0,
347 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
348
349 bool expandPostRAPseudo(MachineInstr &MI) const override;
350
351 void
352 reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
353 Register DestReg, unsigned SubIdx, const MachineInstr &Orig,
354 LaneBitmask UsedLanes = LaneBitmask::getAll()) const override;
355
356 // Splits a V_MOV_B64_DPP_PSEUDO opcode into a pair of v_mov_b32_dpp
357 // instructions. Returns a pair of generated instructions.
358 // Can split either post-RA with physical registers or pre-RA with
359 // virtual registers. In latter case IR needs to be in SSA form and
360 // and a REG_SEQUENCE is produced to define original register.
361 std::pair<MachineInstr*, MachineInstr*>
362 expandMovDPP64(MachineInstr &MI) const;
363
364 // Returns an opcode that can be used to move a value to a \p DstRC
365 // register. If there is no hardware instruction that can store to \p
366 // DstRC, then AMDGPU::COPY is returned.
367 unsigned getMovOpcode(const TargetRegisterClass *DstRC) const;
368
369 const MCInstrDesc &getIndirectRegWriteMovRelPseudo(unsigned VecSize,
370 unsigned EltSize,
371 bool IsSGPR) const;
372
373 const MCInstrDesc &getIndirectGPRIDXPseudo(unsigned VecSize,
374 bool IsIndirectSrc) const;
375 LLVM_READONLY
376 int commuteOpcode(unsigned Opc) const;
377
378 LLVM_READONLY
379 inline int commuteOpcode(const MachineInstr &MI) const {
380 return commuteOpcode(Opc: MI.getOpcode());
381 }
382
383 bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0,
384 unsigned &SrcOpIdx1) const override;
385
386 bool findCommutedOpIndices(const MCInstrDesc &Desc, unsigned &SrcOpIdx0,
387 unsigned &SrcOpIdx1) const;
388
389 bool isBranchOffsetInRange(unsigned BranchOpc,
390 int64_t BrOffset) const override;
391
392 MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;
393
394 /// Return whether the block terminate with divergent branch.
395 /// Note this only work before lowering the pseudo control flow instructions.
396 bool hasDivergentBranch(const MachineBasicBlock *MBB) const;
397
398 void insertIndirectBranch(MachineBasicBlock &MBB,
399 MachineBasicBlock &NewDestBB,
400 MachineBasicBlock &RestoreBB, const DebugLoc &DL,
401 int64_t BrOffset, RegScavenger *RS) const override;
402
403 bool analyzeBranchImpl(MachineBasicBlock &MBB,
404 MachineBasicBlock::iterator I,
405 MachineBasicBlock *&TBB,
406 MachineBasicBlock *&FBB,
407 SmallVectorImpl<MachineOperand> &Cond,
408 bool AllowModify) const;
409
410 bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
411 MachineBasicBlock *&FBB,
412 SmallVectorImpl<MachineOperand> &Cond,
413 bool AllowModify = false) const override;
414
415 unsigned removeBranch(MachineBasicBlock &MBB,
416 int *BytesRemoved = nullptr) const override;
417
418 unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
419 MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
420 const DebugLoc &DL,
421 int *BytesAdded = nullptr) const override;
422
423 bool reverseBranchCondition(
424 SmallVectorImpl<MachineOperand> &Cond) const override;
425
426 bool canInsertSelect(const MachineBasicBlock &MBB,
427 ArrayRef<MachineOperand> Cond, Register DstReg,
428 Register TrueReg, Register FalseReg, int &CondCycles,
429 int &TrueCycles, int &FalseCycles) const override;
430
431 void insertSelect(MachineBasicBlock &MBB,
432 MachineBasicBlock::iterator I, const DebugLoc &DL,
433 Register DstReg, ArrayRef<MachineOperand> Cond,
434 Register TrueReg, Register FalseReg) const override;
435
436 bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
437 Register &SrcReg2, int64_t &CmpMask,
438 int64_t &CmpValue) const override;
439
440 bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
441 Register SrcReg2, int64_t CmpMask, int64_t CmpValue,
442 const MachineRegisterInfo *MRI) const override;
443
444 bool
445 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
446 const MachineInstr &MIb) const override;
447
448 static bool isFoldableCopy(const MachineInstr &MI);
449 static unsigned getFoldableCopySrcIdx(const MachineInstr &MI);
450
451 void removeModOperands(MachineInstr &MI) const;
452
453 void mutateAndCleanupImplicit(MachineInstr &MI,
454 const MCInstrDesc &NewDesc) const;
455
456 /// Return the extracted immediate value in a subregister use from a constant
457 /// materialized in a super register.
458 ///
459 /// e.g. %imm = S_MOV_B64 K[0:63]
460 /// USE %imm.sub1
461 /// This will return K[32:63]
462 static std::optional<int64_t> extractSubregFromImm(int64_t ImmVal,
463 unsigned SubRegIndex);
464
465 bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
466 MachineRegisterInfo *MRI) const final;
467
468 unsigned getMachineCSELookAheadLimit() const override { return 500; }
469
470 MachineInstr *convertToThreeAddress(MachineInstr &MI, LiveVariables *LV,
471 LiveIntervals *LIS) const override;
472
473 bool isSchedulingBoundary(const MachineInstr &MI,
474 const MachineBasicBlock *MBB,
475 const MachineFunction &MF) const override;
476
477 static bool isSALU(const MachineInstr &MI) {
478 return SIInstrFlags::isSALU(O: MI);
479 }
480
481 bool isSALU(uint32_t Opcode) const {
482 return SIInstrFlags::isSALU(O: get(Opcode));
483 }
484
485 static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA) {
486 if (!AllowLDSDMA && isLDSDMA(MI))
487 return false;
488
489 return SIInstrFlags::isVALU(O: MI);
490 }
491
492 /// LDSDMA instructions act as both VALU and memory instructions, thus
493 /// we also tag them as VALU. However, in many places, we do not actually want
494 /// to include LDSDMA instructions in this query. By setting \p AllowLDSDMA to
495 /// false, this will return false for LDSDMA instructions.
496 bool isVALU(uint32_t Opcode, bool AllowLDSDMA) const {
497 if (!AllowLDSDMA && isLDSDMA(Opcode))
498 return false;
499
500 return SIInstrFlags::isVALU(O: get(Opcode));
501 }
502
503 static bool isImage(const MachineInstr &MI) {
504 return SIInstrFlags::isImage(O: MI);
505 }
506
507 bool isImage(uint32_t Opcode) const {
508 return SIInstrFlags::isImage(O: get(Opcode));
509 }
510
511 static bool isVMEM(const MachineInstr &MI) {
512 return SIInstrFlags::isVMEM(O: MI);
513 }
514
515 bool isVMEM(uint32_t Opcode) const {
516 return SIInstrFlags::isVMEM(O: get(Opcode));
517 }
518
519 /// True if MI implicitly drains XCNT.
520 static bool isXcntDrain(const MachineInstr &MI);
521
522 static bool isSOP1(const MachineInstr &MI) {
523 return SIInstrFlags::isSOP1(O: MI);
524 }
525
526 bool isSOP1(uint32_t Opcode) const {
527 return SIInstrFlags::isSOP1(O: get(Opcode));
528 }
529
530 static bool isSOP2(const MachineInstr &MI) {
531 return SIInstrFlags::isSOP2(O: MI);
532 }
533
534 bool isSOP2(uint32_t Opcode) const {
535 return SIInstrFlags::isSOP2(O: get(Opcode));
536 }
537
538 static bool isSOPC(const MachineInstr &MI) {
539 return SIInstrFlags::isSOPC(O: MI);
540 }
541
542 bool isSOPC(uint32_t Opcode) const {
543 return SIInstrFlags::isSOPC(O: get(Opcode));
544 }
545
546 static bool isSOPK(const MachineInstr &MI) {
547 return SIInstrFlags::isSOPK(O: MI);
548 }
549
550 bool isSOPK(uint32_t Opcode) const {
551 return SIInstrFlags::isSOPK(O: get(Opcode));
552 }
553
554 static bool isSOPP(const MachineInstr &MI) {
555 return SIInstrFlags::isSOPP(O: MI);
556 }
557
558 bool isSOPP(uint32_t Opcode) const {
559 return SIInstrFlags::isSOPP(O: get(Opcode));
560 }
561
562 static bool isPacked(const MachineInstr &MI) {
563 return SIInstrFlags::isPacked(O: MI);
564 }
565
566 bool isPacked(uint32_t Opcode) const {
567 return SIInstrFlags::isPacked(O: get(Opcode));
568 }
569
570 static bool isVOP1(const MachineInstr &MI) {
571 return SIInstrFlags::isVOP1(O: MI);
572 }
573
574 bool isVOP1(uint32_t Opcode) const {
575 return SIInstrFlags::isVOP1(O: get(Opcode));
576 }
577
578 static bool isVOP2(const MachineInstr &MI) {
579 return SIInstrFlags::isVOP2(O: MI);
580 }
581
582 bool isVOP2(uint32_t Opcode) const {
583 return SIInstrFlags::isVOP2(O: get(Opcode));
584 }
585
586 static bool isVOP3(const MCInstrDesc &Desc) {
587 return SIInstrFlags::isVOP3(O: Desc);
588 }
589
590 static bool isVOP3(const MachineInstr &MI) { return isVOP3(Desc: MI.getDesc()); }
591
592 bool isVOP3(uint32_t Opcode) const { return isVOP3(Desc: get(Opcode)); }
593
594 static bool isSDWA(const MachineInstr &MI) {
595 return SIInstrFlags::isSDWA(O: MI);
596 }
597
598 bool isSDWA(uint32_t Opcode) const {
599 return SIInstrFlags::isSDWA(O: get(Opcode));
600 }
601
602 static bool isVOPC(const MachineInstr &MI) {
603 return SIInstrFlags::isVOPC(O: MI);
604 }
605
606 bool isVOPC(uint32_t Opcode) const {
607 return SIInstrFlags::isVOPC(O: get(Opcode));
608 }
609
610 static bool isMUBUF(const MachineInstr &MI) {
611 return SIInstrFlags::isMUBUF(O: MI);
612 }
613
614 bool isMUBUF(uint32_t Opcode) const {
615 return SIInstrFlags::isMUBUF(O: get(Opcode));
616 }
617
618 static bool isMTBUF(const MachineInstr &MI) {
619 return SIInstrFlags::isMTBUF(O: MI);
620 }
621
622 bool isMTBUF(uint32_t Opcode) const {
623 return SIInstrFlags::isMTBUF(O: get(Opcode));
624 }
625
626 static bool isBUF(const MachineInstr &MI) {
627 return isMUBUF(MI) || isMTBUF(MI);
628 }
629
630 static bool isSMRD(const MachineInstr &MI) {
631 return SIInstrFlags::isSMRD(O: MI);
632 }
633
634 bool isSMRD(uint32_t Opcode) const {
635 return SIInstrFlags::isSMRD(O: get(Opcode));
636 }
637
638 bool isBufferSMRD(const MachineInstr &MI) const;
639
640 static bool isDS(const MachineInstr &MI) { return SIInstrFlags::isDS(O: MI); }
641
642 bool isDS(uint32_t Opcode) const { return SIInstrFlags::isDS(O: get(Opcode)); }
643
644 static bool isLDSDMA(const MachineInstr &MI) {
645 return (SIInstrFlags::isVALU(O: MI) && (isMUBUF(MI) || isFLAT(MI))) ||
646 SIInstrFlags::usesTENSOR_CNT(O: MI);
647 }
648
649 bool isLDSDMA(uint32_t Opcode) const {
650 return (SIInstrFlags::isVALU(O: get(Opcode)) &&
651 (isMUBUF(Opcode) || isFLAT(Opcode))) ||
652 SIInstrFlags::usesTENSOR_CNT(O: get(Opcode));
653 }
654
655 static bool isGWS(const MachineInstr &MI) { return SIInstrFlags::isGWS(O: MI); }
656
657 bool isGWS(uint32_t Opcode) const { return SIInstrFlags::isGWS(O: get(Opcode)); }
658
659 bool isAlwaysGDS(uint32_t Opcode) const;
660
661 static bool isMIMG(const MachineInstr &MI) {
662 return SIInstrFlags::isMIMG(O: MI);
663 }
664
665 bool isMIMG(uint32_t Opcode) const {
666 return SIInstrFlags::isMIMG(O: get(Opcode));
667 }
668
669 static bool isVIMAGE(const MachineInstr &MI) {
670 return SIInstrFlags::isVIMAGE(O: MI);
671 }
672
673 bool isVIMAGE(uint32_t Opcode) const {
674 return SIInstrFlags::isVIMAGE(O: get(Opcode));
675 }
676
677 static bool isVSAMPLE(const MachineInstr &MI) {
678 return SIInstrFlags::isVSAMPLE(O: MI);
679 }
680
681 bool isVSAMPLE(uint32_t Opcode) const {
682 return SIInstrFlags::isVSAMPLE(O: get(Opcode));
683 }
684
685 static bool isGather4(const MachineInstr &MI) {
686 return SIInstrFlags::isGather4(O: MI);
687 }
688
689 bool isGather4(uint32_t Opcode) const {
690 return SIInstrFlags::isGather4(O: get(Opcode));
691 }
692
693 static bool isFLAT(const MachineInstr &MI) {
694 return SIInstrFlags::isFLAT(O: MI);
695 }
696
697 // Is a FLAT encoded instruction which accesses a specific segment,
698 // i.e. global_* or scratch_*.
699 static bool isSegmentSpecificFLAT(const MachineInstr &MI) {
700 return SIInstrFlags::isSegmentSpecificFLAT(O: MI);
701 }
702
703 bool isSegmentSpecificFLAT(uint32_t Opcode) const {
704 return SIInstrFlags::isSegmentSpecificFLAT(O: get(Opcode));
705 }
706
707 static bool isFLATGlobal(const MachineInstr &MI) {
708 return SIInstrFlags::isFlatGlobal(O: MI);
709 }
710
711 bool isFLATGlobal(uint32_t Opcode) const {
712 return SIInstrFlags::isFlatGlobal(O: get(Opcode));
713 }
714
715 static bool isFLATScratch(const MachineInstr &MI) {
716 return SIInstrFlags::isFlatScratch(O: MI);
717 }
718
719 bool isFLATScratch(uint32_t Opcode) const {
720 return SIInstrFlags::isFlatScratch(O: get(Opcode));
721 }
722
723 // Any FLAT encoded instruction, including global_* and scratch_*.
724 bool isFLAT(uint32_t Opcode) const {
725 return SIInstrFlags::isFLAT(O: get(Opcode));
726 }
727
728 /// \returns true for SCRATCH_ instructions, or FLAT/BUF instructions unless
729 /// the MMOs do not include scratch.
730 /// Conservatively correct; will return true if \p MI cannot be proven
731 /// to not hit scratch.
732 bool mayAccessScratch(const MachineInstr &MI) const;
733
734 /// \returns true for FLAT instructions that can access VMEM.
735 bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const;
736
737 /// \returns true for FLAT instructions that can access LDS.
738 bool mayAccessLDSThroughFlat(const MachineInstr &MI, bool TgSplit) const;
739
740 static bool isBlockLoadStore(uint32_t Opcode) {
741 switch (Opcode) {
742 case AMDGPU::SI_BLOCK_SPILL_V1024_SAVE:
743 case AMDGPU::SI_BLOCK_SPILL_V1024_CFI_SAVE:
744 case AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE:
745 case AMDGPU::SCRATCH_STORE_BLOCK_SADDR:
746 case AMDGPU::SCRATCH_LOAD_BLOCK_SADDR:
747 case AMDGPU::SCRATCH_STORE_BLOCK_SVS:
748 case AMDGPU::SCRATCH_LOAD_BLOCK_SVS:
749 return true;
750 default:
751 return false;
752 }
753 }
754
755 static bool setsSCCIfResultIsNonZero(const MachineInstr &MI) {
756 switch (MI.getOpcode()) {
757 case AMDGPU::S_ABSDIFF_I32:
758 case AMDGPU::S_ABS_I32:
759 case AMDGPU::S_AND_B32:
760 case AMDGPU::S_AND_B64:
761 case AMDGPU::S_ANDN2_B32:
762 case AMDGPU::S_ANDN2_B64:
763 case AMDGPU::S_ASHR_I32:
764 case AMDGPU::S_ASHR_I64:
765 case AMDGPU::S_BCNT0_I32_B32:
766 case AMDGPU::S_BCNT0_I32_B64:
767 case AMDGPU::S_BCNT1_I32_B32:
768 case AMDGPU::S_BCNT1_I32_B64:
769 case AMDGPU::S_BFE_I32:
770 case AMDGPU::S_BFE_I64:
771 case AMDGPU::S_BFE_U32:
772 case AMDGPU::S_BFE_U64:
773 case AMDGPU::S_LSHL_B32:
774 case AMDGPU::S_LSHL_B64:
775 case AMDGPU::S_LSHR_B32:
776 case AMDGPU::S_LSHR_B64:
777 case AMDGPU::S_NAND_B32:
778 case AMDGPU::S_NAND_B64:
779 case AMDGPU::S_NOR_B32:
780 case AMDGPU::S_NOR_B64:
781 case AMDGPU::S_NOT_B32:
782 case AMDGPU::S_NOT_B64:
783 case AMDGPU::S_OR_B32:
784 case AMDGPU::S_OR_B64:
785 case AMDGPU::S_ORN2_B32:
786 case AMDGPU::S_ORN2_B64:
787 case AMDGPU::S_QUADMASK_B32:
788 case AMDGPU::S_QUADMASK_B64:
789 case AMDGPU::S_WQM_B32:
790 case AMDGPU::S_WQM_B64:
791 case AMDGPU::S_XNOR_B32:
792 case AMDGPU::S_XNOR_B64:
793 case AMDGPU::S_XOR_B32:
794 case AMDGPU::S_XOR_B64:
795 return true;
796 default:
797 return false;
798 }
799 }
800
801 static bool isEXP(const MachineInstr &MI) { return SIInstrFlags::isEXP(O: MI); }
802
803 static bool isDualSourceBlendEXP(const MachineInstr &MI) {
804 if (!isEXP(MI))
805 return false;
806 unsigned Target = MI.getOperand(i: 0).getImm();
807 return Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND0 ||
808 Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND1;
809 }
810
811 bool isEXP(uint32_t Opcode) const { return SIInstrFlags::isEXP(O: get(Opcode)); }
812
813 static bool isAtomicNoRet(const MachineInstr &MI) {
814 return SIInstrFlags::isAtomicNoRet(O: MI);
815 }
816
817 bool isAtomicNoRet(uint32_t Opcode) const {
818 return SIInstrFlags::isAtomicNoRet(O: get(Opcode));
819 }
820
821 static bool isAtomicRet(const MachineInstr &MI) {
822 return SIInstrFlags::isAtomicRet(O: MI);
823 }
824
825 bool isAtomicRet(uint32_t Opcode) const {
826 return SIInstrFlags::isAtomicRet(O: get(Opcode));
827 }
828
829 static bool isAtomic(const MachineInstr &MI) {
830 return SIInstrFlags::isAtomic(O: MI);
831 }
832
833 bool isAtomic(uint32_t Opcode) const {
834 return SIInstrFlags::isAtomic(O: get(Opcode));
835 }
836
837 static bool mayWriteLDSThroughDMA(const MachineInstr &MI) {
838 unsigned Opc = MI.getOpcode();
839 // Exclude instructions that read FROM LDS (not write to it)
840 return isLDSDMA(MI) && Opc != AMDGPU::BUFFER_STORE_LDS_DWORD &&
841 Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d2 &&
842 Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d4;
843 }
844
845 static bool isSBarrierSCCWrite(unsigned Opcode) {
846 return Opcode == AMDGPU::S_BARRIER_LEAVE ||
847 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM ||
848 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0;
849 }
850
851 static bool isCBranchVCCZRead(const MachineInstr &MI) {
852 unsigned Opc = MI.getOpcode();
853 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
854 !MI.getOperand(i: 1).isUndef();
855 }
856
857 static bool isWQM(const MachineInstr &MI) { return SIInstrFlags::isWQM(O: MI); }
858
859 bool isWQM(uint32_t Opcode) const { return SIInstrFlags::isWQM(O: get(Opcode)); }
860
861 static bool isDisableWQM(const MachineInstr &MI) {
862 return SIInstrFlags::isDisableWQM(O: MI);
863 }
864
865 bool isDisableWQM(uint32_t Opcode) const {
866 return SIInstrFlags::isDisableWQM(O: get(Opcode));
867 }
868
869 // SI_SPILL_S32_TO_VGPR and SI_RESTORE_S32_FROM_VGPR form a special case of
870 // SGPRs spilling to VGPRs which are SGPR spills but from VALU instructions
871 // therefore we need an explicit check for them since just checking if the
872 // Spill bit is set and what instruction type it came from misclassifies
873 // them.
874 static bool isVGPRSpill(const MachineInstr &MI) {
875 return MI.getOpcode() != AMDGPU::SI_SPILL_S32_TO_VGPR &&
876 MI.getOpcode() != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
877 (isSpill(MI) && isVALU(MI, /*AllowLDSDMA=*/AllowLDSDMA: true));
878 }
879
880 bool isVGPRSpill(uint32_t Opcode) const {
881 return Opcode != AMDGPU::SI_SPILL_S32_TO_VGPR &&
882 Opcode != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
883 (isSpill(Opcode) && isVALU(Opcode, /*AllowLDSDMA=*/AllowLDSDMA: true));
884 }
885
886 static bool isSGPRSpill(const MachineInstr &MI) {
887 return MI.getOpcode() == AMDGPU::SI_SPILL_S32_TO_VGPR ||
888 MI.getOpcode() == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
889 (isSpill(MI) && isSALU(MI));
890 }
891
892 bool isSGPRSpill(uint32_t Opcode) const {
893 return Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR ||
894 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
895 (isSpill(Opcode) && isSALU(Opcode));
896 }
897
898 bool isSpill(uint32_t Opcode) const {
899 return SIInstrFlags::isSpill(O: get(Opcode));
900 }
901
902 static bool isSpill(const MCInstrDesc &Desc) {
903 return SIInstrFlags::isSpill(O: Desc);
904 }
905
906 static bool isSpill(const MachineInstr &MI) { return isSpill(Desc: MI.getDesc()); }
907
908 static bool isWWMRegSpillOpcode(uint32_t Opcode) {
909 return Opcode == AMDGPU::SI_SPILL_WWM_V32_SAVE ||
910 Opcode == AMDGPU::SI_SPILL_WWM_AV32_SAVE ||
911 Opcode == AMDGPU::SI_SPILL_WWM_V32_RESTORE ||
912 Opcode == AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
913 }
914
915 static bool isChainCallOpcode(uint64_t Opcode) {
916 return Opcode == AMDGPU::SI_CS_CHAIN_TC_W32 ||
917 Opcode == AMDGPU::SI_CS_CHAIN_TC_W64;
918 }
919
920 static bool isDPP(const MachineInstr &MI) { return SIInstrFlags::isDPP(O: MI); }
921
922 bool isDPP(uint32_t Opcode) const { return SIInstrFlags::isDPP(O: get(Opcode)); }
923
924 static bool isTRANS(const MachineInstr &MI) {
925 return SIInstrFlags::isTRANS(O: MI);
926 }
927
928 bool isTRANS(uint32_t Opcode) const {
929 return SIInstrFlags::isTRANS(O: get(Opcode));
930 }
931
932 static bool isVOP3P(const MachineInstr &MI) {
933 return SIInstrFlags::isVOP3P(O: MI);
934 }
935
936 bool isVOP3P(uint32_t Opcode) const {
937 return SIInstrFlags::isVOP3P(O: get(Opcode));
938 }
939
940 bool isVOP3PMix(const MachineInstr &MI) const {
941 return isVOP3PMix(Opcode: MI.getOpcode());
942 }
943
944 bool isVOP3PMix(uint16_t Opcode) const {
945 switch (Opcode) {
946 case AMDGPU::V_FMA_MIXHI_F16:
947 case AMDGPU::V_FMA_MIXLO_F16:
948 case AMDGPU::V_FMA_MIX_F32:
949 case AMDGPU::V_MAD_MIXHI_F16:
950 case AMDGPU::V_MAD_MIXLO_F16:
951 case AMDGPU::V_MAD_MIX_F32:
952 return true;
953 default:
954 return false;
955 }
956 }
957
958 static bool isVINTRP(const MachineInstr &MI) {
959 return SIInstrFlags::isVINTRP(O: MI);
960 }
961
962 bool isVINTRP(uint32_t Opcode) const {
963 return SIInstrFlags::isVINTRP(O: get(Opcode));
964 }
965
966 static bool isMAI(const MCInstrDesc &Desc) {
967 return SIInstrFlags::isMAI(O: Desc);
968 }
969
970 static bool isMAI(const MachineInstr &MI) { return isMAI(Desc: MI.getDesc()); }
971
972 bool isMAI(uint32_t Opcode) const { return isMAI(Desc: get(Opcode)); }
973
974 static bool isMFMA(const MachineInstr &MI) {
975 return isMAI(MI) && MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
976 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
977 }
978
979 bool isMFMA(uint32_t Opcode) const {
980 return isMAI(Opcode) && Opcode != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
981 Opcode != AMDGPU::V_ACCVGPR_READ_B32_e64;
982 }
983
984 static bool isDOT(const MachineInstr &MI) { return SIInstrFlags::isDOT(O: MI); }
985
986 static bool isWMMA(const MachineInstr &MI) {
987 return SIInstrFlags::isWMMA(O: MI);
988 }
989
990 bool isWMMA(uint32_t Opcode) const {
991 return SIInstrFlags::isWMMA(O: get(Opcode));
992 }
993
994 static bool isMFMAorWMMA(const MachineInstr &MI) {
995 return isMFMA(MI) || isWMMA(MI) || isSWMMAC(MI);
996 }
997
998 bool isMFMAorWMMA(uint32_t Opcode) const {
999 return isMFMA(Opcode) || isWMMA(Opcode) || isSWMMAC(Opcode);
1000 }
1001
1002 static bool isSWMMAC(const MachineInstr &MI) {
1003 return SIInstrFlags::isSWMMAC(O: MI);
1004 }
1005
1006 bool isSWMMAC(uint32_t Opcode) const {
1007 return SIInstrFlags::isSWMMAC(O: get(Opcode));
1008 }
1009
1010 bool isDOT(uint32_t Opcode) const { return SIInstrFlags::isDOT(O: get(Opcode)); }
1011
1012 bool isXDLWMMA(const MachineInstr &MI) const;
1013
1014 bool isXDL(const MachineInstr &MI) const;
1015
1016 static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opc: Opcode); }
1017
1018 static bool isLDSDIR(const MachineInstr &MI) {
1019 return SIInstrFlags::isLDSDIR(O: MI);
1020 }
1021
1022 bool isLDSDIR(uint32_t Opcode) const {
1023 return SIInstrFlags::isLDSDIR(O: get(Opcode));
1024 }
1025
1026 static bool isVINTERP(const MachineInstr &MI) {
1027 return SIInstrFlags::isVINTERP(O: MI);
1028 }
1029
1030 bool isVINTERP(uint32_t Opcode) const {
1031 return SIInstrFlags::isVINTERP(O: get(Opcode));
1032 }
1033
1034 static bool isScalarUnit(const MachineInstr &MI) {
1035 return SIInstrFlags::isSALU(O: MI) || SIInstrFlags::isSMRD(O: MI);
1036 }
1037
1038 static bool usesVM_CNT(const MachineInstr &MI) {
1039 return SIInstrFlags::usesVM_CNT(O: MI);
1040 }
1041
1042 static bool usesLGKM_CNT(const MachineInstr &MI) {
1043 return SIInstrFlags::usesLGKM_CNT(O: MI);
1044 }
1045
1046 static bool usesASYNC_CNT(const MachineInstr &MI) {
1047 return SIInstrFlags::usesASYNC_CNT(O: MI);
1048 }
1049
1050 bool usesASYNC_CNT(uint32_t Opcode) const {
1051 return SIInstrFlags::usesASYNC_CNT(O: get(Opcode));
1052 }
1053
1054 static bool usesTENSOR_CNT(const MachineInstr &MI) {
1055 return MI.getDesc().TSFlags & SIInstrFlags::TENSOR_CNT;
1056 }
1057
1058 bool usesTENSOR_CNT(uint32_t Opcode) const {
1059 return get(Opcode).TSFlags & SIInstrFlags::TENSOR_CNT;
1060 }
1061
1062 // Most sopk treat the immediate as a signed 16-bit, however some
1063 // use it as unsigned.
1064 static bool sopkIsZext(unsigned Opcode) {
1065 return Opcode == AMDGPU::S_CMPK_EQ_U32 || Opcode == AMDGPU::S_CMPK_LG_U32 ||
1066 Opcode == AMDGPU::S_CMPK_GT_U32 || Opcode == AMDGPU::S_CMPK_GE_U32 ||
1067 Opcode == AMDGPU::S_CMPK_LT_U32 || Opcode == AMDGPU::S_CMPK_LE_U32 ||
1068 Opcode == AMDGPU::S_GETREG_B32 ||
1069 Opcode == AMDGPU::S_GETREG_B32_const;
1070 }
1071
1072 /// \returns true if this is an s_store_dword* instruction. This is more
1073 /// specific than isSMEM && mayStore.
1074 static bool isScalarStore(const MachineInstr &MI) {
1075 return SIInstrFlags::isScalarStore(O: MI);
1076 }
1077
1078 bool isScalarStore(uint32_t Opcode) const {
1079 return SIInstrFlags::isScalarStore(O: get(Opcode));
1080 }
1081
1082 static bool isFixedSize(const MachineInstr &MI) {
1083 return SIInstrFlags::isFixedSize(O: MI);
1084 }
1085
1086 bool isFixedSize(uint32_t Opcode) const {
1087 return SIInstrFlags::isFixedSize(O: get(Opcode));
1088 }
1089
1090 static bool hasFPClamp(const MachineInstr &MI) {
1091 return SIInstrFlags::hasFPClamp(O: MI);
1092 }
1093
1094 bool hasFPClamp(uint32_t Opcode) const {
1095 return SIInstrFlags::hasFPClamp(O: get(Opcode));
1096 }
1097
1098 static bool hasIntClamp(const MachineInstr &MI) {
1099 return SIInstrFlags::hasIntClamp(O: MI);
1100 }
1101
1102 static bool hasSameClamp(const MachineInstr &A, const MachineInstr &B) {
1103 const MCInstrDesc &DA = A.getDesc(), &DB = B.getDesc();
1104 return SIInstrFlags::hasFPClamp(O: DA) == SIInstrFlags::hasFPClamp(O: DB) &&
1105 SIInstrFlags::hasIntClamp(O: DA) == SIInstrFlags::hasIntClamp(O: DB) &&
1106 SIInstrFlags::hasClampLo(O: DA) == SIInstrFlags::hasClampLo(O: DB) &&
1107 SIInstrFlags::hasClampHi(O: DA) == SIInstrFlags::hasClampHi(O: DB);
1108 }
1109
1110 static bool usesFPDPRounding(const MachineInstr &MI) {
1111 return SIInstrFlags::usesFPDPRounding(O: MI);
1112 }
1113
1114 bool usesFPDPRounding(uint32_t Opcode) const {
1115 return SIInstrFlags::usesFPDPRounding(O: get(Opcode));
1116 }
1117
1118 static bool isFPAtomic(const MachineInstr &MI) {
1119 return SIInstrFlags::isFPAtomic(O: MI);
1120 }
1121
1122 bool isFPAtomic(uint32_t Opcode) const {
1123 return SIInstrFlags::isFPAtomic(O: get(Opcode));
1124 }
1125
1126 static bool isNeverUniform(const MachineInstr &MI) {
1127 return SIInstrFlags::isNeverUniform(O: MI);
1128 }
1129
1130 // Check to see if opcode is for a barrier start. Pre gfx12 this is just the
1131 // S_BARRIER, but after support for S_BARRIER_SIGNAL* / S_BARRIER_WAIT we want
1132 // to check for the barrier start (S_BARRIER_SIGNAL*)
1133 bool isBarrierStart(unsigned Opcode) const {
1134 return Opcode == AMDGPU::S_BARRIER ||
1135 Opcode == AMDGPU::S_BARRIER_SIGNAL_M0 ||
1136 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0 ||
1137 Opcode == AMDGPU::S_BARRIER_SIGNAL_IMM ||
1138 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM;
1139 }
1140
1141 bool isBarrier(unsigned Opcode) const {
1142 return isBarrierStart(Opcode) || Opcode == AMDGPU::S_BARRIER_WAIT ||
1143 Opcode == AMDGPU::S_BARRIER_INIT_M0 ||
1144 Opcode == AMDGPU::S_BARRIER_INIT_IMM ||
1145 Opcode == AMDGPU::S_BARRIER_JOIN_IMM ||
1146 Opcode == AMDGPU::S_BARRIER_LEAVE || Opcode == AMDGPU::DS_GWS_INIT ||
1147 Opcode == AMDGPU::DS_GWS_BARRIER;
1148 }
1149
1150 static bool isLoadMonitor(unsigned Opc) {
1151 switch (Opc) {
1152 case AMDGPU::GLOBAL_LOAD_MONITOR_B32:
1153 case AMDGPU::GLOBAL_LOAD_MONITOR_B32_SADDR:
1154 case AMDGPU::GLOBAL_LOAD_MONITOR_B64:
1155 case AMDGPU::GLOBAL_LOAD_MONITOR_B64_SADDR:
1156 case AMDGPU::GLOBAL_LOAD_MONITOR_B128:
1157 case AMDGPU::GLOBAL_LOAD_MONITOR_B128_SADDR:
1158 case AMDGPU::FLAT_LOAD_MONITOR_B32:
1159 case AMDGPU::FLAT_LOAD_MONITOR_B64:
1160 case AMDGPU::FLAT_LOAD_MONITOR_B128:
1161 return true;
1162 default:
1163 return false;
1164 }
1165 }
1166
1167 static bool isGFX12CacheInvOrWBInst(unsigned Opc) {
1168 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||
1169 Opc == AMDGPU::GLOBAL_WBINV;
1170 }
1171
1172 static bool isF16PseudoScalarTrans(unsigned Opcode) {
1173 return Opcode == AMDGPU::V_S_EXP_F16_e64 ||
1174 Opcode == AMDGPU::V_S_LOG_F16_e64 ||
1175 Opcode == AMDGPU::V_S_RCP_F16_e64 ||
1176 Opcode == AMDGPU::V_S_RSQ_F16_e64 ||
1177 Opcode == AMDGPU::V_S_SQRT_F16_e64;
1178 }
1179
1180 static bool doesNotReadTiedSource(const MachineInstr &MI) {
1181 return SIInstrFlags::isTiedSourceNotRead(O: MI);
1182 }
1183
1184 bool doesNotReadTiedSource(uint32_t Opcode) const {
1185 return SIInstrFlags::isTiedSourceNotRead(O: get(Opcode));
1186 }
1187
1188 bool isIGLP(unsigned Opcode) const {
1189 return Opcode == AMDGPU::SCHED_BARRIER ||
1190 Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
1191 }
1192
1193 bool isIGLP(const MachineInstr &MI) const { return isIGLP(Opcode: MI.getOpcode()); }
1194
1195 // Return true if the instruction is mutually exclusive with all non-IGLP DAG
1196 // mutations, requiring all other mutations to be disabled.
1197 bool isIGLPMutationOnly(unsigned Opcode) const {
1198 return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
1199 }
1200
1201 static unsigned getNonSoftWaitcntOpcode(unsigned Opcode) {
1202 switch (Opcode) {
1203 case AMDGPU::S_WAITCNT_soft:
1204 return AMDGPU::S_WAITCNT;
1205 case AMDGPU::S_WAITCNT_VSCNT_soft:
1206 return AMDGPU::S_WAITCNT_VSCNT;
1207 case AMDGPU::S_WAIT_LOADCNT_soft:
1208 return AMDGPU::S_WAIT_LOADCNT;
1209 case AMDGPU::S_WAIT_STORECNT_soft:
1210 return AMDGPU::S_WAIT_STORECNT;
1211 case AMDGPU::S_WAIT_SAMPLECNT_soft:
1212 return AMDGPU::S_WAIT_SAMPLECNT;
1213 case AMDGPU::S_WAIT_BVHCNT_soft:
1214 return AMDGPU::S_WAIT_BVHCNT;
1215 case AMDGPU::S_WAIT_DSCNT_soft:
1216 return AMDGPU::S_WAIT_DSCNT;
1217 case AMDGPU::S_WAIT_KMCNT_soft:
1218 return AMDGPU::S_WAIT_KMCNT;
1219 case AMDGPU::S_WAIT_XCNT_soft:
1220 return AMDGPU::S_WAIT_XCNT;
1221 default:
1222 return Opcode;
1223 }
1224 }
1225
1226 static bool isWaitcnt(unsigned Opcode) {
1227 switch (getNonSoftWaitcntOpcode(Opcode)) {
1228 case AMDGPU::S_WAITCNT:
1229 case AMDGPU::S_WAITCNT_VSCNT:
1230 case AMDGPU::S_WAITCNT_VMCNT:
1231 case AMDGPU::S_WAITCNT_EXPCNT:
1232 case AMDGPU::S_WAITCNT_LGKMCNT:
1233 case AMDGPU::S_WAIT_LOADCNT:
1234 case AMDGPU::S_WAIT_LOADCNT_DSCNT:
1235 case AMDGPU::S_WAIT_STORECNT:
1236 case AMDGPU::S_WAIT_STORECNT_DSCNT:
1237 case AMDGPU::S_WAIT_SAMPLECNT:
1238 case AMDGPU::S_WAIT_BVHCNT:
1239 case AMDGPU::S_WAIT_EXPCNT:
1240 case AMDGPU::S_WAIT_DSCNT:
1241 case AMDGPU::S_WAIT_KMCNT:
1242 case AMDGPU::S_WAIT_XCNT:
1243 case AMDGPU::S_WAIT_IDLE:
1244 return true;
1245 default:
1246 return false;
1247 }
1248 }
1249
1250 bool isVGPRCopy(const MachineInstr &MI) const {
1251 assert(isCopyInstr(MI));
1252 Register Dest = MI.getOperand(i: 0).getReg();
1253 const MachineFunction &MF = *MI.getMF();
1254 const MachineRegisterInfo &MRI = MF.getRegInfo();
1255 return !RI.isSGPRReg(MRI, Reg: Dest);
1256 }
1257
1258 bool hasVGPRUses(const MachineInstr &MI) const {
1259 const MachineFunction &MF = *MI.getMF();
1260 const MachineRegisterInfo &MRI = MF.getRegInfo();
1261 return llvm::any_of(Range: MI.explicit_uses(),
1262 P: [&MRI, this](const MachineOperand &MO) {
1263 return MO.isReg() && RI.isVGPR(MRI, Reg: MO.getReg());});
1264 }
1265
1266 /// Return true if the instruction modifies the mode register.q
1267 static bool modifiesModeRegister(const MachineInstr &MI);
1268
1269 /// This function is used to determine if an instruction can be safely
1270 /// executed under EXEC = 0 without hardware error, indeterminate results,
1271 /// and/or visible effects on future vector execution or outside the shader.
1272 /// Note: as of 2024 the only use of this is SIPreEmitPeephole where it is
1273 /// used in removing branches over short EXEC = 0 sequences.
1274 /// As such it embeds certain assumptions which may not apply to every case
1275 /// of EXEC = 0 execution.
1276 bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const;
1277
1278 /// Returns true if the instruction could potentially depend on the value of
1279 /// exec. If false, exec dependencies may safely be ignored.
1280 bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const;
1281
1282 bool isInlineConstant(const APInt &Imm) const;
1283
1284 bool isInlineConstant(const APFloat &Imm) const;
1285
1286 // Returns true if this non-register operand definitely does not need to be
1287 // encoded as a 32-bit literal. Note that this function handles all kinds of
1288 // operands, not just immediates.
1289 //
1290 // Some operands like FrameIndexes could resolve to an inline immediate value
1291 // that will not require an additional 4-bytes; this function assumes that it
1292 // will.
1293 bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const {
1294 if (!MO.isImm())
1295 return false;
1296 return isInlineConstant(ImmVal: MO.getImm(), OperandType);
1297 }
1298 bool isInlineConstant(int64_t ImmVal, uint8_t OperandType) const;
1299
1300 bool isInlineConstant(const MachineOperand &MO,
1301 const MCOperandInfo &OpInfo) const {
1302 return isInlineConstant(MO, OperandType: OpInfo.OperandType);
1303 }
1304
1305 /// \p returns true if \p UseMO is substituted with \p DefMO in \p MI it would
1306 /// be an inline immediate.
1307 bool isInlineConstant(const MachineInstr &MI,
1308 const MachineOperand &UseMO,
1309 const MachineOperand &DefMO) const {
1310 assert(UseMO.getParent() == &MI);
1311 int OpIdx = UseMO.getOperandNo();
1312 if (OpIdx >= MI.getDesc().NumOperands)
1313 return false;
1314
1315 return isInlineConstant(MO: DefMO, OpInfo: MI.getDesc().operands()[OpIdx]);
1316 }
1317
1318 /// \p returns true if the operand \p OpIdx in \p MI is a valid inline
1319 /// immediate.
1320 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx) const {
1321 const MachineOperand &MO = MI.getOperand(i: OpIdx);
1322 return isInlineConstant(MO, OperandType: MI.getDesc().operands()[OpIdx].OperandType);
1323 }
1324
1325 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
1326 int64_t ImmVal) const {
1327 if (OpIdx >= MI.getDesc().NumOperands)
1328 return false;
1329
1330 if (isCopyInstr(MI)) {
1331 unsigned Size = getOpSize(MI, OpNo: OpIdx);
1332 assert(Size == 8 || Size == 4);
1333
1334 uint8_t OpType = (Size == 8) ?
1335 AMDGPU::OPERAND_REG_IMM_INT64 : AMDGPU::OPERAND_REG_IMM_INT32;
1336 return isInlineConstant(ImmVal, OperandType: OpType);
1337 }
1338
1339 return isInlineConstant(ImmVal, OperandType: MI.getDesc().operands()[OpIdx].OperandType);
1340 }
1341
1342 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
1343 const MachineOperand &MO) const {
1344 return isInlineConstant(MI, OpIdx, ImmVal: MO.getImm());
1345 }
1346
1347 bool isInlineConstant(const MachineOperand &MO) const {
1348 return isInlineConstant(MI: *MO.getParent(), OpIdx: MO.getOperandNo());
1349 }
1350
1351 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
1352 const MachineOperand &MO) const;
1353
1354 bool isLiteralOperandLegal(const MCInstrDesc &InstDesc,
1355 const MCOperandInfo &OpInfo) const;
1356
1357 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
1358 int64_t ImmVal) const;
1359
1360 bool isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
1361 const MachineOperand &MO) const {
1362 return isImmOperandLegal(InstDesc: MI.getDesc(), OpNo, MO);
1363 }
1364
1365 bool isNeverCoissue(MachineInstr &MI) const;
1366
1367 /// Check if this immediate value can be used for AV_MOV_B64_IMM_PSEUDO.
1368 bool isLegalAV64PseudoImm(uint64_t Imm) const;
1369
1370 /// Return true if this 64-bit VALU instruction has a 32-bit encoding.
1371 /// This function will return false if you pass it a 32-bit instruction.
1372 bool hasVALU32BitEncoding(unsigned Opcode) const;
1373
1374 bool physRegUsesConstantBus(const MachineOperand &Reg) const;
1375 bool regUsesConstantBus(const MachineOperand &Reg,
1376 const MachineRegisterInfo &MRI) const;
1377
1378 /// Returns true if this operand uses the constant bus.
1379 bool usesConstantBus(const MachineRegisterInfo &MRI,
1380 const MachineOperand &MO,
1381 const MCOperandInfo &OpInfo) const;
1382
1383 bool usesConstantBus(const MachineRegisterInfo &MRI, const MachineInstr &MI,
1384 int OpIdx) const {
1385 return usesConstantBus(MRI, MO: MI.getOperand(i: OpIdx),
1386 OpInfo: MI.getDesc().operands()[OpIdx]);
1387 }
1388
1389 /// Return true if this instruction has any modifiers.
1390 /// e.g. src[012]_mod, omod, clamp.
1391 bool hasModifiers(unsigned Opcode) const;
1392
1393 bool hasModifiersSet(const MachineInstr &MI, AMDGPU::OpName OpName) const;
1394 bool hasAnyModifiersSet(const MachineInstr &MI) const;
1395
1396 bool canShrink(const MachineInstr &MI,
1397 const MachineRegisterInfo &MRI) const;
1398
1399 MachineInstr *buildShrunkInst(MachineInstr &MI,
1400 unsigned NewOpcode) const;
1401
1402 bool verifyInstruction(const MachineInstr &MI,
1403 StringRef &ErrInfo) const override;
1404
1405 unsigned getVALUOp(const MachineInstr &MI) const;
1406 unsigned getVALUOp(unsigned Opc) const;
1407
1408 void insertScratchExecCopy(MachineFunction &MF, MachineBasicBlock &MBB,
1409 MachineBasicBlock::iterator MBBI,
1410 const DebugLoc &DL, Register Reg, bool IsSCCLive,
1411 SlotIndexes *Indexes = nullptr) const;
1412
1413 void restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
1414 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
1415 Register Reg, SlotIndexes *Indexes = nullptr) const;
1416
1417 MachineInstr *getWholeWaveFunctionSetup(MachineFunction &MF) const;
1418
1419 /// Return the correct register class for \p OpNo. For target-specific
1420 /// instructions, this will return the register class that has been defined
1421 /// in tablegen. For generic instructions, like REG_SEQUENCE it will return
1422 /// the register class of its machine operand.
1423 /// to infer the correct register class base on the other operands.
1424 const TargetRegisterClass *getOpRegClass(const MachineInstr &MI,
1425 unsigned OpNo) const;
1426
1427 /// Return the size in bytes of the operand OpNo on the given
1428 // instruction opcode.
1429 unsigned getOpSize(uint32_t Opcode, unsigned OpNo) const {
1430 const MCOperandInfo &OpInfo = get(Opcode).operands()[OpNo];
1431
1432 if (OpInfo.RegClass == -1) {
1433 // If this is an immediate operand, this must be a 32-bit literal.
1434 assert(OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE);
1435 return 4;
1436 }
1437
1438 return RI.getRegSizeInBits(RC: *RI.getRegClass(i: getOpRegClassID(OpInfo))) / 8;
1439 }
1440
1441 /// This form should usually be preferred since it handles operands
1442 /// with unknown register classes.
1443 unsigned getOpSize(const MachineInstr &MI, unsigned OpNo) const {
1444 const MachineOperand &MO = MI.getOperand(i: OpNo);
1445 if (MO.isReg()) {
1446 if (unsigned SubReg = MO.getSubReg()) {
1447 return RI.getSubRegIdxSize(Idx: SubReg) / 8;
1448 }
1449 }
1450 return RI.getRegSizeInBits(RC: *getOpRegClass(MI, OpNo)) / 8;
1451 }
1452
1453 /// Legalize the \p OpIndex operand of this instruction by inserting
1454 /// a MOV. For example:
1455 /// ADD_I32_e32 VGPR0, 15
1456 /// to
1457 /// MOV VGPR1, 15
1458 /// ADD_I32_e32 VGPR0, VGPR1
1459 ///
1460 /// If the operand being legalized is a register, then a COPY will be used
1461 /// instead of MOV.
1462 void legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const;
1463
1464 /// Check if \p MO is a legal operand if it was the \p OpIdx Operand
1465 /// for \p MI.
1466 bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
1467 const MachineOperand *MO = nullptr) const;
1468
1469 /// Check if \p MO would be a valid operand for the given operand
1470 /// definition \p OpInfo. Note this does not attempt to validate constant bus
1471 /// restrictions (e.g. literal constant usage).
1472 bool isLegalVSrcOperand(const MachineRegisterInfo &MRI,
1473 const MCOperandInfo &OpInfo,
1474 const MachineOperand &MO) const;
1475
1476 /// Check if \p MO (a register operand) is a legal register for the
1477 /// given operand description or operand index.
1478 /// The operand index version provide more legality checks
1479 bool isLegalRegOperand(const MachineRegisterInfo &MRI,
1480 const MCOperandInfo &OpInfo,
1481 const MachineOperand &MO) const;
1482 bool isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
1483 const MachineOperand &MO) const;
1484
1485 /// Check if \p MO would be a legal operand for gfx12+ packed math FP32 or
1486 /// 64 instructions. Packed math FP32/FP64/U64 instructions typically accept
1487 /// SGPRs or VGPRs as source operands. On gfx12+, if a source operand uses
1488 /// SGPRs, the HW can only read the first SGPR and use it for both the low and
1489 /// high operations.
1490 /// \p SrcN can be 0, 1, or 2, representing src0, src1, and src2,
1491 /// respectively. If \p MO is nullptr, the operand corresponding to SrcN will
1492 /// be used.
1493 bool isLegalGFX12PlusPackedMathFP32or64BitOperand(
1494 const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,
1495 const MachineOperand *MO = nullptr) const;
1496
1497 /// Legalize operands in \p MI by either commuting it or inserting a
1498 /// copy of src1.
1499 void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1500
1501 /// Fix operands in \p MI to satisfy constant bus requirements.
1502 void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1503
1504 /// Copy a value from a VGPR (\p SrcReg) to SGPR. The desired register class
1505 /// for the dst register (\p DstRC) can be optionally supplied. This function
1506 /// can only be used when it is know that the value in SrcReg is same across
1507 /// all threads in the wave.
1508 /// \returns The SGPR register that \p SrcReg was copied to.
1509 Register readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,
1510 MachineRegisterInfo &MRI,
1511 const TargetRegisterClass *DstRC = nullptr) const;
1512
1513 void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1514 void legalizeOperandsFLAT(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1515
1516 void legalizeGenericOperand(MachineBasicBlock &InsertMBB,
1517 MachineBasicBlock::iterator I,
1518 const TargetRegisterClass *DstRC,
1519 MachineOperand &Op, MachineRegisterInfo &MRI,
1520 const DebugLoc &DL) const;
1521
1522 /// Legalize all operands in this instruction. This function may create new
1523 /// instructions and control-flow around \p MI. If present, \p MDT is
1524 /// updated.
1525 /// \returns A new basic block that contains \p MI if new blocks were created.
1526 MachineBasicBlock *
1527 legalizeOperands(MachineInstr &MI, MachineDominatorTree *MDT = nullptr) const;
1528
1529 /// Change SADDR form of a FLAT \p Inst to its VADDR form if saddr operand
1530 /// was moved to VGPR. \returns true if succeeded.
1531 bool moveFlatAddrToVGPR(MachineInstr &Inst) const;
1532
1533 /// Fix operands in Inst to fix 16bit SALU to VALU lowering.
1534 void legalizeOperandsVALUt16(MachineInstr &Inst,
1535 MachineRegisterInfo &MRI) const;
1536 void legalizeOperandsVALUt16(MachineInstr &Inst, unsigned OpIdx,
1537 MachineRegisterInfo &MRI) const;
1538
1539 /// Replace the instructions opcode with the equivalent VALU
1540 /// opcode. This function will also move the users of MachineInstruntions
1541 /// in the \p WorkList to the VALU if necessary. If present, \p MDT is
1542 /// updated.
1543 void moveToVALU(SIInstrWorklist &Worklist, MachineDominatorTree *MDT) const;
1544
1545 void
1546 moveToVALUImpl(SIInstrWorklist &Worklist, MachineDominatorTree *MDT,
1547 MachineInstr &Inst,
1548 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
1549 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const;
1550 /// Wrapper function for generating waterfall for instruction \p MI
1551 /// This function take into consideration of related pre & succ instructions
1552 /// (e.g. calling process) into consideratioin
1553 void createWaterFallForSiCall(MachineInstr *MI, MachineDominatorTree *MDT,
1554 ArrayRef<MachineOperand *> ScalarOps,
1555 ArrayRef<Register> PhySGPRs = {}) const;
1556
1557 void insertNoop(MachineBasicBlock &MBB,
1558 MachineBasicBlock::iterator MI) const override;
1559
1560 void insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1561 unsigned Quantity) const override;
1562
1563 /// Build instructions that simulate the behavior of a `s_trap 2` instructions
1564 /// for hardware (namely, gfx11) that runs in PRIV=1 mode. There, s_trap is
1565 /// interpreted as a nop.
1566 MachineBasicBlock *insertSimulatedTrap(MachineRegisterInfo &MRI,
1567 MachineBasicBlock &MBB,
1568 MachineInstr &MI,
1569 const DebugLoc &DL) const;
1570
1571 /// Return the number of wait states that result from executing this
1572 /// instruction.
1573 static unsigned getNumWaitStates(const MachineInstr &MI);
1574
1575 /// Returns the operand named \p Op. If \p MI does not have an
1576 /// operand named \c Op, this function returns nullptr.
1577 LLVM_READONLY
1578 MachineOperand *getNamedOperand(MachineInstr &MI,
1579 AMDGPU::OpName OperandName) const;
1580
1581 LLVM_READONLY
1582 const MachineOperand *getNamedOperand(const MachineInstr &MI,
1583 AMDGPU::OpName OperandName) const {
1584 return getNamedOperand(MI&: const_cast<MachineInstr &>(MI), OperandName);
1585 }
1586
1587 /// Get required immediate operand
1588 int64_t getNamedImmOperand(const MachineInstr &MI,
1589 AMDGPU::OpName OperandName) const {
1590 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OperandName);
1591 return MI.getOperand(i: Idx).getImm();
1592 }
1593
1594 uint64_t getDefaultRsrcDataFormat() const;
1595 uint64_t getScratchRsrcWords23() const;
1596
1597 bool isLowLatencyInstruction(const MachineInstr &MI) const;
1598 bool isHighLatencyDef(int Opc) const override;
1599
1600 /// Return the descriptor of the target-specific machine instruction
1601 /// that corresponds to the specified pseudo or native opcode.
1602 const MCInstrDesc &getMCOpcodeFromPseudo(unsigned Opcode) const {
1603 return get(Opcode: pseudoToMCOpcode(Opcode));
1604 }
1605
1606 Register isStackAccess(const MachineInstr &MI, int &FrameIndex,
1607 TypeSize &MemBytes) const;
1608 Register isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex,
1609 TypeSize &MemBytes) const;
1610
1611 Register isLoadFromStackSlot(const MachineInstr &MI,
1612 int &FrameIndex) const override {
1613 TypeSize MemBytes = TypeSize::getZero();
1614 return isLoadFromStackSlot(MI, FrameIndex, MemBytes);
1615 }
1616
1617 Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex,
1618 TypeSize &MemBytes) const override;
1619
1620 Register isStoreToStackSlot(const MachineInstr &MI,
1621 int &FrameIndex) const override {
1622 TypeSize MemBytes = TypeSize::getZero();
1623 return isStoreToStackSlot(MI, FrameIndex, MemBytes);
1624 }
1625
1626 Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex,
1627 TypeSize &MemBytes) const override;
1628
1629 unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
1630
1631 InstSizeVerifyMode
1632 getInstSizeVerifyMode(const MachineInstr &MI) const override;
1633
1634 bool mayAccessFlatAddressSpace(const MachineInstr &MI) const;
1635
1636 std::pair<unsigned, unsigned>
1637 decomposeMachineOperandsTargetFlags(unsigned TF) const override;
1638
1639 ArrayRef<std::pair<int, const char *>>
1640 getSerializableTargetIndices() const override;
1641
1642 ArrayRef<std::pair<unsigned, const char *>>
1643 getSerializableDirectMachineOperandTargetFlags() const override;
1644
1645 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
1646 getSerializableMachineMemOperandTargetFlags() const override;
1647
1648 ScheduleHazardRecognizer *
1649 CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
1650 const ScheduleDAG *DAG) const override;
1651
1652 ScheduleHazardRecognizer *
1653 CreateTargetPostRAHazardRecognizer(const MachineFunction &MF,
1654 MachineLoopInfo *MLI) const override;
1655
1656 ScheduleHazardRecognizer *
1657 CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
1658 const ScheduleDAGMI *DAG) const override;
1659
1660 unsigned getLiveRangeSplitOpcode(Register Reg,
1661 const MachineFunction &MF) const override;
1662
1663 bool isBasicBlockPrologue(const MachineInstr &MI,
1664 Register Reg = Register()) const override;
1665
1666 bool canAddToBBProlog(const MachineInstr &MI) const;
1667
1668 MachineInstr *createPHIDestinationCopy(MachineBasicBlock &MBB,
1669 MachineBasicBlock::iterator InsPt,
1670 const DebugLoc &DL, Register Src,
1671 Register Dst) const override;
1672
1673 MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB,
1674 MachineBasicBlock::iterator InsPt,
1675 const DebugLoc &DL, Register Src,
1676 unsigned SrcSubReg,
1677 Register Dst) const override;
1678
1679 bool isWave32() const;
1680
1681 bool isVOPDAntidependencyAllowed(const MachineInstr &MI) const;
1682
1683 bool hasRAWDependency(const MachineInstr &FirstMI,
1684 const MachineInstr &SecondMI) const;
1685
1686 /// Return a partially built integer add instruction without carry.
1687 /// Caller must add source operands.
1688 /// For pre-GFX9 it will generate unused carry destination operand.
1689 /// TODO: After GFX9 it should return a no-carry operation.
1690 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1691 MachineBasicBlock::iterator I,
1692 const DebugLoc &DL,
1693 Register DestReg) const;
1694
1695 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1696 MachineBasicBlock::iterator I,
1697 const DebugLoc &DL,
1698 Register DestReg,
1699 RegScavenger &RS) const;
1700
1701 static bool isKillTerminator(unsigned Opcode);
1702 const MCInstrDesc &getKillTerminatorFromPseudo(unsigned Opcode) const;
1703
1704 bool isLegalMUBUFImmOffset(unsigned Imm) const;
1705
1706 static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST);
1707
1708 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
1709 Align Alignment = Align(4)) const;
1710
1711 /// Returns if \p Offset is legal for the subtarget as the offset to a FLAT
1712 /// encoded instruction with the given \p FlatVariant.
1713 bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
1714 AMDGPU::FlatAddrSpace FlatVariant) const;
1715
1716 /// Split \p COffsetVal into {immediate offset field, remainder offset}
1717 /// values.
1718 std::pair<int64_t, int64_t>
1719 splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,
1720 AMDGPU::FlatAddrSpace FlatVariant) const;
1721
1722 /// Returns true if negative offsets are allowed for the given \p FlatVariant.
1723 bool allowNegativeFlatOffset(AMDGPU::FlatAddrSpace FlatVariant) const;
1724
1725 /// \brief Return a target-specific opcode if Opcode is a pseudo instruction.
1726 /// Return -1 if the target-specific opcode for the pseudo instruction does
1727 /// not exist. If Opcode is not a pseudo instruction, this is identity.
1728 int pseudoToMCOpcode(int Opcode) const;
1729
1730 /// \brief Check if this instruction should only be used by assembler.
1731 /// Return true if this opcode should not be used by codegen.
1732 bool isAsmOnlyOpcode(int MCOp) const;
1733
1734 void fixImplicitOperands(MachineInstr &MI) const;
1735
1736 MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
1737 ArrayRef<unsigned> Ops, int FrameIndex,
1738 MachineInstr *&CopyMI,
1739 LiveIntervals *LIS = nullptr,
1740 VirtRegMap *VRM = nullptr) const override;
1741
1742 unsigned getInstrLatency(const InstrItineraryData *ItinData,
1743 const MachineInstr &MI,
1744 unsigned *PredCost = nullptr) const override;
1745
1746 const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override;
1747
1748 ValueUniformity getValueUniformity(const MachineInstr &MI) const final;
1749
1750 ValueUniformity getGenericValueUniformity(const MachineInstr &MI) const;
1751
1752 const MIRFormatter *getMIRFormatter() const override;
1753
1754 static unsigned getDSShaderTypeValue(const MachineFunction &MF);
1755
1756 const TargetSchedModel &getSchedModel() const { return SchedModel; }
1757
1758 void createReadFirstLaneFromCopyToPhysReg(MachineRegisterInfo &MRI,
1759 Register DstReg,
1760 MachineInstr &Inst) const;
1761
1762 void handleCopyToPhysHelper(
1763 SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst,
1764 MachineRegisterInfo &MRI,
1765 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
1766 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const;
1767
1768 // FIXME: This should be removed
1769 // Enforce operand's \p OpName even alignment if required by target.
1770 // This is used if an operand is a 32 bit register but needs to be aligned
1771 // regardless.
1772 void enforceOperandRCAlignment(MachineInstr &MI, AMDGPU::OpName OpName) const;
1773};
1774
1775/// \brief Returns true if a reg:subreg pair P has a TRC class
1776inline bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P,
1777 const TargetRegisterClass &TRC,
1778 MachineRegisterInfo &MRI) {
1779 auto *RC = MRI.getRegClass(Reg: P.Reg);
1780 if (!P.SubReg)
1781 return RC == &TRC;
1782 auto *TRI = MRI.getTargetRegisterInfo();
1783 return RC == TRI->getMatchingSuperRegClass(A: RC, B: &TRC, Idx: P.SubReg);
1784}
1785
1786/// \brief Create RegSubRegPair from a register MachineOperand
1787inline
1788TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O) {
1789 assert(O.isReg());
1790 return TargetInstrInfo::RegSubRegPair(O.getReg(), O.getSubReg());
1791}
1792
1793/// \brief Return the SubReg component from REG_SEQUENCE
1794TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI,
1795 unsigned SubReg);
1796
1797/// \brief Return the defining instruction for a given reg:subreg pair
1798/// skipping copy like instructions and subreg-manipulation pseudos.
1799/// Following another subreg of a reg:subreg isn't supported.
1800MachineInstr *getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
1801 const MachineRegisterInfo &MRI);
1802
1803/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1804/// DefMI and the use at \p UseMI. Should be run on SSA. Currently does not
1805/// attempt to track between blocks.
1806bool execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
1807 Register VReg,
1808 const MachineInstr &DefMI,
1809 const MachineInstr &UseMI);
1810
1811/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1812/// DefMI and all its uses. Should be run on SSA. Currently does not attempt to
1813/// track between blocks.
1814bool execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
1815 Register VReg,
1816 const MachineInstr &DefMI);
1817
1818namespace AMDGPU {
1819
1820 LLVM_READONLY
1821 int32_t getVOPe64(uint32_t Opcode);
1822
1823 LLVM_READONLY
1824 int32_t getVOPe32(uint32_t Opcode);
1825
1826 LLVM_READONLY
1827 int32_t getSDWAOp(uint32_t Opcode);
1828
1829 LLVM_READONLY
1830 int32_t getDPPOp32(uint32_t Opcode);
1831
1832 LLVM_READONLY
1833 int32_t getDPPOp64(uint32_t Opcode);
1834
1835 LLVM_READONLY
1836 int32_t getBasicFromSDWAOp(uint32_t Opcode);
1837
1838 LLVM_READONLY
1839 int32_t getCommuteRev(uint32_t Opcode);
1840
1841 LLVM_READONLY
1842 int32_t getCommuteOrig(uint32_t Opcode);
1843
1844 LLVM_READONLY
1845 int32_t getAddr64Inst(uint32_t Opcode);
1846
1847 /// Check if \p Opcode is an Addr64 opcode.
1848 ///
1849 /// \returns \p Opcode if it is an Addr64 opcode, otherwise -1.
1850 LLVM_READONLY
1851 int32_t getIfAddr64Inst(uint32_t Opcode);
1852
1853 LLVM_READONLY
1854 int32_t getSOPKOp(uint32_t Opcode);
1855
1856 /// \returns SADDR form of a FLAT Global instruction given an \p Opcode
1857 /// of a VADDR form.
1858 LLVM_READONLY
1859 int32_t getGlobalSaddrOp(uint32_t Opcode);
1860
1861 /// \returns VADDR form of a FLAT Global instruction given an \p Opcode
1862 /// of a SADDR form.
1863 LLVM_READONLY
1864 int32_t getGlobalVaddrOp(uint32_t Opcode);
1865
1866 LLVM_READONLY
1867 int32_t getVCMPXNoSDstOp(uint32_t Opcode);
1868
1869 /// \returns ST form with only immediate offset of a FLAT Scratch instruction
1870 /// given an \p Opcode of an SS (SADDR) form.
1871 LLVM_READONLY
1872 int32_t getFlatScratchInstSTfromSS(uint32_t Opcode);
1873
1874 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
1875 /// of an SVS (SADDR + VADDR) form.
1876 LLVM_READONLY
1877 int32_t getFlatScratchInstSVfromSVS(uint32_t Opcode);
1878
1879 /// \returns SS (SADDR) form of a FLAT Scratch instruction given an \p Opcode
1880 /// of an SV (VADDR) form.
1881 LLVM_READONLY
1882 int32_t getFlatScratchInstSSfromSV(uint32_t Opcode);
1883
1884 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
1885 /// of an SS (SADDR) form.
1886 LLVM_READONLY
1887 int32_t getFlatScratchInstSVfromSS(uint32_t Opcode);
1888
1889 /// \returns earlyclobber version of a MAC MFMA is exists.
1890 LLVM_READONLY
1891 int32_t getMFMAEarlyClobberOp(uint32_t Opcode);
1892
1893 /// \returns Version of an MFMA instruction which uses AGPRs for srcC and
1894 /// vdst, given an \p Opcode of an MFMA which uses VGPRs for srcC/vdst.
1895 LLVM_READONLY
1896 int32_t getMFMASrcCVDstAGPROp(uint32_t Opcode);
1897
1898 /// \returns v_cmpx version of a v_cmp instruction.
1899 LLVM_READONLY
1900 int32_t getVCMPXOpFromVCMP(uint32_t Opcode);
1901
1902 const uint64_t RSRC_DATA_FORMAT = 0xf00000000000LL;
1903 const uint64_t RSRC_ELEMENT_SIZE_SHIFT = (32 + 19);
1904 const uint64_t RSRC_INDEX_STRIDE_SHIFT = (32 + 21);
1905 const uint64_t RSRC_TID_ENABLE = UINT64_C(1) << (32 + 23);
1906
1907} // end namespace AMDGPU
1908
1909namespace AMDGPU {
1910enum AsmComments : MachineInstr::AsmPrinterFlagTy {
1911 // For sgpr to vgpr spill instructions
1912 SGPR_SPILL = MachineInstr::TAsmComments
1913};
1914} // namespace AMDGPU
1915
1916namespace SI {
1917namespace KernelInputOffsets {
1918
1919/// Offsets in bytes from the start of the input buffer
1920enum Offsets {
1921 NGROUPS_X = 0,
1922 NGROUPS_Y = 4,
1923 NGROUPS_Z = 8,
1924 GLOBAL_SIZE_X = 12,
1925 GLOBAL_SIZE_Y = 16,
1926 GLOBAL_SIZE_Z = 20,
1927 LOCAL_SIZE_X = 24,
1928 LOCAL_SIZE_Y = 28,
1929 LOCAL_SIZE_Z = 32
1930};
1931
1932} // end namespace KernelInputOffsets
1933} // end namespace SI
1934
1935} // end namespace llvm
1936
1937#endif // LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
1938