1//===- SIInstrInfo.h - SI Instruction Info Interface ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Interface definition for SIInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
15#define LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
16
17#include "AMDGPUMIRFormatter.h"
18#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
19#include "SIRegisterInfo.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/CodeGen/TargetInstrInfo.h"
24#include "llvm/CodeGen/TargetSchedule.h"
25
26#define GET_INSTRINFO_HEADER
27#include "AMDGPUGenInstrInfo.inc"
28
29namespace llvm {
30
31class APInt;
32class GCNSubtarget;
33class MachineDominatorTree;
34class MachineRegisterInfo;
35class RegScavenger;
36class SIMachineFunctionInfo;
37class MCRegisterClass;
38using TargetRegisterClass = MCRegisterClass;
39class ScheduleHazardRecognizer;
40
41namespace AMDGPU {
42enum class AsyncStages : uint32_t;
43} // namespace AMDGPU
44
45constexpr unsigned DefaultMemoryClusterDWordsLimit = 8;
46
47/// Mark the MMO of a uniform load if there are no potentially clobbering stores
48/// on any path from the start of an entry function to this load.
49static const MachineMemOperand::Flags MONoClobber =
50 MachineMemOperand::MOTargetFlag1;
51
52/// Mark the MMO of a load as the last use.
53static const MachineMemOperand::Flags MOLastUse =
54 MachineMemOperand::MOTargetFlag2;
55
56/// Mark the MMO of cooperative load/store atomics.
57static const MachineMemOperand::Flags MOCooperative =
58 MachineMemOperand::MOTargetFlag3;
59
60struct V2PhysSCopyInfo {
61 // Operands that need to replaced by waterfall
62 SmallVector<MachineOperand *> MOs;
63 // Target physical registers replacing the MOs
64 SmallVector<Register> SGPRs;
65};
66/// Mark the MMO of accesses to memory locations that are
67/// never written to by other threads.
68static const MachineMemOperand::Flags MOThreadPrivate =
69 MachineMemOperand::MOTargetFlag4;
70
71/// Use two MMO flags to encode the 2-bit cache fill size field for GFX13+
72static const MachineMemOperand::Flags MOCFSB0 =
73 MachineMemOperand::MOTargetFlag5;
74
75static const MachineMemOperand::Flags MOCFSB1 =
76 MachineMemOperand::MOTargetFlag6;
77
78/// Utility to store machine instructions worklist.
79struct SIInstrWorklist {
80 SIInstrWorklist() = default;
81
82 void insert(MachineInstr *MI);
83
84 MachineInstr *top() const { return InstrList[Front]; }
85
86 void erase_top() {
87 InSet.erase(Ptr: InstrList[Front]);
88 ++Front;
89 }
90
91 bool empty() const { return Front == InstrList.size(); }
92
93 void clear() {
94 InstrList.clear();
95 Front = 0;
96 InSet.clear();
97 DeferredList.clear();
98 }
99
100 bool isDeferred(MachineInstr *MI);
101
102 SetVector<MachineInstr *> &getDeferredList() { return DeferredList; }
103
104private:
105 /// InstrList contains the MachineInstrs.
106 SmallVector<MachineInstr *> InstrList;
107 SmallPtrSet<MachineInstr *, 8> InSet;
108 unsigned Front = 0;
109 /// Deferred instructions are specific MachineInstr
110 /// that will be added by insert method.
111 SetVector<MachineInstr *> DeferredList;
112};
113
114// In namespace llvm so ADL finds it when SIInstrFlags predicates are
115// instantiated with MachineInstr (MachineInstr is in namespace llvm).
116inline uint64_t getTSFlags(const MachineInstr &MI) {
117 return MI.getDesc().TSFlags;
118}
119
120class SIInstrInfo final : public AMDGPUGenInstrInfo {
121 struct ThreeAddressUpdates;
122
123private:
124 const SIRegisterInfo RI;
125 const GCNSubtarget &ST;
126 TargetSchedModel SchedModel;
127 mutable std::unique_ptr<AMDGPUMIRFormatter> Formatter;
128
129 // The inverse predicate should have the negative value.
130 enum BranchPredicate {
131 INVALID_BR = 0,
132 SCC_TRUE = 1,
133 SCC_FALSE = -1,
134 VCCNZ = 2,
135 VCCZ = -2,
136 EXECNZ = -3,
137 EXECZ = 3
138 };
139
140 using SetVectorType = SmallSetVector<MachineInstr *, 32>;
141
142 static unsigned getBranchOpcode(BranchPredicate Cond);
143 static BranchPredicate getBranchPredicate(unsigned Opcode);
144
145public:
146 unsigned buildExtractSubReg(MachineBasicBlock::iterator MI,
147 MachineRegisterInfo &MRI,
148 const MachineOperand &SuperReg,
149 const TargetRegisterClass *SuperRC,
150 unsigned SubIdx,
151 const TargetRegisterClass *SubRC) const;
152 MachineOperand buildExtractSubRegOrImm(
153 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
154 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
155 unsigned SubIdx, const TargetRegisterClass *SubRC) const;
156
157private:
158 bool optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,
159 bool NeedInversion) const;
160
161 bool invertSCCUse(MachineInstr *SCCDef) const;
162
163 void swapOperands(MachineInstr &Inst) const;
164
165 std::pair<bool, MachineBasicBlock *>
166 moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
167 MachineDominatorTree *MDT = nullptr) const;
168
169 void lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
170 MachineDominatorTree *MDT = nullptr) const;
171
172 void lowerScalarAbs(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
173
174 void lowerScalarAbsDiff(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
175
176 void lowerScalarXnor(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
177
178 void splitScalarNotBinop(SIInstrWorklist &Worklist, MachineInstr &Inst,
179 unsigned Opcode) const;
180
181 void splitScalarBinOpN2(SIInstrWorklist &Worklist, MachineInstr &Inst,
182 unsigned Opcode) const;
183
184 void splitScalar64BitUnaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
185 unsigned Opcode, bool Swap = false) const;
186
187 void splitScalar64BitBinaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
188 unsigned Opcode,
189 MachineDominatorTree *MDT = nullptr) const;
190
191 void splitScalarSMulU64(SIInstrWorklist &Worklist, MachineInstr &Inst,
192 MachineDominatorTree *MDT) const;
193
194 void splitScalarSMulPseudo(SIInstrWorklist &Worklist, MachineInstr &Inst,
195 MachineDominatorTree *MDT) const;
196
197 void splitScalar64BitXnor(SIInstrWorklist &Worklist, MachineInstr &Inst,
198 MachineDominatorTree *MDT = nullptr) const;
199
200 void splitScalar64BitBCNT(SIInstrWorklist &Worklist,
201 MachineInstr &Inst) const;
202 void splitScalar64BitBFE(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
203 void splitScalar64BitCountOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
204 unsigned Opcode,
205 MachineDominatorTree *MDT = nullptr) const;
206 void movePackToVALU(SIInstrWorklist &Worklist, MachineRegisterInfo &MRI,
207 MachineInstr &Inst) const;
208
209 void addUsersToMoveToVALUWorklist(Register Reg, MachineRegisterInfo &MRI,
210 SIInstrWorklist &Worklist) const;
211
212 void addSCCDefUsersToVALUWorklist(const MachineOperand &Op,
213 MachineInstr &SCCDefInst,
214 SIInstrWorklist &Worklist,
215 Register NewCond = Register()) const;
216 void addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
217 SIInstrWorklist &Worklist) const;
218
219 const TargetRegisterClass *
220 getDestEquivalentVGPRClass(const MachineInstr &Inst) const;
221
222 bool checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
223 const MachineInstr &MIb) const;
224
225 Register findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const;
226
227 bool verifyCopy(const MachineInstr &MI, const MachineRegisterInfo &MRI,
228 StringRef &ErrInfo) const;
229
230 bool resultDependsOnExec(const MachineInstr &MI) const;
231
232 MachineInstr *convertToThreeAddressImpl(MachineInstr &MI,
233 ThreeAddressUpdates &Updates) const;
234
235protected:
236 /// If the specific machine instruction is a instruction that moves/copies
237 /// value from one register to another register return destination and source
238 /// registers as machine operands.
239 std::optional<DestSourcePair>
240 isCopyInstrImpl(const MachineInstr &MI) const override;
241
242 bool swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,
243 AMDGPU::OpName Src0OpName, MachineOperand &Src1,
244 AMDGPU::OpName Src1OpName) const;
245 bool isNonCommutableDPP(const MachineInstr &MI) const;
246 MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
247 unsigned OpIdx0,
248 unsigned OpIdx1) const override;
249
250public:
251 bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx,
252 unsigned toIdx) const;
253 enum TargetOperandFlags {
254 MO_MASK = 0xf,
255
256 MO_NONE = 0,
257 // MO_GOTPCREL -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
258 MO_GOTPCREL = 1,
259 // MO_GOTPCREL32_LO -> symbol@gotpcrel32@lo -> R_AMDGPU_GOTPCREL32_LO.
260 MO_GOTPCREL32 = 2,
261 MO_GOTPCREL32_LO = 2,
262 // MO_GOTPCREL32_HI -> symbol@gotpcrel32@hi -> R_AMDGPU_GOTPCREL32_HI.
263 MO_GOTPCREL32_HI = 3,
264 // MO_GOTPCREL64 -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
265 MO_GOTPCREL64 = 4,
266 // MO_REL32_LO -> symbol@rel32@lo -> R_AMDGPU_REL32_LO.
267 MO_REL32 = 5,
268 MO_REL32_LO = 5,
269 // MO_REL32_HI -> symbol@rel32@hi -> R_AMDGPU_REL32_HI.
270 MO_REL32_HI = 6,
271 MO_REL64 = 7,
272
273 MO_FAR_BRANCH_OFFSET = 8,
274
275 MO_ABS32_LO = 9,
276 MO_ABS32_HI = 10,
277 MO_ABS64 = 11,
278 };
279
280 explicit SIInstrInfo(const GCNSubtarget &ST);
281
282 const SIRegisterInfo &getRegisterInfo() const {
283 return RI;
284 }
285
286 // FIXME: This is inaccurate and needs to account for use context. Normal asm
287 // constraints should use 64-bit pointers.
288 const TargetRegisterClass *getInlineAsmMemoryOperandRegClass(
289 InlineAsm::ConstraintCode C) const override {
290 switch (C) {
291 case InlineAsm::ConstraintCode::RF:
292 case InlineAsm::ConstraintCode::m:
293 // flat_load/flat_store require a 64-bit VGPR pair for the address.
294 return &AMDGPU::VReg_64RegClass;
295 default:
296 return &AMDGPU::VGPR_32RegClass;
297 }
298 }
299
300 const GCNSubtarget &getSubtarget() const {
301 return ST;
302 }
303
304 bool isReMaterializableImpl(const MachineInstr &MI) const override;
305
306 bool isIgnorableUse(const MachineInstr &MI, unsigned OpIdx) const override;
307
308 bool isSafeToSink(MachineInstr &MI, MachineBasicBlock *SuccToSinkTo,
309 MachineCycleInfo *CI) const override;
310
311 bool areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, int64_t &Offset0,
312 int64_t &Offset1) const override;
313
314 bool isGlobalMemoryObject(const MachineInstr *MI) const override;
315
316 bool getMemOperandsWithOffsetWidth(
317 const MachineInstr &LdSt,
318 SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
319 bool &OffsetIsScalable, LocationSize &Width) const final;
320
321 bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
322 int64_t Offset1, bool OffsetIsScalable1,
323 ArrayRef<const MachineOperand *> BaseOps2,
324 int64_t Offset2, bool OffsetIsScalable2,
325 unsigned ClusterSize,
326 unsigned NumBytes) const override;
327
328 /// Return true if the memory operations described by \p BaseOps1 and
329 /// \p BaseOps2 (as returned by getMemOperandsWithOffsetWidth) address the
330 /// same base. Unlike shouldClusterMemOps, this does not apply the
331 /// MaxMemoryClusterDWords budget, which only bounds pre-RA register pressure.
332 /// Only the first base operand is compared, unlike
333 /// memOpsHaveSameBaseOperands which requires all of them to match.
334 bool memOpsHaveSameBase(ArrayRef<const MachineOperand *> BaseOps1,
335 ArrayRef<const MachineOperand *> BaseOps2) const;
336
337 bool shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0,
338 int64_t Offset1, unsigned NumLoads) const override;
339
340 void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
341 const DebugLoc &DL, Register DestReg, Register SrcReg,
342 bool KillSrc, bool RenamableDest = false,
343 bool RenamableSrc = false) const override;
344
345private:
346 void storeRegToStackSlotImpl(MachineBasicBlock &MBB,
347 MachineBasicBlock::iterator MI, Register SrcReg,
348 bool isKill, int FrameIndex,
349 const TargetRegisterClass *RC, Register VReg,
350 MachineInstr::MIFlag Flags, bool NeedsCFI) const;
351
352public:
353 void storeRegToStackSlotCFI(MachineBasicBlock &MBB,
354 MachineBasicBlock::iterator MI, Register SrcReg,
355 bool isKill, int FrameIndex,
356 const TargetRegisterClass *RC) const;
357
358 bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg,
359 int64_t &ImmVal) const override;
360
361 std::optional<int64_t>
362 getImmOrMaterializedImm(const MachineRegisterInfo &MRI,
363 const MachineOperand &Op,
364 MachineInstr **DefMI = nullptr) const;
365 std::optional<int64_t>
366 getImmOrMaterializedImm(const MachineRegisterInfo &MRI, Register Reg,
367 MachineInstr **DefMI = nullptr) const;
368
369 unsigned getVectorRegSpillSaveOpcode(Register Reg,
370 const TargetRegisterClass *RC,
371 unsigned Size,
372 const SIMachineFunctionInfo &MFI,
373 bool NeedsCFI) const;
374 unsigned
375 getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC,
376 unsigned Size,
377 const SIMachineFunctionInfo &MFI) const;
378
379 void storeRegToStackSlot(
380 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
381 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
382 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
383
384 void loadRegFromStackSlot(
385 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
386 int FrameIndex, const TargetRegisterClass *RC, Register VReg,
387 unsigned SubReg = 0,
388 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
389
390 bool expandPostRAPseudo(MachineInstr &MI) const override;
391
392 void
393 reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
394 Register DestReg, unsigned SubIdx, const MachineInstr &Orig,
395 LaneBitmask UsedLanes = LaneBitmask::getAll()) const override;
396
397 // Splits a V_MOV_B64_DPP_PSEUDO opcode into a pair of v_mov_b32_dpp
398 // instructions. Returns a pair of generated instructions.
399 // Can split either post-RA with physical registers or pre-RA with
400 // virtual registers. In latter case IR needs to be in SSA form and
401 // and a REG_SEQUENCE is produced to define original register.
402 std::pair<MachineInstr*, MachineInstr*>
403 expandMovDPP64(MachineInstr &MI) const;
404
405 // Returns an opcode that can be used to move a value to a \p DstRC
406 // register. If there is no hardware instruction that can store to \p
407 // DstRC, then AMDGPU::COPY is returned.
408 unsigned getMovOpcode(const TargetRegisterClass *DstRC) const;
409
410 const MCInstrDesc &getIndirectRegWriteMovRelPseudo(unsigned VecSize,
411 unsigned EltSize,
412 bool IsSGPR) const;
413
414 const MCInstrDesc &getIndirectGPRIDXPseudo(unsigned VecSize,
415 bool IsIndirectSrc) const;
416 LLVM_READONLY
417 int commuteOpcode(unsigned Opc) const;
418
419 LLVM_READONLY
420 inline int commuteOpcode(const MachineInstr &MI) const {
421 return commuteOpcode(Opc: MI.getOpcode());
422 }
423
424 bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0,
425 unsigned &SrcOpIdx1) const override;
426
427 bool findCommutedOpIndices(const MCInstrDesc &Desc, unsigned &SrcOpIdx0,
428 unsigned &SrcOpIdx1) const;
429
430 bool isBranchOffsetInRange(unsigned BranchOpc,
431 int64_t BrOffset) const override;
432
433 MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;
434
435 /// Return whether the block terminate with divergent branch.
436 /// Note this only work before lowering the pseudo control flow instructions.
437 bool hasDivergentBranch(const MachineBasicBlock *MBB) const;
438
439 void insertIndirectBranch(MachineBasicBlock &MBB,
440 MachineBasicBlock &NewDestBB,
441 MachineBasicBlock &RestoreBB, const DebugLoc &DL,
442 int64_t BrOffset, RegScavenger *RS) const override;
443
444 bool analyzeBranchImpl(MachineBasicBlock &MBB,
445 MachineBasicBlock::iterator I,
446 MachineBasicBlock *&TBB,
447 MachineBasicBlock *&FBB,
448 SmallVectorImpl<MachineOperand> &Cond,
449 bool AllowModify) const;
450
451 bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
452 MachineBasicBlock *&FBB,
453 SmallVectorImpl<MachineOperand> &Cond,
454 bool AllowModify = false) const override;
455
456 unsigned removeBranch(MachineBasicBlock &MBB,
457 int *BytesRemoved = nullptr) const override;
458
459 unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
460 MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
461 const DebugLoc &DL,
462 int *BytesAdded = nullptr) const override;
463
464 bool reverseBranchCondition(
465 SmallVectorImpl<MachineOperand> &Cond) const override;
466
467 std::unique_ptr<PipelinerLoopInfo>
468 analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
469
470 bool canInsertSelect(const MachineBasicBlock &MBB,
471 ArrayRef<MachineOperand> Cond, Register DstReg,
472 Register TrueReg, Register FalseReg,
473 SelectExpansion &Exp) const override;
474
475 void insertSelect(MachineBasicBlock &MBB,
476 MachineBasicBlock::iterator I, const DebugLoc &DL,
477 Register DstReg, ArrayRef<MachineOperand> Cond,
478 Register TrueReg, Register FalseReg) const override;
479
480 bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
481 Register &SrcReg2, int64_t &CmpMask,
482 int64_t &CmpValue) const override;
483
484 bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
485 Register SrcReg2, int64_t CmpMask, int64_t CmpValue,
486 const MachineRegisterInfo *MRI) const override;
487
488 bool
489 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
490 const MachineInstr &MIb) const override;
491
492 static bool isFoldableCopy(const MachineInstr &MI);
493 static unsigned getFoldableCopySrcIdx(const MachineInstr &MI);
494
495 void removeModOperands(MachineInstr &MI) const;
496
497 void mutateAndCleanupImplicit(MachineInstr &MI,
498 const MCInstrDesc &NewDesc) const;
499
500 /// Return the extracted immediate value in a subregister use from a constant
501 /// materialized in a super register.
502 ///
503 /// e.g. %imm = S_MOV_B64 K[0:63]
504 /// USE %imm.sub1
505 /// This will return K[32:63]
506 static std::optional<int64_t> extractSubregFromImm(int64_t ImmVal,
507 unsigned SubRegIndex);
508
509 /// Return the target flags for a global address operand reading subregister
510 /// \p SubRegIndex of a 64-bit global address with \p TargetFlags.
511 /// e.g. %addr = S_MOV_B64 target-flags(amdgpu-abs64) @gv
512 /// USE %addr.sub1
513 /// This will return amdgpu-abs32-hi
514 static std::optional<unsigned>
515 extractSubregFromGlobalAddressFlags(unsigned TargetFlags,
516 unsigned SubRegIndex);
517
518 bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
519 MachineRegisterInfo *MRI) const final;
520
521 unsigned getMachineCSELookAheadLimit() const override { return 500; }
522
523 MachineInstr *convertToThreeAddress(MachineInstr &MI,
524 LiveIntervals *LIS) const override;
525
526 bool isSchedulingBoundary(const MachineInstr &MI,
527 const MachineBasicBlock *MBB,
528 const MachineFunction &MF) const override;
529
530 static bool isSALU(const MachineInstr &MI) {
531 return SIInstrFlags::isSALU(O: MI);
532 }
533
534 bool isSALU(uint32_t Opcode) const {
535 return SIInstrFlags::isSALU(O: get(Opcode));
536 }
537
538 static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA) {
539 if (!AllowLDSDMA && isLDSDMA(MI))
540 return false;
541
542 return SIInstrFlags::isVALU(O: MI);
543 }
544
545 /// LDSDMA instructions act as both VALU and memory instructions, thus
546 /// we also tag them as VALU. However, in many places, we do not actually want
547 /// to include LDSDMA instructions in this query. By setting \p AllowLDSDMA to
548 /// false, this will return false for LDSDMA instructions.
549 bool isVALU(uint32_t Opcode, bool AllowLDSDMA) const {
550 if (!AllowLDSDMA && isLDSDMA(Opcode))
551 return false;
552
553 return SIInstrFlags::isVALU(O: get(Opcode));
554 }
555
556 static bool isImage(const MachineInstr &MI) {
557 return SIInstrFlags::isImage(O: MI);
558 }
559
560 bool isImage(uint32_t Opcode) const {
561 return SIInstrFlags::isImage(O: get(Opcode));
562 }
563
564 static bool isVMEM(const MachineInstr &MI) {
565 return SIInstrFlags::isVMEM(O: MI);
566 }
567
568 bool isVMEM(uint32_t Opcode) const {
569 return SIInstrFlags::isVMEM(O: get(Opcode));
570 }
571
572 /// True if MI implicitly drains XCNT.
573 static bool isXcntDrain(const MachineInstr &MI);
574
575 static bool isSOP1(const MachineInstr &MI) {
576 return SIInstrFlags::isSOP1(O: MI);
577 }
578
579 bool isSOP1(uint32_t Opcode) const {
580 return SIInstrFlags::isSOP1(O: get(Opcode));
581 }
582
583 static bool isSOP2(const MachineInstr &MI) {
584 return SIInstrFlags::isSOP2(O: MI);
585 }
586
587 bool isSOP2(uint32_t Opcode) const {
588 return SIInstrFlags::isSOP2(O: get(Opcode));
589 }
590
591 static bool isSOPC(const MachineInstr &MI) {
592 return SIInstrFlags::isSOPC(O: MI);
593 }
594
595 bool isSOPC(uint32_t Opcode) const {
596 return SIInstrFlags::isSOPC(O: get(Opcode));
597 }
598
599 static bool isSOPK(const MachineInstr &MI) {
600 return SIInstrFlags::isSOPK(O: MI);
601 }
602
603 bool isSOPK(uint32_t Opcode) const {
604 return SIInstrFlags::isSOPK(O: get(Opcode));
605 }
606
607 static bool isSOPP(const MachineInstr &MI) {
608 return SIInstrFlags::isSOPP(O: MI);
609 }
610
611 bool isSOPP(uint32_t Opcode) const {
612 return SIInstrFlags::isSOPP(O: get(Opcode));
613 }
614
615 static bool isPacked(const MachineInstr &MI) {
616 return SIInstrFlags::isPacked(O: MI);
617 }
618
619 bool isPacked(uint32_t Opcode) const {
620 return SIInstrFlags::isPacked(O: get(Opcode));
621 }
622
623 static bool isVOP1(const MachineInstr &MI) {
624 return SIInstrFlags::isVOP1(O: MI);
625 }
626
627 bool isVOP1(uint32_t Opcode) const {
628 return SIInstrFlags::isVOP1(O: get(Opcode));
629 }
630
631 static bool isVOP2(const MachineInstr &MI) {
632 return SIInstrFlags::isVOP2(O: MI);
633 }
634
635 bool isVOP2(uint32_t Opcode) const {
636 return SIInstrFlags::isVOP2(O: get(Opcode));
637 }
638
639 static bool isVOP3(const MCInstrDesc &Desc) {
640 return SIInstrFlags::isVOP3(O: Desc);
641 }
642
643 static bool isVOP3(const MachineInstr &MI) { return isVOP3(Desc: MI.getDesc()); }
644
645 bool isVOP3(uint32_t Opcode) const { return isVOP3(Desc: get(Opcode)); }
646
647 static bool isSDWA(const MachineInstr &MI) {
648 return SIInstrFlags::isSDWA(O: MI);
649 }
650
651 bool isSDWA(uint32_t Opcode) const {
652 return SIInstrFlags::isSDWA(O: get(Opcode));
653 }
654
655 static bool isVOPC(const MachineInstr &MI) {
656 return SIInstrFlags::isVOPC(O: MI);
657 }
658
659 bool isVOPC(uint32_t Opcode) const {
660 return SIInstrFlags::isVOPC(O: get(Opcode));
661 }
662
663 static bool isMUBUF(const MachineInstr &MI) {
664 return SIInstrFlags::isMUBUF(O: MI);
665 }
666
667 bool isMUBUF(uint32_t Opcode) const {
668 return SIInstrFlags::isMUBUF(O: get(Opcode));
669 }
670
671 static bool isMTBUF(const MachineInstr &MI) {
672 return SIInstrFlags::isMTBUF(O: MI);
673 }
674
675 bool isMTBUF(uint32_t Opcode) const {
676 return SIInstrFlags::isMTBUF(O: get(Opcode));
677 }
678
679 static bool isBUF(const MachineInstr &MI) {
680 return isMUBUF(MI) || isMTBUF(MI);
681 }
682
683 static bool isSMRD(const MachineInstr &MI) {
684 return SIInstrFlags::isSMRD(O: MI);
685 }
686
687 bool isSMRD(uint32_t Opcode) const {
688 return SIInstrFlags::isSMRD(O: get(Opcode));
689 }
690
691 bool isBufferSMRD(const MachineInstr &MI) const;
692
693 static bool isDS(const MachineInstr &MI) { return SIInstrFlags::isDS(O: MI); }
694
695 bool isDS(uint32_t Opcode) const { return SIInstrFlags::isDS(O: get(Opcode)); }
696
697 static bool isLDSDMA(const MachineInstr &MI) {
698 return (SIInstrFlags::isVALU(O: MI) && (isMUBUF(MI) || isFLAT(MI))) ||
699 SIInstrFlags::usesTENSOR_CNT(O: MI);
700 }
701
702 bool isLDSDMA(uint32_t Opcode) const {
703 return (SIInstrFlags::isVALU(O: get(Opcode)) &&
704 (isMUBUF(Opcode) || isFLAT(Opcode))) ||
705 SIInstrFlags::usesTENSOR_CNT(O: get(Opcode));
706 }
707
708 static bool isGWS(const MachineInstr &MI) { return SIInstrFlags::isGWS(O: MI); }
709
710 bool isGWS(uint32_t Opcode) const { return SIInstrFlags::isGWS(O: get(Opcode)); }
711
712 bool isAlwaysGDS(uint32_t Opcode) const;
713
714 static bool isMIMG(const MachineInstr &MI) {
715 return SIInstrFlags::isMIMG(O: MI);
716 }
717
718 bool isMIMG(uint32_t Opcode) const {
719 return SIInstrFlags::isMIMG(O: get(Opcode));
720 }
721
722 static bool isVIMAGE(const MachineInstr &MI) {
723 return SIInstrFlags::isVIMAGE(O: MI);
724 }
725
726 bool isVIMAGE(uint32_t Opcode) const {
727 return SIInstrFlags::isVIMAGE(O: get(Opcode));
728 }
729
730 static bool isVSAMPLE(const MachineInstr &MI) {
731 return SIInstrFlags::isVSAMPLE(O: MI);
732 }
733
734 bool isVSAMPLE(uint32_t Opcode) const {
735 return SIInstrFlags::isVSAMPLE(O: get(Opcode));
736 }
737
738 static bool isGather4(const MachineInstr &MI) {
739 return SIInstrFlags::isGather4(O: MI);
740 }
741
742 bool isGather4(uint32_t Opcode) const {
743 return SIInstrFlags::isGather4(O: get(Opcode));
744 }
745
746 static bool isFLAT(const MachineInstr &MI) {
747 return SIInstrFlags::isFLAT(O: MI);
748 }
749
750 // Is a FLAT encoded instruction which accesses a specific segment,
751 // i.e. global_* or scratch_*.
752 static bool isSegmentSpecificFLAT(const MachineInstr &MI) {
753 return SIInstrFlags::isSegmentSpecificFLAT(O: MI);
754 }
755
756 bool isSegmentSpecificFLAT(uint32_t Opcode) const {
757 return SIInstrFlags::isSegmentSpecificFLAT(O: get(Opcode));
758 }
759
760 static bool isFLATGlobal(const MachineInstr &MI) {
761 return SIInstrFlags::isFlatGlobal(O: MI);
762 }
763
764 bool isFLATGlobal(uint32_t Opcode) const {
765 return SIInstrFlags::isFlatGlobal(O: get(Opcode));
766 }
767
768 static bool isFLATScratch(const MachineInstr &MI) {
769 return SIInstrFlags::isFlatScratch(O: MI);
770 }
771
772 bool isFLATScratch(uint32_t Opcode) const {
773 return SIInstrFlags::isFlatScratch(O: get(Opcode));
774 }
775
776 // Any FLAT encoded instruction, including global_* and scratch_*.
777 bool isFLAT(uint32_t Opcode) const {
778 return SIInstrFlags::isFLAT(O: get(Opcode));
779 }
780
781 /// \returns true for SCRATCH_ instructions, or FLAT/BUF instructions unless
782 /// the MMOs do not include scratch.
783 /// Conservatively correct; will return true if \p MI cannot be proven
784 /// to not hit scratch.
785 bool mayAccessScratch(const MachineInstr &MI) const;
786
787 /// \returns true for FLAT instructions that can access VMEM.
788 bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const;
789
790 /// \returns true for FLAT instructions that can access LDS.
791 bool mayAccessLDSThroughFlat(const MachineInstr &MI, bool TgSplit) const;
792
793 static bool isBlockLoadStore(uint32_t Opcode) {
794 switch (Opcode) {
795 case AMDGPU::SI_BLOCK_SPILL_V1024_SAVE:
796 case AMDGPU::SI_BLOCK_SPILL_V1024_CFI_SAVE:
797 case AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE:
798 case AMDGPU::SCRATCH_STORE_BLOCK_SADDR:
799 case AMDGPU::SCRATCH_LOAD_BLOCK_SADDR:
800 case AMDGPU::SCRATCH_STORE_BLOCK_SVS:
801 case AMDGPU::SCRATCH_LOAD_BLOCK_SVS:
802 return true;
803 default:
804 return false;
805 }
806 }
807
808 static bool setsSCCIfResultIsNonZero(const MachineInstr &MI) {
809 switch (MI.getOpcode()) {
810 case AMDGPU::S_ABSDIFF_I32:
811 case AMDGPU::S_ABS_I32:
812 case AMDGPU::S_AND_B32:
813 case AMDGPU::S_AND_B64:
814 case AMDGPU::S_ANDN2_B32:
815 case AMDGPU::S_ANDN2_B64:
816 case AMDGPU::S_ASHR_I32:
817 case AMDGPU::S_ASHR_I64:
818 case AMDGPU::S_BCNT0_I32_B32:
819 case AMDGPU::S_BCNT0_I32_B64:
820 case AMDGPU::S_BCNT1_I32_B32:
821 case AMDGPU::S_BCNT1_I32_B64:
822 case AMDGPU::S_BFE_I32:
823 case AMDGPU::S_BFE_I64:
824 case AMDGPU::S_BFE_U32:
825 case AMDGPU::S_BFE_U64:
826 case AMDGPU::S_LSHL_B32:
827 case AMDGPU::S_LSHL_B64:
828 case AMDGPU::S_LSHR_B32:
829 case AMDGPU::S_LSHR_B64:
830 case AMDGPU::S_NAND_B32:
831 case AMDGPU::S_NAND_B64:
832 case AMDGPU::S_NOR_B32:
833 case AMDGPU::S_NOR_B64:
834 case AMDGPU::S_NOT_B32:
835 case AMDGPU::S_NOT_B64:
836 case AMDGPU::S_OR_B32:
837 case AMDGPU::S_OR_B64:
838 case AMDGPU::S_ORN2_B32:
839 case AMDGPU::S_ORN2_B64:
840 case AMDGPU::S_QUADMASK_B32:
841 case AMDGPU::S_QUADMASK_B64:
842 case AMDGPU::S_WQM_B32:
843 case AMDGPU::S_WQM_B64:
844 case AMDGPU::S_XNOR_B32:
845 case AMDGPU::S_XNOR_B64:
846 case AMDGPU::S_XOR_B32:
847 case AMDGPU::S_XOR_B64:
848 return true;
849 default:
850 return false;
851 }
852 }
853
854 static bool isEXP(const MachineInstr &MI) { return SIInstrFlags::isEXP(O: MI); }
855
856 static bool isDualSourceBlendEXP(const MachineInstr &MI) {
857 if (!isEXP(MI))
858 return false;
859 unsigned Target = MI.getOperand(i: 0).getImm();
860 return Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND0 ||
861 Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND1;
862 }
863
864 bool isEXP(uint32_t Opcode) const { return SIInstrFlags::isEXP(O: get(Opcode)); }
865
866 static bool isAtomicNoRet(const MachineInstr &MI) {
867 return SIInstrFlags::isAtomicNoRet(O: MI);
868 }
869
870 bool isAtomicNoRet(uint32_t Opcode) const {
871 return SIInstrFlags::isAtomicNoRet(O: get(Opcode));
872 }
873
874 static bool isAtomicRet(const MachineInstr &MI) {
875 return SIInstrFlags::isAtomicRet(O: MI);
876 }
877
878 bool isAtomicRet(uint32_t Opcode) const {
879 return SIInstrFlags::isAtomicRet(O: get(Opcode));
880 }
881
882 static bool isAtomic(const MachineInstr &MI) {
883 return SIInstrFlags::isAtomic(O: MI);
884 }
885
886 bool isAtomic(uint32_t Opcode) const {
887 return SIInstrFlags::isAtomic(O: get(Opcode));
888 }
889
890 static bool mayWriteLDSThroughDMA(const MachineInstr &MI) {
891 unsigned Opc = MI.getOpcode();
892 // Exclude instructions that read FROM LDS (not write to it)
893 return isLDSDMA(MI) && Opc != AMDGPU::BUFFER_STORE_LDS_DWORD &&
894 Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d2 &&
895 Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d4;
896 }
897
898 bool isNonAsyncLdsDmaWrite(const MachineInstr &MI) const {
899 return mayWriteLDSThroughDMA(MI) && !isAsync(MI);
900 }
901
902 /// \returns true if \p MI creates an async operation.
903 bool isAsync(const MachineInstr &MI) const {
904 if (!isLDSDMA(MI))
905 return false;
906 if (usesASYNC_CNT(MI) || usesTENSOR_CNT(MI))
907 return true;
908 const MachineOperand *Async = getNamedOperand(MI, OperandName: AMDGPU::OpName::IsAsync);
909 return Async && (Async->getImm());
910 }
911
912 /// \returns the asyncmark stage that the async operation initiated by \p MI
913 /// belongs to, or no stage if \p MI does not initiate an async operation.
914 AMDGPU::AsyncStages getAsyncStage(const MachineInstr &MI) const;
915
916 static bool isSBarrierSCCWrite(unsigned Opcode) {
917 return Opcode == AMDGPU::S_BARRIER_LEAVE ||
918 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM ||
919 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0;
920 }
921
922 static bool isCBranchVCCZRead(const MachineInstr &MI) {
923 unsigned Opc = MI.getOpcode();
924 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
925 !MI.getOperand(i: 1).isUndef();
926 }
927
928 static bool isWQM(const MachineInstr &MI) { return SIInstrFlags::isWQM(O: MI); }
929
930 bool isWQM(uint32_t Opcode) const { return SIInstrFlags::isWQM(O: get(Opcode)); }
931
932 static bool isDisableWQM(const MachineInstr &MI) {
933 return SIInstrFlags::isDisableWQM(O: MI);
934 }
935
936 bool isDisableWQM(uint32_t Opcode) const {
937 return SIInstrFlags::isDisableWQM(O: get(Opcode));
938 }
939
940 // SI_SPILL_S32_TO_VGPR and SI_RESTORE_S32_FROM_VGPR form a special case of
941 // SGPRs spilling to VGPRs which are SGPR spills but from VALU instructions
942 // therefore we need an explicit check for them since just checking if the
943 // Spill bit is set and what instruction type it came from misclassifies
944 // them.
945 static bool isVGPRSpill(const MachineInstr &MI) {
946 return MI.getOpcode() != AMDGPU::SI_SPILL_S32_TO_VGPR &&
947 MI.getOpcode() != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
948 (isSpill(MI) && isVALU(MI, /*AllowLDSDMA=*/AllowLDSDMA: false));
949 }
950
951 bool isVGPRSpill(uint32_t Opcode) const {
952 return Opcode != AMDGPU::SI_SPILL_S32_TO_VGPR &&
953 Opcode != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
954 (isSpill(Opcode) && isVALU(Opcode, /*AllowLDSDMA=*/AllowLDSDMA: false));
955 }
956
957 static bool isSGPRSpill(const MachineInstr &MI) {
958 return MI.getOpcode() == AMDGPU::SI_SPILL_S32_TO_VGPR ||
959 MI.getOpcode() == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
960 (isSpill(MI) && isSALU(MI));
961 }
962
963 bool isSGPRSpill(uint32_t Opcode) const {
964 return Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR ||
965 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
966 (isSpill(Opcode) && isSALU(Opcode));
967 }
968
969 bool isSpill(uint32_t Opcode) const {
970 return SIInstrFlags::isSpill(O: get(Opcode));
971 }
972
973 static bool isSpill(const MCInstrDesc &Desc) {
974 return SIInstrFlags::isSpill(O: Desc);
975 }
976
977 static bool isSpill(const MachineInstr &MI) { return isSpill(Desc: MI.getDesc()); }
978
979 static bool isWWMRegSpillOpcode(uint32_t Opcode) {
980 return Opcode == AMDGPU::SI_SPILL_WWM_V32_SAVE ||
981 Opcode == AMDGPU::SI_SPILL_WWM_AV32_SAVE ||
982 Opcode == AMDGPU::SI_SPILL_WWM_V32_RESTORE ||
983 Opcode == AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
984 }
985
986 static bool isChainCallOpcode(uint64_t Opcode) {
987 return Opcode == AMDGPU::SI_CS_CHAIN_TC_W32 ||
988 Opcode == AMDGPU::SI_CS_CHAIN_TC_W64;
989 }
990
991 static bool isDPP(const MachineInstr &MI) { return SIInstrFlags::isDPP(O: MI); }
992
993 bool isDPP(uint32_t Opcode) const { return SIInstrFlags::isDPP(O: get(Opcode)); }
994
995 // Some opcodes use Src1 for DPP instead of Src0, because the sequencer
996 // transforms them and reverse the order of their operands at runtime.
997 //
998 // Documentation is incomplete on which instructions are effected, so
999 // the implementation is derived from experimentation.
1000 //
1001 // Listed as target-independent pseudos; the per-subtarget MC opcodes
1002 // (V_SUBREV_NC_U32_e32_gfx11 and friends) are all reached through these.
1003 // Defined out of line because GCNSubtarget is incomplete here.
1004 static bool isSrc1DPPRevOpcode(const GCNSubtarget &ST, uint32_t Opcode);
1005
1006 static bool isTRANS(const MachineInstr &MI) {
1007 return SIInstrFlags::isTRANS(O: MI);
1008 }
1009
1010 bool isTRANS(uint32_t Opcode) const {
1011 return SIInstrFlags::isTRANS(O: get(Opcode));
1012 }
1013
1014 static bool isVOP3P(const MachineInstr &MI) {
1015 return SIInstrFlags::isVOP3P(O: MI);
1016 }
1017
1018 bool isVOP3P(uint32_t Opcode) const {
1019 return SIInstrFlags::isVOP3P(O: get(Opcode));
1020 }
1021
1022 bool isVOP3PMix(const MachineInstr &MI) const {
1023 return isVOP3PMix(Opcode: MI.getOpcode());
1024 }
1025
1026 bool isVOP3PMix(uint16_t Opcode) const {
1027 switch (Opcode) {
1028 case AMDGPU::V_FMA_MIXHI_F16:
1029 case AMDGPU::V_FMA_MIXLO_F16:
1030 case AMDGPU::V_FMA_MIX_F32:
1031 case AMDGPU::V_MAD_MIXHI_F16:
1032 case AMDGPU::V_MAD_MIXLO_F16:
1033 case AMDGPU::V_MAD_MIX_F32:
1034 return true;
1035 default:
1036 return false;
1037 }
1038 }
1039
1040 static bool isVINTRP(const MachineInstr &MI) {
1041 return SIInstrFlags::isVINTRP(O: MI);
1042 }
1043
1044 bool isVINTRP(uint32_t Opcode) const {
1045 return SIInstrFlags::isVINTRP(O: get(Opcode));
1046 }
1047
1048 static bool isMAI(const MCInstrDesc &Desc) {
1049 return SIInstrFlags::isMAI(O: Desc);
1050 }
1051
1052 static bool isMAI(const MachineInstr &MI) { return isMAI(Desc: MI.getDesc()); }
1053
1054 bool isMAI(uint32_t Opcode) const { return isMAI(Desc: get(Opcode)); }
1055
1056 static bool isMFMA(const MachineInstr &MI) {
1057 return isMAI(MI) && MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1058 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1059 }
1060
1061 bool isMFMA(uint32_t Opcode) const {
1062 return isMAI(Opcode) && Opcode != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1063 Opcode != AMDGPU::V_ACCVGPR_READ_B32_e64;
1064 }
1065
1066 static bool isDOT(const MachineInstr &MI) { return SIInstrFlags::isDOT(O: MI); }
1067
1068 static bool isWMMA(const MachineInstr &MI) {
1069 return SIInstrFlags::isWMMA(O: MI);
1070 }
1071
1072 bool isWMMA(uint32_t Opcode) const {
1073 return SIInstrFlags::isWMMA(O: get(Opcode));
1074 }
1075
1076 static bool isMFMAorWMMA(const MachineInstr &MI) {
1077 return isMFMA(MI) || isWMMA(MI) || isSWMMAC(MI);
1078 }
1079
1080 bool isMFMAorWMMA(uint32_t Opcode) const {
1081 return isMFMA(Opcode) || isWMMA(Opcode) || isSWMMAC(Opcode);
1082 }
1083
1084 static bool isSWMMAC(const MachineInstr &MI) {
1085 return SIInstrFlags::isSWMMAC(O: MI);
1086 }
1087
1088 bool isSWMMAC(uint32_t Opcode) const {
1089 return SIInstrFlags::isSWMMAC(O: get(Opcode));
1090 }
1091
1092 bool isDOT(uint32_t Opcode) const { return SIInstrFlags::isDOT(O: get(Opcode)); }
1093
1094 bool isXDLWMMA(const MachineInstr &MI) const;
1095
1096 bool isXDL(const MachineInstr &MI) const;
1097
1098 /// On gfx1250, an XDL instruction whose destination spans at least eight
1099 /// VGPRs decrements VA_VDST only after every earlier-issued VALU instruction,
1100 /// of any type, has decremented it. Nothing is implied about the order
1101 /// relative to instructions issued after it.
1102 bool isVAVDSTOrderedXDL(const MachineInstr &MI) const;
1103
1104 static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opc: Opcode); }
1105
1106 static bool isLDSDIR(const MachineInstr &MI) {
1107 return SIInstrFlags::isLDSDIR(O: MI);
1108 }
1109
1110 bool isLDSDIR(uint32_t Opcode) const {
1111 return SIInstrFlags::isLDSDIR(O: get(Opcode));
1112 }
1113
1114 static bool isVINTERP(const MachineInstr &MI) {
1115 return SIInstrFlags::isVINTERP(O: MI);
1116 }
1117
1118 bool isVINTERP(uint32_t Opcode) const {
1119 return SIInstrFlags::isVINTERP(O: get(Opcode));
1120 }
1121
1122 static bool usesVM_CNT(const MachineInstr &MI) {
1123 return SIInstrFlags::usesVM_CNT(O: MI);
1124 }
1125
1126 static bool usesLGKM_CNT(const MachineInstr &MI) {
1127 return SIInstrFlags::usesLGKM_CNT(O: MI);
1128 }
1129
1130 static bool usesASYNC_CNT(const MachineInstr &MI) {
1131 return SIInstrFlags::usesASYNC_CNT(O: MI);
1132 }
1133
1134 bool usesASYNC_CNT(uint32_t Opcode) const {
1135 return SIInstrFlags::usesASYNC_CNT(O: get(Opcode));
1136 }
1137
1138 static bool usesTENSOR_CNT(const MachineInstr &MI) {
1139 return SIInstrFlags::usesTENSOR_CNT(O: MI);
1140 }
1141
1142 bool usesTENSOR_CNT(uint32_t Opcode) const {
1143 return SIInstrFlags::usesTENSOR_CNT(O: get(Opcode));
1144 }
1145
1146 // Most sopk treat the immediate as a signed 16-bit, however some
1147 // use it as unsigned.
1148 static bool sopkIsZext(unsigned Opcode) {
1149 return Opcode == AMDGPU::S_CMPK_EQ_U32 || Opcode == AMDGPU::S_CMPK_LG_U32 ||
1150 Opcode == AMDGPU::S_CMPK_GT_U32 || Opcode == AMDGPU::S_CMPK_GE_U32 ||
1151 Opcode == AMDGPU::S_CMPK_LT_U32 || Opcode == AMDGPU::S_CMPK_LE_U32 ||
1152 Opcode == AMDGPU::S_GETREG_B32 ||
1153 Opcode == AMDGPU::S_GETREG_B32_const;
1154 }
1155
1156 /// \returns true if this is an s_store_dword* instruction. This is more
1157 /// specific than isSMEM && mayStore.
1158 static bool isScalarStore(const MachineInstr &MI) {
1159 return SIInstrFlags::isScalarStore(O: MI);
1160 }
1161
1162 bool isScalarStore(uint32_t Opcode) const {
1163 return SIInstrFlags::isScalarStore(O: get(Opcode));
1164 }
1165
1166 static bool isFixedSize(const MachineInstr &MI) {
1167 return SIInstrFlags::isFixedSize(O: MI);
1168 }
1169
1170 bool isFixedSize(uint32_t Opcode) const {
1171 return SIInstrFlags::isFixedSize(O: get(Opcode));
1172 }
1173
1174 static bool hasFPClamp(const MachineInstr &MI) {
1175 return SIInstrFlags::hasFPClamp(O: MI);
1176 }
1177
1178 bool hasFPClamp(uint32_t Opcode) const {
1179 return SIInstrFlags::hasFPClamp(O: get(Opcode));
1180 }
1181
1182 static bool hasIntClamp(const MachineInstr &MI) {
1183 return SIInstrFlags::hasIntClamp(O: MI);
1184 }
1185
1186 static bool hasSameClamp(const MachineInstr &A, const MachineInstr &B) {
1187 const MCInstrDesc &DA = A.getDesc(), &DB = B.getDesc();
1188 return SIInstrFlags::hasFPClamp(O: DA) == SIInstrFlags::hasFPClamp(O: DB) &&
1189 SIInstrFlags::hasIntClamp(O: DA) == SIInstrFlags::hasIntClamp(O: DB) &&
1190 SIInstrFlags::hasClampLo(O: DA) == SIInstrFlags::hasClampLo(O: DB) &&
1191 SIInstrFlags::hasClampHi(O: DA) == SIInstrFlags::hasClampHi(O: DB);
1192 }
1193
1194 static bool usesFPDPRounding(const MachineInstr &MI) {
1195 return SIInstrFlags::usesFPDPRounding(O: MI);
1196 }
1197
1198 bool usesFPDPRounding(uint32_t Opcode) const {
1199 return SIInstrFlags::usesFPDPRounding(O: get(Opcode));
1200 }
1201
1202 static bool isFPAtomic(const MachineInstr &MI) {
1203 return SIInstrFlags::isFPAtomic(O: MI);
1204 }
1205
1206 bool isFPAtomic(uint32_t Opcode) const {
1207 return SIInstrFlags::isFPAtomic(O: get(Opcode));
1208 }
1209
1210 static bool isNeverUniform(const MachineInstr &MI) {
1211 return SIInstrFlags::isNeverUniform(O: MI);
1212 }
1213
1214 // Check to see if opcode is for a barrier start. Pre gfx12 this is just the
1215 // S_BARRIER, but after support for S_BARRIER_SIGNAL* / S_BARRIER_WAIT we want
1216 // to check for the barrier start (S_BARRIER_SIGNAL*)
1217 bool isBarrierStart(unsigned Opcode) const {
1218 return Opcode == AMDGPU::S_BARRIER ||
1219 Opcode == AMDGPU::S_BARRIER_SIGNAL_M0 ||
1220 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0 ||
1221 Opcode == AMDGPU::S_BARRIER_SIGNAL_IMM ||
1222 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM;
1223 }
1224
1225 bool isBarrier(unsigned Opcode) const {
1226 return isBarrierStart(Opcode) || Opcode == AMDGPU::S_BARRIER_WAIT ||
1227 Opcode == AMDGPU::S_BARRIER_INIT_M0 ||
1228 Opcode == AMDGPU::S_BARRIER_INIT_IMM ||
1229 Opcode == AMDGPU::S_BARRIER_JOIN_IMM ||
1230 Opcode == AMDGPU::S_BARRIER_LEAVE || Opcode == AMDGPU::DS_GWS_INIT ||
1231 Opcode == AMDGPU::DS_GWS_BARRIER;
1232 }
1233
1234 static bool isLoadMonitor(unsigned Opc) {
1235 switch (Opc) {
1236 case AMDGPU::GLOBAL_LOAD_MONITOR_B32:
1237 case AMDGPU::GLOBAL_LOAD_MONITOR_B32_SADDR:
1238 case AMDGPU::GLOBAL_LOAD_MONITOR_B64:
1239 case AMDGPU::GLOBAL_LOAD_MONITOR_B64_SADDR:
1240 case AMDGPU::GLOBAL_LOAD_MONITOR_B128:
1241 case AMDGPU::GLOBAL_LOAD_MONITOR_B128_SADDR:
1242 case AMDGPU::FLAT_LOAD_MONITOR_B32:
1243 case AMDGPU::FLAT_LOAD_MONITOR_B64:
1244 case AMDGPU::FLAT_LOAD_MONITOR_B128:
1245 return true;
1246 default:
1247 return false;
1248 }
1249 }
1250
1251 static bool isGFX12CacheInvOrWBInst(unsigned Opc) {
1252 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||
1253 Opc == AMDGPU::GLOBAL_WBINV;
1254 }
1255
1256 static bool isF16PseudoScalarTrans(unsigned Opcode) {
1257 return Opcode == AMDGPU::V_S_EXP_F16_e64 ||
1258 Opcode == AMDGPU::V_S_LOG_F16_e64 ||
1259 Opcode == AMDGPU::V_S_RCP_F16_e64 ||
1260 Opcode == AMDGPU::V_S_RSQ_F16_e64 ||
1261 Opcode == AMDGPU::V_S_SQRT_F16_e64;
1262 }
1263
1264 static bool isPseudoScalarTrans(unsigned Opcode) {
1265 return isF16PseudoScalarTrans(Opcode) ||
1266 Opcode == AMDGPU::V_S_EXP_F32_e64 ||
1267 Opcode == AMDGPU::V_S_LOG_F32_e64 ||
1268 Opcode == AMDGPU::V_S_RCP_F32_e64 ||
1269 Opcode == AMDGPU::V_S_RSQ_F32_e64 ||
1270 Opcode == AMDGPU::V_S_SQRT_F32_e64;
1271 }
1272
1273 static bool isVPermPk16(unsigned Opcode) {
1274 return Opcode == AMDGPU::V_PERM_PK16_B4_U4_e64 ||
1275 Opcode == AMDGPU::V_PERM_PK16_B6_U4_e64 ||
1276 Opcode == AMDGPU::V_PERM_PK16_B8_U4_e64;
1277 }
1278
1279 // \returns true if \p MI clears the V_PERM_PK16 hazard when it immediately
1280 // follows a V_PERM_PK16 (i.e. \p MI is a "safe" instruction).
1281 bool isVPermPk16SafeInstr(const MachineInstr &MI) const {
1282 unsigned Opc = MI.getOpcode();
1283
1284 // Only VALU ops issue on the pipe that clears the V_PERM_PK16 hazard.
1285 if (!isVALU(MI, /*AllowLDSDMA=*/AllowLDSDMA: false))
1286 return false;
1287 // OP_XDL: matrix (WMMA/SWMMAC/DOT) ops clear the hazard.
1288 if (isXDL(MI))
1289 return true;
1290 // Pseudo-scalar transcendentals (OP32_SCL_T) do NOT clear the hazard.
1291 if (isPseudoScalarTrans(Opcode: Opc))
1292 return false;
1293
1294 // Use the table lookup, not getBlockingCycles(): occupancy is gated off on
1295 // gfx1251 but the hazard applies to both gfx1250 and gfx1251. Gfx1250 table
1296 // is valid enough for gfx1251 w.r.t. v_perm_pk16 safety check here.
1297 // OP_32_T is in the table at 2 and is safe; other table entries (>= 2) are
1298 // not.
1299 unsigned Cycles = getGFX1250BlockingCyclesTable(MI);
1300 return Cycles < 2 || (Cycles == 2 && isTRANS(MI));
1301 }
1302
1303 static bool doesNotReadTiedSource(const MachineInstr &MI) {
1304 return SIInstrFlags::isTiedSourceNotRead(O: MI);
1305 }
1306
1307 bool doesNotReadTiedSource(uint32_t Opcode) const {
1308 return SIInstrFlags::isTiedSourceNotRead(O: get(Opcode));
1309 }
1310
1311 bool isIGLP(unsigned Opcode) const {
1312 return Opcode == AMDGPU::SCHED_BARRIER ||
1313 Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
1314 }
1315
1316 bool isIGLP(const MachineInstr &MI) const { return isIGLP(Opcode: MI.getOpcode()); }
1317
1318 // Return true if the instruction is mutually exclusive with all non-IGLP DAG
1319 // mutations, requiring all other mutations to be disabled.
1320 bool isIGLPMutationOnly(unsigned Opcode) const {
1321 return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
1322 }
1323
1324 static unsigned getNonSoftWaitcntOpcode(unsigned Opcode) {
1325 switch (Opcode) {
1326 case AMDGPU::S_WAITCNT_soft:
1327 return AMDGPU::S_WAITCNT;
1328 case AMDGPU::S_WAITCNT_VSCNT_soft:
1329 return AMDGPU::S_WAITCNT_VSCNT;
1330 case AMDGPU::S_WAIT_LOADCNT_soft:
1331 return AMDGPU::S_WAIT_LOADCNT;
1332 case AMDGPU::S_WAIT_STORECNT_soft:
1333 return AMDGPU::S_WAIT_STORECNT;
1334 case AMDGPU::S_WAIT_SAMPLECNT_soft:
1335 return AMDGPU::S_WAIT_SAMPLECNT;
1336 case AMDGPU::S_WAIT_BVHCNT_soft:
1337 return AMDGPU::S_WAIT_BVHCNT;
1338 case AMDGPU::S_WAIT_DSCNT_soft:
1339 return AMDGPU::S_WAIT_DSCNT;
1340 case AMDGPU::S_WAIT_KMCNT_soft:
1341 return AMDGPU::S_WAIT_KMCNT;
1342 case AMDGPU::S_WAIT_XCNT_soft:
1343 return AMDGPU::S_WAIT_XCNT;
1344 default:
1345 return Opcode;
1346 }
1347 }
1348
1349 static bool isWaitcnt(unsigned Opcode) {
1350 switch (getNonSoftWaitcntOpcode(Opcode)) {
1351 case AMDGPU::S_WAITCNT:
1352 case AMDGPU::S_WAITCNT_VSCNT:
1353 case AMDGPU::S_WAITCNT_VMCNT:
1354 case AMDGPU::S_WAITCNT_EXPCNT:
1355 case AMDGPU::S_WAITCNT_LGKMCNT:
1356 case AMDGPU::S_WAIT_LOADCNT:
1357 case AMDGPU::S_WAIT_LOADCNT_DSCNT:
1358 case AMDGPU::S_WAIT_STORECNT:
1359 case AMDGPU::S_WAIT_STORECNT_DSCNT:
1360 case AMDGPU::S_WAIT_SAMPLECNT:
1361 case AMDGPU::S_WAIT_BVHCNT:
1362 case AMDGPU::S_WAIT_EXPCNT:
1363 case AMDGPU::S_WAIT_DSCNT:
1364 case AMDGPU::S_WAIT_KMCNT:
1365 case AMDGPU::S_WAIT_XCNT:
1366 case AMDGPU::S_WAIT_ASYNCCNT:
1367 case AMDGPU::S_WAIT_TENSORCNT:
1368 case AMDGPU::S_WAIT_IDLE:
1369 return true;
1370 default:
1371 return false;
1372 }
1373 }
1374
1375 bool isVGPRCopy(const MachineInstr &MI) const {
1376 assert(isCopyInstr(MI));
1377 Register Dest = MI.getOperand(i: 0).getReg();
1378 const MachineFunction &MF = *MI.getMF();
1379 const MachineRegisterInfo &MRI = MF.getRegInfo();
1380 return !RI.isSGPRReg(MRI, Reg: Dest);
1381 }
1382
1383 bool hasVGPRUses(const MachineInstr &MI) const {
1384 const MachineFunction &MF = *MI.getMF();
1385 const MachineRegisterInfo &MRI = MF.getRegInfo();
1386 return llvm::any_of(Range: MI.explicit_uses(),
1387 P: [&MRI, this](const MachineOperand &MO) {
1388 return MO.isReg() && RI.isVGPR(MRI, Reg: MO.getReg());});
1389 }
1390
1391 /// Return true if the instruction modifies the mode register.q
1392 static bool modifiesModeRegister(const MachineInstr &MI);
1393
1394 /// This function is used to determine if an instruction can be safely
1395 /// executed under EXEC = 0 without hardware error, indeterminate results,
1396 /// and/or visible effects on future vector execution or outside the shader.
1397 /// Note: as of 2024 the only use of this is SIPreEmitPeephole where it is
1398 /// used in removing branches over short EXEC = 0 sequences.
1399 /// As such it embeds certain assumptions which may not apply to every case
1400 /// of EXEC = 0 execution.
1401 bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const;
1402
1403 /// Returns true if the instruction could potentially depend on the value of
1404 /// exec. If false, exec dependencies may safely be ignored.
1405 bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const;
1406
1407 bool isInlineConstant(const APInt &Imm) const;
1408
1409 bool isInlineConstant(const APFloat &Imm) const;
1410
1411 // Returns true if this non-register operand definitely does not need to be
1412 // encoded as a 32-bit literal. Note that this function handles all kinds of
1413 // operands, not just immediates.
1414 //
1415 // Some operands like FrameIndexes could resolve to an inline immediate value
1416 // that will not require an additional 4-bytes; this function assumes that it
1417 // will.
1418 bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const {
1419 if (!MO.isImm())
1420 return false;
1421 return isInlineConstant(ImmVal: MO.getImm(), OperandType);
1422 }
1423 bool isInlineConstant(int64_t ImmVal, uint8_t OperandType) const;
1424
1425 bool isInlineConstant(const MachineOperand &MO,
1426 const MCOperandInfo &OpInfo) const {
1427 return isInlineConstant(MO, OperandType: OpInfo.OperandType);
1428 }
1429
1430 /// \p returns true if \p UseMO is substituted with \p DefMO in \p MI it would
1431 /// be an inline immediate.
1432 bool isInlineConstant(const MachineInstr &MI,
1433 const MachineOperand &UseMO,
1434 const MachineOperand &DefMO) const {
1435 assert(UseMO.getParent() == &MI);
1436 int OpIdx = UseMO.getOperandNo();
1437 if (OpIdx >= MI.getDesc().NumOperands)
1438 return false;
1439
1440 return isInlineConstant(MO: DefMO, OpInfo: MI.getDesc().operands()[OpIdx]);
1441 }
1442
1443 /// \p returns true if the operand \p OpIdx in \p MI is a valid inline
1444 /// immediate.
1445 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx) const {
1446 const MachineOperand &MO = MI.getOperand(i: OpIdx);
1447 return isInlineConstant(MO, OperandType: MI.getDesc().operands()[OpIdx].OperandType);
1448 }
1449
1450 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
1451 int64_t ImmVal) const {
1452 if (OpIdx >= MI.getDesc().NumOperands)
1453 return false;
1454
1455 if (isCopyInstr(MI)) {
1456 unsigned Size = getOpSize(MI, OpNo: OpIdx);
1457 assert(Size == 8 || Size == 4);
1458
1459 uint8_t OpType = (Size == 8) ?
1460 AMDGPU::OPERAND_REG_IMM_INT64 : AMDGPU::OPERAND_REG_IMM_INT32;
1461 return isInlineConstant(ImmVal, OperandType: OpType);
1462 }
1463
1464 return isInlineConstant(ImmVal, OperandType: MI.getDesc().operands()[OpIdx].OperandType);
1465 }
1466
1467 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
1468 const MachineOperand &MO) const {
1469 return isInlineConstant(MI, OpIdx, ImmVal: MO.getImm());
1470 }
1471
1472 bool isInlineConstant(const MachineOperand &MO) const {
1473 return isInlineConstant(MI: *MO.getParent(), OpIdx: MO.getOperandNo());
1474 }
1475
1476 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
1477 const MachineOperand &MO) const;
1478
1479 bool isLiteralOperandLegal(const MCInstrDesc &InstDesc,
1480 const MCOperandInfo &OpInfo) const;
1481
1482 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
1483 int64_t ImmVal) const;
1484
1485 bool isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
1486 const MachineOperand &MO) const {
1487 return isImmOperandLegal(InstDesc: MI.getDesc(), OpNo, MO);
1488 }
1489
1490 bool isNeverCoissue(MachineInstr &MI) const;
1491
1492 /// Check if this immediate value can be used for AV_MOV_B64_IMM_PSEUDO.
1493 bool isLegalAV64PseudoImm(uint64_t Imm) const;
1494
1495 /// Return true if this 64-bit VALU instruction has a 32-bit encoding.
1496 /// This function will return false if you pass it a 32-bit instruction.
1497 bool hasVALU32BitEncoding(unsigned Opcode) const;
1498
1499 /// Return true if \p Reg is a lane mask that already has 0 in every bit
1500 /// corresponding to a lane that is inactive in EXEC where \p Use executes,
1501 /// so that ANDing it with EXEC there would be a no-op. Requires SSA form.
1502 bool isMaskedByExec(Register Reg, const MachineInstr &Use,
1503 const MachineRegisterInfo &MRI, unsigned Depth = 0) const;
1504
1505 bool physRegUsesConstantBus(const MachineOperand &Reg) const;
1506 bool regUsesConstantBus(const MachineOperand &Reg,
1507 const MachineRegisterInfo &MRI) const;
1508
1509 /// Returns true if this operand uses the constant bus.
1510 bool usesConstantBus(const MachineRegisterInfo &MRI,
1511 const MachineOperand &MO,
1512 const MCOperandInfo &OpInfo) const;
1513
1514 bool usesConstantBus(const MachineRegisterInfo &MRI, const MachineInstr &MI,
1515 int OpIdx) const {
1516 return usesConstantBus(MRI, MO: MI.getOperand(i: OpIdx),
1517 OpInfo: MI.getDesc().operands()[OpIdx]);
1518 }
1519
1520 /// Return true if this instruction has any modifiers.
1521 /// e.g. src[012]_mod, omod, clamp.
1522 bool hasModifiers(unsigned Opcode) const;
1523
1524 bool hasModifiersSet(const MachineInstr &MI, AMDGPU::OpName OpName) const;
1525 bool hasAnyModifiersSet(const MachineInstr &MI) const;
1526
1527 bool canShrink(const MachineInstr &MI,
1528 const MachineRegisterInfo &MRI) const;
1529
1530 MachineInstr *buildShrunkInst(MachineInstr &MI,
1531 unsigned NewOpcode) const;
1532
1533 bool verifyInstruction(const MachineInstr &MI,
1534 StringRef &ErrInfo) const override;
1535
1536 unsigned getVALUOp(const MachineInstr &MI) const;
1537 unsigned getVALUOp(unsigned Opc) const;
1538
1539 void insertScratchExecCopy(MachineFunction &MF, MachineBasicBlock &MBB,
1540 MachineBasicBlock::iterator MBBI,
1541 const DebugLoc &DL, Register Reg, bool IsSCCLive,
1542 SlotIndexes *Indexes = nullptr) const;
1543
1544 void restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
1545 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
1546 Register Reg, SlotIndexes *Indexes = nullptr) const;
1547
1548 MachineInstr *getWholeWaveFunctionSetup(MachineFunction &MF) const;
1549
1550 /// Return the correct register class for \p OpNo. For target-specific
1551 /// instructions, this will return the register class that has been defined
1552 /// in tablegen. For generic instructions, like REG_SEQUENCE it will return
1553 /// the register class of its machine operand.
1554 /// to infer the correct register class base on the other operands.
1555 const TargetRegisterClass *getOpRegClass(const MachineInstr &MI,
1556 unsigned OpNo) const;
1557
1558 /// Return the size in bytes of the operand OpNo on the given
1559 // instruction opcode.
1560 unsigned getOpSize(uint32_t Opcode, unsigned OpNo) const {
1561 const MCOperandInfo &OpInfo = get(Opcode).operands()[OpNo];
1562
1563 if (OpInfo.RegClass == -1) {
1564 // If this is an immediate operand, this must be a 32-bit literal.
1565 assert(OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE);
1566 return 4;
1567 }
1568
1569 return RI.getRegSizeInBits(RC: *RI.getRegClass(i: getOpRegClassID(OpInfo))) / 8;
1570 }
1571
1572 /// This form should usually be preferred since it handles operands
1573 /// with unknown register classes.
1574 unsigned getOpSize(const MachineInstr &MI, unsigned OpNo) const {
1575 const MachineOperand &MO = MI.getOperand(i: OpNo);
1576 if (MO.isReg()) {
1577 if (unsigned SubReg = MO.getSubReg()) {
1578 return RI.getSubRegIdxSize(Idx: SubReg) / 8;
1579 }
1580 }
1581 return RI.getRegSizeInBits(RC: *getOpRegClass(MI, OpNo)) / 8;
1582 }
1583
1584 /// Legalize the \p OpIndex operand of this instruction by inserting
1585 /// a MOV. For example:
1586 /// ADD_I32_e32 VGPR0, 15
1587 /// to
1588 /// MOV VGPR1, 15
1589 /// ADD_I32_e32 VGPR0, VGPR1
1590 ///
1591 /// If the operand being legalized is a register, then a COPY will be used
1592 /// instead of MOV.
1593 void legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const;
1594
1595 /// Check if \p MO is a legal operand if it was the \p OpIdx Operand
1596 /// for \p MI.
1597 bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
1598 const MachineOperand *MO = nullptr) const;
1599
1600 /// Check if \p MO (a register operand) is a legal register for the
1601 /// given operand description or operand index.
1602 /// The operand index version provide more legality checks
1603 bool isLegalRegOperand(const MachineRegisterInfo &MRI,
1604 const MCOperandInfo &OpInfo,
1605 const MachineOperand &MO) const;
1606 bool isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
1607 const MachineOperand &MO) const;
1608
1609 /// Check if \p MO would be a legal operand for a single-SGPR-read
1610 /// instruction.
1611 ///
1612 /// Single-SGPR-read instructions typically accept VGPRs, SGPRs, or immediates
1613 /// as source operands. On gfx12+, if a source operand uses SGPRs, the HW can
1614 /// only read the first SGPR and replicate the value across all lanes. \p SrcN
1615 /// can be 0, 1, or 2, representing src0, src1, and src2, respectively. If \p
1616 /// MO is nullptr, the operand corresponding to \p SrcN will be used. Non-SGPR
1617 /// operands are always considered legal.
1618 bool
1619 isLegalSingleSGPRReadInstOperand(const MachineRegisterInfo &MRI,
1620 const MachineInstr &MI, unsigned SrcN,
1621 const MachineOperand *MO = nullptr) const;
1622
1623 /// Legalize operands in \p MI by either commuting it or inserting a
1624 /// copy of src1.
1625 void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1626
1627 /// Fix operands in \p MI to satisfy constant bus requirements.
1628 void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1629
1630 /// Copy a value from a VGPR (\p SrcReg) to SGPR. The desired register class
1631 /// for the dst register (\p DstRC) can be optionally supplied. This function
1632 /// can only be used when it is know that the value in SrcReg is same across
1633 /// all threads in the wave.
1634 /// \returns The SGPR register that \p SrcReg was copied to.
1635 Register readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,
1636 MachineRegisterInfo &MRI,
1637 const TargetRegisterClass *DstRC = nullptr) const;
1638
1639 void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1640 void legalizeOperandsFLAT(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1641
1642 void legalizeGenericOperand(MachineBasicBlock &InsertMBB,
1643 MachineBasicBlock::iterator I,
1644 const TargetRegisterClass *DstRC,
1645 MachineOperand &Op, MachineRegisterInfo &MRI,
1646 const DebugLoc &DL) const;
1647
1648 /// Legalize all operands in this instruction. This function may create new
1649 /// instructions and control-flow around \p MI. If present, \p MDT is
1650 /// updated.
1651 /// \returns A new basic block that contains \p MI if new blocks were created.
1652 MachineBasicBlock *
1653 legalizeOperands(MachineInstr &MI, MachineDominatorTree *MDT = nullptr) const;
1654
1655 /// Change SADDR form of a FLAT \p Inst to its VADDR form if saddr operand
1656 /// was moved to VGPR. \returns true if succeeded.
1657 bool moveFlatAddrToVGPR(MachineInstr &Inst) const;
1658
1659 /// Fix operands in Inst to fix 16bit SALU to VALU lowering.
1660 void legalizeOperandsVALUt16(MachineInstr &Inst,
1661 MachineRegisterInfo &MRI) const;
1662 void legalizeOperandsVALUt16(MachineInstr &Inst, unsigned OpIdx,
1663 MachineRegisterInfo &MRI) const;
1664
1665 /// Build a 32-bit VGPR whose low half is the 16-bit value \p Src.SrcSubReg
1666 /// and whose high half is undefined. The instructions are inserted before
1667 /// \p I. \returns the new register.
1668 Register buildVGPR32FromLo16(MachineBasicBlock &MBB,
1669 MachineBasicBlock::iterator I,
1670 const DebugLoc &DL, Register Src,
1671 unsigned SrcSubReg) const;
1672
1673 /// Replace the instructions opcode with the equivalent VALU
1674 /// opcode. This function will also move the users of MachineInstruntions
1675 /// in the \p WorkList to the VALU if necessary. If present, \p MDT is
1676 /// updated.
1677 void moveToVALU(SIInstrWorklist &Worklist, MachineDominatorTree *MDT) const;
1678
1679 void
1680 moveToVALUImpl(SIInstrWorklist &Worklist, MachineDominatorTree *MDT,
1681 MachineInstr &Inst,
1682 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
1683 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const;
1684 /// Wrapper function for generating waterfall for instruction \p MI
1685 /// This function take into consideration of related pre & succ instructions
1686 /// (e.g. calling process) into consideratioin
1687 void createWaterFallForSiCall(MachineInstr *MI, MachineDominatorTree *MDT,
1688 ArrayRef<MachineOperand *> ScalarOps,
1689 ArrayRef<Register> PhySGPRs = {}) const;
1690
1691 void insertNoop(MachineBasicBlock &MBB,
1692 MachineBasicBlock::iterator MI) const override;
1693
1694 void insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1695 unsigned Quantity) const override;
1696
1697 /// Build instructions that simulate the behavior of a `s_trap 2` instructions
1698 /// for hardware (namely, gfx11) that runs in PRIV=1 mode. There, s_trap is
1699 /// interpreted as a nop.
1700 MachineBasicBlock *insertSimulatedTrap(MachineRegisterInfo &MRI,
1701 MachineBasicBlock &MBB,
1702 MachineInstr &MI,
1703 const DebugLoc &DL) const;
1704
1705 /// Return the number of wait states that result from executing this
1706 /// instruction.
1707 static unsigned getNumWaitStates(const MachineInstr &MI);
1708
1709 /// Returns the operand named \p Op. If \p MI does not have an
1710 /// operand named \c Op, this function returns nullptr.
1711 LLVM_READONLY
1712 MachineOperand *getNamedOperand(MachineInstr &MI,
1713 AMDGPU::OpName OperandName) const;
1714
1715 LLVM_READONLY
1716 const MachineOperand *getNamedOperand(const MachineInstr &MI,
1717 AMDGPU::OpName OperandName) const {
1718 return getNamedOperand(MI&: const_cast<MachineInstr &>(MI), OperandName);
1719 }
1720
1721 /// Get required immediate operand
1722 int64_t getNamedImmOperand(const MachineInstr &MI,
1723 AMDGPU::OpName OperandName) const {
1724 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OperandName);
1725 return MI.getOperand(i: Idx).getImm();
1726 }
1727
1728 uint64_t getDefaultRsrcDataFormat() const;
1729 uint64_t getScratchRsrcWords23() const;
1730
1731 bool isLowLatencyInstruction(const MachineInstr &MI) const;
1732 bool isHighLatencyDef(int Opc) const override;
1733
1734 /// Return the descriptor of the target-specific machine instruction
1735 /// that corresponds to the specified pseudo or native opcode.
1736 const MCInstrDesc &getMCOpcodeFromPseudo(unsigned Opcode) const {
1737 return get(Opcode: pseudoToMCOpcode(Opcode));
1738 }
1739
1740 Register isStackAccess(const MachineInstr &MI, int &FrameIndex,
1741 TypeSize &MemBytes) const;
1742 Register isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex,
1743 TypeSize &MemBytes) const;
1744
1745 Register isLoadFromStackSlot(const MachineInstr &MI,
1746 int &FrameIndex) const override {
1747 TypeSize MemBytes = TypeSize::getZero();
1748 return isLoadFromStackSlot(MI, FrameIndex, MemBytes);
1749 }
1750
1751 Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex,
1752 TypeSize &MemBytes) const override;
1753
1754 Register isStoreToStackSlot(const MachineInstr &MI,
1755 int &FrameIndex) const override {
1756 TypeSize MemBytes = TypeSize::getZero();
1757 return isStoreToStackSlot(MI, FrameIndex, MemBytes);
1758 }
1759
1760 Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex,
1761 TypeSize &MemBytes) const override;
1762
1763 unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
1764
1765 InstSizeVerifyMode
1766 getInstSizeVerifyMode(const MachineInstr &MI) const override;
1767
1768 std::pair<unsigned, unsigned>
1769 decomposeMachineOperandsTargetFlags(unsigned TF) const override;
1770
1771 ArrayRef<std::pair<int, const char *>>
1772 getSerializableTargetIndices() const override;
1773
1774 ArrayRef<std::pair<unsigned, const char *>>
1775 getSerializableDirectMachineOperandTargetFlags() const override;
1776
1777 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
1778 getSerializableMachineMemOperandTargetFlags() const override;
1779
1780 ScheduleHazardRecognizer *
1781 CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
1782 const ScheduleDAG *DAG) const override;
1783
1784 ScheduleHazardRecognizer *
1785 CreateTargetPostRAHazardRecognizer(const MachineFunction &MF,
1786 MachineLoopInfo *MLI) const override;
1787
1788 ScheduleHazardRecognizer *
1789 CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
1790 const ScheduleDAGMI *DAG) const override;
1791
1792 unsigned getLiveRangeSplitOpcode(Register Reg,
1793 const MachineFunction &MF) const override;
1794
1795 bool isBasicBlockPrologue(const MachineInstr &MI,
1796 Register Reg = Register()) const override;
1797
1798 bool canAddToBBProlog(const MachineInstr &MI) const;
1799
1800 MachineInstr *createPHIDestinationCopy(MachineBasicBlock &MBB,
1801 MachineBasicBlock::iterator InsPt,
1802 const DebugLoc &DL, Register Src,
1803 Register Dst) const override;
1804
1805 MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB,
1806 MachineBasicBlock::iterator InsPt,
1807 const DebugLoc &DL, Register Src,
1808 unsigned SrcSubReg,
1809 Register Dst) const override;
1810
1811 bool isWave32() const;
1812
1813 bool isVOPDAntidependencyAllowed(const MachineInstr &MI) const;
1814
1815 bool hasRAWDependency(const MachineInstr &FirstMI,
1816 const MachineInstr &SecondMI) const;
1817
1818 /// Return a partially built integer add instruction without carry.
1819 /// Caller must add source operands.
1820 /// For pre-GFX9 it will generate unused carry destination operand.
1821 /// TODO: After GFX9 it should return a no-carry operation.
1822 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1823 MachineBasicBlock::iterator I,
1824 const DebugLoc &DL,
1825 Register DestReg) const;
1826
1827 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1828 MachineBasicBlock::iterator I,
1829 const DebugLoc &DL,
1830 Register DestReg,
1831 RegScavenger &RS) const;
1832
1833 static bool isKillTerminator(unsigned Opcode);
1834 const MCInstrDesc &getKillTerminatorFromPseudo(unsigned Opcode) const;
1835
1836 bool isLegalMUBUFImmOffset(unsigned Imm) const;
1837
1838 static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST);
1839
1840 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
1841 Align Alignment = Align(4)) const;
1842
1843 /// Returns if \p Offset is legal for the subtarget as the offset to a FLAT
1844 /// encoded instruction with the given \p FlatVariant.
1845 bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
1846 AMDGPU::FlatAddrSpace FlatVariant) const;
1847
1848 /// Split \p COffsetVal into {immediate offset field, remainder offset}
1849 /// values.
1850 std::pair<int64_t, int64_t>
1851 splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,
1852 AMDGPU::FlatAddrSpace FlatVariant) const;
1853
1854 /// Returns true if negative offsets are allowed for the given \p FlatVariant.
1855 bool allowNegativeFlatOffset(AMDGPU::FlatAddrSpace FlatVariant) const;
1856
1857 /// \brief Return a target-specific opcode if Opcode is a pseudo instruction.
1858 /// Return -1 if the target-specific opcode for the pseudo instruction does
1859 /// not exist. If Opcode is not a pseudo instruction, this is identity.
1860 int pseudoToMCOpcode(int Opcode) const;
1861
1862 /// \brief Check if this instruction should only be used by assembler.
1863 /// Return true if this opcode should not be used by codegen.
1864 bool isAsmOnlyOpcode(int MCOp) const;
1865
1866 void fixImplicitOperands(MachineInstr &MI) const;
1867
1868 MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
1869 ArrayRef<unsigned> Ops, int FrameIndex,
1870 MachineInstr *&CopyMI,
1871 LiveIntervals *LIS = nullptr,
1872 VirtRegMap *VRM = nullptr) const override;
1873
1874 unsigned getInstrLatency(const InstrItineraryData *ItinData,
1875 const MachineInstr &MI,
1876 unsigned *PredCost = nullptr) const override;
1877
1878 unsigned getBlockingCycles(const MachineInstr &MI) const;
1879
1880 /// GFX1250 blocking-cycles table lookup with no occupancy subtarget gate.
1881 /// Returns 0 if \p MI is not in the table. Used as a multi-pass VALU denylist
1882 /// (e.g. V_PERM_PK16 hazard) on both gfx1250 and gfx1251.
1883 unsigned getGFX1250BlockingCyclesTable(const MachineInstr &MI) const;
1884
1885 const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override;
1886
1887 ValueUniformity getValueUniformity(const MachineInstr &MI) const final;
1888
1889 ValueUniformity getGenericValueUniformity(const MachineInstr &MI) const;
1890
1891 const MIRFormatter *getMIRFormatter() const override;
1892
1893 static unsigned getDSShaderTypeValue(const MachineFunction &MF);
1894
1895 const TargetSchedModel &getSchedModel() const { return SchedModel; }
1896
1897 void createReadFirstLaneFromCopyToPhysReg(MachineRegisterInfo &MRI,
1898 Register DstReg,
1899 MachineInstr &Inst) const;
1900
1901 void handleCopyToPhysHelper(
1902 SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst,
1903 MachineRegisterInfo &MRI, MachineDominatorTree *MDT,
1904 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
1905 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const;
1906
1907 // FIXME: This should be removed
1908 // Enforce operand's \p OpName even alignment if required by target.
1909 // This is used if an operand is a 32 bit register but needs to be aligned
1910 // regardless.
1911 void enforceOperandRCAlignment(MachineInstr &MI, AMDGPU::OpName OpName) const;
1912
1913 /// Get the repeat rate for a VALU instruction from the scheduling model.
1914 /// Returns 1 for regular VALU, >1 for long-latency VALU (packed, F64, etc.)
1915 unsigned getRepeatRate(const MachineInstr &MI) const;
1916};
1917
1918/// \brief Returns true if a reg:subreg pair P has a TRC class
1919inline bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P,
1920 const TargetRegisterClass &TRC,
1921 MachineRegisterInfo &MRI) {
1922 auto *RC = MRI.getRegClass(Reg: P.Reg);
1923 if (!P.SubReg)
1924 return RC == &TRC;
1925 auto *TRI = MRI.getTargetRegisterInfo();
1926 return RC == TRI->getMatchingSuperRegClass(A: RC, B: &TRC, Idx: P.SubReg);
1927}
1928
1929/// \brief Create RegSubRegPair from a register MachineOperand
1930inline
1931TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O) {
1932 assert(O.isReg());
1933 return TargetInstrInfo::RegSubRegPair(O.getReg(), O.getSubReg());
1934}
1935
1936/// \brief Return the SubReg component from REG_SEQUENCE
1937TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI,
1938 unsigned SubReg);
1939
1940/// \brief Return the defining instruction for a given reg:subreg pair
1941/// skipping copy like instructions and subreg-manipulation pseudos.
1942/// Following another subreg of a reg:subreg isn't supported.
1943MachineInstr *getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
1944 const MachineRegisterInfo &MRI);
1945
1946/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1947/// DefMI and the use at \p UseMI. Should be run on SSA. Currently does not
1948/// attempt to track between blocks.
1949bool execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
1950 Register VReg,
1951 const MachineInstr &DefMI,
1952 const MachineInstr &UseMI);
1953
1954/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1955/// DefMI and all its uses. Should be run on SSA. Currently does not attempt to
1956/// track between blocks.
1957bool execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
1958 Register VReg,
1959 const MachineInstr &DefMI);
1960
1961namespace AMDGPU {
1962
1963 LLVM_READONLY
1964 int32_t getVOPe64(uint32_t Opcode);
1965
1966 LLVM_READONLY
1967 int32_t getVOPe32(uint32_t Opcode);
1968
1969 LLVM_READONLY
1970 int32_t getSDWAOp(uint32_t Opcode);
1971
1972 LLVM_READONLY
1973 int32_t getDPPOp32(uint32_t Opcode);
1974
1975 LLVM_READONLY
1976 int32_t getDPPOp64(uint32_t Opcode);
1977
1978 LLVM_READONLY
1979 int32_t getBasicFromSDWAOp(uint32_t Opcode);
1980
1981 LLVM_READONLY
1982 int32_t getCommuteRev(uint32_t Opcode);
1983
1984 LLVM_READONLY
1985 int32_t getCommuteOrig(uint32_t Opcode);
1986
1987 LLVM_READONLY
1988 int32_t getAddr64Inst(uint32_t Opcode);
1989
1990 /// Check if \p Opcode is an Addr64 opcode.
1991 ///
1992 /// \returns \p Opcode if it is an Addr64 opcode, otherwise -1.
1993 LLVM_READONLY
1994 int32_t getIfAddr64Inst(uint32_t Opcode);
1995
1996 LLVM_READONLY
1997 int32_t getSOPKOp(uint32_t Opcode);
1998
1999 /// \returns SADDR form of a FLAT Global instruction given an \p Opcode
2000 /// of a VADDR form.
2001 LLVM_READONLY
2002 int32_t getGlobalSaddrOp(uint32_t Opcode);
2003
2004 /// \returns VADDR form of a FLAT Global instruction given an \p Opcode
2005 /// of a SADDR form.
2006 LLVM_READONLY
2007 int32_t getGlobalVaddrOp(uint32_t Opcode);
2008
2009 /// \returns ST form with only immediate offset of a FLAT Scratch instruction
2010 /// given an \p Opcode of an SS (SADDR) form.
2011 LLVM_READONLY
2012 int32_t getFlatScratchInstSTfromSS(uint32_t Opcode);
2013
2014 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
2015 /// of an SVS (SADDR + VADDR) form.
2016 LLVM_READONLY
2017 int32_t getFlatScratchInstSVfromSVS(uint32_t Opcode);
2018
2019 /// \returns SS (SADDR) form of a FLAT Scratch instruction given an \p Opcode
2020 /// of an SV (VADDR) form.
2021 LLVM_READONLY
2022 int32_t getFlatScratchInstSSfromSV(uint32_t Opcode);
2023
2024 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
2025 /// of an SS (SADDR) form.
2026 LLVM_READONLY
2027 int32_t getFlatScratchInstSVfromSS(uint32_t Opcode);
2028
2029 /// \returns earlyclobber version of a MAC MFMA is exists.
2030 LLVM_READONLY
2031 int32_t getMFMAEarlyClobberOp(uint32_t Opcode);
2032
2033 /// \returns Version of an instruction which uses AGPRs for coupled operands
2034 /// given an \p Opcode which uses VGPRs for coupled operands.
2035 LLVM_READONLY
2036 int32_t getAGPRFormOp(uint32_t Opcode);
2037
2038 /// \returns v_cmpx version of a v_cmp instruction.
2039 LLVM_READONLY
2040 int32_t getVCMPXOpFromVCMP(uint32_t Opcode);
2041
2042 const uint64_t RSRC_DATA_FORMAT = 0xf00000000000LL;
2043 const uint64_t RSRC_ELEMENT_SIZE_SHIFT = (32 + 19);
2044 const uint64_t RSRC_INDEX_STRIDE_SHIFT = (32 + 21);
2045 const uint64_t RSRC_TID_ENABLE = UINT64_C(1) << (32 + 23);
2046
2047} // end namespace AMDGPU
2048
2049namespace AMDGPU {
2050enum AsmComments : MachineInstr::AsmPrinterFlagTy {
2051 // For sgpr to vgpr spill instructions
2052 SGPR_SPILL = MachineInstr::TAsmComments
2053};
2054} // namespace AMDGPU
2055
2056namespace SI {
2057namespace KernelInputOffsets {
2058
2059/// Offsets in bytes from the start of the input buffer
2060enum Offsets {
2061 NGROUPS_X = 0,
2062 NGROUPS_Y = 4,
2063 NGROUPS_Z = 8,
2064 GLOBAL_SIZE_X = 12,
2065 GLOBAL_SIZE_Y = 16,
2066 GLOBAL_SIZE_Z = 20,
2067 LOCAL_SIZE_X = 24,
2068 LOCAL_SIZE_Y = 28,
2069 LOCAL_SIZE_Z = 32
2070};
2071
2072} // end namespace KernelInputOffsets
2073} // end namespace SI
2074
2075} // end namespace llvm
2076
2077#endif // LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
2078