1//===- SIInstrInfo.h - SI Instruction Info Interface ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Interface definition for SIInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
15#define LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
16
17#include "AMDGPUMIRFormatter.h"
18#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
19#include "SIRegisterInfo.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/CodeGen/TargetInstrInfo.h"
24#include "llvm/CodeGen/TargetSchedule.h"
25
26#define GET_INSTRINFO_HEADER
27#include "AMDGPUGenInstrInfo.inc"
28
29namespace llvm {
30
31class APInt;
32class GCNSubtarget;
33class MachineDominatorTree;
34class MachineRegisterInfo;
35class RegScavenger;
36class SIMachineFunctionInfo;
37class MCRegisterClass;
38using TargetRegisterClass = MCRegisterClass;
39class ScheduleHazardRecognizer;
40
41constexpr unsigned DefaultMemoryClusterDWordsLimit = 8;
42
43/// Mark the MMO of a uniform load if there are no potentially clobbering stores
44/// on any path from the start of an entry function to this load.
45static const MachineMemOperand::Flags MONoClobber =
46 MachineMemOperand::MOTargetFlag1;
47
48/// Mark the MMO of a load as the last use.
49static const MachineMemOperand::Flags MOLastUse =
50 MachineMemOperand::MOTargetFlag2;
51
52/// Mark the MMO of cooperative load/store atomics.
53static const MachineMemOperand::Flags MOCooperative =
54 MachineMemOperand::MOTargetFlag3;
55
56struct V2PhysSCopyInfo {
57 // Operands that need to replaced by waterfall
58 SmallVector<MachineOperand *> MOs;
59 // Target physical registers replacing the MOs
60 SmallVector<Register> SGPRs;
61};
62/// Mark the MMO of accesses to memory locations that are
63/// never written to by other threads.
64static const MachineMemOperand::Flags MOThreadPrivate =
65 MachineMemOperand::MOTargetFlag4;
66
67/// Utility to store machine instructions worklist.
68struct SIInstrWorklist {
69 SIInstrWorklist() = default;
70
71 void insert(MachineInstr *MI);
72
73 MachineInstr *top() const { return InstrList[Front]; }
74
75 void erase_top() {
76 InSet.erase(Ptr: InstrList[Front]);
77 ++Front;
78 }
79
80 bool empty() const { return Front == InstrList.size(); }
81
82 void clear() {
83 InstrList.clear();
84 Front = 0;
85 InSet.clear();
86 DeferredList.clear();
87 }
88
89 bool isDeferred(MachineInstr *MI);
90
91 SetVector<MachineInstr *> &getDeferredList() { return DeferredList; }
92
93private:
94 /// InstrList contains the MachineInstrs.
95 SmallVector<MachineInstr *> InstrList;
96 SmallPtrSet<MachineInstr *, 8> InSet;
97 unsigned Front = 0;
98 /// Deferred instructions are specific MachineInstr
99 /// that will be added by insert method.
100 SetVector<MachineInstr *> DeferredList;
101};
102
103// In namespace llvm so ADL finds it when SIInstrFlags predicates are
104// instantiated with MachineInstr (MachineInstr is in namespace llvm).
105inline uint64_t getTSFlags(const MachineInstr &MI) {
106 return MI.getDesc().TSFlags;
107}
108
109class SIInstrInfo final : public AMDGPUGenInstrInfo {
110 struct ThreeAddressUpdates;
111
112private:
113 const SIRegisterInfo RI;
114 const GCNSubtarget &ST;
115 TargetSchedModel SchedModel;
116 mutable std::unique_ptr<AMDGPUMIRFormatter> Formatter;
117
118 // The inverse predicate should have the negative value.
119 enum BranchPredicate {
120 INVALID_BR = 0,
121 SCC_TRUE = 1,
122 SCC_FALSE = -1,
123 VCCNZ = 2,
124 VCCZ = -2,
125 EXECNZ = -3,
126 EXECZ = 3
127 };
128
129 using SetVectorType = SmallSetVector<MachineInstr *, 32>;
130
131 static unsigned getBranchOpcode(BranchPredicate Cond);
132 static BranchPredicate getBranchPredicate(unsigned Opcode);
133
134public:
135 unsigned buildExtractSubReg(MachineBasicBlock::iterator MI,
136 MachineRegisterInfo &MRI,
137 const MachineOperand &SuperReg,
138 const TargetRegisterClass *SuperRC,
139 unsigned SubIdx,
140 const TargetRegisterClass *SubRC) const;
141 MachineOperand buildExtractSubRegOrImm(
142 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
143 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
144 unsigned SubIdx, const TargetRegisterClass *SubRC) const;
145
146private:
147 bool optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,
148 bool NeedInversion) const;
149
150 bool invertSCCUse(MachineInstr *SCCDef) const;
151
152 void swapOperands(MachineInstr &Inst) const;
153
154 std::pair<bool, MachineBasicBlock *>
155 moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
156 MachineDominatorTree *MDT = nullptr) const;
157
158 void lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
159 MachineDominatorTree *MDT = nullptr) const;
160
161 void lowerScalarAbs(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
162
163 void lowerScalarAbsDiff(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
164
165 void lowerScalarXnor(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
166
167 void splitScalarNotBinop(SIInstrWorklist &Worklist, MachineInstr &Inst,
168 unsigned Opcode) const;
169
170 void splitScalarBinOpN2(SIInstrWorklist &Worklist, MachineInstr &Inst,
171 unsigned Opcode) const;
172
173 void splitScalar64BitUnaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
174 unsigned Opcode, bool Swap = false) const;
175
176 void splitScalar64BitBinaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
177 unsigned Opcode,
178 MachineDominatorTree *MDT = nullptr) const;
179
180 void splitScalarSMulU64(SIInstrWorklist &Worklist, MachineInstr &Inst,
181 MachineDominatorTree *MDT) const;
182
183 void splitScalarSMulPseudo(SIInstrWorklist &Worklist, MachineInstr &Inst,
184 MachineDominatorTree *MDT) const;
185
186 void splitScalar64BitXnor(SIInstrWorklist &Worklist, MachineInstr &Inst,
187 MachineDominatorTree *MDT = nullptr) const;
188
189 void splitScalar64BitBCNT(SIInstrWorklist &Worklist,
190 MachineInstr &Inst) const;
191 void splitScalar64BitBFE(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
192 void splitScalar64BitCountOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
193 unsigned Opcode,
194 MachineDominatorTree *MDT = nullptr) const;
195 void movePackToVALU(SIInstrWorklist &Worklist, MachineRegisterInfo &MRI,
196 MachineInstr &Inst) const;
197
198 void addUsersToMoveToVALUWorklist(Register Reg, MachineRegisterInfo &MRI,
199 SIInstrWorklist &Worklist) const;
200
201 void addSCCDefUsersToVALUWorklist(const MachineOperand &Op,
202 MachineInstr &SCCDefInst,
203 SIInstrWorklist &Worklist,
204 Register NewCond = Register()) const;
205 void addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
206 SIInstrWorklist &Worklist) const;
207
208 const TargetRegisterClass *
209 getDestEquivalentVGPRClass(const MachineInstr &Inst) const;
210
211 bool checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
212 const MachineInstr &MIb) const;
213
214 Register findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const;
215
216 bool verifyCopy(const MachineInstr &MI, const MachineRegisterInfo &MRI,
217 StringRef &ErrInfo) const;
218
219 bool resultDependsOnExec(const MachineInstr &MI) const;
220
221 MachineInstr *convertToThreeAddressImpl(MachineInstr &MI,
222 ThreeAddressUpdates &Updates) const;
223
224protected:
225 /// If the specific machine instruction is a instruction that moves/copies
226 /// value from one register to another register return destination and source
227 /// registers as machine operands.
228 std::optional<DestSourcePair>
229 isCopyInstrImpl(const MachineInstr &MI) const override;
230
231 bool swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,
232 AMDGPU::OpName Src0OpName, MachineOperand &Src1,
233 AMDGPU::OpName Src1OpName) const;
234 bool isNonCommutableDPP(const MachineInstr &MI) const;
235 MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
236 unsigned OpIdx0,
237 unsigned OpIdx1) const override;
238
239public:
240 bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx,
241 unsigned toIdx) const;
242 enum TargetOperandFlags {
243 MO_MASK = 0xf,
244
245 MO_NONE = 0,
246 // MO_GOTPCREL -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
247 MO_GOTPCREL = 1,
248 // MO_GOTPCREL32_LO -> symbol@gotpcrel32@lo -> R_AMDGPU_GOTPCREL32_LO.
249 MO_GOTPCREL32 = 2,
250 MO_GOTPCREL32_LO = 2,
251 // MO_GOTPCREL32_HI -> symbol@gotpcrel32@hi -> R_AMDGPU_GOTPCREL32_HI.
252 MO_GOTPCREL32_HI = 3,
253 // MO_GOTPCREL64 -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
254 MO_GOTPCREL64 = 4,
255 // MO_REL32_LO -> symbol@rel32@lo -> R_AMDGPU_REL32_LO.
256 MO_REL32 = 5,
257 MO_REL32_LO = 5,
258 // MO_REL32_HI -> symbol@rel32@hi -> R_AMDGPU_REL32_HI.
259 MO_REL32_HI = 6,
260 MO_REL64 = 7,
261
262 MO_FAR_BRANCH_OFFSET = 8,
263
264 MO_ABS32_LO = 9,
265 MO_ABS32_HI = 10,
266 MO_ABS64 = 11,
267 };
268
269 explicit SIInstrInfo(const GCNSubtarget &ST);
270
271 const SIRegisterInfo &getRegisterInfo() const {
272 return RI;
273 }
274
275 // FIXME: This is inaccurate and needs to account for use context. Normal asm
276 // constraints should use 64-bit pointers.
277 const TargetRegisterClass *getInlineAsmMemoryOperandRegClass(
278 InlineAsm::ConstraintCode C) const override {
279 return &AMDGPU::VGPR_32RegClass;
280 }
281
282 const GCNSubtarget &getSubtarget() const {
283 return ST;
284 }
285
286 bool isReMaterializableImpl(const MachineInstr &MI) const override;
287
288 bool isIgnorableUse(const MachineInstr &MI, unsigned OpIdx) const override;
289
290 bool isSafeToSink(MachineInstr &MI, MachineBasicBlock *SuccToSinkTo,
291 MachineCycleInfo *CI) const override;
292
293 bool areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, int64_t &Offset0,
294 int64_t &Offset1) const override;
295
296 bool isGlobalMemoryObject(const MachineInstr *MI) const override;
297
298 bool getMemOperandsWithOffsetWidth(
299 const MachineInstr &LdSt,
300 SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
301 bool &OffsetIsScalable, LocationSize &Width) const final;
302
303 bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
304 int64_t Offset1, bool OffsetIsScalable1,
305 ArrayRef<const MachineOperand *> BaseOps2,
306 int64_t Offset2, bool OffsetIsScalable2,
307 unsigned ClusterSize,
308 unsigned NumBytes) const override;
309
310 /// Return true if the memory operations described by \p BaseOps1 and
311 /// \p BaseOps2 (as returned by getMemOperandsWithOffsetWidth) address the
312 /// same base. Unlike shouldClusterMemOps, this does not apply the
313 /// MaxMemoryClusterDWords budget, which only bounds pre-RA register pressure.
314 /// Only the first base operand is compared, unlike
315 /// memOpsHaveSameBaseOperands which requires all of them to match.
316 bool memOpsHaveSameBase(ArrayRef<const MachineOperand *> BaseOps1,
317 ArrayRef<const MachineOperand *> BaseOps2) const;
318
319 bool shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0,
320 int64_t Offset1, unsigned NumLoads) const override;
321
322 void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
323 const DebugLoc &DL, Register DestReg, Register SrcReg,
324 bool KillSrc, bool RenamableDest = false,
325 bool RenamableSrc = false) const override;
326
327private:
328 void storeRegToStackSlotImpl(MachineBasicBlock &MBB,
329 MachineBasicBlock::iterator MI, Register SrcReg,
330 bool isKill, int FrameIndex,
331 const TargetRegisterClass *RC, Register VReg,
332 MachineInstr::MIFlag Flags, bool NeedsCFI) const;
333
334public:
335 void storeRegToStackSlotCFI(MachineBasicBlock &MBB,
336 MachineBasicBlock::iterator MI, Register SrcReg,
337 bool isKill, int FrameIndex,
338 const TargetRegisterClass *RC) const;
339
340 bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg,
341 int64_t &ImmVal) const override;
342
343 std::optional<int64_t>
344 getImmOrMaterializedImm(const MachineRegisterInfo &MRI,
345 const MachineOperand &Op,
346 MachineInstr **DefMI = nullptr) const;
347 std::optional<int64_t>
348 getImmOrMaterializedImm(const MachineRegisterInfo &MRI, Register Reg,
349 MachineInstr **DefMI = nullptr) const;
350
351 unsigned getVectorRegSpillSaveOpcode(Register Reg,
352 const TargetRegisterClass *RC,
353 unsigned Size,
354 const SIMachineFunctionInfo &MFI,
355 bool NeedsCFI) const;
356 unsigned
357 getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC,
358 unsigned Size,
359 const SIMachineFunctionInfo &MFI) const;
360
361 void storeRegToStackSlot(
362 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
363 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
364 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
365
366 void loadRegFromStackSlot(
367 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
368 int FrameIndex, const TargetRegisterClass *RC, Register VReg,
369 unsigned SubReg = 0,
370 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
371
372 bool expandPostRAPseudo(MachineInstr &MI) const override;
373
374 void
375 reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
376 Register DestReg, unsigned SubIdx, const MachineInstr &Orig,
377 LaneBitmask UsedLanes = LaneBitmask::getAll()) const override;
378
379 // Splits a V_MOV_B64_DPP_PSEUDO opcode into a pair of v_mov_b32_dpp
380 // instructions. Returns a pair of generated instructions.
381 // Can split either post-RA with physical registers or pre-RA with
382 // virtual registers. In latter case IR needs to be in SSA form and
383 // and a REG_SEQUENCE is produced to define original register.
384 std::pair<MachineInstr*, MachineInstr*>
385 expandMovDPP64(MachineInstr &MI) const;
386
387 // Returns an opcode that can be used to move a value to a \p DstRC
388 // register. If there is no hardware instruction that can store to \p
389 // DstRC, then AMDGPU::COPY is returned.
390 unsigned getMovOpcode(const TargetRegisterClass *DstRC) const;
391
392 const MCInstrDesc &getIndirectRegWriteMovRelPseudo(unsigned VecSize,
393 unsigned EltSize,
394 bool IsSGPR) const;
395
396 const MCInstrDesc &getIndirectGPRIDXPseudo(unsigned VecSize,
397 bool IsIndirectSrc) const;
398 LLVM_READONLY
399 int commuteOpcode(unsigned Opc) const;
400
401 LLVM_READONLY
402 inline int commuteOpcode(const MachineInstr &MI) const {
403 return commuteOpcode(Opc: MI.getOpcode());
404 }
405
406 bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0,
407 unsigned &SrcOpIdx1) const override;
408
409 bool findCommutedOpIndices(const MCInstrDesc &Desc, unsigned &SrcOpIdx0,
410 unsigned &SrcOpIdx1) const;
411
412 bool isBranchOffsetInRange(unsigned BranchOpc,
413 int64_t BrOffset) const override;
414
415 MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;
416
417 /// Return whether the block terminate with divergent branch.
418 /// Note this only work before lowering the pseudo control flow instructions.
419 bool hasDivergentBranch(const MachineBasicBlock *MBB) const;
420
421 void insertIndirectBranch(MachineBasicBlock &MBB,
422 MachineBasicBlock &NewDestBB,
423 MachineBasicBlock &RestoreBB, const DebugLoc &DL,
424 int64_t BrOffset, RegScavenger *RS) const override;
425
426 bool analyzeBranchImpl(MachineBasicBlock &MBB,
427 MachineBasicBlock::iterator I,
428 MachineBasicBlock *&TBB,
429 MachineBasicBlock *&FBB,
430 SmallVectorImpl<MachineOperand> &Cond,
431 bool AllowModify) const;
432
433 bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
434 MachineBasicBlock *&FBB,
435 SmallVectorImpl<MachineOperand> &Cond,
436 bool AllowModify = false) const override;
437
438 unsigned removeBranch(MachineBasicBlock &MBB,
439 int *BytesRemoved = nullptr) const override;
440
441 unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
442 MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
443 const DebugLoc &DL,
444 int *BytesAdded = nullptr) const override;
445
446 bool reverseBranchCondition(
447 SmallVectorImpl<MachineOperand> &Cond) const override;
448
449 std::unique_ptr<PipelinerLoopInfo>
450 analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
451
452 bool canInsertSelect(const MachineBasicBlock &MBB,
453 ArrayRef<MachineOperand> Cond, Register DstReg,
454 Register TrueReg, Register FalseReg,
455 SelectExpansion &Exp) const override;
456
457 void insertSelect(MachineBasicBlock &MBB,
458 MachineBasicBlock::iterator I, const DebugLoc &DL,
459 Register DstReg, ArrayRef<MachineOperand> Cond,
460 Register TrueReg, Register FalseReg) const override;
461
462 bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
463 Register &SrcReg2, int64_t &CmpMask,
464 int64_t &CmpValue) const override;
465
466 bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
467 Register SrcReg2, int64_t CmpMask, int64_t CmpValue,
468 const MachineRegisterInfo *MRI) const override;
469
470 bool
471 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
472 const MachineInstr &MIb) const override;
473
474 static bool isFoldableCopy(const MachineInstr &MI);
475 static unsigned getFoldableCopySrcIdx(const MachineInstr &MI);
476
477 void removeModOperands(MachineInstr &MI) const;
478
479 void mutateAndCleanupImplicit(MachineInstr &MI,
480 const MCInstrDesc &NewDesc) const;
481
482 /// Return the extracted immediate value in a subregister use from a constant
483 /// materialized in a super register.
484 ///
485 /// e.g. %imm = S_MOV_B64 K[0:63]
486 /// USE %imm.sub1
487 /// This will return K[32:63]
488 static std::optional<int64_t> extractSubregFromImm(int64_t ImmVal,
489 unsigned SubRegIndex);
490
491 bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
492 MachineRegisterInfo *MRI) const final;
493
494 unsigned getMachineCSELookAheadLimit() const override { return 500; }
495
496 MachineInstr *convertToThreeAddress(MachineInstr &MI,
497 LiveIntervals *LIS) const override;
498
499 bool isSchedulingBoundary(const MachineInstr &MI,
500 const MachineBasicBlock *MBB,
501 const MachineFunction &MF) const override;
502
503 static bool isSALU(const MachineInstr &MI) {
504 return SIInstrFlags::isSALU(O: MI);
505 }
506
507 bool isSALU(uint32_t Opcode) const {
508 return SIInstrFlags::isSALU(O: get(Opcode));
509 }
510
511 static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA) {
512 if (!AllowLDSDMA && isLDSDMA(MI))
513 return false;
514
515 return SIInstrFlags::isVALU(O: MI);
516 }
517
518 /// LDSDMA instructions act as both VALU and memory instructions, thus
519 /// we also tag them as VALU. However, in many places, we do not actually want
520 /// to include LDSDMA instructions in this query. By setting \p AllowLDSDMA to
521 /// false, this will return false for LDSDMA instructions.
522 bool isVALU(uint32_t Opcode, bool AllowLDSDMA) const {
523 if (!AllowLDSDMA && isLDSDMA(Opcode))
524 return false;
525
526 return SIInstrFlags::isVALU(O: get(Opcode));
527 }
528
529 static bool isImage(const MachineInstr &MI) {
530 return SIInstrFlags::isImage(O: MI);
531 }
532
533 bool isImage(uint32_t Opcode) const {
534 return SIInstrFlags::isImage(O: get(Opcode));
535 }
536
537 static bool isVMEM(const MachineInstr &MI) {
538 return SIInstrFlags::isVMEM(O: MI);
539 }
540
541 bool isVMEM(uint32_t Opcode) const {
542 return SIInstrFlags::isVMEM(O: get(Opcode));
543 }
544
545 /// True if MI implicitly drains XCNT.
546 static bool isXcntDrain(const MachineInstr &MI);
547
548 static bool isSOP1(const MachineInstr &MI) {
549 return SIInstrFlags::isSOP1(O: MI);
550 }
551
552 bool isSOP1(uint32_t Opcode) const {
553 return SIInstrFlags::isSOP1(O: get(Opcode));
554 }
555
556 static bool isSOP2(const MachineInstr &MI) {
557 return SIInstrFlags::isSOP2(O: MI);
558 }
559
560 bool isSOP2(uint32_t Opcode) const {
561 return SIInstrFlags::isSOP2(O: get(Opcode));
562 }
563
564 static bool isSOPC(const MachineInstr &MI) {
565 return SIInstrFlags::isSOPC(O: MI);
566 }
567
568 bool isSOPC(uint32_t Opcode) const {
569 return SIInstrFlags::isSOPC(O: get(Opcode));
570 }
571
572 static bool isSOPK(const MachineInstr &MI) {
573 return SIInstrFlags::isSOPK(O: MI);
574 }
575
576 bool isSOPK(uint32_t Opcode) const {
577 return SIInstrFlags::isSOPK(O: get(Opcode));
578 }
579
580 static bool isSOPP(const MachineInstr &MI) {
581 return SIInstrFlags::isSOPP(O: MI);
582 }
583
584 bool isSOPP(uint32_t Opcode) const {
585 return SIInstrFlags::isSOPP(O: get(Opcode));
586 }
587
588 static bool isPacked(const MachineInstr &MI) {
589 return SIInstrFlags::isPacked(O: MI);
590 }
591
592 bool isPacked(uint32_t Opcode) const {
593 return SIInstrFlags::isPacked(O: get(Opcode));
594 }
595
596 static bool isVOP1(const MachineInstr &MI) {
597 return SIInstrFlags::isVOP1(O: MI);
598 }
599
600 bool isVOP1(uint32_t Opcode) const {
601 return SIInstrFlags::isVOP1(O: get(Opcode));
602 }
603
604 static bool isVOP2(const MachineInstr &MI) {
605 return SIInstrFlags::isVOP2(O: MI);
606 }
607
608 bool isVOP2(uint32_t Opcode) const {
609 return SIInstrFlags::isVOP2(O: get(Opcode));
610 }
611
612 static bool isVOP3(const MCInstrDesc &Desc) {
613 return SIInstrFlags::isVOP3(O: Desc);
614 }
615
616 static bool isVOP3(const MachineInstr &MI) { return isVOP3(Desc: MI.getDesc()); }
617
618 bool isVOP3(uint32_t Opcode) const { return isVOP3(Desc: get(Opcode)); }
619
620 static bool isSDWA(const MachineInstr &MI) {
621 return SIInstrFlags::isSDWA(O: MI);
622 }
623
624 bool isSDWA(uint32_t Opcode) const {
625 return SIInstrFlags::isSDWA(O: get(Opcode));
626 }
627
628 static bool isVOPC(const MachineInstr &MI) {
629 return SIInstrFlags::isVOPC(O: MI);
630 }
631
632 bool isVOPC(uint32_t Opcode) const {
633 return SIInstrFlags::isVOPC(O: get(Opcode));
634 }
635
636 static bool isMUBUF(const MachineInstr &MI) {
637 return SIInstrFlags::isMUBUF(O: MI);
638 }
639
640 bool isMUBUF(uint32_t Opcode) const {
641 return SIInstrFlags::isMUBUF(O: get(Opcode));
642 }
643
644 static bool isMTBUF(const MachineInstr &MI) {
645 return SIInstrFlags::isMTBUF(O: MI);
646 }
647
648 bool isMTBUF(uint32_t Opcode) const {
649 return SIInstrFlags::isMTBUF(O: get(Opcode));
650 }
651
652 static bool isBUF(const MachineInstr &MI) {
653 return isMUBUF(MI) || isMTBUF(MI);
654 }
655
656 static bool isSMRD(const MachineInstr &MI) {
657 return SIInstrFlags::isSMRD(O: MI);
658 }
659
660 bool isSMRD(uint32_t Opcode) const {
661 return SIInstrFlags::isSMRD(O: get(Opcode));
662 }
663
664 bool isBufferSMRD(const MachineInstr &MI) const;
665
666 static bool isDS(const MachineInstr &MI) { return SIInstrFlags::isDS(O: MI); }
667
668 bool isDS(uint32_t Opcode) const { return SIInstrFlags::isDS(O: get(Opcode)); }
669
670 static bool isLDSDMA(const MachineInstr &MI) {
671 return (SIInstrFlags::isVALU(O: MI) && (isMUBUF(MI) || isFLAT(MI))) ||
672 SIInstrFlags::usesTENSOR_CNT(O: MI);
673 }
674
675 bool isLDSDMA(uint32_t Opcode) const {
676 return (SIInstrFlags::isVALU(O: get(Opcode)) &&
677 (isMUBUF(Opcode) || isFLAT(Opcode))) ||
678 SIInstrFlags::usesTENSOR_CNT(O: get(Opcode));
679 }
680
681 static bool isGWS(const MachineInstr &MI) { return SIInstrFlags::isGWS(O: MI); }
682
683 bool isGWS(uint32_t Opcode) const { return SIInstrFlags::isGWS(O: get(Opcode)); }
684
685 bool isAlwaysGDS(uint32_t Opcode) const;
686
687 static bool isMIMG(const MachineInstr &MI) {
688 return SIInstrFlags::isMIMG(O: MI);
689 }
690
691 bool isMIMG(uint32_t Opcode) const {
692 return SIInstrFlags::isMIMG(O: get(Opcode));
693 }
694
695 static bool isVIMAGE(const MachineInstr &MI) {
696 return SIInstrFlags::isVIMAGE(O: MI);
697 }
698
699 bool isVIMAGE(uint32_t Opcode) const {
700 return SIInstrFlags::isVIMAGE(O: get(Opcode));
701 }
702
703 static bool isVSAMPLE(const MachineInstr &MI) {
704 return SIInstrFlags::isVSAMPLE(O: MI);
705 }
706
707 bool isVSAMPLE(uint32_t Opcode) const {
708 return SIInstrFlags::isVSAMPLE(O: get(Opcode));
709 }
710
711 static bool isGather4(const MachineInstr &MI) {
712 return SIInstrFlags::isGather4(O: MI);
713 }
714
715 bool isGather4(uint32_t Opcode) const {
716 return SIInstrFlags::isGather4(O: get(Opcode));
717 }
718
719 static bool isFLAT(const MachineInstr &MI) {
720 return SIInstrFlags::isFLAT(O: MI);
721 }
722
723 // Is a FLAT encoded instruction which accesses a specific segment,
724 // i.e. global_* or scratch_*.
725 static bool isSegmentSpecificFLAT(const MachineInstr &MI) {
726 return SIInstrFlags::isSegmentSpecificFLAT(O: MI);
727 }
728
729 bool isSegmentSpecificFLAT(uint32_t Opcode) const {
730 return SIInstrFlags::isSegmentSpecificFLAT(O: get(Opcode));
731 }
732
733 static bool isFLATGlobal(const MachineInstr &MI) {
734 return SIInstrFlags::isFlatGlobal(O: MI);
735 }
736
737 bool isFLATGlobal(uint32_t Opcode) const {
738 return SIInstrFlags::isFlatGlobal(O: get(Opcode));
739 }
740
741 static bool isFLATScratch(const MachineInstr &MI) {
742 return SIInstrFlags::isFlatScratch(O: MI);
743 }
744
745 bool isFLATScratch(uint32_t Opcode) const {
746 return SIInstrFlags::isFlatScratch(O: get(Opcode));
747 }
748
749 // Any FLAT encoded instruction, including global_* and scratch_*.
750 bool isFLAT(uint32_t Opcode) const {
751 return SIInstrFlags::isFLAT(O: get(Opcode));
752 }
753
754 /// \returns true for SCRATCH_ instructions, or FLAT/BUF instructions unless
755 /// the MMOs do not include scratch.
756 /// Conservatively correct; will return true if \p MI cannot be proven
757 /// to not hit scratch.
758 bool mayAccessScratch(const MachineInstr &MI) const;
759
760 /// \returns true for FLAT instructions that can access VMEM.
761 bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const;
762
763 /// \returns true for FLAT instructions that can access LDS.
764 bool mayAccessLDSThroughFlat(const MachineInstr &MI, bool TgSplit) const;
765
766 static bool isBlockLoadStore(uint32_t Opcode) {
767 switch (Opcode) {
768 case AMDGPU::SI_BLOCK_SPILL_V1024_SAVE:
769 case AMDGPU::SI_BLOCK_SPILL_V1024_CFI_SAVE:
770 case AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE:
771 case AMDGPU::SCRATCH_STORE_BLOCK_SADDR:
772 case AMDGPU::SCRATCH_LOAD_BLOCK_SADDR:
773 case AMDGPU::SCRATCH_STORE_BLOCK_SVS:
774 case AMDGPU::SCRATCH_LOAD_BLOCK_SVS:
775 return true;
776 default:
777 return false;
778 }
779 }
780
781 static bool setsSCCIfResultIsNonZero(const MachineInstr &MI) {
782 switch (MI.getOpcode()) {
783 case AMDGPU::S_ABSDIFF_I32:
784 case AMDGPU::S_ABS_I32:
785 case AMDGPU::S_AND_B32:
786 case AMDGPU::S_AND_B64:
787 case AMDGPU::S_ANDN2_B32:
788 case AMDGPU::S_ANDN2_B64:
789 case AMDGPU::S_ASHR_I32:
790 case AMDGPU::S_ASHR_I64:
791 case AMDGPU::S_BCNT0_I32_B32:
792 case AMDGPU::S_BCNT0_I32_B64:
793 case AMDGPU::S_BCNT1_I32_B32:
794 case AMDGPU::S_BCNT1_I32_B64:
795 case AMDGPU::S_BFE_I32:
796 case AMDGPU::S_BFE_I64:
797 case AMDGPU::S_BFE_U32:
798 case AMDGPU::S_BFE_U64:
799 case AMDGPU::S_LSHL_B32:
800 case AMDGPU::S_LSHL_B64:
801 case AMDGPU::S_LSHR_B32:
802 case AMDGPU::S_LSHR_B64:
803 case AMDGPU::S_NAND_B32:
804 case AMDGPU::S_NAND_B64:
805 case AMDGPU::S_NOR_B32:
806 case AMDGPU::S_NOR_B64:
807 case AMDGPU::S_NOT_B32:
808 case AMDGPU::S_NOT_B64:
809 case AMDGPU::S_OR_B32:
810 case AMDGPU::S_OR_B64:
811 case AMDGPU::S_ORN2_B32:
812 case AMDGPU::S_ORN2_B64:
813 case AMDGPU::S_QUADMASK_B32:
814 case AMDGPU::S_QUADMASK_B64:
815 case AMDGPU::S_WQM_B32:
816 case AMDGPU::S_WQM_B64:
817 case AMDGPU::S_XNOR_B32:
818 case AMDGPU::S_XNOR_B64:
819 case AMDGPU::S_XOR_B32:
820 case AMDGPU::S_XOR_B64:
821 return true;
822 default:
823 return false;
824 }
825 }
826
827 static bool isEXP(const MachineInstr &MI) { return SIInstrFlags::isEXP(O: MI); }
828
829 static bool isDualSourceBlendEXP(const MachineInstr &MI) {
830 if (!isEXP(MI))
831 return false;
832 unsigned Target = MI.getOperand(i: 0).getImm();
833 return Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND0 ||
834 Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND1;
835 }
836
837 bool isEXP(uint32_t Opcode) const { return SIInstrFlags::isEXP(O: get(Opcode)); }
838
839 static bool isAtomicNoRet(const MachineInstr &MI) {
840 return SIInstrFlags::isAtomicNoRet(O: MI);
841 }
842
843 bool isAtomicNoRet(uint32_t Opcode) const {
844 return SIInstrFlags::isAtomicNoRet(O: get(Opcode));
845 }
846
847 static bool isAtomicRet(const MachineInstr &MI) {
848 return SIInstrFlags::isAtomicRet(O: MI);
849 }
850
851 bool isAtomicRet(uint32_t Opcode) const {
852 return SIInstrFlags::isAtomicRet(O: get(Opcode));
853 }
854
855 static bool isAtomic(const MachineInstr &MI) {
856 return SIInstrFlags::isAtomic(O: MI);
857 }
858
859 bool isAtomic(uint32_t Opcode) const {
860 return SIInstrFlags::isAtomic(O: get(Opcode));
861 }
862
863 static bool mayWriteLDSThroughDMA(const MachineInstr &MI) {
864 unsigned Opc = MI.getOpcode();
865 // Exclude instructions that read FROM LDS (not write to it)
866 return isLDSDMA(MI) && Opc != AMDGPU::BUFFER_STORE_LDS_DWORD &&
867 Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d2 &&
868 Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d4;
869 }
870
871 static bool isSBarrierSCCWrite(unsigned Opcode) {
872 return Opcode == AMDGPU::S_BARRIER_LEAVE ||
873 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM ||
874 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0;
875 }
876
877 static bool isCBranchVCCZRead(const MachineInstr &MI) {
878 unsigned Opc = MI.getOpcode();
879 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
880 !MI.getOperand(i: 1).isUndef();
881 }
882
883 static bool isWQM(const MachineInstr &MI) { return SIInstrFlags::isWQM(O: MI); }
884
885 bool isWQM(uint32_t Opcode) const { return SIInstrFlags::isWQM(O: get(Opcode)); }
886
887 static bool isDisableWQM(const MachineInstr &MI) {
888 return SIInstrFlags::isDisableWQM(O: MI);
889 }
890
891 bool isDisableWQM(uint32_t Opcode) const {
892 return SIInstrFlags::isDisableWQM(O: get(Opcode));
893 }
894
895 // SI_SPILL_S32_TO_VGPR and SI_RESTORE_S32_FROM_VGPR form a special case of
896 // SGPRs spilling to VGPRs which are SGPR spills but from VALU instructions
897 // therefore we need an explicit check for them since just checking if the
898 // Spill bit is set and what instruction type it came from misclassifies
899 // them.
900 static bool isVGPRSpill(const MachineInstr &MI) {
901 return MI.getOpcode() != AMDGPU::SI_SPILL_S32_TO_VGPR &&
902 MI.getOpcode() != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
903 (isSpill(MI) && isVALU(MI, /*AllowLDSDMA=*/AllowLDSDMA: false));
904 }
905
906 bool isVGPRSpill(uint32_t Opcode) const {
907 return Opcode != AMDGPU::SI_SPILL_S32_TO_VGPR &&
908 Opcode != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
909 (isSpill(Opcode) && isVALU(Opcode, /*AllowLDSDMA=*/AllowLDSDMA: false));
910 }
911
912 static bool isSGPRSpill(const MachineInstr &MI) {
913 return MI.getOpcode() == AMDGPU::SI_SPILL_S32_TO_VGPR ||
914 MI.getOpcode() == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
915 (isSpill(MI) && isSALU(MI));
916 }
917
918 bool isSGPRSpill(uint32_t Opcode) const {
919 return Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR ||
920 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
921 (isSpill(Opcode) && isSALU(Opcode));
922 }
923
924 bool isSpill(uint32_t Opcode) const {
925 return SIInstrFlags::isSpill(O: get(Opcode));
926 }
927
928 static bool isSpill(const MCInstrDesc &Desc) {
929 return SIInstrFlags::isSpill(O: Desc);
930 }
931
932 static bool isSpill(const MachineInstr &MI) { return isSpill(Desc: MI.getDesc()); }
933
934 static bool isWWMRegSpillOpcode(uint32_t Opcode) {
935 return Opcode == AMDGPU::SI_SPILL_WWM_V32_SAVE ||
936 Opcode == AMDGPU::SI_SPILL_WWM_AV32_SAVE ||
937 Opcode == AMDGPU::SI_SPILL_WWM_V32_RESTORE ||
938 Opcode == AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
939 }
940
941 static bool isChainCallOpcode(uint64_t Opcode) {
942 return Opcode == AMDGPU::SI_CS_CHAIN_TC_W32 ||
943 Opcode == AMDGPU::SI_CS_CHAIN_TC_W64;
944 }
945
946 static bool isDPP(const MachineInstr &MI) { return SIInstrFlags::isDPP(O: MI); }
947
948 bool isDPP(uint32_t Opcode) const { return SIInstrFlags::isDPP(O: get(Opcode)); }
949
950 // Some opcodes use Src1 for DPP instead of Src0, because the sequencer
951 // transforms them and reverse the order of their operands at runtime.
952 //
953 // Documentation is incomplete on which instructions are effected, so
954 // the implementation is derived from experimentation.
955 //
956 // Listed as target-independent pseudos; the per-subtarget MC opcodes
957 // (V_SUBREV_NC_U32_e32_gfx11 and friends) are all reached through these.
958 // Defined out of line because GCNSubtarget is incomplete here.
959 static bool isSrc1DPPRevOpcode(const GCNSubtarget &ST, uint32_t Opcode);
960
961 static bool isTRANS(const MachineInstr &MI) {
962 return SIInstrFlags::isTRANS(O: MI);
963 }
964
965 bool isTRANS(uint32_t Opcode) const {
966 return SIInstrFlags::isTRANS(O: get(Opcode));
967 }
968
969 static bool isVOP3P(const MachineInstr &MI) {
970 return SIInstrFlags::isVOP3P(O: MI);
971 }
972
973 bool isVOP3P(uint32_t Opcode) const {
974 return SIInstrFlags::isVOP3P(O: get(Opcode));
975 }
976
977 bool isVOP3PMix(const MachineInstr &MI) const {
978 return isVOP3PMix(Opcode: MI.getOpcode());
979 }
980
981 bool isVOP3PMix(uint16_t Opcode) const {
982 switch (Opcode) {
983 case AMDGPU::V_FMA_MIXHI_F16:
984 case AMDGPU::V_FMA_MIXLO_F16:
985 case AMDGPU::V_FMA_MIX_F32:
986 case AMDGPU::V_MAD_MIXHI_F16:
987 case AMDGPU::V_MAD_MIXLO_F16:
988 case AMDGPU::V_MAD_MIX_F32:
989 return true;
990 default:
991 return false;
992 }
993 }
994
995 static bool isVINTRP(const MachineInstr &MI) {
996 return SIInstrFlags::isVINTRP(O: MI);
997 }
998
999 bool isVINTRP(uint32_t Opcode) const {
1000 return SIInstrFlags::isVINTRP(O: get(Opcode));
1001 }
1002
1003 static bool isMAI(const MCInstrDesc &Desc) {
1004 return SIInstrFlags::isMAI(O: Desc);
1005 }
1006
1007 static bool isMAI(const MachineInstr &MI) { return isMAI(Desc: MI.getDesc()); }
1008
1009 bool isMAI(uint32_t Opcode) const { return isMAI(Desc: get(Opcode)); }
1010
1011 static bool isMFMA(const MachineInstr &MI) {
1012 return isMAI(MI) && MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1013 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1014 }
1015
1016 bool isMFMA(uint32_t Opcode) const {
1017 return isMAI(Opcode) && Opcode != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1018 Opcode != AMDGPU::V_ACCVGPR_READ_B32_e64;
1019 }
1020
1021 static bool isDOT(const MachineInstr &MI) { return SIInstrFlags::isDOT(O: MI); }
1022
1023 static bool isWMMA(const MachineInstr &MI) {
1024 return SIInstrFlags::isWMMA(O: MI);
1025 }
1026
1027 bool isWMMA(uint32_t Opcode) const {
1028 return SIInstrFlags::isWMMA(O: get(Opcode));
1029 }
1030
1031 static bool isMFMAorWMMA(const MachineInstr &MI) {
1032 return isMFMA(MI) || isWMMA(MI) || isSWMMAC(MI);
1033 }
1034
1035 bool isMFMAorWMMA(uint32_t Opcode) const {
1036 return isMFMA(Opcode) || isWMMA(Opcode) || isSWMMAC(Opcode);
1037 }
1038
1039 static bool isSWMMAC(const MachineInstr &MI) {
1040 return SIInstrFlags::isSWMMAC(O: MI);
1041 }
1042
1043 bool isSWMMAC(uint32_t Opcode) const {
1044 return SIInstrFlags::isSWMMAC(O: get(Opcode));
1045 }
1046
1047 bool isDOT(uint32_t Opcode) const { return SIInstrFlags::isDOT(O: get(Opcode)); }
1048
1049 bool isXDLWMMA(const MachineInstr &MI) const;
1050
1051 bool isXDL(const MachineInstr &MI) const;
1052
1053 static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opc: Opcode); }
1054
1055 static bool isLDSDIR(const MachineInstr &MI) {
1056 return SIInstrFlags::isLDSDIR(O: MI);
1057 }
1058
1059 bool isLDSDIR(uint32_t Opcode) const {
1060 return SIInstrFlags::isLDSDIR(O: get(Opcode));
1061 }
1062
1063 static bool isVINTERP(const MachineInstr &MI) {
1064 return SIInstrFlags::isVINTERP(O: MI);
1065 }
1066
1067 bool isVINTERP(uint32_t Opcode) const {
1068 return SIInstrFlags::isVINTERP(O: get(Opcode));
1069 }
1070
1071 static bool usesVM_CNT(const MachineInstr &MI) {
1072 return SIInstrFlags::usesVM_CNT(O: MI);
1073 }
1074
1075 static bool usesLGKM_CNT(const MachineInstr &MI) {
1076 return SIInstrFlags::usesLGKM_CNT(O: MI);
1077 }
1078
1079 static bool usesASYNC_CNT(const MachineInstr &MI) {
1080 return SIInstrFlags::usesASYNC_CNT(O: MI);
1081 }
1082
1083 bool usesASYNC_CNT(uint32_t Opcode) const {
1084 return SIInstrFlags::usesASYNC_CNT(O: get(Opcode));
1085 }
1086
1087 static bool usesTENSOR_CNT(const MachineInstr &MI) {
1088 return SIInstrFlags::usesTENSOR_CNT(O: MI);
1089 }
1090
1091 bool usesTENSOR_CNT(uint32_t Opcode) const {
1092 return SIInstrFlags::usesTENSOR_CNT(O: get(Opcode));
1093 }
1094
1095 // Most sopk treat the immediate as a signed 16-bit, however some
1096 // use it as unsigned.
1097 static bool sopkIsZext(unsigned Opcode) {
1098 return Opcode == AMDGPU::S_CMPK_EQ_U32 || Opcode == AMDGPU::S_CMPK_LG_U32 ||
1099 Opcode == AMDGPU::S_CMPK_GT_U32 || Opcode == AMDGPU::S_CMPK_GE_U32 ||
1100 Opcode == AMDGPU::S_CMPK_LT_U32 || Opcode == AMDGPU::S_CMPK_LE_U32 ||
1101 Opcode == AMDGPU::S_GETREG_B32 ||
1102 Opcode == AMDGPU::S_GETREG_B32_const;
1103 }
1104
1105 /// \returns true if this is an s_store_dword* instruction. This is more
1106 /// specific than isSMEM && mayStore.
1107 static bool isScalarStore(const MachineInstr &MI) {
1108 return SIInstrFlags::isScalarStore(O: MI);
1109 }
1110
1111 bool isScalarStore(uint32_t Opcode) const {
1112 return SIInstrFlags::isScalarStore(O: get(Opcode));
1113 }
1114
1115 static bool isFixedSize(const MachineInstr &MI) {
1116 return SIInstrFlags::isFixedSize(O: MI);
1117 }
1118
1119 bool isFixedSize(uint32_t Opcode) const {
1120 return SIInstrFlags::isFixedSize(O: get(Opcode));
1121 }
1122
1123 static bool hasFPClamp(const MachineInstr &MI) {
1124 return SIInstrFlags::hasFPClamp(O: MI);
1125 }
1126
1127 bool hasFPClamp(uint32_t Opcode) const {
1128 return SIInstrFlags::hasFPClamp(O: get(Opcode));
1129 }
1130
1131 static bool hasIntClamp(const MachineInstr &MI) {
1132 return SIInstrFlags::hasIntClamp(O: MI);
1133 }
1134
1135 static bool hasSameClamp(const MachineInstr &A, const MachineInstr &B) {
1136 const MCInstrDesc &DA = A.getDesc(), &DB = B.getDesc();
1137 return SIInstrFlags::hasFPClamp(O: DA) == SIInstrFlags::hasFPClamp(O: DB) &&
1138 SIInstrFlags::hasIntClamp(O: DA) == SIInstrFlags::hasIntClamp(O: DB) &&
1139 SIInstrFlags::hasClampLo(O: DA) == SIInstrFlags::hasClampLo(O: DB) &&
1140 SIInstrFlags::hasClampHi(O: DA) == SIInstrFlags::hasClampHi(O: DB);
1141 }
1142
1143 static bool usesFPDPRounding(const MachineInstr &MI) {
1144 return SIInstrFlags::usesFPDPRounding(O: MI);
1145 }
1146
1147 bool usesFPDPRounding(uint32_t Opcode) const {
1148 return SIInstrFlags::usesFPDPRounding(O: get(Opcode));
1149 }
1150
1151 static bool isFPAtomic(const MachineInstr &MI) {
1152 return SIInstrFlags::isFPAtomic(O: MI);
1153 }
1154
1155 bool isFPAtomic(uint32_t Opcode) const {
1156 return SIInstrFlags::isFPAtomic(O: get(Opcode));
1157 }
1158
1159 static bool isNeverUniform(const MachineInstr &MI) {
1160 return SIInstrFlags::isNeverUniform(O: MI);
1161 }
1162
1163 // Check to see if opcode is for a barrier start. Pre gfx12 this is just the
1164 // S_BARRIER, but after support for S_BARRIER_SIGNAL* / S_BARRIER_WAIT we want
1165 // to check for the barrier start (S_BARRIER_SIGNAL*)
1166 bool isBarrierStart(unsigned Opcode) const {
1167 return Opcode == AMDGPU::S_BARRIER ||
1168 Opcode == AMDGPU::S_BARRIER_SIGNAL_M0 ||
1169 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0 ||
1170 Opcode == AMDGPU::S_BARRIER_SIGNAL_IMM ||
1171 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM;
1172 }
1173
1174 bool isBarrier(unsigned Opcode) const {
1175 return isBarrierStart(Opcode) || Opcode == AMDGPU::S_BARRIER_WAIT ||
1176 Opcode == AMDGPU::S_BARRIER_INIT_M0 ||
1177 Opcode == AMDGPU::S_BARRIER_INIT_IMM ||
1178 Opcode == AMDGPU::S_BARRIER_JOIN_IMM ||
1179 Opcode == AMDGPU::S_BARRIER_LEAVE || Opcode == AMDGPU::DS_GWS_INIT ||
1180 Opcode == AMDGPU::DS_GWS_BARRIER;
1181 }
1182
1183 static bool isLoadMonitor(unsigned Opc) {
1184 switch (Opc) {
1185 case AMDGPU::GLOBAL_LOAD_MONITOR_B32:
1186 case AMDGPU::GLOBAL_LOAD_MONITOR_B32_SADDR:
1187 case AMDGPU::GLOBAL_LOAD_MONITOR_B64:
1188 case AMDGPU::GLOBAL_LOAD_MONITOR_B64_SADDR:
1189 case AMDGPU::GLOBAL_LOAD_MONITOR_B128:
1190 case AMDGPU::GLOBAL_LOAD_MONITOR_B128_SADDR:
1191 case AMDGPU::FLAT_LOAD_MONITOR_B32:
1192 case AMDGPU::FLAT_LOAD_MONITOR_B64:
1193 case AMDGPU::FLAT_LOAD_MONITOR_B128:
1194 return true;
1195 default:
1196 return false;
1197 }
1198 }
1199
1200 static bool isGFX12CacheInvOrWBInst(unsigned Opc) {
1201 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||
1202 Opc == AMDGPU::GLOBAL_WBINV;
1203 }
1204
1205 static bool isF16PseudoScalarTrans(unsigned Opcode) {
1206 return Opcode == AMDGPU::V_S_EXP_F16_e64 ||
1207 Opcode == AMDGPU::V_S_LOG_F16_e64 ||
1208 Opcode == AMDGPU::V_S_RCP_F16_e64 ||
1209 Opcode == AMDGPU::V_S_RSQ_F16_e64 ||
1210 Opcode == AMDGPU::V_S_SQRT_F16_e64;
1211 }
1212
1213 static bool isPseudoScalarTrans(unsigned Opcode) {
1214 return isF16PseudoScalarTrans(Opcode) ||
1215 Opcode == AMDGPU::V_S_EXP_F32_e64 ||
1216 Opcode == AMDGPU::V_S_LOG_F32_e64 ||
1217 Opcode == AMDGPU::V_S_RCP_F32_e64 ||
1218 Opcode == AMDGPU::V_S_RSQ_F32_e64 ||
1219 Opcode == AMDGPU::V_S_SQRT_F32_e64;
1220 }
1221
1222 static bool isVPermPk16(unsigned Opcode) {
1223 return Opcode == AMDGPU::V_PERM_PK16_B4_U4_e64 ||
1224 Opcode == AMDGPU::V_PERM_PK16_B6_U4_e64 ||
1225 Opcode == AMDGPU::V_PERM_PK16_B8_U4_e64;
1226 }
1227
1228 // \returns true if \p MI clears the V_PERM_PK16 hazard when it immediately
1229 // follows a V_PERM_PK16 (i.e. \p MI is a "safe" instruction).
1230 bool isVPermPk16SafeInstr(const MachineInstr &MI) const {
1231 unsigned Opc = MI.getOpcode();
1232
1233 // Only VALU ops issue on the pipe that clears the V_PERM_PK16 hazard.
1234 if (!isVALU(MI, /*AllowLDSDMA=*/AllowLDSDMA: false))
1235 return false;
1236 // OP_XDL: matrix (WMMA/SWMMAC/DOT) ops clear the hazard.
1237 if (isXDL(MI))
1238 return true;
1239 // Pseudo-scalar transcendentals (OP32_SCL_T) do NOT clear the hazard.
1240 if (isPseudoScalarTrans(Opcode: Opc))
1241 return false;
1242
1243 // Use the table lookup, not getBlockingCycles(): occupancy is gated off on
1244 // gfx1251 but the hazard applies to both gfx1250 and gfx1251. Gfx1250 table
1245 // is valid enough for gfx1251 w.r.t. v_perm_pk16 safety check here.
1246 // OP_32_T is in the table at 2 and is safe; other table entries (>= 2) are
1247 // not.
1248 unsigned Cycles = getGFX1250BlockingCyclesTable(MI);
1249 return Cycles < 2 || (Cycles == 2 && isTRANS(MI));
1250 }
1251
1252 static bool doesNotReadTiedSource(const MachineInstr &MI) {
1253 return SIInstrFlags::isTiedSourceNotRead(O: MI);
1254 }
1255
1256 bool doesNotReadTiedSource(uint32_t Opcode) const {
1257 return SIInstrFlags::isTiedSourceNotRead(O: get(Opcode));
1258 }
1259
1260 bool isIGLP(unsigned Opcode) const {
1261 return Opcode == AMDGPU::SCHED_BARRIER ||
1262 Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
1263 }
1264
1265 bool isIGLP(const MachineInstr &MI) const { return isIGLP(Opcode: MI.getOpcode()); }
1266
1267 // Return true if the instruction is mutually exclusive with all non-IGLP DAG
1268 // mutations, requiring all other mutations to be disabled.
1269 bool isIGLPMutationOnly(unsigned Opcode) const {
1270 return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
1271 }
1272
1273 static unsigned getNonSoftWaitcntOpcode(unsigned Opcode) {
1274 switch (Opcode) {
1275 case AMDGPU::S_WAITCNT_soft:
1276 return AMDGPU::S_WAITCNT;
1277 case AMDGPU::S_WAITCNT_VSCNT_soft:
1278 return AMDGPU::S_WAITCNT_VSCNT;
1279 case AMDGPU::S_WAIT_LOADCNT_soft:
1280 return AMDGPU::S_WAIT_LOADCNT;
1281 case AMDGPU::S_WAIT_STORECNT_soft:
1282 return AMDGPU::S_WAIT_STORECNT;
1283 case AMDGPU::S_WAIT_SAMPLECNT_soft:
1284 return AMDGPU::S_WAIT_SAMPLECNT;
1285 case AMDGPU::S_WAIT_BVHCNT_soft:
1286 return AMDGPU::S_WAIT_BVHCNT;
1287 case AMDGPU::S_WAIT_DSCNT_soft:
1288 return AMDGPU::S_WAIT_DSCNT;
1289 case AMDGPU::S_WAIT_KMCNT_soft:
1290 return AMDGPU::S_WAIT_KMCNT;
1291 case AMDGPU::S_WAIT_XCNT_soft:
1292 return AMDGPU::S_WAIT_XCNT;
1293 default:
1294 return Opcode;
1295 }
1296 }
1297
1298 static bool isWaitcnt(unsigned Opcode) {
1299 switch (getNonSoftWaitcntOpcode(Opcode)) {
1300 case AMDGPU::S_WAITCNT:
1301 case AMDGPU::S_WAITCNT_VSCNT:
1302 case AMDGPU::S_WAITCNT_VMCNT:
1303 case AMDGPU::S_WAITCNT_EXPCNT:
1304 case AMDGPU::S_WAITCNT_LGKMCNT:
1305 case AMDGPU::S_WAIT_LOADCNT:
1306 case AMDGPU::S_WAIT_LOADCNT_DSCNT:
1307 case AMDGPU::S_WAIT_STORECNT:
1308 case AMDGPU::S_WAIT_STORECNT_DSCNT:
1309 case AMDGPU::S_WAIT_SAMPLECNT:
1310 case AMDGPU::S_WAIT_BVHCNT:
1311 case AMDGPU::S_WAIT_EXPCNT:
1312 case AMDGPU::S_WAIT_DSCNT:
1313 case AMDGPU::S_WAIT_KMCNT:
1314 case AMDGPU::S_WAIT_XCNT:
1315 case AMDGPU::S_WAIT_ASYNCCNT:
1316 case AMDGPU::S_WAIT_TENSORCNT:
1317 case AMDGPU::S_WAIT_IDLE:
1318 return true;
1319 default:
1320 return false;
1321 }
1322 }
1323
1324 bool isVGPRCopy(const MachineInstr &MI) const {
1325 assert(isCopyInstr(MI));
1326 Register Dest = MI.getOperand(i: 0).getReg();
1327 const MachineFunction &MF = *MI.getMF();
1328 const MachineRegisterInfo &MRI = MF.getRegInfo();
1329 return !RI.isSGPRReg(MRI, Reg: Dest);
1330 }
1331
1332 bool hasVGPRUses(const MachineInstr &MI) const {
1333 const MachineFunction &MF = *MI.getMF();
1334 const MachineRegisterInfo &MRI = MF.getRegInfo();
1335 return llvm::any_of(Range: MI.explicit_uses(),
1336 P: [&MRI, this](const MachineOperand &MO) {
1337 return MO.isReg() && RI.isVGPR(MRI, Reg: MO.getReg());});
1338 }
1339
1340 /// Return true if the instruction modifies the mode register.q
1341 static bool modifiesModeRegister(const MachineInstr &MI);
1342
1343 /// This function is used to determine if an instruction can be safely
1344 /// executed under EXEC = 0 without hardware error, indeterminate results,
1345 /// and/or visible effects on future vector execution or outside the shader.
1346 /// Note: as of 2024 the only use of this is SIPreEmitPeephole where it is
1347 /// used in removing branches over short EXEC = 0 sequences.
1348 /// As such it embeds certain assumptions which may not apply to every case
1349 /// of EXEC = 0 execution.
1350 bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const;
1351
1352 /// Returns true if the instruction could potentially depend on the value of
1353 /// exec. If false, exec dependencies may safely be ignored.
1354 bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const;
1355
1356 bool isInlineConstant(const APInt &Imm) const;
1357
1358 bool isInlineConstant(const APFloat &Imm) const;
1359
1360 // Returns true if this non-register operand definitely does not need to be
1361 // encoded as a 32-bit literal. Note that this function handles all kinds of
1362 // operands, not just immediates.
1363 //
1364 // Some operands like FrameIndexes could resolve to an inline immediate value
1365 // that will not require an additional 4-bytes; this function assumes that it
1366 // will.
1367 bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const {
1368 if (!MO.isImm())
1369 return false;
1370 return isInlineConstant(ImmVal: MO.getImm(), OperandType);
1371 }
1372 bool isInlineConstant(int64_t ImmVal, uint8_t OperandType) const;
1373
1374 bool isInlineConstant(const MachineOperand &MO,
1375 const MCOperandInfo &OpInfo) const {
1376 return isInlineConstant(MO, OperandType: OpInfo.OperandType);
1377 }
1378
1379 /// \p returns true if \p UseMO is substituted with \p DefMO in \p MI it would
1380 /// be an inline immediate.
1381 bool isInlineConstant(const MachineInstr &MI,
1382 const MachineOperand &UseMO,
1383 const MachineOperand &DefMO) const {
1384 assert(UseMO.getParent() == &MI);
1385 int OpIdx = UseMO.getOperandNo();
1386 if (OpIdx >= MI.getDesc().NumOperands)
1387 return false;
1388
1389 return isInlineConstant(MO: DefMO, OpInfo: MI.getDesc().operands()[OpIdx]);
1390 }
1391
1392 /// \p returns true if the operand \p OpIdx in \p MI is a valid inline
1393 /// immediate.
1394 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx) const {
1395 const MachineOperand &MO = MI.getOperand(i: OpIdx);
1396 return isInlineConstant(MO, OperandType: MI.getDesc().operands()[OpIdx].OperandType);
1397 }
1398
1399 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
1400 int64_t ImmVal) const {
1401 if (OpIdx >= MI.getDesc().NumOperands)
1402 return false;
1403
1404 if (isCopyInstr(MI)) {
1405 unsigned Size = getOpSize(MI, OpNo: OpIdx);
1406 assert(Size == 8 || Size == 4);
1407
1408 uint8_t OpType = (Size == 8) ?
1409 AMDGPU::OPERAND_REG_IMM_INT64 : AMDGPU::OPERAND_REG_IMM_INT32;
1410 return isInlineConstant(ImmVal, OperandType: OpType);
1411 }
1412
1413 return isInlineConstant(ImmVal, OperandType: MI.getDesc().operands()[OpIdx].OperandType);
1414 }
1415
1416 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
1417 const MachineOperand &MO) const {
1418 return isInlineConstant(MI, OpIdx, ImmVal: MO.getImm());
1419 }
1420
1421 bool isInlineConstant(const MachineOperand &MO) const {
1422 return isInlineConstant(MI: *MO.getParent(), OpIdx: MO.getOperandNo());
1423 }
1424
1425 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
1426 const MachineOperand &MO) const;
1427
1428 bool isLiteralOperandLegal(const MCInstrDesc &InstDesc,
1429 const MCOperandInfo &OpInfo) const;
1430
1431 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
1432 int64_t ImmVal) const;
1433
1434 bool isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
1435 const MachineOperand &MO) const {
1436 return isImmOperandLegal(InstDesc: MI.getDesc(), OpNo, MO);
1437 }
1438
1439 bool isNeverCoissue(MachineInstr &MI) const;
1440
1441 /// Check if this immediate value can be used for AV_MOV_B64_IMM_PSEUDO.
1442 bool isLegalAV64PseudoImm(uint64_t Imm) const;
1443
1444 /// Return true if this 64-bit VALU instruction has a 32-bit encoding.
1445 /// This function will return false if you pass it a 32-bit instruction.
1446 bool hasVALU32BitEncoding(unsigned Opcode) const;
1447
1448 /// Return true if \p Reg is a lane mask that already has 0 in every bit
1449 /// corresponding to a lane that is inactive in EXEC where \p Use executes,
1450 /// so that ANDing it with EXEC there would be a no-op. Requires SSA form.
1451 bool isMaskedByExec(Register Reg, const MachineInstr &Use,
1452 const MachineRegisterInfo &MRI, unsigned Depth = 0) const;
1453
1454 bool physRegUsesConstantBus(const MachineOperand &Reg) const;
1455 bool regUsesConstantBus(const MachineOperand &Reg,
1456 const MachineRegisterInfo &MRI) const;
1457
1458 /// Returns true if this operand uses the constant bus.
1459 bool usesConstantBus(const MachineRegisterInfo &MRI,
1460 const MachineOperand &MO,
1461 const MCOperandInfo &OpInfo) const;
1462
1463 bool usesConstantBus(const MachineRegisterInfo &MRI, const MachineInstr &MI,
1464 int OpIdx) const {
1465 return usesConstantBus(MRI, MO: MI.getOperand(i: OpIdx),
1466 OpInfo: MI.getDesc().operands()[OpIdx]);
1467 }
1468
1469 /// Return true if this instruction has any modifiers.
1470 /// e.g. src[012]_mod, omod, clamp.
1471 bool hasModifiers(unsigned Opcode) const;
1472
1473 bool hasModifiersSet(const MachineInstr &MI, AMDGPU::OpName OpName) const;
1474 bool hasAnyModifiersSet(const MachineInstr &MI) const;
1475
1476 bool canShrink(const MachineInstr &MI,
1477 const MachineRegisterInfo &MRI) const;
1478
1479 MachineInstr *buildShrunkInst(MachineInstr &MI,
1480 unsigned NewOpcode) const;
1481
1482 bool verifyInstruction(const MachineInstr &MI,
1483 StringRef &ErrInfo) const override;
1484
1485 unsigned getVALUOp(const MachineInstr &MI) const;
1486 unsigned getVALUOp(unsigned Opc) const;
1487
1488 void insertScratchExecCopy(MachineFunction &MF, MachineBasicBlock &MBB,
1489 MachineBasicBlock::iterator MBBI,
1490 const DebugLoc &DL, Register Reg, bool IsSCCLive,
1491 SlotIndexes *Indexes = nullptr) const;
1492
1493 void restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
1494 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
1495 Register Reg, SlotIndexes *Indexes = nullptr) const;
1496
1497 MachineInstr *getWholeWaveFunctionSetup(MachineFunction &MF) const;
1498
1499 /// Return the correct register class for \p OpNo. For target-specific
1500 /// instructions, this will return the register class that has been defined
1501 /// in tablegen. For generic instructions, like REG_SEQUENCE it will return
1502 /// the register class of its machine operand.
1503 /// to infer the correct register class base on the other operands.
1504 const TargetRegisterClass *getOpRegClass(const MachineInstr &MI,
1505 unsigned OpNo) const;
1506
1507 /// Return the size in bytes of the operand OpNo on the given
1508 // instruction opcode.
1509 unsigned getOpSize(uint32_t Opcode, unsigned OpNo) const {
1510 const MCOperandInfo &OpInfo = get(Opcode).operands()[OpNo];
1511
1512 if (OpInfo.RegClass == -1) {
1513 // If this is an immediate operand, this must be a 32-bit literal.
1514 assert(OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE);
1515 return 4;
1516 }
1517
1518 return RI.getRegSizeInBits(RC: *RI.getRegClass(i: getOpRegClassID(OpInfo))) / 8;
1519 }
1520
1521 /// This form should usually be preferred since it handles operands
1522 /// with unknown register classes.
1523 unsigned getOpSize(const MachineInstr &MI, unsigned OpNo) const {
1524 const MachineOperand &MO = MI.getOperand(i: OpNo);
1525 if (MO.isReg()) {
1526 if (unsigned SubReg = MO.getSubReg()) {
1527 return RI.getSubRegIdxSize(Idx: SubReg) / 8;
1528 }
1529 }
1530 return RI.getRegSizeInBits(RC: *getOpRegClass(MI, OpNo)) / 8;
1531 }
1532
1533 /// Legalize the \p OpIndex operand of this instruction by inserting
1534 /// a MOV. For example:
1535 /// ADD_I32_e32 VGPR0, 15
1536 /// to
1537 /// MOV VGPR1, 15
1538 /// ADD_I32_e32 VGPR0, VGPR1
1539 ///
1540 /// If the operand being legalized is a register, then a COPY will be used
1541 /// instead of MOV.
1542 void legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const;
1543
1544 /// Check if \p MO is a legal operand if it was the \p OpIdx Operand
1545 /// for \p MI.
1546 bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
1547 const MachineOperand *MO = nullptr) const;
1548
1549 /// Check if \p MO (a register operand) is a legal register for the
1550 /// given operand description or operand index.
1551 /// The operand index version provide more legality checks
1552 bool isLegalRegOperand(const MachineRegisterInfo &MRI,
1553 const MCOperandInfo &OpInfo,
1554 const MachineOperand &MO) const;
1555 bool isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
1556 const MachineOperand &MO) const;
1557
1558 /// Check if \p MO would be a legal operand for a single-SGPR-read
1559 /// instruction.
1560 ///
1561 /// Single-SGPR-read instructions typically accept VGPRs, SGPRs, or immediates
1562 /// as source operands. On gfx12+, if a source operand uses SGPRs, the HW can
1563 /// only read the first SGPR and replicate the value across all lanes. \p SrcN
1564 /// can be 0, 1, or 2, representing src0, src1, and src2, respectively. If \p
1565 /// MO is nullptr, the operand corresponding to \p SrcN will be used. Non-SGPR
1566 /// operands are always considered legal.
1567 bool
1568 isLegalSingleSGPRReadInstOperand(const MachineRegisterInfo &MRI,
1569 const MachineInstr &MI, unsigned SrcN,
1570 const MachineOperand *MO = nullptr) const;
1571
1572 /// Check if \p MI has legal GFX950 v_pk_fma op_sel.
1573 bool isLegalPkFMAF32OpSel(const MachineInstr &MI) const;
1574
1575 /// Decompose a GFX950 v_pk_fma_f32 with invalid op_sel into two independent
1576 /// scalar V_FMA_F32_e64 instructions, replacing \p MI.
1577 void decomposePkFMAF32(MachineInstr &MI, MachineRegisterInfo &MRI) const;
1578
1579 /// Legalize operands in \p MI by either commuting it or inserting a
1580 /// copy of src1.
1581 void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1582
1583 /// Fix operands in \p MI to satisfy constant bus requirements.
1584 void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1585
1586 /// Copy a value from a VGPR (\p SrcReg) to SGPR. The desired register class
1587 /// for the dst register (\p DstRC) can be optionally supplied. This function
1588 /// can only be used when it is know that the value in SrcReg is same across
1589 /// all threads in the wave.
1590 /// \returns The SGPR register that \p SrcReg was copied to.
1591 Register readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,
1592 MachineRegisterInfo &MRI,
1593 const TargetRegisterClass *DstRC = nullptr) const;
1594
1595 void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1596 void legalizeOperandsFLAT(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1597
1598 void legalizeGenericOperand(MachineBasicBlock &InsertMBB,
1599 MachineBasicBlock::iterator I,
1600 const TargetRegisterClass *DstRC,
1601 MachineOperand &Op, MachineRegisterInfo &MRI,
1602 const DebugLoc &DL) const;
1603
1604 /// Legalize all operands in this instruction. This function may create new
1605 /// instructions and control-flow around \p MI. If present, \p MDT is
1606 /// updated.
1607 /// \returns A new basic block that contains \p MI if new blocks were created.
1608 MachineBasicBlock *
1609 legalizeOperands(MachineInstr &MI, MachineDominatorTree *MDT = nullptr) const;
1610
1611 /// Change SADDR form of a FLAT \p Inst to its VADDR form if saddr operand
1612 /// was moved to VGPR. \returns true if succeeded.
1613 bool moveFlatAddrToVGPR(MachineInstr &Inst) const;
1614
1615 /// Fix operands in Inst to fix 16bit SALU to VALU lowering.
1616 void legalizeOperandsVALUt16(MachineInstr &Inst,
1617 MachineRegisterInfo &MRI) const;
1618 void legalizeOperandsVALUt16(MachineInstr &Inst, unsigned OpIdx,
1619 MachineRegisterInfo &MRI) const;
1620
1621 /// Replace the instructions opcode with the equivalent VALU
1622 /// opcode. This function will also move the users of MachineInstruntions
1623 /// in the \p WorkList to the VALU if necessary. If present, \p MDT is
1624 /// updated.
1625 void moveToVALU(SIInstrWorklist &Worklist, MachineDominatorTree *MDT) const;
1626
1627 void
1628 moveToVALUImpl(SIInstrWorklist &Worklist, MachineDominatorTree *MDT,
1629 MachineInstr &Inst,
1630 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
1631 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const;
1632 /// Wrapper function for generating waterfall for instruction \p MI
1633 /// This function take into consideration of related pre & succ instructions
1634 /// (e.g. calling process) into consideratioin
1635 void createWaterFallForSiCall(MachineInstr *MI, MachineDominatorTree *MDT,
1636 ArrayRef<MachineOperand *> ScalarOps,
1637 ArrayRef<Register> PhySGPRs = {}) const;
1638
1639 void insertNoop(MachineBasicBlock &MBB,
1640 MachineBasicBlock::iterator MI) const override;
1641
1642 void insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1643 unsigned Quantity) const override;
1644
1645 /// Build instructions that simulate the behavior of a `s_trap 2` instructions
1646 /// for hardware (namely, gfx11) that runs in PRIV=1 mode. There, s_trap is
1647 /// interpreted as a nop.
1648 MachineBasicBlock *insertSimulatedTrap(MachineRegisterInfo &MRI,
1649 MachineBasicBlock &MBB,
1650 MachineInstr &MI,
1651 const DebugLoc &DL) const;
1652
1653 /// Return the number of wait states that result from executing this
1654 /// instruction.
1655 static unsigned getNumWaitStates(const MachineInstr &MI);
1656
1657 /// Returns the operand named \p Op. If \p MI does not have an
1658 /// operand named \c Op, this function returns nullptr.
1659 LLVM_READONLY
1660 MachineOperand *getNamedOperand(MachineInstr &MI,
1661 AMDGPU::OpName OperandName) const;
1662
1663 LLVM_READONLY
1664 const MachineOperand *getNamedOperand(const MachineInstr &MI,
1665 AMDGPU::OpName OperandName) const {
1666 return getNamedOperand(MI&: const_cast<MachineInstr &>(MI), OperandName);
1667 }
1668
1669 /// Get required immediate operand
1670 int64_t getNamedImmOperand(const MachineInstr &MI,
1671 AMDGPU::OpName OperandName) const {
1672 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OperandName);
1673 return MI.getOperand(i: Idx).getImm();
1674 }
1675
1676 uint64_t getDefaultRsrcDataFormat() const;
1677 uint64_t getScratchRsrcWords23() const;
1678
1679 bool isLowLatencyInstruction(const MachineInstr &MI) const;
1680 bool isHighLatencyDef(int Opc) const override;
1681
1682 /// Return the descriptor of the target-specific machine instruction
1683 /// that corresponds to the specified pseudo or native opcode.
1684 const MCInstrDesc &getMCOpcodeFromPseudo(unsigned Opcode) const {
1685 return get(Opcode: pseudoToMCOpcode(Opcode));
1686 }
1687
1688 Register isStackAccess(const MachineInstr &MI, int &FrameIndex,
1689 TypeSize &MemBytes) const;
1690 Register isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex,
1691 TypeSize &MemBytes) const;
1692
1693 Register isLoadFromStackSlot(const MachineInstr &MI,
1694 int &FrameIndex) const override {
1695 TypeSize MemBytes = TypeSize::getZero();
1696 return isLoadFromStackSlot(MI, FrameIndex, MemBytes);
1697 }
1698
1699 Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex,
1700 TypeSize &MemBytes) const override;
1701
1702 Register isStoreToStackSlot(const MachineInstr &MI,
1703 int &FrameIndex) const override {
1704 TypeSize MemBytes = TypeSize::getZero();
1705 return isStoreToStackSlot(MI, FrameIndex, MemBytes);
1706 }
1707
1708 Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex,
1709 TypeSize &MemBytes) const override;
1710
1711 unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
1712
1713 InstSizeVerifyMode
1714 getInstSizeVerifyMode(const MachineInstr &MI) const override;
1715
1716 std::pair<unsigned, unsigned>
1717 decomposeMachineOperandsTargetFlags(unsigned TF) const override;
1718
1719 ArrayRef<std::pair<int, const char *>>
1720 getSerializableTargetIndices() const override;
1721
1722 ArrayRef<std::pair<unsigned, const char *>>
1723 getSerializableDirectMachineOperandTargetFlags() const override;
1724
1725 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
1726 getSerializableMachineMemOperandTargetFlags() const override;
1727
1728 ScheduleHazardRecognizer *
1729 CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
1730 const ScheduleDAG *DAG) const override;
1731
1732 ScheduleHazardRecognizer *
1733 CreateTargetPostRAHazardRecognizer(const MachineFunction &MF,
1734 MachineLoopInfo *MLI) const override;
1735
1736 ScheduleHazardRecognizer *
1737 CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
1738 const ScheduleDAGMI *DAG) const override;
1739
1740 unsigned getLiveRangeSplitOpcode(Register Reg,
1741 const MachineFunction &MF) const override;
1742
1743 bool isBasicBlockPrologue(const MachineInstr &MI,
1744 Register Reg = Register()) const override;
1745
1746 bool canAddToBBProlog(const MachineInstr &MI) const;
1747
1748 MachineInstr *createPHIDestinationCopy(MachineBasicBlock &MBB,
1749 MachineBasicBlock::iterator InsPt,
1750 const DebugLoc &DL, Register Src,
1751 Register Dst) const override;
1752
1753 MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB,
1754 MachineBasicBlock::iterator InsPt,
1755 const DebugLoc &DL, Register Src,
1756 unsigned SrcSubReg,
1757 Register Dst) const override;
1758
1759 bool isWave32() const;
1760
1761 bool isVOPDAntidependencyAllowed(const MachineInstr &MI) const;
1762
1763 bool hasRAWDependency(const MachineInstr &FirstMI,
1764 const MachineInstr &SecondMI) const;
1765
1766 /// Return a partially built integer add instruction without carry.
1767 /// Caller must add source operands.
1768 /// For pre-GFX9 it will generate unused carry destination operand.
1769 /// TODO: After GFX9 it should return a no-carry operation.
1770 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1771 MachineBasicBlock::iterator I,
1772 const DebugLoc &DL,
1773 Register DestReg) const;
1774
1775 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1776 MachineBasicBlock::iterator I,
1777 const DebugLoc &DL,
1778 Register DestReg,
1779 RegScavenger &RS) const;
1780
1781 static bool isKillTerminator(unsigned Opcode);
1782 const MCInstrDesc &getKillTerminatorFromPseudo(unsigned Opcode) const;
1783
1784 bool isLegalMUBUFImmOffset(unsigned Imm) const;
1785
1786 static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST);
1787
1788 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
1789 Align Alignment = Align(4)) const;
1790
1791 /// Returns if \p Offset is legal for the subtarget as the offset to a FLAT
1792 /// encoded instruction with the given \p FlatVariant.
1793 bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
1794 AMDGPU::FlatAddrSpace FlatVariant) const;
1795
1796 /// Split \p COffsetVal into {immediate offset field, remainder offset}
1797 /// values.
1798 std::pair<int64_t, int64_t>
1799 splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,
1800 AMDGPU::FlatAddrSpace FlatVariant) const;
1801
1802 /// Returns true if negative offsets are allowed for the given \p FlatVariant.
1803 bool allowNegativeFlatOffset(AMDGPU::FlatAddrSpace FlatVariant) const;
1804
1805 /// \brief Return a target-specific opcode if Opcode is a pseudo instruction.
1806 /// Return -1 if the target-specific opcode for the pseudo instruction does
1807 /// not exist. If Opcode is not a pseudo instruction, this is identity.
1808 int pseudoToMCOpcode(int Opcode) const;
1809
1810 /// \brief Check if this instruction should only be used by assembler.
1811 /// Return true if this opcode should not be used by codegen.
1812 bool isAsmOnlyOpcode(int MCOp) const;
1813
1814 void fixImplicitOperands(MachineInstr &MI) const;
1815
1816 MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
1817 ArrayRef<unsigned> Ops, int FrameIndex,
1818 MachineInstr *&CopyMI,
1819 LiveIntervals *LIS = nullptr,
1820 VirtRegMap *VRM = nullptr) const override;
1821
1822 unsigned getInstrLatency(const InstrItineraryData *ItinData,
1823 const MachineInstr &MI,
1824 unsigned *PredCost = nullptr) const override;
1825
1826 unsigned getBlockingCycles(const MachineInstr &MI) const;
1827
1828 /// GFX1250 blocking-cycles table lookup with no occupancy subtarget gate.
1829 /// Returns 0 if \p MI is not in the table. Used as a multi-pass VALU denylist
1830 /// (e.g. V_PERM_PK16 hazard) on both gfx1250 and gfx1251.
1831 unsigned getGFX1250BlockingCyclesTable(const MachineInstr &MI) const;
1832
1833 const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override;
1834
1835 ValueUniformity getValueUniformity(const MachineInstr &MI) const final;
1836
1837 ValueUniformity getGenericValueUniformity(const MachineInstr &MI) const;
1838
1839 const MIRFormatter *getMIRFormatter() const override;
1840
1841 static unsigned getDSShaderTypeValue(const MachineFunction &MF);
1842
1843 const TargetSchedModel &getSchedModel() const { return SchedModel; }
1844
1845 void createReadFirstLaneFromCopyToPhysReg(MachineRegisterInfo &MRI,
1846 Register DstReg,
1847 MachineInstr &Inst) const;
1848
1849 void handleCopyToPhysHelper(
1850 SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst,
1851 MachineRegisterInfo &MRI,
1852 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
1853 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const;
1854
1855 // FIXME: This should be removed
1856 // Enforce operand's \p OpName even alignment if required by target.
1857 // This is used if an operand is a 32 bit register but needs to be aligned
1858 // regardless.
1859 void enforceOperandRCAlignment(MachineInstr &MI, AMDGPU::OpName OpName) const;
1860
1861 /// Get the repeat rate for a VALU instruction from the scheduling model.
1862 /// Returns 1 for regular VALU, >1 for long-latency VALU (packed, F64, etc.)
1863 unsigned getRepeatRate(const MachineInstr &MI) const;
1864};
1865
1866/// \brief Returns true if a reg:subreg pair P has a TRC class
1867inline bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P,
1868 const TargetRegisterClass &TRC,
1869 MachineRegisterInfo &MRI) {
1870 auto *RC = MRI.getRegClass(Reg: P.Reg);
1871 if (!P.SubReg)
1872 return RC == &TRC;
1873 auto *TRI = MRI.getTargetRegisterInfo();
1874 return RC == TRI->getMatchingSuperRegClass(A: RC, B: &TRC, Idx: P.SubReg);
1875}
1876
1877/// \brief Create RegSubRegPair from a register MachineOperand
1878inline
1879TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O) {
1880 assert(O.isReg());
1881 return TargetInstrInfo::RegSubRegPair(O.getReg(), O.getSubReg());
1882}
1883
1884/// \brief Return the SubReg component from REG_SEQUENCE
1885TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI,
1886 unsigned SubReg);
1887
1888/// \brief Return the defining instruction for a given reg:subreg pair
1889/// skipping copy like instructions and subreg-manipulation pseudos.
1890/// Following another subreg of a reg:subreg isn't supported.
1891MachineInstr *getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
1892 const MachineRegisterInfo &MRI);
1893
1894/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1895/// DefMI and the use at \p UseMI. Should be run on SSA. Currently does not
1896/// attempt to track between blocks.
1897bool execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
1898 Register VReg,
1899 const MachineInstr &DefMI,
1900 const MachineInstr &UseMI);
1901
1902/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1903/// DefMI and all its uses. Should be run on SSA. Currently does not attempt to
1904/// track between blocks.
1905bool execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
1906 Register VReg,
1907 const MachineInstr &DefMI);
1908
1909namespace AMDGPU {
1910
1911 LLVM_READONLY
1912 int32_t getVOPe64(uint32_t Opcode);
1913
1914 LLVM_READONLY
1915 int32_t getVOPe32(uint32_t Opcode);
1916
1917 LLVM_READONLY
1918 int32_t getSDWAOp(uint32_t Opcode);
1919
1920 LLVM_READONLY
1921 int32_t getDPPOp32(uint32_t Opcode);
1922
1923 LLVM_READONLY
1924 int32_t getDPPOp64(uint32_t Opcode);
1925
1926 LLVM_READONLY
1927 int32_t getBasicFromSDWAOp(uint32_t Opcode);
1928
1929 LLVM_READONLY
1930 int32_t getCommuteRev(uint32_t Opcode);
1931
1932 LLVM_READONLY
1933 int32_t getCommuteOrig(uint32_t Opcode);
1934
1935 LLVM_READONLY
1936 int32_t getAddr64Inst(uint32_t Opcode);
1937
1938 /// Check if \p Opcode is an Addr64 opcode.
1939 ///
1940 /// \returns \p Opcode if it is an Addr64 opcode, otherwise -1.
1941 LLVM_READONLY
1942 int32_t getIfAddr64Inst(uint32_t Opcode);
1943
1944 LLVM_READONLY
1945 int32_t getSOPKOp(uint32_t Opcode);
1946
1947 /// \returns SADDR form of a FLAT Global instruction given an \p Opcode
1948 /// of a VADDR form.
1949 LLVM_READONLY
1950 int32_t getGlobalSaddrOp(uint32_t Opcode);
1951
1952 /// \returns VADDR form of a FLAT Global instruction given an \p Opcode
1953 /// of a SADDR form.
1954 LLVM_READONLY
1955 int32_t getGlobalVaddrOp(uint32_t Opcode);
1956
1957 /// \returns ST form with only immediate offset of a FLAT Scratch instruction
1958 /// given an \p Opcode of an SS (SADDR) form.
1959 LLVM_READONLY
1960 int32_t getFlatScratchInstSTfromSS(uint32_t Opcode);
1961
1962 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
1963 /// of an SVS (SADDR + VADDR) form.
1964 LLVM_READONLY
1965 int32_t getFlatScratchInstSVfromSVS(uint32_t Opcode);
1966
1967 /// \returns SS (SADDR) form of a FLAT Scratch instruction given an \p Opcode
1968 /// of an SV (VADDR) form.
1969 LLVM_READONLY
1970 int32_t getFlatScratchInstSSfromSV(uint32_t Opcode);
1971
1972 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
1973 /// of an SS (SADDR) form.
1974 LLVM_READONLY
1975 int32_t getFlatScratchInstSVfromSS(uint32_t Opcode);
1976
1977 /// \returns earlyclobber version of a MAC MFMA is exists.
1978 LLVM_READONLY
1979 int32_t getMFMAEarlyClobberOp(uint32_t Opcode);
1980
1981 /// \returns Version of an instruction which uses AGPRs for coupled operands
1982 /// given an \p Opcode which uses VGPRs for coupled operands.
1983 LLVM_READONLY
1984 int32_t getAGPRFormOp(uint32_t Opcode);
1985
1986 /// \returns v_cmpx version of a v_cmp instruction.
1987 LLVM_READONLY
1988 int32_t getVCMPXOpFromVCMP(uint32_t Opcode);
1989
1990 const uint64_t RSRC_DATA_FORMAT = 0xf00000000000LL;
1991 const uint64_t RSRC_ELEMENT_SIZE_SHIFT = (32 + 19);
1992 const uint64_t RSRC_INDEX_STRIDE_SHIFT = (32 + 21);
1993 const uint64_t RSRC_TID_ENABLE = UINT64_C(1) << (32 + 23);
1994
1995} // end namespace AMDGPU
1996
1997namespace AMDGPU {
1998enum AsmComments : MachineInstr::AsmPrinterFlagTy {
1999 // For sgpr to vgpr spill instructions
2000 SGPR_SPILL = MachineInstr::TAsmComments
2001};
2002} // namespace AMDGPU
2003
2004namespace SI {
2005namespace KernelInputOffsets {
2006
2007/// Offsets in bytes from the start of the input buffer
2008enum Offsets {
2009 NGROUPS_X = 0,
2010 NGROUPS_Y = 4,
2011 NGROUPS_Z = 8,
2012 GLOBAL_SIZE_X = 12,
2013 GLOBAL_SIZE_Y = 16,
2014 GLOBAL_SIZE_Z = 20,
2015 LOCAL_SIZE_X = 24,
2016 LOCAL_SIZE_Y = 28,
2017 LOCAL_SIZE_Z = 32
2018};
2019
2020} // end namespace KernelInputOffsets
2021} // end namespace SI
2022
2023} // end namespace llvm
2024
2025#endif // LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
2026