1//===- SIInstrInfo.h - SI Instruction Info Interface ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Interface definition for SIInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
15#define LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
16
17#include "AMDGPUMIRFormatter.h"
18#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
19#include "SIRegisterInfo.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/CodeGen/TargetInstrInfo.h"
23#include "llvm/CodeGen/TargetSchedule.h"
24
25#define GET_INSTRINFO_HEADER
26#include "AMDGPUGenInstrInfo.inc"
27
28namespace llvm {
29
30class APInt;
31class GCNSubtarget;
32class LiveVariables;
33class MachineDominatorTree;
34class MachineRegisterInfo;
35class RegScavenger;
36class SIMachineFunctionInfo;
37class TargetRegisterClass;
38class ScheduleHazardRecognizer;
39
40constexpr unsigned DefaultMemoryClusterDWordsLimit = 8;
41
42/// Mark the MMO of a uniform load if there are no potentially clobbering stores
43/// on any path from the start of an entry function to this load.
44static const MachineMemOperand::Flags MONoClobber =
45 MachineMemOperand::MOTargetFlag1;
46
47/// Mark the MMO of a load as the last use.
48static const MachineMemOperand::Flags MOLastUse =
49 MachineMemOperand::MOTargetFlag2;
50
51/// Mark the MMO of cooperative load/store atomics.
52static const MachineMemOperand::Flags MOCooperative =
53 MachineMemOperand::MOTargetFlag3;
54
55struct V2PhysSCopyInfo {
56 // Operands that need to replaced by waterfall
57 SmallVector<MachineOperand *> MOs;
58 // Target physical registers replacing the MOs
59 SmallVector<Register> SGPRs;
60};
61/// Mark the MMO of accesses to memory locations that are
62/// never written to by other threads.
63static const MachineMemOperand::Flags MOThreadPrivate =
64 MachineMemOperand::MOTargetFlag4;
65
66/// Utility to store machine instructions worklist.
67struct SIInstrWorklist {
68 SIInstrWorklist() = default;
69
70 void insert(MachineInstr *MI);
71
72 MachineInstr *top() const {
73 const auto *iter = InstrList.begin();
74 return *iter;
75 }
76
77 void erase_top() {
78 const auto *iter = InstrList.begin();
79 InstrList.erase(I: iter);
80 }
81
82 bool empty() const { return InstrList.empty(); }
83
84 void clear() {
85 InstrList.clear();
86 DeferredList.clear();
87 }
88
89 bool isDeferred(MachineInstr *MI);
90
91 SetVector<MachineInstr *> &getDeferredList() { return DeferredList; }
92
93private:
94 /// InstrList contains the MachineInstrs.
95 SetVector<MachineInstr *> InstrList;
96 /// Deferred instructions are specific MachineInstr
97 /// that will be added by insert method.
98 SetVector<MachineInstr *> DeferredList;
99};
100
101// In namespace llvm so ADL finds it when SIInstrFlags predicates are
102// instantiated with MachineInstr (MachineInstr is in namespace llvm).
103inline uint64_t getTSFlags(const MachineInstr &MI) {
104 return MI.getDesc().TSFlags;
105}
106
107class SIInstrInfo final : public AMDGPUGenInstrInfo {
108 struct ThreeAddressUpdates;
109
110private:
111 const SIRegisterInfo RI;
112 const GCNSubtarget &ST;
113 TargetSchedModel SchedModel;
114 mutable std::unique_ptr<AMDGPUMIRFormatter> Formatter;
115
116 // The inverse predicate should have the negative value.
117 enum BranchPredicate {
118 INVALID_BR = 0,
119 SCC_TRUE = 1,
120 SCC_FALSE = -1,
121 VCCNZ = 2,
122 VCCZ = -2,
123 EXECNZ = -3,
124 EXECZ = 3
125 };
126
127 using SetVectorType = SmallSetVector<MachineInstr *, 32>;
128
129 static unsigned getBranchOpcode(BranchPredicate Cond);
130 static BranchPredicate getBranchPredicate(unsigned Opcode);
131
132public:
133 unsigned buildExtractSubReg(MachineBasicBlock::iterator MI,
134 MachineRegisterInfo &MRI,
135 const MachineOperand &SuperReg,
136 const TargetRegisterClass *SuperRC,
137 unsigned SubIdx,
138 const TargetRegisterClass *SubRC) const;
139 MachineOperand buildExtractSubRegOrImm(
140 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
141 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
142 unsigned SubIdx, const TargetRegisterClass *SubRC) const;
143
144private:
145 bool optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,
146 bool NeedInversion) const;
147
148 bool invertSCCUse(MachineInstr *SCCDef) const;
149
150 void swapOperands(MachineInstr &Inst) const;
151
152 std::pair<bool, MachineBasicBlock *>
153 moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
154 MachineDominatorTree *MDT = nullptr) const;
155
156 void lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
157 MachineDominatorTree *MDT = nullptr) const;
158
159 void lowerScalarAbs(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
160
161 void lowerScalarAbsDiff(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
162
163 void lowerScalarXnor(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
164
165 void splitScalarNotBinop(SIInstrWorklist &Worklist, MachineInstr &Inst,
166 unsigned Opcode) const;
167
168 void splitScalarBinOpN2(SIInstrWorklist &Worklist, MachineInstr &Inst,
169 unsigned Opcode) const;
170
171 void splitScalar64BitUnaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
172 unsigned Opcode, bool Swap = false) const;
173
174 void splitScalar64BitBinaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
175 unsigned Opcode,
176 MachineDominatorTree *MDT = nullptr) const;
177
178 void splitScalarSMulU64(SIInstrWorklist &Worklist, MachineInstr &Inst,
179 MachineDominatorTree *MDT) const;
180
181 void splitScalarSMulPseudo(SIInstrWorklist &Worklist, MachineInstr &Inst,
182 MachineDominatorTree *MDT) const;
183
184 void splitScalar64BitXnor(SIInstrWorklist &Worklist, MachineInstr &Inst,
185 MachineDominatorTree *MDT = nullptr) const;
186
187 void splitScalar64BitBCNT(SIInstrWorklist &Worklist,
188 MachineInstr &Inst) const;
189 void splitScalar64BitBFE(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
190 void splitScalar64BitCountOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
191 unsigned Opcode,
192 MachineDominatorTree *MDT = nullptr) const;
193 void movePackToVALU(SIInstrWorklist &Worklist, MachineRegisterInfo &MRI,
194 MachineInstr &Inst) const;
195
196 void addUsersToMoveToVALUWorklist(Register Reg, MachineRegisterInfo &MRI,
197 SIInstrWorklist &Worklist) const;
198
199 void addSCCDefUsersToVALUWorklist(const MachineOperand &Op,
200 MachineInstr &SCCDefInst,
201 SIInstrWorklist &Worklist,
202 Register NewCond = Register()) const;
203 void addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
204 SIInstrWorklist &Worklist) const;
205
206 const TargetRegisterClass *
207 getDestEquivalentVGPRClass(const MachineInstr &Inst) const;
208
209 bool checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
210 const MachineInstr &MIb) const;
211
212 Register findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const;
213
214 bool verifyCopy(const MachineInstr &MI, const MachineRegisterInfo &MRI,
215 StringRef &ErrInfo) const;
216
217 bool resultDependsOnExec(const MachineInstr &MI) const;
218
219 MachineInstr *convertToThreeAddressImpl(MachineInstr &MI,
220 ThreeAddressUpdates &Updates) const;
221
222protected:
223 /// If the specific machine instruction is a instruction that moves/copies
224 /// value from one register to another register return destination and source
225 /// registers as machine operands.
226 std::optional<DestSourcePair>
227 isCopyInstrImpl(const MachineInstr &MI) const override;
228
229 bool swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,
230 AMDGPU::OpName Src0OpName, MachineOperand &Src1,
231 AMDGPU::OpName Src1OpName) const;
232 bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx,
233 unsigned toIdx) const;
234 MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
235 unsigned OpIdx0,
236 unsigned OpIdx1) const override;
237
238public:
239 enum TargetOperandFlags {
240 MO_MASK = 0xf,
241
242 MO_NONE = 0,
243 // MO_GOTPCREL -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
244 MO_GOTPCREL = 1,
245 // MO_GOTPCREL32_LO -> symbol@gotpcrel32@lo -> R_AMDGPU_GOTPCREL32_LO.
246 MO_GOTPCREL32 = 2,
247 MO_GOTPCREL32_LO = 2,
248 // MO_GOTPCREL32_HI -> symbol@gotpcrel32@hi -> R_AMDGPU_GOTPCREL32_HI.
249 MO_GOTPCREL32_HI = 3,
250 // MO_GOTPCREL64 -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
251 MO_GOTPCREL64 = 4,
252 // MO_REL32_LO -> symbol@rel32@lo -> R_AMDGPU_REL32_LO.
253 MO_REL32 = 5,
254 MO_REL32_LO = 5,
255 // MO_REL32_HI -> symbol@rel32@hi -> R_AMDGPU_REL32_HI.
256 MO_REL32_HI = 6,
257 MO_REL64 = 7,
258
259 MO_FAR_BRANCH_OFFSET = 8,
260
261 MO_ABS32_LO = 9,
262 MO_ABS32_HI = 10,
263 MO_ABS64 = 11,
264 };
265
266 explicit SIInstrInfo(const GCNSubtarget &ST);
267
268 const SIRegisterInfo &getRegisterInfo() const {
269 return RI;
270 }
271
272 const GCNSubtarget &getSubtarget() const {
273 return ST;
274 }
275
276 bool isReMaterializableImpl(const MachineInstr &MI) const override;
277
278 bool isIgnorableUse(const MachineOperand &MO) const override;
279
280 bool isSafeToSink(MachineInstr &MI, MachineBasicBlock *SuccToSinkTo,
281 MachineCycleInfo *CI) const override;
282
283 bool areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, int64_t &Offset0,
284 int64_t &Offset1) const override;
285
286 bool isGlobalMemoryObject(const MachineInstr *MI) const override;
287
288 bool getMemOperandsWithOffsetWidth(
289 const MachineInstr &LdSt,
290 SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
291 bool &OffsetIsScalable, LocationSize &Width,
292 const TargetRegisterInfo *TRI) const final;
293
294 bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
295 int64_t Offset1, bool OffsetIsScalable1,
296 ArrayRef<const MachineOperand *> BaseOps2,
297 int64_t Offset2, bool OffsetIsScalable2,
298 unsigned ClusterSize,
299 unsigned NumBytes) const override;
300
301 bool shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0,
302 int64_t Offset1, unsigned NumLoads) const override;
303
304 void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
305 const DebugLoc &DL, Register DestReg, Register SrcReg,
306 bool KillSrc, bool RenamableDest = false,
307 bool RenamableSrc = false) const override;
308
309private:
310 void storeRegToStackSlotImpl(MachineBasicBlock &MBB,
311 MachineBasicBlock::iterator MI, Register SrcReg,
312 bool isKill, int FrameIndex,
313 const TargetRegisterClass *RC, Register VReg,
314 MachineInstr::MIFlag Flags, bool NeedsCFI) const;
315
316public:
317 void storeRegToStackSlotCFI(MachineBasicBlock &MBB,
318 MachineBasicBlock::iterator MI, Register SrcReg,
319 bool isKill, int FrameIndex,
320 const TargetRegisterClass *RC) const;
321
322 bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg,
323 int64_t &ImmVal) const override;
324
325 std::optional<int64_t> getImmOrMaterializedImm(MachineOperand &Op) const;
326
327 unsigned getVectorRegSpillSaveOpcode(Register Reg,
328 const TargetRegisterClass *RC,
329 unsigned Size,
330 const SIMachineFunctionInfo &MFI,
331 bool NeedsCFI) const;
332 unsigned
333 getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC,
334 unsigned Size,
335 const SIMachineFunctionInfo &MFI) const;
336
337 void storeRegToStackSlot(
338 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
339 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
340 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
341
342 void loadRegFromStackSlot(
343 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
344 int FrameIndex, const TargetRegisterClass *RC, Register VReg,
345 unsigned SubReg = 0,
346 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
347
348 bool expandPostRAPseudo(MachineInstr &MI) const override;
349
350 void
351 reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
352 Register DestReg, unsigned SubIdx, const MachineInstr &Orig,
353 LaneBitmask UsedLanes = LaneBitmask::getAll()) const override;
354
355 // Splits a V_MOV_B64_DPP_PSEUDO opcode into a pair of v_mov_b32_dpp
356 // instructions. Returns a pair of generated instructions.
357 // Can split either post-RA with physical registers or pre-RA with
358 // virtual registers. In latter case IR needs to be in SSA form and
359 // and a REG_SEQUENCE is produced to define original register.
360 std::pair<MachineInstr*, MachineInstr*>
361 expandMovDPP64(MachineInstr &MI) const;
362
363 // Returns an opcode that can be used to move a value to a \p DstRC
364 // register. If there is no hardware instruction that can store to \p
365 // DstRC, then AMDGPU::COPY is returned.
366 unsigned getMovOpcode(const TargetRegisterClass *DstRC) const;
367
368 const MCInstrDesc &getIndirectRegWriteMovRelPseudo(unsigned VecSize,
369 unsigned EltSize,
370 bool IsSGPR) const;
371
372 const MCInstrDesc &getIndirectGPRIDXPseudo(unsigned VecSize,
373 bool IsIndirectSrc) const;
374 LLVM_READONLY
375 int commuteOpcode(unsigned Opc) const;
376
377 LLVM_READONLY
378 inline int commuteOpcode(const MachineInstr &MI) const {
379 return commuteOpcode(Opc: MI.getOpcode());
380 }
381
382 bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0,
383 unsigned &SrcOpIdx1) const override;
384
385 bool findCommutedOpIndices(const MCInstrDesc &Desc, unsigned &SrcOpIdx0,
386 unsigned &SrcOpIdx1) const;
387
388 bool isBranchOffsetInRange(unsigned BranchOpc,
389 int64_t BrOffset) const override;
390
391 MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;
392
393 /// Return whether the block terminate with divergent branch.
394 /// Note this only work before lowering the pseudo control flow instructions.
395 bool hasDivergentBranch(const MachineBasicBlock *MBB) const;
396
397 void insertIndirectBranch(MachineBasicBlock &MBB,
398 MachineBasicBlock &NewDestBB,
399 MachineBasicBlock &RestoreBB, const DebugLoc &DL,
400 int64_t BrOffset, RegScavenger *RS) const override;
401
402 bool analyzeBranchImpl(MachineBasicBlock &MBB,
403 MachineBasicBlock::iterator I,
404 MachineBasicBlock *&TBB,
405 MachineBasicBlock *&FBB,
406 SmallVectorImpl<MachineOperand> &Cond,
407 bool AllowModify) const;
408
409 bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
410 MachineBasicBlock *&FBB,
411 SmallVectorImpl<MachineOperand> &Cond,
412 bool AllowModify = false) const override;
413
414 unsigned removeBranch(MachineBasicBlock &MBB,
415 int *BytesRemoved = nullptr) const override;
416
417 unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
418 MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
419 const DebugLoc &DL,
420 int *BytesAdded = nullptr) const override;
421
422 bool reverseBranchCondition(
423 SmallVectorImpl<MachineOperand> &Cond) const override;
424
425 bool canInsertSelect(const MachineBasicBlock &MBB,
426 ArrayRef<MachineOperand> Cond, Register DstReg,
427 Register TrueReg, Register FalseReg, int &CondCycles,
428 int &TrueCycles, int &FalseCycles) const override;
429
430 void insertSelect(MachineBasicBlock &MBB,
431 MachineBasicBlock::iterator I, const DebugLoc &DL,
432 Register DstReg, ArrayRef<MachineOperand> Cond,
433 Register TrueReg, Register FalseReg) const override;
434
435 bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
436 Register &SrcReg2, int64_t &CmpMask,
437 int64_t &CmpValue) const override;
438
439 bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
440 Register SrcReg2, int64_t CmpMask, int64_t CmpValue,
441 const MachineRegisterInfo *MRI) const override;
442
443 bool
444 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
445 const MachineInstr &MIb) const override;
446
447 static bool isFoldableCopy(const MachineInstr &MI);
448 static unsigned getFoldableCopySrcIdx(const MachineInstr &MI);
449
450 void removeModOperands(MachineInstr &MI) const;
451
452 void mutateAndCleanupImplicit(MachineInstr &MI,
453 const MCInstrDesc &NewDesc) const;
454
455 /// Return the extracted immediate value in a subregister use from a constant
456 /// materialized in a super register.
457 ///
458 /// e.g. %imm = S_MOV_B64 K[0:63]
459 /// USE %imm.sub1
460 /// This will return K[32:63]
461 static std::optional<int64_t> extractSubregFromImm(int64_t ImmVal,
462 unsigned SubRegIndex);
463
464 bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
465 MachineRegisterInfo *MRI) const final;
466
467 unsigned getMachineCSELookAheadLimit() const override { return 500; }
468
469 MachineInstr *convertToThreeAddress(MachineInstr &MI, LiveVariables *LV,
470 LiveIntervals *LIS) const override;
471
472 bool isSchedulingBoundary(const MachineInstr &MI,
473 const MachineBasicBlock *MBB,
474 const MachineFunction &MF) const override;
475
476 static bool isSALU(const MachineInstr &MI) {
477 return SIInstrFlags::isSALU(O: MI);
478 }
479
480 bool isSALU(uint32_t Opcode) const {
481 return SIInstrFlags::isSALU(O: get(Opcode));
482 }
483
484 static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA) {
485 if (!AllowLDSDMA && isLDSDMA(MI))
486 return false;
487
488 return SIInstrFlags::isVALU(O: MI);
489 }
490
491 /// LDSDMA instructions act as both VALU and memory instructions, thus
492 /// we also tag them as VALU. However, in many places, we do not actually want
493 /// to include LDSDMA instructions in this query. By setting \p AllowLDSDMA to
494 /// false, this will return false for LDSDMA instructions.
495 bool isVALU(uint32_t Opcode, bool AllowLDSDMA) const {
496 if (!AllowLDSDMA && isLDSDMA(Opcode))
497 return false;
498
499 return SIInstrFlags::isVALU(O: get(Opcode));
500 }
501
502 static bool isImage(const MachineInstr &MI) {
503 return SIInstrFlags::isImage(O: MI);
504 }
505
506 bool isImage(uint32_t Opcode) const {
507 return SIInstrFlags::isImage(O: get(Opcode));
508 }
509
510 static bool isVMEM(const MachineInstr &MI) {
511 return SIInstrFlags::isVMEM(O: MI);
512 }
513
514 bool isVMEM(uint32_t Opcode) const {
515 return SIInstrFlags::isVMEM(O: get(Opcode));
516 }
517
518 /// True if MI implicitly drains XCNT.
519 static bool isXcntDrain(const MachineInstr &MI);
520
521 static bool isSOP1(const MachineInstr &MI) {
522 return SIInstrFlags::isSOP1(O: MI);
523 }
524
525 bool isSOP1(uint32_t Opcode) const {
526 return SIInstrFlags::isSOP1(O: get(Opcode));
527 }
528
529 static bool isSOP2(const MachineInstr &MI) {
530 return SIInstrFlags::isSOP2(O: MI);
531 }
532
533 bool isSOP2(uint32_t Opcode) const {
534 return SIInstrFlags::isSOP2(O: get(Opcode));
535 }
536
537 static bool isSOPC(const MachineInstr &MI) {
538 return SIInstrFlags::isSOPC(O: MI);
539 }
540
541 bool isSOPC(uint32_t Opcode) const {
542 return SIInstrFlags::isSOPC(O: get(Opcode));
543 }
544
545 static bool isSOPK(const MachineInstr &MI) {
546 return SIInstrFlags::isSOPK(O: MI);
547 }
548
549 bool isSOPK(uint32_t Opcode) const {
550 return SIInstrFlags::isSOPK(O: get(Opcode));
551 }
552
553 static bool isSOPP(const MachineInstr &MI) {
554 return SIInstrFlags::isSOPP(O: MI);
555 }
556
557 bool isSOPP(uint32_t Opcode) const {
558 return SIInstrFlags::isSOPP(O: get(Opcode));
559 }
560
561 static bool isPacked(const MachineInstr &MI) {
562 return SIInstrFlags::isPacked(O: MI);
563 }
564
565 bool isPacked(uint32_t Opcode) const {
566 return SIInstrFlags::isPacked(O: get(Opcode));
567 }
568
569 static bool isVOP1(const MachineInstr &MI) {
570 return SIInstrFlags::isVOP1(O: MI);
571 }
572
573 bool isVOP1(uint32_t Opcode) const {
574 return SIInstrFlags::isVOP1(O: get(Opcode));
575 }
576
577 static bool isVOP2(const MachineInstr &MI) {
578 return SIInstrFlags::isVOP2(O: MI);
579 }
580
581 bool isVOP2(uint32_t Opcode) const {
582 return SIInstrFlags::isVOP2(O: get(Opcode));
583 }
584
585 static bool isVOP3(const MCInstrDesc &Desc) {
586 return SIInstrFlags::isVOP3(O: Desc);
587 }
588
589 static bool isVOP3(const MachineInstr &MI) { return isVOP3(Desc: MI.getDesc()); }
590
591 bool isVOP3(uint32_t Opcode) const { return isVOP3(Desc: get(Opcode)); }
592
593 static bool isSDWA(const MachineInstr &MI) {
594 return SIInstrFlags::isSDWA(O: MI);
595 }
596
597 bool isSDWA(uint32_t Opcode) const {
598 return SIInstrFlags::isSDWA(O: get(Opcode));
599 }
600
601 static bool isVOPC(const MachineInstr &MI) {
602 return SIInstrFlags::isVOPC(O: MI);
603 }
604
605 bool isVOPC(uint32_t Opcode) const {
606 return SIInstrFlags::isVOPC(O: get(Opcode));
607 }
608
609 static bool isMUBUF(const MachineInstr &MI) {
610 return SIInstrFlags::isMUBUF(O: MI);
611 }
612
613 bool isMUBUF(uint32_t Opcode) const {
614 return SIInstrFlags::isMUBUF(O: get(Opcode));
615 }
616
617 static bool isMTBUF(const MachineInstr &MI) {
618 return SIInstrFlags::isMTBUF(O: MI);
619 }
620
621 bool isMTBUF(uint32_t Opcode) const {
622 return SIInstrFlags::isMTBUF(O: get(Opcode));
623 }
624
625 static bool isBUF(const MachineInstr &MI) {
626 return isMUBUF(MI) || isMTBUF(MI);
627 }
628
629 static bool isSMRD(const MachineInstr &MI) {
630 return SIInstrFlags::isSMRD(O: MI);
631 }
632
633 bool isSMRD(uint32_t Opcode) const {
634 return SIInstrFlags::isSMRD(O: get(Opcode));
635 }
636
637 bool isBufferSMRD(const MachineInstr &MI) const;
638
639 static bool isDS(const MachineInstr &MI) { return SIInstrFlags::isDS(O: MI); }
640
641 bool isDS(uint32_t Opcode) const { return SIInstrFlags::isDS(O: get(Opcode)); }
642
643 static bool isLDSDMA(const MachineInstr &MI) {
644 return (SIInstrFlags::isVALU(O: MI) && (isMUBUF(MI) || isFLAT(MI))) ||
645 SIInstrFlags::usesTENSOR_CNT(O: MI);
646 }
647
648 bool isLDSDMA(uint32_t Opcode) const {
649 return (SIInstrFlags::isVALU(O: get(Opcode)) &&
650 (isMUBUF(Opcode) || isFLAT(Opcode))) ||
651 SIInstrFlags::usesTENSOR_CNT(O: get(Opcode));
652 }
653
654 static bool isGWS(const MachineInstr &MI) { return SIInstrFlags::isGWS(O: MI); }
655
656 bool isGWS(uint32_t Opcode) const { return SIInstrFlags::isGWS(O: get(Opcode)); }
657
658 bool isAlwaysGDS(uint32_t Opcode) const;
659
660 static bool isMIMG(const MachineInstr &MI) {
661 return SIInstrFlags::isMIMG(O: MI);
662 }
663
664 bool isMIMG(uint32_t Opcode) const {
665 return SIInstrFlags::isMIMG(O: get(Opcode));
666 }
667
668 static bool isVIMAGE(const MachineInstr &MI) {
669 return SIInstrFlags::isVIMAGE(O: MI);
670 }
671
672 bool isVIMAGE(uint32_t Opcode) const {
673 return SIInstrFlags::isVIMAGE(O: get(Opcode));
674 }
675
676 static bool isVSAMPLE(const MachineInstr &MI) {
677 return SIInstrFlags::isVSAMPLE(O: MI);
678 }
679
680 bool isVSAMPLE(uint32_t Opcode) const {
681 return SIInstrFlags::isVSAMPLE(O: get(Opcode));
682 }
683
684 static bool isGather4(const MachineInstr &MI) {
685 return SIInstrFlags::isGather4(O: MI);
686 }
687
688 bool isGather4(uint32_t Opcode) const {
689 return SIInstrFlags::isGather4(O: get(Opcode));
690 }
691
692 static bool isFLAT(const MachineInstr &MI) {
693 return SIInstrFlags::isFLAT(O: MI);
694 }
695
696 // Is a FLAT encoded instruction which accesses a specific segment,
697 // i.e. global_* or scratch_*.
698 static bool isSegmentSpecificFLAT(const MachineInstr &MI) {
699 return SIInstrFlags::isSegmentSpecificFLAT(O: MI);
700 }
701
702 bool isSegmentSpecificFLAT(uint32_t Opcode) const {
703 return SIInstrFlags::isSegmentSpecificFLAT(O: get(Opcode));
704 }
705
706 static bool isFLATGlobal(const MachineInstr &MI) {
707 return SIInstrFlags::isFlatGlobal(O: MI);
708 }
709
710 bool isFLATGlobal(uint32_t Opcode) const {
711 return SIInstrFlags::isFlatGlobal(O: get(Opcode));
712 }
713
714 static bool isFLATScratch(const MachineInstr &MI) {
715 return SIInstrFlags::isFlatScratch(O: MI);
716 }
717
718 bool isFLATScratch(uint32_t Opcode) const {
719 return SIInstrFlags::isFlatScratch(O: get(Opcode));
720 }
721
722 // Any FLAT encoded instruction, including global_* and scratch_*.
723 bool isFLAT(uint32_t Opcode) const {
724 return SIInstrFlags::isFLAT(O: get(Opcode));
725 }
726
727 /// \returns true for SCRATCH_ instructions, or FLAT/BUF instructions unless
728 /// the MMOs do not include scratch.
729 /// Conservatively correct; will return true if \p MI cannot be proven
730 /// to not hit scratch.
731 bool mayAccessScratch(const MachineInstr &MI) const;
732
733 /// \returns true for FLAT instructions that can access VMEM.
734 bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const;
735
736 /// \returns true for FLAT instructions that can access LDS.
737 bool mayAccessLDSThroughFlat(const MachineInstr &MI) const;
738
739 static bool isBlockLoadStore(uint32_t Opcode) {
740 switch (Opcode) {
741 case AMDGPU::SI_BLOCK_SPILL_V1024_SAVE:
742 case AMDGPU::SI_BLOCK_SPILL_V1024_CFI_SAVE:
743 case AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE:
744 case AMDGPU::SCRATCH_STORE_BLOCK_SADDR:
745 case AMDGPU::SCRATCH_LOAD_BLOCK_SADDR:
746 case AMDGPU::SCRATCH_STORE_BLOCK_SVS:
747 case AMDGPU::SCRATCH_LOAD_BLOCK_SVS:
748 return true;
749 default:
750 return false;
751 }
752 }
753
754 static bool setsSCCIfResultIsNonZero(const MachineInstr &MI) {
755 switch (MI.getOpcode()) {
756 case AMDGPU::S_ABSDIFF_I32:
757 case AMDGPU::S_ABS_I32:
758 case AMDGPU::S_AND_B32:
759 case AMDGPU::S_AND_B64:
760 case AMDGPU::S_ANDN2_B32:
761 case AMDGPU::S_ANDN2_B64:
762 case AMDGPU::S_ASHR_I32:
763 case AMDGPU::S_ASHR_I64:
764 case AMDGPU::S_BCNT0_I32_B32:
765 case AMDGPU::S_BCNT0_I32_B64:
766 case AMDGPU::S_BCNT1_I32_B32:
767 case AMDGPU::S_BCNT1_I32_B64:
768 case AMDGPU::S_BFE_I32:
769 case AMDGPU::S_BFE_I64:
770 case AMDGPU::S_BFE_U32:
771 case AMDGPU::S_BFE_U64:
772 case AMDGPU::S_LSHL_B32:
773 case AMDGPU::S_LSHL_B64:
774 case AMDGPU::S_LSHR_B32:
775 case AMDGPU::S_LSHR_B64:
776 case AMDGPU::S_NAND_B32:
777 case AMDGPU::S_NAND_B64:
778 case AMDGPU::S_NOR_B32:
779 case AMDGPU::S_NOR_B64:
780 case AMDGPU::S_NOT_B32:
781 case AMDGPU::S_NOT_B64:
782 case AMDGPU::S_OR_B32:
783 case AMDGPU::S_OR_B64:
784 case AMDGPU::S_ORN2_B32:
785 case AMDGPU::S_ORN2_B64:
786 case AMDGPU::S_QUADMASK_B32:
787 case AMDGPU::S_QUADMASK_B64:
788 case AMDGPU::S_WQM_B32:
789 case AMDGPU::S_WQM_B64:
790 case AMDGPU::S_XNOR_B32:
791 case AMDGPU::S_XNOR_B64:
792 case AMDGPU::S_XOR_B32:
793 case AMDGPU::S_XOR_B64:
794 return true;
795 default:
796 return false;
797 }
798 }
799
800 static bool isEXP(const MachineInstr &MI) { return SIInstrFlags::isEXP(O: MI); }
801
802 static bool isDualSourceBlendEXP(const MachineInstr &MI) {
803 if (!isEXP(MI))
804 return false;
805 unsigned Target = MI.getOperand(i: 0).getImm();
806 return Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND0 ||
807 Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND1;
808 }
809
810 bool isEXP(uint32_t Opcode) const { return SIInstrFlags::isEXP(O: get(Opcode)); }
811
812 static bool isAtomicNoRet(const MachineInstr &MI) {
813 return SIInstrFlags::isAtomicNoRet(O: MI);
814 }
815
816 bool isAtomicNoRet(uint32_t Opcode) const {
817 return SIInstrFlags::isAtomicNoRet(O: get(Opcode));
818 }
819
820 static bool isAtomicRet(const MachineInstr &MI) {
821 return SIInstrFlags::isAtomicRet(O: MI);
822 }
823
824 bool isAtomicRet(uint32_t Opcode) const {
825 return SIInstrFlags::isAtomicRet(O: get(Opcode));
826 }
827
828 static bool isAtomic(const MachineInstr &MI) {
829 return SIInstrFlags::isAtomic(O: MI);
830 }
831
832 bool isAtomic(uint32_t Opcode) const {
833 return SIInstrFlags::isAtomic(O: get(Opcode));
834 }
835
836 static bool mayWriteLDSThroughDMA(const MachineInstr &MI) {
837 unsigned Opc = MI.getOpcode();
838 // Exclude instructions that read FROM LDS (not write to it)
839 return isLDSDMA(MI) && Opc != AMDGPU::BUFFER_STORE_LDS_DWORD &&
840 Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d2 &&
841 Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d4;
842 }
843
844 static bool isSBarrierSCCWrite(unsigned Opcode) {
845 return Opcode == AMDGPU::S_BARRIER_LEAVE ||
846 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM ||
847 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0;
848 }
849
850 static bool isCBranchVCCZRead(const MachineInstr &MI) {
851 unsigned Opc = MI.getOpcode();
852 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
853 !MI.getOperand(i: 1).isUndef();
854 }
855
856 static bool isWQM(const MachineInstr &MI) { return SIInstrFlags::isWQM(O: MI); }
857
858 bool isWQM(uint32_t Opcode) const { return SIInstrFlags::isWQM(O: get(Opcode)); }
859
860 static bool isDisableWQM(const MachineInstr &MI) {
861 return SIInstrFlags::isDisableWQM(O: MI);
862 }
863
864 bool isDisableWQM(uint32_t Opcode) const {
865 return SIInstrFlags::isDisableWQM(O: get(Opcode));
866 }
867
868 // SI_SPILL_S32_TO_VGPR and SI_RESTORE_S32_FROM_VGPR form a special case of
869 // SGPRs spilling to VGPRs which are SGPR spills but from VALU instructions
870 // therefore we need an explicit check for them since just checking if the
871 // Spill bit is set and what instruction type it came from misclassifies
872 // them.
873 static bool isVGPRSpill(const MachineInstr &MI) {
874 return MI.getOpcode() != AMDGPU::SI_SPILL_S32_TO_VGPR &&
875 MI.getOpcode() != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
876 (isSpill(MI) && isVALU(MI, /*AllowLDSDMA=*/AllowLDSDMA: true));
877 }
878
879 bool isVGPRSpill(uint32_t Opcode) const {
880 return Opcode != AMDGPU::SI_SPILL_S32_TO_VGPR &&
881 Opcode != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
882 (isSpill(Opcode) && isVALU(Opcode, /*AllowLDSDMA=*/AllowLDSDMA: true));
883 }
884
885 static bool isSGPRSpill(const MachineInstr &MI) {
886 return MI.getOpcode() == AMDGPU::SI_SPILL_S32_TO_VGPR ||
887 MI.getOpcode() == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
888 (isSpill(MI) && isSALU(MI));
889 }
890
891 bool isSGPRSpill(uint32_t Opcode) const {
892 return Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR ||
893 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
894 (isSpill(Opcode) && isSALU(Opcode));
895 }
896
897 bool isSpill(uint32_t Opcode) const {
898 return SIInstrFlags::isSpill(O: get(Opcode));
899 }
900
901 static bool isSpill(const MCInstrDesc &Desc) {
902 return SIInstrFlags::isSpill(O: Desc);
903 }
904
905 static bool isSpill(const MachineInstr &MI) { return isSpill(Desc: MI.getDesc()); }
906
907 static bool isWWMRegSpillOpcode(uint32_t Opcode) {
908 return Opcode == AMDGPU::SI_SPILL_WWM_V32_SAVE ||
909 Opcode == AMDGPU::SI_SPILL_WWM_AV32_SAVE ||
910 Opcode == AMDGPU::SI_SPILL_WWM_V32_RESTORE ||
911 Opcode == AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
912 }
913
914 static bool isChainCallOpcode(uint64_t Opcode) {
915 return Opcode == AMDGPU::SI_CS_CHAIN_TC_W32 ||
916 Opcode == AMDGPU::SI_CS_CHAIN_TC_W64;
917 }
918
919 static bool isDPP(const MachineInstr &MI) { return SIInstrFlags::isDPP(O: MI); }
920
921 bool isDPP(uint32_t Opcode) const { return SIInstrFlags::isDPP(O: get(Opcode)); }
922
923 static bool isTRANS(const MachineInstr &MI) {
924 return SIInstrFlags::isTRANS(O: MI);
925 }
926
927 bool isTRANS(uint32_t Opcode) const {
928 return SIInstrFlags::isTRANS(O: get(Opcode));
929 }
930
931 static bool isVOP3P(const MachineInstr &MI) {
932 return SIInstrFlags::isVOP3P(O: MI);
933 }
934
935 bool isVOP3P(uint32_t Opcode) const {
936 return SIInstrFlags::isVOP3P(O: get(Opcode));
937 }
938
939 bool isVOP3PMix(const MachineInstr &MI) const {
940 return isVOP3PMix(Opcode: MI.getOpcode());
941 }
942
943 bool isVOP3PMix(uint16_t Opcode) const {
944 switch (Opcode) {
945 case AMDGPU::V_FMA_MIXHI_F16:
946 case AMDGPU::V_FMA_MIXLO_F16:
947 case AMDGPU::V_FMA_MIX_F32:
948 case AMDGPU::V_MAD_MIXHI_F16:
949 case AMDGPU::V_MAD_MIXLO_F16:
950 case AMDGPU::V_MAD_MIX_F32:
951 return true;
952 default:
953 return false;
954 }
955 }
956
957 static bool isVINTRP(const MachineInstr &MI) {
958 return SIInstrFlags::isVINTRP(O: MI);
959 }
960
961 bool isVINTRP(uint32_t Opcode) const {
962 return SIInstrFlags::isVINTRP(O: get(Opcode));
963 }
964
965 static bool isMAI(const MCInstrDesc &Desc) {
966 return SIInstrFlags::isMAI(O: Desc);
967 }
968
969 static bool isMAI(const MachineInstr &MI) { return isMAI(Desc: MI.getDesc()); }
970
971 bool isMAI(uint32_t Opcode) const { return isMAI(Desc: get(Opcode)); }
972
973 static bool isMFMA(const MachineInstr &MI) {
974 return isMAI(MI) && MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
975 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
976 }
977
978 bool isMFMA(uint32_t Opcode) const {
979 return isMAI(Opcode) && Opcode != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
980 Opcode != AMDGPU::V_ACCVGPR_READ_B32_e64;
981 }
982
983 static bool isDOT(const MachineInstr &MI) { return SIInstrFlags::isDOT(O: MI); }
984
985 static bool isWMMA(const MachineInstr &MI) {
986 return SIInstrFlags::isWMMA(O: MI);
987 }
988
989 bool isWMMA(uint32_t Opcode) const {
990 return SIInstrFlags::isWMMA(O: get(Opcode));
991 }
992
993 static bool isMFMAorWMMA(const MachineInstr &MI) {
994 return isMFMA(MI) || isWMMA(MI) || isSWMMAC(MI);
995 }
996
997 bool isMFMAorWMMA(uint32_t Opcode) const {
998 return isMFMA(Opcode) || isWMMA(Opcode) || isSWMMAC(Opcode);
999 }
1000
1001 static bool isSWMMAC(const MachineInstr &MI) {
1002 return SIInstrFlags::isSWMMAC(O: MI);
1003 }
1004
1005 bool isSWMMAC(uint32_t Opcode) const {
1006 return SIInstrFlags::isSWMMAC(O: get(Opcode));
1007 }
1008
1009 bool isDOT(uint32_t Opcode) const { return SIInstrFlags::isDOT(O: get(Opcode)); }
1010
1011 bool isXDLWMMA(const MachineInstr &MI) const;
1012
1013 bool isXDL(const MachineInstr &MI) const;
1014
1015 static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opc: Opcode); }
1016
1017 static bool isLDSDIR(const MachineInstr &MI) {
1018 return SIInstrFlags::isLDSDIR(O: MI);
1019 }
1020
1021 bool isLDSDIR(uint32_t Opcode) const {
1022 return SIInstrFlags::isLDSDIR(O: get(Opcode));
1023 }
1024
1025 static bool isVINTERP(const MachineInstr &MI) {
1026 return SIInstrFlags::isVINTERP(O: MI);
1027 }
1028
1029 bool isVINTERP(uint32_t Opcode) const {
1030 return SIInstrFlags::isVINTERP(O: get(Opcode));
1031 }
1032
1033 static bool isScalarUnit(const MachineInstr &MI) {
1034 return SIInstrFlags::isSALU(O: MI) || SIInstrFlags::isSMRD(O: MI);
1035 }
1036
1037 static bool usesVM_CNT(const MachineInstr &MI) {
1038 return SIInstrFlags::usesVM_CNT(O: MI);
1039 }
1040
1041 static bool usesLGKM_CNT(const MachineInstr &MI) {
1042 return SIInstrFlags::usesLGKM_CNT(O: MI);
1043 }
1044
1045 static bool usesASYNC_CNT(const MachineInstr &MI) {
1046 return SIInstrFlags::usesASYNC_CNT(O: MI);
1047 }
1048
1049 bool usesASYNC_CNT(uint32_t Opcode) const {
1050 return SIInstrFlags::usesASYNC_CNT(O: get(Opcode));
1051 }
1052
1053 static bool usesTENSOR_CNT(const MachineInstr &MI) {
1054 return MI.getDesc().TSFlags & SIInstrFlags::TENSOR_CNT;
1055 }
1056
1057 bool usesTENSOR_CNT(uint32_t Opcode) const {
1058 return get(Opcode).TSFlags & SIInstrFlags::TENSOR_CNT;
1059 }
1060
1061 // Most sopk treat the immediate as a signed 16-bit, however some
1062 // use it as unsigned.
1063 static bool sopkIsZext(unsigned Opcode) {
1064 return Opcode == AMDGPU::S_CMPK_EQ_U32 || Opcode == AMDGPU::S_CMPK_LG_U32 ||
1065 Opcode == AMDGPU::S_CMPK_GT_U32 || Opcode == AMDGPU::S_CMPK_GE_U32 ||
1066 Opcode == AMDGPU::S_CMPK_LT_U32 || Opcode == AMDGPU::S_CMPK_LE_U32 ||
1067 Opcode == AMDGPU::S_GETREG_B32 ||
1068 Opcode == AMDGPU::S_GETREG_B32_const;
1069 }
1070
1071 /// \returns true if this is an s_store_dword* instruction. This is more
1072 /// specific than isSMEM && mayStore.
1073 static bool isScalarStore(const MachineInstr &MI) {
1074 return SIInstrFlags::isScalarStore(O: MI);
1075 }
1076
1077 bool isScalarStore(uint32_t Opcode) const {
1078 return SIInstrFlags::isScalarStore(O: get(Opcode));
1079 }
1080
1081 static bool isFixedSize(const MachineInstr &MI) {
1082 return SIInstrFlags::isFixedSize(O: MI);
1083 }
1084
1085 bool isFixedSize(uint32_t Opcode) const {
1086 return SIInstrFlags::isFixedSize(O: get(Opcode));
1087 }
1088
1089 static bool hasFPClamp(const MachineInstr &MI) {
1090 return SIInstrFlags::hasFPClamp(O: MI);
1091 }
1092
1093 bool hasFPClamp(uint32_t Opcode) const {
1094 return SIInstrFlags::hasFPClamp(O: get(Opcode));
1095 }
1096
1097 static bool hasIntClamp(const MachineInstr &MI) {
1098 return SIInstrFlags::hasIntClamp(O: MI);
1099 }
1100
1101 static bool hasSameClamp(const MachineInstr &A, const MachineInstr &B) {
1102 const MCInstrDesc &DA = A.getDesc(), &DB = B.getDesc();
1103 return SIInstrFlags::hasFPClamp(O: DA) == SIInstrFlags::hasFPClamp(O: DB) &&
1104 SIInstrFlags::hasIntClamp(O: DA) == SIInstrFlags::hasIntClamp(O: DB) &&
1105 SIInstrFlags::hasClampLo(O: DA) == SIInstrFlags::hasClampLo(O: DB) &&
1106 SIInstrFlags::hasClampHi(O: DA) == SIInstrFlags::hasClampHi(O: DB);
1107 }
1108
1109 static bool usesFPDPRounding(const MachineInstr &MI) {
1110 return SIInstrFlags::usesFPDPRounding(O: MI);
1111 }
1112
1113 bool usesFPDPRounding(uint32_t Opcode) const {
1114 return SIInstrFlags::usesFPDPRounding(O: get(Opcode));
1115 }
1116
1117 static bool isFPAtomic(const MachineInstr &MI) {
1118 return SIInstrFlags::isFPAtomic(O: MI);
1119 }
1120
1121 bool isFPAtomic(uint32_t Opcode) const {
1122 return SIInstrFlags::isFPAtomic(O: get(Opcode));
1123 }
1124
1125 static bool isNeverUniform(const MachineInstr &MI) {
1126 return SIInstrFlags::isNeverUniform(O: MI);
1127 }
1128
1129 // Check to see if opcode is for a barrier start. Pre gfx12 this is just the
1130 // S_BARRIER, but after support for S_BARRIER_SIGNAL* / S_BARRIER_WAIT we want
1131 // to check for the barrier start (S_BARRIER_SIGNAL*)
1132 bool isBarrierStart(unsigned Opcode) const {
1133 return Opcode == AMDGPU::S_BARRIER ||
1134 Opcode == AMDGPU::S_BARRIER_SIGNAL_M0 ||
1135 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0 ||
1136 Opcode == AMDGPU::S_BARRIER_SIGNAL_IMM ||
1137 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM;
1138 }
1139
1140 bool isBarrier(unsigned Opcode) const {
1141 return isBarrierStart(Opcode) || Opcode == AMDGPU::S_BARRIER_WAIT ||
1142 Opcode == AMDGPU::S_BARRIER_INIT_M0 ||
1143 Opcode == AMDGPU::S_BARRIER_INIT_IMM ||
1144 Opcode == AMDGPU::S_BARRIER_JOIN_IMM ||
1145 Opcode == AMDGPU::S_BARRIER_LEAVE || Opcode == AMDGPU::DS_GWS_INIT ||
1146 Opcode == AMDGPU::DS_GWS_BARRIER;
1147 }
1148
1149 static bool isLoadMonitor(unsigned Opc) {
1150 switch (Opc) {
1151 case AMDGPU::GLOBAL_LOAD_MONITOR_B32:
1152 case AMDGPU::GLOBAL_LOAD_MONITOR_B32_SADDR:
1153 case AMDGPU::GLOBAL_LOAD_MONITOR_B64:
1154 case AMDGPU::GLOBAL_LOAD_MONITOR_B64_SADDR:
1155 case AMDGPU::GLOBAL_LOAD_MONITOR_B128:
1156 case AMDGPU::GLOBAL_LOAD_MONITOR_B128_SADDR:
1157 case AMDGPU::FLAT_LOAD_MONITOR_B32:
1158 case AMDGPU::FLAT_LOAD_MONITOR_B64:
1159 case AMDGPU::FLAT_LOAD_MONITOR_B128:
1160 return true;
1161 default:
1162 return false;
1163 }
1164 }
1165
1166 static bool isGFX12CacheInvOrWBInst(unsigned Opc) {
1167 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||
1168 Opc == AMDGPU::GLOBAL_WBINV;
1169 }
1170
1171 static bool isF16PseudoScalarTrans(unsigned Opcode) {
1172 return Opcode == AMDGPU::V_S_EXP_F16_e64 ||
1173 Opcode == AMDGPU::V_S_LOG_F16_e64 ||
1174 Opcode == AMDGPU::V_S_RCP_F16_e64 ||
1175 Opcode == AMDGPU::V_S_RSQ_F16_e64 ||
1176 Opcode == AMDGPU::V_S_SQRT_F16_e64;
1177 }
1178
1179 static bool doesNotReadTiedSource(const MachineInstr &MI) {
1180 return SIInstrFlags::isTiedSourceNotRead(O: MI);
1181 }
1182
1183 bool doesNotReadTiedSource(uint32_t Opcode) const {
1184 return SIInstrFlags::isTiedSourceNotRead(O: get(Opcode));
1185 }
1186
1187 bool isIGLP(unsigned Opcode) const {
1188 return Opcode == AMDGPU::SCHED_BARRIER ||
1189 Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
1190 }
1191
1192 bool isIGLP(const MachineInstr &MI) const { return isIGLP(Opcode: MI.getOpcode()); }
1193
1194 // Return true if the instruction is mutually exclusive with all non-IGLP DAG
1195 // mutations, requiring all other mutations to be disabled.
1196 bool isIGLPMutationOnly(unsigned Opcode) const {
1197 return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
1198 }
1199
1200 static unsigned getNonSoftWaitcntOpcode(unsigned Opcode) {
1201 switch (Opcode) {
1202 case AMDGPU::S_WAITCNT_soft:
1203 return AMDGPU::S_WAITCNT;
1204 case AMDGPU::S_WAITCNT_VSCNT_soft:
1205 return AMDGPU::S_WAITCNT_VSCNT;
1206 case AMDGPU::S_WAIT_LOADCNT_soft:
1207 return AMDGPU::S_WAIT_LOADCNT;
1208 case AMDGPU::S_WAIT_STORECNT_soft:
1209 return AMDGPU::S_WAIT_STORECNT;
1210 case AMDGPU::S_WAIT_SAMPLECNT_soft:
1211 return AMDGPU::S_WAIT_SAMPLECNT;
1212 case AMDGPU::S_WAIT_BVHCNT_soft:
1213 return AMDGPU::S_WAIT_BVHCNT;
1214 case AMDGPU::S_WAIT_DSCNT_soft:
1215 return AMDGPU::S_WAIT_DSCNT;
1216 case AMDGPU::S_WAIT_KMCNT_soft:
1217 return AMDGPU::S_WAIT_KMCNT;
1218 case AMDGPU::S_WAIT_XCNT_soft:
1219 return AMDGPU::S_WAIT_XCNT;
1220 default:
1221 return Opcode;
1222 }
1223 }
1224
1225 static bool isWaitcnt(unsigned Opcode) {
1226 switch (getNonSoftWaitcntOpcode(Opcode)) {
1227 case AMDGPU::S_WAITCNT:
1228 case AMDGPU::S_WAITCNT_VSCNT:
1229 case AMDGPU::S_WAITCNT_VMCNT:
1230 case AMDGPU::S_WAITCNT_EXPCNT:
1231 case AMDGPU::S_WAITCNT_LGKMCNT:
1232 case AMDGPU::S_WAIT_LOADCNT:
1233 case AMDGPU::S_WAIT_LOADCNT_DSCNT:
1234 case AMDGPU::S_WAIT_STORECNT:
1235 case AMDGPU::S_WAIT_STORECNT_DSCNT:
1236 case AMDGPU::S_WAIT_SAMPLECNT:
1237 case AMDGPU::S_WAIT_BVHCNT:
1238 case AMDGPU::S_WAIT_EXPCNT:
1239 case AMDGPU::S_WAIT_DSCNT:
1240 case AMDGPU::S_WAIT_KMCNT:
1241 case AMDGPU::S_WAIT_XCNT:
1242 case AMDGPU::S_WAIT_IDLE:
1243 return true;
1244 default:
1245 return false;
1246 }
1247 }
1248
1249 bool isVGPRCopy(const MachineInstr &MI) const {
1250 assert(isCopyInstr(MI));
1251 Register Dest = MI.getOperand(i: 0).getReg();
1252 const MachineFunction &MF = *MI.getMF();
1253 const MachineRegisterInfo &MRI = MF.getRegInfo();
1254 return !RI.isSGPRReg(MRI, Reg: Dest);
1255 }
1256
1257 bool hasVGPRUses(const MachineInstr &MI) const {
1258 const MachineFunction &MF = *MI.getMF();
1259 const MachineRegisterInfo &MRI = MF.getRegInfo();
1260 return llvm::any_of(Range: MI.explicit_uses(),
1261 P: [&MRI, this](const MachineOperand &MO) {
1262 return MO.isReg() && RI.isVGPR(MRI, Reg: MO.getReg());});
1263 }
1264
1265 /// Return true if the instruction modifies the mode register.q
1266 static bool modifiesModeRegister(const MachineInstr &MI);
1267
1268 /// This function is used to determine if an instruction can be safely
1269 /// executed under EXEC = 0 without hardware error, indeterminate results,
1270 /// and/or visible effects on future vector execution or outside the shader.
1271 /// Note: as of 2024 the only use of this is SIPreEmitPeephole where it is
1272 /// used in removing branches over short EXEC = 0 sequences.
1273 /// As such it embeds certain assumptions which may not apply to every case
1274 /// of EXEC = 0 execution.
1275 bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const;
1276
1277 /// Returns true if the instruction could potentially depend on the value of
1278 /// exec. If false, exec dependencies may safely be ignored.
1279 bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const;
1280
1281 bool isInlineConstant(const APInt &Imm) const;
1282
1283 bool isInlineConstant(const APFloat &Imm) const;
1284
1285 // Returns true if this non-register operand definitely does not need to be
1286 // encoded as a 32-bit literal. Note that this function handles all kinds of
1287 // operands, not just immediates.
1288 //
1289 // Some operands like FrameIndexes could resolve to an inline immediate value
1290 // that will not require an additional 4-bytes; this function assumes that it
1291 // will.
1292 bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const {
1293 if (!MO.isImm())
1294 return false;
1295 return isInlineConstant(ImmVal: MO.getImm(), OperandType);
1296 }
1297 bool isInlineConstant(int64_t ImmVal, uint8_t OperandType) const;
1298
1299 bool isInlineConstant(const MachineOperand &MO,
1300 const MCOperandInfo &OpInfo) const {
1301 return isInlineConstant(MO, OperandType: OpInfo.OperandType);
1302 }
1303
1304 /// \p returns true if \p UseMO is substituted with \p DefMO in \p MI it would
1305 /// be an inline immediate.
1306 bool isInlineConstant(const MachineInstr &MI,
1307 const MachineOperand &UseMO,
1308 const MachineOperand &DefMO) const {
1309 assert(UseMO.getParent() == &MI);
1310 int OpIdx = UseMO.getOperandNo();
1311 if (OpIdx >= MI.getDesc().NumOperands)
1312 return false;
1313
1314 return isInlineConstant(MO: DefMO, OpInfo: MI.getDesc().operands()[OpIdx]);
1315 }
1316
1317 /// \p returns true if the operand \p OpIdx in \p MI is a valid inline
1318 /// immediate.
1319 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx) const {
1320 const MachineOperand &MO = MI.getOperand(i: OpIdx);
1321 return isInlineConstant(MO, OperandType: MI.getDesc().operands()[OpIdx].OperandType);
1322 }
1323
1324 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
1325 int64_t ImmVal) const {
1326 if (OpIdx >= MI.getDesc().NumOperands)
1327 return false;
1328
1329 if (isCopyInstr(MI)) {
1330 unsigned Size = getOpSize(MI, OpNo: OpIdx);
1331 assert(Size == 8 || Size == 4);
1332
1333 uint8_t OpType = (Size == 8) ?
1334 AMDGPU::OPERAND_REG_IMM_INT64 : AMDGPU::OPERAND_REG_IMM_INT32;
1335 return isInlineConstant(ImmVal, OperandType: OpType);
1336 }
1337
1338 return isInlineConstant(ImmVal, OperandType: MI.getDesc().operands()[OpIdx].OperandType);
1339 }
1340
1341 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
1342 const MachineOperand &MO) const {
1343 return isInlineConstant(MI, OpIdx, ImmVal: MO.getImm());
1344 }
1345
1346 bool isInlineConstant(const MachineOperand &MO) const {
1347 return isInlineConstant(MI: *MO.getParent(), OpIdx: MO.getOperandNo());
1348 }
1349
1350 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
1351 const MachineOperand &MO) const;
1352
1353 bool isLiteralOperandLegal(const MCInstrDesc &InstDesc,
1354 const MCOperandInfo &OpInfo) const;
1355
1356 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
1357 int64_t ImmVal) const;
1358
1359 bool isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
1360 const MachineOperand &MO) const {
1361 return isImmOperandLegal(InstDesc: MI.getDesc(), OpNo, MO);
1362 }
1363
1364 bool isNeverCoissue(MachineInstr &MI) const;
1365
1366 /// Check if this immediate value can be used for AV_MOV_B64_IMM_PSEUDO.
1367 bool isLegalAV64PseudoImm(uint64_t Imm) const;
1368
1369 /// Return true if this 64-bit VALU instruction has a 32-bit encoding.
1370 /// This function will return false if you pass it a 32-bit instruction.
1371 bool hasVALU32BitEncoding(unsigned Opcode) const;
1372
1373 bool physRegUsesConstantBus(const MachineOperand &Reg) const;
1374 bool regUsesConstantBus(const MachineOperand &Reg,
1375 const MachineRegisterInfo &MRI) const;
1376
1377 /// Returns true if this operand uses the constant bus.
1378 bool usesConstantBus(const MachineRegisterInfo &MRI,
1379 const MachineOperand &MO,
1380 const MCOperandInfo &OpInfo) const;
1381
1382 bool usesConstantBus(const MachineRegisterInfo &MRI, const MachineInstr &MI,
1383 int OpIdx) const {
1384 return usesConstantBus(MRI, MO: MI.getOperand(i: OpIdx),
1385 OpInfo: MI.getDesc().operands()[OpIdx]);
1386 }
1387
1388 /// Return true if this instruction has any modifiers.
1389 /// e.g. src[012]_mod, omod, clamp.
1390 bool hasModifiers(unsigned Opcode) const;
1391
1392 bool hasModifiersSet(const MachineInstr &MI, AMDGPU::OpName OpName) const;
1393 bool hasAnyModifiersSet(const MachineInstr &MI) const;
1394
1395 bool canShrink(const MachineInstr &MI,
1396 const MachineRegisterInfo &MRI) const;
1397
1398 MachineInstr *buildShrunkInst(MachineInstr &MI,
1399 unsigned NewOpcode) const;
1400
1401 bool verifyInstruction(const MachineInstr &MI,
1402 StringRef &ErrInfo) const override;
1403
1404 unsigned getVALUOp(const MachineInstr &MI) const;
1405 unsigned getVALUOp(unsigned Opc) const;
1406
1407 void insertScratchExecCopy(MachineFunction &MF, MachineBasicBlock &MBB,
1408 MachineBasicBlock::iterator MBBI,
1409 const DebugLoc &DL, Register Reg, bool IsSCCLive,
1410 SlotIndexes *Indexes = nullptr) const;
1411
1412 void restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
1413 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
1414 Register Reg, SlotIndexes *Indexes = nullptr) const;
1415
1416 MachineInstr *getWholeWaveFunctionSetup(MachineFunction &MF) const;
1417
1418 /// Return the correct register class for \p OpNo. For target-specific
1419 /// instructions, this will return the register class that has been defined
1420 /// in tablegen. For generic instructions, like REG_SEQUENCE it will return
1421 /// the register class of its machine operand.
1422 /// to infer the correct register class base on the other operands.
1423 const TargetRegisterClass *getOpRegClass(const MachineInstr &MI,
1424 unsigned OpNo) const;
1425
1426 /// Return the size in bytes of the operand OpNo on the given
1427 // instruction opcode.
1428 unsigned getOpSize(uint32_t Opcode, unsigned OpNo) const {
1429 const MCOperandInfo &OpInfo = get(Opcode).operands()[OpNo];
1430
1431 if (OpInfo.RegClass == -1) {
1432 // If this is an immediate operand, this must be a 32-bit literal.
1433 assert(OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE);
1434 return 4;
1435 }
1436
1437 return RI.getRegSizeInBits(RC: *RI.getRegClass(i: getOpRegClassID(OpInfo))) / 8;
1438 }
1439
1440 /// This form should usually be preferred since it handles operands
1441 /// with unknown register classes.
1442 unsigned getOpSize(const MachineInstr &MI, unsigned OpNo) const {
1443 const MachineOperand &MO = MI.getOperand(i: OpNo);
1444 if (MO.isReg()) {
1445 if (unsigned SubReg = MO.getSubReg()) {
1446 return RI.getSubRegIdxSize(Idx: SubReg) / 8;
1447 }
1448 }
1449 return RI.getRegSizeInBits(RC: *getOpRegClass(MI, OpNo)) / 8;
1450 }
1451
1452 /// Legalize the \p OpIndex operand of this instruction by inserting
1453 /// a MOV. For example:
1454 /// ADD_I32_e32 VGPR0, 15
1455 /// to
1456 /// MOV VGPR1, 15
1457 /// ADD_I32_e32 VGPR0, VGPR1
1458 ///
1459 /// If the operand being legalized is a register, then a COPY will be used
1460 /// instead of MOV.
1461 void legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const;
1462
1463 /// Check if \p MO is a legal operand if it was the \p OpIdx Operand
1464 /// for \p MI.
1465 bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
1466 const MachineOperand *MO = nullptr) const;
1467
1468 /// Check if \p MO would be a valid operand for the given operand
1469 /// definition \p OpInfo. Note this does not attempt to validate constant bus
1470 /// restrictions (e.g. literal constant usage).
1471 bool isLegalVSrcOperand(const MachineRegisterInfo &MRI,
1472 const MCOperandInfo &OpInfo,
1473 const MachineOperand &MO) const;
1474
1475 /// Check if \p MO (a register operand) is a legal register for the
1476 /// given operand description or operand index.
1477 /// The operand index version provide more legality checks
1478 bool isLegalRegOperand(const MachineRegisterInfo &MRI,
1479 const MCOperandInfo &OpInfo,
1480 const MachineOperand &MO) const;
1481 bool isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
1482 const MachineOperand &MO) const;
1483
1484 /// Check if \p MO would be a legal operand for gfx12+ packed math FP32 or
1485 /// 64 instructions. Packed math FP32/FP64/U64 instructions typically accept
1486 /// SGPRs or VGPRs as source operands. On gfx12+, if a source operand uses
1487 /// SGPRs, the HW can only read the first SGPR and use it for both the low and
1488 /// high operations.
1489 /// \p SrcN can be 0, 1, or 2, representing src0, src1, and src2,
1490 /// respectively. If \p MO is nullptr, the operand corresponding to SrcN will
1491 /// be used.
1492 bool isLegalGFX12PlusPackedMathFP32or64BitOperand(
1493 const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,
1494 const MachineOperand *MO = nullptr) const;
1495
1496 /// Legalize operands in \p MI by either commuting it or inserting a
1497 /// copy of src1.
1498 void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1499
1500 /// Fix operands in \p MI to satisfy constant bus requirements.
1501 void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1502
1503 /// Copy a value from a VGPR (\p SrcReg) to SGPR. The desired register class
1504 /// for the dst register (\p DstRC) can be optionally supplied. This function
1505 /// can only be used when it is know that the value in SrcReg is same across
1506 /// all threads in the wave.
1507 /// \returns The SGPR register that \p SrcReg was copied to.
1508 Register readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,
1509 MachineRegisterInfo &MRI,
1510 const TargetRegisterClass *DstRC = nullptr) const;
1511
1512 void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1513 void legalizeOperandsFLAT(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1514
1515 void legalizeGenericOperand(MachineBasicBlock &InsertMBB,
1516 MachineBasicBlock::iterator I,
1517 const TargetRegisterClass *DstRC,
1518 MachineOperand &Op, MachineRegisterInfo &MRI,
1519 const DebugLoc &DL) const;
1520
1521 /// Legalize all operands in this instruction. This function may create new
1522 /// instructions and control-flow around \p MI. If present, \p MDT is
1523 /// updated.
1524 /// \returns A new basic block that contains \p MI if new blocks were created.
1525 MachineBasicBlock *
1526 legalizeOperands(MachineInstr &MI, MachineDominatorTree *MDT = nullptr) const;
1527
1528 /// Change SADDR form of a FLAT \p Inst to its VADDR form if saddr operand
1529 /// was moved to VGPR. \returns true if succeeded.
1530 bool moveFlatAddrToVGPR(MachineInstr &Inst) const;
1531
1532 /// Fix operands in Inst to fix 16bit SALU to VALU lowering.
1533 void legalizeOperandsVALUt16(MachineInstr &Inst,
1534 MachineRegisterInfo &MRI) const;
1535 void legalizeOperandsVALUt16(MachineInstr &Inst, unsigned OpIdx,
1536 MachineRegisterInfo &MRI) const;
1537
1538 /// Replace the instructions opcode with the equivalent VALU
1539 /// opcode. This function will also move the users of MachineInstruntions
1540 /// in the \p WorkList to the VALU if necessary. If present, \p MDT is
1541 /// updated.
1542 void moveToVALU(SIInstrWorklist &Worklist, MachineDominatorTree *MDT) const;
1543
1544 void
1545 moveToVALUImpl(SIInstrWorklist &Worklist, MachineDominatorTree *MDT,
1546 MachineInstr &Inst,
1547 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
1548 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const;
1549 /// Wrapper function for generating waterfall for instruction \p MI
1550 /// This function take into consideration of related pre & succ instructions
1551 /// (e.g. calling process) into consideratioin
1552 void createWaterFallForSiCall(MachineInstr *MI, MachineDominatorTree *MDT,
1553 ArrayRef<MachineOperand *> ScalarOps,
1554 ArrayRef<Register> PhySGPRs = {}) const;
1555
1556 void insertNoop(MachineBasicBlock &MBB,
1557 MachineBasicBlock::iterator MI) const override;
1558
1559 void insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1560 unsigned Quantity) const override;
1561
1562 /// Build instructions that simulate the behavior of a `s_trap 2` instructions
1563 /// for hardware (namely, gfx11) that runs in PRIV=1 mode. There, s_trap is
1564 /// interpreted as a nop.
1565 MachineBasicBlock *insertSimulatedTrap(MachineRegisterInfo &MRI,
1566 MachineBasicBlock &MBB,
1567 MachineInstr &MI,
1568 const DebugLoc &DL) const;
1569
1570 /// Return the number of wait states that result from executing this
1571 /// instruction.
1572 static unsigned getNumWaitStates(const MachineInstr &MI);
1573
1574 /// Returns the operand named \p Op. If \p MI does not have an
1575 /// operand named \c Op, this function returns nullptr.
1576 LLVM_READONLY
1577 MachineOperand *getNamedOperand(MachineInstr &MI,
1578 AMDGPU::OpName OperandName) const;
1579
1580 LLVM_READONLY
1581 const MachineOperand *getNamedOperand(const MachineInstr &MI,
1582 AMDGPU::OpName OperandName) const {
1583 return getNamedOperand(MI&: const_cast<MachineInstr &>(MI), OperandName);
1584 }
1585
1586 /// Get required immediate operand
1587 int64_t getNamedImmOperand(const MachineInstr &MI,
1588 AMDGPU::OpName OperandName) const {
1589 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OperandName);
1590 return MI.getOperand(i: Idx).getImm();
1591 }
1592
1593 uint64_t getDefaultRsrcDataFormat() const;
1594 uint64_t getScratchRsrcWords23() const;
1595
1596 bool isLowLatencyInstruction(const MachineInstr &MI) const;
1597 bool isHighLatencyDef(int Opc) const override;
1598
1599 /// Return the descriptor of the target-specific machine instruction
1600 /// that corresponds to the specified pseudo or native opcode.
1601 const MCInstrDesc &getMCOpcodeFromPseudo(unsigned Opcode) const {
1602 return get(Opcode: pseudoToMCOpcode(Opcode));
1603 }
1604
1605 Register isStackAccess(const MachineInstr &MI, int &FrameIndex,
1606 TypeSize &MemBytes) const;
1607 Register isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex,
1608 TypeSize &MemBytes) const;
1609
1610 Register isLoadFromStackSlot(const MachineInstr &MI,
1611 int &FrameIndex) const override {
1612 TypeSize MemBytes = TypeSize::getZero();
1613 return isLoadFromStackSlot(MI, FrameIndex, MemBytes);
1614 }
1615
1616 Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex,
1617 TypeSize &MemBytes) const override;
1618
1619 Register isStoreToStackSlot(const MachineInstr &MI,
1620 int &FrameIndex) const override {
1621 TypeSize MemBytes = TypeSize::getZero();
1622 return isStoreToStackSlot(MI, FrameIndex, MemBytes);
1623 }
1624
1625 Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex,
1626 TypeSize &MemBytes) const override;
1627
1628 unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
1629
1630 InstSizeVerifyMode
1631 getInstSizeVerifyMode(const MachineInstr &MI) const override;
1632
1633 bool mayAccessFlatAddressSpace(const MachineInstr &MI) const;
1634
1635 std::pair<unsigned, unsigned>
1636 decomposeMachineOperandsTargetFlags(unsigned TF) const override;
1637
1638 ArrayRef<std::pair<int, const char *>>
1639 getSerializableTargetIndices() const override;
1640
1641 ArrayRef<std::pair<unsigned, const char *>>
1642 getSerializableDirectMachineOperandTargetFlags() const override;
1643
1644 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
1645 getSerializableMachineMemOperandTargetFlags() const override;
1646
1647 ScheduleHazardRecognizer *
1648 CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
1649 const ScheduleDAG *DAG) const override;
1650
1651 ScheduleHazardRecognizer *
1652 CreateTargetPostRAHazardRecognizer(const MachineFunction &MF,
1653 MachineLoopInfo *MLI) const override;
1654
1655 ScheduleHazardRecognizer *
1656 CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
1657 const ScheduleDAGMI *DAG) const override;
1658
1659 unsigned getLiveRangeSplitOpcode(Register Reg,
1660 const MachineFunction &MF) const override;
1661
1662 bool isBasicBlockPrologue(const MachineInstr &MI,
1663 Register Reg = Register()) const override;
1664
1665 bool canAddToBBProlog(const MachineInstr &MI) const;
1666
1667 MachineInstr *createPHIDestinationCopy(MachineBasicBlock &MBB,
1668 MachineBasicBlock::iterator InsPt,
1669 const DebugLoc &DL, Register Src,
1670 Register Dst) const override;
1671
1672 MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB,
1673 MachineBasicBlock::iterator InsPt,
1674 const DebugLoc &DL, Register Src,
1675 unsigned SrcSubReg,
1676 Register Dst) const override;
1677
1678 bool isWave32() const;
1679
1680 bool isVOPDAntidependencyAllowed(const MachineInstr &MI) const;
1681
1682 bool hasRAWDependency(const MachineInstr &FirstMI,
1683 const MachineInstr &SecondMI) const;
1684
1685 /// Return a partially built integer add instruction without carry.
1686 /// Caller must add source operands.
1687 /// For pre-GFX9 it will generate unused carry destination operand.
1688 /// TODO: After GFX9 it should return a no-carry operation.
1689 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1690 MachineBasicBlock::iterator I,
1691 const DebugLoc &DL,
1692 Register DestReg) const;
1693
1694 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1695 MachineBasicBlock::iterator I,
1696 const DebugLoc &DL,
1697 Register DestReg,
1698 RegScavenger &RS) const;
1699
1700 static bool isKillTerminator(unsigned Opcode);
1701 const MCInstrDesc &getKillTerminatorFromPseudo(unsigned Opcode) const;
1702
1703 bool isLegalMUBUFImmOffset(unsigned Imm) const;
1704
1705 static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST);
1706
1707 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
1708 Align Alignment = Align(4)) const;
1709
1710 /// Returns if \p Offset is legal for the subtarget as the offset to a FLAT
1711 /// encoded instruction with the given \p FlatVariant.
1712 bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
1713 AMDGPU::FlatAddrSpace FlatVariant) const;
1714
1715 /// Split \p COffsetVal into {immediate offset field, remainder offset}
1716 /// values.
1717 std::pair<int64_t, int64_t>
1718 splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,
1719 AMDGPU::FlatAddrSpace FlatVariant) const;
1720
1721 /// Returns true if negative offsets are allowed for the given \p FlatVariant.
1722 bool allowNegativeFlatOffset(AMDGPU::FlatAddrSpace FlatVariant) const;
1723
1724 /// \brief Return a target-specific opcode if Opcode is a pseudo instruction.
1725 /// Return -1 if the target-specific opcode for the pseudo instruction does
1726 /// not exist. If Opcode is not a pseudo instruction, this is identity.
1727 int pseudoToMCOpcode(int Opcode) const;
1728
1729 /// \brief Check if this instruction should only be used by assembler.
1730 /// Return true if this opcode should not be used by codegen.
1731 bool isAsmOnlyOpcode(int MCOp) const;
1732
1733 void fixImplicitOperands(MachineInstr &MI) const;
1734
1735 MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
1736 ArrayRef<unsigned> Ops, int FrameIndex,
1737 MachineInstr *&CopyMI,
1738 LiveIntervals *LIS = nullptr,
1739 VirtRegMap *VRM = nullptr) const override;
1740
1741 unsigned getInstrLatency(const InstrItineraryData *ItinData,
1742 const MachineInstr &MI,
1743 unsigned *PredCost = nullptr) const override;
1744
1745 const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override;
1746
1747 ValueUniformity getValueUniformity(const MachineInstr &MI) const final;
1748
1749 ValueUniformity getGenericValueUniformity(const MachineInstr &MI) const;
1750
1751 const MIRFormatter *getMIRFormatter() const override;
1752
1753 static unsigned getDSShaderTypeValue(const MachineFunction &MF);
1754
1755 const TargetSchedModel &getSchedModel() const { return SchedModel; }
1756
1757 void createReadFirstLaneFromCopyToPhysReg(MachineRegisterInfo &MRI,
1758 Register DstReg,
1759 MachineInstr &Inst) const;
1760
1761 void handleCopyToPhysHelper(
1762 SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst,
1763 MachineRegisterInfo &MRI,
1764 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
1765 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const;
1766
1767 // FIXME: This should be removed
1768 // Enforce operand's \p OpName even alignment if required by target.
1769 // This is used if an operand is a 32 bit register but needs to be aligned
1770 // regardless.
1771 void enforceOperandRCAlignment(MachineInstr &MI, AMDGPU::OpName OpName) const;
1772};
1773
1774/// \brief Returns true if a reg:subreg pair P has a TRC class
1775inline bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P,
1776 const TargetRegisterClass &TRC,
1777 MachineRegisterInfo &MRI) {
1778 auto *RC = MRI.getRegClass(Reg: P.Reg);
1779 if (!P.SubReg)
1780 return RC == &TRC;
1781 auto *TRI = MRI.getTargetRegisterInfo();
1782 return RC == TRI->getMatchingSuperRegClass(A: RC, B: &TRC, Idx: P.SubReg);
1783}
1784
1785/// \brief Create RegSubRegPair from a register MachineOperand
1786inline
1787TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O) {
1788 assert(O.isReg());
1789 return TargetInstrInfo::RegSubRegPair(O.getReg(), O.getSubReg());
1790}
1791
1792/// \brief Return the SubReg component from REG_SEQUENCE
1793TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI,
1794 unsigned SubReg);
1795
1796/// \brief Return the defining instruction for a given reg:subreg pair
1797/// skipping copy like instructions and subreg-manipulation pseudos.
1798/// Following another subreg of a reg:subreg isn't supported.
1799MachineInstr *getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
1800 const MachineRegisterInfo &MRI);
1801
1802/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1803/// DefMI and the use at \p UseMI. Should be run on SSA. Currently does not
1804/// attempt to track between blocks.
1805bool execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
1806 Register VReg,
1807 const MachineInstr &DefMI,
1808 const MachineInstr &UseMI);
1809
1810/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1811/// DefMI and all its uses. Should be run on SSA. Currently does not attempt to
1812/// track between blocks.
1813bool execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
1814 Register VReg,
1815 const MachineInstr &DefMI);
1816
1817namespace AMDGPU {
1818
1819 LLVM_READONLY
1820 int32_t getVOPe64(uint32_t Opcode);
1821
1822 LLVM_READONLY
1823 int32_t getVOPe32(uint32_t Opcode);
1824
1825 LLVM_READONLY
1826 int32_t getSDWAOp(uint32_t Opcode);
1827
1828 LLVM_READONLY
1829 int32_t getDPPOp32(uint32_t Opcode);
1830
1831 LLVM_READONLY
1832 int32_t getDPPOp64(uint32_t Opcode);
1833
1834 LLVM_READONLY
1835 int32_t getBasicFromSDWAOp(uint32_t Opcode);
1836
1837 LLVM_READONLY
1838 int32_t getCommuteRev(uint32_t Opcode);
1839
1840 LLVM_READONLY
1841 int32_t getCommuteOrig(uint32_t Opcode);
1842
1843 LLVM_READONLY
1844 int32_t getAddr64Inst(uint32_t Opcode);
1845
1846 /// Check if \p Opcode is an Addr64 opcode.
1847 ///
1848 /// \returns \p Opcode if it is an Addr64 opcode, otherwise -1.
1849 LLVM_READONLY
1850 int32_t getIfAddr64Inst(uint32_t Opcode);
1851
1852 LLVM_READONLY
1853 int32_t getSOPKOp(uint32_t Opcode);
1854
1855 /// \returns SADDR form of a FLAT Global instruction given an \p Opcode
1856 /// of a VADDR form.
1857 LLVM_READONLY
1858 int32_t getGlobalSaddrOp(uint32_t Opcode);
1859
1860 /// \returns VADDR form of a FLAT Global instruction given an \p Opcode
1861 /// of a SADDR form.
1862 LLVM_READONLY
1863 int32_t getGlobalVaddrOp(uint32_t Opcode);
1864
1865 LLVM_READONLY
1866 int32_t getVCMPXNoSDstOp(uint32_t Opcode);
1867
1868 /// \returns ST form with only immediate offset of a FLAT Scratch instruction
1869 /// given an \p Opcode of an SS (SADDR) form.
1870 LLVM_READONLY
1871 int32_t getFlatScratchInstSTfromSS(uint32_t Opcode);
1872
1873 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
1874 /// of an SVS (SADDR + VADDR) form.
1875 LLVM_READONLY
1876 int32_t getFlatScratchInstSVfromSVS(uint32_t Opcode);
1877
1878 /// \returns SS (SADDR) form of a FLAT Scratch instruction given an \p Opcode
1879 /// of an SV (VADDR) form.
1880 LLVM_READONLY
1881 int32_t getFlatScratchInstSSfromSV(uint32_t Opcode);
1882
1883 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
1884 /// of an SS (SADDR) form.
1885 LLVM_READONLY
1886 int32_t getFlatScratchInstSVfromSS(uint32_t Opcode);
1887
1888 /// \returns earlyclobber version of a MAC MFMA is exists.
1889 LLVM_READONLY
1890 int32_t getMFMAEarlyClobberOp(uint32_t Opcode);
1891
1892 /// \returns Version of an MFMA instruction which uses AGPRs for srcC and
1893 /// vdst, given an \p Opcode of an MFMA which uses VGPRs for srcC/vdst.
1894 LLVM_READONLY
1895 int32_t getMFMASrcCVDstAGPROp(uint32_t Opcode);
1896
1897 /// \returns v_cmpx version of a v_cmp instruction.
1898 LLVM_READONLY
1899 int32_t getVCMPXOpFromVCMP(uint32_t Opcode);
1900
1901 const uint64_t RSRC_DATA_FORMAT = 0xf00000000000LL;
1902 const uint64_t RSRC_ELEMENT_SIZE_SHIFT = (32 + 19);
1903 const uint64_t RSRC_INDEX_STRIDE_SHIFT = (32 + 21);
1904 const uint64_t RSRC_TID_ENABLE = UINT64_C(1) << (32 + 23);
1905
1906} // end namespace AMDGPU
1907
1908namespace AMDGPU {
1909enum AsmComments : MachineInstr::AsmPrinterFlagTy {
1910 // For sgpr to vgpr spill instructions
1911 SGPR_SPILL = MachineInstr::TAsmComments
1912};
1913} // namespace AMDGPU
1914
1915namespace SI {
1916namespace KernelInputOffsets {
1917
1918/// Offsets in bytes from the start of the input buffer
1919enum Offsets {
1920 NGROUPS_X = 0,
1921 NGROUPS_Y = 4,
1922 NGROUPS_Z = 8,
1923 GLOBAL_SIZE_X = 12,
1924 GLOBAL_SIZE_Y = 16,
1925 GLOBAL_SIZE_Z = 20,
1926 LOCAL_SIZE_X = 24,
1927 LOCAL_SIZE_Y = 28,
1928 LOCAL_SIZE_Z = 32
1929};
1930
1931} // end namespace KernelInputOffsets
1932} // end namespace SI
1933
1934} // end namespace llvm
1935
1936#endif // LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
1937