1//===- SIInstrInfo.h - SI Instruction Info Interface ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Interface definition for SIInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
15#define LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
16
17#include "AMDGPUMIRFormatter.h"
18#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
19#include "SIRegisterInfo.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/CodeGen/TargetInstrInfo.h"
23#include "llvm/CodeGen/TargetSchedule.h"
24
25#define GET_INSTRINFO_HEADER
26#include "AMDGPUGenInstrInfo.inc"
27
28namespace llvm {
29
30class APInt;
31class GCNSubtarget;
32class LiveVariables;
33class MachineDominatorTree;
34class MachineRegisterInfo;
35class RegScavenger;
36class SIMachineFunctionInfo;
37class TargetRegisterClass;
38class ScheduleHazardRecognizer;
39
40constexpr unsigned DefaultMemoryClusterDWordsLimit = 8;
41
42/// Mark the MMO of a uniform load if there are no potentially clobbering stores
43/// on any path from the start of an entry function to this load.
44static const MachineMemOperand::Flags MONoClobber =
45 MachineMemOperand::MOTargetFlag1;
46
47/// Mark the MMO of a load as the last use.
48static const MachineMemOperand::Flags MOLastUse =
49 MachineMemOperand::MOTargetFlag2;
50
51/// Mark the MMO of cooperative load/store atomics.
52static const MachineMemOperand::Flags MOCooperative =
53 MachineMemOperand::MOTargetFlag3;
54
55/// Mark the MMO of accesses to memory locations that are
56/// never written to by other threads.
57static const MachineMemOperand::Flags MOThreadPrivate =
58 MachineMemOperand::MOTargetFlag4;
59
60/// Utility to store machine instructions worklist.
61struct SIInstrWorklist {
62 SIInstrWorklist() = default;
63
64 void insert(MachineInstr *MI);
65
66 MachineInstr *top() const {
67 const auto *iter = InstrList.begin();
68 return *iter;
69 }
70
71 void erase_top() {
72 const auto *iter = InstrList.begin();
73 InstrList.erase(I: iter);
74 }
75
76 bool empty() const { return InstrList.empty(); }
77
78 void clear() {
79 InstrList.clear();
80 DeferredList.clear();
81 }
82
83 bool isDeferred(MachineInstr *MI);
84
85 SetVector<MachineInstr *> &getDeferredList() { return DeferredList; }
86
87private:
88 /// InstrList contains the MachineInstrs.
89 SetVector<MachineInstr *> InstrList;
90 /// Deferred instructions are specific MachineInstr
91 /// that will be added by insert method.
92 SetVector<MachineInstr *> DeferredList;
93};
94
95class SIInstrInfo final : public AMDGPUGenInstrInfo {
96 struct ThreeAddressUpdates;
97
98private:
99 const SIRegisterInfo RI;
100 const GCNSubtarget &ST;
101 TargetSchedModel SchedModel;
102 mutable std::unique_ptr<AMDGPUMIRFormatter> Formatter;
103
104 // The inverse predicate should have the negative value.
105 enum BranchPredicate {
106 INVALID_BR = 0,
107 SCC_TRUE = 1,
108 SCC_FALSE = -1,
109 VCCNZ = 2,
110 VCCZ = -2,
111 EXECNZ = -3,
112 EXECZ = 3
113 };
114
115 using SetVectorType = SmallSetVector<MachineInstr *, 32>;
116
117 static unsigned getBranchOpcode(BranchPredicate Cond);
118 static BranchPredicate getBranchPredicate(unsigned Opcode);
119
120public:
121 unsigned buildExtractSubReg(MachineBasicBlock::iterator MI,
122 MachineRegisterInfo &MRI,
123 const MachineOperand &SuperReg,
124 const TargetRegisterClass *SuperRC,
125 unsigned SubIdx,
126 const TargetRegisterClass *SubRC) const;
127 MachineOperand buildExtractSubRegOrImm(
128 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
129 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
130 unsigned SubIdx, const TargetRegisterClass *SubRC) const;
131
132private:
133 bool optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,
134 bool NeedInversion) const;
135
136 bool invertSCCUse(MachineInstr *SCCDef) const;
137
138 void swapOperands(MachineInstr &Inst) const;
139
140 std::pair<bool, MachineBasicBlock *>
141 moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
142 MachineDominatorTree *MDT = nullptr) const;
143
144 void lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
145 MachineDominatorTree *MDT = nullptr) const;
146
147 void lowerScalarAbs(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
148
149 void lowerScalarAbsDiff(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
150
151 void lowerScalarXnor(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
152
153 void splitScalarNotBinop(SIInstrWorklist &Worklist, MachineInstr &Inst,
154 unsigned Opcode) const;
155
156 void splitScalarBinOpN2(SIInstrWorklist &Worklist, MachineInstr &Inst,
157 unsigned Opcode) const;
158
159 void splitScalar64BitUnaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
160 unsigned Opcode, bool Swap = false) const;
161
162 void splitScalar64BitBinaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
163 unsigned Opcode,
164 MachineDominatorTree *MDT = nullptr) const;
165
166 void splitScalarSMulU64(SIInstrWorklist &Worklist, MachineInstr &Inst,
167 MachineDominatorTree *MDT) const;
168
169 void splitScalarSMulPseudo(SIInstrWorklist &Worklist, MachineInstr &Inst,
170 MachineDominatorTree *MDT) const;
171
172 void splitScalar64BitXnor(SIInstrWorklist &Worklist, MachineInstr &Inst,
173 MachineDominatorTree *MDT = nullptr) const;
174
175 void splitScalar64BitBCNT(SIInstrWorklist &Worklist,
176 MachineInstr &Inst) const;
177 void splitScalar64BitBFE(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
178 void splitScalar64BitCountOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
179 unsigned Opcode,
180 MachineDominatorTree *MDT = nullptr) const;
181 void movePackToVALU(SIInstrWorklist &Worklist, MachineRegisterInfo &MRI,
182 MachineInstr &Inst) const;
183
184 void addUsersToMoveToVALUWorklist(Register Reg, MachineRegisterInfo &MRI,
185 SIInstrWorklist &Worklist) const;
186
187 void addSCCDefUsersToVALUWorklist(const MachineOperand &Op,
188 MachineInstr &SCCDefInst,
189 SIInstrWorklist &Worklist,
190 Register NewCond = Register()) const;
191 void addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
192 SIInstrWorklist &Worklist) const;
193
194 const TargetRegisterClass *
195 getDestEquivalentVGPRClass(const MachineInstr &Inst) const;
196
197 bool checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
198 const MachineInstr &MIb) const;
199
200 Register findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const;
201
202 bool verifyCopy(const MachineInstr &MI, const MachineRegisterInfo &MRI,
203 StringRef &ErrInfo) const;
204
205 bool resultDependsOnExec(const MachineInstr &MI) const;
206
207 MachineInstr *convertToThreeAddressImpl(MachineInstr &MI,
208 ThreeAddressUpdates &Updates) const;
209
210protected:
211 /// If the specific machine instruction is a instruction that moves/copies
212 /// value from one register to another register return destination and source
213 /// registers as machine operands.
214 std::optional<DestSourcePair>
215 isCopyInstrImpl(const MachineInstr &MI) const override;
216
217 bool swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,
218 AMDGPU::OpName Src0OpName, MachineOperand &Src1,
219 AMDGPU::OpName Src1OpName) const;
220 bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx,
221 unsigned toIdx) const;
222 MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
223 unsigned OpIdx0,
224 unsigned OpIdx1) const override;
225
226public:
227 enum TargetOperandFlags {
228 MO_MASK = 0xf,
229
230 MO_NONE = 0,
231 // MO_GOTPCREL -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
232 MO_GOTPCREL = 1,
233 // MO_GOTPCREL32_LO -> symbol@gotpcrel32@lo -> R_AMDGPU_GOTPCREL32_LO.
234 MO_GOTPCREL32 = 2,
235 MO_GOTPCREL32_LO = 2,
236 // MO_GOTPCREL32_HI -> symbol@gotpcrel32@hi -> R_AMDGPU_GOTPCREL32_HI.
237 MO_GOTPCREL32_HI = 3,
238 // MO_GOTPCREL64 -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
239 MO_GOTPCREL64 = 4,
240 // MO_REL32_LO -> symbol@rel32@lo -> R_AMDGPU_REL32_LO.
241 MO_REL32 = 5,
242 MO_REL32_LO = 5,
243 // MO_REL32_HI -> symbol@rel32@hi -> R_AMDGPU_REL32_HI.
244 MO_REL32_HI = 6,
245 MO_REL64 = 7,
246
247 MO_FAR_BRANCH_OFFSET = 8,
248
249 MO_ABS32_LO = 9,
250 MO_ABS32_HI = 10,
251 MO_ABS64 = 11,
252 };
253
254 explicit SIInstrInfo(const GCNSubtarget &ST);
255
256 const SIRegisterInfo &getRegisterInfo() const {
257 return RI;
258 }
259
260 const GCNSubtarget &getSubtarget() const {
261 return ST;
262 }
263
264 bool isReMaterializableImpl(const MachineInstr &MI) const override;
265
266 bool isIgnorableUse(const MachineOperand &MO) const override;
267
268 bool isSafeToSink(MachineInstr &MI, MachineBasicBlock *SuccToSinkTo,
269 MachineCycleInfo *CI) const override;
270
271 bool areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, int64_t &Offset0,
272 int64_t &Offset1) const override;
273
274 bool isGlobalMemoryObject(const MachineInstr *MI) const override;
275
276 bool getMemOperandsWithOffsetWidth(
277 const MachineInstr &LdSt,
278 SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
279 bool &OffsetIsScalable, LocationSize &Width,
280 const TargetRegisterInfo *TRI) const final;
281
282 bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
283 int64_t Offset1, bool OffsetIsScalable1,
284 ArrayRef<const MachineOperand *> BaseOps2,
285 int64_t Offset2, bool OffsetIsScalable2,
286 unsigned ClusterSize,
287 unsigned NumBytes) const override;
288
289 bool shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0,
290 int64_t Offset1, unsigned NumLoads) const override;
291
292 void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
293 const DebugLoc &DL, Register DestReg, Register SrcReg,
294 bool KillSrc, bool RenamableDest = false,
295 bool RenamableSrc = false) const override;
296
297 const TargetRegisterClass *getPreferredSelectRegClass(
298 unsigned Size) const;
299
300 Register insertNE(MachineBasicBlock *MBB,
301 MachineBasicBlock::iterator I, const DebugLoc &DL,
302 Register SrcReg, int Value) const;
303
304 Register insertEQ(MachineBasicBlock *MBB,
305 MachineBasicBlock::iterator I, const DebugLoc &DL,
306 Register SrcReg, int Value) const;
307
308 bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg,
309 int64_t &ImmVal) const override;
310
311 std::optional<int64_t> getImmOrMaterializedImm(MachineOperand &Op) const;
312
313 unsigned getVectorRegSpillSaveOpcode(Register Reg,
314 const TargetRegisterClass *RC,
315 unsigned Size,
316 const SIMachineFunctionInfo &MFI) const;
317 unsigned
318 getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC,
319 unsigned Size,
320 const SIMachineFunctionInfo &MFI) const;
321
322 void storeRegToStackSlot(
323 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
324 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
325 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
326
327 void loadRegFromStackSlot(
328 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
329 int FrameIndex, const TargetRegisterClass *RC, Register VReg,
330 unsigned SubReg = 0,
331 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
332
333 bool expandPostRAPseudo(MachineInstr &MI) const override;
334
335 void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
336 Register DestReg, unsigned SubIdx,
337 const MachineInstr &Orig) const override;
338
339 // Splits a V_MOV_B64_DPP_PSEUDO opcode into a pair of v_mov_b32_dpp
340 // instructions. Returns a pair of generated instructions.
341 // Can split either post-RA with physical registers or pre-RA with
342 // virtual registers. In latter case IR needs to be in SSA form and
343 // and a REG_SEQUENCE is produced to define original register.
344 std::pair<MachineInstr*, MachineInstr*>
345 expandMovDPP64(MachineInstr &MI) const;
346
347 // Returns an opcode that can be used to move a value to a \p DstRC
348 // register. If there is no hardware instruction that can store to \p
349 // DstRC, then AMDGPU::COPY is returned.
350 unsigned getMovOpcode(const TargetRegisterClass *DstRC) const;
351
352 const MCInstrDesc &getIndirectRegWriteMovRelPseudo(unsigned VecSize,
353 unsigned EltSize,
354 bool IsSGPR) const;
355
356 const MCInstrDesc &getIndirectGPRIDXPseudo(unsigned VecSize,
357 bool IsIndirectSrc) const;
358 LLVM_READONLY
359 int commuteOpcode(unsigned Opc) const;
360
361 LLVM_READONLY
362 inline int commuteOpcode(const MachineInstr &MI) const {
363 return commuteOpcode(Opc: MI.getOpcode());
364 }
365
366 bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0,
367 unsigned &SrcOpIdx1) const override;
368
369 bool findCommutedOpIndices(const MCInstrDesc &Desc, unsigned &SrcOpIdx0,
370 unsigned &SrcOpIdx1) const;
371
372 bool isBranchOffsetInRange(unsigned BranchOpc,
373 int64_t BrOffset) const override;
374
375 MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;
376
377 /// Return whether the block terminate with divergent branch.
378 /// Note this only work before lowering the pseudo control flow instructions.
379 bool hasDivergentBranch(const MachineBasicBlock *MBB) const;
380
381 void insertIndirectBranch(MachineBasicBlock &MBB,
382 MachineBasicBlock &NewDestBB,
383 MachineBasicBlock &RestoreBB, const DebugLoc &DL,
384 int64_t BrOffset, RegScavenger *RS) const override;
385
386 bool analyzeBranchImpl(MachineBasicBlock &MBB,
387 MachineBasicBlock::iterator I,
388 MachineBasicBlock *&TBB,
389 MachineBasicBlock *&FBB,
390 SmallVectorImpl<MachineOperand> &Cond,
391 bool AllowModify) const;
392
393 bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
394 MachineBasicBlock *&FBB,
395 SmallVectorImpl<MachineOperand> &Cond,
396 bool AllowModify = false) const override;
397
398 unsigned removeBranch(MachineBasicBlock &MBB,
399 int *BytesRemoved = nullptr) const override;
400
401 unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
402 MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
403 const DebugLoc &DL,
404 int *BytesAdded = nullptr) const override;
405
406 bool reverseBranchCondition(
407 SmallVectorImpl<MachineOperand> &Cond) const override;
408
409 bool canInsertSelect(const MachineBasicBlock &MBB,
410 ArrayRef<MachineOperand> Cond, Register DstReg,
411 Register TrueReg, Register FalseReg, int &CondCycles,
412 int &TrueCycles, int &FalseCycles) const override;
413
414 void insertSelect(MachineBasicBlock &MBB,
415 MachineBasicBlock::iterator I, const DebugLoc &DL,
416 Register DstReg, ArrayRef<MachineOperand> Cond,
417 Register TrueReg, Register FalseReg) const override;
418
419 void insertVectorSelect(MachineBasicBlock &MBB,
420 MachineBasicBlock::iterator I, const DebugLoc &DL,
421 Register DstReg, ArrayRef<MachineOperand> Cond,
422 Register TrueReg, Register FalseReg) const;
423
424 bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
425 Register &SrcReg2, int64_t &CmpMask,
426 int64_t &CmpValue) const override;
427
428 bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
429 Register SrcReg2, int64_t CmpMask, int64_t CmpValue,
430 const MachineRegisterInfo *MRI) const override;
431
432 bool
433 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
434 const MachineInstr &MIb) const override;
435
436 static bool isFoldableCopy(const MachineInstr &MI);
437 static unsigned getFoldableCopySrcIdx(const MachineInstr &MI);
438
439 void removeModOperands(MachineInstr &MI) const;
440
441 void mutateAndCleanupImplicit(MachineInstr &MI,
442 const MCInstrDesc &NewDesc) const;
443
444 /// Return the extracted immediate value in a subregister use from a constant
445 /// materialized in a super register.
446 ///
447 /// e.g. %imm = S_MOV_B64 K[0:63]
448 /// USE %imm.sub1
449 /// This will return K[32:63]
450 static std::optional<int64_t> extractSubregFromImm(int64_t ImmVal,
451 unsigned SubRegIndex);
452
453 bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
454 MachineRegisterInfo *MRI) const final;
455
456 unsigned getMachineCSELookAheadLimit() const override { return 500; }
457
458 MachineInstr *convertToThreeAddress(MachineInstr &MI, LiveVariables *LV,
459 LiveIntervals *LIS) const override;
460
461 bool isSchedulingBoundary(const MachineInstr &MI,
462 const MachineBasicBlock *MBB,
463 const MachineFunction &MF) const override;
464
465 static bool isSALU(const MachineInstr &MI) {
466 return MI.getDesc().TSFlags & SIInstrFlags::SALU;
467 }
468
469 bool isSALU(uint32_t Opcode) const {
470 return get(Opcode).TSFlags & SIInstrFlags::SALU;
471 }
472
473 static bool isVALU(const MachineInstr &MI) {
474 return MI.getDesc().TSFlags & SIInstrFlags::VALU;
475 }
476
477 bool isVALU(uint32_t Opcode) const {
478 return get(Opcode).TSFlags & SIInstrFlags::VALU;
479 }
480
481 static bool isImage(const MachineInstr &MI) {
482 return isMIMG(MI) || isVSAMPLE(MI) || isVIMAGE(MI);
483 }
484
485 bool isImage(uint32_t Opcode) const {
486 return isMIMG(Opcode) || isVSAMPLE(Opcode) || isVIMAGE(Opcode);
487 }
488
489 static bool isVMEM(const MachineInstr &MI) {
490 return isMUBUF(MI) || isMTBUF(MI) || isImage(MI) || isFLAT(MI);
491 }
492
493 bool isVMEM(uint32_t Opcode) const {
494 return isMUBUF(Opcode) || isMTBUF(Opcode) || isImage(Opcode);
495 }
496
497 static bool isSOP1(const MachineInstr &MI) {
498 return MI.getDesc().TSFlags & SIInstrFlags::SOP1;
499 }
500
501 bool isSOP1(uint32_t Opcode) const {
502 return get(Opcode).TSFlags & SIInstrFlags::SOP1;
503 }
504
505 static bool isSOP2(const MachineInstr &MI) {
506 return MI.getDesc().TSFlags & SIInstrFlags::SOP2;
507 }
508
509 bool isSOP2(uint32_t Opcode) const {
510 return get(Opcode).TSFlags & SIInstrFlags::SOP2;
511 }
512
513 static bool isSOPC(const MachineInstr &MI) {
514 return MI.getDesc().TSFlags & SIInstrFlags::SOPC;
515 }
516
517 bool isSOPC(uint32_t Opcode) const {
518 return get(Opcode).TSFlags & SIInstrFlags::SOPC;
519 }
520
521 static bool isSOPK(const MachineInstr &MI) {
522 return MI.getDesc().TSFlags & SIInstrFlags::SOPK;
523 }
524
525 bool isSOPK(uint32_t Opcode) const {
526 return get(Opcode).TSFlags & SIInstrFlags::SOPK;
527 }
528
529 static bool isSOPP(const MachineInstr &MI) {
530 return MI.getDesc().TSFlags & SIInstrFlags::SOPP;
531 }
532
533 bool isSOPP(uint32_t Opcode) const {
534 return get(Opcode).TSFlags & SIInstrFlags::SOPP;
535 }
536
537 static bool isPacked(const MachineInstr &MI) {
538 return MI.getDesc().TSFlags & SIInstrFlags::IsPacked;
539 }
540
541 bool isPacked(uint32_t Opcode) const {
542 return get(Opcode).TSFlags & SIInstrFlags::IsPacked;
543 }
544
545 static bool isVOP1(const MachineInstr &MI) {
546 return MI.getDesc().TSFlags & SIInstrFlags::VOP1;
547 }
548
549 bool isVOP1(uint32_t Opcode) const {
550 return get(Opcode).TSFlags & SIInstrFlags::VOP1;
551 }
552
553 static bool isVOP2(const MachineInstr &MI) {
554 return MI.getDesc().TSFlags & SIInstrFlags::VOP2;
555 }
556
557 bool isVOP2(uint32_t Opcode) const {
558 return get(Opcode).TSFlags & SIInstrFlags::VOP2;
559 }
560
561 static bool isVOP3(const MCInstrDesc &Desc) {
562 return Desc.TSFlags & SIInstrFlags::VOP3;
563 }
564
565 static bool isVOP3(const MachineInstr &MI) { return isVOP3(Desc: MI.getDesc()); }
566
567 bool isVOP3(uint32_t Opcode) const { return isVOP3(Desc: get(Opcode)); }
568
569 static bool isSDWA(const MachineInstr &MI) {
570 return MI.getDesc().TSFlags & SIInstrFlags::SDWA;
571 }
572
573 bool isSDWA(uint32_t Opcode) const {
574 return get(Opcode).TSFlags & SIInstrFlags::SDWA;
575 }
576
577 static bool isVOPC(const MachineInstr &MI) {
578 return MI.getDesc().TSFlags & SIInstrFlags::VOPC;
579 }
580
581 bool isVOPC(uint32_t Opcode) const {
582 return get(Opcode).TSFlags & SIInstrFlags::VOPC;
583 }
584
585 static bool isMUBUF(const MachineInstr &MI) {
586 return MI.getDesc().TSFlags & SIInstrFlags::MUBUF;
587 }
588
589 bool isMUBUF(uint32_t Opcode) const {
590 return get(Opcode).TSFlags & SIInstrFlags::MUBUF;
591 }
592
593 static bool isMTBUF(const MachineInstr &MI) {
594 return MI.getDesc().TSFlags & SIInstrFlags::MTBUF;
595 }
596
597 bool isMTBUF(uint32_t Opcode) const {
598 return get(Opcode).TSFlags & SIInstrFlags::MTBUF;
599 }
600
601 static bool isBUF(const MachineInstr &MI) {
602 return isMUBUF(MI) || isMTBUF(MI);
603 }
604
605 static bool isSMRD(const MachineInstr &MI) {
606 return MI.getDesc().TSFlags & SIInstrFlags::SMRD;
607 }
608
609 bool isSMRD(uint32_t Opcode) const {
610 return get(Opcode).TSFlags & SIInstrFlags::SMRD;
611 }
612
613 bool isBufferSMRD(const MachineInstr &MI) const;
614
615 static bool isDS(const MachineInstr &MI) {
616 return MI.getDesc().TSFlags & SIInstrFlags::DS;
617 }
618
619 bool isDS(uint32_t Opcode) const {
620 return get(Opcode).TSFlags & SIInstrFlags::DS;
621 }
622
623 static bool isLDSDMA(const MachineInstr &MI) {
624 return (isVALU(MI) && (isMUBUF(MI) || isFLAT(MI))) ||
625 (MI.getDesc().TSFlags & SIInstrFlags::TENSOR_CNT);
626 }
627
628 bool isLDSDMA(uint32_t Opcode) {
629 return (isVALU(Opcode) && (isMUBUF(Opcode) || isFLAT(Opcode))) ||
630 (get(Opcode).TSFlags & SIInstrFlags::TENSOR_CNT);
631 }
632
633 static bool isGWS(const MachineInstr &MI) {
634 return MI.getDesc().TSFlags & SIInstrFlags::GWS;
635 }
636
637 bool isGWS(uint32_t Opcode) const {
638 return get(Opcode).TSFlags & SIInstrFlags::GWS;
639 }
640
641 bool isAlwaysGDS(uint32_t Opcode) const;
642
643 static bool isMIMG(const MachineInstr &MI) {
644 return MI.getDesc().TSFlags & SIInstrFlags::MIMG;
645 }
646
647 bool isMIMG(uint32_t Opcode) const {
648 return get(Opcode).TSFlags & SIInstrFlags::MIMG;
649 }
650
651 static bool isVIMAGE(const MachineInstr &MI) {
652 return MI.getDesc().TSFlags & SIInstrFlags::VIMAGE;
653 }
654
655 bool isVIMAGE(uint32_t Opcode) const {
656 return get(Opcode).TSFlags & SIInstrFlags::VIMAGE;
657 }
658
659 static bool isVSAMPLE(const MachineInstr &MI) {
660 return MI.getDesc().TSFlags & SIInstrFlags::VSAMPLE;
661 }
662
663 bool isVSAMPLE(uint32_t Opcode) const {
664 return get(Opcode).TSFlags & SIInstrFlags::VSAMPLE;
665 }
666
667 static bool isGather4(const MachineInstr &MI) {
668 return MI.getDesc().TSFlags & SIInstrFlags::Gather4;
669 }
670
671 bool isGather4(uint32_t Opcode) const {
672 return get(Opcode).TSFlags & SIInstrFlags::Gather4;
673 }
674
675 static bool isFLAT(const MachineInstr &MI) {
676 return MI.getDesc().TSFlags & SIInstrFlags::FLAT;
677 }
678
679 // Is a FLAT encoded instruction which accesses a specific segment,
680 // i.e. global_* or scratch_*.
681 static bool isSegmentSpecificFLAT(const MachineInstr &MI) {
682 auto Flags = MI.getDesc().TSFlags;
683 return Flags & (SIInstrFlags::FlatGlobal | SIInstrFlags::FlatScratch);
684 }
685
686 bool isSegmentSpecificFLAT(uint32_t Opcode) const {
687 auto Flags = get(Opcode).TSFlags;
688 return Flags & (SIInstrFlags::FlatGlobal | SIInstrFlags::FlatScratch);
689 }
690
691 static bool isFLATGlobal(const MachineInstr &MI) {
692 return MI.getDesc().TSFlags & SIInstrFlags::FlatGlobal;
693 }
694
695 bool isFLATGlobal(uint32_t Opcode) const {
696 return get(Opcode).TSFlags & SIInstrFlags::FlatGlobal;
697 }
698
699 static bool isFLATScratch(const MachineInstr &MI) {
700 return MI.getDesc().TSFlags & SIInstrFlags::FlatScratch;
701 }
702
703 bool isFLATScratch(uint32_t Opcode) const {
704 return get(Opcode).TSFlags & SIInstrFlags::FlatScratch;
705 }
706
707 // Any FLAT encoded instruction, including global_* and scratch_*.
708 bool isFLAT(uint32_t Opcode) const {
709 return get(Opcode).TSFlags & SIInstrFlags::FLAT;
710 }
711
712 /// \returns true for SCRATCH_ instructions, or FLAT/BUF instructions unless
713 /// the MMOs do not include scratch.
714 /// Conservatively correct; will return true if \p MI cannot be proven
715 /// to not hit scratch.
716 bool mayAccessScratch(const MachineInstr &MI) const;
717
718 /// \returns true for FLAT instructions that can access VMEM.
719 bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const;
720
721 /// \returns true for FLAT instructions that can access LDS.
722 bool mayAccessLDSThroughFlat(const MachineInstr &MI) const;
723
724 static bool isBlockLoadStore(uint32_t Opcode) {
725 switch (Opcode) {
726 case AMDGPU::SI_BLOCK_SPILL_V1024_SAVE:
727 case AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE:
728 case AMDGPU::SCRATCH_STORE_BLOCK_SADDR:
729 case AMDGPU::SCRATCH_LOAD_BLOCK_SADDR:
730 case AMDGPU::SCRATCH_STORE_BLOCK_SVS:
731 case AMDGPU::SCRATCH_LOAD_BLOCK_SVS:
732 return true;
733 default:
734 return false;
735 }
736 }
737
738 static bool setsSCCIfResultIsNonZero(const MachineInstr &MI) {
739 switch (MI.getOpcode()) {
740 case AMDGPU::S_ABSDIFF_I32:
741 case AMDGPU::S_ABS_I32:
742 case AMDGPU::S_AND_B32:
743 case AMDGPU::S_AND_B64:
744 case AMDGPU::S_ANDN2_B32:
745 case AMDGPU::S_ANDN2_B64:
746 case AMDGPU::S_ASHR_I32:
747 case AMDGPU::S_ASHR_I64:
748 case AMDGPU::S_BCNT0_I32_B32:
749 case AMDGPU::S_BCNT0_I32_B64:
750 case AMDGPU::S_BCNT1_I32_B32:
751 case AMDGPU::S_BCNT1_I32_B64:
752 case AMDGPU::S_BFE_I32:
753 case AMDGPU::S_BFE_I64:
754 case AMDGPU::S_BFE_U32:
755 case AMDGPU::S_BFE_U64:
756 case AMDGPU::S_LSHL_B32:
757 case AMDGPU::S_LSHL_B64:
758 case AMDGPU::S_LSHR_B32:
759 case AMDGPU::S_LSHR_B64:
760 case AMDGPU::S_NAND_B32:
761 case AMDGPU::S_NAND_B64:
762 case AMDGPU::S_NOR_B32:
763 case AMDGPU::S_NOR_B64:
764 case AMDGPU::S_NOT_B32:
765 case AMDGPU::S_NOT_B64:
766 case AMDGPU::S_OR_B32:
767 case AMDGPU::S_OR_B64:
768 case AMDGPU::S_ORN2_B32:
769 case AMDGPU::S_ORN2_B64:
770 case AMDGPU::S_QUADMASK_B32:
771 case AMDGPU::S_QUADMASK_B64:
772 case AMDGPU::S_WQM_B32:
773 case AMDGPU::S_WQM_B64:
774 case AMDGPU::S_XNOR_B32:
775 case AMDGPU::S_XNOR_B64:
776 case AMDGPU::S_XOR_B32:
777 case AMDGPU::S_XOR_B64:
778 return true;
779 default:
780 return false;
781 }
782 }
783
784 static bool isEXP(const MachineInstr &MI) {
785 return MI.getDesc().TSFlags & SIInstrFlags::EXP;
786 }
787
788 static bool isDualSourceBlendEXP(const MachineInstr &MI) {
789 if (!isEXP(MI))
790 return false;
791 unsigned Target = MI.getOperand(i: 0).getImm();
792 return Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND0 ||
793 Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND1;
794 }
795
796 bool isEXP(uint32_t Opcode) const {
797 return get(Opcode).TSFlags & SIInstrFlags::EXP;
798 }
799
800 static bool isAtomicNoRet(const MachineInstr &MI) {
801 return MI.getDesc().TSFlags & SIInstrFlags::IsAtomicNoRet;
802 }
803
804 bool isAtomicNoRet(uint32_t Opcode) const {
805 return get(Opcode).TSFlags & SIInstrFlags::IsAtomicNoRet;
806 }
807
808 static bool isAtomicRet(const MachineInstr &MI) {
809 return MI.getDesc().TSFlags & SIInstrFlags::IsAtomicRet;
810 }
811
812 bool isAtomicRet(uint32_t Opcode) const {
813 return get(Opcode).TSFlags & SIInstrFlags::IsAtomicRet;
814 }
815
816 static bool isAtomic(const MachineInstr &MI) {
817 return MI.getDesc().TSFlags & (SIInstrFlags::IsAtomicRet |
818 SIInstrFlags::IsAtomicNoRet);
819 }
820
821 bool isAtomic(uint32_t Opcode) const {
822 return get(Opcode).TSFlags & (SIInstrFlags::IsAtomicRet |
823 SIInstrFlags::IsAtomicNoRet);
824 }
825
826 static bool mayWriteLDSThroughDMA(const MachineInstr &MI) {
827 unsigned Opc = MI.getOpcode();
828 // Exclude instructions that read FROM LDS (not write to it)
829 return isLDSDMA(MI) && Opc != AMDGPU::BUFFER_STORE_LDS_DWORD &&
830 Opc != AMDGPU::TENSOR_STORE_FROM_LDS &&
831 Opc != AMDGPU::TENSOR_STORE_FROM_LDS_D2;
832 }
833
834 static bool isSBarrierSCCWrite(unsigned Opcode) {
835 return Opcode == AMDGPU::S_BARRIER_LEAVE ||
836 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM ||
837 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0;
838 }
839
840 static bool isCBranchVCCZRead(const MachineInstr &MI) {
841 unsigned Opc = MI.getOpcode();
842 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
843 !MI.getOperand(i: 1).isUndef();
844 }
845
846 static bool isWQM(const MachineInstr &MI) {
847 return MI.getDesc().TSFlags & SIInstrFlags::WQM;
848 }
849
850 bool isWQM(uint32_t Opcode) const {
851 return get(Opcode).TSFlags & SIInstrFlags::WQM;
852 }
853
854 static bool isDisableWQM(const MachineInstr &MI) {
855 return MI.getDesc().TSFlags & SIInstrFlags::DisableWQM;
856 }
857
858 bool isDisableWQM(uint32_t Opcode) const {
859 return get(Opcode).TSFlags & SIInstrFlags::DisableWQM;
860 }
861
862 // SI_SPILL_S32_TO_VGPR and SI_RESTORE_S32_FROM_VGPR form a special case of
863 // SGPRs spilling to VGPRs which are SGPR spills but from VALU instructions
864 // therefore we need an explicit check for them since just checking if the
865 // Spill bit is set and what instruction type it came from misclassifies
866 // them.
867 static bool isVGPRSpill(const MachineInstr &MI) {
868 return MI.getOpcode() != AMDGPU::SI_SPILL_S32_TO_VGPR &&
869 MI.getOpcode() != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
870 (isSpill(MI) && isVALU(MI));
871 }
872
873 bool isVGPRSpill(uint32_t Opcode) const {
874 return Opcode != AMDGPU::SI_SPILL_S32_TO_VGPR &&
875 Opcode != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
876 (isSpill(Opcode) && isVALU(Opcode));
877 }
878
879 static bool isSGPRSpill(const MachineInstr &MI) {
880 return MI.getOpcode() == AMDGPU::SI_SPILL_S32_TO_VGPR ||
881 MI.getOpcode() == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
882 (isSpill(MI) && isSALU(MI));
883 }
884
885 bool isSGPRSpill(uint32_t Opcode) const {
886 return Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR ||
887 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
888 (isSpill(Opcode) && isSALU(Opcode));
889 }
890
891 bool isSpill(uint32_t Opcode) const {
892 return get(Opcode).TSFlags & SIInstrFlags::Spill;
893 }
894
895 static bool isSpill(const MCInstrDesc &Desc) {
896 return Desc.TSFlags & SIInstrFlags::Spill;
897 }
898
899 static bool isSpill(const MachineInstr &MI) { return isSpill(Desc: MI.getDesc()); }
900
901 static bool isWWMRegSpillOpcode(uint32_t Opcode) {
902 return Opcode == AMDGPU::SI_SPILL_WWM_V32_SAVE ||
903 Opcode == AMDGPU::SI_SPILL_WWM_AV32_SAVE ||
904 Opcode == AMDGPU::SI_SPILL_WWM_V32_RESTORE ||
905 Opcode == AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
906 }
907
908 static bool isChainCallOpcode(uint64_t Opcode) {
909 return Opcode == AMDGPU::SI_CS_CHAIN_TC_W32 ||
910 Opcode == AMDGPU::SI_CS_CHAIN_TC_W64;
911 }
912
913 static bool isDPP(const MachineInstr &MI) {
914 return MI.getDesc().TSFlags & SIInstrFlags::DPP;
915 }
916
917 bool isDPP(uint32_t Opcode) const {
918 return get(Opcode).TSFlags & SIInstrFlags::DPP;
919 }
920
921 static bool isTRANS(const MachineInstr &MI) {
922 return MI.getDesc().TSFlags & SIInstrFlags::TRANS;
923 }
924
925 bool isTRANS(uint32_t Opcode) const {
926 return get(Opcode).TSFlags & SIInstrFlags::TRANS;
927 }
928
929 static bool isVOP3P(const MachineInstr &MI) {
930 return MI.getDesc().TSFlags & SIInstrFlags::VOP3P;
931 }
932
933 bool isVOP3P(uint32_t Opcode) const {
934 return get(Opcode).TSFlags & SIInstrFlags::VOP3P;
935 }
936
937 static bool isVINTRP(const MachineInstr &MI) {
938 return MI.getDesc().TSFlags & SIInstrFlags::VINTRP;
939 }
940
941 bool isVINTRP(uint32_t Opcode) const {
942 return get(Opcode).TSFlags & SIInstrFlags::VINTRP;
943 }
944
945 static bool isMAI(const MCInstrDesc &Desc) {
946 return Desc.TSFlags & SIInstrFlags::IsMAI;
947 }
948
949 static bool isMAI(const MachineInstr &MI) { return isMAI(Desc: MI.getDesc()); }
950
951 bool isMAI(uint32_t Opcode) const { return isMAI(Desc: get(Opcode)); }
952
953 static bool isMFMA(const MachineInstr &MI) {
954 return isMAI(MI) && MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
955 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
956 }
957
958 bool isMFMA(uint32_t Opcode) const {
959 return isMAI(Opcode) && Opcode != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
960 Opcode != AMDGPU::V_ACCVGPR_READ_B32_e64;
961 }
962
963 static bool isDOT(const MachineInstr &MI) {
964 return MI.getDesc().TSFlags & SIInstrFlags::IsDOT;
965 }
966
967 static bool isWMMA(const MachineInstr &MI) {
968 return MI.getDesc().TSFlags & SIInstrFlags::IsWMMA;
969 }
970
971 bool isWMMA(uint32_t Opcode) const {
972 return get(Opcode).TSFlags & SIInstrFlags::IsWMMA;
973 }
974
975 static bool isMFMAorWMMA(const MachineInstr &MI) {
976 return isMFMA(MI) || isWMMA(MI) || isSWMMAC(MI);
977 }
978
979 bool isMFMAorWMMA(uint32_t Opcode) const {
980 return isMFMA(Opcode) || isWMMA(Opcode) || isSWMMAC(Opcode);
981 }
982
983 static bool isSWMMAC(const MachineInstr &MI) {
984 return MI.getDesc().TSFlags & SIInstrFlags::IsSWMMAC;
985 }
986
987 bool isSWMMAC(uint32_t Opcode) const {
988 return get(Opcode).TSFlags & SIInstrFlags::IsSWMMAC;
989 }
990
991 bool isDOT(uint32_t Opcode) const {
992 return get(Opcode).TSFlags & SIInstrFlags::IsDOT;
993 }
994
995 bool isXDLWMMA(const MachineInstr &MI) const;
996
997 bool isXDL(const MachineInstr &MI) const;
998
999 static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opc: Opcode); }
1000
1001 static bool isLDSDIR(const MachineInstr &MI) {
1002 return MI.getDesc().TSFlags & SIInstrFlags::LDSDIR;
1003 }
1004
1005 bool isLDSDIR(uint32_t Opcode) const {
1006 return get(Opcode).TSFlags & SIInstrFlags::LDSDIR;
1007 }
1008
1009 static bool isVINTERP(const MachineInstr &MI) {
1010 return MI.getDesc().TSFlags & SIInstrFlags::VINTERP;
1011 }
1012
1013 bool isVINTERP(uint32_t Opcode) const {
1014 return get(Opcode).TSFlags & SIInstrFlags::VINTERP;
1015 }
1016
1017 static bool isScalarUnit(const MachineInstr &MI) {
1018 return MI.getDesc().TSFlags & (SIInstrFlags::SALU | SIInstrFlags::SMRD);
1019 }
1020
1021 static bool usesVM_CNT(const MachineInstr &MI) {
1022 return MI.getDesc().TSFlags & SIInstrFlags::VM_CNT;
1023 }
1024
1025 static bool usesLGKM_CNT(const MachineInstr &MI) {
1026 return MI.getDesc().TSFlags & SIInstrFlags::LGKM_CNT;
1027 }
1028
1029 static bool usesASYNC_CNT(const MachineInstr &MI) {
1030 return MI.getDesc().TSFlags & SIInstrFlags::ASYNC_CNT;
1031 }
1032
1033 bool usesASYNC_CNT(uint32_t Opcode) const {
1034 return get(Opcode).TSFlags & SIInstrFlags::ASYNC_CNT;
1035 }
1036
1037 // Most sopk treat the immediate as a signed 16-bit, however some
1038 // use it as unsigned.
1039 static bool sopkIsZext(unsigned Opcode) {
1040 return Opcode == AMDGPU::S_CMPK_EQ_U32 || Opcode == AMDGPU::S_CMPK_LG_U32 ||
1041 Opcode == AMDGPU::S_CMPK_GT_U32 || Opcode == AMDGPU::S_CMPK_GE_U32 ||
1042 Opcode == AMDGPU::S_CMPK_LT_U32 || Opcode == AMDGPU::S_CMPK_LE_U32 ||
1043 Opcode == AMDGPU::S_GETREG_B32 ||
1044 Opcode == AMDGPU::S_GETREG_B32_const;
1045 }
1046
1047 /// \returns true if this is an s_store_dword* instruction. This is more
1048 /// specific than isSMEM && mayStore.
1049 static bool isScalarStore(const MachineInstr &MI) {
1050 return MI.getDesc().TSFlags & SIInstrFlags::SCALAR_STORE;
1051 }
1052
1053 bool isScalarStore(uint32_t Opcode) const {
1054 return get(Opcode).TSFlags & SIInstrFlags::SCALAR_STORE;
1055 }
1056
1057 static bool isFixedSize(const MachineInstr &MI) {
1058 return MI.getDesc().TSFlags & SIInstrFlags::FIXED_SIZE;
1059 }
1060
1061 bool isFixedSize(uint32_t Opcode) const {
1062 return get(Opcode).TSFlags & SIInstrFlags::FIXED_SIZE;
1063 }
1064
1065 static bool hasFPClamp(const MachineInstr &MI) {
1066 return MI.getDesc().TSFlags & SIInstrFlags::FPClamp;
1067 }
1068
1069 bool hasFPClamp(uint32_t Opcode) const {
1070 return get(Opcode).TSFlags & SIInstrFlags::FPClamp;
1071 }
1072
1073 static bool hasIntClamp(const MachineInstr &MI) {
1074 return MI.getDesc().TSFlags & SIInstrFlags::IntClamp;
1075 }
1076
1077 uint64_t getClampMask(const MachineInstr &MI) const {
1078 const uint64_t ClampFlags = SIInstrFlags::FPClamp |
1079 SIInstrFlags::IntClamp |
1080 SIInstrFlags::ClampLo |
1081 SIInstrFlags::ClampHi;
1082 return MI.getDesc().TSFlags & ClampFlags;
1083 }
1084
1085 static bool usesFPDPRounding(const MachineInstr &MI) {
1086 return MI.getDesc().TSFlags & SIInstrFlags::FPDPRounding;
1087 }
1088
1089 bool usesFPDPRounding(uint32_t Opcode) const {
1090 return get(Opcode).TSFlags & SIInstrFlags::FPDPRounding;
1091 }
1092
1093 static bool isFPAtomic(const MachineInstr &MI) {
1094 return MI.getDesc().TSFlags & SIInstrFlags::FPAtomic;
1095 }
1096
1097 bool isFPAtomic(uint32_t Opcode) const {
1098 return get(Opcode).TSFlags & SIInstrFlags::FPAtomic;
1099 }
1100
1101 static bool isNeverUniform(const MachineInstr &MI) {
1102 return MI.getDesc().TSFlags & SIInstrFlags::IsNeverUniform;
1103 }
1104
1105 // Check to see if opcode is for a barrier start. Pre gfx12 this is just the
1106 // S_BARRIER, but after support for S_BARRIER_SIGNAL* / S_BARRIER_WAIT we want
1107 // to check for the barrier start (S_BARRIER_SIGNAL*)
1108 bool isBarrierStart(unsigned Opcode) const {
1109 return Opcode == AMDGPU::S_BARRIER ||
1110 Opcode == AMDGPU::S_BARRIER_SIGNAL_M0 ||
1111 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0 ||
1112 Opcode == AMDGPU::S_BARRIER_SIGNAL_IMM ||
1113 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM;
1114 }
1115
1116 bool isBarrier(unsigned Opcode) const {
1117 return isBarrierStart(Opcode) || Opcode == AMDGPU::S_BARRIER_WAIT ||
1118 Opcode == AMDGPU::S_BARRIER_INIT_M0 ||
1119 Opcode == AMDGPU::S_BARRIER_INIT_IMM ||
1120 Opcode == AMDGPU::S_BARRIER_JOIN_IMM ||
1121 Opcode == AMDGPU::S_BARRIER_LEAVE || Opcode == AMDGPU::DS_GWS_INIT ||
1122 Opcode == AMDGPU::DS_GWS_BARRIER;
1123 }
1124
1125 static bool isGFX12CacheInvOrWBInst(unsigned Opc) {
1126 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||
1127 Opc == AMDGPU::GLOBAL_WBINV;
1128 }
1129
1130 static bool isF16PseudoScalarTrans(unsigned Opcode) {
1131 return Opcode == AMDGPU::V_S_EXP_F16_e64 ||
1132 Opcode == AMDGPU::V_S_LOG_F16_e64 ||
1133 Opcode == AMDGPU::V_S_RCP_F16_e64 ||
1134 Opcode == AMDGPU::V_S_RSQ_F16_e64 ||
1135 Opcode == AMDGPU::V_S_SQRT_F16_e64;
1136 }
1137
1138 static bool doesNotReadTiedSource(const MachineInstr &MI) {
1139 return MI.getDesc().TSFlags & SIInstrFlags::TiedSourceNotRead;
1140 }
1141
1142 bool doesNotReadTiedSource(uint32_t Opcode) const {
1143 return get(Opcode).TSFlags & SIInstrFlags::TiedSourceNotRead;
1144 }
1145
1146 bool isIGLP(unsigned Opcode) const {
1147 return Opcode == AMDGPU::SCHED_BARRIER ||
1148 Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
1149 }
1150
1151 bool isIGLP(const MachineInstr &MI) const { return isIGLP(Opcode: MI.getOpcode()); }
1152
1153 // Return true if the instruction is mutually exclusive with all non-IGLP DAG
1154 // mutations, requiring all other mutations to be disabled.
1155 bool isIGLPMutationOnly(unsigned Opcode) const {
1156 return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
1157 }
1158
1159 static unsigned getNonSoftWaitcntOpcode(unsigned Opcode) {
1160 switch (Opcode) {
1161 case AMDGPU::S_WAITCNT_soft:
1162 return AMDGPU::S_WAITCNT;
1163 case AMDGPU::S_WAITCNT_VSCNT_soft:
1164 return AMDGPU::S_WAITCNT_VSCNT;
1165 case AMDGPU::S_WAIT_LOADCNT_soft:
1166 return AMDGPU::S_WAIT_LOADCNT;
1167 case AMDGPU::S_WAIT_STORECNT_soft:
1168 return AMDGPU::S_WAIT_STORECNT;
1169 case AMDGPU::S_WAIT_SAMPLECNT_soft:
1170 return AMDGPU::S_WAIT_SAMPLECNT;
1171 case AMDGPU::S_WAIT_BVHCNT_soft:
1172 return AMDGPU::S_WAIT_BVHCNT;
1173 case AMDGPU::S_WAIT_DSCNT_soft:
1174 return AMDGPU::S_WAIT_DSCNT;
1175 case AMDGPU::S_WAIT_KMCNT_soft:
1176 return AMDGPU::S_WAIT_KMCNT;
1177 case AMDGPU::S_WAIT_XCNT_soft:
1178 return AMDGPU::S_WAIT_XCNT;
1179 default:
1180 return Opcode;
1181 }
1182 }
1183
1184 static bool isWaitcnt(unsigned Opcode) {
1185 switch (getNonSoftWaitcntOpcode(Opcode)) {
1186 case AMDGPU::S_WAITCNT:
1187 case AMDGPU::S_WAITCNT_VSCNT:
1188 case AMDGPU::S_WAITCNT_VMCNT:
1189 case AMDGPU::S_WAITCNT_EXPCNT:
1190 case AMDGPU::S_WAITCNT_LGKMCNT:
1191 case AMDGPU::S_WAIT_LOADCNT:
1192 case AMDGPU::S_WAIT_LOADCNT_DSCNT:
1193 case AMDGPU::S_WAIT_STORECNT:
1194 case AMDGPU::S_WAIT_STORECNT_DSCNT:
1195 case AMDGPU::S_WAIT_SAMPLECNT:
1196 case AMDGPU::S_WAIT_BVHCNT:
1197 case AMDGPU::S_WAIT_EXPCNT:
1198 case AMDGPU::S_WAIT_DSCNT:
1199 case AMDGPU::S_WAIT_KMCNT:
1200 case AMDGPU::S_WAIT_IDLE:
1201 return true;
1202 default:
1203 return false;
1204 }
1205 }
1206
1207 bool isVGPRCopy(const MachineInstr &MI) const {
1208 assert(isCopyInstr(MI));
1209 Register Dest = MI.getOperand(i: 0).getReg();
1210 const MachineFunction &MF = *MI.getMF();
1211 const MachineRegisterInfo &MRI = MF.getRegInfo();
1212 return !RI.isSGPRReg(MRI, Reg: Dest);
1213 }
1214
1215 bool hasVGPRUses(const MachineInstr &MI) const {
1216 const MachineFunction &MF = *MI.getMF();
1217 const MachineRegisterInfo &MRI = MF.getRegInfo();
1218 return llvm::any_of(Range: MI.explicit_uses(),
1219 P: [&MRI, this](const MachineOperand &MO) {
1220 return MO.isReg() && RI.isVGPR(MRI, Reg: MO.getReg());});
1221 }
1222
1223 /// Return true if the instruction modifies the mode register.q
1224 static bool modifiesModeRegister(const MachineInstr &MI);
1225
1226 /// This function is used to determine if an instruction can be safely
1227 /// executed under EXEC = 0 without hardware error, indeterminate results,
1228 /// and/or visible effects on future vector execution or outside the shader.
1229 /// Note: as of 2024 the only use of this is SIPreEmitPeephole where it is
1230 /// used in removing branches over short EXEC = 0 sequences.
1231 /// As such it embeds certain assumptions which may not apply to every case
1232 /// of EXEC = 0 execution.
1233 bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const;
1234
1235 /// Returns true if the instruction could potentially depend on the value of
1236 /// exec. If false, exec dependencies may safely be ignored.
1237 bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const;
1238
1239 bool isInlineConstant(const APInt &Imm) const;
1240
1241 bool isInlineConstant(const APFloat &Imm) const;
1242
1243 // Returns true if this non-register operand definitely does not need to be
1244 // encoded as a 32-bit literal. Note that this function handles all kinds of
1245 // operands, not just immediates.
1246 //
1247 // Some operands like FrameIndexes could resolve to an inline immediate value
1248 // that will not require an additional 4-bytes; this function assumes that it
1249 // will.
1250 bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const {
1251 if (!MO.isImm())
1252 return false;
1253 return isInlineConstant(ImmVal: MO.getImm(), OperandType);
1254 }
1255 bool isInlineConstant(int64_t ImmVal, uint8_t OperandType) const;
1256
1257 bool isInlineConstant(const MachineOperand &MO,
1258 const MCOperandInfo &OpInfo) const {
1259 return isInlineConstant(MO, OperandType: OpInfo.OperandType);
1260 }
1261
1262 /// \p returns true if \p UseMO is substituted with \p DefMO in \p MI it would
1263 /// be an inline immediate.
1264 bool isInlineConstant(const MachineInstr &MI,
1265 const MachineOperand &UseMO,
1266 const MachineOperand &DefMO) const {
1267 assert(UseMO.getParent() == &MI);
1268 int OpIdx = UseMO.getOperandNo();
1269 if (OpIdx >= MI.getDesc().NumOperands)
1270 return false;
1271
1272 return isInlineConstant(MO: DefMO, OpInfo: MI.getDesc().operands()[OpIdx]);
1273 }
1274
1275 /// \p returns true if the operand \p OpIdx in \p MI is a valid inline
1276 /// immediate.
1277 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx) const {
1278 const MachineOperand &MO = MI.getOperand(i: OpIdx);
1279 return isInlineConstant(MO, OperandType: MI.getDesc().operands()[OpIdx].OperandType);
1280 }
1281
1282 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
1283 int64_t ImmVal) const {
1284 if (OpIdx >= MI.getDesc().NumOperands)
1285 return false;
1286
1287 if (isCopyInstr(MI)) {
1288 unsigned Size = getOpSize(MI, OpNo: OpIdx);
1289 assert(Size == 8 || Size == 4);
1290
1291 uint8_t OpType = (Size == 8) ?
1292 AMDGPU::OPERAND_REG_IMM_INT64 : AMDGPU::OPERAND_REG_IMM_INT32;
1293 return isInlineConstant(ImmVal, OperandType: OpType);
1294 }
1295
1296 return isInlineConstant(ImmVal, OperandType: MI.getDesc().operands()[OpIdx].OperandType);
1297 }
1298
1299 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
1300 const MachineOperand &MO) const {
1301 return isInlineConstant(MI, OpIdx, ImmVal: MO.getImm());
1302 }
1303
1304 bool isInlineConstant(const MachineOperand &MO) const {
1305 return isInlineConstant(MI: *MO.getParent(), OpIdx: MO.getOperandNo());
1306 }
1307
1308 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
1309 const MachineOperand &MO) const;
1310
1311 bool isLiteralOperandLegal(const MCInstrDesc &InstDesc,
1312 const MCOperandInfo &OpInfo) const;
1313
1314 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
1315 int64_t ImmVal) const;
1316
1317 bool isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
1318 const MachineOperand &MO) const {
1319 return isImmOperandLegal(InstDesc: MI.getDesc(), OpNo, MO);
1320 }
1321
1322 bool isNeverCoissue(MachineInstr &MI) const;
1323
1324 /// Check if this immediate value can be used for AV_MOV_B64_IMM_PSEUDO.
1325 bool isLegalAV64PseudoImm(uint64_t Imm) const;
1326
1327 /// Return true if this 64-bit VALU instruction has a 32-bit encoding.
1328 /// This function will return false if you pass it a 32-bit instruction.
1329 bool hasVALU32BitEncoding(unsigned Opcode) const;
1330
1331 bool physRegUsesConstantBus(const MachineOperand &Reg) const;
1332 bool regUsesConstantBus(const MachineOperand &Reg,
1333 const MachineRegisterInfo &MRI) const;
1334
1335 /// Returns true if this operand uses the constant bus.
1336 bool usesConstantBus(const MachineRegisterInfo &MRI,
1337 const MachineOperand &MO,
1338 const MCOperandInfo &OpInfo) const;
1339
1340 bool usesConstantBus(const MachineRegisterInfo &MRI, const MachineInstr &MI,
1341 int OpIdx) const {
1342 return usesConstantBus(MRI, MO: MI.getOperand(i: OpIdx),
1343 OpInfo: MI.getDesc().operands()[OpIdx]);
1344 }
1345
1346 /// Return true if this instruction has any modifiers.
1347 /// e.g. src[012]_mod, omod, clamp.
1348 bool hasModifiers(unsigned Opcode) const;
1349
1350 bool hasModifiersSet(const MachineInstr &MI, AMDGPU::OpName OpName) const;
1351 bool hasAnyModifiersSet(const MachineInstr &MI) const;
1352
1353 bool canShrink(const MachineInstr &MI,
1354 const MachineRegisterInfo &MRI) const;
1355
1356 MachineInstr *buildShrunkInst(MachineInstr &MI,
1357 unsigned NewOpcode) const;
1358
1359 bool verifyInstruction(const MachineInstr &MI,
1360 StringRef &ErrInfo) const override;
1361
1362 unsigned getVALUOp(const MachineInstr &MI) const;
1363
1364 void insertScratchExecCopy(MachineFunction &MF, MachineBasicBlock &MBB,
1365 MachineBasicBlock::iterator MBBI,
1366 const DebugLoc &DL, Register Reg, bool IsSCCLive,
1367 SlotIndexes *Indexes = nullptr) const;
1368
1369 void restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
1370 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
1371 Register Reg, SlotIndexes *Indexes = nullptr) const;
1372
1373 MachineInstr *getWholeWaveFunctionSetup(MachineFunction &MF) const;
1374
1375 /// Return the correct register class for \p OpNo. For target-specific
1376 /// instructions, this will return the register class that has been defined
1377 /// in tablegen. For generic instructions, like REG_SEQUENCE it will return
1378 /// the register class of its machine operand.
1379 /// to infer the correct register class base on the other operands.
1380 const TargetRegisterClass *getOpRegClass(const MachineInstr &MI,
1381 unsigned OpNo) const;
1382
1383 /// Return the size in bytes of the operand OpNo on the given
1384 // instruction opcode.
1385 unsigned getOpSize(uint32_t Opcode, unsigned OpNo) const {
1386 const MCOperandInfo &OpInfo = get(Opcode).operands()[OpNo];
1387
1388 if (OpInfo.RegClass == -1) {
1389 // If this is an immediate operand, this must be a 32-bit literal.
1390 assert(OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE);
1391 return 4;
1392 }
1393
1394 return RI.getRegSizeInBits(RC: *RI.getRegClass(i: getOpRegClassID(OpInfo))) / 8;
1395 }
1396
1397 /// This form should usually be preferred since it handles operands
1398 /// with unknown register classes.
1399 unsigned getOpSize(const MachineInstr &MI, unsigned OpNo) const {
1400 const MachineOperand &MO = MI.getOperand(i: OpNo);
1401 if (MO.isReg()) {
1402 if (unsigned SubReg = MO.getSubReg()) {
1403 return RI.getSubRegIdxSize(Idx: SubReg) / 8;
1404 }
1405 }
1406 return RI.getRegSizeInBits(RC: *getOpRegClass(MI, OpNo)) / 8;
1407 }
1408
1409 /// Legalize the \p OpIndex operand of this instruction by inserting
1410 /// a MOV. For example:
1411 /// ADD_I32_e32 VGPR0, 15
1412 /// to
1413 /// MOV VGPR1, 15
1414 /// ADD_I32_e32 VGPR0, VGPR1
1415 ///
1416 /// If the operand being legalized is a register, then a COPY will be used
1417 /// instead of MOV.
1418 void legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const;
1419
1420 /// Check if \p MO is a legal operand if it was the \p OpIdx Operand
1421 /// for \p MI.
1422 bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
1423 const MachineOperand *MO = nullptr) const;
1424
1425 /// Check if \p MO would be a valid operand for the given operand
1426 /// definition \p OpInfo. Note this does not attempt to validate constant bus
1427 /// restrictions (e.g. literal constant usage).
1428 bool isLegalVSrcOperand(const MachineRegisterInfo &MRI,
1429 const MCOperandInfo &OpInfo,
1430 const MachineOperand &MO) const;
1431
1432 /// Check if \p MO (a register operand) is a legal register for the
1433 /// given operand description or operand index.
1434 /// The operand index version provide more legality checks
1435 bool isLegalRegOperand(const MachineRegisterInfo &MRI,
1436 const MCOperandInfo &OpInfo,
1437 const MachineOperand &MO) const;
1438 bool isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
1439 const MachineOperand &MO) const;
1440
1441 /// Check if \p MO would be a legal operand for gfx12+ packed math FP32
1442 /// instructions. Packed math FP32 instructions typically accept SGPRs or
1443 /// VGPRs as source operands. On gfx12+, if a source operand uses SGPRs, the
1444 /// HW can only read the first SGPR and use it for both the low and high
1445 /// operations.
1446 /// \p SrcN can be 0, 1, or 2, representing src0, src1, and src2,
1447 /// respectively. If \p MO is nullptr, the operand corresponding to SrcN will
1448 /// be used.
1449 bool isLegalGFX12PlusPackedMathFP32Operand(
1450 const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,
1451 const MachineOperand *MO = nullptr) const;
1452
1453 /// Legalize operands in \p MI by either commuting it or inserting a
1454 /// copy of src1.
1455 void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1456
1457 /// Fix operands in \p MI to satisfy constant bus requirements.
1458 void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1459
1460 /// Copy a value from a VGPR (\p SrcReg) to SGPR. The desired register class
1461 /// for the dst register (\p DstRC) can be optionally supplied. This function
1462 /// can only be used when it is know that the value in SrcReg is same across
1463 /// all threads in the wave.
1464 /// \returns The SGPR register that \p SrcReg was copied to.
1465 Register readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,
1466 MachineRegisterInfo &MRI,
1467 const TargetRegisterClass *DstRC = nullptr) const;
1468
1469 void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1470 void legalizeOperandsFLAT(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1471
1472 void legalizeGenericOperand(MachineBasicBlock &InsertMBB,
1473 MachineBasicBlock::iterator I,
1474 const TargetRegisterClass *DstRC,
1475 MachineOperand &Op, MachineRegisterInfo &MRI,
1476 const DebugLoc &DL) const;
1477
1478 /// Legalize all operands in this instruction. This function may create new
1479 /// instructions and control-flow around \p MI. If present, \p MDT is
1480 /// updated.
1481 /// \returns A new basic block that contains \p MI if new blocks were created.
1482 MachineBasicBlock *
1483 legalizeOperands(MachineInstr &MI, MachineDominatorTree *MDT = nullptr) const;
1484
1485 /// Change SADDR form of a FLAT \p Inst to its VADDR form if saddr operand
1486 /// was moved to VGPR. \returns true if succeeded.
1487 bool moveFlatAddrToVGPR(MachineInstr &Inst) const;
1488
1489 /// Fix operands in Inst to fix 16bit SALU to VALU lowering.
1490 void legalizeOperandsVALUt16(MachineInstr &Inst,
1491 MachineRegisterInfo &MRI) const;
1492 void legalizeOperandsVALUt16(MachineInstr &Inst, unsigned OpIdx,
1493 MachineRegisterInfo &MRI) const;
1494
1495 /// Replace the instructions opcode with the equivalent VALU
1496 /// opcode. This function will also move the users of MachineInstruntions
1497 /// in the \p WorkList to the VALU if necessary. If present, \p MDT is
1498 /// updated.
1499 void moveToVALU(SIInstrWorklist &Worklist, MachineDominatorTree *MDT) const;
1500
1501 void moveToVALUImpl(SIInstrWorklist &Worklist, MachineDominatorTree *MDT,
1502 MachineInstr &Inst) const;
1503
1504 void insertNoop(MachineBasicBlock &MBB,
1505 MachineBasicBlock::iterator MI) const override;
1506
1507 void insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1508 unsigned Quantity) const override;
1509
1510 void insertReturn(MachineBasicBlock &MBB) const;
1511
1512 /// Build instructions that simulate the behavior of a `s_trap 2` instructions
1513 /// for hardware (namely, gfx11) that runs in PRIV=1 mode. There, s_trap is
1514 /// interpreted as a nop.
1515 MachineBasicBlock *insertSimulatedTrap(MachineRegisterInfo &MRI,
1516 MachineBasicBlock &MBB,
1517 MachineInstr &MI,
1518 const DebugLoc &DL) const;
1519
1520 /// Return the number of wait states that result from executing this
1521 /// instruction.
1522 static unsigned getNumWaitStates(const MachineInstr &MI);
1523
1524 /// Returns the operand named \p Op. If \p MI does not have an
1525 /// operand named \c Op, this function returns nullptr.
1526 LLVM_READONLY
1527 MachineOperand *getNamedOperand(MachineInstr &MI,
1528 AMDGPU::OpName OperandName) const;
1529
1530 LLVM_READONLY
1531 const MachineOperand *getNamedOperand(const MachineInstr &MI,
1532 AMDGPU::OpName OperandName) const {
1533 return getNamedOperand(MI&: const_cast<MachineInstr &>(MI), OperandName);
1534 }
1535
1536 /// Get required immediate operand
1537 int64_t getNamedImmOperand(const MachineInstr &MI,
1538 AMDGPU::OpName OperandName) const {
1539 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OperandName);
1540 return MI.getOperand(i: Idx).getImm();
1541 }
1542
1543 uint64_t getDefaultRsrcDataFormat() const;
1544 uint64_t getScratchRsrcWords23() const;
1545
1546 bool isLowLatencyInstruction(const MachineInstr &MI) const;
1547 bool isHighLatencyDef(int Opc) const override;
1548
1549 /// Return the descriptor of the target-specific machine instruction
1550 /// that corresponds to the specified pseudo or native opcode.
1551 const MCInstrDesc &getMCOpcodeFromPseudo(unsigned Opcode) const {
1552 return get(Opcode: pseudoToMCOpcode(Opcode));
1553 }
1554
1555 Register isStackAccess(const MachineInstr &MI, int &FrameIndex) const;
1556 Register isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex) const;
1557
1558 Register isLoadFromStackSlot(const MachineInstr &MI,
1559 int &FrameIndex) const override;
1560 Register isStoreToStackSlot(const MachineInstr &MI,
1561 int &FrameIndex) const override;
1562
1563 unsigned getInstBundleSize(const MachineInstr &MI) const;
1564 unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
1565
1566 bool mayAccessFlatAddressSpace(const MachineInstr &MI) const;
1567
1568 std::pair<unsigned, unsigned>
1569 decomposeMachineOperandsTargetFlags(unsigned TF) const override;
1570
1571 ArrayRef<std::pair<int, const char *>>
1572 getSerializableTargetIndices() const override;
1573
1574 ArrayRef<std::pair<unsigned, const char *>>
1575 getSerializableDirectMachineOperandTargetFlags() const override;
1576
1577 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
1578 getSerializableMachineMemOperandTargetFlags() const override;
1579
1580 ScheduleHazardRecognizer *
1581 CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
1582 const ScheduleDAG *DAG) const override;
1583
1584 ScheduleHazardRecognizer *
1585 CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const override;
1586
1587 ScheduleHazardRecognizer *
1588 CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
1589 const ScheduleDAGMI *DAG) const override;
1590
1591 unsigned getLiveRangeSplitOpcode(Register Reg,
1592 const MachineFunction &MF) const override;
1593
1594 bool isBasicBlockPrologue(const MachineInstr &MI,
1595 Register Reg = Register()) const override;
1596
1597 bool canAddToBBProlog(const MachineInstr &MI) const;
1598
1599 MachineInstr *createPHIDestinationCopy(MachineBasicBlock &MBB,
1600 MachineBasicBlock::iterator InsPt,
1601 const DebugLoc &DL, Register Src,
1602 Register Dst) const override;
1603
1604 MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB,
1605 MachineBasicBlock::iterator InsPt,
1606 const DebugLoc &DL, Register Src,
1607 unsigned SrcSubReg,
1608 Register Dst) const override;
1609
1610 bool isWave32() const;
1611
1612 /// Return a partially built integer add instruction without carry.
1613 /// Caller must add source operands.
1614 /// For pre-GFX9 it will generate unused carry destination operand.
1615 /// TODO: After GFX9 it should return a no-carry operation.
1616 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1617 MachineBasicBlock::iterator I,
1618 const DebugLoc &DL,
1619 Register DestReg) const;
1620
1621 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1622 MachineBasicBlock::iterator I,
1623 const DebugLoc &DL,
1624 Register DestReg,
1625 RegScavenger &RS) const;
1626
1627 static bool isKillTerminator(unsigned Opcode);
1628 const MCInstrDesc &getKillTerminatorFromPseudo(unsigned Opcode) const;
1629
1630 bool isLegalMUBUFImmOffset(unsigned Imm) const;
1631
1632 static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST);
1633
1634 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
1635 Align Alignment = Align(4)) const;
1636
1637 /// Returns if \p Offset is legal for the subtarget as the offset to a FLAT
1638 /// encoded instruction with the given \p FlatVariant.
1639 bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
1640 uint64_t FlatVariant) const;
1641
1642 /// Split \p COffsetVal into {immediate offset field, remainder offset}
1643 /// values.
1644 std::pair<int64_t, int64_t> splitFlatOffset(int64_t COffsetVal,
1645 unsigned AddrSpace,
1646 uint64_t FlatVariant) const;
1647
1648 /// Returns true if negative offsets are allowed for the given \p FlatVariant.
1649 bool allowNegativeFlatOffset(uint64_t FlatVariant) const;
1650
1651 /// \brief Return a target-specific opcode if Opcode is a pseudo instruction.
1652 /// Return -1 if the target-specific opcode for the pseudo instruction does
1653 /// not exist. If Opcode is not a pseudo instruction, this is identity.
1654 int pseudoToMCOpcode(int Opcode) const;
1655
1656 /// \brief Check if this instruction should only be used by assembler.
1657 /// Return true if this opcode should not be used by codegen.
1658 bool isAsmOnlyOpcode(int MCOp) const;
1659
1660 void fixImplicitOperands(MachineInstr &MI) const;
1661
1662 MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
1663 ArrayRef<unsigned> Ops,
1664 MachineBasicBlock::iterator InsertPt,
1665 int FrameIndex,
1666 LiveIntervals *LIS = nullptr,
1667 VirtRegMap *VRM = nullptr) const override;
1668
1669 unsigned getInstrLatency(const InstrItineraryData *ItinData,
1670 const MachineInstr &MI,
1671 unsigned *PredCost = nullptr) const override;
1672
1673 const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override;
1674
1675 InstructionUniformity
1676 getInstructionUniformity(const MachineInstr &MI) const final;
1677
1678 InstructionUniformity
1679 getGenericInstructionUniformity(const MachineInstr &MI) const;
1680
1681 const MIRFormatter *getMIRFormatter() const override;
1682
1683 static unsigned getDSShaderTypeValue(const MachineFunction &MF);
1684
1685 const TargetSchedModel &getSchedModel() const { return SchedModel; }
1686
1687 // FIXME: This should be removed
1688 // Enforce operand's \p OpName even alignment if required by target.
1689 // This is used if an operand is a 32 bit register but needs to be aligned
1690 // regardless.
1691 void enforceOperandRCAlignment(MachineInstr &MI, AMDGPU::OpName OpName) const;
1692};
1693
1694/// \brief Returns true if a reg:subreg pair P has a TRC class
1695inline bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P,
1696 const TargetRegisterClass &TRC,
1697 MachineRegisterInfo &MRI) {
1698 auto *RC = MRI.getRegClass(Reg: P.Reg);
1699 if (!P.SubReg)
1700 return RC == &TRC;
1701 auto *TRI = MRI.getTargetRegisterInfo();
1702 return RC == TRI->getMatchingSuperRegClass(A: RC, B: &TRC, Idx: P.SubReg);
1703}
1704
1705/// \brief Create RegSubRegPair from a register MachineOperand
1706inline
1707TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O) {
1708 assert(O.isReg());
1709 return TargetInstrInfo::RegSubRegPair(O.getReg(), O.getSubReg());
1710}
1711
1712/// \brief Return the SubReg component from REG_SEQUENCE
1713TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI,
1714 unsigned SubReg);
1715
1716/// \brief Return the defining instruction for a given reg:subreg pair
1717/// skipping copy like instructions and subreg-manipulation pseudos.
1718/// Following another subreg of a reg:subreg isn't supported.
1719MachineInstr *getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
1720 const MachineRegisterInfo &MRI);
1721
1722/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1723/// DefMI and the use at \p UseMI. Should be run on SSA. Currently does not
1724/// attempt to track between blocks.
1725bool execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
1726 Register VReg,
1727 const MachineInstr &DefMI,
1728 const MachineInstr &UseMI);
1729
1730/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1731/// DefMI and all its uses. Should be run on SSA. Currently does not attempt to
1732/// track between blocks.
1733bool execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
1734 Register VReg,
1735 const MachineInstr &DefMI);
1736
1737namespace AMDGPU {
1738
1739 LLVM_READONLY
1740 int32_t getVOPe64(uint32_t Opcode);
1741
1742 LLVM_READONLY
1743 int32_t getVOPe32(uint32_t Opcode);
1744
1745 LLVM_READONLY
1746 int32_t getSDWAOp(uint32_t Opcode);
1747
1748 LLVM_READONLY
1749 int32_t getDPPOp32(uint32_t Opcode);
1750
1751 LLVM_READONLY
1752 int32_t getDPPOp64(uint32_t Opcode);
1753
1754 LLVM_READONLY
1755 int32_t getBasicFromSDWAOp(uint32_t Opcode);
1756
1757 LLVM_READONLY
1758 int32_t getCommuteRev(uint32_t Opcode);
1759
1760 LLVM_READONLY
1761 int32_t getCommuteOrig(uint32_t Opcode);
1762
1763 LLVM_READONLY
1764 int32_t getAddr64Inst(uint32_t Opcode);
1765
1766 /// Check if \p Opcode is an Addr64 opcode.
1767 ///
1768 /// \returns \p Opcode if it is an Addr64 opcode, otherwise -1.
1769 LLVM_READONLY
1770 int32_t getIfAddr64Inst(uint32_t Opcode);
1771
1772 LLVM_READONLY
1773 int32_t getSOPKOp(uint32_t Opcode);
1774
1775 /// \returns SADDR form of a FLAT Global instruction given an \p Opcode
1776 /// of a VADDR form.
1777 LLVM_READONLY
1778 int32_t getGlobalSaddrOp(uint32_t Opcode);
1779
1780 /// \returns VADDR form of a FLAT Global instruction given an \p Opcode
1781 /// of a SADDR form.
1782 LLVM_READONLY
1783 int32_t getGlobalVaddrOp(uint32_t Opcode);
1784
1785 LLVM_READONLY
1786 int32_t getVCMPXNoSDstOp(uint32_t Opcode);
1787
1788 /// \returns ST form with only immediate offset of a FLAT Scratch instruction
1789 /// given an \p Opcode of an SS (SADDR) form.
1790 LLVM_READONLY
1791 int32_t getFlatScratchInstSTfromSS(uint32_t Opcode);
1792
1793 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
1794 /// of an SVS (SADDR + VADDR) form.
1795 LLVM_READONLY
1796 int32_t getFlatScratchInstSVfromSVS(uint32_t Opcode);
1797
1798 /// \returns SS (SADDR) form of a FLAT Scratch instruction given an \p Opcode
1799 /// of an SV (VADDR) form.
1800 LLVM_READONLY
1801 int32_t getFlatScratchInstSSfromSV(uint32_t Opcode);
1802
1803 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
1804 /// of an SS (SADDR) form.
1805 LLVM_READONLY
1806 int32_t getFlatScratchInstSVfromSS(uint32_t Opcode);
1807
1808 /// \returns earlyclobber version of a MAC MFMA is exists.
1809 LLVM_READONLY
1810 int32_t getMFMAEarlyClobberOp(uint32_t Opcode);
1811
1812 /// \returns Version of an MFMA instruction which uses AGPRs for srcC and
1813 /// vdst, given an \p Opcode of an MFMA which uses VGPRs for srcC/vdst.
1814 LLVM_READONLY
1815 int32_t getMFMASrcCVDstAGPROp(uint32_t Opcode);
1816
1817 /// \returns v_cmpx version of a v_cmp instruction.
1818 LLVM_READONLY
1819 int32_t getVCMPXOpFromVCMP(uint32_t Opcode);
1820
1821 const uint64_t RSRC_DATA_FORMAT = 0xf00000000000LL;
1822 const uint64_t RSRC_ELEMENT_SIZE_SHIFT = (32 + 19);
1823 const uint64_t RSRC_INDEX_STRIDE_SHIFT = (32 + 21);
1824 const uint64_t RSRC_TID_ENABLE = UINT64_C(1) << (32 + 23);
1825
1826} // end namespace AMDGPU
1827
1828namespace AMDGPU {
1829enum AsmComments {
1830 // For sgpr to vgpr spill instructions
1831 SGPR_SPILL = MachineInstr::TAsmComments
1832};
1833} // namespace AMDGPU
1834
1835namespace SI {
1836namespace KernelInputOffsets {
1837
1838/// Offsets in bytes from the start of the input buffer
1839enum Offsets {
1840 NGROUPS_X = 0,
1841 NGROUPS_Y = 4,
1842 NGROUPS_Z = 8,
1843 GLOBAL_SIZE_X = 12,
1844 GLOBAL_SIZE_Y = 16,
1845 GLOBAL_SIZE_Z = 20,
1846 LOCAL_SIZE_X = 24,
1847 LOCAL_SIZE_Y = 28,
1848 LOCAL_SIZE_Z = 32
1849};
1850
1851} // end namespace KernelInputOffsets
1852} // end namespace SI
1853
1854} // end namespace llvm
1855
1856#endif // LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
1857