1//===- SIInstrInfo.h - SI Instruction Info Interface ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Interface definition for SIInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
15#define LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
16
17#include "AMDGPUMIRFormatter.h"
18#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
19#include "SIRegisterInfo.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/CodeGen/TargetInstrInfo.h"
24#include "llvm/CodeGen/TargetSchedule.h"
25
26#define GET_INSTRINFO_HEADER
27#include "AMDGPUGenInstrInfo.inc"
28
29namespace llvm {
30
31class APInt;
32class GCNSubtarget;
33class LiveVariables;
34class MachineDominatorTree;
35class MachineRegisterInfo;
36class RegScavenger;
37class SIMachineFunctionInfo;
38class MCRegisterClass;
39using TargetRegisterClass = MCRegisterClass;
40class ScheduleHazardRecognizer;
41
42constexpr unsigned DefaultMemoryClusterDWordsLimit = 8;
43
44/// Mark the MMO of a uniform load if there are no potentially clobbering stores
45/// on any path from the start of an entry function to this load.
46static const MachineMemOperand::Flags MONoClobber =
47 MachineMemOperand::MOTargetFlag1;
48
49/// Mark the MMO of a load as the last use.
50static const MachineMemOperand::Flags MOLastUse =
51 MachineMemOperand::MOTargetFlag2;
52
53/// Mark the MMO of cooperative load/store atomics.
54static const MachineMemOperand::Flags MOCooperative =
55 MachineMemOperand::MOTargetFlag3;
56
57struct V2PhysSCopyInfo {
58 // Operands that need to replaced by waterfall
59 SmallVector<MachineOperand *> MOs;
60 // Target physical registers replacing the MOs
61 SmallVector<Register> SGPRs;
62};
63/// Mark the MMO of accesses to memory locations that are
64/// never written to by other threads.
65static const MachineMemOperand::Flags MOThreadPrivate =
66 MachineMemOperand::MOTargetFlag4;
67
68/// Utility to store machine instructions worklist.
69struct SIInstrWorklist {
70 SIInstrWorklist() = default;
71
72 void insert(MachineInstr *MI);
73
74 MachineInstr *top() const { return InstrList[Front]; }
75
76 void erase_top() {
77 InSet.erase(Ptr: InstrList[Front]);
78 ++Front;
79 }
80
81 bool empty() const { return Front == InstrList.size(); }
82
83 void clear() {
84 InstrList.clear();
85 Front = 0;
86 InSet.clear();
87 DeferredList.clear();
88 }
89
90 bool isDeferred(MachineInstr *MI);
91
92 SetVector<MachineInstr *> &getDeferredList() { return DeferredList; }
93
94private:
95 /// InstrList contains the MachineInstrs.
96 SmallVector<MachineInstr *> InstrList;
97 SmallPtrSet<MachineInstr *, 8> InSet;
98 unsigned Front = 0;
99 /// Deferred instructions are specific MachineInstr
100 /// that will be added by insert method.
101 SetVector<MachineInstr *> DeferredList;
102};
103
104// In namespace llvm so ADL finds it when SIInstrFlags predicates are
105// instantiated with MachineInstr (MachineInstr is in namespace llvm).
106inline uint64_t getTSFlags(const MachineInstr &MI) {
107 return MI.getDesc().TSFlags;
108}
109
110class SIInstrInfo final : public AMDGPUGenInstrInfo {
111 struct ThreeAddressUpdates;
112
113private:
114 const SIRegisterInfo RI;
115 const GCNSubtarget &ST;
116 TargetSchedModel SchedModel;
117 mutable std::unique_ptr<AMDGPUMIRFormatter> Formatter;
118
119 // The inverse predicate should have the negative value.
120 enum BranchPredicate {
121 INVALID_BR = 0,
122 SCC_TRUE = 1,
123 SCC_FALSE = -1,
124 VCCNZ = 2,
125 VCCZ = -2,
126 EXECNZ = -3,
127 EXECZ = 3
128 };
129
130 using SetVectorType = SmallSetVector<MachineInstr *, 32>;
131
132 static unsigned getBranchOpcode(BranchPredicate Cond);
133 static BranchPredicate getBranchPredicate(unsigned Opcode);
134
135public:
136 unsigned buildExtractSubReg(MachineBasicBlock::iterator MI,
137 MachineRegisterInfo &MRI,
138 const MachineOperand &SuperReg,
139 const TargetRegisterClass *SuperRC,
140 unsigned SubIdx,
141 const TargetRegisterClass *SubRC) const;
142 MachineOperand buildExtractSubRegOrImm(
143 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
144 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
145 unsigned SubIdx, const TargetRegisterClass *SubRC) const;
146
147private:
148 bool optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,
149 bool NeedInversion) const;
150
151 bool invertSCCUse(MachineInstr *SCCDef) const;
152
153 void swapOperands(MachineInstr &Inst) const;
154
155 std::pair<bool, MachineBasicBlock *>
156 moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
157 MachineDominatorTree *MDT = nullptr) const;
158
159 void lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
160 MachineDominatorTree *MDT = nullptr) const;
161
162 void lowerScalarAbs(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
163
164 void lowerScalarAbsDiff(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
165
166 void lowerScalarXnor(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
167
168 void splitScalarNotBinop(SIInstrWorklist &Worklist, MachineInstr &Inst,
169 unsigned Opcode) const;
170
171 void splitScalarBinOpN2(SIInstrWorklist &Worklist, MachineInstr &Inst,
172 unsigned Opcode) const;
173
174 void splitScalar64BitUnaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
175 unsigned Opcode, bool Swap = false) const;
176
177 void splitScalar64BitBinaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
178 unsigned Opcode,
179 MachineDominatorTree *MDT = nullptr) const;
180
181 void splitScalarSMulU64(SIInstrWorklist &Worklist, MachineInstr &Inst,
182 MachineDominatorTree *MDT) const;
183
184 void splitScalarSMulPseudo(SIInstrWorklist &Worklist, MachineInstr &Inst,
185 MachineDominatorTree *MDT) const;
186
187 void splitScalar64BitXnor(SIInstrWorklist &Worklist, MachineInstr &Inst,
188 MachineDominatorTree *MDT = nullptr) const;
189
190 void splitScalar64BitBCNT(SIInstrWorklist &Worklist,
191 MachineInstr &Inst) const;
192 void splitScalar64BitBFE(SIInstrWorklist &Worklist, MachineInstr &Inst) const;
193 void splitScalar64BitCountOp(SIInstrWorklist &Worklist, MachineInstr &Inst,
194 unsigned Opcode,
195 MachineDominatorTree *MDT = nullptr) const;
196 void movePackToVALU(SIInstrWorklist &Worklist, MachineRegisterInfo &MRI,
197 MachineInstr &Inst) const;
198
199 void addUsersToMoveToVALUWorklist(Register Reg, MachineRegisterInfo &MRI,
200 SIInstrWorklist &Worklist) const;
201
202 void addSCCDefUsersToVALUWorklist(const MachineOperand &Op,
203 MachineInstr &SCCDefInst,
204 SIInstrWorklist &Worklist,
205 Register NewCond = Register()) const;
206 void addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
207 SIInstrWorklist &Worklist) const;
208
209 const TargetRegisterClass *
210 getDestEquivalentVGPRClass(const MachineInstr &Inst) const;
211
212 bool checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
213 const MachineInstr &MIb) const;
214
215 Register findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const;
216
217 bool verifyCopy(const MachineInstr &MI, const MachineRegisterInfo &MRI,
218 StringRef &ErrInfo) const;
219
220 bool resultDependsOnExec(const MachineInstr &MI) const;
221
222 MachineInstr *convertToThreeAddressImpl(MachineInstr &MI,
223 ThreeAddressUpdates &Updates) const;
224
225protected:
226 /// If the specific machine instruction is a instruction that moves/copies
227 /// value from one register to another register return destination and source
228 /// registers as machine operands.
229 std::optional<DestSourcePair>
230 isCopyInstrImpl(const MachineInstr &MI) const override;
231
232 bool swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,
233 AMDGPU::OpName Src0OpName, MachineOperand &Src1,
234 AMDGPU::OpName Src1OpName) const;
235 bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx,
236 unsigned toIdx) const;
237 bool isNonCommutableDPP(const MachineInstr &MI) const;
238 MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
239 unsigned OpIdx0,
240 unsigned OpIdx1) const override;
241
242public:
243 enum TargetOperandFlags {
244 MO_MASK = 0xf,
245
246 MO_NONE = 0,
247 // MO_GOTPCREL -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
248 MO_GOTPCREL = 1,
249 // MO_GOTPCREL32_LO -> symbol@gotpcrel32@lo -> R_AMDGPU_GOTPCREL32_LO.
250 MO_GOTPCREL32 = 2,
251 MO_GOTPCREL32_LO = 2,
252 // MO_GOTPCREL32_HI -> symbol@gotpcrel32@hi -> R_AMDGPU_GOTPCREL32_HI.
253 MO_GOTPCREL32_HI = 3,
254 // MO_GOTPCREL64 -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL.
255 MO_GOTPCREL64 = 4,
256 // MO_REL32_LO -> symbol@rel32@lo -> R_AMDGPU_REL32_LO.
257 MO_REL32 = 5,
258 MO_REL32_LO = 5,
259 // MO_REL32_HI -> symbol@rel32@hi -> R_AMDGPU_REL32_HI.
260 MO_REL32_HI = 6,
261 MO_REL64 = 7,
262
263 MO_FAR_BRANCH_OFFSET = 8,
264
265 MO_ABS32_LO = 9,
266 MO_ABS32_HI = 10,
267 MO_ABS64 = 11,
268 };
269
270 explicit SIInstrInfo(const GCNSubtarget &ST);
271
272 const SIRegisterInfo &getRegisterInfo() const {
273 return RI;
274 }
275
276 // FIXME: This is inaccurate and needs to account for use context. Normal asm
277 // constraints should use 64-bit pointers.
278 const TargetRegisterClass *getInlineAsmMemoryOperandRegClass(
279 InlineAsm::ConstraintCode C) const override {
280 return &AMDGPU::VGPR_32RegClass;
281 }
282
283 const GCNSubtarget &getSubtarget() const {
284 return ST;
285 }
286
287 bool isReMaterializableImpl(const MachineInstr &MI) const override;
288
289 bool isIgnorableUse(const MachineInstr &MI, unsigned OpIdx) const override;
290
291 bool isSafeToSink(MachineInstr &MI, MachineBasicBlock *SuccToSinkTo,
292 MachineCycleInfo *CI) const override;
293
294 bool areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, int64_t &Offset0,
295 int64_t &Offset1) const override;
296
297 bool isGlobalMemoryObject(const MachineInstr *MI) const override;
298
299 bool getMemOperandsWithOffsetWidth(
300 const MachineInstr &LdSt,
301 SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
302 bool &OffsetIsScalable, LocationSize &Width,
303 const TargetRegisterInfo *TRI) const final;
304
305 bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
306 int64_t Offset1, bool OffsetIsScalable1,
307 ArrayRef<const MachineOperand *> BaseOps2,
308 int64_t Offset2, bool OffsetIsScalable2,
309 unsigned ClusterSize,
310 unsigned NumBytes) const override;
311
312 bool shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0,
313 int64_t Offset1, unsigned NumLoads) const override;
314
315 void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
316 const DebugLoc &DL, Register DestReg, Register SrcReg,
317 bool KillSrc, bool RenamableDest = false,
318 bool RenamableSrc = false) const override;
319
320private:
321 void storeRegToStackSlotImpl(MachineBasicBlock &MBB,
322 MachineBasicBlock::iterator MI, Register SrcReg,
323 bool isKill, int FrameIndex,
324 const TargetRegisterClass *RC, Register VReg,
325 MachineInstr::MIFlag Flags, bool NeedsCFI) const;
326
327public:
328 void storeRegToStackSlotCFI(MachineBasicBlock &MBB,
329 MachineBasicBlock::iterator MI, Register SrcReg,
330 bool isKill, int FrameIndex,
331 const TargetRegisterClass *RC) const;
332
333 bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg,
334 int64_t &ImmVal) const override;
335
336 std::optional<int64_t>
337 getImmOrMaterializedImm(const MachineRegisterInfo &MRI,
338 const MachineOperand &Op,
339 MachineInstr **DefMI = nullptr) const;
340 std::optional<int64_t>
341 getImmOrMaterializedImm(const MachineRegisterInfo &MRI, Register Reg,
342 MachineInstr **DefMI = nullptr) const;
343
344 unsigned getVectorRegSpillSaveOpcode(Register Reg,
345 const TargetRegisterClass *RC,
346 unsigned Size,
347 const SIMachineFunctionInfo &MFI,
348 bool NeedsCFI) const;
349 unsigned
350 getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC,
351 unsigned Size,
352 const SIMachineFunctionInfo &MFI) const;
353
354 void storeRegToStackSlot(
355 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
356 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
357 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
358
359 void loadRegFromStackSlot(
360 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
361 int FrameIndex, const TargetRegisterClass *RC, Register VReg,
362 unsigned SubReg = 0,
363 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
364
365 bool expandPostRAPseudo(MachineInstr &MI) const override;
366
367 void
368 reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
369 Register DestReg, unsigned SubIdx, const MachineInstr &Orig,
370 LaneBitmask UsedLanes = LaneBitmask::getAll()) const override;
371
372 // Splits a V_MOV_B64_DPP_PSEUDO opcode into a pair of v_mov_b32_dpp
373 // instructions. Returns a pair of generated instructions.
374 // Can split either post-RA with physical registers or pre-RA with
375 // virtual registers. In latter case IR needs to be in SSA form and
376 // and a REG_SEQUENCE is produced to define original register.
377 std::pair<MachineInstr*, MachineInstr*>
378 expandMovDPP64(MachineInstr &MI) const;
379
380 // Returns an opcode that can be used to move a value to a \p DstRC
381 // register. If there is no hardware instruction that can store to \p
382 // DstRC, then AMDGPU::COPY is returned.
383 unsigned getMovOpcode(const TargetRegisterClass *DstRC) const;
384
385 const MCInstrDesc &getIndirectRegWriteMovRelPseudo(unsigned VecSize,
386 unsigned EltSize,
387 bool IsSGPR) const;
388
389 const MCInstrDesc &getIndirectGPRIDXPseudo(unsigned VecSize,
390 bool IsIndirectSrc) const;
391 LLVM_READONLY
392 int commuteOpcode(unsigned Opc) const;
393
394 LLVM_READONLY
395 inline int commuteOpcode(const MachineInstr &MI) const {
396 return commuteOpcode(Opc: MI.getOpcode());
397 }
398
399 bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0,
400 unsigned &SrcOpIdx1) const override;
401
402 bool findCommutedOpIndices(const MCInstrDesc &Desc, unsigned &SrcOpIdx0,
403 unsigned &SrcOpIdx1) const;
404
405 bool isBranchOffsetInRange(unsigned BranchOpc,
406 int64_t BrOffset) const override;
407
408 MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;
409
410 /// Return whether the block terminate with divergent branch.
411 /// Note this only work before lowering the pseudo control flow instructions.
412 bool hasDivergentBranch(const MachineBasicBlock *MBB) const;
413
414 void insertIndirectBranch(MachineBasicBlock &MBB,
415 MachineBasicBlock &NewDestBB,
416 MachineBasicBlock &RestoreBB, const DebugLoc &DL,
417 int64_t BrOffset, RegScavenger *RS) const override;
418
419 bool analyzeBranchImpl(MachineBasicBlock &MBB,
420 MachineBasicBlock::iterator I,
421 MachineBasicBlock *&TBB,
422 MachineBasicBlock *&FBB,
423 SmallVectorImpl<MachineOperand> &Cond,
424 bool AllowModify) const;
425
426 bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
427 MachineBasicBlock *&FBB,
428 SmallVectorImpl<MachineOperand> &Cond,
429 bool AllowModify = false) const override;
430
431 unsigned removeBranch(MachineBasicBlock &MBB,
432 int *BytesRemoved = nullptr) const override;
433
434 unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
435 MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
436 const DebugLoc &DL,
437 int *BytesAdded = nullptr) const override;
438
439 bool reverseBranchCondition(
440 SmallVectorImpl<MachineOperand> &Cond) const override;
441
442 std::unique_ptr<PipelinerLoopInfo>
443 analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
444
445 bool canInsertSelect(const MachineBasicBlock &MBB,
446 ArrayRef<MachineOperand> Cond, Register DstReg,
447 Register TrueReg, Register FalseReg, int &CondCycles,
448 int &TrueCycles, int &FalseCycles) const override;
449
450 void insertSelect(MachineBasicBlock &MBB,
451 MachineBasicBlock::iterator I, const DebugLoc &DL,
452 Register DstReg, ArrayRef<MachineOperand> Cond,
453 Register TrueReg, Register FalseReg) const override;
454
455 bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
456 Register &SrcReg2, int64_t &CmpMask,
457 int64_t &CmpValue) const override;
458
459 bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
460 Register SrcReg2, int64_t CmpMask, int64_t CmpValue,
461 const MachineRegisterInfo *MRI) const override;
462
463 bool
464 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
465 const MachineInstr &MIb) const override;
466
467 static bool isFoldableCopy(const MachineInstr &MI);
468 static unsigned getFoldableCopySrcIdx(const MachineInstr &MI);
469
470 void removeModOperands(MachineInstr &MI) const;
471
472 void mutateAndCleanupImplicit(MachineInstr &MI,
473 const MCInstrDesc &NewDesc) const;
474
475 /// Return the extracted immediate value in a subregister use from a constant
476 /// materialized in a super register.
477 ///
478 /// e.g. %imm = S_MOV_B64 K[0:63]
479 /// USE %imm.sub1
480 /// This will return K[32:63]
481 static std::optional<int64_t> extractSubregFromImm(int64_t ImmVal,
482 unsigned SubRegIndex);
483
484 bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
485 MachineRegisterInfo *MRI) const final;
486
487 unsigned getMachineCSELookAheadLimit() const override { return 500; }
488
489 MachineInstr *convertToThreeAddress(MachineInstr &MI, LiveVariables *LV,
490 LiveIntervals *LIS) const override;
491
492 bool isSchedulingBoundary(const MachineInstr &MI,
493 const MachineBasicBlock *MBB,
494 const MachineFunction &MF) const override;
495
496 static bool isSALU(const MachineInstr &MI) {
497 return SIInstrFlags::isSALU(O: MI);
498 }
499
500 bool isSALU(uint32_t Opcode) const {
501 return SIInstrFlags::isSALU(O: get(Opcode));
502 }
503
504 static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA) {
505 if (!AllowLDSDMA && isLDSDMA(MI))
506 return false;
507
508 return SIInstrFlags::isVALU(O: MI);
509 }
510
511 /// LDSDMA instructions act as both VALU and memory instructions, thus
512 /// we also tag them as VALU. However, in many places, we do not actually want
513 /// to include LDSDMA instructions in this query. By setting \p AllowLDSDMA to
514 /// false, this will return false for LDSDMA instructions.
515 bool isVALU(uint32_t Opcode, bool AllowLDSDMA) const {
516 if (!AllowLDSDMA && isLDSDMA(Opcode))
517 return false;
518
519 return SIInstrFlags::isVALU(O: get(Opcode));
520 }
521
522 static bool isImage(const MachineInstr &MI) {
523 return SIInstrFlags::isImage(O: MI);
524 }
525
526 bool isImage(uint32_t Opcode) const {
527 return SIInstrFlags::isImage(O: get(Opcode));
528 }
529
530 static bool isVMEM(const MachineInstr &MI) {
531 return SIInstrFlags::isVMEM(O: MI);
532 }
533
534 bool isVMEM(uint32_t Opcode) const {
535 return SIInstrFlags::isVMEM(O: get(Opcode));
536 }
537
538 /// True if MI implicitly drains XCNT.
539 static bool isXcntDrain(const MachineInstr &MI);
540
541 static bool isSOP1(const MachineInstr &MI) {
542 return SIInstrFlags::isSOP1(O: MI);
543 }
544
545 bool isSOP1(uint32_t Opcode) const {
546 return SIInstrFlags::isSOP1(O: get(Opcode));
547 }
548
549 static bool isSOP2(const MachineInstr &MI) {
550 return SIInstrFlags::isSOP2(O: MI);
551 }
552
553 bool isSOP2(uint32_t Opcode) const {
554 return SIInstrFlags::isSOP2(O: get(Opcode));
555 }
556
557 static bool isSOPC(const MachineInstr &MI) {
558 return SIInstrFlags::isSOPC(O: MI);
559 }
560
561 bool isSOPC(uint32_t Opcode) const {
562 return SIInstrFlags::isSOPC(O: get(Opcode));
563 }
564
565 static bool isSOPK(const MachineInstr &MI) {
566 return SIInstrFlags::isSOPK(O: MI);
567 }
568
569 bool isSOPK(uint32_t Opcode) const {
570 return SIInstrFlags::isSOPK(O: get(Opcode));
571 }
572
573 static bool isSOPP(const MachineInstr &MI) {
574 return SIInstrFlags::isSOPP(O: MI);
575 }
576
577 bool isSOPP(uint32_t Opcode) const {
578 return SIInstrFlags::isSOPP(O: get(Opcode));
579 }
580
581 static bool isPacked(const MachineInstr &MI) {
582 return SIInstrFlags::isPacked(O: MI);
583 }
584
585 bool isPacked(uint32_t Opcode) const {
586 return SIInstrFlags::isPacked(O: get(Opcode));
587 }
588
589 static bool isVOP1(const MachineInstr &MI) {
590 return SIInstrFlags::isVOP1(O: MI);
591 }
592
593 bool isVOP1(uint32_t Opcode) const {
594 return SIInstrFlags::isVOP1(O: get(Opcode));
595 }
596
597 static bool isVOP2(const MachineInstr &MI) {
598 return SIInstrFlags::isVOP2(O: MI);
599 }
600
601 bool isVOP2(uint32_t Opcode) const {
602 return SIInstrFlags::isVOP2(O: get(Opcode));
603 }
604
605 static bool isVOP3(const MCInstrDesc &Desc) {
606 return SIInstrFlags::isVOP3(O: Desc);
607 }
608
609 static bool isVOP3(const MachineInstr &MI) { return isVOP3(Desc: MI.getDesc()); }
610
611 bool isVOP3(uint32_t Opcode) const { return isVOP3(Desc: get(Opcode)); }
612
613 static bool isSDWA(const MachineInstr &MI) {
614 return SIInstrFlags::isSDWA(O: MI);
615 }
616
617 bool isSDWA(uint32_t Opcode) const {
618 return SIInstrFlags::isSDWA(O: get(Opcode));
619 }
620
621 static bool isVOPC(const MachineInstr &MI) {
622 return SIInstrFlags::isVOPC(O: MI);
623 }
624
625 bool isVOPC(uint32_t Opcode) const {
626 return SIInstrFlags::isVOPC(O: get(Opcode));
627 }
628
629 static bool isMUBUF(const MachineInstr &MI) {
630 return SIInstrFlags::isMUBUF(O: MI);
631 }
632
633 bool isMUBUF(uint32_t Opcode) const {
634 return SIInstrFlags::isMUBUF(O: get(Opcode));
635 }
636
637 static bool isMTBUF(const MachineInstr &MI) {
638 return SIInstrFlags::isMTBUF(O: MI);
639 }
640
641 bool isMTBUF(uint32_t Opcode) const {
642 return SIInstrFlags::isMTBUF(O: get(Opcode));
643 }
644
645 static bool isBUF(const MachineInstr &MI) {
646 return isMUBUF(MI) || isMTBUF(MI);
647 }
648
649 static bool isSMRD(const MachineInstr &MI) {
650 return SIInstrFlags::isSMRD(O: MI);
651 }
652
653 bool isSMRD(uint32_t Opcode) const {
654 return SIInstrFlags::isSMRD(O: get(Opcode));
655 }
656
657 bool isBufferSMRD(const MachineInstr &MI) const;
658
659 static bool isDS(const MachineInstr &MI) { return SIInstrFlags::isDS(O: MI); }
660
661 bool isDS(uint32_t Opcode) const { return SIInstrFlags::isDS(O: get(Opcode)); }
662
663 static bool isLDSDMA(const MachineInstr &MI) {
664 return (SIInstrFlags::isVALU(O: MI) && (isMUBUF(MI) || isFLAT(MI))) ||
665 SIInstrFlags::usesTENSOR_CNT(O: MI);
666 }
667
668 bool isLDSDMA(uint32_t Opcode) const {
669 return (SIInstrFlags::isVALU(O: get(Opcode)) &&
670 (isMUBUF(Opcode) || isFLAT(Opcode))) ||
671 SIInstrFlags::usesTENSOR_CNT(O: get(Opcode));
672 }
673
674 static bool isGWS(const MachineInstr &MI) { return SIInstrFlags::isGWS(O: MI); }
675
676 bool isGWS(uint32_t Opcode) const { return SIInstrFlags::isGWS(O: get(Opcode)); }
677
678 bool isAlwaysGDS(uint32_t Opcode) const;
679
680 static bool isMIMG(const MachineInstr &MI) {
681 return SIInstrFlags::isMIMG(O: MI);
682 }
683
684 bool isMIMG(uint32_t Opcode) const {
685 return SIInstrFlags::isMIMG(O: get(Opcode));
686 }
687
688 static bool isVIMAGE(const MachineInstr &MI) {
689 return SIInstrFlags::isVIMAGE(O: MI);
690 }
691
692 bool isVIMAGE(uint32_t Opcode) const {
693 return SIInstrFlags::isVIMAGE(O: get(Opcode));
694 }
695
696 static bool isVSAMPLE(const MachineInstr &MI) {
697 return SIInstrFlags::isVSAMPLE(O: MI);
698 }
699
700 bool isVSAMPLE(uint32_t Opcode) const {
701 return SIInstrFlags::isVSAMPLE(O: get(Opcode));
702 }
703
704 static bool isGather4(const MachineInstr &MI) {
705 return SIInstrFlags::isGather4(O: MI);
706 }
707
708 bool isGather4(uint32_t Opcode) const {
709 return SIInstrFlags::isGather4(O: get(Opcode));
710 }
711
712 static bool isFLAT(const MachineInstr &MI) {
713 return SIInstrFlags::isFLAT(O: MI);
714 }
715
716 // Is a FLAT encoded instruction which accesses a specific segment,
717 // i.e. global_* or scratch_*.
718 static bool isSegmentSpecificFLAT(const MachineInstr &MI) {
719 return SIInstrFlags::isSegmentSpecificFLAT(O: MI);
720 }
721
722 bool isSegmentSpecificFLAT(uint32_t Opcode) const {
723 return SIInstrFlags::isSegmentSpecificFLAT(O: get(Opcode));
724 }
725
726 static bool isFLATGlobal(const MachineInstr &MI) {
727 return SIInstrFlags::isFlatGlobal(O: MI);
728 }
729
730 bool isFLATGlobal(uint32_t Opcode) const {
731 return SIInstrFlags::isFlatGlobal(O: get(Opcode));
732 }
733
734 static bool isFLATScratch(const MachineInstr &MI) {
735 return SIInstrFlags::isFlatScratch(O: MI);
736 }
737
738 bool isFLATScratch(uint32_t Opcode) const {
739 return SIInstrFlags::isFlatScratch(O: get(Opcode));
740 }
741
742 // Any FLAT encoded instruction, including global_* and scratch_*.
743 bool isFLAT(uint32_t Opcode) const {
744 return SIInstrFlags::isFLAT(O: get(Opcode));
745 }
746
747 /// \returns true for SCRATCH_ instructions, or FLAT/BUF instructions unless
748 /// the MMOs do not include scratch.
749 /// Conservatively correct; will return true if \p MI cannot be proven
750 /// to not hit scratch.
751 bool mayAccessScratch(const MachineInstr &MI) const;
752
753 /// \returns true for FLAT instructions that can access VMEM.
754 bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const;
755
756 /// \returns true for FLAT instructions that can access LDS.
757 bool mayAccessLDSThroughFlat(const MachineInstr &MI, bool TgSplit) const;
758
759 static bool isBlockLoadStore(uint32_t Opcode) {
760 switch (Opcode) {
761 case AMDGPU::SI_BLOCK_SPILL_V1024_SAVE:
762 case AMDGPU::SI_BLOCK_SPILL_V1024_CFI_SAVE:
763 case AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE:
764 case AMDGPU::SCRATCH_STORE_BLOCK_SADDR:
765 case AMDGPU::SCRATCH_LOAD_BLOCK_SADDR:
766 case AMDGPU::SCRATCH_STORE_BLOCK_SVS:
767 case AMDGPU::SCRATCH_LOAD_BLOCK_SVS:
768 return true;
769 default:
770 return false;
771 }
772 }
773
774 static bool setsSCCIfResultIsNonZero(const MachineInstr &MI) {
775 switch (MI.getOpcode()) {
776 case AMDGPU::S_ABSDIFF_I32:
777 case AMDGPU::S_ABS_I32:
778 case AMDGPU::S_AND_B32:
779 case AMDGPU::S_AND_B64:
780 case AMDGPU::S_ANDN2_B32:
781 case AMDGPU::S_ANDN2_B64:
782 case AMDGPU::S_ASHR_I32:
783 case AMDGPU::S_ASHR_I64:
784 case AMDGPU::S_BCNT0_I32_B32:
785 case AMDGPU::S_BCNT0_I32_B64:
786 case AMDGPU::S_BCNT1_I32_B32:
787 case AMDGPU::S_BCNT1_I32_B64:
788 case AMDGPU::S_BFE_I32:
789 case AMDGPU::S_BFE_I64:
790 case AMDGPU::S_BFE_U32:
791 case AMDGPU::S_BFE_U64:
792 case AMDGPU::S_LSHL_B32:
793 case AMDGPU::S_LSHL_B64:
794 case AMDGPU::S_LSHR_B32:
795 case AMDGPU::S_LSHR_B64:
796 case AMDGPU::S_NAND_B32:
797 case AMDGPU::S_NAND_B64:
798 case AMDGPU::S_NOR_B32:
799 case AMDGPU::S_NOR_B64:
800 case AMDGPU::S_NOT_B32:
801 case AMDGPU::S_NOT_B64:
802 case AMDGPU::S_OR_B32:
803 case AMDGPU::S_OR_B64:
804 case AMDGPU::S_ORN2_B32:
805 case AMDGPU::S_ORN2_B64:
806 case AMDGPU::S_QUADMASK_B32:
807 case AMDGPU::S_QUADMASK_B64:
808 case AMDGPU::S_WQM_B32:
809 case AMDGPU::S_WQM_B64:
810 case AMDGPU::S_XNOR_B32:
811 case AMDGPU::S_XNOR_B64:
812 case AMDGPU::S_XOR_B32:
813 case AMDGPU::S_XOR_B64:
814 return true;
815 default:
816 return false;
817 }
818 }
819
820 static bool isEXP(const MachineInstr &MI) { return SIInstrFlags::isEXP(O: MI); }
821
822 static bool isDualSourceBlendEXP(const MachineInstr &MI) {
823 if (!isEXP(MI))
824 return false;
825 unsigned Target = MI.getOperand(i: 0).getImm();
826 return Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND0 ||
827 Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND1;
828 }
829
830 bool isEXP(uint32_t Opcode) const { return SIInstrFlags::isEXP(O: get(Opcode)); }
831
832 static bool isAtomicNoRet(const MachineInstr &MI) {
833 return SIInstrFlags::isAtomicNoRet(O: MI);
834 }
835
836 bool isAtomicNoRet(uint32_t Opcode) const {
837 return SIInstrFlags::isAtomicNoRet(O: get(Opcode));
838 }
839
840 static bool isAtomicRet(const MachineInstr &MI) {
841 return SIInstrFlags::isAtomicRet(O: MI);
842 }
843
844 bool isAtomicRet(uint32_t Opcode) const {
845 return SIInstrFlags::isAtomicRet(O: get(Opcode));
846 }
847
848 static bool isAtomic(const MachineInstr &MI) {
849 return SIInstrFlags::isAtomic(O: MI);
850 }
851
852 bool isAtomic(uint32_t Opcode) const {
853 return SIInstrFlags::isAtomic(O: get(Opcode));
854 }
855
856 static bool mayWriteLDSThroughDMA(const MachineInstr &MI) {
857 unsigned Opc = MI.getOpcode();
858 // Exclude instructions that read FROM LDS (not write to it)
859 return isLDSDMA(MI) && Opc != AMDGPU::BUFFER_STORE_LDS_DWORD &&
860 Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d2 &&
861 Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d4;
862 }
863
864 static bool isSBarrierSCCWrite(unsigned Opcode) {
865 return Opcode == AMDGPU::S_BARRIER_LEAVE ||
866 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM ||
867 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0;
868 }
869
870 static bool isCBranchVCCZRead(const MachineInstr &MI) {
871 unsigned Opc = MI.getOpcode();
872 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
873 !MI.getOperand(i: 1).isUndef();
874 }
875
876 static bool isWQM(const MachineInstr &MI) { return SIInstrFlags::isWQM(O: MI); }
877
878 bool isWQM(uint32_t Opcode) const { return SIInstrFlags::isWQM(O: get(Opcode)); }
879
880 static bool isDisableWQM(const MachineInstr &MI) {
881 return SIInstrFlags::isDisableWQM(O: MI);
882 }
883
884 bool isDisableWQM(uint32_t Opcode) const {
885 return SIInstrFlags::isDisableWQM(O: get(Opcode));
886 }
887
888 // SI_SPILL_S32_TO_VGPR and SI_RESTORE_S32_FROM_VGPR form a special case of
889 // SGPRs spilling to VGPRs which are SGPR spills but from VALU instructions
890 // therefore we need an explicit check for them since just checking if the
891 // Spill bit is set and what instruction type it came from misclassifies
892 // them.
893 static bool isVGPRSpill(const MachineInstr &MI) {
894 return MI.getOpcode() != AMDGPU::SI_SPILL_S32_TO_VGPR &&
895 MI.getOpcode() != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
896 (isSpill(MI) && isVALU(MI, /*AllowLDSDMA=*/AllowLDSDMA: false));
897 }
898
899 bool isVGPRSpill(uint32_t Opcode) const {
900 return Opcode != AMDGPU::SI_SPILL_S32_TO_VGPR &&
901 Opcode != AMDGPU::SI_RESTORE_S32_FROM_VGPR &&
902 (isSpill(Opcode) && isVALU(Opcode, /*AllowLDSDMA=*/AllowLDSDMA: false));
903 }
904
905 static bool isSGPRSpill(const MachineInstr &MI) {
906 return MI.getOpcode() == AMDGPU::SI_SPILL_S32_TO_VGPR ||
907 MI.getOpcode() == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
908 (isSpill(MI) && isSALU(MI));
909 }
910
911 bool isSGPRSpill(uint32_t Opcode) const {
912 return Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR ||
913 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
914 (isSpill(Opcode) && isSALU(Opcode));
915 }
916
917 bool isSpill(uint32_t Opcode) const {
918 return SIInstrFlags::isSpill(O: get(Opcode));
919 }
920
921 static bool isSpill(const MCInstrDesc &Desc) {
922 return SIInstrFlags::isSpill(O: Desc);
923 }
924
925 static bool isSpill(const MachineInstr &MI) { return isSpill(Desc: MI.getDesc()); }
926
927 static bool isWWMRegSpillOpcode(uint32_t Opcode) {
928 return Opcode == AMDGPU::SI_SPILL_WWM_V32_SAVE ||
929 Opcode == AMDGPU::SI_SPILL_WWM_AV32_SAVE ||
930 Opcode == AMDGPU::SI_SPILL_WWM_V32_RESTORE ||
931 Opcode == AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
932 }
933
934 static bool isChainCallOpcode(uint64_t Opcode) {
935 return Opcode == AMDGPU::SI_CS_CHAIN_TC_W32 ||
936 Opcode == AMDGPU::SI_CS_CHAIN_TC_W64;
937 }
938
939 static bool isDPP(const MachineInstr &MI) { return SIInstrFlags::isDPP(O: MI); }
940
941 bool isDPP(uint32_t Opcode) const { return SIInstrFlags::isDPP(O: get(Opcode)); }
942
943 // Some opcodes use Src1 for DPP instead of Src0, because the sequencer
944 // transforms them and reverse the order of their operands at runtime.
945 //
946 // Documentation is incomplete on which instructions are effected, so
947 // the implementation is derived from experimentation.
948 //
949 // Listed as target-independent pseudos; the per-subtarget MC opcodes
950 // (V_SUBREV_NC_U32_e32_gfx11 and friends) are all reached through these.
951 // Defined out of line because GCNSubtarget is incomplete here.
952 static bool isSrc1DPPRevOpcode(const GCNSubtarget &ST, uint32_t Opcode);
953
954 static bool isTRANS(const MachineInstr &MI) {
955 return SIInstrFlags::isTRANS(O: MI);
956 }
957
958 bool isTRANS(uint32_t Opcode) const {
959 return SIInstrFlags::isTRANS(O: get(Opcode));
960 }
961
962 static bool isVOP3P(const MachineInstr &MI) {
963 return SIInstrFlags::isVOP3P(O: MI);
964 }
965
966 bool isVOP3P(uint32_t Opcode) const {
967 return SIInstrFlags::isVOP3P(O: get(Opcode));
968 }
969
970 bool isVOP3PMix(const MachineInstr &MI) const {
971 return isVOP3PMix(Opcode: MI.getOpcode());
972 }
973
974 bool isVOP3PMix(uint16_t Opcode) const {
975 switch (Opcode) {
976 case AMDGPU::V_FMA_MIXHI_F16:
977 case AMDGPU::V_FMA_MIXLO_F16:
978 case AMDGPU::V_FMA_MIX_F32:
979 case AMDGPU::V_MAD_MIXHI_F16:
980 case AMDGPU::V_MAD_MIXLO_F16:
981 case AMDGPU::V_MAD_MIX_F32:
982 return true;
983 default:
984 return false;
985 }
986 }
987
988 static bool isVINTRP(const MachineInstr &MI) {
989 return SIInstrFlags::isVINTRP(O: MI);
990 }
991
992 bool isVINTRP(uint32_t Opcode) const {
993 return SIInstrFlags::isVINTRP(O: get(Opcode));
994 }
995
996 static bool isMAI(const MCInstrDesc &Desc) {
997 return SIInstrFlags::isMAI(O: Desc);
998 }
999
1000 static bool isMAI(const MachineInstr &MI) { return isMAI(Desc: MI.getDesc()); }
1001
1002 bool isMAI(uint32_t Opcode) const { return isMAI(Desc: get(Opcode)); }
1003
1004 static bool isMFMA(const MachineInstr &MI) {
1005 return isMAI(MI) && MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1006 MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64;
1007 }
1008
1009 bool isMFMA(uint32_t Opcode) const {
1010 return isMAI(Opcode) && Opcode != AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
1011 Opcode != AMDGPU::V_ACCVGPR_READ_B32_e64;
1012 }
1013
1014 static bool isDOT(const MachineInstr &MI) { return SIInstrFlags::isDOT(O: MI); }
1015
1016 static bool isWMMA(const MachineInstr &MI) {
1017 return SIInstrFlags::isWMMA(O: MI);
1018 }
1019
1020 bool isWMMA(uint32_t Opcode) const {
1021 return SIInstrFlags::isWMMA(O: get(Opcode));
1022 }
1023
1024 static bool isMFMAorWMMA(const MachineInstr &MI) {
1025 return isMFMA(MI) || isWMMA(MI) || isSWMMAC(MI);
1026 }
1027
1028 bool isMFMAorWMMA(uint32_t Opcode) const {
1029 return isMFMA(Opcode) || isWMMA(Opcode) || isSWMMAC(Opcode);
1030 }
1031
1032 static bool isSWMMAC(const MachineInstr &MI) {
1033 return SIInstrFlags::isSWMMAC(O: MI);
1034 }
1035
1036 bool isSWMMAC(uint32_t Opcode) const {
1037 return SIInstrFlags::isSWMMAC(O: get(Opcode));
1038 }
1039
1040 bool isDOT(uint32_t Opcode) const { return SIInstrFlags::isDOT(O: get(Opcode)); }
1041
1042 bool isXDLWMMA(const MachineInstr &MI) const;
1043
1044 bool isXDL(const MachineInstr &MI) const;
1045
1046 static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opc: Opcode); }
1047
1048 static bool isLDSDIR(const MachineInstr &MI) {
1049 return SIInstrFlags::isLDSDIR(O: MI);
1050 }
1051
1052 bool isLDSDIR(uint32_t Opcode) const {
1053 return SIInstrFlags::isLDSDIR(O: get(Opcode));
1054 }
1055
1056 static bool isVINTERP(const MachineInstr &MI) {
1057 return SIInstrFlags::isVINTERP(O: MI);
1058 }
1059
1060 bool isVINTERP(uint32_t Opcode) const {
1061 return SIInstrFlags::isVINTERP(O: get(Opcode));
1062 }
1063
1064 static bool isScalarUnit(const MachineInstr &MI) {
1065 return SIInstrFlags::isSALU(O: MI) || SIInstrFlags::isSMRD(O: MI);
1066 }
1067
1068 static bool usesVM_CNT(const MachineInstr &MI) {
1069 return SIInstrFlags::usesVM_CNT(O: MI);
1070 }
1071
1072 static bool usesLGKM_CNT(const MachineInstr &MI) {
1073 return SIInstrFlags::usesLGKM_CNT(O: MI);
1074 }
1075
1076 static bool usesASYNC_CNT(const MachineInstr &MI) {
1077 return SIInstrFlags::usesASYNC_CNT(O: MI);
1078 }
1079
1080 bool usesASYNC_CNT(uint32_t Opcode) const {
1081 return SIInstrFlags::usesASYNC_CNT(O: get(Opcode));
1082 }
1083
1084 static bool usesTENSOR_CNT(const MachineInstr &MI) {
1085 return SIInstrFlags::usesTENSOR_CNT(O: MI);
1086 }
1087
1088 bool usesTENSOR_CNT(uint32_t Opcode) const {
1089 return SIInstrFlags::usesTENSOR_CNT(O: get(Opcode));
1090 }
1091
1092 // Most sopk treat the immediate as a signed 16-bit, however some
1093 // use it as unsigned.
1094 static bool sopkIsZext(unsigned Opcode) {
1095 return Opcode == AMDGPU::S_CMPK_EQ_U32 || Opcode == AMDGPU::S_CMPK_LG_U32 ||
1096 Opcode == AMDGPU::S_CMPK_GT_U32 || Opcode == AMDGPU::S_CMPK_GE_U32 ||
1097 Opcode == AMDGPU::S_CMPK_LT_U32 || Opcode == AMDGPU::S_CMPK_LE_U32 ||
1098 Opcode == AMDGPU::S_GETREG_B32 ||
1099 Opcode == AMDGPU::S_GETREG_B32_const;
1100 }
1101
1102 /// \returns true if this is an s_store_dword* instruction. This is more
1103 /// specific than isSMEM && mayStore.
1104 static bool isScalarStore(const MachineInstr &MI) {
1105 return SIInstrFlags::isScalarStore(O: MI);
1106 }
1107
1108 bool isScalarStore(uint32_t Opcode) const {
1109 return SIInstrFlags::isScalarStore(O: get(Opcode));
1110 }
1111
1112 static bool isFixedSize(const MachineInstr &MI) {
1113 return SIInstrFlags::isFixedSize(O: MI);
1114 }
1115
1116 bool isFixedSize(uint32_t Opcode) const {
1117 return SIInstrFlags::isFixedSize(O: get(Opcode));
1118 }
1119
1120 static bool hasFPClamp(const MachineInstr &MI) {
1121 return SIInstrFlags::hasFPClamp(O: MI);
1122 }
1123
1124 bool hasFPClamp(uint32_t Opcode) const {
1125 return SIInstrFlags::hasFPClamp(O: get(Opcode));
1126 }
1127
1128 static bool hasIntClamp(const MachineInstr &MI) {
1129 return SIInstrFlags::hasIntClamp(O: MI);
1130 }
1131
1132 static bool hasSameClamp(const MachineInstr &A, const MachineInstr &B) {
1133 const MCInstrDesc &DA = A.getDesc(), &DB = B.getDesc();
1134 return SIInstrFlags::hasFPClamp(O: DA) == SIInstrFlags::hasFPClamp(O: DB) &&
1135 SIInstrFlags::hasIntClamp(O: DA) == SIInstrFlags::hasIntClamp(O: DB) &&
1136 SIInstrFlags::hasClampLo(O: DA) == SIInstrFlags::hasClampLo(O: DB) &&
1137 SIInstrFlags::hasClampHi(O: DA) == SIInstrFlags::hasClampHi(O: DB);
1138 }
1139
1140 static bool usesFPDPRounding(const MachineInstr &MI) {
1141 return SIInstrFlags::usesFPDPRounding(O: MI);
1142 }
1143
1144 bool usesFPDPRounding(uint32_t Opcode) const {
1145 return SIInstrFlags::usesFPDPRounding(O: get(Opcode));
1146 }
1147
1148 static bool isFPAtomic(const MachineInstr &MI) {
1149 return SIInstrFlags::isFPAtomic(O: MI);
1150 }
1151
1152 bool isFPAtomic(uint32_t Opcode) const {
1153 return SIInstrFlags::isFPAtomic(O: get(Opcode));
1154 }
1155
1156 static bool isNeverUniform(const MachineInstr &MI) {
1157 return SIInstrFlags::isNeverUniform(O: MI);
1158 }
1159
1160 // Check to see if opcode is for a barrier start. Pre gfx12 this is just the
1161 // S_BARRIER, but after support for S_BARRIER_SIGNAL* / S_BARRIER_WAIT we want
1162 // to check for the barrier start (S_BARRIER_SIGNAL*)
1163 bool isBarrierStart(unsigned Opcode) const {
1164 return Opcode == AMDGPU::S_BARRIER ||
1165 Opcode == AMDGPU::S_BARRIER_SIGNAL_M0 ||
1166 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0 ||
1167 Opcode == AMDGPU::S_BARRIER_SIGNAL_IMM ||
1168 Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM;
1169 }
1170
1171 bool isBarrier(unsigned Opcode) const {
1172 return isBarrierStart(Opcode) || Opcode == AMDGPU::S_BARRIER_WAIT ||
1173 Opcode == AMDGPU::S_BARRIER_INIT_M0 ||
1174 Opcode == AMDGPU::S_BARRIER_INIT_IMM ||
1175 Opcode == AMDGPU::S_BARRIER_JOIN_IMM ||
1176 Opcode == AMDGPU::S_BARRIER_LEAVE || Opcode == AMDGPU::DS_GWS_INIT ||
1177 Opcode == AMDGPU::DS_GWS_BARRIER;
1178 }
1179
1180 static bool isLoadMonitor(unsigned Opc) {
1181 switch (Opc) {
1182 case AMDGPU::GLOBAL_LOAD_MONITOR_B32:
1183 case AMDGPU::GLOBAL_LOAD_MONITOR_B32_SADDR:
1184 case AMDGPU::GLOBAL_LOAD_MONITOR_B64:
1185 case AMDGPU::GLOBAL_LOAD_MONITOR_B64_SADDR:
1186 case AMDGPU::GLOBAL_LOAD_MONITOR_B128:
1187 case AMDGPU::GLOBAL_LOAD_MONITOR_B128_SADDR:
1188 case AMDGPU::FLAT_LOAD_MONITOR_B32:
1189 case AMDGPU::FLAT_LOAD_MONITOR_B64:
1190 case AMDGPU::FLAT_LOAD_MONITOR_B128:
1191 return true;
1192 default:
1193 return false;
1194 }
1195 }
1196
1197 static bool isGFX12CacheInvOrWBInst(unsigned Opc) {
1198 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||
1199 Opc == AMDGPU::GLOBAL_WBINV;
1200 }
1201
1202 static bool doesNotReadTiedSource(const MachineInstr &MI) {
1203 return SIInstrFlags::isTiedSourceNotRead(O: MI);
1204 }
1205
1206 bool doesNotReadTiedSource(uint32_t Opcode) const {
1207 return SIInstrFlags::isTiedSourceNotRead(O: get(Opcode));
1208 }
1209
1210 bool isIGLP(unsigned Opcode) const {
1211 return Opcode == AMDGPU::SCHED_BARRIER ||
1212 Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
1213 }
1214
1215 bool isIGLP(const MachineInstr &MI) const { return isIGLP(Opcode: MI.getOpcode()); }
1216
1217 // Return true if the instruction is mutually exclusive with all non-IGLP DAG
1218 // mutations, requiring all other mutations to be disabled.
1219 bool isIGLPMutationOnly(unsigned Opcode) const {
1220 return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT;
1221 }
1222
1223 static unsigned getNonSoftWaitcntOpcode(unsigned Opcode) {
1224 switch (Opcode) {
1225 case AMDGPU::S_WAITCNT_soft:
1226 return AMDGPU::S_WAITCNT;
1227 case AMDGPU::S_WAITCNT_VSCNT_soft:
1228 return AMDGPU::S_WAITCNT_VSCNT;
1229 case AMDGPU::S_WAIT_LOADCNT_soft:
1230 return AMDGPU::S_WAIT_LOADCNT;
1231 case AMDGPU::S_WAIT_STORECNT_soft:
1232 return AMDGPU::S_WAIT_STORECNT;
1233 case AMDGPU::S_WAIT_SAMPLECNT_soft:
1234 return AMDGPU::S_WAIT_SAMPLECNT;
1235 case AMDGPU::S_WAIT_BVHCNT_soft:
1236 return AMDGPU::S_WAIT_BVHCNT;
1237 case AMDGPU::S_WAIT_DSCNT_soft:
1238 return AMDGPU::S_WAIT_DSCNT;
1239 case AMDGPU::S_WAIT_KMCNT_soft:
1240 return AMDGPU::S_WAIT_KMCNT;
1241 case AMDGPU::S_WAIT_XCNT_soft:
1242 return AMDGPU::S_WAIT_XCNT;
1243 default:
1244 return Opcode;
1245 }
1246 }
1247
1248 static bool isWaitcnt(unsigned Opcode) {
1249 switch (getNonSoftWaitcntOpcode(Opcode)) {
1250 case AMDGPU::S_WAITCNT:
1251 case AMDGPU::S_WAITCNT_VSCNT:
1252 case AMDGPU::S_WAITCNT_VMCNT:
1253 case AMDGPU::S_WAITCNT_EXPCNT:
1254 case AMDGPU::S_WAITCNT_LGKMCNT:
1255 case AMDGPU::S_WAIT_LOADCNT:
1256 case AMDGPU::S_WAIT_LOADCNT_DSCNT:
1257 case AMDGPU::S_WAIT_STORECNT:
1258 case AMDGPU::S_WAIT_STORECNT_DSCNT:
1259 case AMDGPU::S_WAIT_SAMPLECNT:
1260 case AMDGPU::S_WAIT_BVHCNT:
1261 case AMDGPU::S_WAIT_EXPCNT:
1262 case AMDGPU::S_WAIT_DSCNT:
1263 case AMDGPU::S_WAIT_KMCNT:
1264 case AMDGPU::S_WAIT_XCNT:
1265 case AMDGPU::S_WAIT_IDLE:
1266 return true;
1267 default:
1268 return false;
1269 }
1270 }
1271
1272 bool isVGPRCopy(const MachineInstr &MI) const {
1273 assert(isCopyInstr(MI));
1274 Register Dest = MI.getOperand(i: 0).getReg();
1275 const MachineFunction &MF = *MI.getMF();
1276 const MachineRegisterInfo &MRI = MF.getRegInfo();
1277 return !RI.isSGPRReg(MRI, Reg: Dest);
1278 }
1279
1280 bool hasVGPRUses(const MachineInstr &MI) const {
1281 const MachineFunction &MF = *MI.getMF();
1282 const MachineRegisterInfo &MRI = MF.getRegInfo();
1283 return llvm::any_of(Range: MI.explicit_uses(),
1284 P: [&MRI, this](const MachineOperand &MO) {
1285 return MO.isReg() && RI.isVGPR(MRI, Reg: MO.getReg());});
1286 }
1287
1288 /// Return true if the instruction modifies the mode register.q
1289 static bool modifiesModeRegister(const MachineInstr &MI);
1290
1291 /// This function is used to determine if an instruction can be safely
1292 /// executed under EXEC = 0 without hardware error, indeterminate results,
1293 /// and/or visible effects on future vector execution or outside the shader.
1294 /// Note: as of 2024 the only use of this is SIPreEmitPeephole where it is
1295 /// used in removing branches over short EXEC = 0 sequences.
1296 /// As such it embeds certain assumptions which may not apply to every case
1297 /// of EXEC = 0 execution.
1298 bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const;
1299
1300 /// Returns true if the instruction could potentially depend on the value of
1301 /// exec. If false, exec dependencies may safely be ignored.
1302 bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const;
1303
1304 bool isInlineConstant(const APInt &Imm) const;
1305
1306 bool isInlineConstant(const APFloat &Imm) const;
1307
1308 // Returns true if this non-register operand definitely does not need to be
1309 // encoded as a 32-bit literal. Note that this function handles all kinds of
1310 // operands, not just immediates.
1311 //
1312 // Some operands like FrameIndexes could resolve to an inline immediate value
1313 // that will not require an additional 4-bytes; this function assumes that it
1314 // will.
1315 bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const {
1316 if (!MO.isImm())
1317 return false;
1318 return isInlineConstant(ImmVal: MO.getImm(), OperandType);
1319 }
1320 bool isInlineConstant(int64_t ImmVal, uint8_t OperandType) const;
1321
1322 bool isInlineConstant(const MachineOperand &MO,
1323 const MCOperandInfo &OpInfo) const {
1324 return isInlineConstant(MO, OperandType: OpInfo.OperandType);
1325 }
1326
1327 /// \p returns true if \p UseMO is substituted with \p DefMO in \p MI it would
1328 /// be an inline immediate.
1329 bool isInlineConstant(const MachineInstr &MI,
1330 const MachineOperand &UseMO,
1331 const MachineOperand &DefMO) const {
1332 assert(UseMO.getParent() == &MI);
1333 int OpIdx = UseMO.getOperandNo();
1334 if (OpIdx >= MI.getDesc().NumOperands)
1335 return false;
1336
1337 return isInlineConstant(MO: DefMO, OpInfo: MI.getDesc().operands()[OpIdx]);
1338 }
1339
1340 /// \p returns true if the operand \p OpIdx in \p MI is a valid inline
1341 /// immediate.
1342 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx) const {
1343 const MachineOperand &MO = MI.getOperand(i: OpIdx);
1344 return isInlineConstant(MO, OperandType: MI.getDesc().operands()[OpIdx].OperandType);
1345 }
1346
1347 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
1348 int64_t ImmVal) const {
1349 if (OpIdx >= MI.getDesc().NumOperands)
1350 return false;
1351
1352 if (isCopyInstr(MI)) {
1353 unsigned Size = getOpSize(MI, OpNo: OpIdx);
1354 assert(Size == 8 || Size == 4);
1355
1356 uint8_t OpType = (Size == 8) ?
1357 AMDGPU::OPERAND_REG_IMM_INT64 : AMDGPU::OPERAND_REG_IMM_INT32;
1358 return isInlineConstant(ImmVal, OperandType: OpType);
1359 }
1360
1361 return isInlineConstant(ImmVal, OperandType: MI.getDesc().operands()[OpIdx].OperandType);
1362 }
1363
1364 bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx,
1365 const MachineOperand &MO) const {
1366 return isInlineConstant(MI, OpIdx, ImmVal: MO.getImm());
1367 }
1368
1369 bool isInlineConstant(const MachineOperand &MO) const {
1370 return isInlineConstant(MI: *MO.getParent(), OpIdx: MO.getOperandNo());
1371 }
1372
1373 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
1374 const MachineOperand &MO) const;
1375
1376 bool isLiteralOperandLegal(const MCInstrDesc &InstDesc,
1377 const MCOperandInfo &OpInfo) const;
1378
1379 bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
1380 int64_t ImmVal) const;
1381
1382 bool isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
1383 const MachineOperand &MO) const {
1384 return isImmOperandLegal(InstDesc: MI.getDesc(), OpNo, MO);
1385 }
1386
1387 bool isNeverCoissue(MachineInstr &MI) const;
1388
1389 /// Check if this immediate value can be used for AV_MOV_B64_IMM_PSEUDO.
1390 bool isLegalAV64PseudoImm(uint64_t Imm) const;
1391
1392 /// Return true if this 64-bit VALU instruction has a 32-bit encoding.
1393 /// This function will return false if you pass it a 32-bit instruction.
1394 bool hasVALU32BitEncoding(unsigned Opcode) const;
1395
1396 bool physRegUsesConstantBus(const MachineOperand &Reg) const;
1397 bool regUsesConstantBus(const MachineOperand &Reg,
1398 const MachineRegisterInfo &MRI) const;
1399
1400 /// Returns true if this operand uses the constant bus.
1401 bool usesConstantBus(const MachineRegisterInfo &MRI,
1402 const MachineOperand &MO,
1403 const MCOperandInfo &OpInfo) const;
1404
1405 bool usesConstantBus(const MachineRegisterInfo &MRI, const MachineInstr &MI,
1406 int OpIdx) const {
1407 return usesConstantBus(MRI, MO: MI.getOperand(i: OpIdx),
1408 OpInfo: MI.getDesc().operands()[OpIdx]);
1409 }
1410
1411 /// Return true if this instruction has any modifiers.
1412 /// e.g. src[012]_mod, omod, clamp.
1413 bool hasModifiers(unsigned Opcode) const;
1414
1415 bool hasModifiersSet(const MachineInstr &MI, AMDGPU::OpName OpName) const;
1416 bool hasAnyModifiersSet(const MachineInstr &MI) const;
1417
1418 bool canShrink(const MachineInstr &MI,
1419 const MachineRegisterInfo &MRI) const;
1420
1421 MachineInstr *buildShrunkInst(MachineInstr &MI,
1422 unsigned NewOpcode) const;
1423
1424 bool verifyInstruction(const MachineInstr &MI,
1425 StringRef &ErrInfo) const override;
1426
1427 unsigned getVALUOp(const MachineInstr &MI) const;
1428 unsigned getVALUOp(unsigned Opc) const;
1429
1430 void insertScratchExecCopy(MachineFunction &MF, MachineBasicBlock &MBB,
1431 MachineBasicBlock::iterator MBBI,
1432 const DebugLoc &DL, Register Reg, bool IsSCCLive,
1433 SlotIndexes *Indexes = nullptr) const;
1434
1435 void restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
1436 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
1437 Register Reg, SlotIndexes *Indexes = nullptr) const;
1438
1439 MachineInstr *getWholeWaveFunctionSetup(MachineFunction &MF) const;
1440
1441 /// Return the correct register class for \p OpNo. For target-specific
1442 /// instructions, this will return the register class that has been defined
1443 /// in tablegen. For generic instructions, like REG_SEQUENCE it will return
1444 /// the register class of its machine operand.
1445 /// to infer the correct register class base on the other operands.
1446 const TargetRegisterClass *getOpRegClass(const MachineInstr &MI,
1447 unsigned OpNo) const;
1448
1449 /// Return the size in bytes of the operand OpNo on the given
1450 // instruction opcode.
1451 unsigned getOpSize(uint32_t Opcode, unsigned OpNo) const {
1452 const MCOperandInfo &OpInfo = get(Opcode).operands()[OpNo];
1453
1454 if (OpInfo.RegClass == -1) {
1455 // If this is an immediate operand, this must be a 32-bit literal.
1456 assert(OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE);
1457 return 4;
1458 }
1459
1460 return RI.getRegSizeInBits(RC: *RI.getRegClass(i: getOpRegClassID(OpInfo))) / 8;
1461 }
1462
1463 /// This form should usually be preferred since it handles operands
1464 /// with unknown register classes.
1465 unsigned getOpSize(const MachineInstr &MI, unsigned OpNo) const {
1466 const MachineOperand &MO = MI.getOperand(i: OpNo);
1467 if (MO.isReg()) {
1468 if (unsigned SubReg = MO.getSubReg()) {
1469 return RI.getSubRegIdxSize(Idx: SubReg) / 8;
1470 }
1471 }
1472 return RI.getRegSizeInBits(RC: *getOpRegClass(MI, OpNo)) / 8;
1473 }
1474
1475 /// Legalize the \p OpIndex operand of this instruction by inserting
1476 /// a MOV. For example:
1477 /// ADD_I32_e32 VGPR0, 15
1478 /// to
1479 /// MOV VGPR1, 15
1480 /// ADD_I32_e32 VGPR0, VGPR1
1481 ///
1482 /// If the operand being legalized is a register, then a COPY will be used
1483 /// instead of MOV.
1484 void legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const;
1485
1486 /// Check if \p MO is a legal operand if it was the \p OpIdx Operand
1487 /// for \p MI.
1488 bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
1489 const MachineOperand *MO = nullptr) const;
1490
1491 /// Check if \p MO would be a valid operand for the given operand
1492 /// definition \p OpInfo. Note this does not attempt to validate constant bus
1493 /// restrictions (e.g. literal constant usage).
1494 bool isLegalVSrcOperand(const MachineRegisterInfo &MRI,
1495 const MCOperandInfo &OpInfo,
1496 const MachineOperand &MO) const;
1497
1498 /// Check if \p MO (a register operand) is a legal register for the
1499 /// given operand description or operand index.
1500 /// The operand index version provide more legality checks
1501 bool isLegalRegOperand(const MachineRegisterInfo &MRI,
1502 const MCOperandInfo &OpInfo,
1503 const MachineOperand &MO) const;
1504 bool isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
1505 const MachineOperand &MO) const;
1506
1507 /// Check if \p MO would be a legal operand for a single-SGPR-read
1508 /// instruction.
1509 ///
1510 /// Single-SGPR-read instructions typically accept VGPRs, SGPRs, or immediates
1511 /// as source operands. On gfx12+, if a source operand uses SGPRs, the HW can
1512 /// only read the first SGPR and replicate the value across all lanes. \p SrcN
1513 /// can be 0, 1, or 2, representing src0, src1, and src2, respectively. If \p
1514 /// MO is nullptr, the operand corresponding to \p SrcN will be used. Non-SGPR
1515 /// operands are always considered legal.
1516 bool
1517 isLegalSingleSGPRReadInstOperand(const MachineRegisterInfo &MRI,
1518 const MachineInstr &MI, unsigned SrcN,
1519 const MachineOperand *MO = nullptr) const;
1520
1521 /// Legalize operands in \p MI by either commuting it or inserting a
1522 /// copy of src1.
1523 void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1524
1525 /// Fix operands in \p MI to satisfy constant bus requirements.
1526 void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1527
1528 /// Copy a value from a VGPR (\p SrcReg) to SGPR. The desired register class
1529 /// for the dst register (\p DstRC) can be optionally supplied. This function
1530 /// can only be used when it is know that the value in SrcReg is same across
1531 /// all threads in the wave.
1532 /// \returns The SGPR register that \p SrcReg was copied to.
1533 Register readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,
1534 MachineRegisterInfo &MRI,
1535 const TargetRegisterClass *DstRC = nullptr) const;
1536
1537 void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1538 void legalizeOperandsFLAT(MachineRegisterInfo &MRI, MachineInstr &MI) const;
1539
1540 void legalizeGenericOperand(MachineBasicBlock &InsertMBB,
1541 MachineBasicBlock::iterator I,
1542 const TargetRegisterClass *DstRC,
1543 MachineOperand &Op, MachineRegisterInfo &MRI,
1544 const DebugLoc &DL) const;
1545
1546 /// Legalize all operands in this instruction. This function may create new
1547 /// instructions and control-flow around \p MI. If present, \p MDT is
1548 /// updated.
1549 /// \returns A new basic block that contains \p MI if new blocks were created.
1550 MachineBasicBlock *
1551 legalizeOperands(MachineInstr &MI, MachineDominatorTree *MDT = nullptr) const;
1552
1553 /// Change SADDR form of a FLAT \p Inst to its VADDR form if saddr operand
1554 /// was moved to VGPR. \returns true if succeeded.
1555 bool moveFlatAddrToVGPR(MachineInstr &Inst) const;
1556
1557 /// Fix operands in Inst to fix 16bit SALU to VALU lowering.
1558 void legalizeOperandsVALUt16(MachineInstr &Inst,
1559 MachineRegisterInfo &MRI) const;
1560 void legalizeOperandsVALUt16(MachineInstr &Inst, unsigned OpIdx,
1561 MachineRegisterInfo &MRI) const;
1562
1563 /// Replace the instructions opcode with the equivalent VALU
1564 /// opcode. This function will also move the users of MachineInstruntions
1565 /// in the \p WorkList to the VALU if necessary. If present, \p MDT is
1566 /// updated.
1567 void moveToVALU(SIInstrWorklist &Worklist, MachineDominatorTree *MDT) const;
1568
1569 void
1570 moveToVALUImpl(SIInstrWorklist &Worklist, MachineDominatorTree *MDT,
1571 MachineInstr &Inst,
1572 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
1573 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const;
1574 /// Wrapper function for generating waterfall for instruction \p MI
1575 /// This function take into consideration of related pre & succ instructions
1576 /// (e.g. calling process) into consideratioin
1577 void createWaterFallForSiCall(MachineInstr *MI, MachineDominatorTree *MDT,
1578 ArrayRef<MachineOperand *> ScalarOps,
1579 ArrayRef<Register> PhySGPRs = {}) const;
1580
1581 void insertNoop(MachineBasicBlock &MBB,
1582 MachineBasicBlock::iterator MI) const override;
1583
1584 void insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
1585 unsigned Quantity) const override;
1586
1587 /// Build instructions that simulate the behavior of a `s_trap 2` instructions
1588 /// for hardware (namely, gfx11) that runs in PRIV=1 mode. There, s_trap is
1589 /// interpreted as a nop.
1590 MachineBasicBlock *insertSimulatedTrap(MachineRegisterInfo &MRI,
1591 MachineBasicBlock &MBB,
1592 MachineInstr &MI,
1593 const DebugLoc &DL) const;
1594
1595 /// Return the number of wait states that result from executing this
1596 /// instruction.
1597 static unsigned getNumWaitStates(const MachineInstr &MI);
1598
1599 /// Returns the operand named \p Op. If \p MI does not have an
1600 /// operand named \c Op, this function returns nullptr.
1601 LLVM_READONLY
1602 MachineOperand *getNamedOperand(MachineInstr &MI,
1603 AMDGPU::OpName OperandName) const;
1604
1605 LLVM_READONLY
1606 const MachineOperand *getNamedOperand(const MachineInstr &MI,
1607 AMDGPU::OpName OperandName) const {
1608 return getNamedOperand(MI&: const_cast<MachineInstr &>(MI), OperandName);
1609 }
1610
1611 /// Get required immediate operand
1612 int64_t getNamedImmOperand(const MachineInstr &MI,
1613 AMDGPU::OpName OperandName) const {
1614 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OperandName);
1615 return MI.getOperand(i: Idx).getImm();
1616 }
1617
1618 uint64_t getDefaultRsrcDataFormat() const;
1619 uint64_t getScratchRsrcWords23() const;
1620
1621 bool isLowLatencyInstruction(const MachineInstr &MI) const;
1622 bool isHighLatencyDef(int Opc) const override;
1623
1624 /// Return the descriptor of the target-specific machine instruction
1625 /// that corresponds to the specified pseudo or native opcode.
1626 const MCInstrDesc &getMCOpcodeFromPseudo(unsigned Opcode) const {
1627 return get(Opcode: pseudoToMCOpcode(Opcode));
1628 }
1629
1630 Register isStackAccess(const MachineInstr &MI, int &FrameIndex,
1631 TypeSize &MemBytes) const;
1632 Register isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex,
1633 TypeSize &MemBytes) const;
1634
1635 Register isLoadFromStackSlot(const MachineInstr &MI,
1636 int &FrameIndex) const override {
1637 TypeSize MemBytes = TypeSize::getZero();
1638 return isLoadFromStackSlot(MI, FrameIndex, MemBytes);
1639 }
1640
1641 Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex,
1642 TypeSize &MemBytes) const override;
1643
1644 Register isStoreToStackSlot(const MachineInstr &MI,
1645 int &FrameIndex) const override {
1646 TypeSize MemBytes = TypeSize::getZero();
1647 return isStoreToStackSlot(MI, FrameIndex, MemBytes);
1648 }
1649
1650 Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex,
1651 TypeSize &MemBytes) const override;
1652
1653 unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
1654
1655 InstSizeVerifyMode
1656 getInstSizeVerifyMode(const MachineInstr &MI) const override;
1657
1658 bool mayAccessFlatAddressSpace(const MachineInstr &MI) const;
1659
1660 std::pair<unsigned, unsigned>
1661 decomposeMachineOperandsTargetFlags(unsigned TF) const override;
1662
1663 ArrayRef<std::pair<int, const char *>>
1664 getSerializableTargetIndices() const override;
1665
1666 ArrayRef<std::pair<unsigned, const char *>>
1667 getSerializableDirectMachineOperandTargetFlags() const override;
1668
1669 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
1670 getSerializableMachineMemOperandTargetFlags() const override;
1671
1672 ScheduleHazardRecognizer *
1673 CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
1674 const ScheduleDAG *DAG) const override;
1675
1676 ScheduleHazardRecognizer *
1677 CreateTargetPostRAHazardRecognizer(const MachineFunction &MF,
1678 MachineLoopInfo *MLI) const override;
1679
1680 ScheduleHazardRecognizer *
1681 CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
1682 const ScheduleDAGMI *DAG) const override;
1683
1684 unsigned getLiveRangeSplitOpcode(Register Reg,
1685 const MachineFunction &MF) const override;
1686
1687 bool isBasicBlockPrologue(const MachineInstr &MI,
1688 Register Reg = Register()) const override;
1689
1690 bool canAddToBBProlog(const MachineInstr &MI) const;
1691
1692 MachineInstr *createPHIDestinationCopy(MachineBasicBlock &MBB,
1693 MachineBasicBlock::iterator InsPt,
1694 const DebugLoc &DL, Register Src,
1695 Register Dst) const override;
1696
1697 MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB,
1698 MachineBasicBlock::iterator InsPt,
1699 const DebugLoc &DL, Register Src,
1700 unsigned SrcSubReg,
1701 Register Dst) const override;
1702
1703 bool isWave32() const;
1704
1705 bool isVOPDAntidependencyAllowed(const MachineInstr &MI) const;
1706
1707 bool hasRAWDependency(const MachineInstr &FirstMI,
1708 const MachineInstr &SecondMI) const;
1709
1710 /// Return a partially built integer add instruction without carry.
1711 /// Caller must add source operands.
1712 /// For pre-GFX9 it will generate unused carry destination operand.
1713 /// TODO: After GFX9 it should return a no-carry operation.
1714 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1715 MachineBasicBlock::iterator I,
1716 const DebugLoc &DL,
1717 Register DestReg) const;
1718
1719 MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB,
1720 MachineBasicBlock::iterator I,
1721 const DebugLoc &DL,
1722 Register DestReg,
1723 RegScavenger &RS) const;
1724
1725 static bool isKillTerminator(unsigned Opcode);
1726 const MCInstrDesc &getKillTerminatorFromPseudo(unsigned Opcode) const;
1727
1728 bool isLegalMUBUFImmOffset(unsigned Imm) const;
1729
1730 static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST);
1731
1732 bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset,
1733 Align Alignment = Align(4)) const;
1734
1735 /// Returns if \p Offset is legal for the subtarget as the offset to a FLAT
1736 /// encoded instruction with the given \p FlatVariant.
1737 bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
1738 AMDGPU::FlatAddrSpace FlatVariant) const;
1739
1740 /// Split \p COffsetVal into {immediate offset field, remainder offset}
1741 /// values.
1742 std::pair<int64_t, int64_t>
1743 splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,
1744 AMDGPU::FlatAddrSpace FlatVariant) const;
1745
1746 /// Returns true if negative offsets are allowed for the given \p FlatVariant.
1747 bool allowNegativeFlatOffset(AMDGPU::FlatAddrSpace FlatVariant) const;
1748
1749 /// \brief Return a target-specific opcode if Opcode is a pseudo instruction.
1750 /// Return -1 if the target-specific opcode for the pseudo instruction does
1751 /// not exist. If Opcode is not a pseudo instruction, this is identity.
1752 int pseudoToMCOpcode(int Opcode) const;
1753
1754 /// \brief Check if this instruction should only be used by assembler.
1755 /// Return true if this opcode should not be used by codegen.
1756 bool isAsmOnlyOpcode(int MCOp) const;
1757
1758 void fixImplicitOperands(MachineInstr &MI) const;
1759
1760 MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
1761 ArrayRef<unsigned> Ops, int FrameIndex,
1762 MachineInstr *&CopyMI,
1763 LiveIntervals *LIS = nullptr,
1764 VirtRegMap *VRM = nullptr) const override;
1765
1766 unsigned getInstrLatency(const InstrItineraryData *ItinData,
1767 const MachineInstr &MI,
1768 unsigned *PredCost = nullptr) const override;
1769
1770 unsigned getBlockingCycles(const MachineInstr &MI) const;
1771
1772 const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override;
1773
1774 ValueUniformity getValueUniformity(const MachineInstr &MI) const final;
1775
1776 ValueUniformity getGenericValueUniformity(const MachineInstr &MI) const;
1777
1778 const MIRFormatter *getMIRFormatter() const override;
1779
1780 static unsigned getDSShaderTypeValue(const MachineFunction &MF);
1781
1782 const TargetSchedModel &getSchedModel() const { return SchedModel; }
1783
1784 void createReadFirstLaneFromCopyToPhysReg(MachineRegisterInfo &MRI,
1785 Register DstReg,
1786 MachineInstr &Inst) const;
1787
1788 void handleCopyToPhysHelper(
1789 SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst,
1790 MachineRegisterInfo &MRI,
1791 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
1792 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const;
1793
1794 // FIXME: This should be removed
1795 // Enforce operand's \p OpName even alignment if required by target.
1796 // This is used if an operand is a 32 bit register but needs to be aligned
1797 // regardless.
1798 void enforceOperandRCAlignment(MachineInstr &MI, AMDGPU::OpName OpName) const;
1799
1800 /// Get the repeat rate for a VALU instruction from the scheduling model.
1801 /// Returns 1 for regular VALU, >1 for long-latency VALU (packed, F64, etc.)
1802 unsigned getRepeatRate(const MachineInstr &MI) const;
1803};
1804
1805/// \brief Returns true if a reg:subreg pair P has a TRC class
1806inline bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P,
1807 const TargetRegisterClass &TRC,
1808 MachineRegisterInfo &MRI) {
1809 auto *RC = MRI.getRegClass(Reg: P.Reg);
1810 if (!P.SubReg)
1811 return RC == &TRC;
1812 auto *TRI = MRI.getTargetRegisterInfo();
1813 return RC == TRI->getMatchingSuperRegClass(A: RC, B: &TRC, Idx: P.SubReg);
1814}
1815
1816/// \brief Create RegSubRegPair from a register MachineOperand
1817inline
1818TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O) {
1819 assert(O.isReg());
1820 return TargetInstrInfo::RegSubRegPair(O.getReg(), O.getSubReg());
1821}
1822
1823/// \brief Return the SubReg component from REG_SEQUENCE
1824TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI,
1825 unsigned SubReg);
1826
1827/// \brief Return the defining instruction for a given reg:subreg pair
1828/// skipping copy like instructions and subreg-manipulation pseudos.
1829/// Following another subreg of a reg:subreg isn't supported.
1830MachineInstr *getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
1831 const MachineRegisterInfo &MRI);
1832
1833/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1834/// DefMI and the use at \p UseMI. Should be run on SSA. Currently does not
1835/// attempt to track between blocks.
1836bool execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
1837 Register VReg,
1838 const MachineInstr &DefMI,
1839 const MachineInstr &UseMI);
1840
1841/// \brief Return false if EXEC is not changed between the def of \p VReg at \p
1842/// DefMI and all its uses. Should be run on SSA. Currently does not attempt to
1843/// track between blocks.
1844bool execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
1845 Register VReg,
1846 const MachineInstr &DefMI);
1847
1848namespace AMDGPU {
1849
1850 LLVM_READONLY
1851 int32_t getVOPe64(uint32_t Opcode);
1852
1853 LLVM_READONLY
1854 int32_t getVOPe32(uint32_t Opcode);
1855
1856 LLVM_READONLY
1857 int32_t getSDWAOp(uint32_t Opcode);
1858
1859 LLVM_READONLY
1860 int32_t getDPPOp32(uint32_t Opcode);
1861
1862 LLVM_READONLY
1863 int32_t getDPPOp64(uint32_t Opcode);
1864
1865 LLVM_READONLY
1866 int32_t getBasicFromSDWAOp(uint32_t Opcode);
1867
1868 LLVM_READONLY
1869 int32_t getCommuteRev(uint32_t Opcode);
1870
1871 LLVM_READONLY
1872 int32_t getCommuteOrig(uint32_t Opcode);
1873
1874 LLVM_READONLY
1875 int32_t getAddr64Inst(uint32_t Opcode);
1876
1877 /// Check if \p Opcode is an Addr64 opcode.
1878 ///
1879 /// \returns \p Opcode if it is an Addr64 opcode, otherwise -1.
1880 LLVM_READONLY
1881 int32_t getIfAddr64Inst(uint32_t Opcode);
1882
1883 LLVM_READONLY
1884 int32_t getSOPKOp(uint32_t Opcode);
1885
1886 /// \returns SADDR form of a FLAT Global instruction given an \p Opcode
1887 /// of a VADDR form.
1888 LLVM_READONLY
1889 int32_t getGlobalSaddrOp(uint32_t Opcode);
1890
1891 /// \returns VADDR form of a FLAT Global instruction given an \p Opcode
1892 /// of a SADDR form.
1893 LLVM_READONLY
1894 int32_t getGlobalVaddrOp(uint32_t Opcode);
1895
1896 /// \returns ST form with only immediate offset of a FLAT Scratch instruction
1897 /// given an \p Opcode of an SS (SADDR) form.
1898 LLVM_READONLY
1899 int32_t getFlatScratchInstSTfromSS(uint32_t Opcode);
1900
1901 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
1902 /// of an SVS (SADDR + VADDR) form.
1903 LLVM_READONLY
1904 int32_t getFlatScratchInstSVfromSVS(uint32_t Opcode);
1905
1906 /// \returns SS (SADDR) form of a FLAT Scratch instruction given an \p Opcode
1907 /// of an SV (VADDR) form.
1908 LLVM_READONLY
1909 int32_t getFlatScratchInstSSfromSV(uint32_t Opcode);
1910
1911 /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode
1912 /// of an SS (SADDR) form.
1913 LLVM_READONLY
1914 int32_t getFlatScratchInstSVfromSS(uint32_t Opcode);
1915
1916 /// \returns earlyclobber version of a MAC MFMA is exists.
1917 LLVM_READONLY
1918 int32_t getMFMAEarlyClobberOp(uint32_t Opcode);
1919
1920 /// \returns Version of an instruction which uses AGPRs for coupled operands
1921 /// given an \p Opcode which uses VGPRs for coupled operands.
1922 LLVM_READONLY
1923 int32_t getAGPRFormOp(uint32_t Opcode);
1924
1925 /// \returns v_cmpx version of a v_cmp instruction.
1926 LLVM_READONLY
1927 int32_t getVCMPXOpFromVCMP(uint32_t Opcode);
1928
1929 const uint64_t RSRC_DATA_FORMAT = 0xf00000000000LL;
1930 const uint64_t RSRC_ELEMENT_SIZE_SHIFT = (32 + 19);
1931 const uint64_t RSRC_INDEX_STRIDE_SHIFT = (32 + 21);
1932 const uint64_t RSRC_TID_ENABLE = UINT64_C(1) << (32 + 23);
1933
1934} // end namespace AMDGPU
1935
1936namespace AMDGPU {
1937enum AsmComments : MachineInstr::AsmPrinterFlagTy {
1938 // For sgpr to vgpr spill instructions
1939 SGPR_SPILL = MachineInstr::TAsmComments
1940};
1941} // namespace AMDGPU
1942
1943namespace SI {
1944namespace KernelInputOffsets {
1945
1946/// Offsets in bytes from the start of the input buffer
1947enum Offsets {
1948 NGROUPS_X = 0,
1949 NGROUPS_Y = 4,
1950 NGROUPS_Z = 8,
1951 GLOBAL_SIZE_X = 12,
1952 GLOBAL_SIZE_Y = 16,
1953 GLOBAL_SIZE_Z = 20,
1954 LOCAL_SIZE_X = 24,
1955 LOCAL_SIZE_Y = 28,
1956 LOCAL_SIZE_Z = 32
1957};
1958
1959} // end namespace KernelInputOffsets
1960} // end namespace SI
1961
1962} // end namespace llvm
1963
1964#endif // LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H
1965