| 1 | //===- SIInstrInfo.h - SI Instruction Info Interface ------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | /// \file |
| 10 | /// Interface definition for SIInstrInfo. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #ifndef LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H |
| 15 | #define LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H |
| 16 | |
| 17 | #include "AMDGPUMIRFormatter.h" |
| 18 | #include "MCTargetDesc/AMDGPUMCTargetDesc.h" |
| 19 | #include "SIRegisterInfo.h" |
| 20 | #include "Utils/AMDGPUBaseInfo.h" |
| 21 | #include "llvm/ADT/SetVector.h" |
| 22 | #include "llvm/ADT/SmallPtrSet.h" |
| 23 | #include "llvm/CodeGen/TargetInstrInfo.h" |
| 24 | #include "llvm/CodeGen/TargetSchedule.h" |
| 25 | |
| 26 | #define |
| 27 | #include "AMDGPUGenInstrInfo.inc" |
| 28 | |
| 29 | namespace llvm { |
| 30 | |
| 31 | class APInt; |
| 32 | class GCNSubtarget; |
| 33 | class MachineDominatorTree; |
| 34 | class MachineRegisterInfo; |
| 35 | class RegScavenger; |
| 36 | class SIMachineFunctionInfo; |
| 37 | class MCRegisterClass; |
| 38 | using TargetRegisterClass = MCRegisterClass; |
| 39 | class ScheduleHazardRecognizer; |
| 40 | |
| 41 | namespace AMDGPU { |
| 42 | enum class AsyncStages : uint32_t; |
| 43 | } // namespace AMDGPU |
| 44 | |
| 45 | constexpr unsigned DefaultMemoryClusterDWordsLimit = 8; |
| 46 | |
| 47 | /// Mark the MMO of a uniform load if there are no potentially clobbering stores |
| 48 | /// on any path from the start of an entry function to this load. |
| 49 | static const MachineMemOperand::Flags MONoClobber = |
| 50 | MachineMemOperand::MOTargetFlag1; |
| 51 | |
| 52 | /// Mark the MMO of a load as the last use. |
| 53 | static const MachineMemOperand::Flags MOLastUse = |
| 54 | MachineMemOperand::MOTargetFlag2; |
| 55 | |
| 56 | /// Mark the MMO of cooperative load/store atomics. |
| 57 | static const MachineMemOperand::Flags MOCooperative = |
| 58 | MachineMemOperand::MOTargetFlag3; |
| 59 | |
| 60 | struct V2PhysSCopyInfo { |
| 61 | // Operands that need to replaced by waterfall |
| 62 | SmallVector<MachineOperand *> MOs; |
| 63 | // Target physical registers replacing the MOs |
| 64 | SmallVector<Register> SGPRs; |
| 65 | }; |
| 66 | /// Mark the MMO of accesses to memory locations that are |
| 67 | /// never written to by other threads. |
| 68 | static const MachineMemOperand::Flags MOThreadPrivate = |
| 69 | MachineMemOperand::MOTargetFlag4; |
| 70 | |
| 71 | /// Use two MMO flags to encode the 2-bit cache fill size field for GFX13+ |
| 72 | static const MachineMemOperand::Flags MOCFSB0 = |
| 73 | MachineMemOperand::MOTargetFlag5; |
| 74 | |
| 75 | static const MachineMemOperand::Flags MOCFSB1 = |
| 76 | MachineMemOperand::MOTargetFlag6; |
| 77 | |
| 78 | /// Utility to store machine instructions worklist. |
| 79 | struct SIInstrWorklist { |
| 80 | SIInstrWorklist() = default; |
| 81 | |
| 82 | void insert(MachineInstr *MI); |
| 83 | |
| 84 | MachineInstr *top() const { return InstrList[Front]; } |
| 85 | |
| 86 | void erase_top() { |
| 87 | InSet.erase(Ptr: InstrList[Front]); |
| 88 | ++Front; |
| 89 | } |
| 90 | |
| 91 | bool empty() const { return Front == InstrList.size(); } |
| 92 | |
| 93 | void clear() { |
| 94 | InstrList.clear(); |
| 95 | Front = 0; |
| 96 | InSet.clear(); |
| 97 | DeferredList.clear(); |
| 98 | } |
| 99 | |
| 100 | bool isDeferred(MachineInstr *MI); |
| 101 | |
| 102 | SetVector<MachineInstr *> &getDeferredList() { return DeferredList; } |
| 103 | |
| 104 | private: |
| 105 | /// InstrList contains the MachineInstrs. |
| 106 | SmallVector<MachineInstr *> InstrList; |
| 107 | SmallPtrSet<MachineInstr *, 8> InSet; |
| 108 | unsigned Front = 0; |
| 109 | /// Deferred instructions are specific MachineInstr |
| 110 | /// that will be added by insert method. |
| 111 | SetVector<MachineInstr *> DeferredList; |
| 112 | }; |
| 113 | |
| 114 | // In namespace llvm so ADL finds it when SIInstrFlags predicates are |
| 115 | // instantiated with MachineInstr (MachineInstr is in namespace llvm). |
| 116 | inline uint64_t getTSFlags(const MachineInstr &MI) { |
| 117 | return MI.getDesc().TSFlags; |
| 118 | } |
| 119 | |
| 120 | class SIInstrInfo final : public AMDGPUGenInstrInfo { |
| 121 | struct ThreeAddressUpdates; |
| 122 | |
| 123 | private: |
| 124 | const SIRegisterInfo RI; |
| 125 | const GCNSubtarget &ST; |
| 126 | TargetSchedModel SchedModel; |
| 127 | mutable std::unique_ptr<AMDGPUMIRFormatter> Formatter; |
| 128 | |
| 129 | // The inverse predicate should have the negative value. |
| 130 | enum BranchPredicate { |
| 131 | INVALID_BR = 0, |
| 132 | SCC_TRUE = 1, |
| 133 | SCC_FALSE = -1, |
| 134 | VCCNZ = 2, |
| 135 | VCCZ = -2, |
| 136 | EXECNZ = -3, |
| 137 | EXECZ = 3 |
| 138 | }; |
| 139 | |
| 140 | using SetVectorType = SmallSetVector<MachineInstr *, 32>; |
| 141 | |
| 142 | static unsigned getBranchOpcode(BranchPredicate Cond); |
| 143 | static BranchPredicate getBranchPredicate(unsigned Opcode); |
| 144 | |
| 145 | public: |
| 146 | unsigned buildExtractSubReg(MachineBasicBlock::iterator MI, |
| 147 | MachineRegisterInfo &MRI, |
| 148 | const MachineOperand &SuperReg, |
| 149 | const TargetRegisterClass *SuperRC, |
| 150 | unsigned SubIdx, |
| 151 | const TargetRegisterClass *SubRC) const; |
| 152 | MachineOperand ( |
| 153 | MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI, |
| 154 | const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC, |
| 155 | unsigned SubIdx, const TargetRegisterClass *SubRC) const; |
| 156 | |
| 157 | private: |
| 158 | bool optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine, |
| 159 | bool NeedInversion) const; |
| 160 | |
| 161 | bool invertSCCUse(MachineInstr *SCCDef) const; |
| 162 | |
| 163 | void swapOperands(MachineInstr &Inst) const; |
| 164 | |
| 165 | std::pair<bool, MachineBasicBlock *> |
| 166 | moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst, |
| 167 | MachineDominatorTree *MDT = nullptr) const; |
| 168 | |
| 169 | void lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst, |
| 170 | MachineDominatorTree *MDT = nullptr) const; |
| 171 | |
| 172 | void lowerScalarAbs(SIInstrWorklist &Worklist, MachineInstr &Inst) const; |
| 173 | |
| 174 | void lowerScalarAbsDiff(SIInstrWorklist &Worklist, MachineInstr &Inst) const; |
| 175 | |
| 176 | void lowerScalarXnor(SIInstrWorklist &Worklist, MachineInstr &Inst) const; |
| 177 | |
| 178 | void splitScalarNotBinop(SIInstrWorklist &Worklist, MachineInstr &Inst, |
| 179 | unsigned Opcode) const; |
| 180 | |
| 181 | void splitScalarBinOpN2(SIInstrWorklist &Worklist, MachineInstr &Inst, |
| 182 | unsigned Opcode) const; |
| 183 | |
| 184 | void splitScalar64BitUnaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst, |
| 185 | unsigned Opcode, bool Swap = false) const; |
| 186 | |
| 187 | void splitScalar64BitBinaryOp(SIInstrWorklist &Worklist, MachineInstr &Inst, |
| 188 | unsigned Opcode, |
| 189 | MachineDominatorTree *MDT = nullptr) const; |
| 190 | |
| 191 | void splitScalarSMulU64(SIInstrWorklist &Worklist, MachineInstr &Inst, |
| 192 | MachineDominatorTree *MDT) const; |
| 193 | |
| 194 | void splitScalarSMulPseudo(SIInstrWorklist &Worklist, MachineInstr &Inst, |
| 195 | MachineDominatorTree *MDT) const; |
| 196 | |
| 197 | void splitScalar64BitXnor(SIInstrWorklist &Worklist, MachineInstr &Inst, |
| 198 | MachineDominatorTree *MDT = nullptr) const; |
| 199 | |
| 200 | void splitScalar64BitBCNT(SIInstrWorklist &Worklist, |
| 201 | MachineInstr &Inst) const; |
| 202 | void splitScalar64BitBFE(SIInstrWorklist &Worklist, MachineInstr &Inst) const; |
| 203 | void splitScalar64BitCountOp(SIInstrWorklist &Worklist, MachineInstr &Inst, |
| 204 | unsigned Opcode, |
| 205 | MachineDominatorTree *MDT = nullptr) const; |
| 206 | void movePackToVALU(SIInstrWorklist &Worklist, MachineRegisterInfo &MRI, |
| 207 | MachineInstr &Inst) const; |
| 208 | |
| 209 | void addUsersToMoveToVALUWorklist(Register Reg, MachineRegisterInfo &MRI, |
| 210 | SIInstrWorklist &Worklist) const; |
| 211 | |
| 212 | void addSCCDefUsersToVALUWorklist(const MachineOperand &Op, |
| 213 | MachineInstr &SCCDefInst, |
| 214 | SIInstrWorklist &Worklist, |
| 215 | Register NewCond = Register()) const; |
| 216 | void addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst, |
| 217 | SIInstrWorklist &Worklist) const; |
| 218 | |
| 219 | const TargetRegisterClass * |
| 220 | getDestEquivalentVGPRClass(const MachineInstr &Inst) const; |
| 221 | |
| 222 | bool checkInstOffsetsDoNotOverlap(const MachineInstr &MIa, |
| 223 | const MachineInstr &MIb) const; |
| 224 | |
| 225 | Register findUsedSGPR(const MachineInstr &MI, int OpIndices[3]) const; |
| 226 | |
| 227 | bool verifyCopy(const MachineInstr &MI, const MachineRegisterInfo &MRI, |
| 228 | StringRef &ErrInfo) const; |
| 229 | |
| 230 | bool resultDependsOnExec(const MachineInstr &MI) const; |
| 231 | |
| 232 | MachineInstr *convertToThreeAddressImpl(MachineInstr &MI, |
| 233 | ThreeAddressUpdates &Updates) const; |
| 234 | |
| 235 | protected: |
| 236 | /// If the specific machine instruction is a instruction that moves/copies |
| 237 | /// value from one register to another register return destination and source |
| 238 | /// registers as machine operands. |
| 239 | std::optional<DestSourcePair> |
| 240 | isCopyInstrImpl(const MachineInstr &MI) const override; |
| 241 | |
| 242 | bool swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0, |
| 243 | AMDGPU::OpName Src0OpName, MachineOperand &Src1, |
| 244 | AMDGPU::OpName Src1OpName) const; |
| 245 | bool isNonCommutableDPP(const MachineInstr &MI) const; |
| 246 | MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI, |
| 247 | unsigned OpIdx0, |
| 248 | unsigned OpIdx1) const override; |
| 249 | |
| 250 | public: |
| 251 | bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx, |
| 252 | unsigned toIdx) const; |
| 253 | enum TargetOperandFlags { |
| 254 | MO_MASK = 0xf, |
| 255 | |
| 256 | MO_NONE = 0, |
| 257 | // MO_GOTPCREL -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL. |
| 258 | MO_GOTPCREL = 1, |
| 259 | // MO_GOTPCREL32_LO -> symbol@gotpcrel32@lo -> R_AMDGPU_GOTPCREL32_LO. |
| 260 | MO_GOTPCREL32 = 2, |
| 261 | MO_GOTPCREL32_LO = 2, |
| 262 | // MO_GOTPCREL32_HI -> symbol@gotpcrel32@hi -> R_AMDGPU_GOTPCREL32_HI. |
| 263 | MO_GOTPCREL32_HI = 3, |
| 264 | // MO_GOTPCREL64 -> symbol@GOTPCREL -> R_AMDGPU_GOTPCREL. |
| 265 | MO_GOTPCREL64 = 4, |
| 266 | // MO_REL32_LO -> symbol@rel32@lo -> R_AMDGPU_REL32_LO. |
| 267 | MO_REL32 = 5, |
| 268 | MO_REL32_LO = 5, |
| 269 | // MO_REL32_HI -> symbol@rel32@hi -> R_AMDGPU_REL32_HI. |
| 270 | MO_REL32_HI = 6, |
| 271 | MO_REL64 = 7, |
| 272 | |
| 273 | MO_FAR_BRANCH_OFFSET = 8, |
| 274 | |
| 275 | MO_ABS32_LO = 9, |
| 276 | MO_ABS32_HI = 10, |
| 277 | MO_ABS64 = 11, |
| 278 | }; |
| 279 | |
| 280 | explicit SIInstrInfo(const GCNSubtarget &ST); |
| 281 | |
| 282 | const SIRegisterInfo &getRegisterInfo() const { |
| 283 | return RI; |
| 284 | } |
| 285 | |
| 286 | // FIXME: This is inaccurate and needs to account for use context. Normal asm |
| 287 | // constraints should use 64-bit pointers. |
| 288 | const TargetRegisterClass *getInlineAsmMemoryOperandRegClass( |
| 289 | InlineAsm::ConstraintCode C) const override { |
| 290 | switch (C) { |
| 291 | case InlineAsm::ConstraintCode::RF: |
| 292 | case InlineAsm::ConstraintCode::m: |
| 293 | // flat_load/flat_store require a 64-bit VGPR pair for the address. |
| 294 | return &AMDGPU::VReg_64RegClass; |
| 295 | default: |
| 296 | return &AMDGPU::VGPR_32RegClass; |
| 297 | } |
| 298 | } |
| 299 | |
| 300 | const GCNSubtarget &getSubtarget() const { |
| 301 | return ST; |
| 302 | } |
| 303 | |
| 304 | bool isReMaterializableImpl(const MachineInstr &MI) const override; |
| 305 | |
| 306 | bool isIgnorableUse(const MachineInstr &MI, unsigned OpIdx) const override; |
| 307 | |
| 308 | bool isSafeToSink(MachineInstr &MI, MachineBasicBlock *SuccToSinkTo, |
| 309 | MachineCycleInfo *CI) const override; |
| 310 | |
| 311 | bool areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, int64_t &Offset0, |
| 312 | int64_t &Offset1) const override; |
| 313 | |
| 314 | bool isGlobalMemoryObject(const MachineInstr *MI) const override; |
| 315 | |
| 316 | bool getMemOperandsWithOffsetWidth( |
| 317 | const MachineInstr &LdSt, |
| 318 | SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset, |
| 319 | bool &OffsetIsScalable, LocationSize &Width) const final; |
| 320 | |
| 321 | bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1, |
| 322 | int64_t Offset1, bool OffsetIsScalable1, |
| 323 | ArrayRef<const MachineOperand *> BaseOps2, |
| 324 | int64_t Offset2, bool OffsetIsScalable2, |
| 325 | unsigned ClusterSize, |
| 326 | unsigned NumBytes) const override; |
| 327 | |
| 328 | /// Return true if the memory operations described by \p BaseOps1 and |
| 329 | /// \p BaseOps2 (as returned by getMemOperandsWithOffsetWidth) address the |
| 330 | /// same base. Unlike shouldClusterMemOps, this does not apply the |
| 331 | /// MaxMemoryClusterDWords budget, which only bounds pre-RA register pressure. |
| 332 | /// Only the first base operand is compared, unlike |
| 333 | /// memOpsHaveSameBaseOperands which requires all of them to match. |
| 334 | bool memOpsHaveSameBase(ArrayRef<const MachineOperand *> BaseOps1, |
| 335 | ArrayRef<const MachineOperand *> BaseOps2) const; |
| 336 | |
| 337 | bool shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, int64_t Offset0, |
| 338 | int64_t Offset1, unsigned NumLoads) const override; |
| 339 | |
| 340 | void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, |
| 341 | const DebugLoc &DL, Register DestReg, Register SrcReg, |
| 342 | bool KillSrc, bool RenamableDest = false, |
| 343 | bool RenamableSrc = false) const override; |
| 344 | |
| 345 | private: |
| 346 | void storeRegToStackSlotImpl(MachineBasicBlock &MBB, |
| 347 | MachineBasicBlock::iterator MI, Register SrcReg, |
| 348 | bool isKill, int FrameIndex, |
| 349 | const TargetRegisterClass *RC, Register VReg, |
| 350 | MachineInstr::MIFlag Flags, bool NeedsCFI) const; |
| 351 | |
| 352 | public: |
| 353 | void storeRegToStackSlotCFI(MachineBasicBlock &MBB, |
| 354 | MachineBasicBlock::iterator MI, Register SrcReg, |
| 355 | bool isKill, int FrameIndex, |
| 356 | const TargetRegisterClass *RC) const; |
| 357 | |
| 358 | bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg, |
| 359 | int64_t &ImmVal) const override; |
| 360 | |
| 361 | std::optional<int64_t> |
| 362 | getImmOrMaterializedImm(const MachineRegisterInfo &MRI, |
| 363 | const MachineOperand &Op, |
| 364 | MachineInstr **DefMI = nullptr) const; |
| 365 | std::optional<int64_t> |
| 366 | getImmOrMaterializedImm(const MachineRegisterInfo &MRI, Register Reg, |
| 367 | MachineInstr **DefMI = nullptr) const; |
| 368 | |
| 369 | unsigned getVectorRegSpillSaveOpcode(Register Reg, |
| 370 | const TargetRegisterClass *RC, |
| 371 | unsigned Size, |
| 372 | const SIMachineFunctionInfo &MFI, |
| 373 | bool NeedsCFI) const; |
| 374 | unsigned |
| 375 | getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC, |
| 376 | unsigned Size, |
| 377 | const SIMachineFunctionInfo &MFI) const; |
| 378 | |
| 379 | void storeRegToStackSlot( |
| 380 | MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, |
| 381 | bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg, |
| 382 | MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override; |
| 383 | |
| 384 | void loadRegFromStackSlot( |
| 385 | MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, |
| 386 | int FrameIndex, const TargetRegisterClass *RC, Register VReg, |
| 387 | unsigned SubReg = 0, |
| 388 | MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override; |
| 389 | |
| 390 | bool expandPostRAPseudo(MachineInstr &MI) const override; |
| 391 | |
| 392 | void |
| 393 | reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, |
| 394 | Register DestReg, unsigned SubIdx, const MachineInstr &Orig, |
| 395 | LaneBitmask UsedLanes = LaneBitmask::getAll()) const override; |
| 396 | |
| 397 | // Splits a V_MOV_B64_DPP_PSEUDO opcode into a pair of v_mov_b32_dpp |
| 398 | // instructions. Returns a pair of generated instructions. |
| 399 | // Can split either post-RA with physical registers or pre-RA with |
| 400 | // virtual registers. In latter case IR needs to be in SSA form and |
| 401 | // and a REG_SEQUENCE is produced to define original register. |
| 402 | std::pair<MachineInstr*, MachineInstr*> |
| 403 | expandMovDPP64(MachineInstr &MI) const; |
| 404 | |
| 405 | // Returns an opcode that can be used to move a value to a \p DstRC |
| 406 | // register. If there is no hardware instruction that can store to \p |
| 407 | // DstRC, then AMDGPU::COPY is returned. |
| 408 | unsigned getMovOpcode(const TargetRegisterClass *DstRC) const; |
| 409 | |
| 410 | const MCInstrDesc &getIndirectRegWriteMovRelPseudo(unsigned VecSize, |
| 411 | unsigned EltSize, |
| 412 | bool IsSGPR) const; |
| 413 | |
| 414 | const MCInstrDesc &getIndirectGPRIDXPseudo(unsigned VecSize, |
| 415 | bool IsIndirectSrc) const; |
| 416 | LLVM_READONLY |
| 417 | int commuteOpcode(unsigned Opc) const; |
| 418 | |
| 419 | LLVM_READONLY |
| 420 | inline int commuteOpcode(const MachineInstr &MI) const { |
| 421 | return commuteOpcode(Opc: MI.getOpcode()); |
| 422 | } |
| 423 | |
| 424 | bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx0, |
| 425 | unsigned &SrcOpIdx1) const override; |
| 426 | |
| 427 | bool findCommutedOpIndices(const MCInstrDesc &Desc, unsigned &SrcOpIdx0, |
| 428 | unsigned &SrcOpIdx1) const; |
| 429 | |
| 430 | bool isBranchOffsetInRange(unsigned BranchOpc, |
| 431 | int64_t BrOffset) const override; |
| 432 | |
| 433 | MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override; |
| 434 | |
| 435 | /// Return whether the block terminate with divergent branch. |
| 436 | /// Note this only work before lowering the pseudo control flow instructions. |
| 437 | bool hasDivergentBranch(const MachineBasicBlock *MBB) const; |
| 438 | |
| 439 | void insertIndirectBranch(MachineBasicBlock &MBB, |
| 440 | MachineBasicBlock &NewDestBB, |
| 441 | MachineBasicBlock &RestoreBB, const DebugLoc &DL, |
| 442 | int64_t BrOffset, RegScavenger *RS) const override; |
| 443 | |
| 444 | bool analyzeBranchImpl(MachineBasicBlock &MBB, |
| 445 | MachineBasicBlock::iterator I, |
| 446 | MachineBasicBlock *&TBB, |
| 447 | MachineBasicBlock *&FBB, |
| 448 | SmallVectorImpl<MachineOperand> &Cond, |
| 449 | bool AllowModify) const; |
| 450 | |
| 451 | bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, |
| 452 | MachineBasicBlock *&FBB, |
| 453 | SmallVectorImpl<MachineOperand> &Cond, |
| 454 | bool AllowModify = false) const override; |
| 455 | |
| 456 | unsigned removeBranch(MachineBasicBlock &MBB, |
| 457 | int *BytesRemoved = nullptr) const override; |
| 458 | |
| 459 | unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, |
| 460 | MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond, |
| 461 | const DebugLoc &DL, |
| 462 | int *BytesAdded = nullptr) const override; |
| 463 | |
| 464 | bool reverseBranchCondition( |
| 465 | SmallVectorImpl<MachineOperand> &Cond) const override; |
| 466 | |
| 467 | std::unique_ptr<PipelinerLoopInfo> |
| 468 | analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override; |
| 469 | |
| 470 | bool canInsertSelect(const MachineBasicBlock &MBB, |
| 471 | ArrayRef<MachineOperand> Cond, Register DstReg, |
| 472 | Register TrueReg, Register FalseReg, |
| 473 | SelectExpansion &Exp) const override; |
| 474 | |
| 475 | void insertSelect(MachineBasicBlock &MBB, |
| 476 | MachineBasicBlock::iterator I, const DebugLoc &DL, |
| 477 | Register DstReg, ArrayRef<MachineOperand> Cond, |
| 478 | Register TrueReg, Register FalseReg) const override; |
| 479 | |
| 480 | bool analyzeCompare(const MachineInstr &MI, Register &SrcReg, |
| 481 | Register &SrcReg2, int64_t &CmpMask, |
| 482 | int64_t &CmpValue) const override; |
| 483 | |
| 484 | bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, |
| 485 | Register SrcReg2, int64_t CmpMask, int64_t CmpValue, |
| 486 | const MachineRegisterInfo *MRI) const override; |
| 487 | |
| 488 | bool |
| 489 | areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, |
| 490 | const MachineInstr &MIb) const override; |
| 491 | |
| 492 | static bool isFoldableCopy(const MachineInstr &MI); |
| 493 | static unsigned getFoldableCopySrcIdx(const MachineInstr &MI); |
| 494 | |
| 495 | void removeModOperands(MachineInstr &MI) const; |
| 496 | |
| 497 | void mutateAndCleanupImplicit(MachineInstr &MI, |
| 498 | const MCInstrDesc &NewDesc) const; |
| 499 | |
| 500 | /// Return the extracted immediate value in a subregister use from a constant |
| 501 | /// materialized in a super register. |
| 502 | /// |
| 503 | /// e.g. %imm = S_MOV_B64 K[0:63] |
| 504 | /// USE %imm.sub1 |
| 505 | /// This will return K[32:63] |
| 506 | static std::optional<int64_t> (int64_t ImmVal, |
| 507 | unsigned SubRegIndex); |
| 508 | |
| 509 | /// Return the target flags for a global address operand reading subregister |
| 510 | /// \p SubRegIndex of a 64-bit global address with \p TargetFlags. |
| 511 | /// e.g. %addr = S_MOV_B64 target-flags(amdgpu-abs64) @gv |
| 512 | /// USE %addr.sub1 |
| 513 | /// This will return amdgpu-abs32-hi |
| 514 | static std::optional<unsigned> |
| 515 | (unsigned TargetFlags, |
| 516 | unsigned SubRegIndex); |
| 517 | |
| 518 | bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg, |
| 519 | MachineRegisterInfo *MRI) const final; |
| 520 | |
| 521 | unsigned getMachineCSELookAheadLimit() const override { return 500; } |
| 522 | |
| 523 | MachineInstr *convertToThreeAddress(MachineInstr &MI, |
| 524 | LiveIntervals *LIS) const override; |
| 525 | |
| 526 | bool isSchedulingBoundary(const MachineInstr &MI, |
| 527 | const MachineBasicBlock *MBB, |
| 528 | const MachineFunction &MF) const override; |
| 529 | |
| 530 | static bool isSALU(const MachineInstr &MI) { |
| 531 | return SIInstrFlags::isSALU(O: MI); |
| 532 | } |
| 533 | |
| 534 | bool isSALU(uint32_t Opcode) const { |
| 535 | return SIInstrFlags::isSALU(O: get(Opcode)); |
| 536 | } |
| 537 | |
| 538 | static bool isVALU(const MachineInstr &MI, bool AllowLDSDMA) { |
| 539 | if (!AllowLDSDMA && isLDSDMA(MI)) |
| 540 | return false; |
| 541 | |
| 542 | return SIInstrFlags::isVALU(O: MI); |
| 543 | } |
| 544 | |
| 545 | /// LDSDMA instructions act as both VALU and memory instructions, thus |
| 546 | /// we also tag them as VALU. However, in many places, we do not actually want |
| 547 | /// to include LDSDMA instructions in this query. By setting \p AllowLDSDMA to |
| 548 | /// false, this will return false for LDSDMA instructions. |
| 549 | bool isVALU(uint32_t Opcode, bool AllowLDSDMA) const { |
| 550 | if (!AllowLDSDMA && isLDSDMA(Opcode)) |
| 551 | return false; |
| 552 | |
| 553 | return SIInstrFlags::isVALU(O: get(Opcode)); |
| 554 | } |
| 555 | |
| 556 | static bool isImage(const MachineInstr &MI) { |
| 557 | return SIInstrFlags::isImage(O: MI); |
| 558 | } |
| 559 | |
| 560 | bool isImage(uint32_t Opcode) const { |
| 561 | return SIInstrFlags::isImage(O: get(Opcode)); |
| 562 | } |
| 563 | |
| 564 | static bool isVMEM(const MachineInstr &MI) { |
| 565 | return SIInstrFlags::isVMEM(O: MI); |
| 566 | } |
| 567 | |
| 568 | bool isVMEM(uint32_t Opcode) const { |
| 569 | return SIInstrFlags::isVMEM(O: get(Opcode)); |
| 570 | } |
| 571 | |
| 572 | /// True if MI implicitly drains XCNT. |
| 573 | static bool isXcntDrain(const MachineInstr &MI); |
| 574 | |
| 575 | static bool isSOP1(const MachineInstr &MI) { |
| 576 | return SIInstrFlags::isSOP1(O: MI); |
| 577 | } |
| 578 | |
| 579 | bool isSOP1(uint32_t Opcode) const { |
| 580 | return SIInstrFlags::isSOP1(O: get(Opcode)); |
| 581 | } |
| 582 | |
| 583 | static bool isSOP2(const MachineInstr &MI) { |
| 584 | return SIInstrFlags::isSOP2(O: MI); |
| 585 | } |
| 586 | |
| 587 | bool isSOP2(uint32_t Opcode) const { |
| 588 | return SIInstrFlags::isSOP2(O: get(Opcode)); |
| 589 | } |
| 590 | |
| 591 | static bool isSOPC(const MachineInstr &MI) { |
| 592 | return SIInstrFlags::isSOPC(O: MI); |
| 593 | } |
| 594 | |
| 595 | bool isSOPC(uint32_t Opcode) const { |
| 596 | return SIInstrFlags::isSOPC(O: get(Opcode)); |
| 597 | } |
| 598 | |
| 599 | static bool isSOPK(const MachineInstr &MI) { |
| 600 | return SIInstrFlags::isSOPK(O: MI); |
| 601 | } |
| 602 | |
| 603 | bool isSOPK(uint32_t Opcode) const { |
| 604 | return SIInstrFlags::isSOPK(O: get(Opcode)); |
| 605 | } |
| 606 | |
| 607 | static bool isSOPP(const MachineInstr &MI) { |
| 608 | return SIInstrFlags::isSOPP(O: MI); |
| 609 | } |
| 610 | |
| 611 | bool isSOPP(uint32_t Opcode) const { |
| 612 | return SIInstrFlags::isSOPP(O: get(Opcode)); |
| 613 | } |
| 614 | |
| 615 | static bool isPacked(const MachineInstr &MI) { |
| 616 | return SIInstrFlags::isPacked(O: MI); |
| 617 | } |
| 618 | |
| 619 | bool isPacked(uint32_t Opcode) const { |
| 620 | return SIInstrFlags::isPacked(O: get(Opcode)); |
| 621 | } |
| 622 | |
| 623 | static bool isVOP1(const MachineInstr &MI) { |
| 624 | return SIInstrFlags::isVOP1(O: MI); |
| 625 | } |
| 626 | |
| 627 | bool isVOP1(uint32_t Opcode) const { |
| 628 | return SIInstrFlags::isVOP1(O: get(Opcode)); |
| 629 | } |
| 630 | |
| 631 | static bool isVOP2(const MachineInstr &MI) { |
| 632 | return SIInstrFlags::isVOP2(O: MI); |
| 633 | } |
| 634 | |
| 635 | bool isVOP2(uint32_t Opcode) const { |
| 636 | return SIInstrFlags::isVOP2(O: get(Opcode)); |
| 637 | } |
| 638 | |
| 639 | static bool isVOP3(const MCInstrDesc &Desc) { |
| 640 | return SIInstrFlags::isVOP3(O: Desc); |
| 641 | } |
| 642 | |
| 643 | static bool isVOP3(const MachineInstr &MI) { return isVOP3(Desc: MI.getDesc()); } |
| 644 | |
| 645 | bool isVOP3(uint32_t Opcode) const { return isVOP3(Desc: get(Opcode)); } |
| 646 | |
| 647 | static bool isSDWA(const MachineInstr &MI) { |
| 648 | return SIInstrFlags::isSDWA(O: MI); |
| 649 | } |
| 650 | |
| 651 | bool isSDWA(uint32_t Opcode) const { |
| 652 | return SIInstrFlags::isSDWA(O: get(Opcode)); |
| 653 | } |
| 654 | |
| 655 | static bool isVOPC(const MachineInstr &MI) { |
| 656 | return SIInstrFlags::isVOPC(O: MI); |
| 657 | } |
| 658 | |
| 659 | bool isVOPC(uint32_t Opcode) const { |
| 660 | return SIInstrFlags::isVOPC(O: get(Opcode)); |
| 661 | } |
| 662 | |
| 663 | static bool isMUBUF(const MachineInstr &MI) { |
| 664 | return SIInstrFlags::isMUBUF(O: MI); |
| 665 | } |
| 666 | |
| 667 | bool isMUBUF(uint32_t Opcode) const { |
| 668 | return SIInstrFlags::isMUBUF(O: get(Opcode)); |
| 669 | } |
| 670 | |
| 671 | static bool isMTBUF(const MachineInstr &MI) { |
| 672 | return SIInstrFlags::isMTBUF(O: MI); |
| 673 | } |
| 674 | |
| 675 | bool isMTBUF(uint32_t Opcode) const { |
| 676 | return SIInstrFlags::isMTBUF(O: get(Opcode)); |
| 677 | } |
| 678 | |
| 679 | static bool isBUF(const MachineInstr &MI) { |
| 680 | return isMUBUF(MI) || isMTBUF(MI); |
| 681 | } |
| 682 | |
| 683 | static bool isSMRD(const MachineInstr &MI) { |
| 684 | return SIInstrFlags::isSMRD(O: MI); |
| 685 | } |
| 686 | |
| 687 | bool isSMRD(uint32_t Opcode) const { |
| 688 | return SIInstrFlags::isSMRD(O: get(Opcode)); |
| 689 | } |
| 690 | |
| 691 | bool isBufferSMRD(const MachineInstr &MI) const; |
| 692 | |
| 693 | static bool isDS(const MachineInstr &MI) { return SIInstrFlags::isDS(O: MI); } |
| 694 | |
| 695 | bool isDS(uint32_t Opcode) const { return SIInstrFlags::isDS(O: get(Opcode)); } |
| 696 | |
| 697 | static bool isLDSDMA(const MachineInstr &MI) { |
| 698 | return (SIInstrFlags::isVALU(O: MI) && (isMUBUF(MI) || isFLAT(MI))) || |
| 699 | SIInstrFlags::usesTENSOR_CNT(O: MI); |
| 700 | } |
| 701 | |
| 702 | bool isLDSDMA(uint32_t Opcode) const { |
| 703 | return (SIInstrFlags::isVALU(O: get(Opcode)) && |
| 704 | (isMUBUF(Opcode) || isFLAT(Opcode))) || |
| 705 | SIInstrFlags::usesTENSOR_CNT(O: get(Opcode)); |
| 706 | } |
| 707 | |
| 708 | static bool isGWS(const MachineInstr &MI) { return SIInstrFlags::isGWS(O: MI); } |
| 709 | |
| 710 | bool isGWS(uint32_t Opcode) const { return SIInstrFlags::isGWS(O: get(Opcode)); } |
| 711 | |
| 712 | bool isAlwaysGDS(uint32_t Opcode) const; |
| 713 | |
| 714 | static bool isMIMG(const MachineInstr &MI) { |
| 715 | return SIInstrFlags::isMIMG(O: MI); |
| 716 | } |
| 717 | |
| 718 | bool isMIMG(uint32_t Opcode) const { |
| 719 | return SIInstrFlags::isMIMG(O: get(Opcode)); |
| 720 | } |
| 721 | |
| 722 | static bool isVIMAGE(const MachineInstr &MI) { |
| 723 | return SIInstrFlags::isVIMAGE(O: MI); |
| 724 | } |
| 725 | |
| 726 | bool isVIMAGE(uint32_t Opcode) const { |
| 727 | return SIInstrFlags::isVIMAGE(O: get(Opcode)); |
| 728 | } |
| 729 | |
| 730 | static bool isVSAMPLE(const MachineInstr &MI) { |
| 731 | return SIInstrFlags::isVSAMPLE(O: MI); |
| 732 | } |
| 733 | |
| 734 | bool isVSAMPLE(uint32_t Opcode) const { |
| 735 | return SIInstrFlags::isVSAMPLE(O: get(Opcode)); |
| 736 | } |
| 737 | |
| 738 | static bool isGather4(const MachineInstr &MI) { |
| 739 | return SIInstrFlags::isGather4(O: MI); |
| 740 | } |
| 741 | |
| 742 | bool isGather4(uint32_t Opcode) const { |
| 743 | return SIInstrFlags::isGather4(O: get(Opcode)); |
| 744 | } |
| 745 | |
| 746 | static bool isFLAT(const MachineInstr &MI) { |
| 747 | return SIInstrFlags::isFLAT(O: MI); |
| 748 | } |
| 749 | |
| 750 | // Is a FLAT encoded instruction which accesses a specific segment, |
| 751 | // i.e. global_* or scratch_*. |
| 752 | static bool isSegmentSpecificFLAT(const MachineInstr &MI) { |
| 753 | return SIInstrFlags::isSegmentSpecificFLAT(O: MI); |
| 754 | } |
| 755 | |
| 756 | bool isSegmentSpecificFLAT(uint32_t Opcode) const { |
| 757 | return SIInstrFlags::isSegmentSpecificFLAT(O: get(Opcode)); |
| 758 | } |
| 759 | |
| 760 | static bool isFLATGlobal(const MachineInstr &MI) { |
| 761 | return SIInstrFlags::isFlatGlobal(O: MI); |
| 762 | } |
| 763 | |
| 764 | bool isFLATGlobal(uint32_t Opcode) const { |
| 765 | return SIInstrFlags::isFlatGlobal(O: get(Opcode)); |
| 766 | } |
| 767 | |
| 768 | static bool isFLATScratch(const MachineInstr &MI) { |
| 769 | return SIInstrFlags::isFlatScratch(O: MI); |
| 770 | } |
| 771 | |
| 772 | bool isFLATScratch(uint32_t Opcode) const { |
| 773 | return SIInstrFlags::isFlatScratch(O: get(Opcode)); |
| 774 | } |
| 775 | |
| 776 | // Any FLAT encoded instruction, including global_* and scratch_*. |
| 777 | bool isFLAT(uint32_t Opcode) const { |
| 778 | return SIInstrFlags::isFLAT(O: get(Opcode)); |
| 779 | } |
| 780 | |
| 781 | /// \returns true for SCRATCH_ instructions, or FLAT/BUF instructions unless |
| 782 | /// the MMOs do not include scratch. |
| 783 | /// Conservatively correct; will return true if \p MI cannot be proven |
| 784 | /// to not hit scratch. |
| 785 | bool mayAccessScratch(const MachineInstr &MI) const; |
| 786 | |
| 787 | /// \returns true for FLAT instructions that can access VMEM. |
| 788 | bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const; |
| 789 | |
| 790 | /// \returns true for FLAT instructions that can access LDS. |
| 791 | bool mayAccessLDSThroughFlat(const MachineInstr &MI, bool TgSplit) const; |
| 792 | |
| 793 | static bool isBlockLoadStore(uint32_t Opcode) { |
| 794 | switch (Opcode) { |
| 795 | case AMDGPU::SI_BLOCK_SPILL_V1024_SAVE: |
| 796 | case AMDGPU::SI_BLOCK_SPILL_V1024_CFI_SAVE: |
| 797 | case AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE: |
| 798 | case AMDGPU::SCRATCH_STORE_BLOCK_SADDR: |
| 799 | case AMDGPU::SCRATCH_LOAD_BLOCK_SADDR: |
| 800 | case AMDGPU::SCRATCH_STORE_BLOCK_SVS: |
| 801 | case AMDGPU::SCRATCH_LOAD_BLOCK_SVS: |
| 802 | return true; |
| 803 | default: |
| 804 | return false; |
| 805 | } |
| 806 | } |
| 807 | |
| 808 | static bool setsSCCIfResultIsNonZero(const MachineInstr &MI) { |
| 809 | switch (MI.getOpcode()) { |
| 810 | case AMDGPU::S_ABSDIFF_I32: |
| 811 | case AMDGPU::S_ABS_I32: |
| 812 | case AMDGPU::S_AND_B32: |
| 813 | case AMDGPU::S_AND_B64: |
| 814 | case AMDGPU::S_ANDN2_B32: |
| 815 | case AMDGPU::S_ANDN2_B64: |
| 816 | case AMDGPU::S_ASHR_I32: |
| 817 | case AMDGPU::S_ASHR_I64: |
| 818 | case AMDGPU::S_BCNT0_I32_B32: |
| 819 | case AMDGPU::S_BCNT0_I32_B64: |
| 820 | case AMDGPU::S_BCNT1_I32_B32: |
| 821 | case AMDGPU::S_BCNT1_I32_B64: |
| 822 | case AMDGPU::S_BFE_I32: |
| 823 | case AMDGPU::S_BFE_I64: |
| 824 | case AMDGPU::S_BFE_U32: |
| 825 | case AMDGPU::S_BFE_U64: |
| 826 | case AMDGPU::S_LSHL_B32: |
| 827 | case AMDGPU::S_LSHL_B64: |
| 828 | case AMDGPU::S_LSHR_B32: |
| 829 | case AMDGPU::S_LSHR_B64: |
| 830 | case AMDGPU::S_NAND_B32: |
| 831 | case AMDGPU::S_NAND_B64: |
| 832 | case AMDGPU::S_NOR_B32: |
| 833 | case AMDGPU::S_NOR_B64: |
| 834 | case AMDGPU::S_NOT_B32: |
| 835 | case AMDGPU::S_NOT_B64: |
| 836 | case AMDGPU::S_OR_B32: |
| 837 | case AMDGPU::S_OR_B64: |
| 838 | case AMDGPU::S_ORN2_B32: |
| 839 | case AMDGPU::S_ORN2_B64: |
| 840 | case AMDGPU::S_QUADMASK_B32: |
| 841 | case AMDGPU::S_QUADMASK_B64: |
| 842 | case AMDGPU::S_WQM_B32: |
| 843 | case AMDGPU::S_WQM_B64: |
| 844 | case AMDGPU::S_XNOR_B32: |
| 845 | case AMDGPU::S_XNOR_B64: |
| 846 | case AMDGPU::S_XOR_B32: |
| 847 | case AMDGPU::S_XOR_B64: |
| 848 | return true; |
| 849 | default: |
| 850 | return false; |
| 851 | } |
| 852 | } |
| 853 | |
| 854 | static bool isEXP(const MachineInstr &MI) { return SIInstrFlags::isEXP(O: MI); } |
| 855 | |
| 856 | static bool isDualSourceBlendEXP(const MachineInstr &MI) { |
| 857 | if (!isEXP(MI)) |
| 858 | return false; |
| 859 | unsigned Target = MI.getOperand(i: 0).getImm(); |
| 860 | return Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND0 || |
| 861 | Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND1; |
| 862 | } |
| 863 | |
| 864 | bool isEXP(uint32_t Opcode) const { return SIInstrFlags::isEXP(O: get(Opcode)); } |
| 865 | |
| 866 | static bool isAtomicNoRet(const MachineInstr &MI) { |
| 867 | return SIInstrFlags::isAtomicNoRet(O: MI); |
| 868 | } |
| 869 | |
| 870 | bool isAtomicNoRet(uint32_t Opcode) const { |
| 871 | return SIInstrFlags::isAtomicNoRet(O: get(Opcode)); |
| 872 | } |
| 873 | |
| 874 | static bool isAtomicRet(const MachineInstr &MI) { |
| 875 | return SIInstrFlags::isAtomicRet(O: MI); |
| 876 | } |
| 877 | |
| 878 | bool isAtomicRet(uint32_t Opcode) const { |
| 879 | return SIInstrFlags::isAtomicRet(O: get(Opcode)); |
| 880 | } |
| 881 | |
| 882 | static bool isAtomic(const MachineInstr &MI) { |
| 883 | return SIInstrFlags::isAtomic(O: MI); |
| 884 | } |
| 885 | |
| 886 | bool isAtomic(uint32_t Opcode) const { |
| 887 | return SIInstrFlags::isAtomic(O: get(Opcode)); |
| 888 | } |
| 889 | |
| 890 | static bool mayWriteLDSThroughDMA(const MachineInstr &MI) { |
| 891 | unsigned Opc = MI.getOpcode(); |
| 892 | // Exclude instructions that read FROM LDS (not write to it) |
| 893 | return isLDSDMA(MI) && Opc != AMDGPU::BUFFER_STORE_LDS_DWORD && |
| 894 | Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d2 && |
| 895 | Opc != AMDGPU::TENSOR_STORE_FROM_LDS_d4; |
| 896 | } |
| 897 | |
| 898 | bool isNonAsyncLdsDmaWrite(const MachineInstr &MI) const { |
| 899 | return mayWriteLDSThroughDMA(MI) && !isAsync(MI); |
| 900 | } |
| 901 | |
| 902 | /// \returns true if \p MI creates an async operation. |
| 903 | bool isAsync(const MachineInstr &MI) const { |
| 904 | if (!isLDSDMA(MI)) |
| 905 | return false; |
| 906 | if (usesASYNC_CNT(MI) || usesTENSOR_CNT(MI)) |
| 907 | return true; |
| 908 | const MachineOperand *Async = getNamedOperand(MI, OperandName: AMDGPU::OpName::IsAsync); |
| 909 | return Async && (Async->getImm()); |
| 910 | } |
| 911 | |
| 912 | /// \returns the asyncmark stage that the async operation initiated by \p MI |
| 913 | /// belongs to, or no stage if \p MI does not initiate an async operation. |
| 914 | AMDGPU::AsyncStages getAsyncStage(const MachineInstr &MI) const; |
| 915 | |
| 916 | static bool isSBarrierSCCWrite(unsigned Opcode) { |
| 917 | return Opcode == AMDGPU::S_BARRIER_LEAVE || |
| 918 | Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM || |
| 919 | Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0; |
| 920 | } |
| 921 | |
| 922 | static bool isCBranchVCCZRead(const MachineInstr &MI) { |
| 923 | unsigned Opc = MI.getOpcode(); |
| 924 | return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) && |
| 925 | !MI.getOperand(i: 1).isUndef(); |
| 926 | } |
| 927 | |
| 928 | static bool isWQM(const MachineInstr &MI) { return SIInstrFlags::isWQM(O: MI); } |
| 929 | |
| 930 | bool isWQM(uint32_t Opcode) const { return SIInstrFlags::isWQM(O: get(Opcode)); } |
| 931 | |
| 932 | static bool isDisableWQM(const MachineInstr &MI) { |
| 933 | return SIInstrFlags::isDisableWQM(O: MI); |
| 934 | } |
| 935 | |
| 936 | bool isDisableWQM(uint32_t Opcode) const { |
| 937 | return SIInstrFlags::isDisableWQM(O: get(Opcode)); |
| 938 | } |
| 939 | |
| 940 | // SI_SPILL_S32_TO_VGPR and SI_RESTORE_S32_FROM_VGPR form a special case of |
| 941 | // SGPRs spilling to VGPRs which are SGPR spills but from VALU instructions |
| 942 | // therefore we need an explicit check for them since just checking if the |
| 943 | // Spill bit is set and what instruction type it came from misclassifies |
| 944 | // them. |
| 945 | static bool isVGPRSpill(const MachineInstr &MI) { |
| 946 | return MI.getOpcode() != AMDGPU::SI_SPILL_S32_TO_VGPR && |
| 947 | MI.getOpcode() != AMDGPU::SI_RESTORE_S32_FROM_VGPR && |
| 948 | (isSpill(MI) && isVALU(MI, /*AllowLDSDMA=*/AllowLDSDMA: false)); |
| 949 | } |
| 950 | |
| 951 | bool isVGPRSpill(uint32_t Opcode) const { |
| 952 | return Opcode != AMDGPU::SI_SPILL_S32_TO_VGPR && |
| 953 | Opcode != AMDGPU::SI_RESTORE_S32_FROM_VGPR && |
| 954 | (isSpill(Opcode) && isVALU(Opcode, /*AllowLDSDMA=*/AllowLDSDMA: false)); |
| 955 | } |
| 956 | |
| 957 | static bool isSGPRSpill(const MachineInstr &MI) { |
| 958 | return MI.getOpcode() == AMDGPU::SI_SPILL_S32_TO_VGPR || |
| 959 | MI.getOpcode() == AMDGPU::SI_RESTORE_S32_FROM_VGPR || |
| 960 | (isSpill(MI) && isSALU(MI)); |
| 961 | } |
| 962 | |
| 963 | bool isSGPRSpill(uint32_t Opcode) const { |
| 964 | return Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR || |
| 965 | Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR || |
| 966 | (isSpill(Opcode) && isSALU(Opcode)); |
| 967 | } |
| 968 | |
| 969 | bool isSpill(uint32_t Opcode) const { |
| 970 | return SIInstrFlags::isSpill(O: get(Opcode)); |
| 971 | } |
| 972 | |
| 973 | static bool isSpill(const MCInstrDesc &Desc) { |
| 974 | return SIInstrFlags::isSpill(O: Desc); |
| 975 | } |
| 976 | |
| 977 | static bool isSpill(const MachineInstr &MI) { return isSpill(Desc: MI.getDesc()); } |
| 978 | |
| 979 | static bool isWWMRegSpillOpcode(uint32_t Opcode) { |
| 980 | return Opcode == AMDGPU::SI_SPILL_WWM_V32_SAVE || |
| 981 | Opcode == AMDGPU::SI_SPILL_WWM_AV32_SAVE || |
| 982 | Opcode == AMDGPU::SI_SPILL_WWM_V32_RESTORE || |
| 983 | Opcode == AMDGPU::SI_SPILL_WWM_AV32_RESTORE; |
| 984 | } |
| 985 | |
| 986 | static bool isChainCallOpcode(uint64_t Opcode) { |
| 987 | return Opcode == AMDGPU::SI_CS_CHAIN_TC_W32 || |
| 988 | Opcode == AMDGPU::SI_CS_CHAIN_TC_W64; |
| 989 | } |
| 990 | |
| 991 | static bool isDPP(const MachineInstr &MI) { return SIInstrFlags::isDPP(O: MI); } |
| 992 | |
| 993 | bool isDPP(uint32_t Opcode) const { return SIInstrFlags::isDPP(O: get(Opcode)); } |
| 994 | |
| 995 | // Some opcodes use Src1 for DPP instead of Src0, because the sequencer |
| 996 | // transforms them and reverse the order of their operands at runtime. |
| 997 | // |
| 998 | // Documentation is incomplete on which instructions are effected, so |
| 999 | // the implementation is derived from experimentation. |
| 1000 | // |
| 1001 | // Listed as target-independent pseudos; the per-subtarget MC opcodes |
| 1002 | // (V_SUBREV_NC_U32_e32_gfx11 and friends) are all reached through these. |
| 1003 | // Defined out of line because GCNSubtarget is incomplete here. |
| 1004 | static bool isSrc1DPPRevOpcode(const GCNSubtarget &ST, uint32_t Opcode); |
| 1005 | |
| 1006 | static bool isTRANS(const MachineInstr &MI) { |
| 1007 | return SIInstrFlags::isTRANS(O: MI); |
| 1008 | } |
| 1009 | |
| 1010 | bool isTRANS(uint32_t Opcode) const { |
| 1011 | return SIInstrFlags::isTRANS(O: get(Opcode)); |
| 1012 | } |
| 1013 | |
| 1014 | static bool isVOP3P(const MachineInstr &MI) { |
| 1015 | return SIInstrFlags::isVOP3P(O: MI); |
| 1016 | } |
| 1017 | |
| 1018 | bool isVOP3P(uint32_t Opcode) const { |
| 1019 | return SIInstrFlags::isVOP3P(O: get(Opcode)); |
| 1020 | } |
| 1021 | |
| 1022 | bool isVOP3PMix(const MachineInstr &MI) const { |
| 1023 | return isVOP3PMix(Opcode: MI.getOpcode()); |
| 1024 | } |
| 1025 | |
| 1026 | bool isVOP3PMix(uint16_t Opcode) const { |
| 1027 | switch (Opcode) { |
| 1028 | case AMDGPU::V_FMA_MIXHI_F16: |
| 1029 | case AMDGPU::V_FMA_MIXLO_F16: |
| 1030 | case AMDGPU::V_FMA_MIX_F32: |
| 1031 | case AMDGPU::V_MAD_MIXHI_F16: |
| 1032 | case AMDGPU::V_MAD_MIXLO_F16: |
| 1033 | case AMDGPU::V_MAD_MIX_F32: |
| 1034 | return true; |
| 1035 | default: |
| 1036 | return false; |
| 1037 | } |
| 1038 | } |
| 1039 | |
| 1040 | static bool isVINTRP(const MachineInstr &MI) { |
| 1041 | return SIInstrFlags::isVINTRP(O: MI); |
| 1042 | } |
| 1043 | |
| 1044 | bool isVINTRP(uint32_t Opcode) const { |
| 1045 | return SIInstrFlags::isVINTRP(O: get(Opcode)); |
| 1046 | } |
| 1047 | |
| 1048 | static bool isMAI(const MCInstrDesc &Desc) { |
| 1049 | return SIInstrFlags::isMAI(O: Desc); |
| 1050 | } |
| 1051 | |
| 1052 | static bool isMAI(const MachineInstr &MI) { return isMAI(Desc: MI.getDesc()); } |
| 1053 | |
| 1054 | bool isMAI(uint32_t Opcode) const { return isMAI(Desc: get(Opcode)); } |
| 1055 | |
| 1056 | static bool isMFMA(const MachineInstr &MI) { |
| 1057 | return isMAI(MI) && MI.getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && |
| 1058 | MI.getOpcode() != AMDGPU::V_ACCVGPR_READ_B32_e64; |
| 1059 | } |
| 1060 | |
| 1061 | bool isMFMA(uint32_t Opcode) const { |
| 1062 | return isMAI(Opcode) && Opcode != AMDGPU::V_ACCVGPR_WRITE_B32_e64 && |
| 1063 | Opcode != AMDGPU::V_ACCVGPR_READ_B32_e64; |
| 1064 | } |
| 1065 | |
| 1066 | static bool isDOT(const MachineInstr &MI) { return SIInstrFlags::isDOT(O: MI); } |
| 1067 | |
| 1068 | static bool isWMMA(const MachineInstr &MI) { |
| 1069 | return SIInstrFlags::isWMMA(O: MI); |
| 1070 | } |
| 1071 | |
| 1072 | bool isWMMA(uint32_t Opcode) const { |
| 1073 | return SIInstrFlags::isWMMA(O: get(Opcode)); |
| 1074 | } |
| 1075 | |
| 1076 | static bool isMFMAorWMMA(const MachineInstr &MI) { |
| 1077 | return isMFMA(MI) || isWMMA(MI) || isSWMMAC(MI); |
| 1078 | } |
| 1079 | |
| 1080 | bool isMFMAorWMMA(uint32_t Opcode) const { |
| 1081 | return isMFMA(Opcode) || isWMMA(Opcode) || isSWMMAC(Opcode); |
| 1082 | } |
| 1083 | |
| 1084 | static bool isSWMMAC(const MachineInstr &MI) { |
| 1085 | return SIInstrFlags::isSWMMAC(O: MI); |
| 1086 | } |
| 1087 | |
| 1088 | bool isSWMMAC(uint32_t Opcode) const { |
| 1089 | return SIInstrFlags::isSWMMAC(O: get(Opcode)); |
| 1090 | } |
| 1091 | |
| 1092 | bool isDOT(uint32_t Opcode) const { return SIInstrFlags::isDOT(O: get(Opcode)); } |
| 1093 | |
| 1094 | bool isXDLWMMA(const MachineInstr &MI) const; |
| 1095 | |
| 1096 | bool isXDL(const MachineInstr &MI) const; |
| 1097 | |
| 1098 | /// On gfx1250, an XDL instruction whose destination spans at least eight |
| 1099 | /// VGPRs decrements VA_VDST only after every earlier-issued VALU instruction, |
| 1100 | /// of any type, has decremented it. Nothing is implied about the order |
| 1101 | /// relative to instructions issued after it. |
| 1102 | bool isVAVDSTOrderedXDL(const MachineInstr &MI) const; |
| 1103 | |
| 1104 | static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opc: Opcode); } |
| 1105 | |
| 1106 | static bool isLDSDIR(const MachineInstr &MI) { |
| 1107 | return SIInstrFlags::isLDSDIR(O: MI); |
| 1108 | } |
| 1109 | |
| 1110 | bool isLDSDIR(uint32_t Opcode) const { |
| 1111 | return SIInstrFlags::isLDSDIR(O: get(Opcode)); |
| 1112 | } |
| 1113 | |
| 1114 | static bool isVINTERP(const MachineInstr &MI) { |
| 1115 | return SIInstrFlags::isVINTERP(O: MI); |
| 1116 | } |
| 1117 | |
| 1118 | bool isVINTERP(uint32_t Opcode) const { |
| 1119 | return SIInstrFlags::isVINTERP(O: get(Opcode)); |
| 1120 | } |
| 1121 | |
| 1122 | static bool usesVM_CNT(const MachineInstr &MI) { |
| 1123 | return SIInstrFlags::usesVM_CNT(O: MI); |
| 1124 | } |
| 1125 | |
| 1126 | static bool usesLGKM_CNT(const MachineInstr &MI) { |
| 1127 | return SIInstrFlags::usesLGKM_CNT(O: MI); |
| 1128 | } |
| 1129 | |
| 1130 | static bool usesASYNC_CNT(const MachineInstr &MI) { |
| 1131 | return SIInstrFlags::usesASYNC_CNT(O: MI); |
| 1132 | } |
| 1133 | |
| 1134 | bool usesASYNC_CNT(uint32_t Opcode) const { |
| 1135 | return SIInstrFlags::usesASYNC_CNT(O: get(Opcode)); |
| 1136 | } |
| 1137 | |
| 1138 | static bool usesTENSOR_CNT(const MachineInstr &MI) { |
| 1139 | return SIInstrFlags::usesTENSOR_CNT(O: MI); |
| 1140 | } |
| 1141 | |
| 1142 | bool usesTENSOR_CNT(uint32_t Opcode) const { |
| 1143 | return SIInstrFlags::usesTENSOR_CNT(O: get(Opcode)); |
| 1144 | } |
| 1145 | |
| 1146 | // Most sopk treat the immediate as a signed 16-bit, however some |
| 1147 | // use it as unsigned. |
| 1148 | static bool sopkIsZext(unsigned Opcode) { |
| 1149 | return Opcode == AMDGPU::S_CMPK_EQ_U32 || Opcode == AMDGPU::S_CMPK_LG_U32 || |
| 1150 | Opcode == AMDGPU::S_CMPK_GT_U32 || Opcode == AMDGPU::S_CMPK_GE_U32 || |
| 1151 | Opcode == AMDGPU::S_CMPK_LT_U32 || Opcode == AMDGPU::S_CMPK_LE_U32 || |
| 1152 | Opcode == AMDGPU::S_GETREG_B32 || |
| 1153 | Opcode == AMDGPU::S_GETREG_B32_const; |
| 1154 | } |
| 1155 | |
| 1156 | /// \returns true if this is an s_store_dword* instruction. This is more |
| 1157 | /// specific than isSMEM && mayStore. |
| 1158 | static bool isScalarStore(const MachineInstr &MI) { |
| 1159 | return SIInstrFlags::isScalarStore(O: MI); |
| 1160 | } |
| 1161 | |
| 1162 | bool isScalarStore(uint32_t Opcode) const { |
| 1163 | return SIInstrFlags::isScalarStore(O: get(Opcode)); |
| 1164 | } |
| 1165 | |
| 1166 | static bool isFixedSize(const MachineInstr &MI) { |
| 1167 | return SIInstrFlags::isFixedSize(O: MI); |
| 1168 | } |
| 1169 | |
| 1170 | bool isFixedSize(uint32_t Opcode) const { |
| 1171 | return SIInstrFlags::isFixedSize(O: get(Opcode)); |
| 1172 | } |
| 1173 | |
| 1174 | static bool hasFPClamp(const MachineInstr &MI) { |
| 1175 | return SIInstrFlags::hasFPClamp(O: MI); |
| 1176 | } |
| 1177 | |
| 1178 | bool hasFPClamp(uint32_t Opcode) const { |
| 1179 | return SIInstrFlags::hasFPClamp(O: get(Opcode)); |
| 1180 | } |
| 1181 | |
| 1182 | static bool hasIntClamp(const MachineInstr &MI) { |
| 1183 | return SIInstrFlags::hasIntClamp(O: MI); |
| 1184 | } |
| 1185 | |
| 1186 | static bool hasSameClamp(const MachineInstr &A, const MachineInstr &B) { |
| 1187 | const MCInstrDesc &DA = A.getDesc(), &DB = B.getDesc(); |
| 1188 | return SIInstrFlags::hasFPClamp(O: DA) == SIInstrFlags::hasFPClamp(O: DB) && |
| 1189 | SIInstrFlags::hasIntClamp(O: DA) == SIInstrFlags::hasIntClamp(O: DB) && |
| 1190 | SIInstrFlags::hasClampLo(O: DA) == SIInstrFlags::hasClampLo(O: DB) && |
| 1191 | SIInstrFlags::hasClampHi(O: DA) == SIInstrFlags::hasClampHi(O: DB); |
| 1192 | } |
| 1193 | |
| 1194 | static bool usesFPDPRounding(const MachineInstr &MI) { |
| 1195 | return SIInstrFlags::usesFPDPRounding(O: MI); |
| 1196 | } |
| 1197 | |
| 1198 | bool usesFPDPRounding(uint32_t Opcode) const { |
| 1199 | return SIInstrFlags::usesFPDPRounding(O: get(Opcode)); |
| 1200 | } |
| 1201 | |
| 1202 | static bool isFPAtomic(const MachineInstr &MI) { |
| 1203 | return SIInstrFlags::isFPAtomic(O: MI); |
| 1204 | } |
| 1205 | |
| 1206 | bool isFPAtomic(uint32_t Opcode) const { |
| 1207 | return SIInstrFlags::isFPAtomic(O: get(Opcode)); |
| 1208 | } |
| 1209 | |
| 1210 | static bool isNeverUniform(const MachineInstr &MI) { |
| 1211 | return SIInstrFlags::isNeverUniform(O: MI); |
| 1212 | } |
| 1213 | |
| 1214 | // Check to see if opcode is for a barrier start. Pre gfx12 this is just the |
| 1215 | // S_BARRIER, but after support for S_BARRIER_SIGNAL* / S_BARRIER_WAIT we want |
| 1216 | // to check for the barrier start (S_BARRIER_SIGNAL*) |
| 1217 | bool isBarrierStart(unsigned Opcode) const { |
| 1218 | return Opcode == AMDGPU::S_BARRIER || |
| 1219 | Opcode == AMDGPU::S_BARRIER_SIGNAL_M0 || |
| 1220 | Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0 || |
| 1221 | Opcode == AMDGPU::S_BARRIER_SIGNAL_IMM || |
| 1222 | Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM; |
| 1223 | } |
| 1224 | |
| 1225 | bool isBarrier(unsigned Opcode) const { |
| 1226 | return isBarrierStart(Opcode) || Opcode == AMDGPU::S_BARRIER_WAIT || |
| 1227 | Opcode == AMDGPU::S_BARRIER_INIT_M0 || |
| 1228 | Opcode == AMDGPU::S_BARRIER_INIT_IMM || |
| 1229 | Opcode == AMDGPU::S_BARRIER_JOIN_IMM || |
| 1230 | Opcode == AMDGPU::S_BARRIER_LEAVE || Opcode == AMDGPU::DS_GWS_INIT || |
| 1231 | Opcode == AMDGPU::DS_GWS_BARRIER; |
| 1232 | } |
| 1233 | |
| 1234 | static bool isLoadMonitor(unsigned Opc) { |
| 1235 | switch (Opc) { |
| 1236 | case AMDGPU::GLOBAL_LOAD_MONITOR_B32: |
| 1237 | case AMDGPU::GLOBAL_LOAD_MONITOR_B32_SADDR: |
| 1238 | case AMDGPU::GLOBAL_LOAD_MONITOR_B64: |
| 1239 | case AMDGPU::GLOBAL_LOAD_MONITOR_B64_SADDR: |
| 1240 | case AMDGPU::GLOBAL_LOAD_MONITOR_B128: |
| 1241 | case AMDGPU::GLOBAL_LOAD_MONITOR_B128_SADDR: |
| 1242 | case AMDGPU::FLAT_LOAD_MONITOR_B32: |
| 1243 | case AMDGPU::FLAT_LOAD_MONITOR_B64: |
| 1244 | case AMDGPU::FLAT_LOAD_MONITOR_B128: |
| 1245 | return true; |
| 1246 | default: |
| 1247 | return false; |
| 1248 | } |
| 1249 | } |
| 1250 | |
| 1251 | static bool isGFX12CacheInvOrWBInst(unsigned Opc) { |
| 1252 | return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB || |
| 1253 | Opc == AMDGPU::GLOBAL_WBINV; |
| 1254 | } |
| 1255 | |
| 1256 | static bool isF16PseudoScalarTrans(unsigned Opcode) { |
| 1257 | return Opcode == AMDGPU::V_S_EXP_F16_e64 || |
| 1258 | Opcode == AMDGPU::V_S_LOG_F16_e64 || |
| 1259 | Opcode == AMDGPU::V_S_RCP_F16_e64 || |
| 1260 | Opcode == AMDGPU::V_S_RSQ_F16_e64 || |
| 1261 | Opcode == AMDGPU::V_S_SQRT_F16_e64; |
| 1262 | } |
| 1263 | |
| 1264 | static bool isPseudoScalarTrans(unsigned Opcode) { |
| 1265 | return isF16PseudoScalarTrans(Opcode) || |
| 1266 | Opcode == AMDGPU::V_S_EXP_F32_e64 || |
| 1267 | Opcode == AMDGPU::V_S_LOG_F32_e64 || |
| 1268 | Opcode == AMDGPU::V_S_RCP_F32_e64 || |
| 1269 | Opcode == AMDGPU::V_S_RSQ_F32_e64 || |
| 1270 | Opcode == AMDGPU::V_S_SQRT_F32_e64; |
| 1271 | } |
| 1272 | |
| 1273 | static bool isVPermPk16(unsigned Opcode) { |
| 1274 | return Opcode == AMDGPU::V_PERM_PK16_B4_U4_e64 || |
| 1275 | Opcode == AMDGPU::V_PERM_PK16_B6_U4_e64 || |
| 1276 | Opcode == AMDGPU::V_PERM_PK16_B8_U4_e64; |
| 1277 | } |
| 1278 | |
| 1279 | // \returns true if \p MI clears the V_PERM_PK16 hazard when it immediately |
| 1280 | // follows a V_PERM_PK16 (i.e. \p MI is a "safe" instruction). |
| 1281 | bool isVPermPk16SafeInstr(const MachineInstr &MI) const { |
| 1282 | unsigned Opc = MI.getOpcode(); |
| 1283 | |
| 1284 | // Only VALU ops issue on the pipe that clears the V_PERM_PK16 hazard. |
| 1285 | if (!isVALU(MI, /*AllowLDSDMA=*/AllowLDSDMA: false)) |
| 1286 | return false; |
| 1287 | // OP_XDL: matrix (WMMA/SWMMAC/DOT) ops clear the hazard. |
| 1288 | if (isXDL(MI)) |
| 1289 | return true; |
| 1290 | // Pseudo-scalar transcendentals (OP32_SCL_T) do NOT clear the hazard. |
| 1291 | if (isPseudoScalarTrans(Opcode: Opc)) |
| 1292 | return false; |
| 1293 | |
| 1294 | // Use the table lookup, not getBlockingCycles(): occupancy is gated off on |
| 1295 | // gfx1251 but the hazard applies to both gfx1250 and gfx1251. Gfx1250 table |
| 1296 | // is valid enough for gfx1251 w.r.t. v_perm_pk16 safety check here. |
| 1297 | // OP_32_T is in the table at 2 and is safe; other table entries (>= 2) are |
| 1298 | // not. |
| 1299 | unsigned Cycles = getGFX1250BlockingCyclesTable(MI); |
| 1300 | return Cycles < 2 || (Cycles == 2 && isTRANS(MI)); |
| 1301 | } |
| 1302 | |
| 1303 | static bool doesNotReadTiedSource(const MachineInstr &MI) { |
| 1304 | return SIInstrFlags::isTiedSourceNotRead(O: MI); |
| 1305 | } |
| 1306 | |
| 1307 | bool doesNotReadTiedSource(uint32_t Opcode) const { |
| 1308 | return SIInstrFlags::isTiedSourceNotRead(O: get(Opcode)); |
| 1309 | } |
| 1310 | |
| 1311 | bool isIGLP(unsigned Opcode) const { |
| 1312 | return Opcode == AMDGPU::SCHED_BARRIER || |
| 1313 | Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT; |
| 1314 | } |
| 1315 | |
| 1316 | bool isIGLP(const MachineInstr &MI) const { return isIGLP(Opcode: MI.getOpcode()); } |
| 1317 | |
| 1318 | // Return true if the instruction is mutually exclusive with all non-IGLP DAG |
| 1319 | // mutations, requiring all other mutations to be disabled. |
| 1320 | bool isIGLPMutationOnly(unsigned Opcode) const { |
| 1321 | return Opcode == AMDGPU::SCHED_GROUP_BARRIER || Opcode == AMDGPU::IGLP_OPT; |
| 1322 | } |
| 1323 | |
| 1324 | static unsigned getNonSoftWaitcntOpcode(unsigned Opcode) { |
| 1325 | switch (Opcode) { |
| 1326 | case AMDGPU::S_WAITCNT_soft: |
| 1327 | return AMDGPU::S_WAITCNT; |
| 1328 | case AMDGPU::S_WAITCNT_VSCNT_soft: |
| 1329 | return AMDGPU::S_WAITCNT_VSCNT; |
| 1330 | case AMDGPU::S_WAIT_LOADCNT_soft: |
| 1331 | return AMDGPU::S_WAIT_LOADCNT; |
| 1332 | case AMDGPU::S_WAIT_STORECNT_soft: |
| 1333 | return AMDGPU::S_WAIT_STORECNT; |
| 1334 | case AMDGPU::S_WAIT_SAMPLECNT_soft: |
| 1335 | return AMDGPU::S_WAIT_SAMPLECNT; |
| 1336 | case AMDGPU::S_WAIT_BVHCNT_soft: |
| 1337 | return AMDGPU::S_WAIT_BVHCNT; |
| 1338 | case AMDGPU::S_WAIT_DSCNT_soft: |
| 1339 | return AMDGPU::S_WAIT_DSCNT; |
| 1340 | case AMDGPU::S_WAIT_KMCNT_soft: |
| 1341 | return AMDGPU::S_WAIT_KMCNT; |
| 1342 | case AMDGPU::S_WAIT_XCNT_soft: |
| 1343 | return AMDGPU::S_WAIT_XCNT; |
| 1344 | default: |
| 1345 | return Opcode; |
| 1346 | } |
| 1347 | } |
| 1348 | |
| 1349 | static bool isWaitcnt(unsigned Opcode) { |
| 1350 | switch (getNonSoftWaitcntOpcode(Opcode)) { |
| 1351 | case AMDGPU::S_WAITCNT: |
| 1352 | case AMDGPU::S_WAITCNT_VSCNT: |
| 1353 | case AMDGPU::S_WAITCNT_VMCNT: |
| 1354 | case AMDGPU::S_WAITCNT_EXPCNT: |
| 1355 | case AMDGPU::S_WAITCNT_LGKMCNT: |
| 1356 | case AMDGPU::S_WAIT_LOADCNT: |
| 1357 | case AMDGPU::S_WAIT_LOADCNT_DSCNT: |
| 1358 | case AMDGPU::S_WAIT_STORECNT: |
| 1359 | case AMDGPU::S_WAIT_STORECNT_DSCNT: |
| 1360 | case AMDGPU::S_WAIT_SAMPLECNT: |
| 1361 | case AMDGPU::S_WAIT_BVHCNT: |
| 1362 | case AMDGPU::S_WAIT_EXPCNT: |
| 1363 | case AMDGPU::S_WAIT_DSCNT: |
| 1364 | case AMDGPU::S_WAIT_KMCNT: |
| 1365 | case AMDGPU::S_WAIT_XCNT: |
| 1366 | case AMDGPU::S_WAIT_ASYNCCNT: |
| 1367 | case AMDGPU::S_WAIT_TENSORCNT: |
| 1368 | case AMDGPU::S_WAIT_IDLE: |
| 1369 | return true; |
| 1370 | default: |
| 1371 | return false; |
| 1372 | } |
| 1373 | } |
| 1374 | |
| 1375 | bool isVGPRCopy(const MachineInstr &MI) const { |
| 1376 | assert(isCopyInstr(MI)); |
| 1377 | Register Dest = MI.getOperand(i: 0).getReg(); |
| 1378 | const MachineFunction &MF = *MI.getMF(); |
| 1379 | const MachineRegisterInfo &MRI = MF.getRegInfo(); |
| 1380 | return !RI.isSGPRReg(MRI, Reg: Dest); |
| 1381 | } |
| 1382 | |
| 1383 | bool hasVGPRUses(const MachineInstr &MI) const { |
| 1384 | const MachineFunction &MF = *MI.getMF(); |
| 1385 | const MachineRegisterInfo &MRI = MF.getRegInfo(); |
| 1386 | return llvm::any_of(Range: MI.explicit_uses(), |
| 1387 | P: [&MRI, this](const MachineOperand &MO) { |
| 1388 | return MO.isReg() && RI.isVGPR(MRI, Reg: MO.getReg());}); |
| 1389 | } |
| 1390 | |
| 1391 | /// Return true if the instruction modifies the mode register.q |
| 1392 | static bool modifiesModeRegister(const MachineInstr &MI); |
| 1393 | |
| 1394 | /// This function is used to determine if an instruction can be safely |
| 1395 | /// executed under EXEC = 0 without hardware error, indeterminate results, |
| 1396 | /// and/or visible effects on future vector execution or outside the shader. |
| 1397 | /// Note: as of 2024 the only use of this is SIPreEmitPeephole where it is |
| 1398 | /// used in removing branches over short EXEC = 0 sequences. |
| 1399 | /// As such it embeds certain assumptions which may not apply to every case |
| 1400 | /// of EXEC = 0 execution. |
| 1401 | bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const; |
| 1402 | |
| 1403 | /// Returns true if the instruction could potentially depend on the value of |
| 1404 | /// exec. If false, exec dependencies may safely be ignored. |
| 1405 | bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const; |
| 1406 | |
| 1407 | bool isInlineConstant(const APInt &Imm) const; |
| 1408 | |
| 1409 | bool isInlineConstant(const APFloat &Imm) const; |
| 1410 | |
| 1411 | // Returns true if this non-register operand definitely does not need to be |
| 1412 | // encoded as a 32-bit literal. Note that this function handles all kinds of |
| 1413 | // operands, not just immediates. |
| 1414 | // |
| 1415 | // Some operands like FrameIndexes could resolve to an inline immediate value |
| 1416 | // that will not require an additional 4-bytes; this function assumes that it |
| 1417 | // will. |
| 1418 | bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const { |
| 1419 | if (!MO.isImm()) |
| 1420 | return false; |
| 1421 | return isInlineConstant(ImmVal: MO.getImm(), OperandType); |
| 1422 | } |
| 1423 | bool isInlineConstant(int64_t ImmVal, uint8_t OperandType) const; |
| 1424 | |
| 1425 | bool isInlineConstant(const MachineOperand &MO, |
| 1426 | const MCOperandInfo &OpInfo) const { |
| 1427 | return isInlineConstant(MO, OperandType: OpInfo.OperandType); |
| 1428 | } |
| 1429 | |
| 1430 | /// \p returns true if \p UseMO is substituted with \p DefMO in \p MI it would |
| 1431 | /// be an inline immediate. |
| 1432 | bool isInlineConstant(const MachineInstr &MI, |
| 1433 | const MachineOperand &UseMO, |
| 1434 | const MachineOperand &DefMO) const { |
| 1435 | assert(UseMO.getParent() == &MI); |
| 1436 | int OpIdx = UseMO.getOperandNo(); |
| 1437 | if (OpIdx >= MI.getDesc().NumOperands) |
| 1438 | return false; |
| 1439 | |
| 1440 | return isInlineConstant(MO: DefMO, OpInfo: MI.getDesc().operands()[OpIdx]); |
| 1441 | } |
| 1442 | |
| 1443 | /// \p returns true if the operand \p OpIdx in \p MI is a valid inline |
| 1444 | /// immediate. |
| 1445 | bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx) const { |
| 1446 | const MachineOperand &MO = MI.getOperand(i: OpIdx); |
| 1447 | return isInlineConstant(MO, OperandType: MI.getDesc().operands()[OpIdx].OperandType); |
| 1448 | } |
| 1449 | |
| 1450 | bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx, |
| 1451 | int64_t ImmVal) const { |
| 1452 | if (OpIdx >= MI.getDesc().NumOperands) |
| 1453 | return false; |
| 1454 | |
| 1455 | if (isCopyInstr(MI)) { |
| 1456 | unsigned Size = getOpSize(MI, OpNo: OpIdx); |
| 1457 | assert(Size == 8 || Size == 4); |
| 1458 | |
| 1459 | uint8_t OpType = (Size == 8) ? |
| 1460 | AMDGPU::OPERAND_REG_IMM_INT64 : AMDGPU::OPERAND_REG_IMM_INT32; |
| 1461 | return isInlineConstant(ImmVal, OperandType: OpType); |
| 1462 | } |
| 1463 | |
| 1464 | return isInlineConstant(ImmVal, OperandType: MI.getDesc().operands()[OpIdx].OperandType); |
| 1465 | } |
| 1466 | |
| 1467 | bool isInlineConstant(const MachineInstr &MI, unsigned OpIdx, |
| 1468 | const MachineOperand &MO) const { |
| 1469 | return isInlineConstant(MI, OpIdx, ImmVal: MO.getImm()); |
| 1470 | } |
| 1471 | |
| 1472 | bool isInlineConstant(const MachineOperand &MO) const { |
| 1473 | return isInlineConstant(MI: *MO.getParent(), OpIdx: MO.getOperandNo()); |
| 1474 | } |
| 1475 | |
| 1476 | bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo, |
| 1477 | const MachineOperand &MO) const; |
| 1478 | |
| 1479 | bool isLiteralOperandLegal(const MCInstrDesc &InstDesc, |
| 1480 | const MCOperandInfo &OpInfo) const; |
| 1481 | |
| 1482 | bool isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo, |
| 1483 | int64_t ImmVal) const; |
| 1484 | |
| 1485 | bool isImmOperandLegal(const MachineInstr &MI, unsigned OpNo, |
| 1486 | const MachineOperand &MO) const { |
| 1487 | return isImmOperandLegal(InstDesc: MI.getDesc(), OpNo, MO); |
| 1488 | } |
| 1489 | |
| 1490 | bool isNeverCoissue(MachineInstr &MI) const; |
| 1491 | |
| 1492 | /// Check if this immediate value can be used for AV_MOV_B64_IMM_PSEUDO. |
| 1493 | bool isLegalAV64PseudoImm(uint64_t Imm) const; |
| 1494 | |
| 1495 | /// Return true if this 64-bit VALU instruction has a 32-bit encoding. |
| 1496 | /// This function will return false if you pass it a 32-bit instruction. |
| 1497 | bool hasVALU32BitEncoding(unsigned Opcode) const; |
| 1498 | |
| 1499 | /// Return true if \p Reg is a lane mask that already has 0 in every bit |
| 1500 | /// corresponding to a lane that is inactive in EXEC where \p Use executes, |
| 1501 | /// so that ANDing it with EXEC there would be a no-op. Requires SSA form. |
| 1502 | bool isMaskedByExec(Register Reg, const MachineInstr &Use, |
| 1503 | const MachineRegisterInfo &MRI, unsigned Depth = 0) const; |
| 1504 | |
| 1505 | bool physRegUsesConstantBus(const MachineOperand &Reg) const; |
| 1506 | bool regUsesConstantBus(const MachineOperand &Reg, |
| 1507 | const MachineRegisterInfo &MRI) const; |
| 1508 | |
| 1509 | /// Returns true if this operand uses the constant bus. |
| 1510 | bool usesConstantBus(const MachineRegisterInfo &MRI, |
| 1511 | const MachineOperand &MO, |
| 1512 | const MCOperandInfo &OpInfo) const; |
| 1513 | |
| 1514 | bool usesConstantBus(const MachineRegisterInfo &MRI, const MachineInstr &MI, |
| 1515 | int OpIdx) const { |
| 1516 | return usesConstantBus(MRI, MO: MI.getOperand(i: OpIdx), |
| 1517 | OpInfo: MI.getDesc().operands()[OpIdx]); |
| 1518 | } |
| 1519 | |
| 1520 | /// Return true if this instruction has any modifiers. |
| 1521 | /// e.g. src[012]_mod, omod, clamp. |
| 1522 | bool hasModifiers(unsigned Opcode) const; |
| 1523 | |
| 1524 | bool (const MachineInstr &MI, AMDGPU::OpName OpName) const; |
| 1525 | bool (const MachineInstr &MI) const; |
| 1526 | |
| 1527 | bool canShrink(const MachineInstr &MI, |
| 1528 | const MachineRegisterInfo &MRI) const; |
| 1529 | |
| 1530 | MachineInstr *buildShrunkInst(MachineInstr &MI, |
| 1531 | unsigned NewOpcode) const; |
| 1532 | |
| 1533 | bool verifyInstruction(const MachineInstr &MI, |
| 1534 | StringRef &ErrInfo) const override; |
| 1535 | |
| 1536 | unsigned getVALUOp(const MachineInstr &MI) const; |
| 1537 | unsigned getVALUOp(unsigned Opc) const; |
| 1538 | |
| 1539 | void insertScratchExecCopy(MachineFunction &MF, MachineBasicBlock &MBB, |
| 1540 | MachineBasicBlock::iterator MBBI, |
| 1541 | const DebugLoc &DL, Register Reg, bool IsSCCLive, |
| 1542 | SlotIndexes *Indexes = nullptr) const; |
| 1543 | |
| 1544 | void restoreExec(MachineFunction &MF, MachineBasicBlock &MBB, |
| 1545 | MachineBasicBlock::iterator MBBI, const DebugLoc &DL, |
| 1546 | Register Reg, SlotIndexes *Indexes = nullptr) const; |
| 1547 | |
| 1548 | MachineInstr *getWholeWaveFunctionSetup(MachineFunction &MF) const; |
| 1549 | |
| 1550 | /// Return the correct register class for \p OpNo. For target-specific |
| 1551 | /// instructions, this will return the register class that has been defined |
| 1552 | /// in tablegen. For generic instructions, like REG_SEQUENCE it will return |
| 1553 | /// the register class of its machine operand. |
| 1554 | /// to infer the correct register class base on the other operands. |
| 1555 | const TargetRegisterClass *getOpRegClass(const MachineInstr &MI, |
| 1556 | unsigned OpNo) const; |
| 1557 | |
| 1558 | /// Return the size in bytes of the operand OpNo on the given |
| 1559 | // instruction opcode. |
| 1560 | unsigned getOpSize(uint32_t Opcode, unsigned OpNo) const { |
| 1561 | const MCOperandInfo &OpInfo = get(Opcode).operands()[OpNo]; |
| 1562 | |
| 1563 | if (OpInfo.RegClass == -1) { |
| 1564 | // If this is an immediate operand, this must be a 32-bit literal. |
| 1565 | assert(OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE); |
| 1566 | return 4; |
| 1567 | } |
| 1568 | |
| 1569 | return RI.getRegSizeInBits(RC: *RI.getRegClass(i: getOpRegClassID(OpInfo))) / 8; |
| 1570 | } |
| 1571 | |
| 1572 | /// This form should usually be preferred since it handles operands |
| 1573 | /// with unknown register classes. |
| 1574 | unsigned getOpSize(const MachineInstr &MI, unsigned OpNo) const { |
| 1575 | const MachineOperand &MO = MI.getOperand(i: OpNo); |
| 1576 | if (MO.isReg()) { |
| 1577 | if (unsigned SubReg = MO.getSubReg()) { |
| 1578 | return RI.getSubRegIdxSize(Idx: SubReg) / 8; |
| 1579 | } |
| 1580 | } |
| 1581 | return RI.getRegSizeInBits(RC: *getOpRegClass(MI, OpNo)) / 8; |
| 1582 | } |
| 1583 | |
| 1584 | /// Legalize the \p OpIndex operand of this instruction by inserting |
| 1585 | /// a MOV. For example: |
| 1586 | /// ADD_I32_e32 VGPR0, 15 |
| 1587 | /// to |
| 1588 | /// MOV VGPR1, 15 |
| 1589 | /// ADD_I32_e32 VGPR0, VGPR1 |
| 1590 | /// |
| 1591 | /// If the operand being legalized is a register, then a COPY will be used |
| 1592 | /// instead of MOV. |
| 1593 | void legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const; |
| 1594 | |
| 1595 | /// Check if \p MO is a legal operand if it was the \p OpIdx Operand |
| 1596 | /// for \p MI. |
| 1597 | bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx, |
| 1598 | const MachineOperand *MO = nullptr) const; |
| 1599 | |
| 1600 | /// Check if \p MO (a register operand) is a legal register for the |
| 1601 | /// given operand description or operand index. |
| 1602 | /// The operand index version provide more legality checks |
| 1603 | bool isLegalRegOperand(const MachineRegisterInfo &MRI, |
| 1604 | const MCOperandInfo &OpInfo, |
| 1605 | const MachineOperand &MO) const; |
| 1606 | bool isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx, |
| 1607 | const MachineOperand &MO) const; |
| 1608 | |
| 1609 | /// Check if \p MO would be a legal operand for a single-SGPR-read |
| 1610 | /// instruction. |
| 1611 | /// |
| 1612 | /// Single-SGPR-read instructions typically accept VGPRs, SGPRs, or immediates |
| 1613 | /// as source operands. On gfx12+, if a source operand uses SGPRs, the HW can |
| 1614 | /// only read the first SGPR and replicate the value across all lanes. \p SrcN |
| 1615 | /// can be 0, 1, or 2, representing src0, src1, and src2, respectively. If \p |
| 1616 | /// MO is nullptr, the operand corresponding to \p SrcN will be used. Non-SGPR |
| 1617 | /// operands are always considered legal. |
| 1618 | bool |
| 1619 | isLegalSingleSGPRReadInstOperand(const MachineRegisterInfo &MRI, |
| 1620 | const MachineInstr &MI, unsigned SrcN, |
| 1621 | const MachineOperand *MO = nullptr) const; |
| 1622 | |
| 1623 | /// Legalize operands in \p MI by either commuting it or inserting a |
| 1624 | /// copy of src1. |
| 1625 | void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const; |
| 1626 | |
| 1627 | /// Fix operands in \p MI to satisfy constant bus requirements. |
| 1628 | void legalizeOperandsVOP3(MachineRegisterInfo &MRI, MachineInstr &MI) const; |
| 1629 | |
| 1630 | /// Copy a value from a VGPR (\p SrcReg) to SGPR. The desired register class |
| 1631 | /// for the dst register (\p DstRC) can be optionally supplied. This function |
| 1632 | /// can only be used when it is know that the value in SrcReg is same across |
| 1633 | /// all threads in the wave. |
| 1634 | /// \returns The SGPR register that \p SrcReg was copied to. |
| 1635 | Register readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI, |
| 1636 | MachineRegisterInfo &MRI, |
| 1637 | const TargetRegisterClass *DstRC = nullptr) const; |
| 1638 | |
| 1639 | void legalizeOperandsSMRD(MachineRegisterInfo &MRI, MachineInstr &MI) const; |
| 1640 | void legalizeOperandsFLAT(MachineRegisterInfo &MRI, MachineInstr &MI) const; |
| 1641 | |
| 1642 | void legalizeGenericOperand(MachineBasicBlock &InsertMBB, |
| 1643 | MachineBasicBlock::iterator I, |
| 1644 | const TargetRegisterClass *DstRC, |
| 1645 | MachineOperand &Op, MachineRegisterInfo &MRI, |
| 1646 | const DebugLoc &DL) const; |
| 1647 | |
| 1648 | /// Legalize all operands in this instruction. This function may create new |
| 1649 | /// instructions and control-flow around \p MI. If present, \p MDT is |
| 1650 | /// updated. |
| 1651 | /// \returns A new basic block that contains \p MI if new blocks were created. |
| 1652 | MachineBasicBlock * |
| 1653 | legalizeOperands(MachineInstr &MI, MachineDominatorTree *MDT = nullptr) const; |
| 1654 | |
| 1655 | /// Change SADDR form of a FLAT \p Inst to its VADDR form if saddr operand |
| 1656 | /// was moved to VGPR. \returns true if succeeded. |
| 1657 | bool moveFlatAddrToVGPR(MachineInstr &Inst) const; |
| 1658 | |
| 1659 | /// Fix operands in Inst to fix 16bit SALU to VALU lowering. |
| 1660 | void legalizeOperandsVALUt16(MachineInstr &Inst, |
| 1661 | MachineRegisterInfo &MRI) const; |
| 1662 | void legalizeOperandsVALUt16(MachineInstr &Inst, unsigned OpIdx, |
| 1663 | MachineRegisterInfo &MRI) const; |
| 1664 | |
| 1665 | /// Build a 32-bit VGPR whose low half is the 16-bit value \p Src.SrcSubReg |
| 1666 | /// and whose high half is undefined. The instructions are inserted before |
| 1667 | /// \p I. \returns the new register. |
| 1668 | Register buildVGPR32FromLo16(MachineBasicBlock &MBB, |
| 1669 | MachineBasicBlock::iterator I, |
| 1670 | const DebugLoc &DL, Register Src, |
| 1671 | unsigned SrcSubReg) const; |
| 1672 | |
| 1673 | /// Replace the instructions opcode with the equivalent VALU |
| 1674 | /// opcode. This function will also move the users of MachineInstruntions |
| 1675 | /// in the \p WorkList to the VALU if necessary. If present, \p MDT is |
| 1676 | /// updated. |
| 1677 | void moveToVALU(SIInstrWorklist &Worklist, MachineDominatorTree *MDT) const; |
| 1678 | |
| 1679 | void |
| 1680 | moveToVALUImpl(SIInstrWorklist &Worklist, MachineDominatorTree *MDT, |
| 1681 | MachineInstr &Inst, |
| 1682 | DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls, |
| 1683 | DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const; |
| 1684 | /// Wrapper function for generating waterfall for instruction \p MI |
| 1685 | /// This function take into consideration of related pre & succ instructions |
| 1686 | /// (e.g. calling process) into consideratioin |
| 1687 | void createWaterFallForSiCall(MachineInstr *MI, MachineDominatorTree *MDT, |
| 1688 | ArrayRef<MachineOperand *> ScalarOps, |
| 1689 | ArrayRef<Register> PhySGPRs = {}) const; |
| 1690 | |
| 1691 | void insertNoop(MachineBasicBlock &MBB, |
| 1692 | MachineBasicBlock::iterator MI) const override; |
| 1693 | |
| 1694 | void insertNoops(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, |
| 1695 | unsigned Quantity) const override; |
| 1696 | |
| 1697 | /// Build instructions that simulate the behavior of a `s_trap 2` instructions |
| 1698 | /// for hardware (namely, gfx11) that runs in PRIV=1 mode. There, s_trap is |
| 1699 | /// interpreted as a nop. |
| 1700 | MachineBasicBlock *insertSimulatedTrap(MachineRegisterInfo &MRI, |
| 1701 | MachineBasicBlock &MBB, |
| 1702 | MachineInstr &MI, |
| 1703 | const DebugLoc &DL) const; |
| 1704 | |
| 1705 | /// Return the number of wait states that result from executing this |
| 1706 | /// instruction. |
| 1707 | static unsigned getNumWaitStates(const MachineInstr &MI); |
| 1708 | |
| 1709 | /// Returns the operand named \p Op. If \p MI does not have an |
| 1710 | /// operand named \c Op, this function returns nullptr. |
| 1711 | LLVM_READONLY |
| 1712 | MachineOperand *getNamedOperand(MachineInstr &MI, |
| 1713 | AMDGPU::OpName OperandName) const; |
| 1714 | |
| 1715 | LLVM_READONLY |
| 1716 | const MachineOperand *getNamedOperand(const MachineInstr &MI, |
| 1717 | AMDGPU::OpName OperandName) const { |
| 1718 | return getNamedOperand(MI&: const_cast<MachineInstr &>(MI), OperandName); |
| 1719 | } |
| 1720 | |
| 1721 | /// Get required immediate operand |
| 1722 | int64_t getNamedImmOperand(const MachineInstr &MI, |
| 1723 | AMDGPU::OpName OperandName) const { |
| 1724 | int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OperandName); |
| 1725 | return MI.getOperand(i: Idx).getImm(); |
| 1726 | } |
| 1727 | |
| 1728 | uint64_t getDefaultRsrcDataFormat() const; |
| 1729 | uint64_t getScratchRsrcWords23() const; |
| 1730 | |
| 1731 | bool isLowLatencyInstruction(const MachineInstr &MI) const; |
| 1732 | bool isHighLatencyDef(int Opc) const override; |
| 1733 | |
| 1734 | /// Return the descriptor of the target-specific machine instruction |
| 1735 | /// that corresponds to the specified pseudo or native opcode. |
| 1736 | const MCInstrDesc &getMCOpcodeFromPseudo(unsigned Opcode) const { |
| 1737 | return get(Opcode: pseudoToMCOpcode(Opcode)); |
| 1738 | } |
| 1739 | |
| 1740 | Register isStackAccess(const MachineInstr &MI, int &FrameIndex, |
| 1741 | TypeSize &MemBytes) const; |
| 1742 | Register isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex, |
| 1743 | TypeSize &MemBytes) const; |
| 1744 | |
| 1745 | Register isLoadFromStackSlot(const MachineInstr &MI, |
| 1746 | int &FrameIndex) const override { |
| 1747 | TypeSize MemBytes = TypeSize::getZero(); |
| 1748 | return isLoadFromStackSlot(MI, FrameIndex, MemBytes); |
| 1749 | } |
| 1750 | |
| 1751 | Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex, |
| 1752 | TypeSize &MemBytes) const override; |
| 1753 | |
| 1754 | Register isStoreToStackSlot(const MachineInstr &MI, |
| 1755 | int &FrameIndex) const override { |
| 1756 | TypeSize MemBytes = TypeSize::getZero(); |
| 1757 | return isStoreToStackSlot(MI, FrameIndex, MemBytes); |
| 1758 | } |
| 1759 | |
| 1760 | Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex, |
| 1761 | TypeSize &MemBytes) const override; |
| 1762 | |
| 1763 | unsigned getInstSizeInBytes(const MachineInstr &MI) const override; |
| 1764 | |
| 1765 | InstSizeVerifyMode |
| 1766 | getInstSizeVerifyMode(const MachineInstr &MI) const override; |
| 1767 | |
| 1768 | std::pair<unsigned, unsigned> |
| 1769 | decomposeMachineOperandsTargetFlags(unsigned TF) const override; |
| 1770 | |
| 1771 | ArrayRef<std::pair<int, const char *>> |
| 1772 | getSerializableTargetIndices() const override; |
| 1773 | |
| 1774 | ArrayRef<std::pair<unsigned, const char *>> |
| 1775 | getSerializableDirectMachineOperandTargetFlags() const override; |
| 1776 | |
| 1777 | ArrayRef<std::pair<MachineMemOperand::Flags, const char *>> |
| 1778 | getSerializableMachineMemOperandTargetFlags() const override; |
| 1779 | |
| 1780 | ScheduleHazardRecognizer * |
| 1781 | CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, |
| 1782 | const ScheduleDAG *DAG) const override; |
| 1783 | |
| 1784 | ScheduleHazardRecognizer * |
| 1785 | CreateTargetPostRAHazardRecognizer(const MachineFunction &MF, |
| 1786 | MachineLoopInfo *MLI) const override; |
| 1787 | |
| 1788 | ScheduleHazardRecognizer * |
| 1789 | CreateTargetMIHazardRecognizer(const InstrItineraryData *II, |
| 1790 | const ScheduleDAGMI *DAG) const override; |
| 1791 | |
| 1792 | unsigned getLiveRangeSplitOpcode(Register Reg, |
| 1793 | const MachineFunction &MF) const override; |
| 1794 | |
| 1795 | bool isBasicBlockPrologue(const MachineInstr &MI, |
| 1796 | Register Reg = Register()) const override; |
| 1797 | |
| 1798 | bool canAddToBBProlog(const MachineInstr &MI) const; |
| 1799 | |
| 1800 | MachineInstr *createPHIDestinationCopy(MachineBasicBlock &MBB, |
| 1801 | MachineBasicBlock::iterator InsPt, |
| 1802 | const DebugLoc &DL, Register Src, |
| 1803 | Register Dst) const override; |
| 1804 | |
| 1805 | MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB, |
| 1806 | MachineBasicBlock::iterator InsPt, |
| 1807 | const DebugLoc &DL, Register Src, |
| 1808 | unsigned SrcSubReg, |
| 1809 | Register Dst) const override; |
| 1810 | |
| 1811 | bool isWave32() const; |
| 1812 | |
| 1813 | bool isVOPDAntidependencyAllowed(const MachineInstr &MI) const; |
| 1814 | |
| 1815 | bool hasRAWDependency(const MachineInstr &FirstMI, |
| 1816 | const MachineInstr &SecondMI) const; |
| 1817 | |
| 1818 | /// Return a partially built integer add instruction without carry. |
| 1819 | /// Caller must add source operands. |
| 1820 | /// For pre-GFX9 it will generate unused carry destination operand. |
| 1821 | /// TODO: After GFX9 it should return a no-carry operation. |
| 1822 | MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB, |
| 1823 | MachineBasicBlock::iterator I, |
| 1824 | const DebugLoc &DL, |
| 1825 | Register DestReg) const; |
| 1826 | |
| 1827 | MachineInstrBuilder getAddNoCarry(MachineBasicBlock &MBB, |
| 1828 | MachineBasicBlock::iterator I, |
| 1829 | const DebugLoc &DL, |
| 1830 | Register DestReg, |
| 1831 | RegScavenger &RS) const; |
| 1832 | |
| 1833 | static bool isKillTerminator(unsigned Opcode); |
| 1834 | const MCInstrDesc &getKillTerminatorFromPseudo(unsigned Opcode) const; |
| 1835 | |
| 1836 | bool isLegalMUBUFImmOffset(unsigned Imm) const; |
| 1837 | |
| 1838 | static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST); |
| 1839 | |
| 1840 | bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, |
| 1841 | Align Alignment = Align(4)) const; |
| 1842 | |
| 1843 | /// Returns if \p Offset is legal for the subtarget as the offset to a FLAT |
| 1844 | /// encoded instruction with the given \p FlatVariant. |
| 1845 | bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace, |
| 1846 | AMDGPU::FlatAddrSpace FlatVariant) const; |
| 1847 | |
| 1848 | /// Split \p COffsetVal into {immediate offset field, remainder offset} |
| 1849 | /// values. |
| 1850 | std::pair<int64_t, int64_t> |
| 1851 | splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace, |
| 1852 | AMDGPU::FlatAddrSpace FlatVariant) const; |
| 1853 | |
| 1854 | /// Returns true if negative offsets are allowed for the given \p FlatVariant. |
| 1855 | bool allowNegativeFlatOffset(AMDGPU::FlatAddrSpace FlatVariant) const; |
| 1856 | |
| 1857 | /// \brief Return a target-specific opcode if Opcode is a pseudo instruction. |
| 1858 | /// Return -1 if the target-specific opcode for the pseudo instruction does |
| 1859 | /// not exist. If Opcode is not a pseudo instruction, this is identity. |
| 1860 | int pseudoToMCOpcode(int Opcode) const; |
| 1861 | |
| 1862 | /// \brief Check if this instruction should only be used by assembler. |
| 1863 | /// Return true if this opcode should not be used by codegen. |
| 1864 | bool isAsmOnlyOpcode(int MCOp) const; |
| 1865 | |
| 1866 | void fixImplicitOperands(MachineInstr &MI) const; |
| 1867 | |
| 1868 | MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, |
| 1869 | ArrayRef<unsigned> Ops, int FrameIndex, |
| 1870 | MachineInstr *&CopyMI, |
| 1871 | LiveIntervals *LIS = nullptr, |
| 1872 | VirtRegMap *VRM = nullptr) const override; |
| 1873 | |
| 1874 | unsigned getInstrLatency(const InstrItineraryData *ItinData, |
| 1875 | const MachineInstr &MI, |
| 1876 | unsigned *PredCost = nullptr) const override; |
| 1877 | |
| 1878 | unsigned getBlockingCycles(const MachineInstr &MI) const; |
| 1879 | |
| 1880 | /// GFX1250 blocking-cycles table lookup with no occupancy subtarget gate. |
| 1881 | /// Returns 0 if \p MI is not in the table. Used as a multi-pass VALU denylist |
| 1882 | /// (e.g. V_PERM_PK16 hazard) on both gfx1250 and gfx1251. |
| 1883 | unsigned getGFX1250BlockingCyclesTable(const MachineInstr &MI) const; |
| 1884 | |
| 1885 | const MachineOperand &getCalleeOperand(const MachineInstr &MI) const override; |
| 1886 | |
| 1887 | ValueUniformity getValueUniformity(const MachineInstr &MI) const final; |
| 1888 | |
| 1889 | ValueUniformity getGenericValueUniformity(const MachineInstr &MI) const; |
| 1890 | |
| 1891 | const MIRFormatter *getMIRFormatter() const override; |
| 1892 | |
| 1893 | static unsigned getDSShaderTypeValue(const MachineFunction &MF); |
| 1894 | |
| 1895 | const TargetSchedModel &getSchedModel() const { return SchedModel; } |
| 1896 | |
| 1897 | void createReadFirstLaneFromCopyToPhysReg(MachineRegisterInfo &MRI, |
| 1898 | Register DstReg, |
| 1899 | MachineInstr &Inst) const; |
| 1900 | |
| 1901 | void handleCopyToPhysHelper( |
| 1902 | SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst, |
| 1903 | MachineRegisterInfo &MRI, MachineDominatorTree *MDT, |
| 1904 | DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls, |
| 1905 | DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const; |
| 1906 | |
| 1907 | // FIXME: This should be removed |
| 1908 | // Enforce operand's \p OpName even alignment if required by target. |
| 1909 | // This is used if an operand is a 32 bit register but needs to be aligned |
| 1910 | // regardless. |
| 1911 | void enforceOperandRCAlignment(MachineInstr &MI, AMDGPU::OpName OpName) const; |
| 1912 | |
| 1913 | /// Get the repeat rate for a VALU instruction from the scheduling model. |
| 1914 | /// Returns 1 for regular VALU, >1 for long-latency VALU (packed, F64, etc.) |
| 1915 | unsigned getRepeatRate(const MachineInstr &MI) const; |
| 1916 | }; |
| 1917 | |
| 1918 | /// \brief Returns true if a reg:subreg pair P has a TRC class |
| 1919 | inline bool isOfRegClass(const TargetInstrInfo::RegSubRegPair &P, |
| 1920 | const TargetRegisterClass &TRC, |
| 1921 | MachineRegisterInfo &MRI) { |
| 1922 | auto *RC = MRI.getRegClass(Reg: P.Reg); |
| 1923 | if (!P.SubReg) |
| 1924 | return RC == &TRC; |
| 1925 | auto *TRI = MRI.getTargetRegisterInfo(); |
| 1926 | return RC == TRI->getMatchingSuperRegClass(A: RC, B: &TRC, Idx: P.SubReg); |
| 1927 | } |
| 1928 | |
| 1929 | /// \brief Create RegSubRegPair from a register MachineOperand |
| 1930 | inline |
| 1931 | TargetInstrInfo::RegSubRegPair getRegSubRegPair(const MachineOperand &O) { |
| 1932 | assert(O.isReg()); |
| 1933 | return TargetInstrInfo::RegSubRegPair(O.getReg(), O.getSubReg()); |
| 1934 | } |
| 1935 | |
| 1936 | /// \brief Return the SubReg component from REG_SEQUENCE |
| 1937 | TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI, |
| 1938 | unsigned SubReg); |
| 1939 | |
| 1940 | /// \brief Return the defining instruction for a given reg:subreg pair |
| 1941 | /// skipping copy like instructions and subreg-manipulation pseudos. |
| 1942 | /// Following another subreg of a reg:subreg isn't supported. |
| 1943 | MachineInstr *getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P, |
| 1944 | const MachineRegisterInfo &MRI); |
| 1945 | |
| 1946 | /// \brief Return false if EXEC is not changed between the def of \p VReg at \p |
| 1947 | /// DefMI and the use at \p UseMI. Should be run on SSA. Currently does not |
| 1948 | /// attempt to track between blocks. |
| 1949 | bool execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI, |
| 1950 | Register VReg, |
| 1951 | const MachineInstr &DefMI, |
| 1952 | const MachineInstr &UseMI); |
| 1953 | |
| 1954 | /// \brief Return false if EXEC is not changed between the def of \p VReg at \p |
| 1955 | /// DefMI and all its uses. Should be run on SSA. Currently does not attempt to |
| 1956 | /// track between blocks. |
| 1957 | bool execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI, |
| 1958 | Register VReg, |
| 1959 | const MachineInstr &DefMI); |
| 1960 | |
| 1961 | namespace AMDGPU { |
| 1962 | |
| 1963 | LLVM_READONLY |
| 1964 | int32_t getVOPe64(uint32_t Opcode); |
| 1965 | |
| 1966 | LLVM_READONLY |
| 1967 | int32_t getVOPe32(uint32_t Opcode); |
| 1968 | |
| 1969 | LLVM_READONLY |
| 1970 | int32_t getSDWAOp(uint32_t Opcode); |
| 1971 | |
| 1972 | LLVM_READONLY |
| 1973 | int32_t getDPPOp32(uint32_t Opcode); |
| 1974 | |
| 1975 | LLVM_READONLY |
| 1976 | int32_t getDPPOp64(uint32_t Opcode); |
| 1977 | |
| 1978 | LLVM_READONLY |
| 1979 | int32_t getBasicFromSDWAOp(uint32_t Opcode); |
| 1980 | |
| 1981 | LLVM_READONLY |
| 1982 | int32_t getCommuteRev(uint32_t Opcode); |
| 1983 | |
| 1984 | LLVM_READONLY |
| 1985 | int32_t getCommuteOrig(uint32_t Opcode); |
| 1986 | |
| 1987 | LLVM_READONLY |
| 1988 | int32_t getAddr64Inst(uint32_t Opcode); |
| 1989 | |
| 1990 | /// Check if \p Opcode is an Addr64 opcode. |
| 1991 | /// |
| 1992 | /// \returns \p Opcode if it is an Addr64 opcode, otherwise -1. |
| 1993 | LLVM_READONLY |
| 1994 | int32_t getIfAddr64Inst(uint32_t Opcode); |
| 1995 | |
| 1996 | LLVM_READONLY |
| 1997 | int32_t getSOPKOp(uint32_t Opcode); |
| 1998 | |
| 1999 | /// \returns SADDR form of a FLAT Global instruction given an \p Opcode |
| 2000 | /// of a VADDR form. |
| 2001 | LLVM_READONLY |
| 2002 | int32_t getGlobalSaddrOp(uint32_t Opcode); |
| 2003 | |
| 2004 | /// \returns VADDR form of a FLAT Global instruction given an \p Opcode |
| 2005 | /// of a SADDR form. |
| 2006 | LLVM_READONLY |
| 2007 | int32_t getGlobalVaddrOp(uint32_t Opcode); |
| 2008 | |
| 2009 | /// \returns ST form with only immediate offset of a FLAT Scratch instruction |
| 2010 | /// given an \p Opcode of an SS (SADDR) form. |
| 2011 | LLVM_READONLY |
| 2012 | int32_t getFlatScratchInstSTfromSS(uint32_t Opcode); |
| 2013 | |
| 2014 | /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode |
| 2015 | /// of an SVS (SADDR + VADDR) form. |
| 2016 | LLVM_READONLY |
| 2017 | int32_t getFlatScratchInstSVfromSVS(uint32_t Opcode); |
| 2018 | |
| 2019 | /// \returns SS (SADDR) form of a FLAT Scratch instruction given an \p Opcode |
| 2020 | /// of an SV (VADDR) form. |
| 2021 | LLVM_READONLY |
| 2022 | int32_t getFlatScratchInstSSfromSV(uint32_t Opcode); |
| 2023 | |
| 2024 | /// \returns SV (VADDR) form of a FLAT Scratch instruction given an \p Opcode |
| 2025 | /// of an SS (SADDR) form. |
| 2026 | LLVM_READONLY |
| 2027 | int32_t getFlatScratchInstSVfromSS(uint32_t Opcode); |
| 2028 | |
| 2029 | /// \returns earlyclobber version of a MAC MFMA is exists. |
| 2030 | LLVM_READONLY |
| 2031 | int32_t getMFMAEarlyClobberOp(uint32_t Opcode); |
| 2032 | |
| 2033 | /// \returns Version of an instruction which uses AGPRs for coupled operands |
| 2034 | /// given an \p Opcode which uses VGPRs for coupled operands. |
| 2035 | LLVM_READONLY |
| 2036 | int32_t getAGPRFormOp(uint32_t Opcode); |
| 2037 | |
| 2038 | /// \returns v_cmpx version of a v_cmp instruction. |
| 2039 | LLVM_READONLY |
| 2040 | int32_t getVCMPXOpFromVCMP(uint32_t Opcode); |
| 2041 | |
| 2042 | const uint64_t RSRC_DATA_FORMAT = 0xf00000000000LL; |
| 2043 | const uint64_t RSRC_ELEMENT_SIZE_SHIFT = (32 + 19); |
| 2044 | const uint64_t RSRC_INDEX_STRIDE_SHIFT = (32 + 21); |
| 2045 | const uint64_t RSRC_TID_ENABLE = UINT64_C(1) << (32 + 23); |
| 2046 | |
| 2047 | } // end namespace AMDGPU |
| 2048 | |
| 2049 | namespace AMDGPU { |
| 2050 | enum : MachineInstr::AsmPrinterFlagTy { |
| 2051 | // For sgpr to vgpr spill instructions |
| 2052 | SGPR_SPILL = MachineInstr::TAsmComments |
| 2053 | }; |
| 2054 | } // namespace AMDGPU |
| 2055 | |
| 2056 | namespace SI { |
| 2057 | namespace KernelInputOffsets { |
| 2058 | |
| 2059 | /// Offsets in bytes from the start of the input buffer |
| 2060 | enum Offsets { |
| 2061 | NGROUPS_X = 0, |
| 2062 | NGROUPS_Y = 4, |
| 2063 | NGROUPS_Z = 8, |
| 2064 | GLOBAL_SIZE_X = 12, |
| 2065 | GLOBAL_SIZE_Y = 16, |
| 2066 | GLOBAL_SIZE_Z = 20, |
| 2067 | LOCAL_SIZE_X = 24, |
| 2068 | LOCAL_SIZE_Y = 28, |
| 2069 | LOCAL_SIZE_Z = 32 |
| 2070 | }; |
| 2071 | |
| 2072 | } // end namespace KernelInputOffsets |
| 2073 | } // end namespace SI |
| 2074 | |
| 2075 | } // end namespace llvm |
| 2076 | |
| 2077 | #endif // LLVM_LIB_TARGET_AMDGPU_SIINSTRINFO_H |
| 2078 | |