| 1 | //===- AMDGPUGlobalISelUtils.cpp ---------------------------------*- C++ -*-==// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "AMDGPUGlobalISelUtils.h" |
| 10 | #include "AMDGPURegisterBankInfo.h" |
| 11 | #include "MCTargetDesc/AMDGPUMCTargetDesc.h" |
| 12 | #include "llvm/ADT/DenseSet.h" |
| 13 | #include "llvm/CodeGen/GlobalISel/GISelValueTracking.h" |
| 14 | #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" |
| 15 | #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" |
| 16 | #include "llvm/CodeGen/GlobalISel/Utils.h" |
| 17 | #include "llvm/CodeGen/LowLevelTypeUtils.h" |
| 18 | #include "llvm/CodeGenTypes/LowLevelType.h" |
| 19 | #include "llvm/IR/Constants.h" |
| 20 | #include "llvm/IR/IntrinsicsAMDGPU.h" |
| 21 | |
| 22 | using namespace llvm; |
| 23 | using namespace AMDGPU; |
| 24 | using namespace MIPatternMatch; |
| 25 | |
| 26 | bool AMDGPU::isUnscaledIEEEDivFmas(const MachineInstr &MI, |
| 27 | const MachineRegisterInfo &MRI) { |
| 28 | Register ScaleReg = MI.getOperand(i: 5).getReg(); |
| 29 | MachineInstr *ScaleDef = getDefIgnoringCopies(Reg: ScaleReg, MRI); |
| 30 | if (ScaleDef && ScaleDef->getOpcode() == AMDGPU::G_AMDGPU_COPY_VCC_SCC) |
| 31 | ScaleReg = ScaleDef->getOperand(i: 1).getReg(); |
| 32 | auto Scale = getIConstantVRegValWithLookThrough(VReg: ScaleReg, MRI); |
| 33 | if (!Scale || !Scale->Value.isZero()) |
| 34 | return false; |
| 35 | |
| 36 | const fltSemantics &Semantics = |
| 37 | getFltSemanticForLLT(Ty: MRI.getType(Reg: MI.getOperand(i: 0).getReg())); |
| 38 | return MI.getMF()->getDenormalMode(FPType: Semantics).Input == DenormalMode::IEEE; |
| 39 | } |
| 40 | |
| 41 | std::pair<Register, unsigned> |
| 42 | AMDGPU::getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg, |
| 43 | GISelValueTracking *ValueTracking, |
| 44 | bool CheckNUW) { |
| 45 | MachineInstr *Def = getDefIgnoringCopies(Reg, MRI); |
| 46 | if (Def->getOpcode() == TargetOpcode::G_CONSTANT) { |
| 47 | unsigned Offset; |
| 48 | const MachineOperand &Op = Def->getOperand(i: 1); |
| 49 | if (Op.isImm()) |
| 50 | Offset = Op.getImm(); |
| 51 | else |
| 52 | Offset = Op.getCImm()->getZExtValue(); |
| 53 | |
| 54 | return std::pair(Register(), Offset); |
| 55 | } |
| 56 | |
| 57 | int64_t Offset; |
| 58 | if (Def->getOpcode() == TargetOpcode::G_ADD) { |
| 59 | // A 32-bit (address + offset) should not cause unsigned 32-bit integer |
| 60 | // wraparound, because s_load instructions perform the addition in 64 bits. |
| 61 | if (CheckNUW && !Def->getFlag(Flag: MachineInstr::NoUWrap)) { |
| 62 | assert(MRI.getType(Reg).getScalarSizeInBits() == 32); |
| 63 | return std::pair(Reg, 0); |
| 64 | } |
| 65 | // TODO: Handle G_OR used for add case |
| 66 | if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI, P: m_ICst(Cst&: Offset))) |
| 67 | return std::pair(Def->getOperand(i: 1).getReg(), Offset); |
| 68 | |
| 69 | // FIXME: matcher should ignore copies |
| 70 | if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI, P: m_Copy(Src: m_ICst(Cst&: Offset)))) |
| 71 | return std::pair(Def->getOperand(i: 1).getReg(), Offset); |
| 72 | } |
| 73 | |
| 74 | Register Base; |
| 75 | if (mi_match(R: Reg, MRI, P: m_GOr(L: m_Reg(R&: Base), R: m_ICst(Cst&: Offset))) && |
| 76 | (Def->getFlag(Flag: MachineInstr::Disjoint) || |
| 77 | (ValueTracking && ValueTracking->maskedValueIsZero( |
| 78 | Val: Base, Mask: APInt(32, Offset, /*isSigned=*/true))))) |
| 79 | return std::pair(Base, Offset); |
| 80 | |
| 81 | // Handle G_PTRTOINT (G_PTR_ADD base, const) case |
| 82 | if (Def->getOpcode() == TargetOpcode::G_PTRTOINT) { |
| 83 | MachineInstr *Base; |
| 84 | Register PtrAdd = Def->getOperand(i: 1).getReg(); |
| 85 | uint32_t Flags; |
| 86 | if (mi_match(R: PtrAdd, MRI, |
| 87 | P: m_GPtrAdd(L: m_MInstr(MI&: Base), R: m_ICst(Cst&: Offset), Flags: m_MIFlags(Flags)))) { |
| 88 | // Same check as for G_ADD; nuw comes from getelementptr inbounds. |
| 89 | if (CheckNUW && !(Flags & MachineInstr::NoUWrap)) { |
| 90 | assert(MRI.getType(Reg).getScalarSizeInBits() == 32); |
| 91 | return std::pair(Reg, 0); |
| 92 | } |
| 93 | // If Base was int converted to pointer, simply return int and offset. |
| 94 | if (Base->getOpcode() == TargetOpcode::G_INTTOPTR) |
| 95 | return std::pair(Base->getOperand(i: 1).getReg(), Offset); |
| 96 | |
| 97 | // Register returned here will be of pointer type. |
| 98 | return std::pair(Base->getOperand(i: 0).getReg(), Offset); |
| 99 | } |
| 100 | } |
| 101 | |
| 102 | return std::pair(Reg, 0); |
| 103 | } |
| 104 | |
| 105 | IntrinsicLaneMaskAnalyzer::IntrinsicLaneMaskAnalyzer(MachineFunction &MF) |
| 106 | : MRI(MF.getRegInfo()) { |
| 107 | initLaneMaskIntrinsics(MF); |
| 108 | } |
| 109 | |
| 110 | bool IntrinsicLaneMaskAnalyzer::isS32S64LaneMask(Register Reg) const { |
| 111 | return S32S64LaneMask.contains(V: Reg); |
| 112 | } |
| 113 | |
| 114 | void IntrinsicLaneMaskAnalyzer::initLaneMaskIntrinsics(MachineFunction &MF) { |
| 115 | for (auto &MBB : MF) { |
| 116 | for (auto &MI : MBB) { |
| 117 | GIntrinsic *GI = dyn_cast<GIntrinsic>(Val: &MI); |
| 118 | if (GI && GI->is(ID: Intrinsic::amdgcn_if_break)) { |
| 119 | S32S64LaneMask.insert(V: MI.getOperand(i: 3).getReg()); |
| 120 | S32S64LaneMask.insert(V: MI.getOperand(i: 0).getReg()); |
| 121 | } |
| 122 | |
| 123 | if (MI.getOpcode() == AMDGPU::SI_IF || |
| 124 | MI.getOpcode() == AMDGPU::SI_ELSE) { |
| 125 | S32S64LaneMask.insert(V: MI.getOperand(i: 0).getReg()); |
| 126 | } |
| 127 | } |
| 128 | } |
| 129 | } |
| 130 | |
| 131 | static LLT getReadAnyLaneSplitTy(LLT Ty) { |
| 132 | if (Ty.isVector()) { |
| 133 | LLT ElTy = Ty.getElementType(); |
| 134 | if (ElTy.getSizeInBits() == 16) |
| 135 | return LLT::fixed_vector(NumElements: 2, ScalarTy: ElTy); |
| 136 | // S32, S64 or pointer |
| 137 | return ElTy; |
| 138 | } |
| 139 | |
| 140 | // Large scalars and 64-bit pointers |
| 141 | return LLT::integer(SizeInBits: 32); |
| 142 | } |
| 143 | |
| 144 | template <typename ReadLaneFnTy> |
| 145 | static Register buildReadLane(MachineIRBuilder &, Register, |
| 146 | const RegisterBankInfo &, ReadLaneFnTy); |
| 147 | |
| 148 | template <typename ReadLaneFnTy> |
| 149 | static void |
| 150 | unmergeReadAnyLane(MachineIRBuilder &B, SmallVectorImpl<Register> &SgprDstParts, |
| 151 | LLT UnmergeTy, Register VgprSrc, const RegisterBankInfo &RBI, |
| 152 | ReadLaneFnTy BuildRL) { |
| 153 | const RegisterBank *VgprRB = &RBI.getRegBank(ID: AMDGPU::VGPRRegBankID); |
| 154 | auto Unmerge = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: UnmergeTy}, Op: VgprSrc); |
| 155 | for (unsigned i = 0; i < Unmerge->getNumOperands() - 1; ++i) { |
| 156 | SgprDstParts.push_back(Elt: buildReadLane(B, Unmerge.getReg(Idx: i), RBI, BuildRL)); |
| 157 | } |
| 158 | } |
| 159 | |
| 160 | template <typename ReadLaneFnTy> |
| 161 | static Register buildReadLane(MachineIRBuilder &B, Register VgprSrc, |
| 162 | const RegisterBankInfo &RBI, |
| 163 | ReadLaneFnTy BuildRL) { |
| 164 | LLT Ty = B.getMRI()->getType(Reg: VgprSrc); |
| 165 | const RegisterBank *SgprRB = &RBI.getRegBank(ID: AMDGPU::SGPRRegBankID); |
| 166 | if (Ty.getSizeInBits() == 32) { |
| 167 | Register SgprDst = B.getMRI()->createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty}); |
| 168 | return BuildRL(B, SgprDst, VgprSrc).getReg(0); |
| 169 | } |
| 170 | |
| 171 | SmallVector<Register, 8> SgprDstParts; |
| 172 | unmergeReadAnyLane(B, SgprDstParts, getReadAnyLaneSplitTy(Ty), VgprSrc, RBI, |
| 173 | BuildRL); |
| 174 | |
| 175 | return B.buildMergeLikeInstr(Res: {SgprRB, Ty}, Ops: SgprDstParts).getReg(Idx: 0); |
| 176 | } |
| 177 | |
| 178 | template <typename ReadLaneFnTy> |
| 179 | static void buildReadLane(MachineIRBuilder &B, Register SgprDst, |
| 180 | Register VgprSrc, const RegisterBankInfo &RBI, |
| 181 | ReadLaneFnTy BuildReadLane) { |
| 182 | LLT Ty = B.getMRI()->getType(Reg: VgprSrc); |
| 183 | if (Ty.getSizeInBits() == 32) { |
| 184 | BuildReadLane(B, SgprDst, VgprSrc); |
| 185 | return; |
| 186 | } |
| 187 | |
| 188 | SmallVector<Register, 8> SgprDstParts; |
| 189 | unmergeReadAnyLane(B, SgprDstParts, getReadAnyLaneSplitTy(Ty), VgprSrc, RBI, |
| 190 | BuildReadLane); |
| 191 | |
| 192 | B.buildMergeLikeInstr(Res: SgprDst, Ops: SgprDstParts).getReg(Idx: 0); |
| 193 | } |
| 194 | |
| 195 | void AMDGPU::buildReadAnyLane(MachineIRBuilder &B, Register SgprDst, |
| 196 | Register VgprSrc, const RegisterBankInfo &RBI) { |
| 197 | return buildReadLane( |
| 198 | B, SgprDst, VgprSrc, RBI, |
| 199 | BuildReadLane: [](MachineIRBuilder &B, Register SgprDst, Register VgprSrc) { |
| 200 | return B.buildInstr(Opc: AMDGPU::G_AMDGPU_READANYLANE, DstOps: {SgprDst}, SrcOps: {VgprSrc}); |
| 201 | }); |
| 202 | } |
| 203 | |
| 204 | void AMDGPU::buildReadFirstLane(MachineIRBuilder &B, Register SgprDst, |
| 205 | Register VgprSrc, const RegisterBankInfo &RBI) { |
| 206 | return buildReadLane( |
| 207 | B, SgprDst, VgprSrc, RBI, |
| 208 | BuildReadLane: [](MachineIRBuilder &B, Register SgprDst, Register VgprSrc) { |
| 209 | return B.buildIntrinsic(ID: Intrinsic::amdgcn_readfirstlane, Res: SgprDst) |
| 210 | .addReg(RegNo: VgprSrc); |
| 211 | }); |
| 212 | } |
| 213 | |