1//===- AMDGPUGlobalISelUtils.cpp ---------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUGlobalISelUtils.h"
10#include "AMDGPURegisterBankInfo.h"
11#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12#include "llvm/ADT/DenseSet.h"
13#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
14#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
15#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
16#include "llvm/CodeGen/GlobalISel/Utils.h"
17#include "llvm/CodeGen/LowLevelTypeUtils.h"
18#include "llvm/CodeGenTypes/LowLevelType.h"
19#include "llvm/IR/Constants.h"
20#include "llvm/IR/IntrinsicsAMDGPU.h"
21
22using namespace llvm;
23using namespace AMDGPU;
24using namespace MIPatternMatch;
25
26bool AMDGPU::isUnscaledIEEEDivFmas(const MachineInstr &MI,
27 const MachineRegisterInfo &MRI) {
28 Register ScaleReg = MI.getOperand(i: 5).getReg();
29 MachineInstr *ScaleDef = getDefIgnoringCopies(Reg: ScaleReg, MRI);
30 if (ScaleDef && ScaleDef->getOpcode() == AMDGPU::G_AMDGPU_COPY_VCC_SCC)
31 ScaleReg = ScaleDef->getOperand(i: 1).getReg();
32 auto Scale = getIConstantVRegValWithLookThrough(VReg: ScaleReg, MRI);
33 if (!Scale || !Scale->Value.isZero())
34 return false;
35
36 const fltSemantics &Semantics =
37 getFltSemanticForLLT(Ty: MRI.getType(Reg: MI.getOperand(i: 0).getReg()));
38 return MI.getMF()->getDenormalMode(FPType: Semantics).Input == DenormalMode::IEEE;
39}
40
41std::pair<Register, unsigned>
42AMDGPU::getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg,
43 GISelValueTracking *ValueTracking,
44 bool CheckNUW) {
45 MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
46 if (Def->getOpcode() == TargetOpcode::G_CONSTANT) {
47 unsigned Offset;
48 const MachineOperand &Op = Def->getOperand(i: 1);
49 if (Op.isImm())
50 Offset = Op.getImm();
51 else
52 Offset = Op.getCImm()->getZExtValue();
53
54 return std::pair(Register(), Offset);
55 }
56
57 int64_t Offset;
58 if (Def->getOpcode() == TargetOpcode::G_ADD) {
59 // A 32-bit (address + offset) should not cause unsigned 32-bit integer
60 // wraparound, because s_load instructions perform the addition in 64 bits.
61 if (CheckNUW && !Def->getFlag(Flag: MachineInstr::NoUWrap)) {
62 assert(MRI.getType(Reg).getScalarSizeInBits() == 32);
63 return std::pair(Reg, 0);
64 }
65 // TODO: Handle G_OR used for add case
66 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI, P: m_ICst(Cst&: Offset)))
67 return std::pair(Def->getOperand(i: 1).getReg(), Offset);
68
69 // FIXME: matcher should ignore copies
70 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI, P: m_Copy(Src: m_ICst(Cst&: Offset))))
71 return std::pair(Def->getOperand(i: 1).getReg(), Offset);
72 }
73
74 Register Base;
75 if (mi_match(R: Reg, MRI, P: m_GOr(L: m_Reg(R&: Base), R: m_ICst(Cst&: Offset))) &&
76 (Def->getFlag(Flag: MachineInstr::Disjoint) ||
77 (ValueTracking && ValueTracking->maskedValueIsZero(
78 Val: Base, Mask: APInt(32, Offset, /*isSigned=*/true)))))
79 return std::pair(Base, Offset);
80
81 // Handle G_PTRTOINT (G_PTR_ADD base, const) case
82 if (Def->getOpcode() == TargetOpcode::G_PTRTOINT) {
83 MachineInstr *Base;
84 Register PtrAdd = Def->getOperand(i: 1).getReg();
85 uint32_t Flags;
86 if (mi_match(R: PtrAdd, MRI,
87 P: m_GPtrAdd(L: m_MInstr(MI&: Base), R: m_ICst(Cst&: Offset), Flags: m_MIFlags(Flags)))) {
88 // Same check as for G_ADD; nuw comes from getelementptr inbounds.
89 if (CheckNUW && !(Flags & MachineInstr::NoUWrap)) {
90 assert(MRI.getType(Reg).getScalarSizeInBits() == 32);
91 return std::pair(Reg, 0);
92 }
93 // If Base was int converted to pointer, simply return int and offset.
94 if (Base->getOpcode() == TargetOpcode::G_INTTOPTR)
95 return std::pair(Base->getOperand(i: 1).getReg(), Offset);
96
97 // Register returned here will be of pointer type.
98 return std::pair(Base->getOperand(i: 0).getReg(), Offset);
99 }
100 }
101
102 return std::pair(Reg, 0);
103}
104
105IntrinsicLaneMaskAnalyzer::IntrinsicLaneMaskAnalyzer(MachineFunction &MF)
106 : MRI(MF.getRegInfo()) {
107 initLaneMaskIntrinsics(MF);
108}
109
110bool IntrinsicLaneMaskAnalyzer::isS32S64LaneMask(Register Reg) const {
111 return S32S64LaneMask.contains(V: Reg);
112}
113
114void IntrinsicLaneMaskAnalyzer::initLaneMaskIntrinsics(MachineFunction &MF) {
115 for (auto &MBB : MF) {
116 for (auto &MI : MBB) {
117 GIntrinsic *GI = dyn_cast<GIntrinsic>(Val: &MI);
118 if (GI && GI->is(ID: Intrinsic::amdgcn_if_break)) {
119 S32S64LaneMask.insert(V: MI.getOperand(i: 3).getReg());
120 S32S64LaneMask.insert(V: MI.getOperand(i: 0).getReg());
121 }
122
123 if (MI.getOpcode() == AMDGPU::SI_IF ||
124 MI.getOpcode() == AMDGPU::SI_ELSE) {
125 S32S64LaneMask.insert(V: MI.getOperand(i: 0).getReg());
126 }
127 }
128 }
129}
130
131static LLT getReadAnyLaneSplitTy(LLT Ty) {
132 if (Ty.isVector()) {
133 LLT ElTy = Ty.getElementType();
134 if (ElTy.getSizeInBits() == 16)
135 return LLT::fixed_vector(NumElements: 2, ScalarTy: ElTy);
136 // S32, S64 or pointer
137 return ElTy;
138 }
139
140 // Large scalars and 64-bit pointers
141 return LLT::integer(SizeInBits: 32);
142}
143
144template <typename ReadLaneFnTy>
145static Register buildReadLane(MachineIRBuilder &, Register,
146 const RegisterBankInfo &, ReadLaneFnTy);
147
148template <typename ReadLaneFnTy>
149static void
150unmergeReadAnyLane(MachineIRBuilder &B, SmallVectorImpl<Register> &SgprDstParts,
151 LLT UnmergeTy, Register VgprSrc, const RegisterBankInfo &RBI,
152 ReadLaneFnTy BuildRL) {
153 const RegisterBank *VgprRB = &RBI.getRegBank(ID: AMDGPU::VGPRRegBankID);
154 auto Unmerge = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: UnmergeTy}, Op: VgprSrc);
155 for (unsigned i = 0; i < Unmerge->getNumOperands() - 1; ++i) {
156 SgprDstParts.push_back(Elt: buildReadLane(B, Unmerge.getReg(Idx: i), RBI, BuildRL));
157 }
158}
159
160template <typename ReadLaneFnTy>
161static Register buildReadLane(MachineIRBuilder &B, Register VgprSrc,
162 const RegisterBankInfo &RBI,
163 ReadLaneFnTy BuildRL) {
164 LLT Ty = B.getMRI()->getType(Reg: VgprSrc);
165 const RegisterBank *SgprRB = &RBI.getRegBank(ID: AMDGPU::SGPRRegBankID);
166 if (Ty.getSizeInBits() == 32) {
167 Register SgprDst = B.getMRI()->createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty});
168 return BuildRL(B, SgprDst, VgprSrc).getReg(0);
169 }
170
171 SmallVector<Register, 8> SgprDstParts;
172 unmergeReadAnyLane(B, SgprDstParts, getReadAnyLaneSplitTy(Ty), VgprSrc, RBI,
173 BuildRL);
174
175 return B.buildMergeLikeInstr(Res: {SgprRB, Ty}, Ops: SgprDstParts).getReg(Idx: 0);
176}
177
178template <typename ReadLaneFnTy>
179static void buildReadLane(MachineIRBuilder &B, Register SgprDst,
180 Register VgprSrc, const RegisterBankInfo &RBI,
181 ReadLaneFnTy BuildReadLane) {
182 LLT Ty = B.getMRI()->getType(Reg: VgprSrc);
183 if (Ty.getSizeInBits() == 32) {
184 BuildReadLane(B, SgprDst, VgprSrc);
185 return;
186 }
187
188 SmallVector<Register, 8> SgprDstParts;
189 unmergeReadAnyLane(B, SgprDstParts, getReadAnyLaneSplitTy(Ty), VgprSrc, RBI,
190 BuildReadLane);
191
192 B.buildMergeLikeInstr(Res: SgprDst, Ops: SgprDstParts).getReg(Idx: 0);
193}
194
195void AMDGPU::buildReadAnyLane(MachineIRBuilder &B, Register SgprDst,
196 Register VgprSrc, const RegisterBankInfo &RBI) {
197 return buildReadLane(
198 B, SgprDst, VgprSrc, RBI,
199 BuildReadLane: [](MachineIRBuilder &B, Register SgprDst, Register VgprSrc) {
200 return B.buildInstr(Opc: AMDGPU::G_AMDGPU_READANYLANE, DstOps: {SgprDst}, SrcOps: {VgprSrc});
201 });
202}
203
204void AMDGPU::buildReadFirstLane(MachineIRBuilder &B, Register SgprDst,
205 Register VgprSrc, const RegisterBankInfo &RBI) {
206 return buildReadLane(
207 B, SgprDst, VgprSrc, RBI,
208 BuildReadLane: [](MachineIRBuilder &B, Register SgprDst, Register VgprSrc) {
209 return B.buildIntrinsic(ID: Intrinsic::amdgcn_readfirstlane, Res: SgprDst)
210 .addReg(RegNo: VgprSrc);
211 });
212}
213