1//===- AMDGPUGlobalISelUtils.cpp ---------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUGlobalISelUtils.h"
10#include "AMDGPURegisterBankInfo.h"
11#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12#include "llvm/ADT/DenseSet.h"
13#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
14#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
15#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
16#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
17#include "llvm/CodeGenTypes/LowLevelType.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/IntrinsicsAMDGPU.h"
20
21using namespace llvm;
22using namespace AMDGPU;
23using namespace MIPatternMatch;
24
25std::pair<Register, unsigned>
26AMDGPU::getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg,
27 GISelValueTracking *ValueTracking,
28 bool CheckNUW) {
29 MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
30 if (Def->getOpcode() == TargetOpcode::G_CONSTANT) {
31 unsigned Offset;
32 const MachineOperand &Op = Def->getOperand(i: 1);
33 if (Op.isImm())
34 Offset = Op.getImm();
35 else
36 Offset = Op.getCImm()->getZExtValue();
37
38 return std::pair(Register(), Offset);
39 }
40
41 int64_t Offset;
42 if (Def->getOpcode() == TargetOpcode::G_ADD) {
43 // A 32-bit (address + offset) should not cause unsigned 32-bit integer
44 // wraparound, because s_load instructions perform the addition in 64 bits.
45 if (CheckNUW && !Def->getFlag(Flag: MachineInstr::NoUWrap)) {
46 assert(MRI.getType(Reg).getScalarSizeInBits() == 32);
47 return std::pair(Reg, 0);
48 }
49 // TODO: Handle G_OR used for add case
50 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI, P: m_ICst(Cst&: Offset)))
51 return std::pair(Def->getOperand(i: 1).getReg(), Offset);
52
53 // FIXME: matcher should ignore copies
54 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI, P: m_Copy(Src: m_ICst(Cst&: Offset))))
55 return std::pair(Def->getOperand(i: 1).getReg(), Offset);
56 }
57
58 Register Base;
59 if (ValueTracking && mi_match(R: Reg, MRI, P: m_GOr(L: m_Reg(R&: Base), R: m_ICst(Cst&: Offset))) &&
60 ValueTracking->maskedValueIsZero(Val: Base,
61 Mask: APInt(32, Offset, /*isSigned=*/true)))
62 return std::pair(Base, Offset);
63
64 // Handle G_PTRTOINT (G_PTR_ADD base, const) case
65 if (Def->getOpcode() == TargetOpcode::G_PTRTOINT) {
66 MachineInstr *Base;
67 Register PtrAdd = Def->getOperand(i: 1).getReg();
68 uint32_t Flags;
69 if (mi_match(R: PtrAdd, MRI,
70 P: m_GPtrAdd(L: m_MInstr(MI&: Base), R: m_ICst(Cst&: Offset), Flags: m_MIFlags(Flags)))) {
71 // Same check as for G_ADD; nuw comes from getelementptr inbounds.
72 if (CheckNUW && !(Flags & MachineInstr::NoUWrap)) {
73 assert(MRI.getType(Reg).getScalarSizeInBits() == 32);
74 return std::pair(Reg, 0);
75 }
76 // If Base was int converted to pointer, simply return int and offset.
77 if (Base->getOpcode() == TargetOpcode::G_INTTOPTR)
78 return std::pair(Base->getOperand(i: 1).getReg(), Offset);
79
80 // Register returned here will be of pointer type.
81 return std::pair(Base->getOperand(i: 0).getReg(), Offset);
82 }
83 }
84
85 return std::pair(Reg, 0);
86}
87
88IntrinsicLaneMaskAnalyzer::IntrinsicLaneMaskAnalyzer(MachineFunction &MF)
89 : MRI(MF.getRegInfo()) {
90 initLaneMaskIntrinsics(MF);
91}
92
93bool IntrinsicLaneMaskAnalyzer::isS32S64LaneMask(Register Reg) const {
94 return S32S64LaneMask.contains(V: Reg);
95}
96
97void IntrinsicLaneMaskAnalyzer::initLaneMaskIntrinsics(MachineFunction &MF) {
98 for (auto &MBB : MF) {
99 for (auto &MI : MBB) {
100 GIntrinsic *GI = dyn_cast<GIntrinsic>(Val: &MI);
101 if (GI && GI->is(ID: Intrinsic::amdgcn_if_break)) {
102 S32S64LaneMask.insert(V: MI.getOperand(i: 3).getReg());
103 S32S64LaneMask.insert(V: MI.getOperand(i: 0).getReg());
104 }
105
106 if (MI.getOpcode() == AMDGPU::SI_IF ||
107 MI.getOpcode() == AMDGPU::SI_ELSE) {
108 S32S64LaneMask.insert(V: MI.getOperand(i: 0).getReg());
109 }
110 }
111 }
112}
113
114static LLT getReadAnyLaneSplitTy(LLT Ty) {
115 if (Ty.isVector()) {
116 LLT ElTy = Ty.getElementType();
117 if (ElTy.getSizeInBits() == 16)
118 return LLT::fixed_vector(NumElements: 2, ScalarTy: ElTy);
119 // S32, S64 or pointer
120 return ElTy;
121 }
122
123 // Large scalars and 64-bit pointers
124 return LLT::integer(SizeInBits: 32);
125}
126
127template <typename ReadLaneFnTy>
128static Register buildReadLane(MachineIRBuilder &, Register,
129 const RegisterBankInfo &, ReadLaneFnTy);
130
131template <typename ReadLaneFnTy>
132static void
133unmergeReadAnyLane(MachineIRBuilder &B, SmallVectorImpl<Register> &SgprDstParts,
134 LLT UnmergeTy, Register VgprSrc, const RegisterBankInfo &RBI,
135 ReadLaneFnTy BuildRL) {
136 const RegisterBank *VgprRB = &RBI.getRegBank(ID: AMDGPU::VGPRRegBankID);
137 auto Unmerge = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: UnmergeTy}, Op: VgprSrc);
138 for (unsigned i = 0; i < Unmerge->getNumOperands() - 1; ++i) {
139 SgprDstParts.push_back(Elt: buildReadLane(B, Unmerge.getReg(Idx: i), RBI, BuildRL));
140 }
141}
142
143template <typename ReadLaneFnTy>
144static Register buildReadLane(MachineIRBuilder &B, Register VgprSrc,
145 const RegisterBankInfo &RBI,
146 ReadLaneFnTy BuildRL) {
147 LLT Ty = B.getMRI()->getType(Reg: VgprSrc);
148 const RegisterBank *SgprRB = &RBI.getRegBank(ID: AMDGPU::SGPRRegBankID);
149 if (Ty.getSizeInBits() == 32) {
150 Register SgprDst = B.getMRI()->createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty});
151 return BuildRL(B, SgprDst, VgprSrc).getReg(0);
152 }
153
154 SmallVector<Register, 8> SgprDstParts;
155 unmergeReadAnyLane(B, SgprDstParts, getReadAnyLaneSplitTy(Ty), VgprSrc, RBI,
156 BuildRL);
157
158 return B.buildMergeLikeInstr(Res: {SgprRB, Ty}, Ops: SgprDstParts).getReg(Idx: 0);
159}
160
161template <typename ReadLaneFnTy>
162static void buildReadLane(MachineIRBuilder &B, Register SgprDst,
163 Register VgprSrc, const RegisterBankInfo &RBI,
164 ReadLaneFnTy BuildReadLane) {
165 LLT Ty = B.getMRI()->getType(Reg: VgprSrc);
166 if (Ty.getSizeInBits() == 32) {
167 BuildReadLane(B, SgprDst, VgprSrc);
168 return;
169 }
170
171 SmallVector<Register, 8> SgprDstParts;
172 unmergeReadAnyLane(B, SgprDstParts, getReadAnyLaneSplitTy(Ty), VgprSrc, RBI,
173 BuildReadLane);
174
175 B.buildMergeLikeInstr(Res: SgprDst, Ops: SgprDstParts).getReg(Idx: 0);
176}
177
178void AMDGPU::buildReadAnyLane(MachineIRBuilder &B, Register SgprDst,
179 Register VgprSrc, const RegisterBankInfo &RBI) {
180 return buildReadLane(
181 B, SgprDst, VgprSrc, RBI,
182 BuildReadLane: [](MachineIRBuilder &B, Register SgprDst, Register VgprSrc) {
183 return B.buildInstr(Opc: AMDGPU::G_AMDGPU_READANYLANE, DstOps: {SgprDst}, SrcOps: {VgprSrc});
184 });
185}
186
187void AMDGPU::buildReadFirstLane(MachineIRBuilder &B, Register SgprDst,
188 Register VgprSrc, const RegisterBankInfo &RBI) {
189 return buildReadLane(
190 B, SgprDst, VgprSrc, RBI,
191 BuildReadLane: [](MachineIRBuilder &B, Register SgprDst, Register VgprSrc) {
192 return B.buildIntrinsic(ID: Intrinsic::amdgcn_readfirstlane, Res: SgprDst)
193 .addReg(RegNo: VgprSrc);
194 });
195}
196