1//=== lib/CodeGen/GlobalISel/AMDGPURegBankCombiner.cpp ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass does combining of machine instructions at the generic MI level,
10// after register banks are known.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "AMDGPULegalizerInfo.h"
16#include "AMDGPURegisterBankInfo.h"
17#include "GCNSubtarget.h"
18#include "SIMachineFunctionInfo.h"
19#include "llvm/CodeGen/GlobalISel/Combiner.h"
20#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
21#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
22#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
23#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
24#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
25#include "llvm/CodeGen/MachineDominators.h"
26#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
27#include "llvm/CodeGen/MachinePassManager.h"
28#include "llvm/CodeGen/TargetPassConfig.h"
29#include "llvm/Target/TargetMachine.h"
30
31#define GET_GICOMBINER_DEPS
32#include "AMDGPUGenPreLegalizeGICombiner.inc"
33#undef GET_GICOMBINER_DEPS
34
35#define DEBUG_TYPE "amdgpu-regbank-combiner"
36
37using namespace llvm;
38using namespace MIPatternMatch;
39
40namespace {
41#define GET_GICOMBINER_TYPES
42#include "AMDGPUGenRegBankGICombiner.inc"
43#undef GET_GICOMBINER_TYPES
44
45class AMDGPURegBankCombinerImpl : public Combiner {
46protected:
47 const AMDGPURegBankCombinerImplRuleConfig &RuleConfig;
48 const GCNSubtarget &STI;
49 const RegisterBankInfo &RBI;
50 const TargetRegisterInfo &TRI;
51 const SIInstrInfo &TII;
52 const CombinerHelper Helper;
53
54public:
55 AMDGPURegBankCombinerImpl(
56 MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &VT,
57 GISelCSEInfo *CSEInfo,
58 const AMDGPURegBankCombinerImplRuleConfig &RuleConfig,
59 const GCNSubtarget &STI, MachineDominatorTree *MDT,
60 const LegalizerInfo *LI);
61
62 static const char *getName() { return "AMDGPURegBankCombinerImpl"; }
63
64 bool tryCombineAll(MachineInstr &I) const override;
65
66 bool isVgprRegBank(Register Reg) const;
67 Register getAsVgpr(Register Reg) const;
68
69 struct MinMaxMedOpc {
70 unsigned Min, Max, Med;
71 };
72
73 struct Med3MatchInfo {
74 unsigned Opc;
75 Register Val0, Val1, Val2;
76 };
77
78 struct MinMaxToMinMax3MatchInfo {
79 unsigned Opc;
80 Register Val0, Val1, Val2;
81 };
82
83 MinMaxMedOpc getMinMaxPair(unsigned Opc) const;
84
85 template <class m_Cst, typename CstTy>
86 bool matchMed(MachineInstr &MI, MachineRegisterInfo &MRI, MinMaxMedOpc MMMOpc,
87 Register &Val, CstTy &K0, CstTy &K1) const;
88
89 bool matchIntMinMaxToMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo) const;
90 bool matchFPMinMaxToMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo) const;
91 bool matchFPMinMaxToClamp(MachineInstr &MI, Register &Reg) const;
92 bool matchFPMed3ToClamp(MachineInstr &MI, Register &Reg) const;
93 void applyMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo) const;
94 void applyClamp(MachineInstr &MI, Register &Reg) const;
95
96 void applyCanonicalizeZextShiftAmt(MachineInstr &MI, MachineInstr &Ext) const;
97
98 bool combineD16Load(MachineInstr &MI) const;
99 bool applyD16Load(unsigned D16Opc, MachineInstr &DstMI,
100 MachineInstr *SmallLoad, Register ToOverwriteD16) const;
101
102 bool matchMinMaxToMinMax3(MachineInstr &MI,
103 MinMaxToMinMax3MatchInfo &MatchInfo) const;
104 void applyMinMaxToMinMax3(MachineInstr &MI,
105 MinMaxToMinMax3MatchInfo &MatchInfo) const;
106
107private:
108 SIModeRegisterDefaults getMode() const;
109 bool getIEEE() const;
110 bool getDX10Clamp() const;
111 bool isFminnumIeee(const MachineInstr &MI) const;
112 bool isFCst(MachineInstr *MI) const;
113 bool isClampZeroToOne(MachineInstr *K0, MachineInstr *K1) const;
114
115#define GET_GICOMBINER_CLASS_MEMBERS
116#define AMDGPUSubtarget GCNSubtarget
117#include "AMDGPUGenRegBankGICombiner.inc"
118#undef GET_GICOMBINER_CLASS_MEMBERS
119#undef AMDGPUSubtarget
120};
121
122#define GET_GICOMBINER_IMPL
123#define AMDGPUSubtarget GCNSubtarget
124#include "AMDGPUGenRegBankGICombiner.inc"
125#undef AMDGPUSubtarget
126#undef GET_GICOMBINER_IMPL
127
128AMDGPURegBankCombinerImpl::AMDGPURegBankCombinerImpl(
129 MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &VT,
130 GISelCSEInfo *CSEInfo,
131 const AMDGPURegBankCombinerImplRuleConfig &RuleConfig,
132 const GCNSubtarget &STI, MachineDominatorTree *MDT, const LegalizerInfo *LI)
133 : Combiner(MF, CInfo, &VT, CSEInfo), RuleConfig(RuleConfig), STI(STI),
134 RBI(*STI.getRegBankInfo()), TRI(*STI.getRegisterInfo()),
135 TII(*STI.getInstrInfo()),
136 Helper(Observer, B, /*IsPreLegalize*/ false, &VT, MDT, LI),
137#define GET_GICOMBINER_CONSTRUCTOR_INITS
138#include "AMDGPUGenRegBankGICombiner.inc"
139#undef GET_GICOMBINER_CONSTRUCTOR_INITS
140{
141}
142
143bool AMDGPURegBankCombinerImpl::isVgprRegBank(Register Reg) const {
144 return RBI.getRegBank(Reg, MRI, TRI)->getID() == AMDGPU::VGPRRegBankID;
145}
146
147Register AMDGPURegBankCombinerImpl::getAsVgpr(Register Reg) const {
148 if (isVgprRegBank(Reg))
149 return Reg;
150
151 // Search for existing copy of Reg to vgpr.
152 for (MachineInstr &Use : MRI.use_instructions(Reg)) {
153 Register Def = Use.getOperand(i: 0).getReg();
154 if (Use.getOpcode() == AMDGPU::COPY && isVgprRegBank(Reg: Def))
155 return Def;
156 }
157
158 // Copy Reg to vgpr.
159 Register VgprReg = B.buildCopy(Res: MRI.getType(Reg), Op: Reg).getReg(Idx: 0);
160 MRI.setRegBank(Reg: VgprReg, RegBank: RBI.getRegBank(ID: AMDGPU::VGPRRegBankID));
161 return VgprReg;
162}
163
164AMDGPURegBankCombinerImpl::MinMaxMedOpc
165AMDGPURegBankCombinerImpl::getMinMaxPair(unsigned Opc) const {
166 switch (Opc) {
167 default:
168 llvm_unreachable("Unsupported opcode");
169 case AMDGPU::G_SMAX:
170 case AMDGPU::G_SMIN:
171 return {.Min: AMDGPU::G_SMIN, .Max: AMDGPU::G_SMAX, .Med: AMDGPU::G_AMDGPU_SMED3};
172 case AMDGPU::G_UMAX:
173 case AMDGPU::G_UMIN:
174 return {.Min: AMDGPU::G_UMIN, .Max: AMDGPU::G_UMAX, .Med: AMDGPU::G_AMDGPU_UMED3};
175 case AMDGPU::G_FMAXNUM:
176 case AMDGPU::G_FMINNUM:
177 return {.Min: AMDGPU::G_FMINNUM, .Max: AMDGPU::G_FMAXNUM, .Med: AMDGPU::G_AMDGPU_FMED3};
178 case AMDGPU::G_FMAXNUM_IEEE:
179 case AMDGPU::G_FMINNUM_IEEE:
180 return {.Min: AMDGPU::G_FMINNUM_IEEE, .Max: AMDGPU::G_FMAXNUM_IEEE,
181 .Med: AMDGPU::G_AMDGPU_FMED3};
182 }
183}
184
185template <class m_Cst, typename CstTy>
186bool AMDGPURegBankCombinerImpl::matchMed(MachineInstr &MI,
187 MachineRegisterInfo &MRI,
188 MinMaxMedOpc MMMOpc, Register &Val,
189 CstTy &K0, CstTy &K1) const {
190 // 4 operand commutes of: min(max(Val, K0), K1).
191 // Find K1 from outer instr: min(max(...), K1) or min(K1, max(...)).
192 // Find K0 and Val from inner instr: max(K0, Val) or max(Val, K0).
193 // 4 operand commutes of: max(min(Val, K1), K0).
194 // Find K0 from outer instr: max(min(...), K0) or max(K0, min(...)).
195 // Find K1 and Val from inner instr: min(K1, Val) or min(Val, K1).
196 return mi_match(
197 MI, MRI,
198 m_any_of(
199 m_CommutativeBinOp(
200 MMMOpc.Min, m_CommutativeBinOp(MMMOpc.Max, m_Reg(R&: Val), m_Cst(K0)),
201 m_Cst(K1)),
202 m_CommutativeBinOp(
203 MMMOpc.Max, m_CommutativeBinOp(MMMOpc.Min, m_Reg(R&: Val), m_Cst(K1)),
204 m_Cst(K0))));
205}
206
207bool AMDGPURegBankCombinerImpl::matchIntMinMaxToMed3(
208 MachineInstr &MI, Med3MatchInfo &MatchInfo) const {
209 Register Dst = MI.getOperand(i: 0).getReg();
210 if (!isVgprRegBank(Reg: Dst))
211 return false;
212
213 // med3 for i16 is only available on gfx9+, and not available for v2i16.
214 LLT Ty = MRI.getType(Reg: Dst);
215 if ((Ty != LLT::scalar(SizeInBits: 16) || !STI.hasMed3_16()) && Ty != LLT::scalar(SizeInBits: 32))
216 return false;
217
218 MinMaxMedOpc OpcodeTriple = getMinMaxPair(Opc: MI.getOpcode());
219 Register Val;
220 std::optional<ValueAndVReg> K0, K1;
221 // Match min(max(Val, K0), K1) or max(min(Val, K1), K0). Then see if K0 <= K1.
222 if (!matchMed<GCstAndRegMatch>(MI, MRI, MMMOpc: OpcodeTriple, Val, K0, K1))
223 return false;
224
225 if (OpcodeTriple.Med == AMDGPU::G_AMDGPU_SMED3 && K0->Value.sgt(RHS: K1->Value))
226 return false;
227 if (OpcodeTriple.Med == AMDGPU::G_AMDGPU_UMED3 && K0->Value.ugt(RHS: K1->Value))
228 return false;
229
230 MatchInfo = {.Opc: OpcodeTriple.Med, .Val0: Val, .Val1: K0->VReg, .Val2: K1->VReg};
231 return true;
232}
233
234// fmed3(NaN, K0, K1) = min(min(NaN, K0), K1)
235// ieee = true : min/max(SNaN, K) = QNaN, min/max(QNaN, K) = K
236// ieee = false : min/max(NaN, K) = K
237// clamp(NaN) = dx10_clamp ? 0.0 : NaN
238// Consider values of min(max(Val, K0), K1) and max(min(Val, K1), K0) as input.
239// Other operand commutes (see matchMed) give same result since min and max are
240// commutative.
241
242// Try to replace fp min(max(Val, K0), K1) or max(min(Val, K1), K0), KO<=K1
243// with fmed3(Val, K0, K1) or clamp(Val). Clamp requires K0 = 0.0 and K1 = 1.0.
244// Val = SNaN only for ieee = true
245// fmed3(SNaN, K0, K1) = min(min(SNaN, K0), K1) = min(QNaN, K1) = K1
246// min(max(SNaN, K0), K1) = min(QNaN, K1) = K1
247// max(min(SNaN, K1), K0) = max(K1, K0) = K1
248// Val = NaN,ieee = false or Val = QNaN,ieee = true
249// fmed3(NaN, K0, K1) = min(min(NaN, K0), K1) = min(K0, K1) = K0
250// min(max(NaN, K0), K1) = min(K0, K1) = K0 (can clamp when dx10_clamp = true)
251// max(min(NaN, K1), K0) = max(K1, K0) = K1 != K0
252bool AMDGPURegBankCombinerImpl::matchFPMinMaxToMed3(
253 MachineInstr &MI, Med3MatchInfo &MatchInfo) const {
254 Register Dst = MI.getOperand(i: 0).getReg();
255 // Perform combine only when the destination is a VGPR.
256 if (!isVgprRegBank(Reg: Dst))
257 return false;
258
259 LLT Ty = MRI.getType(Reg: Dst);
260
261 // med3 for f16 is only available on gfx9+, and not available for v2f16.
262 if ((Ty != LLT::scalar(SizeInBits: 16) || !STI.hasMed3_16()) && Ty != LLT::scalar(SizeInBits: 32))
263 return false;
264
265 auto OpcodeTriple = getMinMaxPair(Opc: MI.getOpcode());
266
267 Register Val;
268 std::optional<FPValueAndVReg> K0, K1;
269 // Match min(max(Val, K0), K1) or max(min(Val, K1), K0). Then see if K0 <= K1.
270 if (!matchMed<GFCstAndRegMatch>(MI, MRI, MMMOpc: OpcodeTriple, Val, K0, K1))
271 return false;
272
273 if (K0->Value > K1->Value)
274 return false;
275
276 // For IEEE=false perform combine only when it's safe to assume that there are
277 // no NaN inputs. Most often MI is marked with nnan fast math flag.
278 // For IEEE=true consider NaN inputs. fmed3(NaN, K0, K1) is equivalent to
279 // min(min(NaN, K0), K1). Safe to fold for min(max(Val, K0), K1) since inner
280 // nodes(max/min) have same behavior when one input is NaN and other isn't.
281 // Don't consider max(min(SNaN, K1), K0) since there is no isKnownNeverQNaN,
282 // also post-legalizer inputs to min/max are fcanonicalized (never SNaN).
283 if ((getIEEE() && isFminnumIeee(MI)) || VT->isKnownNeverNaN(Val: Dst)) {
284 // Don't fold single use constant that can't be inlined.
285 if ((!MRI.hasOneNonDBGUse(RegNo: K0->VReg) || TII.isInlineConstant(Imm: K0->Value)) &&
286 (!MRI.hasOneNonDBGUse(RegNo: K1->VReg) || TII.isInlineConstant(Imm: K1->Value))) {
287 MatchInfo = {.Opc: OpcodeTriple.Med, .Val0: Val, .Val1: K0->VReg, .Val2: K1->VReg};
288 return true;
289 }
290 }
291
292 return false;
293}
294
295bool AMDGPURegBankCombinerImpl::matchFPMinMaxToClamp(MachineInstr &MI,
296 Register &Reg) const {
297 // Perform combine only when the destination is a VGPR.
298 if (!isVgprRegBank(Reg: MI.getOperand(i: 0).getReg()))
299 return false;
300
301 // Clamp is available on all types after regbankselect (f16, f32, f64, v2f16).
302 auto OpcodeTriple = getMinMaxPair(Opc: MI.getOpcode());
303 Register Val;
304 std::optional<FPValueAndVReg> K0, K1;
305 // Match min(max(Val, K0), K1) or max(min(Val, K1), K0).
306 if (!matchMed<GFCstOrSplatGFCstMatch>(MI, MRI, MMMOpc: OpcodeTriple, Val, K0, K1))
307 return false;
308
309 if (!K0->Value.isPosZero() || !K1->Value.isOne())
310 return false;
311
312 // For IEEE=false perform combine only when it's safe to assume that there are
313 // no NaN inputs. Most often MI is marked with nnan fast math flag.
314 // For IEEE=true consider NaN inputs. Only min(max(QNaN, 0.0), 1.0) evaluates
315 // to 0.0 requires dx10_clamp = true.
316 if ((getIEEE() && getDX10Clamp() && isFminnumIeee(MI) &&
317 VT->isKnownNeverSNaN(Val)) ||
318 VT->isKnownNeverNaN(Val: MI.getOperand(i: 0).getReg())) {
319 Reg = Val;
320 return true;
321 }
322
323 return false;
324}
325
326// Replacing fmed3(NaN, 0.0, 1.0) with clamp. Requires dx10_clamp = true.
327// Val = SNaN only for ieee = true. It is important which operand is NaN.
328// min(min(SNaN, 0.0), 1.0) = min(QNaN, 1.0) = 1.0
329// min(min(SNaN, 1.0), 0.0) = min(QNaN, 0.0) = 0.0
330// min(min(0.0, 1.0), SNaN) = min(0.0, SNaN) = QNaN
331// Val = NaN,ieee = false or Val = QNaN,ieee = true
332// min(min(NaN, 0.0), 1.0) = min(0.0, 1.0) = 0.0
333// min(min(NaN, 1.0), 0.0) = min(1.0, 0.0) = 0.0
334// min(min(0.0, 1.0), NaN) = min(0.0, NaN) = 0.0
335bool AMDGPURegBankCombinerImpl::matchFPMed3ToClamp(MachineInstr &MI,
336 Register &Reg) const {
337 // Perform combine only when the destination is a VGPR.
338 if (!isVgprRegBank(Reg: MI.getOperand(i: 0).getReg()))
339 return false;
340
341 // In llvm-ir, clamp is often represented as an intrinsic call to
342 // @llvm.amdgcn.fmed3.f32(%Val, 0.0, 1.0). Check for other operand orders.
343 MachineInstr *Src0 = getDefIgnoringCopies(Reg: MI.getOperand(i: 1).getReg(), MRI);
344 MachineInstr *Src1 = getDefIgnoringCopies(Reg: MI.getOperand(i: 2).getReg(), MRI);
345 MachineInstr *Src2 = getDefIgnoringCopies(Reg: MI.getOperand(i: 3).getReg(), MRI);
346
347 if (isFCst(MI: Src0) && !isFCst(MI: Src1))
348 std::swap(a&: Src0, b&: Src1);
349 if (isFCst(MI: Src1) && !isFCst(MI: Src2))
350 std::swap(a&: Src1, b&: Src2);
351 if (isFCst(MI: Src0) && !isFCst(MI: Src1))
352 std::swap(a&: Src0, b&: Src1);
353 if (!isClampZeroToOne(K0: Src1, K1: Src2))
354 return false;
355
356 Register Val = Src0->getOperand(i: 0).getReg();
357
358 auto isOp3Zero = [&]() {
359 MachineInstr *Op3 = getDefIgnoringCopies(Reg: MI.getOperand(i: 3).getReg(), MRI);
360 if (Op3->getOpcode() == TargetOpcode::G_FCONSTANT)
361 return Op3->getOperand(i: 1).getFPImm()->isPosZero();
362 return false;
363 };
364 // For IEEE=false perform combine only when it's safe to assume that there are
365 // no NaN inputs. Most often MI is marked with nnan fast math flag.
366 // For IEEE=true consider NaN inputs. Requires dx10_clamp = true. Safe to fold
367 // when Val could be QNaN. If Val can also be SNaN third input should be 0.0.
368 if (VT->isKnownNeverNaN(Val: MI.getOperand(i: 0).getReg()) ||
369 (getIEEE() && getDX10Clamp() &&
370 (VT->isKnownNeverSNaN(Val) || isOp3Zero()))) {
371 Reg = Val;
372 return true;
373 }
374
375 return false;
376}
377
378void AMDGPURegBankCombinerImpl::applyClamp(MachineInstr &MI,
379 Register &Reg) const {
380 B.buildInstr(Opc: AMDGPU::G_AMDGPU_CLAMP, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Reg},
381 Flags: MI.getFlags());
382 MI.eraseFromParent();
383}
384
385void AMDGPURegBankCombinerImpl::applyMed3(MachineInstr &MI,
386 Med3MatchInfo &MatchInfo) const {
387 B.buildInstr(Opc: MatchInfo.Opc, DstOps: {MI.getOperand(i: 0)},
388 SrcOps: {getAsVgpr(Reg: MatchInfo.Val0), getAsVgpr(Reg: MatchInfo.Val1),
389 getAsVgpr(Reg: MatchInfo.Val2)},
390 Flags: MI.getFlags());
391 MI.eraseFromParent();
392}
393
394void AMDGPURegBankCombinerImpl::applyCanonicalizeZextShiftAmt(
395 MachineInstr &MI, MachineInstr &Ext) const {
396 unsigned ShOpc = MI.getOpcode();
397 assert(ShOpc == AMDGPU::G_SHL || ShOpc == AMDGPU::G_LSHR ||
398 ShOpc == AMDGPU::G_ASHR);
399 assert(Ext.getOpcode() == AMDGPU::G_ZEXT);
400
401 Register AmtReg = Ext.getOperand(i: 1).getReg();
402 Register ShDst = MI.getOperand(i: 0).getReg();
403 Register ShSrc = MI.getOperand(i: 1).getReg();
404
405 LLT ExtAmtTy = MRI.getType(Reg: Ext.getOperand(i: 0).getReg());
406 LLT AmtTy = MRI.getType(Reg: AmtReg);
407
408 auto &RB = *MRI.getRegBank(Reg: AmtReg);
409
410 auto NewExt = B.buildAnyExt(Res: ExtAmtTy, Op: AmtReg);
411 auto Mask = B.buildConstant(
412 Res: ExtAmtTy, Val: maskTrailingOnes<uint64_t>(N: AmtTy.getScalarSizeInBits()));
413 auto And = B.buildAnd(Dst: ExtAmtTy, Src0: NewExt, Src1: Mask);
414 B.buildInstr(Opc: ShOpc, DstOps: {ShDst}, SrcOps: {ShSrc, And});
415
416 MRI.setRegBank(Reg: NewExt.getReg(Idx: 0), RegBank: RB);
417 MRI.setRegBank(Reg: Mask.getReg(Idx: 0), RegBank: RB);
418 MRI.setRegBank(Reg: And.getReg(Idx: 0), RegBank: RB);
419 MI.eraseFromParent();
420}
421
422bool AMDGPURegBankCombinerImpl::combineD16Load(MachineInstr &MI) const {
423 Register Dst;
424 MachineInstr *Load, *SextLoad;
425 const int64_t CleanLo16 = 0xFFFFFFFFFFFF0000;
426 const int64_t CleanHi16 = 0x000000000000FFFF;
427
428 // Load lo
429 if (mi_match(R: MI.getOperand(i: 1).getReg(), MRI,
430 P: m_GOr(L: m_GAnd(L: m_GBitcast(Src: m_Reg(R&: Dst)),
431 R: m_Copy(Src: m_SpecificICst(RequestedValue: CleanLo16))),
432 R: m_MInstr(MI&: Load)))) {
433
434 if (Load->getOpcode() == AMDGPU::G_ZEXTLOAD) {
435 const MachineMemOperand *MMO = *Load->memoperands_begin();
436 unsigned LoadSize = MMO->getSizeInBits().getValue();
437 if (LoadSize == 8)
438 return applyD16Load(D16Opc: AMDGPU::G_AMDGPU_LOAD_D16_LO_U8, DstMI&: MI, SmallLoad: Load, ToOverwriteD16: Dst);
439 if (LoadSize == 16)
440 return applyD16Load(D16Opc: AMDGPU::G_AMDGPU_LOAD_D16_LO, DstMI&: MI, SmallLoad: Load, ToOverwriteD16: Dst);
441 return false;
442 }
443
444 // s32 Load_lo16 holds SextLoad i8, Load_hi16 is zero.
445 // fake16: and (sextload i8 -> s32), 0xFFFF
446 // true16: zext (sextload i8 -> s16) -> s32
447 if (mi_match(
448 R: Load, MRI,
449 P: m_GAnd(L: m_MInstr(MI&: SextLoad), R: m_Copy(Src: m_SpecificICst(RequestedValue: CleanHi16)))) ||
450 mi_match(R: Load, MRI,
451 P: m_GZExt(Src: m_all_of(preds: m_SpecificType(Ty: LLT::scalar(SizeInBits: 16)),
452 preds: m_MInstr(MI&: SextLoad))))) {
453 if (SextLoad->getOpcode() != AMDGPU::G_SEXTLOAD)
454 return false;
455
456 const MachineMemOperand *MMO = *SextLoad->memoperands_begin();
457 if (MMO->getSizeInBits().getValue() != 8)
458 return false;
459
460 return applyD16Load(D16Opc: AMDGPU::G_AMDGPU_LOAD_D16_LO_I8, DstMI&: MI, SmallLoad: SextLoad, ToOverwriteD16: Dst);
461 }
462
463 return false;
464 }
465
466 // Load hi
467 if (mi_match(R: MI.getOperand(i: 1).getReg(), MRI,
468 P: m_GOr(L: m_GAnd(L: m_GBitcast(Src: m_Reg(R&: Dst)),
469 R: m_Copy(Src: m_SpecificICst(RequestedValue: CleanHi16))),
470 R: m_GShl(L: m_MInstr(MI&: Load), R: m_Copy(Src: m_SpecificICst(RequestedValue: 16)))))) {
471
472 if (Load->getOpcode() == AMDGPU::G_ZEXTLOAD) {
473 const MachineMemOperand *MMO = *Load->memoperands_begin();
474 unsigned LoadSize = MMO->getSizeInBits().getValue();
475 if (LoadSize == 8)
476 return applyD16Load(D16Opc: AMDGPU::G_AMDGPU_LOAD_D16_HI_U8, DstMI&: MI, SmallLoad: Load, ToOverwriteD16: Dst);
477 if (LoadSize == 16)
478 return applyD16Load(D16Opc: AMDGPU::G_AMDGPU_LOAD_D16_HI, DstMI&: MI, SmallLoad: Load, ToOverwriteD16: Dst);
479 return false;
480 }
481
482 // s32 Load_lo16 holds SextLoad i8, Load_hi16 is zero.
483 // fake16: and (sextload i8 -> s32), 0xFFFF
484 // true16: zext (sextload i8 -> s16) -> s32
485 if (mi_match(
486 R: Load, MRI,
487 P: m_GAnd(L: m_MInstr(MI&: SextLoad), R: m_Copy(Src: m_SpecificICst(RequestedValue: CleanHi16)))) ||
488 mi_match(R: Load, MRI,
489 P: m_GZExt(Src: m_all_of(preds: m_SpecificType(Ty: LLT::scalar(SizeInBits: 16)),
490 preds: m_MInstr(MI&: SextLoad))))) {
491 if (SextLoad->getOpcode() != AMDGPU::G_SEXTLOAD)
492 return false;
493
494 const MachineMemOperand *MMO = *SextLoad->memoperands_begin();
495 if (MMO->getSizeInBits().getValue() != 8)
496 return false;
497
498 return applyD16Load(D16Opc: AMDGPU::G_AMDGPU_LOAD_D16_HI_I8, DstMI&: MI, SmallLoad: SextLoad, ToOverwriteD16: Dst);
499 }
500
501 return false;
502 }
503
504 return false;
505}
506
507void AMDGPURegBankCombinerImpl::applyMinMaxToMinMax3(
508 MachineInstr &MI, MinMaxToMinMax3MatchInfo &MatchInfo) const {
509 B.buildInstr(Opc: MatchInfo.Opc, DstOps: {MI.getOperand(i: 0)},
510 SrcOps: {MatchInfo.Val0, MatchInfo.Val1, MatchInfo.Val2}, Flags: MI.getFlags());
511 MI.eraseFromParent();
512 return;
513}
514
515// min(min(a, b), c) == min(a, min(b, c)) == min3(a, b, c)
516// supported scalar type: S32 S16 U32 U16 F32 F16
517bool AMDGPURegBankCombinerImpl::matchMinMaxToMinMax3(
518 MachineInstr &MI, MinMaxToMinMax3MatchInfo &MatchInfo) const {
519 Register Dst = MI.getOperand(i: 0).getReg();
520 Register Src1 = MI.getOperand(i: 1).getReg();
521 Register Src2 = MI.getOperand(i: 2).getReg();
522 // If the register is SGPR, don't optimize it.
523 if (!(isVgprRegBank(Reg: Dst) && isVgprRegBank(Reg: Src1) && isVgprRegBank(Reg: Src2))) {
524 return false;
525 }
526
527 LLT Ty = MRI.getType(Reg: Dst);
528 unsigned Opc = MI.getOpcode();
529 if (!(Ty == LLT::scalar(SizeInBits: 32) ||
530 (Ty == LLT::scalar(SizeInBits: 16) && STI.hasMin3Max3_16())))
531 return false;
532
533 Register R0, R1, R2;
534 if (!mi_match(MI, MRI,
535 P: m_CommutativeBinOp(
536 Opcode: Opc, L: m_OneNonDBGUse(SP: m_BinOp(Opcode: Opc, L: m_Reg(R&: R0), R: m_Reg(R&: R1))),
537 R: m_Reg(R&: R2)))) {
538 return false;
539 }
540
541 unsigned AMDGPUOpc = 0;
542 switch (Opc) {
543 case AMDGPU::G_SMAX:
544 AMDGPUOpc = AMDGPU::G_AMDGPU_SMAX3;
545 break;
546 case AMDGPU::G_SMIN:
547 AMDGPUOpc = AMDGPU::G_AMDGPU_SMIN3;
548 break;
549 case AMDGPU::G_UMAX:
550 AMDGPUOpc = AMDGPU::G_AMDGPU_UMAX3;
551 break;
552 case AMDGPU::G_UMIN:
553 AMDGPUOpc = AMDGPU::G_AMDGPU_UMIN3;
554 break;
555 case AMDGPU::G_FMAXNUM:
556 case AMDGPU::G_FMAXNUM_IEEE:
557 AMDGPUOpc = AMDGPU::G_AMDGPU_FMAX3;
558 break;
559 case AMDGPU::G_FMINNUM:
560 case AMDGPU::G_FMINNUM_IEEE:
561 AMDGPUOpc = AMDGPU::G_AMDGPU_FMIN3;
562 break;
563 case AMDGPU::G_FMAXIMUM:
564 case AMDGPU::G_FMAXIMUMNUM:
565 AMDGPUOpc = AMDGPU::G_AMDGPU_FMAXIMUM3;
566 break;
567 case AMDGPU::G_FMINIMUM:
568 case AMDGPU::G_FMINIMUMNUM:
569 AMDGPUOpc = AMDGPU::G_AMDGPU_FMINIMUM3;
570 break;
571 default:
572 return false;
573 }
574
575 MatchInfo = {.Opc: AMDGPUOpc, .Val0: R0, .Val1: R1, .Val2: R2};
576 return true;
577}
578
579bool AMDGPURegBankCombinerImpl::applyD16Load(
580 unsigned D16Opc, MachineInstr &DstMI, MachineInstr *SmallLoad,
581 Register SrcReg32ToOverwriteD16) const {
582 Register DstReg = DstMI.getOperand(i: 0).getReg();
583 LLT SrcTy = MRI.getType(Reg: SrcReg32ToOverwriteD16);
584
585 // Dst and Src for D16 load need to have same type.
586 Register D16Dst =
587 SrcTy == MRI.getType(Reg: DstReg)
588 ? DstReg
589 : MRI.createVirtualRegister(RegAttr: {.RCOrRB: MRI.getRegBank(Reg: DstReg), .Ty: SrcTy});
590
591 B.buildInstr(Opc: D16Opc, DstOps: {D16Dst},
592 SrcOps: {SmallLoad->getOperand(i: 1).getReg(), SrcReg32ToOverwriteD16})
593 .setMemRefs(SmallLoad->memoperands());
594
595 if (D16Dst != DstReg)
596 B.buildBitcast(Dst: DstReg, Src: D16Dst);
597
598 DstMI.eraseFromParent();
599 return true;
600}
601
602SIModeRegisterDefaults AMDGPURegBankCombinerImpl::getMode() const {
603 return MF.getInfo<SIMachineFunctionInfo>()->getMode();
604}
605
606bool AMDGPURegBankCombinerImpl::getIEEE() const { return getMode().IEEE; }
607
608bool AMDGPURegBankCombinerImpl::getDX10Clamp() const {
609 return getMode().DX10Clamp;
610}
611
612bool AMDGPURegBankCombinerImpl::isFminnumIeee(const MachineInstr &MI) const {
613 return MI.getOpcode() == AMDGPU::G_FMINNUM_IEEE;
614}
615
616bool AMDGPURegBankCombinerImpl::isFCst(MachineInstr *MI) const {
617 return MI->getOpcode() == AMDGPU::G_FCONSTANT;
618}
619
620bool AMDGPURegBankCombinerImpl::isClampZeroToOne(MachineInstr *K0,
621 MachineInstr *K1) const {
622 if (isFCst(MI: K0) && isFCst(MI: K1)) {
623 const ConstantFP *KO_FPImm = K0->getOperand(i: 1).getFPImm();
624 const ConstantFP *K1_FPImm = K1->getOperand(i: 1).getFPImm();
625 return (KO_FPImm->isPosZero() && K1_FPImm->isOne()) ||
626 (KO_FPImm->isOne() && K1_FPImm->isPosZero());
627 }
628 return false;
629}
630
631static bool runCombiner(MachineFunction &MF,
632 function_ref<GISelValueTracking *()> GetVT,
633 function_ref<MachineDominatorTree *()> GetMDT,
634 bool EnableOpt) {
635 AMDGPURegBankCombinerImplRuleConfig RuleConfig;
636 if (!RuleConfig.parseCommandLineOption())
637 reportFatalUsageError(reason: "Invalid rule identifier");
638
639 if (MF.getProperties().hasFailedISel())
640 return false;
641
642 const Function &F = MF.getFunction();
643 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
644 const auto *LI = ST.getLegalizerInfo();
645
646 CombinerInfo CInfo(/*AllowIllegalOps=*/false, /*ShouldLegalizeIllegal=*/true,
647 LI, EnableOpt, F.hasOptSize(), F.hasMinSize());
648 // Disable fixed-point iteration to reduce compile-time
649 CInfo.MaxIterations = 1;
650 CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
651 // RegBankSelect seems not to leave dead instructions, so a full DCE pass is
652 // unnecessary.
653 CInfo.EnableFullDCE = false;
654
655 GISelValueTracking *VT = GetVT();
656 MachineDominatorTree *MDT = GetMDT();
657 AMDGPURegBankCombinerImpl Impl(MF, CInfo, *VT, /*CSEInfo=*/nullptr,
658 RuleConfig, ST, MDT, LI);
659 return Impl.combineMachineInstrs();
660}
661
662// Pass boilerplate
663// ================
664
665class AMDGPURegBankCombinerLegacy : public MachineFunctionPass {
666public:
667 static char ID;
668
669 AMDGPURegBankCombinerLegacy(bool IsOptLevelNone = false)
670 : MachineFunctionPass(ID), IsOptLevelNone(IsOptLevelNone) {}
671
672 StringRef getPassName() const override { return "AMDGPURegBankCombiner"; }
673
674 bool runOnMachineFunction(MachineFunction &MF) override;
675
676 void getAnalysisUsage(AnalysisUsage &AU) const override;
677
678private:
679 bool IsOptLevelNone;
680};
681} // end anonymous namespace
682
683void AMDGPURegBankCombinerLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
684 AU.setPreservesCFG();
685 getSelectionDAGFallbackAnalysisUsage(AU);
686 AU.addRequired<GISelValueTrackingAnalysisLegacy>();
687 AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
688 if (!IsOptLevelNone) {
689 AU.addRequired<MachineDominatorTreeWrapperPass>();
690 }
691 MachineFunctionPass::getAnalysisUsage(AU);
692}
693
694bool AMDGPURegBankCombinerLegacy::runOnMachineFunction(MachineFunction &MF) {
695 const Function &F = MF.getFunction();
696 bool EnableOpt =
697 MF.getTarget().getOptLevel() != CodeGenOptLevel::None && !skipFunction(F);
698
699 return runCombiner(
700 MF,
701 GetVT: [&]() {
702 return &getAnalysis<GISelValueTrackingAnalysisLegacy>().get(MF);
703 },
704 GetMDT: [&]() -> MachineDominatorTree * {
705 return IsOptLevelNone ? nullptr
706 : &getAnalysis<MachineDominatorTreeWrapperPass>()
707 .getDomTree();
708 },
709 EnableOpt);
710}
711
712char AMDGPURegBankCombinerLegacy::ID = 0;
713INITIALIZE_PASS_BEGIN(AMDGPURegBankCombinerLegacy, DEBUG_TYPE,
714 "Combine AMDGPU machine instrs after regbankselect",
715 false, false)
716INITIALIZE_PASS_DEPENDENCY(GISelValueTrackingAnalysisLegacy)
717INITIALIZE_PASS_END(AMDGPURegBankCombinerLegacy, DEBUG_TYPE,
718 "Combine AMDGPU machine instrs after regbankselect", false,
719 false)
720
721FunctionPass *llvm::createAMDGPURegBankCombinerLegacy(bool IsOptLevelNone) {
722 return new AMDGPURegBankCombinerLegacy(IsOptLevelNone);
723}
724
725AMDGPURegBankCombinerPass::AMDGPURegBankCombinerPass(bool IsOptLevelNone)
726 : IsOptLevelNone(IsOptLevelNone) {}
727
728PreservedAnalyses
729AMDGPURegBankCombinerPass::run(MachineFunction &MF,
730 MachineFunctionAnalysisManager &MFAM) {
731 const Function &F = MF.getFunction();
732 bool EnableOpt =
733 MF.getTarget().getOptLevel() != CodeGenOptLevel::None && !F.hasOptNone();
734
735 if (!runCombiner(
736 MF, GetVT: [&]() { return &MFAM.getResult<GISelValueTrackingAnalysis>(IR&: MF); },
737 GetMDT: [&]() -> MachineDominatorTree * {
738 return IsOptLevelNone
739 ? nullptr
740 : &MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
741 },
742 EnableOpt))
743 return PreservedAnalyses::all();
744
745 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
746 PA.preserveSet<CFGAnalyses>();
747 PA.preserve<GISelValueTrackingAnalysis>();
748 return PA;
749}
750