1//=== lib/CodeGen/GlobalISel/AMDGPURegBankCombiner.cpp ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass does combining of machine instructions at the generic MI level,
10// after register banks are known.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "AMDGPULegalizerInfo.h"
16#include "AMDGPURegisterBankInfo.h"
17#include "GCNSubtarget.h"
18#include "SIMachineFunctionInfo.h"
19#include "llvm/CodeGen/GlobalISel/Combiner.h"
20#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
21#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
22#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
23#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
24#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
25#include "llvm/CodeGen/MachineDominators.h"
26#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
27#include "llvm/CodeGen/MachinePassManager.h"
28#include "llvm/CodeGen/TargetPassConfig.h"
29#include "llvm/IR/IntrinsicsAMDGPU.h"
30#include "llvm/Target/TargetMachine.h"
31
32#define GET_GICOMBINER_DEPS
33#include "AMDGPUGenPreLegalizeGICombiner.inc"
34#undef GET_GICOMBINER_DEPS
35
36#define DEBUG_TYPE "amdgpu-regbank-combiner"
37
38using namespace llvm;
39using namespace MIPatternMatch;
40
41namespace {
42#define GET_GICOMBINER_TYPES
43#include "AMDGPUGenRegBankGICombiner.inc"
44#undef GET_GICOMBINER_TYPES
45
46class AMDGPURegBankCombinerImpl : public Combiner {
47protected:
48 const AMDGPURegBankCombinerImplRuleConfig &RuleConfig;
49 const GCNSubtarget &STI;
50 const RegisterBankInfo &RBI;
51 const TargetRegisterInfo &TRI;
52 const SIInstrInfo &TII;
53 const CombinerHelper Helper;
54
55public:
56 AMDGPURegBankCombinerImpl(
57 MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &VT,
58 GISelCSEInfo *CSEInfo,
59 const AMDGPURegBankCombinerImplRuleConfig &RuleConfig,
60 const GCNSubtarget &STI, MachineDominatorTree *MDT,
61 const LegalizerInfo *LI);
62
63 static const char *getName() { return "AMDGPURegBankCombinerImpl"; }
64
65 bool tryCombineAll(MachineInstr &I) const override;
66
67 bool isVgprRegBank(Register Reg) const;
68 Register getAsVgpr(Register Reg) const;
69
70 struct MinMaxMedOpc {
71 unsigned Min, Max, Med;
72 };
73
74 struct Med3MatchInfo {
75 unsigned Opc;
76 Register Val0, Val1, Val2;
77 };
78
79 struct MinMaxToMinMax3MatchInfo {
80 unsigned Opc;
81 Register Val0, Val1, Val2;
82 };
83
84 MinMaxMedOpc getMinMaxPair(unsigned Opc) const;
85
86 template <class m_Cst, typename CstTy>
87 bool matchMed(MachineInstr &MI, MachineRegisterInfo &MRI, MinMaxMedOpc MMMOpc,
88 Register &Val, CstTy &K0, CstTy &K1) const;
89
90 bool matchIntMinMaxToMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo) const;
91 bool matchFPMinMaxToMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo) const;
92 bool matchFPMinMaxToClamp(MachineInstr &MI, Register &Reg) const;
93 bool matchFPMed3ToClamp(MachineInstr &MI, Register &Reg) const;
94 void applyMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo) const;
95 void applyClamp(MachineInstr &MI, Register &Reg) const;
96
97 void applyCanonicalizeZextShiftAmt(MachineInstr &MI, MachineInstr &Ext) const;
98
99 bool combineD16Load(MachineInstr &MI) const;
100 void applyD16Load(unsigned D16Opc, MachineInstr &DstMI,
101 MachineInstr *SmallLoad, Register ToOverwriteD16,
102 MachineInstr &InsertPt) const;
103
104 bool matchMinMaxToMinMax3(MachineInstr &MI,
105 MinMaxToMinMax3MatchInfo &MatchInfo) const;
106 void applyMinMaxToMinMax3(MachineInstr &MI,
107 MinMaxToMinMax3MatchInfo &MatchInfo) const;
108
109 bool matchCombine_mul24(MachineInstr &MI) const;
110 void applyCombine_mul24(MachineInstr &MI) const;
111
112private:
113 SIModeRegisterDefaults getMode() const;
114 bool getIEEE() const;
115 bool getDX10Clamp() const;
116 bool isFminnumIeee(const MachineInstr &MI) const;
117 bool isFminnum(const MachineInstr &MI) const;
118 bool isFminnumLike(const MachineInstr &MI) const;
119 bool isFCst(MachineInstr *MI) const;
120 bool isClampZeroToOne(MachineInstr *K0, MachineInstr *K1) const;
121
122#define GET_GICOMBINER_CLASS_MEMBERS
123#define AMDGPUSubtarget GCNSubtarget
124#include "AMDGPUGenRegBankGICombiner.inc"
125#undef GET_GICOMBINER_CLASS_MEMBERS
126#undef AMDGPUSubtarget
127};
128
129#define GET_GICOMBINER_IMPL
130#define AMDGPUSubtarget GCNSubtarget
131#include "AMDGPUGenRegBankGICombiner.inc"
132#undef AMDGPUSubtarget
133#undef GET_GICOMBINER_IMPL
134
135AMDGPURegBankCombinerImpl::AMDGPURegBankCombinerImpl(
136 MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &VT,
137 GISelCSEInfo *CSEInfo,
138 const AMDGPURegBankCombinerImplRuleConfig &RuleConfig,
139 const GCNSubtarget &STI, MachineDominatorTree *MDT, const LegalizerInfo *LI)
140 : Combiner(MF, CInfo, &VT, CSEInfo), RuleConfig(RuleConfig), STI(STI),
141 RBI(*STI.getRegBankInfo()), TRI(*STI.getRegisterInfo()),
142 TII(*STI.getInstrInfo()),
143 Helper(Observer, B, /*IsPreLegalize*/ false, &VT, MDT, LI),
144#define GET_GICOMBINER_CONSTRUCTOR_INITS
145#include "AMDGPUGenRegBankGICombiner.inc"
146#undef GET_GICOMBINER_CONSTRUCTOR_INITS
147{
148}
149
150bool AMDGPURegBankCombinerImpl::isVgprRegBank(Register Reg) const {
151 return RBI.getRegBank(Reg, MRI, TRI)->getID() == AMDGPU::VGPRRegBankID;
152}
153
154Register AMDGPURegBankCombinerImpl::getAsVgpr(Register Reg) const {
155 if (isVgprRegBank(Reg))
156 return Reg;
157
158 // Search for existing copy of Reg to vgpr.
159 for (MachineInstr &Use : MRI.use_instructions(Reg)) {
160 Register Def = Use.getOperand(i: 0).getReg();
161 if (Use.getOpcode() == AMDGPU::COPY && isVgprRegBank(Reg: Def))
162 return Def;
163 }
164
165 // Copy Reg to vgpr.
166 Register VgprReg = B.buildCopy(Res: MRI.getType(Reg), Op: Reg).getReg(Idx: 0);
167 MRI.setRegBank(Reg: VgprReg, RegBank: RBI.getRegBank(ID: AMDGPU::VGPRRegBankID));
168 return VgprReg;
169}
170
171AMDGPURegBankCombinerImpl::MinMaxMedOpc
172AMDGPURegBankCombinerImpl::getMinMaxPair(unsigned Opc) const {
173 switch (Opc) {
174 default:
175 llvm_unreachable("Unsupported opcode");
176 case AMDGPU::G_SMAX:
177 case AMDGPU::G_SMIN:
178 return {.Min: AMDGPU::G_SMIN, .Max: AMDGPU::G_SMAX, .Med: AMDGPU::G_AMDGPU_SMED3};
179 case AMDGPU::G_UMAX:
180 case AMDGPU::G_UMIN:
181 return {.Min: AMDGPU::G_UMIN, .Max: AMDGPU::G_UMAX, .Med: AMDGPU::G_AMDGPU_UMED3};
182 case AMDGPU::G_FMAXNUM:
183 case AMDGPU::G_FMINNUM:
184 return {.Min: AMDGPU::G_FMINNUM, .Max: AMDGPU::G_FMAXNUM, .Med: AMDGPU::G_AMDGPU_FMED3};
185 case AMDGPU::G_FMAXNUM_IEEE:
186 case AMDGPU::G_FMINNUM_IEEE:
187 return {.Min: AMDGPU::G_FMINNUM_IEEE, .Max: AMDGPU::G_FMAXNUM_IEEE,
188 .Med: AMDGPU::G_AMDGPU_FMED3};
189 }
190}
191
192template <class m_Cst, typename CstTy>
193bool AMDGPURegBankCombinerImpl::matchMed(MachineInstr &MI,
194 MachineRegisterInfo &MRI,
195 MinMaxMedOpc MMMOpc, Register &Val,
196 CstTy &K0, CstTy &K1) const {
197 // 4 operand commutes of: min(max(Val, K0), K1).
198 // Find K1 from outer instr: min(max(...), K1) or min(K1, max(...)).
199 // Find K0 and Val from inner instr: max(K0, Val) or max(Val, K0).
200 // 4 operand commutes of: max(min(Val, K1), K0).
201 // Find K0 from outer instr: max(min(...), K0) or max(K0, min(...)).
202 // Find K1 and Val from inner instr: min(K1, Val) or min(Val, K1).
203 return mi_match(
204 MI, MRI,
205 m_any_of(
206 m_CommutativeBinOp(
207 MMMOpc.Min, m_CommutativeBinOp(MMMOpc.Max, m_Reg(R&: Val), m_Cst(K0)),
208 m_Cst(K1)),
209 m_CommutativeBinOp(
210 MMMOpc.Max, m_CommutativeBinOp(MMMOpc.Min, m_Reg(R&: Val), m_Cst(K1)),
211 m_Cst(K0))));
212}
213
214bool AMDGPURegBankCombinerImpl::matchIntMinMaxToMed3(
215 MachineInstr &MI, Med3MatchInfo &MatchInfo) const {
216 Register Dst = MI.getOperand(i: 0).getReg();
217 if (!isVgprRegBank(Reg: Dst))
218 return false;
219
220 // med3 for i16 is only available on gfx9+, and not available for v2i16.
221 LLT Ty = MRI.getType(Reg: Dst);
222 if ((Ty != LLT::scalar(SizeInBits: 16) || !STI.hasMed3_16()) && Ty != LLT::scalar(SizeInBits: 32))
223 return false;
224
225 MinMaxMedOpc OpcodeTriple = getMinMaxPair(Opc: MI.getOpcode());
226 Register Val;
227 std::optional<ValueAndVReg> K0, K1;
228 // Match min(max(Val, K0), K1) or max(min(Val, K1), K0). Then see if K0 <= K1.
229 if (!matchMed<GCstAndRegMatch>(MI, MRI, MMMOpc: OpcodeTriple, Val, K0, K1))
230 return false;
231
232 if (OpcodeTriple.Med == AMDGPU::G_AMDGPU_SMED3 && K0->Value.sgt(RHS: K1->Value))
233 return false;
234 if (OpcodeTriple.Med == AMDGPU::G_AMDGPU_UMED3 && K0->Value.ugt(RHS: K1->Value))
235 return false;
236
237 MatchInfo = {.Opc: OpcodeTriple.Med, .Val0: Val, .Val1: K0->VReg, .Val2: K1->VReg};
238 return true;
239}
240
241// fmed3(NaN, K0, K1) = min(min(NaN, K0), K1)
242// ieee = true : min/max(SNaN, K) = QNaN, min/max(QNaN, K) = K
243// ieee = false : min/max(NaN, K) = K
244// clamp(NaN) = dx10_clamp ? 0.0 : NaN
245// Consider values of min(max(Val, K0), K1) and max(min(Val, K1), K0) as input.
246// Other operand commutes (see matchMed) give same result since min and max are
247// commutative.
248
249// Try to replace fp min(max(Val, K0), K1) or max(min(Val, K1), K0), KO<=K1
250// with fmed3(Val, K0, K1) or clamp(Val). Clamp requires K0 = 0.0 and K1 = 1.0.
251// Val = SNaN only for ieee = true
252// fmed3(SNaN, K0, K1) = min(min(SNaN, K0), K1) = min(QNaN, K1) = K1
253// min(max(SNaN, K0), K1) = min(QNaN, K1) = K1
254// max(min(SNaN, K1), K0) = max(K1, K0) = K1
255// Val = NaN,ieee = false or Val = QNaN,ieee = true
256// fmed3(NaN, K0, K1) = min(min(NaN, K0), K1) = min(K0, K1) = K0
257// min(max(NaN, K0), K1) = min(K0, K1) = K0 (can clamp when dx10_clamp = true)
258// max(min(NaN, K1), K0) = max(K1, K0) = K1 != K0
259bool AMDGPURegBankCombinerImpl::matchFPMinMaxToMed3(
260 MachineInstr &MI, Med3MatchInfo &MatchInfo) const {
261 Register Dst = MI.getOperand(i: 0).getReg();
262 // Perform combine only when the destination is a VGPR.
263 if (!isVgprRegBank(Reg: Dst))
264 return false;
265
266 LLT Ty = MRI.getType(Reg: Dst);
267
268 // med3 for f16 is only available on gfx9+, and not available for v2f16.
269 if ((Ty != LLT::scalar(SizeInBits: 16) || !STI.hasMed3_16()) && Ty != LLT::scalar(SizeInBits: 32))
270 return false;
271
272 auto OpcodeTriple = getMinMaxPair(Opc: MI.getOpcode());
273
274 Register Val;
275 std::optional<FPValueAndVReg> K0, K1;
276 // Match min(max(Val, K0), K1) or max(min(Val, K1), K0). Then see if K0 <= K1.
277 if (!matchMed<GFCstAndRegMatch>(MI, MRI, MMMOpc: OpcodeTriple, Val, K0, K1))
278 return false;
279
280 if (K0->Value > K1->Value)
281 return false;
282
283 // For IEEE=false perform combine only when it's safe to assume that there are
284 // no NaN inputs. Most often MI is marked with nnan fast math flag.
285 // For IEEE=true consider NaN inputs. fmed3(NaN, K0, K1) is equivalent to
286 // min(min(NaN, K0), K1). Safe to fold for min(max(Val, K0), K1) since inner
287 // nodes(max/min) have same behavior when one input is NaN and other isn't.
288 // Don't consider max(min(SNaN, K1), K0) since there is no isKnownNeverQNaN,
289 // also post-legalizer inputs to min/max are fcanonicalized (never SNaN).
290 if ((getIEEE() && isFminnumLike(MI)) || VT->isKnownNeverNaN(Val: Dst)) {
291 // Don't fold single use constant that can't be inlined.
292 if ((!MRI.hasOneNonDBGUse(RegNo: K0->VReg) || TII.isInlineConstant(Imm: K0->Value)) &&
293 (!MRI.hasOneNonDBGUse(RegNo: K1->VReg) || TII.isInlineConstant(Imm: K1->Value))) {
294 MatchInfo = {.Opc: OpcodeTriple.Med, .Val0: Val, .Val1: K0->VReg, .Val2: K1->VReg};
295 return true;
296 }
297 }
298
299 return false;
300}
301
302bool AMDGPURegBankCombinerImpl::matchFPMinMaxToClamp(MachineInstr &MI,
303 Register &Reg) const {
304 // Perform combine only when the destination is a VGPR.
305 if (!isVgprRegBank(Reg: MI.getOperand(i: 0).getReg()))
306 return false;
307
308 // Clamp is available on all types after regbankselect (f16, f32, f64, v2f16).
309 auto OpcodeTriple = getMinMaxPair(Opc: MI.getOpcode());
310 Register Val;
311 std::optional<FPValueAndVReg> K0, K1;
312 // Match min(max(Val, K0), K1) or max(min(Val, K1), K0).
313 if (!matchMed<GFCstOrSplatGFCstMatch>(MI, MRI, MMMOpc: OpcodeTriple, Val, K0, K1))
314 return false;
315
316 if (!K0->Value.isPosZero() || !K1->Value.isOne())
317 return false;
318
319 // For IEEE=false perform combine only when it's safe to assume that there are
320 // no NaN inputs. Most often MI is marked with nnan fast math flag.
321 // For IEEE=true consider NaN inputs. Only min(max(QNaN, 0.0), 1.0) evaluates
322 // to 0.0 requires dx10_clamp = true.
323 if ((getIEEE() && getDX10Clamp() && isFminnumLike(MI) &&
324 VT->isKnownNeverSNaN(Val)) ||
325 VT->isKnownNeverNaN(Val: MI.getOperand(i: 0).getReg())) {
326 Reg = Val;
327 return true;
328 }
329
330 return false;
331}
332
333// Replacing fmed3(NaN, 0.0, 1.0) with clamp. Requires dx10_clamp = true.
334// Val = SNaN only for ieee = true. It is important which operand is NaN.
335// min(min(SNaN, 0.0), 1.0) = min(QNaN, 1.0) = 1.0
336// min(min(SNaN, 1.0), 0.0) = min(QNaN, 0.0) = 0.0
337// min(min(0.0, 1.0), SNaN) = min(0.0, SNaN) = QNaN
338// Val = NaN,ieee = false or Val = QNaN,ieee = true
339// min(min(NaN, 0.0), 1.0) = min(0.0, 1.0) = 0.0
340// min(min(NaN, 1.0), 0.0) = min(1.0, 0.0) = 0.0
341// min(min(0.0, 1.0), NaN) = min(0.0, NaN) = 0.0
342bool AMDGPURegBankCombinerImpl::matchFPMed3ToClamp(MachineInstr &MI,
343 Register &Reg) const {
344 // Perform combine only when the destination is a VGPR.
345 if (!isVgprRegBank(Reg: MI.getOperand(i: 0).getReg()))
346 return false;
347
348 // In llvm-ir, clamp is often represented as an intrinsic call to
349 // @llvm.amdgcn.fmed3.f32(%Val, 0.0, 1.0). Check for other operand orders.
350 MachineInstr *Src0 = getDefIgnoringCopies(Reg: MI.getOperand(i: 1).getReg(), MRI);
351 MachineInstr *Src1 = getDefIgnoringCopies(Reg: MI.getOperand(i: 2).getReg(), MRI);
352 MachineInstr *Src2 = getDefIgnoringCopies(Reg: MI.getOperand(i: 3).getReg(), MRI);
353
354 if (isFCst(MI: Src0) && !isFCst(MI: Src1))
355 std::swap(a&: Src0, b&: Src1);
356 if (isFCst(MI: Src1) && !isFCst(MI: Src2))
357 std::swap(a&: Src1, b&: Src2);
358 if (isFCst(MI: Src0) && !isFCst(MI: Src1))
359 std::swap(a&: Src0, b&: Src1);
360 if (!isClampZeroToOne(K0: Src1, K1: Src2))
361 return false;
362
363 Register Val = Src0->getOperand(i: 0).getReg();
364
365 auto isOp3Zero = [&]() {
366 MachineInstr *Op3 = getDefIgnoringCopies(Reg: MI.getOperand(i: 3).getReg(), MRI);
367 if (Op3->getOpcode() == TargetOpcode::G_FCONSTANT)
368 return Op3->getOperand(i: 1).getFPImm()->isPosZero();
369 return false;
370 };
371 // For IEEE=false perform combine only when it's safe to assume that there are
372 // no NaN inputs. Most often MI is marked with nnan fast math flag.
373 // For IEEE=true consider NaN inputs. Requires dx10_clamp = true. Safe to fold
374 // when Val could be QNaN. If Val can also be SNaN third input should be 0.0.
375 if (VT->isKnownNeverNaN(Val: MI.getOperand(i: 0).getReg()) ||
376 (getIEEE() && getDX10Clamp() &&
377 (VT->isKnownNeverSNaN(Val) || isOp3Zero()))) {
378 Reg = Val;
379 return true;
380 }
381
382 return false;
383}
384
385void AMDGPURegBankCombinerImpl::applyClamp(MachineInstr &MI,
386 Register &Reg) const {
387 B.buildInstr(Opc: AMDGPU::G_AMDGPU_CLAMP, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Reg},
388 Flags: MI.getFlags());
389 MI.eraseFromParent();
390}
391
392void AMDGPURegBankCombinerImpl::applyMed3(MachineInstr &MI,
393 Med3MatchInfo &MatchInfo) const {
394 B.buildInstr(Opc: MatchInfo.Opc, DstOps: {MI.getOperand(i: 0)},
395 SrcOps: {getAsVgpr(Reg: MatchInfo.Val0), getAsVgpr(Reg: MatchInfo.Val1),
396 getAsVgpr(Reg: MatchInfo.Val2)},
397 Flags: MI.getFlags());
398 MI.eraseFromParent();
399}
400
401void AMDGPURegBankCombinerImpl::applyCanonicalizeZextShiftAmt(
402 MachineInstr &MI, MachineInstr &Ext) const {
403 unsigned ShOpc = MI.getOpcode();
404 assert(ShOpc == AMDGPU::G_SHL || ShOpc == AMDGPU::G_LSHR ||
405 ShOpc == AMDGPU::G_ASHR);
406 assert(Ext.getOpcode() == AMDGPU::G_ZEXT);
407
408 Register AmtReg = Ext.getOperand(i: 1).getReg();
409 Register ShDst = MI.getOperand(i: 0).getReg();
410 Register ShSrc = MI.getOperand(i: 1).getReg();
411
412 LLT ExtAmtTy = MRI.getType(Reg: Ext.getOperand(i: 0).getReg());
413 LLT AmtTy = MRI.getType(Reg: AmtReg);
414
415 auto &RB = *MRI.getRegBank(Reg: AmtReg);
416
417 auto NewExt = B.buildAnyExt(Res: ExtAmtTy, Op: AmtReg);
418 auto Mask = B.buildConstant(
419 Res: ExtAmtTy, Val: maskTrailingOnes<uint64_t>(N: AmtTy.getScalarSizeInBits()));
420 auto And = B.buildAnd(Dst: ExtAmtTy, Src0: NewExt, Src1: Mask);
421 B.buildInstr(Opc: ShOpc, DstOps: {ShDst}, SrcOps: {ShSrc, And});
422
423 MRI.setRegBank(Reg: NewExt.getReg(Idx: 0), RegBank: RB);
424 MRI.setRegBank(Reg: Mask.getReg(Idx: 0), RegBank: RB);
425 MRI.setRegBank(Reg: And.getReg(Idx: 0), RegBank: RB);
426 MI.eraseFromParent();
427}
428
429bool AMDGPURegBankCombinerImpl::combineD16Load(MachineInstr &MI) const {
430 Register Dst;
431 MachineInstr *Load;
432 GSExtLoad *SextLoad;
433 GZExtLoad *ZextLoad;
434 const MachineMemOperand *MMO;
435 const int64_t CleanLo16 = 0xFFFFFFFFFFFF0000;
436 const int64_t CleanHi16 = 0x000000000000FFFF;
437
438 auto TryApplyD16Load = [&](unsigned D16Opc, GAnyLoad *SmallLoad) {
439 LLT DstTy = MRI.getType(Reg: Dst);
440 if (DstTy != LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::integer(SizeInBits: 16)) &&
441 DstTy != LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::float16()))
442 return false;
443
444 MachineInstr *DstDef = MRI.getVRegDef(Reg: Dst);
445 MachineInstr *InsertPt = SmallLoad;
446 if (SmallLoad->isSimple() && isObviouslySafeToFold(MI&: *SmallLoad, IntoMI&: MI))
447 InsertPt = &MI;
448 else if (!Helper.dominates(DefMI: *DstDef, UseMI: *SmallLoad))
449 return false;
450
451 applyD16Load(D16Opc, DstMI&: MI, SmallLoad, ToOverwriteD16: Dst, InsertPt&: *InsertPt);
452 return true;
453 };
454
455 // Load lo
456 if (mi_match(R: MI.getOperand(i: 1).getReg(), MRI,
457 P: m_OneUse(SP: m_GOr(L: m_GAnd(L: m_GBitcast(Src: m_Reg(R&: Dst)),
458 R: m_Copy(Src: m_SpecificICst(RequestedValue: CleanLo16))),
459 R: m_OneUse(SP: m_MInstr(MI&: Load)))))) {
460 if (mi_match(R: Load, MRI, P: m_GZExtLoad(Inst&: ZextLoad, MMO: m_MMO(MMO)))) {
461 unsigned LoadSize = MMO->getSizeInBits().getValue();
462 if (LoadSize == 8)
463 return TryApplyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_LO_U8, ZextLoad);
464 if (LoadSize == 16)
465 return TryApplyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_LO, ZextLoad);
466
467 return false;
468 }
469
470 // s32 Load_lo16 holds SextLoad i8, Load_hi16 is zero.
471 // fake16: and (sextload i8 -> s32), 0xFFFF
472 // true16: zext (sextload i8 -> s16) -> s32
473 if (mi_match(R: Load, MRI,
474 P: m_any_of(preds: m_GAnd(L: m_OneUse(SP: m_GSExtLoad(Inst&: SextLoad, MMO: m_MMO(MMO))),
475 R: m_Copy(Src: m_SpecificICst(RequestedValue: CleanHi16))),
476 preds: m_GZExt(Src: m_all_of(
477 preds: m_SpecificType(Ty: LLT::scalar(SizeInBits: 16)),
478 preds: m_OneUse(SP: m_GSExtLoad(Inst&: SextLoad, MMO: m_MMO(MMO)))))))) {
479 unsigned LoadSize = MMO->getSizeInBits().getValue();
480 if (LoadSize == 8)
481 return TryApplyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_LO_I8, SextLoad);
482 }
483
484 return false;
485 }
486
487 // Load hi
488 if (mi_match(R: MI.getOperand(i: 1).getReg(), MRI,
489 P: m_OneUse(SP: m_GOr(L: m_GAnd(L: m_GBitcast(Src: m_Reg(R&: Dst)),
490 R: m_Copy(Src: m_SpecificICst(RequestedValue: CleanHi16))),
491 R: m_OneUse(SP: m_GShl(L: m_OneUse(SP: m_MInstr(MI&: Load)),
492 R: m_Copy(Src: m_SpecificICst(RequestedValue: 16)))))))) {
493 if (mi_match(R: Load, MRI, P: m_GZExtLoad(Inst&: ZextLoad, MMO: m_MMO(MMO)))) {
494 unsigned LoadSize = MMO->getSizeInBits().getValue();
495 if (LoadSize == 8)
496 return TryApplyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_HI_U8, ZextLoad);
497 if (LoadSize == 16)
498 return TryApplyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_HI, ZextLoad);
499
500 return false;
501 }
502
503 // The shift discards the upper half of the sign-extending load.
504 if (mi_match(R: Load, MRI,
505 P: m_any_of(preds: m_GSExtLoad(Inst&: SextLoad, MMO: m_MMO(MMO)),
506 preds: m_GAnd(L: m_OneUse(SP: m_GSExtLoad(Inst&: SextLoad, MMO: m_MMO(MMO))),
507 R: m_Copy(Src: m_SpecificICst(RequestedValue: CleanHi16))),
508 preds: m_GZExt(Src: m_all_of(
509 preds: m_SpecificType(Ty: LLT::scalar(SizeInBits: 16)),
510 preds: m_OneUse(SP: m_GSExtLoad(Inst&: SextLoad, MMO: m_MMO(MMO)))))))) {
511 unsigned LoadSize = MMO->getSizeInBits().getValue();
512 if (LoadSize == 8)
513 return TryApplyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_HI_I8, SextLoad);
514 }
515
516 return false;
517 }
518
519 return false;
520}
521
522void AMDGPURegBankCombinerImpl::applyMinMaxToMinMax3(
523 MachineInstr &MI, MinMaxToMinMax3MatchInfo &MatchInfo) const {
524 B.buildInstr(Opc: MatchInfo.Opc, DstOps: {MI.getOperand(i: 0)},
525 SrcOps: {MatchInfo.Val0, MatchInfo.Val1, MatchInfo.Val2}, Flags: MI.getFlags());
526 MI.eraseFromParent();
527 return;
528}
529
530bool AMDGPURegBankCombinerImpl::matchCombine_mul24(MachineInstr &MI) const {
531 Register Dst = MI.getOperand(i: 0).getReg();
532 if (MRI.getType(Reg: Dst) != LLT::scalar(SizeInBits: 32) || !isVgprRegBank(Reg: Dst))
533 return false;
534 Register Src0 = MI.getOperand(i: 1).getReg();
535 Register Src1 = MI.getOperand(i: 2).getReg();
536 if (STI.hasMulU24() && VT->getKnownBits(R: Src0).countMinLeadingZeros() >= 8 &&
537 VT->getKnownBits(R: Src1).countMinLeadingZeros() >= 8)
538 return true;
539 return STI.hasMulI24() && VT->computeNumSignBits(R: Src0) >= 9 &&
540 VT->computeNumSignBits(R: Src1) >= 9;
541}
542
543void AMDGPURegBankCombinerImpl::applyCombine_mul24(MachineInstr &MI) const {
544 Register Dst = MI.getOperand(i: 0).getReg();
545 Register Src0 = MI.getOperand(i: 1).getReg();
546 Register Src1 = MI.getOperand(i: 2).getReg();
547 Intrinsic::ID IntrID =
548 (STI.hasMulU24() && VT->getKnownBits(R: Src0).countMinLeadingZeros() >= 8 &&
549 VT->getKnownBits(R: Src1).countMinLeadingZeros() >= 8)
550 ? Intrinsic::amdgcn_mul_u24
551 : Intrinsic::amdgcn_mul_i24;
552 B.buildIntrinsic(ID: IntrID, Res: ArrayRef<Register>{Dst}).addUse(RegNo: Src0).addUse(RegNo: Src1);
553 MI.eraseFromParent();
554}
555
556// min(min(a, b), c) == min(a, min(b, c)) == min3(a, b, c)
557// supported scalar type: S32 S16 U32 U16 F32 F16
558bool AMDGPURegBankCombinerImpl::matchMinMaxToMinMax3(
559 MachineInstr &MI, MinMaxToMinMax3MatchInfo &MatchInfo) const {
560 Register Dst = MI.getOperand(i: 0).getReg();
561 Register Src1 = MI.getOperand(i: 1).getReg();
562 Register Src2 = MI.getOperand(i: 2).getReg();
563 // If the register is SGPR, don't optimize it.
564 if (!(isVgprRegBank(Reg: Dst) && isVgprRegBank(Reg: Src1) && isVgprRegBank(Reg: Src2))) {
565 return false;
566 }
567
568 LLT Ty = MRI.getType(Reg: Dst);
569 unsigned Opc = MI.getOpcode();
570 if (!(Ty == LLT::scalar(SizeInBits: 32) ||
571 (Ty == LLT::scalar(SizeInBits: 16) && STI.hasMin3Max3_16())))
572 return false;
573
574 Register R0, R1, R2;
575 if (!mi_match(MI, MRI,
576 P: m_CommutativeBinOp(Opcode: Opc,
577 L: m_OneUse(SP: m_BinOp(Opcode: Opc, L: m_Reg(R&: R0), R: m_Reg(R&: R1))),
578 R: m_Reg(R&: R2)))) {
579 return false;
580 }
581
582 unsigned AMDGPUOpc = 0;
583 switch (Opc) {
584 case AMDGPU::G_SMAX:
585 AMDGPUOpc = AMDGPU::G_AMDGPU_SMAX3;
586 break;
587 case AMDGPU::G_SMIN:
588 AMDGPUOpc = AMDGPU::G_AMDGPU_SMIN3;
589 break;
590 case AMDGPU::G_UMAX:
591 AMDGPUOpc = AMDGPU::G_AMDGPU_UMAX3;
592 break;
593 case AMDGPU::G_UMIN:
594 AMDGPUOpc = AMDGPU::G_AMDGPU_UMIN3;
595 break;
596 case AMDGPU::G_FMAXNUM:
597 case AMDGPU::G_FMAXNUM_IEEE:
598 case AMDGPU::G_FMAXIMUMNUM:
599 AMDGPUOpc = AMDGPU::G_AMDGPU_FMAX3;
600 break;
601 case AMDGPU::G_FMINNUM:
602 case AMDGPU::G_FMINNUM_IEEE:
603 case AMDGPU::G_FMINIMUMNUM:
604 AMDGPUOpc = AMDGPU::G_AMDGPU_FMIN3;
605 break;
606 case AMDGPU::G_FMAXIMUM:
607 AMDGPUOpc = AMDGPU::G_AMDGPU_FMAXIMUM3;
608 break;
609 case AMDGPU::G_FMINIMUM:
610 AMDGPUOpc = AMDGPU::G_AMDGPU_FMINIMUM3;
611 break;
612 default:
613 return false;
614 }
615
616 MatchInfo = {.Opc: AMDGPUOpc, .Val0: R0, .Val1: R1, .Val2: R2};
617 return true;
618}
619
620void AMDGPURegBankCombinerImpl::applyD16Load(unsigned D16Opc,
621 MachineInstr &DstMI,
622 MachineInstr *SmallLoad,
623 Register SrcReg32ToOverwriteD16,
624 MachineInstr &InsertPt) const {
625 Register DstReg = DstMI.getOperand(i: 0).getReg();
626 LLT SrcTy = MRI.getType(Reg: SrcReg32ToOverwriteD16);
627
628 // Dst and Src for D16 load need to have same type.
629 Register D16Dst =
630 SrcTy == MRI.getType(Reg: DstReg)
631 ? DstReg
632 : MRI.createVirtualRegister(RegAttr: {.RCOrRB: MRI.getRegBank(Reg: DstReg), .Ty: SrcTy});
633
634 B.setInstrAndDebugLoc(InsertPt);
635 B.setDebugLoc(SmallLoad->getDebugLoc());
636 B.buildInstr(Opc: D16Opc, DstOps: {D16Dst},
637 SrcOps: {SmallLoad->getOperand(i: 1).getReg(), SrcReg32ToOverwriteD16})
638 .setMemRefs(SmallLoad->memoperands());
639
640 if (D16Dst != DstReg)
641 B.buildBitcast(Dst: DstReg, Src: D16Dst);
642
643 // Recursive cleanup may erase SmallLoad.
644 Register LoadDst = SmallLoad->getOperand(i: 0).getReg();
645 eraseInstr(MI&: DstMI, MRI);
646 // Volatile and ordered atomic loads are not trivially dead.
647 if (MachineInstr *OldLoad = MRI.getVRegDef(Reg: LoadDst))
648 eraseInstr(MI&: *OldLoad, MRI);
649}
650
651SIModeRegisterDefaults AMDGPURegBankCombinerImpl::getMode() const {
652 return MF.getInfo<SIMachineFunctionInfo>()->getMode();
653}
654
655bool AMDGPURegBankCombinerImpl::getIEEE() const { return getMode().IEEE; }
656
657bool AMDGPURegBankCombinerImpl::getDX10Clamp() const {
658 return getMode().DX10Clamp;
659}
660
661bool AMDGPURegBankCombinerImpl::isFminnumIeee(const MachineInstr &MI) const {
662 return MI.getOpcode() == AMDGPU::G_FMINNUM_IEEE;
663}
664
665bool AMDGPURegBankCombinerImpl::isFminnum(const MachineInstr &MI) const {
666 return MI.getOpcode() == AMDGPU::G_FMINNUM;
667}
668
669bool AMDGPURegBankCombinerImpl::isFminnumLike(const MachineInstr &MI) const {
670 return isFminnumIeee(MI) || isFminnum(MI);
671}
672
673bool AMDGPURegBankCombinerImpl::isFCst(MachineInstr *MI) const {
674 return MI->getOpcode() == AMDGPU::G_FCONSTANT;
675}
676
677bool AMDGPURegBankCombinerImpl::isClampZeroToOne(MachineInstr *K0,
678 MachineInstr *K1) const {
679 if (isFCst(MI: K0) && isFCst(MI: K1)) {
680 const ConstantFP *KO_FPImm = K0->getOperand(i: 1).getFPImm();
681 const ConstantFP *K1_FPImm = K1->getOperand(i: 1).getFPImm();
682 return (KO_FPImm->isPosZero() && K1_FPImm->isOne()) ||
683 (KO_FPImm->isOne() && K1_FPImm->isPosZero());
684 }
685 return false;
686}
687
688static bool runCombiner(MachineFunction &MF,
689 function_ref<GISelValueTracking *()> GetVT,
690 function_ref<MachineDominatorTree *()> GetMDT,
691 bool EnableOpt) {
692 AMDGPURegBankCombinerImplRuleConfig RuleConfig;
693 if (!RuleConfig.parseCommandLineOption())
694 reportFatalUsageError(reason: "Invalid rule identifier");
695
696 if (MF.getProperties().hasFailedISel())
697 return false;
698
699 const Function &F = MF.getFunction();
700 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
701 const auto *LI = ST.getLegalizerInfo();
702
703 CombinerInfo CInfo(/*AllowIllegalOps=*/false, /*ShouldLegalizeIllegal=*/true,
704 LI, EnableOpt, F.hasOptSize(), F.hasMinSize());
705 // Disable fixed-point iteration to reduce compile-time
706 CInfo.MaxIterations = 1;
707 CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
708 // RegBankSelect seems not to leave dead instructions, so a full DCE pass is
709 // unnecessary.
710 CInfo.EnableFullDCE = false;
711
712 GISelValueTracking *VT = GetVT();
713 MachineDominatorTree *MDT = GetMDT();
714 AMDGPURegBankCombinerImpl Impl(MF, CInfo, *VT, /*CSEInfo=*/nullptr,
715 RuleConfig, ST, MDT, LI);
716 return Impl.combineMachineInstrs();
717}
718
719// Pass boilerplate
720// ================
721
722class AMDGPURegBankCombinerLegacy : public MachineFunctionPass {
723public:
724 static char ID;
725
726 AMDGPURegBankCombinerLegacy(bool IsOptLevelNone = false)
727 : MachineFunctionPass(ID), IsOptLevelNone(IsOptLevelNone) {}
728
729 StringRef getPassName() const override { return "AMDGPURegBankCombiner"; }
730
731 bool runOnMachineFunction(MachineFunction &MF) override;
732
733 void getAnalysisUsage(AnalysisUsage &AU) const override;
734
735private:
736 bool IsOptLevelNone;
737};
738} // end anonymous namespace
739
740void AMDGPURegBankCombinerLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
741 AU.setPreservesCFG();
742 getSelectionDAGFallbackAnalysisUsage(AU);
743 AU.addRequired<GISelValueTrackingAnalysisLegacy>();
744 AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
745 if (!IsOptLevelNone) {
746 AU.addRequired<MachineDominatorTreeWrapperPass>();
747 }
748 MachineFunctionPass::getAnalysisUsage(AU);
749}
750
751bool AMDGPURegBankCombinerLegacy::runOnMachineFunction(MachineFunction &MF) {
752 const Function &F = MF.getFunction();
753 bool EnableOpt =
754 MF.getTarget().getOptLevel() != CodeGenOptLevel::None && !skipFunction(F);
755
756 return runCombiner(
757 MF,
758 GetVT: [&]() {
759 return &getAnalysis<GISelValueTrackingAnalysisLegacy>().get(MF);
760 },
761 GetMDT: [&]() -> MachineDominatorTree * {
762 return IsOptLevelNone ? nullptr
763 : &getAnalysis<MachineDominatorTreeWrapperPass>()
764 .getDomTree();
765 },
766 EnableOpt);
767}
768
769char AMDGPURegBankCombinerLegacy::ID = 0;
770INITIALIZE_PASS_BEGIN(AMDGPURegBankCombinerLegacy, DEBUG_TYPE,
771 "Combine AMDGPU machine instrs after regbankselect",
772 false, false)
773INITIALIZE_PASS_DEPENDENCY(GISelValueTrackingAnalysisLegacy)
774INITIALIZE_PASS_END(AMDGPURegBankCombinerLegacy, DEBUG_TYPE,
775 "Combine AMDGPU machine instrs after regbankselect", false,
776 false)
777
778FunctionPass *llvm::createAMDGPURegBankCombinerLegacy(bool IsOptLevelNone) {
779 return new AMDGPURegBankCombinerLegacy(IsOptLevelNone);
780}
781
782AMDGPURegBankCombinerPass::AMDGPURegBankCombinerPass(bool IsOptLevelNone)
783 : IsOptLevelNone(IsOptLevelNone) {}
784
785PreservedAnalyses
786AMDGPURegBankCombinerPass::run(MachineFunction &MF,
787 MachineFunctionAnalysisManager &MFAM) {
788 const Function &F = MF.getFunction();
789 bool EnableOpt = MF.getTarget().getOptLevel() != CodeGenOptLevel::None &&
790 !F.hasOptNone() && !shouldSkipOptimizationForOptBisect(IR: F);
791
792 if (!runCombiner(
793 MF, GetVT: [&]() { return &MFAM.getResult<GISelValueTrackingAnalysis>(IR&: MF); },
794 GetMDT: [&]() -> MachineDominatorTree * {
795 return IsOptLevelNone
796 ? nullptr
797 : &MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
798 },
799 EnableOpt))
800 return PreservedAnalyses::all();
801
802 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
803 PA.preserveSet<CFGAnalyses>();
804 PA.preserve<GISelValueTrackingAnalysis>();
805 return PA;
806}
807