1//=== lib/CodeGen/GlobalISel/AMDGPUPreLegalizerCombiner.cpp ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass does combining of machine instructions at the generic MI level,
10// before the legalizer.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "AMDGPUCombinerHelper.h"
16#include "AMDGPULegalizerInfo.h"
17#include "GCNSubtarget.h"
18#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
19#include "llvm/CodeGen/GlobalISel/Combiner.h"
20#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
21#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
22#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
23#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
24#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
25#include "llvm/CodeGen/MachineDominators.h"
26#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
27#include "llvm/CodeGen/MachinePassManager.h"
28#include "llvm/CodeGen/TargetPassConfig.h"
29#include "llvm/Target/TargetMachine.h"
30
31#define GET_GICOMBINER_DEPS
32#include "AMDGPUGenPreLegalizeGICombiner.inc"
33#undef GET_GICOMBINER_DEPS
34
35#define DEBUG_TYPE "amdgpu-prelegalizer-combiner"
36
37using namespace llvm;
38using namespace MIPatternMatch;
39namespace {
40
41#define GET_GICOMBINER_TYPES
42#include "AMDGPUGenPreLegalizeGICombiner.inc"
43#undef GET_GICOMBINER_TYPES
44
45class AMDGPUPreLegalizerCombinerImpl : public Combiner {
46protected:
47 const AMDGPUPreLegalizerCombinerImplRuleConfig &RuleConfig;
48 const GCNSubtarget &STI;
49 const AMDGPUCombinerHelper Helper;
50
51public:
52 AMDGPUPreLegalizerCombinerImpl(
53 MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &VT,
54 GISelCSEInfo *CSEInfo,
55 const AMDGPUPreLegalizerCombinerImplRuleConfig &RuleConfig,
56 const GCNSubtarget &STI, MachineDominatorTree *MDT,
57 const LegalizerInfo *LI);
58
59 static const char *getName() { return "AMDGPUPreLegalizerCombinerImpl"; }
60
61 bool tryCombineAllImpl(MachineInstr &MI) const;
62 bool tryCombineAll(MachineInstr &I) const override;
63
64 struct ClampI64ToI16MatchInfo {
65 int64_t Cmp1 = 0;
66 int64_t Cmp2 = 0;
67 Register Origin;
68 };
69
70 bool matchClampI64ToI16(MachineInstr &MI, const MachineRegisterInfo &MRI,
71 const MachineFunction &MF,
72 ClampI64ToI16MatchInfo &MatchInfo) const;
73
74 void applyClampI64ToI16(MachineInstr &MI,
75 const ClampI64ToI16MatchInfo &MatchInfo) const;
76
77private:
78#define GET_GICOMBINER_CLASS_MEMBERS
79#define AMDGPUSubtarget GCNSubtarget
80#include "AMDGPUGenPreLegalizeGICombiner.inc"
81#undef GET_GICOMBINER_CLASS_MEMBERS
82#undef AMDGPUSubtarget
83};
84
85#define GET_GICOMBINER_IMPL
86#define AMDGPUSubtarget GCNSubtarget
87#include "AMDGPUGenPreLegalizeGICombiner.inc"
88#undef AMDGPUSubtarget
89#undef GET_GICOMBINER_IMPL
90
91AMDGPUPreLegalizerCombinerImpl::AMDGPUPreLegalizerCombinerImpl(
92 MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &VT,
93 GISelCSEInfo *CSEInfo,
94 const AMDGPUPreLegalizerCombinerImplRuleConfig &RuleConfig,
95 const GCNSubtarget &STI, MachineDominatorTree *MDT, const LegalizerInfo *LI)
96 : Combiner(MF, CInfo, &VT, CSEInfo), RuleConfig(RuleConfig), STI(STI),
97 Helper(Observer, B, /*IsPreLegalize*/ true, &VT, MDT, LI, STI),
98#define GET_GICOMBINER_CONSTRUCTOR_INITS
99#include "AMDGPUGenPreLegalizeGICombiner.inc"
100#undef GET_GICOMBINER_CONSTRUCTOR_INITS
101{
102}
103
104bool AMDGPUPreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
105 if (tryCombineAllImpl(I&: MI))
106 return true;
107 return false;
108}
109
110bool AMDGPUPreLegalizerCombinerImpl::matchClampI64ToI16(
111 MachineInstr &MI, const MachineRegisterInfo &MRI, const MachineFunction &MF,
112 ClampI64ToI16MatchInfo &MatchInfo) const {
113 assert(MI.getOpcode() == TargetOpcode::G_TRUNC && "Invalid instruction!");
114
115 // Try to find a pattern where an i64 value should get clamped to short.
116 const LLT SrcType = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
117 if (SrcType != LLT::scalar(SizeInBits: 64))
118 return false;
119
120 const LLT DstType = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
121 if (DstType != LLT::scalar(SizeInBits: 16))
122 return false;
123
124 Register Base;
125
126 // Lo must not exceed Hi: with inverted bounds smin(smax(X, Lo), Hi) is
127 // constant, but the med3 built below would still clamp X to [Hi, Lo].
128 auto IsApplicableForCombine = [&MatchInfo](bool OuterIsMin) -> bool {
129 const int64_t Lo = OuterIsMin ? MatchInfo.Cmp2 : MatchInfo.Cmp1;
130 const int64_t Hi = OuterIsMin ? MatchInfo.Cmp1 : MatchInfo.Cmp2;
131
132 // Range-check first so Hi - Lo below can't overflow.
133 const int64_t Min = std::numeric_limits<int16_t>::min();
134 const int64_t Max = std::numeric_limits<int16_t>::max();
135 if (Lo < Min || Lo > Max || Hi < Min || Hi > Max)
136 return false;
137
138 // Reject inverted bounds, and bounds so close there is no need to clamp.
139 return Hi - Lo > 1;
140 };
141
142 // Try to match a combination of min / max MIR opcodes.
143 if (mi_match(R: MI.getOperand(i: 1).getReg(), MRI,
144 P: m_GSMin(L: m_Reg(R&: Base), R: m_ICst(Cst&: MatchInfo.Cmp1)))) {
145 if (mi_match(R: Base, MRI,
146 P: m_GSMax(L: m_Reg(R&: MatchInfo.Origin), R: m_ICst(Cst&: MatchInfo.Cmp2)))) {
147 return IsApplicableForCombine(/*OuterIsMin=*/true);
148 }
149 }
150
151 if (mi_match(R: MI.getOperand(i: 1).getReg(), MRI,
152 P: m_GSMax(L: m_Reg(R&: Base), R: m_ICst(Cst&: MatchInfo.Cmp1)))) {
153 if (mi_match(R: Base, MRI,
154 P: m_GSMin(L: m_Reg(R&: MatchInfo.Origin), R: m_ICst(Cst&: MatchInfo.Cmp2)))) {
155 return IsApplicableForCombine(/*OuterIsMin=*/false);
156 }
157 }
158
159 return false;
160}
161
162// We want to find a combination of instructions that
163// gets generated when an i64 gets clamped to i16.
164// The corresponding pattern is:
165// G_MAX / G_MAX for i16 <= G_TRUNC i64.
166// This can be efficiently written as following:
167// v_cvt_pk_i16_i32 v0, v0, v1
168// v_med3_i32 v0, Clamp_Min, v0, Clamp_Max
169void AMDGPUPreLegalizerCombinerImpl::applyClampI64ToI16(
170 MachineInstr &MI, const ClampI64ToI16MatchInfo &MatchInfo) const {
171
172 Register Src = MatchInfo.Origin;
173 assert(MI.getMF()->getRegInfo().getType(Src) == LLT::scalar(64));
174 const LLT I32 = LLT::integer(SizeInBits: 32);
175
176 auto Unmerge = B.buildUnmerge(Res: I32, Op: Src);
177
178 assert(MI.getOpcode() != AMDGPU::G_AMDGPU_CVT_PK_I16_I32);
179
180 const LLT V2S16 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
181 auto CvtPk =
182 B.buildInstr(Opc: AMDGPU::G_AMDGPU_CVT_PK_I16_I32, DstOps: {V2S16},
183 SrcOps: {Unmerge.getReg(Idx: 0), Unmerge.getReg(Idx: 1)}, Flags: MI.getFlags());
184
185 auto MinBoundary = std::min(a: MatchInfo.Cmp1, b: MatchInfo.Cmp2);
186 auto MaxBoundary = std::max(a: MatchInfo.Cmp1, b: MatchInfo.Cmp2);
187 auto MinBoundaryDst = B.buildConstant(Res: I32, Val: MinBoundary);
188 auto MaxBoundaryDst = B.buildConstant(Res: I32, Val: MaxBoundary);
189
190 auto Bitcast = B.buildBitcast(Dst: {I32}, Src: CvtPk);
191
192 auto Med3 = B.buildInstr(
193 Opc: AMDGPU::G_AMDGPU_SMED3, DstOps: {I32},
194 SrcOps: {MinBoundaryDst.getReg(Idx: 0), Bitcast.getReg(Idx: 0), MaxBoundaryDst.getReg(Idx: 0)},
195 Flags: MI.getFlags());
196
197 B.buildTrunc(Res: MI.getOperand(i: 0).getReg(), Op: Med3);
198
199 MI.eraseFromParent();
200}
201
202static bool runCombiner(MachineFunction &MF,
203 function_ref<GISelCSEInfo *()> GetCSEInfo,
204 function_ref<GISelValueTracking *()> GetVT,
205 function_ref<MachineDominatorTree *()> GetMDT,
206 bool EnableOpt) {
207 AMDGPUPreLegalizerCombinerImplRuleConfig RuleConfig;
208 if (!RuleConfig.parseCommandLineOption())
209 reportFatalUsageError(reason: "Invalid rule identifier");
210
211 // If the ISel pipeline failed, do not bother running that pass.
212 if (MF.getProperties().hasFailedISel())
213 return false;
214
215 const GCNSubtarget &STI = MF.getSubtarget<GCNSubtarget>();
216 const Function &F = MF.getFunction();
217 CombinerInfo CInfo(/*AllowIllegalOps=*/true, /*ShouldLegalizeIllegal=*/false,
218 nullptr, EnableOpt, F.hasOptSize(), F.hasMinSize());
219 // Disable fixed-point iteration to reduce compile-time
220 CInfo.MaxIterations = 1;
221 CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
222 // This is the first Combiner, so the input IR might contain dead
223 // instructions.
224 CInfo.EnableFullDCE = true;
225
226 GISelValueTracking *VT = GetVT();
227 GISelCSEInfo *CSEInfo = GetCSEInfo();
228 MachineDominatorTree *MDT = GetMDT();
229 AMDGPUPreLegalizerCombinerImpl Impl(MF, CInfo, *VT, CSEInfo, RuleConfig, STI,
230 MDT, STI.getLegalizerInfo());
231 return Impl.combineMachineInstrs();
232}
233
234// Pass boilerplate
235// ================
236
237class AMDGPUPreLegalizerCombinerLegacy : public MachineFunctionPass {
238public:
239 static char ID;
240
241 AMDGPUPreLegalizerCombinerLegacy(bool IsOptLevelNone = false)
242 : MachineFunctionPass(ID), IsOptLevelNone(IsOptLevelNone) {}
243
244 StringRef getPassName() const override {
245 return "AMDGPUPreLegalizerCombiner";
246 }
247
248 bool runOnMachineFunction(MachineFunction &MF) override;
249
250 void getAnalysisUsage(AnalysisUsage &AU) const override;
251
252private:
253 bool IsOptLevelNone;
254};
255} // end anonymous namespace
256
257void AMDGPUPreLegalizerCombinerLegacy::getAnalysisUsage(
258 AnalysisUsage &AU) const {
259 AU.addRequired<TargetPassConfig>();
260 AU.setPreservesCFG();
261 getSelectionDAGFallbackAnalysisUsage(AU);
262 AU.addRequired<GISelValueTrackingAnalysisLegacy>();
263 AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
264 if (!IsOptLevelNone) {
265 AU.addRequired<MachineDominatorTreeWrapperPass>();
266 }
267
268 AU.addRequired<GISelCSEAnalysisWrapperPass>();
269 AU.addPreserved<GISelCSEAnalysisWrapperPass>();
270 MachineFunctionPass::getAnalysisUsage(AU);
271}
272
273bool AMDGPUPreLegalizerCombinerLegacy::runOnMachineFunction(
274 MachineFunction &MF) {
275 const Function &F = MF.getFunction();
276 bool EnableOpt =
277 MF.getTarget().getOptLevel() != CodeGenOptLevel::None && !skipFunction(F);
278 return runCombiner(
279 MF,
280 GetCSEInfo: [&]() {
281 // Enable CSE.
282 GISelCSEAnalysisWrapper &Wrapper =
283 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
284 return &Wrapper.get(CSEOpt: getAnalysis<TargetPassConfig>().getCSEConfig());
285 },
286 GetVT: [&]() {
287 return &getAnalysis<GISelValueTrackingAnalysisLegacy>().get(MF);
288 },
289 GetMDT: [&]() -> MachineDominatorTree * {
290 return IsOptLevelNone ? nullptr
291 : &getAnalysis<MachineDominatorTreeWrapperPass>()
292 .getDomTree();
293 },
294 EnableOpt);
295}
296
297char AMDGPUPreLegalizerCombinerLegacy::ID = 0;
298INITIALIZE_PASS_BEGIN(AMDGPUPreLegalizerCombinerLegacy, DEBUG_TYPE,
299 "Combine AMDGPU machine instrs before legalization",
300 false, false)
301INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
302INITIALIZE_PASS_DEPENDENCY(GISelValueTrackingAnalysisLegacy)
303INITIALIZE_PASS_END(AMDGPUPreLegalizerCombinerLegacy, DEBUG_TYPE,
304 "Combine AMDGPU machine instrs before legalization", false,
305 false)
306
307FunctionPass *
308llvm::createAMDGPUPreLegalizeCombinerLegacyPass(bool IsOptLevelNone) {
309 return new AMDGPUPreLegalizerCombinerLegacy(IsOptLevelNone);
310}
311
312PreservedAnalyses
313AMDGPUPreLegalizerCombinerPass::run(MachineFunction &MF,
314 MachineFunctionAnalysisManager &MFAM) {
315 bool IsOptLevelNone = MF.getTarget().getOptLevel() == CodeGenOptLevel::None;
316
317 if (!runCombiner(
318 MF, GetCSEInfo: [&]() { return MFAM.getResult<GISelCSEAnalysis>(IR&: MF).get(); },
319 GetVT: [&]() { return &MFAM.getResult<GISelValueTrackingAnalysis>(IR&: MF); },
320 GetMDT: [&]() -> MachineDominatorTree * {
321 return IsOptLevelNone
322 ? nullptr
323 : &MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
324 },
325 /*EnableOpt=*/!IsOptLevelNone))
326 return PreservedAnalyses::all();
327
328 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
329 PA.preserveSet<CFGAnalyses>();
330 PA.preserve<GISelValueTrackingAnalysis>();
331 PA.preserve<GISelCSEAnalysis>();
332 return PA;
333}
334