1//=== lib/CodeGen/GlobalISel/AMDGPUCombinerHelper.cpp ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUCombinerHelper.h"
10#include "GCNSubtarget.h"
11#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
12#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
13#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
14#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
15#include "llvm/IR/IntrinsicsAMDGPU.h"
16#include "llvm/Target/TargetMachine.h"
17
18using namespace llvm;
19using namespace MIPatternMatch;
20
21AMDGPUCombinerHelper::AMDGPUCombinerHelper(
22 GISelChangeObserver &Observer, MachineIRBuilder &B, bool IsPreLegalize,
23 GISelValueTracking *VT, MachineDominatorTree *MDT, const LegalizerInfo *LI,
24 const GCNSubtarget &STI)
25 : CombinerHelper(Observer, B, IsPreLegalize, VT, MDT, LI), STI(STI),
26 TII(*STI.getInstrInfo()) {}
27
28LLVM_READNONE
29static bool fnegFoldsIntoMI(const MachineInstr &MI) {
30 switch (MI.getOpcode()) {
31 case AMDGPU::G_FADD:
32 case AMDGPU::G_FSUB:
33 case AMDGPU::G_FMUL:
34 case AMDGPU::G_FMA:
35 case AMDGPU::G_FMAD:
36 case AMDGPU::G_FMINNUM:
37 case AMDGPU::G_FMAXNUM:
38 case AMDGPU::G_FMINNUM_IEEE:
39 case AMDGPU::G_FMAXNUM_IEEE:
40 case AMDGPU::G_FMINIMUM:
41 case AMDGPU::G_FMAXIMUM:
42 case AMDGPU::G_FSIN:
43 case AMDGPU::G_FPEXT:
44 case AMDGPU::G_INTRINSIC_TRUNC:
45 case AMDGPU::G_FPTRUNC:
46 case AMDGPU::G_FRINT:
47 case AMDGPU::G_FNEARBYINT:
48 case AMDGPU::G_INTRINSIC_ROUND:
49 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
50 case AMDGPU::G_FCANONICALIZE:
51 case AMDGPU::G_AMDGPU_RCP_IFLAG:
52 case AMDGPU::G_AMDGPU_FMIN_LEGACY:
53 case AMDGPU::G_AMDGPU_FMAX_LEGACY:
54 return true;
55 case AMDGPU::G_INTRINSIC: {
56 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: MI).getIntrinsicID();
57 switch (IntrinsicID) {
58 case Intrinsic::amdgcn_rcp:
59 case Intrinsic::amdgcn_rcp_legacy:
60 case Intrinsic::amdgcn_sin:
61 case Intrinsic::amdgcn_fmul_legacy:
62 case Intrinsic::amdgcn_fmed3:
63 case Intrinsic::amdgcn_fma_legacy:
64 return true;
65 default:
66 return false;
67 }
68 }
69 default:
70 return false;
71 }
72}
73
74/// \p returns true if the operation will definitely need to use a 64-bit
75/// encoding, and thus will use a VOP3 encoding regardless of the source
76/// modifiers.
77LLVM_READONLY
78static bool opMustUseVOP3Encoding(const MachineInstr &MI,
79 const MachineRegisterInfo &MRI) {
80 return MI.getNumOperands() > (isa<GIntrinsic>(Val: MI) ? 4u : 3u) ||
81 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getScalarSizeInBits() == 64;
82}
83
84// Most FP instructions support source modifiers.
85LLVM_READONLY
86static bool hasSourceMods(const MachineInstr &MI) {
87 if (!MI.memoperands().empty())
88 return false;
89
90 switch (MI.getOpcode()) {
91 case AMDGPU::COPY:
92 case AMDGPU::G_SELECT:
93 case AMDGPU::G_FDIV:
94 case AMDGPU::G_FREM:
95 case TargetOpcode::INLINEASM:
96 case TargetOpcode::INLINEASM_BR:
97 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
98 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
99 case AMDGPU::G_BITCAST:
100 case AMDGPU::G_ANYEXT:
101 case AMDGPU::G_BUILD_VECTOR:
102 case AMDGPU::G_BUILD_VECTOR_TRUNC:
103 case AMDGPU::G_PHI:
104 return false;
105 case AMDGPU::G_INTRINSIC:
106 case AMDGPU::G_INTRINSIC_CONVERGENT: {
107 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: MI).getIntrinsicID();
108 switch (IntrinsicID) {
109 case Intrinsic::amdgcn_interp_p1:
110 case Intrinsic::amdgcn_interp_p2:
111 case Intrinsic::amdgcn_interp_mov:
112 case Intrinsic::amdgcn_interp_p1_f16:
113 case Intrinsic::amdgcn_interp_p2_f16:
114 case Intrinsic::amdgcn_div_scale:
115 return false;
116 default:
117 return true;
118 }
119 }
120 default:
121 return true;
122 }
123}
124
125static bool allUsesHaveSourceMods(MachineInstr &MI, MachineRegisterInfo &MRI,
126 unsigned CostThreshold = 4) {
127 // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus
128 // it is truly free to use a source modifier in all cases. If there are
129 // multiple users but for each one will necessitate using VOP3, there will be
130 // a code size increase. Try to avoid increasing code size unless we know it
131 // will save on the instruction count.
132 unsigned NumMayIncreaseSize = 0;
133 Register Dst = MI.getOperand(i: 0).getReg();
134 for (const MachineInstr &Use : MRI.use_nodbg_instructions(Reg: Dst)) {
135 if (!hasSourceMods(MI: Use))
136 return false;
137
138 if (!opMustUseVOP3Encoding(MI: Use, MRI)) {
139 if (++NumMayIncreaseSize > CostThreshold)
140 return false;
141 }
142 }
143 return true;
144}
145
146static bool mayIgnoreSignedZero(const MachineInstr &MI) {
147 return MI.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
148}
149
150static bool isInv2Pi(const APFloat &APF) {
151 static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118));
152 static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983));
153 static const APFloat KF64(APFloat::IEEEdouble(),
154 APInt(64, 0x3fc45f306dc9c882));
155
156 return APF.bitwiseIsEqual(RHS: KF16) || APF.bitwiseIsEqual(RHS: KF32) ||
157 APF.bitwiseIsEqual(RHS: KF64);
158}
159
160// 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an
161// additional cost to negate them.
162static bool isConstantCostlierToNegate(MachineInstr &MI, Register Reg,
163 MachineRegisterInfo &MRI) {
164 std::optional<FPValueAndVReg> FPValReg;
165 if (mi_match(R: Reg, MRI, P: m_GFCstOrSplat(FPValReg))) {
166 if (FPValReg->Value.isZero() && !FPValReg->Value.isNegative())
167 return true;
168
169 const GCNSubtarget &ST = MI.getMF()->getSubtarget<GCNSubtarget>();
170 if (ST.hasInv2PiInlineImm() && isInv2Pi(APF: FPValReg->Value))
171 return true;
172 }
173 return false;
174}
175
176bool AMDGPUCombinerHelper::canIgnoreLegacyMinMaxTies(const MachineInstr &MI,
177 Register LHS,
178 Register RHS) const {
179 if (mayIgnoreSignedZero(MI))
180 return true;
181 return VT &&
182 (VT->isKnownNeverLogicalZero(Val: LHS) || VT->isKnownNeverLogicalZero(Val: RHS));
183}
184
185static unsigned inverseMinMax(unsigned Opc) {
186 switch (Opc) {
187 case AMDGPU::G_FMAXNUM:
188 return AMDGPU::G_FMINNUM;
189 case AMDGPU::G_FMINNUM:
190 return AMDGPU::G_FMAXNUM;
191 case AMDGPU::G_FMAXNUM_IEEE:
192 return AMDGPU::G_FMINNUM_IEEE;
193 case AMDGPU::G_FMINNUM_IEEE:
194 return AMDGPU::G_FMAXNUM_IEEE;
195 case AMDGPU::G_FMAXIMUM:
196 return AMDGPU::G_FMINIMUM;
197 case AMDGPU::G_FMINIMUM:
198 return AMDGPU::G_FMAXIMUM;
199 case AMDGPU::G_AMDGPU_FMAX_LEGACY:
200 return AMDGPU::G_AMDGPU_FMIN_LEGACY;
201 case AMDGPU::G_AMDGPU_FMIN_LEGACY:
202 return AMDGPU::G_AMDGPU_FMAX_LEGACY;
203 default:
204 llvm_unreachable("invalid min/max opcode");
205 }
206}
207
208bool AMDGPUCombinerHelper::matchFoldableFneg(MachineInstr &MI,
209 MachineInstr *&MatchInfo) const {
210 Register Src = MI.getOperand(i: 1).getReg();
211 MatchInfo = MRI.getVRegDef(Reg: Src);
212
213 // If the input has multiple uses and we can either fold the negate down, or
214 // the other uses cannot, give up. This both prevents unprofitable
215 // transformations and infinite loops: we won't repeatedly try to fold around
216 // a negate that has no 'good' form.
217 if (MRI.hasOneNonDBGUse(RegNo: Src)) {
218 if (allUsesHaveSourceMods(MI, MRI, CostThreshold: 0))
219 return false;
220 } else {
221 if (fnegFoldsIntoMI(MI: *MatchInfo) &&
222 (allUsesHaveSourceMods(MI, MRI) ||
223 !allUsesHaveSourceMods(MI&: *MatchInfo, MRI)))
224 return false;
225 }
226
227 switch (MatchInfo->getOpcode()) {
228 case AMDGPU::G_AMDGPU_FMIN_LEGACY:
229 case AMDGPU::G_AMDGPU_FMAX_LEGACY:
230 if (isConstantCostlierToNegate(MI&: *MatchInfo,
231 Reg: MatchInfo->getOperand(i: 2).getReg(), MRI))
232 return false;
233 // Swapping min<->max flips which operand a signed zero tie selects.
234 return canIgnoreLegacyMinMaxTies(MI: *MatchInfo,
235 LHS: MatchInfo->getOperand(i: 1).getReg(),
236 RHS: MatchInfo->getOperand(i: 2).getReg());
237 case AMDGPU::G_FMINNUM:
238 case AMDGPU::G_FMAXNUM:
239 case AMDGPU::G_FMINNUM_IEEE:
240 case AMDGPU::G_FMAXNUM_IEEE:
241 case AMDGPU::G_FMINIMUM:
242 case AMDGPU::G_FMAXIMUM:
243 // 0 doesn't have a negated inline immediate.
244 return !isConstantCostlierToNegate(MI&: *MatchInfo,
245 Reg: MatchInfo->getOperand(i: 2).getReg(), MRI);
246 case AMDGPU::G_FADD:
247 case AMDGPU::G_FSUB:
248 case AMDGPU::G_FMA:
249 case AMDGPU::G_FMAD:
250 return mayIgnoreSignedZero(MI: *MatchInfo);
251 case AMDGPU::G_FMUL:
252 case AMDGPU::G_FPEXT:
253 case AMDGPU::G_INTRINSIC_TRUNC:
254 case AMDGPU::G_FPTRUNC:
255 case AMDGPU::G_FRINT:
256 case AMDGPU::G_FNEARBYINT:
257 case AMDGPU::G_INTRINSIC_ROUND:
258 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
259 case AMDGPU::G_FSIN:
260 case AMDGPU::G_FCANONICALIZE:
261 case AMDGPU::G_AMDGPU_RCP_IFLAG:
262 return true;
263 case AMDGPU::G_INTRINSIC:
264 case AMDGPU::G_INTRINSIC_CONVERGENT: {
265 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: MatchInfo)->getIntrinsicID();
266 switch (IntrinsicID) {
267 case Intrinsic::amdgcn_rcp:
268 case Intrinsic::amdgcn_rcp_legacy:
269 case Intrinsic::amdgcn_sin:
270 case Intrinsic::amdgcn_fmul_legacy:
271 case Intrinsic::amdgcn_fmed3:
272 return true;
273 case Intrinsic::amdgcn_fma_legacy:
274 return mayIgnoreSignedZero(MI: *MatchInfo);
275 default:
276 return false;
277 }
278 }
279 default:
280 return false;
281 }
282}
283
284void AMDGPUCombinerHelper::applyFoldableFneg(MachineInstr &MI,
285 MachineInstr *&MatchInfo) const {
286 // Transform:
287 // %A = inst %Op1, ...
288 // %B = fneg %A
289 //
290 // into:
291 //
292 // (if %A has one use, specifically fneg above)
293 // %B = inst (maybe fneg %Op1), ...
294 //
295 // (if %A has multiple uses)
296 // %B = inst (maybe fneg %Op1), ...
297 // %A = fneg %B
298
299 // Replace register in operand with a register holding negated value.
300 auto NegateOperand = [&](MachineOperand &Op) {
301 Register Reg = Op.getReg();
302 if (!mi_match(R: Reg, MRI, P: m_GFNeg(Src: m_Reg(R&: Reg))))
303 Reg = Builder.buildFNeg(Dst: MRI.getType(Reg), Src0: Reg).getReg(Idx: 0);
304 replaceRegOpWith(MRI, FromRegOp&: Op, ToReg: Reg);
305 };
306
307 // Replace either register in operands with a register holding negated value.
308 auto NegateEitherOperand = [&](MachineOperand &X, MachineOperand &Y) {
309 Register XReg = X.getReg();
310 Register YReg = Y.getReg();
311 if (mi_match(R: XReg, MRI, P: m_GFNeg(Src: m_Reg(R&: XReg))))
312 replaceRegOpWith(MRI, FromRegOp&: X, ToReg: XReg);
313 else if (mi_match(R: YReg, MRI, P: m_GFNeg(Src: m_Reg(R&: YReg))))
314 replaceRegOpWith(MRI, FromRegOp&: Y, ToReg: YReg);
315 else {
316 YReg = Builder.buildFNeg(Dst: MRI.getType(Reg: YReg), Src0: YReg).getReg(Idx: 0);
317 replaceRegOpWith(MRI, FromRegOp&: Y, ToReg: YReg);
318 }
319 };
320
321 Builder.setInstrAndDebugLoc(*MatchInfo);
322
323 // Negate appropriate operands so that resulting value of MatchInfo is
324 // negated.
325 switch (MatchInfo->getOpcode()) {
326 case AMDGPU::G_FADD:
327 case AMDGPU::G_FSUB:
328 NegateOperand(MatchInfo->getOperand(i: 1));
329 NegateOperand(MatchInfo->getOperand(i: 2));
330 break;
331 case AMDGPU::G_FMUL:
332 NegateEitherOperand(MatchInfo->getOperand(i: 1), MatchInfo->getOperand(i: 2));
333 break;
334 case AMDGPU::G_FMINNUM:
335 case AMDGPU::G_FMAXNUM:
336 case AMDGPU::G_FMINNUM_IEEE:
337 case AMDGPU::G_FMAXNUM_IEEE:
338 case AMDGPU::G_FMINIMUM:
339 case AMDGPU::G_FMAXIMUM:
340 case AMDGPU::G_AMDGPU_FMIN_LEGACY:
341 case AMDGPU::G_AMDGPU_FMAX_LEGACY: {
342 NegateOperand(MatchInfo->getOperand(i: 1));
343 NegateOperand(MatchInfo->getOperand(i: 2));
344 unsigned Opposite = inverseMinMax(Opc: MatchInfo->getOpcode());
345 replaceOpcodeWith(FromMI&: *MatchInfo, ToOpcode: Opposite);
346 break;
347 }
348 case AMDGPU::G_FMA:
349 case AMDGPU::G_FMAD:
350 NegateEitherOperand(MatchInfo->getOperand(i: 1), MatchInfo->getOperand(i: 2));
351 NegateOperand(MatchInfo->getOperand(i: 3));
352 break;
353 case AMDGPU::G_FPEXT:
354 case AMDGPU::G_INTRINSIC_TRUNC:
355 case AMDGPU::G_FRINT:
356 case AMDGPU::G_FNEARBYINT:
357 case AMDGPU::G_INTRINSIC_ROUND:
358 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
359 case AMDGPU::G_FSIN:
360 case AMDGPU::G_FCANONICALIZE:
361 case AMDGPU::G_AMDGPU_RCP_IFLAG:
362 case AMDGPU::G_FPTRUNC:
363 NegateOperand(MatchInfo->getOperand(i: 1));
364 break;
365 case AMDGPU::G_INTRINSIC:
366 case AMDGPU::G_INTRINSIC_CONVERGENT: {
367 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: MatchInfo)->getIntrinsicID();
368 switch (IntrinsicID) {
369 case Intrinsic::amdgcn_rcp:
370 case Intrinsic::amdgcn_rcp_legacy:
371 case Intrinsic::amdgcn_sin:
372 NegateOperand(MatchInfo->getOperand(i: 2));
373 break;
374 case Intrinsic::amdgcn_fmul_legacy:
375 NegateEitherOperand(MatchInfo->getOperand(i: 2), MatchInfo->getOperand(i: 3));
376 break;
377 case Intrinsic::amdgcn_fmed3:
378 NegateOperand(MatchInfo->getOperand(i: 2));
379 NegateOperand(MatchInfo->getOperand(i: 3));
380 NegateOperand(MatchInfo->getOperand(i: 4));
381 break;
382 case Intrinsic::amdgcn_fma_legacy:
383 NegateEitherOperand(MatchInfo->getOperand(i: 2), MatchInfo->getOperand(i: 3));
384 NegateOperand(MatchInfo->getOperand(i: 4));
385 break;
386 default:
387 llvm_unreachable("folding fneg not supported for this intrinsic");
388 }
389 break;
390 }
391 default:
392 llvm_unreachable("folding fneg not supported for this instruction");
393 }
394
395 Register Dst = MI.getOperand(i: 0).getReg();
396 Register MatchInfoDst = MatchInfo->getOperand(i: 0).getReg();
397
398 if (MRI.hasOneNonDBGUse(RegNo: MatchInfoDst)) {
399 // MatchInfo now has negated value so use that instead of old Dst.
400 replaceRegWith(MRI, FromReg: Dst, ToReg: MatchInfoDst);
401 } else {
402 // We want to swap all uses of Dst with uses of MatchInfoDst and vice versa
403 // but replaceRegWith will replace defs as well. It is easier to replace one
404 // def with a new register.
405 LLT Type = MRI.getType(Reg: Dst);
406 Register NegatedMatchInfo = MRI.createGenericVirtualRegister(Ty: Type);
407 replaceRegOpWith(MRI, FromRegOp&: MatchInfo->getOperand(i: 0), ToReg: NegatedMatchInfo);
408
409 // MatchInfo now has negated value so use that instead of old Dst.
410 replaceRegWith(MRI, FromReg: Dst, ToReg: NegatedMatchInfo);
411
412 // Recreate non negated value for other uses of old MatchInfoDst
413 auto NextInst = ++MatchInfo->getIterator();
414 Builder.setInstrAndDebugLoc(*NextInst);
415 Builder.buildFNeg(Dst: MatchInfoDst, Src0: NegatedMatchInfo, Flags: MI.getFlags());
416 }
417
418 MI.eraseFromParent();
419}
420
421bool AMDGPUCombinerHelper::matchFoldFAbsFptrunc(MachineInstr &Fabs,
422 MachineInstr &Fptrunc) const {
423 Register Round = Fptrunc.getOperand(i: 0).getReg();
424 if (!MRI.hasOneNonDBGUse(RegNo: Round))
425 return false;
426
427 LLT SrcTy = MRI.getType(Reg: Fptrunc.getOperand(i: 1).getReg());
428 return isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FABS, {SrcTy}});
429}
430
431void AMDGPUCombinerHelper::applyFoldFAbsFptrunc(MachineInstr &Fabs,
432 MachineInstr &Fptrunc) const {
433 // fabs (fptrunc x) -> fptrunc (fabs x)
434 Register Dst = Fabs.getOperand(i: 0).getReg();
435 Register Src = Fptrunc.getOperand(i: 1).getReg();
436 Builder.setInstrAndDebugLoc(Fabs);
437 Register Abs =
438 Builder.buildFAbs(Dst: MRI.getType(Reg: Src), Src0: Src, Flags: Fabs.getFlags()).getReg(Idx: 0);
439 Builder.buildFPTrunc(Res: Dst, Op: Abs, Flags: Fptrunc.getFlags());
440 Fabs.eraseFromParent();
441}
442
443// TODO: Should return converted value / extension source and avoid introducing
444// intermediate fptruncs in the apply function.
445static bool isFPExtFromF16OrConst(const MachineRegisterInfo &MRI,
446 Register Reg) {
447 Register SrcReg;
448 if (mi_match(R: Reg, MRI, P: m_GFPExt(Src: m_Reg(R&: SrcReg))))
449 return MRI.getType(Reg: SrcReg) == LLT::float16();
450
451 const ConstantFP *FPImm;
452 if (mi_match(R: Reg, MRI, P: m_GFCst(C&: FPImm))) {
453 APFloat Val = FPImm->getValueAPF();
454 bool LosesInfo = true;
455 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
456 return !LosesInfo;
457 }
458
459 return false;
460}
461
462bool AMDGPUCombinerHelper::matchExpandPromotedF16FMed3(MachineInstr &MI,
463 Register Src0,
464 Register Src1,
465 Register Src2) const {
466 assert(MI.getOpcode() == TargetOpcode::G_FPTRUNC);
467 Register SrcReg = MI.getOperand(i: 1).getReg();
468 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()) != LLT::float16())
469 return false;
470 if (!MRI.hasOneNonDBGUse(RegNo: SrcReg) || MRI.getType(Reg: SrcReg) != LLT::float32())
471 return false;
472
473 return isFPExtFromF16OrConst(MRI, Reg: Src0) && isFPExtFromF16OrConst(MRI, Reg: Src1) &&
474 isFPExtFromF16OrConst(MRI, Reg: Src2);
475}
476
477void AMDGPUCombinerHelper::applyExpandPromotedF16FMed3(MachineInstr &MI,
478 Register Src0,
479 Register Src1,
480 Register Src2) const {
481 // We expect fptrunc (fpext x) to fold out, and to constant fold any constant
482 // sources.
483 Src0 = Builder.buildFPTrunc(Res: LLT::float16(), Op: Src0).getReg(Idx: 0);
484 Src1 = Builder.buildFPTrunc(Res: LLT::float16(), Op: Src1).getReg(Idx: 0);
485 Src2 = Builder.buildFPTrunc(Res: LLT::float16(), Op: Src2).getReg(Idx: 0);
486
487 LLT Ty = MRI.getType(Reg: Src0);
488 auto A1 = Builder.buildFMinNumIEEE(Dst: Ty, Src0, Src1);
489 auto B1 = Builder.buildFMaxNumIEEE(Dst: Ty, Src0, Src1);
490 auto C1 = Builder.buildFMaxNumIEEE(Dst: Ty, Src0: A1, Src1: Src2);
491 Builder.buildFMinNumIEEE(Dst: MI.getOperand(i: 0), Src0: B1, Src1: C1);
492 MI.eraseFromParent();
493}
494
495bool AMDGPUCombinerHelper::matchCombineFmulWithSelectToFldexp(
496 MachineInstr &MI, MachineInstr &Sel,
497 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
498 assert(MI.getOpcode() == TargetOpcode::G_FMUL);
499 assert(Sel.getOpcode() == TargetOpcode::G_SELECT);
500 assert(MI.getOperand(2).getReg() == Sel.getOperand(0).getReg());
501
502 Register Dst = MI.getOperand(i: 0).getReg();
503 LLT DestTy = MRI.getType(Reg: Dst);
504 LLT ScalarDestTy = DestTy.getScalarType();
505
506 // TODO: Expected float type in ScalarDestTy
507 if ((ScalarDestTy != LLT::float64() && ScalarDestTy != LLT::float32() &&
508 ScalarDestTy != LLT::float16()) ||
509 !MRI.hasOneNonDBGUse(RegNo: Sel.getOperand(i: 0).getReg()))
510 return false;
511
512 Register SelectCondReg = Sel.getOperand(i: 1).getReg();
513 Register SelectTrueReg = Sel.getOperand(i: 2).getReg();
514 Register SelectFalseReg = Sel.getOperand(i: 3).getReg();
515
516 const auto SelectTrueVal =
517 isConstantOrConstantSplatVectorFP(Def: SelectTrueReg, MRI);
518 if (!SelectTrueVal)
519 return false;
520 const auto SelectFalseVal =
521 isConstantOrConstantSplatVectorFP(Def: SelectFalseReg, MRI);
522 if (!SelectFalseVal)
523 return false;
524
525 if (SelectTrueVal->isNegative() != SelectFalseVal->isNegative())
526 return false;
527
528 // For f32, only non-inline constants should be transformed.
529 // TODO: Expected float32
530 if (ScalarDestTy == LLT::float32() && TII.isInlineConstant(Imm: *SelectTrueVal) &&
531 TII.isInlineConstant(Imm: *SelectFalseVal))
532 return false;
533
534 int SelectTrueLog2Val = SelectTrueVal->getExactLog2Abs();
535 if (SelectTrueLog2Val == INT_MIN)
536 return false;
537 int SelectFalseLog2Val = SelectFalseVal->getExactLog2Abs();
538 if (SelectFalseLog2Val == INT_MIN)
539 return false;
540
541 MatchInfo = [=, &MI](MachineIRBuilder &Builder) {
542 LLT IntDestTy = DestTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 32));
543 auto NewSel = Builder.buildSelect(
544 Res: IntDestTy, Tst: SelectCondReg,
545 Op0: Builder.buildConstant(Res: IntDestTy, Val: SelectTrueLog2Val),
546 Op1: Builder.buildConstant(Res: IntDestTy, Val: SelectFalseLog2Val));
547
548 Register XReg = MI.getOperand(i: 1).getReg();
549 if (SelectTrueVal->isNegative()) {
550 auto NegX =
551 Builder.buildFNeg(Dst: DestTy, Src0: XReg, Flags: MRI.getVRegDef(Reg: XReg)->getFlags());
552 Builder.buildFLdexp(Dst, Src0: NegX, Src1: NewSel, Flags: MI.getFlags());
553 } else {
554 Builder.buildFLdexp(Dst, Src0: XReg, Src1: NewSel, Flags: MI.getFlags());
555 }
556 };
557
558 return true;
559}
560
561bool AMDGPUCombinerHelper::matchConstantIs32BitMask(Register Reg) const {
562 auto Res = getIConstantVRegValWithLookThrough(VReg: Reg, MRI);
563 if (!Res)
564 return false;
565
566 const uint64_t Val = Res->Value.getZExtValue();
567 unsigned MaskIdx = 0;
568 unsigned MaskLen = 0;
569 if (!isShiftedMask_64(Value: Val, MaskIdx, MaskLen))
570 return false;
571
572 // Check if low 32 bits or high 32 bits are all ones.
573 return MaskLen >= 32 && ((MaskIdx == 0) || (MaskIdx == 64 - MaskLen));
574}
575