1//=== lib/CodeGen/GlobalISel/AMDGPUCombinerHelper.cpp ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUCombinerHelper.h"
10#include "GCNSubtarget.h"
11#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
13#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
14#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
15#include "llvm/IR/IntrinsicsAMDGPU.h"
16#include "llvm/Target/TargetMachine.h"
17
18using namespace llvm;
19using namespace MIPatternMatch;
20
21AMDGPUCombinerHelper::AMDGPUCombinerHelper(
22 GISelChangeObserver &Observer, MachineIRBuilder &B, bool IsPreLegalize,
23 GISelValueTracking *VT, MachineDominatorTree *MDT, const LegalizerInfo *LI,
24 const GCNSubtarget &STI)
25 : CombinerHelper(Observer, B, IsPreLegalize, VT, MDT, LI), STI(STI),
26 TII(*STI.getInstrInfo()) {}
27
28LLVM_READNONE
29static bool fnegFoldsIntoMI(const MachineInstr &MI) {
30 switch (MI.getOpcode()) {
31 case AMDGPU::G_FADD:
32 case AMDGPU::G_FSUB:
33 case AMDGPU::G_FMUL:
34 case AMDGPU::G_FMA:
35 case AMDGPU::G_FMAD:
36 case AMDGPU::G_FMINNUM:
37 case AMDGPU::G_FMAXNUM:
38 case AMDGPU::G_FMINNUM_IEEE:
39 case AMDGPU::G_FMAXNUM_IEEE:
40 case AMDGPU::G_FMINIMUM:
41 case AMDGPU::G_FMAXIMUM:
42 case AMDGPU::G_FSIN:
43 case AMDGPU::G_FPEXT:
44 case AMDGPU::G_INTRINSIC_TRUNC:
45 case AMDGPU::G_FPTRUNC:
46 case AMDGPU::G_FRINT:
47 case AMDGPU::G_FNEARBYINT:
48 case AMDGPU::G_INTRINSIC_ROUND:
49 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
50 case AMDGPU::G_FCANONICALIZE:
51 case AMDGPU::G_AMDGPU_RCP_IFLAG:
52 case AMDGPU::G_AMDGPU_FMIN_LEGACY:
53 case AMDGPU::G_AMDGPU_FMAX_LEGACY:
54 return true;
55 case AMDGPU::G_INTRINSIC: {
56 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: MI).getIntrinsicID();
57 switch (IntrinsicID) {
58 case Intrinsic::amdgcn_rcp:
59 case Intrinsic::amdgcn_rcp_legacy:
60 case Intrinsic::amdgcn_sin:
61 case Intrinsic::amdgcn_fmul_legacy:
62 case Intrinsic::amdgcn_fmed3:
63 case Intrinsic::amdgcn_fma_legacy:
64 return true;
65 default:
66 return false;
67 }
68 }
69 default:
70 return false;
71 }
72}
73
74/// \p returns true if the operation will definitely need to use a 64-bit
75/// encoding, and thus will use a VOP3 encoding regardless of the source
76/// modifiers.
77LLVM_READONLY
78static bool opMustUseVOP3Encoding(const MachineInstr &MI,
79 const MachineRegisterInfo &MRI) {
80 return MI.getNumOperands() > (isa<GIntrinsic>(Val: MI) ? 4u : 3u) ||
81 MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getScalarSizeInBits() == 64;
82}
83
84// Most FP instructions support source modifiers.
85LLVM_READONLY
86static bool hasSourceMods(const MachineInstr &MI) {
87 if (!MI.memoperands().empty())
88 return false;
89
90 switch (MI.getOpcode()) {
91 case AMDGPU::COPY:
92 case AMDGPU::G_SELECT:
93 case AMDGPU::G_FDIV:
94 case AMDGPU::G_FREM:
95 case TargetOpcode::INLINEASM:
96 case TargetOpcode::INLINEASM_BR:
97 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
98 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
99 case AMDGPU::G_BITCAST:
100 case AMDGPU::G_ANYEXT:
101 case AMDGPU::G_BUILD_VECTOR:
102 case AMDGPU::G_BUILD_VECTOR_TRUNC:
103 case AMDGPU::G_PHI:
104 return false;
105 case AMDGPU::G_INTRINSIC:
106 case AMDGPU::G_INTRINSIC_CONVERGENT: {
107 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: MI).getIntrinsicID();
108 switch (IntrinsicID) {
109 case Intrinsic::amdgcn_interp_p1:
110 case Intrinsic::amdgcn_interp_p2:
111 case Intrinsic::amdgcn_interp_mov:
112 case Intrinsic::amdgcn_interp_p1_f16:
113 case Intrinsic::amdgcn_interp_p2_f16:
114 case Intrinsic::amdgcn_div_scale:
115 return false;
116 default:
117 return true;
118 }
119 }
120 default:
121 return true;
122 }
123}
124
125static bool allUsesHaveSourceMods(MachineInstr &MI, MachineRegisterInfo &MRI,
126 unsigned CostThreshold = 4) {
127 // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus
128 // it is truly free to use a source modifier in all cases. If there are
129 // multiple users but for each one will necessitate using VOP3, there will be
130 // a code size increase. Try to avoid increasing code size unless we know it
131 // will save on the instruction count.
132 unsigned NumMayIncreaseSize = 0;
133 Register Dst = MI.getOperand(i: 0).getReg();
134 for (const MachineInstr &Use : MRI.use_nodbg_instructions(Reg: Dst)) {
135 if (!hasSourceMods(MI: Use))
136 return false;
137
138 if (!opMustUseVOP3Encoding(MI: Use, MRI)) {
139 if (++NumMayIncreaseSize > CostThreshold)
140 return false;
141 }
142 }
143 return true;
144}
145
146static bool mayIgnoreSignedZero(MachineInstr &MI) {
147 return MI.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
148}
149
150static bool isInv2Pi(const APFloat &APF) {
151 static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118));
152 static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983));
153 static const APFloat KF64(APFloat::IEEEdouble(),
154 APInt(64, 0x3fc45f306dc9c882));
155
156 return APF.bitwiseIsEqual(RHS: KF16) || APF.bitwiseIsEqual(RHS: KF32) ||
157 APF.bitwiseIsEqual(RHS: KF64);
158}
159
160// 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an
161// additional cost to negate them.
162static bool isConstantCostlierToNegate(MachineInstr &MI, Register Reg,
163 MachineRegisterInfo &MRI) {
164 std::optional<FPValueAndVReg> FPValReg;
165 if (mi_match(R: Reg, MRI, P: m_GFCstOrSplat(FPValReg))) {
166 if (FPValReg->Value.isZero() && !FPValReg->Value.isNegative())
167 return true;
168
169 const GCNSubtarget &ST = MI.getMF()->getSubtarget<GCNSubtarget>();
170 if (ST.hasInv2PiInlineImm() && isInv2Pi(APF: FPValReg->Value))
171 return true;
172 }
173 return false;
174}
175
176static unsigned inverseMinMax(unsigned Opc) {
177 switch (Opc) {
178 case AMDGPU::G_FMAXNUM:
179 return AMDGPU::G_FMINNUM;
180 case AMDGPU::G_FMINNUM:
181 return AMDGPU::G_FMAXNUM;
182 case AMDGPU::G_FMAXNUM_IEEE:
183 return AMDGPU::G_FMINNUM_IEEE;
184 case AMDGPU::G_FMINNUM_IEEE:
185 return AMDGPU::G_FMAXNUM_IEEE;
186 case AMDGPU::G_FMAXIMUM:
187 return AMDGPU::G_FMINIMUM;
188 case AMDGPU::G_FMINIMUM:
189 return AMDGPU::G_FMAXIMUM;
190 case AMDGPU::G_AMDGPU_FMAX_LEGACY:
191 return AMDGPU::G_AMDGPU_FMIN_LEGACY;
192 case AMDGPU::G_AMDGPU_FMIN_LEGACY:
193 return AMDGPU::G_AMDGPU_FMAX_LEGACY;
194 default:
195 llvm_unreachable("invalid min/max opcode");
196 }
197}
198
199bool AMDGPUCombinerHelper::matchFoldableFneg(MachineInstr &MI,
200 MachineInstr *&MatchInfo) const {
201 Register Src = MI.getOperand(i: 1).getReg();
202 MatchInfo = MRI.getVRegDef(Reg: Src);
203
204 // If the input has multiple uses and we can either fold the negate down, or
205 // the other uses cannot, give up. This both prevents unprofitable
206 // transformations and infinite loops: we won't repeatedly try to fold around
207 // a negate that has no 'good' form.
208 if (MRI.hasOneNonDBGUse(RegNo: Src)) {
209 if (allUsesHaveSourceMods(MI, MRI, CostThreshold: 0))
210 return false;
211 } else {
212 if (fnegFoldsIntoMI(MI: *MatchInfo) &&
213 (allUsesHaveSourceMods(MI, MRI) ||
214 !allUsesHaveSourceMods(MI&: *MatchInfo, MRI)))
215 return false;
216 }
217
218 switch (MatchInfo->getOpcode()) {
219 case AMDGPU::G_FMINNUM:
220 case AMDGPU::G_FMAXNUM:
221 case AMDGPU::G_FMINNUM_IEEE:
222 case AMDGPU::G_FMAXNUM_IEEE:
223 case AMDGPU::G_FMINIMUM:
224 case AMDGPU::G_FMAXIMUM:
225 case AMDGPU::G_AMDGPU_FMIN_LEGACY:
226 case AMDGPU::G_AMDGPU_FMAX_LEGACY:
227 // 0 doesn't have a negated inline immediate.
228 return !isConstantCostlierToNegate(MI&: *MatchInfo,
229 Reg: MatchInfo->getOperand(i: 2).getReg(), MRI);
230 case AMDGPU::G_FADD:
231 case AMDGPU::G_FSUB:
232 case AMDGPU::G_FMA:
233 case AMDGPU::G_FMAD:
234 return mayIgnoreSignedZero(MI&: *MatchInfo);
235 case AMDGPU::G_FMUL:
236 case AMDGPU::G_FPEXT:
237 case AMDGPU::G_INTRINSIC_TRUNC:
238 case AMDGPU::G_FPTRUNC:
239 case AMDGPU::G_FRINT:
240 case AMDGPU::G_FNEARBYINT:
241 case AMDGPU::G_INTRINSIC_ROUND:
242 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
243 case AMDGPU::G_FSIN:
244 case AMDGPU::G_FCANONICALIZE:
245 case AMDGPU::G_AMDGPU_RCP_IFLAG:
246 return true;
247 case AMDGPU::G_INTRINSIC:
248 case AMDGPU::G_INTRINSIC_CONVERGENT: {
249 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: MatchInfo)->getIntrinsicID();
250 switch (IntrinsicID) {
251 case Intrinsic::amdgcn_rcp:
252 case Intrinsic::amdgcn_rcp_legacy:
253 case Intrinsic::amdgcn_sin:
254 case Intrinsic::amdgcn_fmul_legacy:
255 case Intrinsic::amdgcn_fmed3:
256 return true;
257 case Intrinsic::amdgcn_fma_legacy:
258 return mayIgnoreSignedZero(MI&: *MatchInfo);
259 default:
260 return false;
261 }
262 }
263 default:
264 return false;
265 }
266}
267
268void AMDGPUCombinerHelper::applyFoldableFneg(MachineInstr &MI,
269 MachineInstr *&MatchInfo) const {
270 // Transform:
271 // %A = inst %Op1, ...
272 // %B = fneg %A
273 //
274 // into:
275 //
276 // (if %A has one use, specifically fneg above)
277 // %B = inst (maybe fneg %Op1), ...
278 //
279 // (if %A has multiple uses)
280 // %B = inst (maybe fneg %Op1), ...
281 // %A = fneg %B
282
283 // Replace register in operand with a register holding negated value.
284 auto NegateOperand = [&](MachineOperand &Op) {
285 Register Reg = Op.getReg();
286 if (!mi_match(R: Reg, MRI, P: m_GFNeg(Src: m_Reg(R&: Reg))))
287 Reg = Builder.buildFNeg(Dst: MRI.getType(Reg), Src0: Reg).getReg(Idx: 0);
288 replaceRegOpWith(MRI, FromRegOp&: Op, ToReg: Reg);
289 };
290
291 // Replace either register in operands with a register holding negated value.
292 auto NegateEitherOperand = [&](MachineOperand &X, MachineOperand &Y) {
293 Register XReg = X.getReg();
294 Register YReg = Y.getReg();
295 if (mi_match(R: XReg, MRI, P: m_GFNeg(Src: m_Reg(R&: XReg))))
296 replaceRegOpWith(MRI, FromRegOp&: X, ToReg: XReg);
297 else if (mi_match(R: YReg, MRI, P: m_GFNeg(Src: m_Reg(R&: YReg))))
298 replaceRegOpWith(MRI, FromRegOp&: Y, ToReg: YReg);
299 else {
300 YReg = Builder.buildFNeg(Dst: MRI.getType(Reg: YReg), Src0: YReg).getReg(Idx: 0);
301 replaceRegOpWith(MRI, FromRegOp&: Y, ToReg: YReg);
302 }
303 };
304
305 Builder.setInstrAndDebugLoc(*MatchInfo);
306
307 // Negate appropriate operands so that resulting value of MatchInfo is
308 // negated.
309 switch (MatchInfo->getOpcode()) {
310 case AMDGPU::G_FADD:
311 case AMDGPU::G_FSUB:
312 NegateOperand(MatchInfo->getOperand(i: 1));
313 NegateOperand(MatchInfo->getOperand(i: 2));
314 break;
315 case AMDGPU::G_FMUL:
316 NegateEitherOperand(MatchInfo->getOperand(i: 1), MatchInfo->getOperand(i: 2));
317 break;
318 case AMDGPU::G_FMINNUM:
319 case AMDGPU::G_FMAXNUM:
320 case AMDGPU::G_FMINNUM_IEEE:
321 case AMDGPU::G_FMAXNUM_IEEE:
322 case AMDGPU::G_FMINIMUM:
323 case AMDGPU::G_FMAXIMUM:
324 case AMDGPU::G_AMDGPU_FMIN_LEGACY:
325 case AMDGPU::G_AMDGPU_FMAX_LEGACY: {
326 NegateOperand(MatchInfo->getOperand(i: 1));
327 NegateOperand(MatchInfo->getOperand(i: 2));
328 unsigned Opposite = inverseMinMax(Opc: MatchInfo->getOpcode());
329 replaceOpcodeWith(FromMI&: *MatchInfo, ToOpcode: Opposite);
330 break;
331 }
332 case AMDGPU::G_FMA:
333 case AMDGPU::G_FMAD:
334 NegateEitherOperand(MatchInfo->getOperand(i: 1), MatchInfo->getOperand(i: 2));
335 NegateOperand(MatchInfo->getOperand(i: 3));
336 break;
337 case AMDGPU::G_FPEXT:
338 case AMDGPU::G_INTRINSIC_TRUNC:
339 case AMDGPU::G_FRINT:
340 case AMDGPU::G_FNEARBYINT:
341 case AMDGPU::G_INTRINSIC_ROUND:
342 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
343 case AMDGPU::G_FSIN:
344 case AMDGPU::G_FCANONICALIZE:
345 case AMDGPU::G_AMDGPU_RCP_IFLAG:
346 case AMDGPU::G_FPTRUNC:
347 NegateOperand(MatchInfo->getOperand(i: 1));
348 break;
349 case AMDGPU::G_INTRINSIC:
350 case AMDGPU::G_INTRINSIC_CONVERGENT: {
351 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: MatchInfo)->getIntrinsicID();
352 switch (IntrinsicID) {
353 case Intrinsic::amdgcn_rcp:
354 case Intrinsic::amdgcn_rcp_legacy:
355 case Intrinsic::amdgcn_sin:
356 NegateOperand(MatchInfo->getOperand(i: 2));
357 break;
358 case Intrinsic::amdgcn_fmul_legacy:
359 NegateEitherOperand(MatchInfo->getOperand(i: 2), MatchInfo->getOperand(i: 3));
360 break;
361 case Intrinsic::amdgcn_fmed3:
362 NegateOperand(MatchInfo->getOperand(i: 2));
363 NegateOperand(MatchInfo->getOperand(i: 3));
364 NegateOperand(MatchInfo->getOperand(i: 4));
365 break;
366 case Intrinsic::amdgcn_fma_legacy:
367 NegateEitherOperand(MatchInfo->getOperand(i: 2), MatchInfo->getOperand(i: 3));
368 NegateOperand(MatchInfo->getOperand(i: 4));
369 break;
370 default:
371 llvm_unreachable("folding fneg not supported for this intrinsic");
372 }
373 break;
374 }
375 default:
376 llvm_unreachable("folding fneg not supported for this instruction");
377 }
378
379 Register Dst = MI.getOperand(i: 0).getReg();
380 Register MatchInfoDst = MatchInfo->getOperand(i: 0).getReg();
381
382 if (MRI.hasOneNonDBGUse(RegNo: MatchInfoDst)) {
383 // MatchInfo now has negated value so use that instead of old Dst.
384 replaceRegWith(MRI, FromReg: Dst, ToReg: MatchInfoDst);
385 } else {
386 // We want to swap all uses of Dst with uses of MatchInfoDst and vice versa
387 // but replaceRegWith will replace defs as well. It is easier to replace one
388 // def with a new register.
389 LLT Type = MRI.getType(Reg: Dst);
390 Register NegatedMatchInfo = MRI.createGenericVirtualRegister(Ty: Type);
391 replaceRegOpWith(MRI, FromRegOp&: MatchInfo->getOperand(i: 0), ToReg: NegatedMatchInfo);
392
393 // MatchInfo now has negated value so use that instead of old Dst.
394 replaceRegWith(MRI, FromReg: Dst, ToReg: NegatedMatchInfo);
395
396 // Recreate non negated value for other uses of old MatchInfoDst
397 auto NextInst = ++MatchInfo->getIterator();
398 Builder.setInstrAndDebugLoc(*NextInst);
399 Builder.buildFNeg(Dst: MatchInfoDst, Src0: NegatedMatchInfo, Flags: MI.getFlags());
400 }
401
402 MI.eraseFromParent();
403}
404
405bool AMDGPUCombinerHelper::matchFoldFAbsFptrunc(MachineInstr &Fabs,
406 MachineInstr &Fptrunc) const {
407 Register Round = Fptrunc.getOperand(i: 0).getReg();
408 if (!MRI.hasOneNonDBGUse(RegNo: Round))
409 return false;
410
411 LLT SrcTy = MRI.getType(Reg: Fptrunc.getOperand(i: 1).getReg());
412 return isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FABS, {SrcTy}});
413}
414
415void AMDGPUCombinerHelper::applyFoldFAbsFptrunc(MachineInstr &Fabs,
416 MachineInstr &Fptrunc) const {
417 // fabs (fptrunc x) -> fptrunc (fabs x)
418 Register Dst = Fabs.getOperand(i: 0).getReg();
419 Register Src = Fptrunc.getOperand(i: 1).getReg();
420 Builder.setInstrAndDebugLoc(Fabs);
421 Register Abs =
422 Builder.buildFAbs(Dst: MRI.getType(Reg: Src), Src0: Src, Flags: Fabs.getFlags()).getReg(Idx: 0);
423 Builder.buildFPTrunc(Res: Dst, Op: Abs, Flags: Fptrunc.getFlags());
424 Fabs.eraseFromParent();
425}
426
427// TODO: Should return converted value / extension source and avoid introducing
428// intermediate fptruncs in the apply function.
429static bool isFPExtFromF16OrConst(const MachineRegisterInfo &MRI,
430 Register Reg) {
431 const MachineInstr *Def = MRI.getVRegDef(Reg);
432 if (Def->getOpcode() == TargetOpcode::G_FPEXT) {
433 Register SrcReg = Def->getOperand(i: 1).getReg();
434 return MRI.getType(Reg: SrcReg) == LLT::float16();
435 }
436
437 if (Def->getOpcode() == TargetOpcode::G_FCONSTANT) {
438 APFloat Val = Def->getOperand(i: 1).getFPImm()->getValueAPF();
439 bool LosesInfo = true;
440 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
441 return !LosesInfo;
442 }
443
444 return false;
445}
446
447bool AMDGPUCombinerHelper::matchExpandPromotedF16FMed3(MachineInstr &MI,
448 Register Src0,
449 Register Src1,
450 Register Src2) const {
451 assert(MI.getOpcode() == TargetOpcode::G_FPTRUNC);
452 Register SrcReg = MI.getOperand(i: 1).getReg();
453 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()) != LLT::float16())
454 return false;
455 if (!MRI.hasOneNonDBGUse(RegNo: SrcReg) || MRI.getType(Reg: SrcReg) != LLT::float32())
456 return false;
457
458 return isFPExtFromF16OrConst(MRI, Reg: Src0) && isFPExtFromF16OrConst(MRI, Reg: Src1) &&
459 isFPExtFromF16OrConst(MRI, Reg: Src2);
460}
461
462void AMDGPUCombinerHelper::applyExpandPromotedF16FMed3(MachineInstr &MI,
463 Register Src0,
464 Register Src1,
465 Register Src2) const {
466 // We expect fptrunc (fpext x) to fold out, and to constant fold any constant
467 // sources.
468 Src0 = Builder.buildFPTrunc(Res: LLT::float16(), Op: Src0).getReg(Idx: 0);
469 Src1 = Builder.buildFPTrunc(Res: LLT::float16(), Op: Src1).getReg(Idx: 0);
470 Src2 = Builder.buildFPTrunc(Res: LLT::float16(), Op: Src2).getReg(Idx: 0);
471
472 LLT Ty = MRI.getType(Reg: Src0);
473 auto A1 = Builder.buildFMinNumIEEE(Dst: Ty, Src0, Src1);
474 auto B1 = Builder.buildFMaxNumIEEE(Dst: Ty, Src0, Src1);
475 auto C1 = Builder.buildFMaxNumIEEE(Dst: Ty, Src0: A1, Src1: Src2);
476 Builder.buildFMinNumIEEE(Dst: MI.getOperand(i: 0), Src0: B1, Src1: C1);
477 MI.eraseFromParent();
478}
479
480bool AMDGPUCombinerHelper::matchCombineFmulWithSelectToFldexp(
481 MachineInstr &MI, MachineInstr &Sel,
482 std::function<void(MachineIRBuilder &)> &MatchInfo) const {
483 assert(MI.getOpcode() == TargetOpcode::G_FMUL);
484 assert(Sel.getOpcode() == TargetOpcode::G_SELECT);
485 assert(MI.getOperand(2).getReg() == Sel.getOperand(0).getReg());
486
487 Register Dst = MI.getOperand(i: 0).getReg();
488 LLT DestTy = MRI.getType(Reg: Dst);
489 LLT ScalarDestTy = DestTy.getScalarType();
490
491 // TODO: Expected float type in ScalarDestTy
492 if ((ScalarDestTy != LLT::float64() && ScalarDestTy != LLT::float32() &&
493 ScalarDestTy != LLT::float16()) ||
494 !MRI.hasOneNonDBGUse(RegNo: Sel.getOperand(i: 0).getReg()))
495 return false;
496
497 Register SelectCondReg = Sel.getOperand(i: 1).getReg();
498 Register SelectTrueReg = Sel.getOperand(i: 2).getReg();
499 Register SelectFalseReg = Sel.getOperand(i: 3).getReg();
500
501 const auto SelectTrueVal =
502 isConstantOrConstantSplatVectorFP(Def: SelectTrueReg, MRI);
503 if (!SelectTrueVal)
504 return false;
505 const auto SelectFalseVal =
506 isConstantOrConstantSplatVectorFP(Def: SelectFalseReg, MRI);
507 if (!SelectFalseVal)
508 return false;
509
510 if (SelectTrueVal->isNegative() != SelectFalseVal->isNegative())
511 return false;
512
513 // For f32, only non-inline constants should be transformed.
514 // TODO: Expected float32
515 if (ScalarDestTy == LLT::float32() && TII.isInlineConstant(Imm: *SelectTrueVal) &&
516 TII.isInlineConstant(Imm: *SelectFalseVal))
517 return false;
518
519 int SelectTrueLog2Val = SelectTrueVal->getExactLog2Abs();
520 if (SelectTrueLog2Val == INT_MIN)
521 return false;
522 int SelectFalseLog2Val = SelectFalseVal->getExactLog2Abs();
523 if (SelectFalseLog2Val == INT_MIN)
524 return false;
525
526 MatchInfo = [=, &MI](MachineIRBuilder &Builder) {
527 LLT IntDestTy = DestTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 32));
528 auto NewSel = Builder.buildSelect(
529 Res: IntDestTy, Tst: SelectCondReg,
530 Op0: Builder.buildConstant(Res: IntDestTy, Val: SelectTrueLog2Val),
531 Op1: Builder.buildConstant(Res: IntDestTy, Val: SelectFalseLog2Val));
532
533 Register XReg = MI.getOperand(i: 1).getReg();
534 if (SelectTrueVal->isNegative()) {
535 auto NegX =
536 Builder.buildFNeg(Dst: DestTy, Src0: XReg, Flags: MRI.getVRegDef(Reg: XReg)->getFlags());
537 Builder.buildFLdexp(Dst, Src0: NegX, Src1: NewSel, Flags: MI.getFlags());
538 } else {
539 Builder.buildFLdexp(Dst, Src0: XReg, Src1: NewSel, Flags: MI.getFlags());
540 }
541 };
542
543 return true;
544}
545
546bool AMDGPUCombinerHelper::matchConstantIs32BitMask(Register Reg) const {
547 auto Res = getIConstantVRegValWithLookThrough(VReg: Reg, MRI);
548 if (!Res)
549 return false;
550
551 const uint64_t Val = Res->Value.getZExtValue();
552 unsigned MaskIdx = 0;
553 unsigned MaskLen = 0;
554 if (!isShiftedMask_64(Value: Val, MaskIdx, MaskLen))
555 return false;
556
557 // Check if low 32 bits or high 32 bits are all ones.
558 return MaskLen >= 32 && ((MaskIdx == 0) || (MaskIdx == 64 - MaskLen));
559}
560