| 1 | //=== lib/CodeGen/GlobalISel/AMDGPUCombinerHelper.cpp ---------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "AMDGPUCombinerHelper.h" |
| 10 | #include "GCNSubtarget.h" |
| 11 | #include "llvm/CodeGen/GlobalISel/GISelValueTracking.h" |
| 12 | #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" |
| 13 | #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" |
| 14 | #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" |
| 15 | #include "llvm/IR/IntrinsicsAMDGPU.h" |
| 16 | #include "llvm/Target/TargetMachine.h" |
| 17 | |
| 18 | using namespace llvm; |
| 19 | using namespace MIPatternMatch; |
| 20 | |
| 21 | AMDGPUCombinerHelper::AMDGPUCombinerHelper( |
| 22 | GISelChangeObserver &Observer, MachineIRBuilder &B, bool IsPreLegalize, |
| 23 | GISelValueTracking *VT, MachineDominatorTree *MDT, const LegalizerInfo *LI, |
| 24 | const GCNSubtarget &STI) |
| 25 | : CombinerHelper(Observer, B, IsPreLegalize, VT, MDT, LI), STI(STI), |
| 26 | TII(*STI.getInstrInfo()) {} |
| 27 | |
| 28 | LLVM_READNONE |
| 29 | static bool fnegFoldsIntoMI(const MachineInstr &MI) { |
| 30 | switch (MI.getOpcode()) { |
| 31 | case AMDGPU::G_FADD: |
| 32 | case AMDGPU::G_FSUB: |
| 33 | case AMDGPU::G_FMUL: |
| 34 | case AMDGPU::G_FMA: |
| 35 | case AMDGPU::G_FMAD: |
| 36 | case AMDGPU::G_FMINNUM: |
| 37 | case AMDGPU::G_FMAXNUM: |
| 38 | case AMDGPU::G_FMINNUM_IEEE: |
| 39 | case AMDGPU::G_FMAXNUM_IEEE: |
| 40 | case AMDGPU::G_FMINIMUM: |
| 41 | case AMDGPU::G_FMAXIMUM: |
| 42 | case AMDGPU::G_FSIN: |
| 43 | case AMDGPU::G_FPEXT: |
| 44 | case AMDGPU::G_INTRINSIC_TRUNC: |
| 45 | case AMDGPU::G_FPTRUNC: |
| 46 | case AMDGPU::G_FRINT: |
| 47 | case AMDGPU::G_FNEARBYINT: |
| 48 | case AMDGPU::G_INTRINSIC_ROUND: |
| 49 | case AMDGPU::G_INTRINSIC_ROUNDEVEN: |
| 50 | case AMDGPU::G_FCANONICALIZE: |
| 51 | case AMDGPU::G_AMDGPU_RCP_IFLAG: |
| 52 | case AMDGPU::G_AMDGPU_FMIN_LEGACY: |
| 53 | case AMDGPU::G_AMDGPU_FMAX_LEGACY: |
| 54 | return true; |
| 55 | case AMDGPU::G_INTRINSIC: { |
| 56 | Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: MI).getIntrinsicID(); |
| 57 | switch (IntrinsicID) { |
| 58 | case Intrinsic::amdgcn_rcp: |
| 59 | case Intrinsic::amdgcn_rcp_legacy: |
| 60 | case Intrinsic::amdgcn_sin: |
| 61 | case Intrinsic::amdgcn_fmul_legacy: |
| 62 | case Intrinsic::amdgcn_fmed3: |
| 63 | case Intrinsic::amdgcn_fma_legacy: |
| 64 | return true; |
| 65 | default: |
| 66 | return false; |
| 67 | } |
| 68 | } |
| 69 | default: |
| 70 | return false; |
| 71 | } |
| 72 | } |
| 73 | |
| 74 | /// \p returns true if the operation will definitely need to use a 64-bit |
| 75 | /// encoding, and thus will use a VOP3 encoding regardless of the source |
| 76 | /// modifiers. |
| 77 | LLVM_READONLY |
| 78 | static bool opMustUseVOP3Encoding(const MachineInstr &MI, |
| 79 | const MachineRegisterInfo &MRI) { |
| 80 | return MI.getNumOperands() > (isa<GIntrinsic>(Val: MI) ? 4u : 3u) || |
| 81 | MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getScalarSizeInBits() == 64; |
| 82 | } |
| 83 | |
| 84 | // Most FP instructions support source modifiers. |
| 85 | LLVM_READONLY |
| 86 | static bool hasSourceMods(const MachineInstr &MI) { |
| 87 | if (!MI.memoperands().empty()) |
| 88 | return false; |
| 89 | |
| 90 | switch (MI.getOpcode()) { |
| 91 | case AMDGPU::COPY: |
| 92 | case AMDGPU::G_SELECT: |
| 93 | case AMDGPU::G_FDIV: |
| 94 | case AMDGPU::G_FREM: |
| 95 | case TargetOpcode::INLINEASM: |
| 96 | case TargetOpcode::INLINEASM_BR: |
| 97 | case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: |
| 98 | case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: |
| 99 | case AMDGPU::G_BITCAST: |
| 100 | case AMDGPU::G_ANYEXT: |
| 101 | case AMDGPU::G_BUILD_VECTOR: |
| 102 | case AMDGPU::G_BUILD_VECTOR_TRUNC: |
| 103 | case AMDGPU::G_PHI: |
| 104 | return false; |
| 105 | case AMDGPU::G_INTRINSIC: |
| 106 | case AMDGPU::G_INTRINSIC_CONVERGENT: { |
| 107 | Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: MI).getIntrinsicID(); |
| 108 | switch (IntrinsicID) { |
| 109 | case Intrinsic::amdgcn_interp_p1: |
| 110 | case Intrinsic::amdgcn_interp_p2: |
| 111 | case Intrinsic::amdgcn_interp_mov: |
| 112 | case Intrinsic::amdgcn_interp_p1_f16: |
| 113 | case Intrinsic::amdgcn_interp_p2_f16: |
| 114 | case Intrinsic::amdgcn_div_scale: |
| 115 | return false; |
| 116 | default: |
| 117 | return true; |
| 118 | } |
| 119 | } |
| 120 | default: |
| 121 | return true; |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | static bool allUsesHaveSourceMods(MachineInstr &MI, MachineRegisterInfo &MRI, |
| 126 | unsigned CostThreshold = 4) { |
| 127 | // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus |
| 128 | // it is truly free to use a source modifier in all cases. If there are |
| 129 | // multiple users but for each one will necessitate using VOP3, there will be |
| 130 | // a code size increase. Try to avoid increasing code size unless we know it |
| 131 | // will save on the instruction count. |
| 132 | unsigned NumMayIncreaseSize = 0; |
| 133 | Register Dst = MI.getOperand(i: 0).getReg(); |
| 134 | for (const MachineInstr &Use : MRI.use_nodbg_instructions(Reg: Dst)) { |
| 135 | if (!hasSourceMods(MI: Use)) |
| 136 | return false; |
| 137 | |
| 138 | if (!opMustUseVOP3Encoding(MI: Use, MRI)) { |
| 139 | if (++NumMayIncreaseSize > CostThreshold) |
| 140 | return false; |
| 141 | } |
| 142 | } |
| 143 | return true; |
| 144 | } |
| 145 | |
| 146 | static bool mayIgnoreSignedZero(const MachineInstr &MI) { |
| 147 | return MI.getFlag(Flag: MachineInstr::MIFlag::FmNsz); |
| 148 | } |
| 149 | |
| 150 | static bool isInv2Pi(const APFloat &APF) { |
| 151 | static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118)); |
| 152 | static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983)); |
| 153 | static const APFloat KF64(APFloat::IEEEdouble(), |
| 154 | APInt(64, 0x3fc45f306dc9c882)); |
| 155 | |
| 156 | return APF.bitwiseIsEqual(RHS: KF16) || APF.bitwiseIsEqual(RHS: KF32) || |
| 157 | APF.bitwiseIsEqual(RHS: KF64); |
| 158 | } |
| 159 | |
| 160 | // 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an |
| 161 | // additional cost to negate them. |
| 162 | static bool isConstantCostlierToNegate(MachineInstr &MI, Register Reg, |
| 163 | MachineRegisterInfo &MRI) { |
| 164 | std::optional<FPValueAndVReg> FPValReg; |
| 165 | if (mi_match(R: Reg, MRI, P: m_GFCstOrSplat(FPValReg))) { |
| 166 | if (FPValReg->Value.isZero() && !FPValReg->Value.isNegative()) |
| 167 | return true; |
| 168 | |
| 169 | const GCNSubtarget &ST = MI.getMF()->getSubtarget<GCNSubtarget>(); |
| 170 | if (ST.hasInv2PiInlineImm() && isInv2Pi(APF: FPValReg->Value)) |
| 171 | return true; |
| 172 | } |
| 173 | return false; |
| 174 | } |
| 175 | |
| 176 | bool AMDGPUCombinerHelper::canIgnoreLegacyMinMaxTies(const MachineInstr &MI, |
| 177 | Register LHS, |
| 178 | Register RHS) const { |
| 179 | if (mayIgnoreSignedZero(MI)) |
| 180 | return true; |
| 181 | return VT && |
| 182 | (VT->isKnownNeverLogicalZero(Val: LHS) || VT->isKnownNeverLogicalZero(Val: RHS)); |
| 183 | } |
| 184 | |
| 185 | static unsigned inverseMinMax(unsigned Opc) { |
| 186 | switch (Opc) { |
| 187 | case AMDGPU::G_FMAXNUM: |
| 188 | return AMDGPU::G_FMINNUM; |
| 189 | case AMDGPU::G_FMINNUM: |
| 190 | return AMDGPU::G_FMAXNUM; |
| 191 | case AMDGPU::G_FMAXNUM_IEEE: |
| 192 | return AMDGPU::G_FMINNUM_IEEE; |
| 193 | case AMDGPU::G_FMINNUM_IEEE: |
| 194 | return AMDGPU::G_FMAXNUM_IEEE; |
| 195 | case AMDGPU::G_FMAXIMUM: |
| 196 | return AMDGPU::G_FMINIMUM; |
| 197 | case AMDGPU::G_FMINIMUM: |
| 198 | return AMDGPU::G_FMAXIMUM; |
| 199 | case AMDGPU::G_AMDGPU_FMAX_LEGACY: |
| 200 | return AMDGPU::G_AMDGPU_FMIN_LEGACY; |
| 201 | case AMDGPU::G_AMDGPU_FMIN_LEGACY: |
| 202 | return AMDGPU::G_AMDGPU_FMAX_LEGACY; |
| 203 | default: |
| 204 | llvm_unreachable("invalid min/max opcode" ); |
| 205 | } |
| 206 | } |
| 207 | |
| 208 | bool AMDGPUCombinerHelper::matchFoldableFneg(MachineInstr &MI, |
| 209 | MachineInstr *&MatchInfo) const { |
| 210 | Register Src = MI.getOperand(i: 1).getReg(); |
| 211 | MatchInfo = MRI.getVRegDef(Reg: Src); |
| 212 | |
| 213 | // If the input has multiple uses and we can either fold the negate down, or |
| 214 | // the other uses cannot, give up. This both prevents unprofitable |
| 215 | // transformations and infinite loops: we won't repeatedly try to fold around |
| 216 | // a negate that has no 'good' form. |
| 217 | if (MRI.hasOneNonDBGUse(RegNo: Src)) { |
| 218 | if (allUsesHaveSourceMods(MI, MRI, CostThreshold: 0)) |
| 219 | return false; |
| 220 | } else { |
| 221 | if (fnegFoldsIntoMI(MI: *MatchInfo) && |
| 222 | (allUsesHaveSourceMods(MI, MRI) || |
| 223 | !allUsesHaveSourceMods(MI&: *MatchInfo, MRI))) |
| 224 | return false; |
| 225 | } |
| 226 | |
| 227 | switch (MatchInfo->getOpcode()) { |
| 228 | case AMDGPU::G_AMDGPU_FMIN_LEGACY: |
| 229 | case AMDGPU::G_AMDGPU_FMAX_LEGACY: |
| 230 | if (isConstantCostlierToNegate(MI&: *MatchInfo, |
| 231 | Reg: MatchInfo->getOperand(i: 2).getReg(), MRI)) |
| 232 | return false; |
| 233 | // Swapping min<->max flips which operand a signed zero tie selects. |
| 234 | return canIgnoreLegacyMinMaxTies(MI: *MatchInfo, |
| 235 | LHS: MatchInfo->getOperand(i: 1).getReg(), |
| 236 | RHS: MatchInfo->getOperand(i: 2).getReg()); |
| 237 | case AMDGPU::G_FMINNUM: |
| 238 | case AMDGPU::G_FMAXNUM: |
| 239 | case AMDGPU::G_FMINNUM_IEEE: |
| 240 | case AMDGPU::G_FMAXNUM_IEEE: |
| 241 | case AMDGPU::G_FMINIMUM: |
| 242 | case AMDGPU::G_FMAXIMUM: |
| 243 | // 0 doesn't have a negated inline immediate. |
| 244 | return !isConstantCostlierToNegate(MI&: *MatchInfo, |
| 245 | Reg: MatchInfo->getOperand(i: 2).getReg(), MRI); |
| 246 | case AMDGPU::G_FADD: |
| 247 | case AMDGPU::G_FSUB: |
| 248 | case AMDGPU::G_FMA: |
| 249 | case AMDGPU::G_FMAD: |
| 250 | return mayIgnoreSignedZero(MI: *MatchInfo); |
| 251 | case AMDGPU::G_FMUL: |
| 252 | case AMDGPU::G_FPEXT: |
| 253 | case AMDGPU::G_INTRINSIC_TRUNC: |
| 254 | case AMDGPU::G_FPTRUNC: |
| 255 | case AMDGPU::G_FRINT: |
| 256 | case AMDGPU::G_FNEARBYINT: |
| 257 | case AMDGPU::G_INTRINSIC_ROUND: |
| 258 | case AMDGPU::G_INTRINSIC_ROUNDEVEN: |
| 259 | case AMDGPU::G_FSIN: |
| 260 | case AMDGPU::G_FCANONICALIZE: |
| 261 | case AMDGPU::G_AMDGPU_RCP_IFLAG: |
| 262 | return true; |
| 263 | case AMDGPU::G_INTRINSIC: |
| 264 | case AMDGPU::G_INTRINSIC_CONVERGENT: { |
| 265 | Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: MatchInfo)->getIntrinsicID(); |
| 266 | switch (IntrinsicID) { |
| 267 | case Intrinsic::amdgcn_rcp: |
| 268 | case Intrinsic::amdgcn_rcp_legacy: |
| 269 | case Intrinsic::amdgcn_sin: |
| 270 | case Intrinsic::amdgcn_fmul_legacy: |
| 271 | case Intrinsic::amdgcn_fmed3: |
| 272 | return true; |
| 273 | case Intrinsic::amdgcn_fma_legacy: |
| 274 | return mayIgnoreSignedZero(MI: *MatchInfo); |
| 275 | default: |
| 276 | return false; |
| 277 | } |
| 278 | } |
| 279 | default: |
| 280 | return false; |
| 281 | } |
| 282 | } |
| 283 | |
| 284 | void AMDGPUCombinerHelper::applyFoldableFneg(MachineInstr &MI, |
| 285 | MachineInstr *&MatchInfo) const { |
| 286 | // Transform: |
| 287 | // %A = inst %Op1, ... |
| 288 | // %B = fneg %A |
| 289 | // |
| 290 | // into: |
| 291 | // |
| 292 | // (if %A has one use, specifically fneg above) |
| 293 | // %B = inst (maybe fneg %Op1), ... |
| 294 | // |
| 295 | // (if %A has multiple uses) |
| 296 | // %B = inst (maybe fneg %Op1), ... |
| 297 | // %A = fneg %B |
| 298 | |
| 299 | // Replace register in operand with a register holding negated value. |
| 300 | auto NegateOperand = [&](MachineOperand &Op) { |
| 301 | Register Reg = Op.getReg(); |
| 302 | if (!mi_match(R: Reg, MRI, P: m_GFNeg(Src: m_Reg(R&: Reg)))) |
| 303 | Reg = Builder.buildFNeg(Dst: MRI.getType(Reg), Src0: Reg).getReg(Idx: 0); |
| 304 | replaceRegOpWith(MRI, FromRegOp&: Op, ToReg: Reg); |
| 305 | }; |
| 306 | |
| 307 | // Replace either register in operands with a register holding negated value. |
| 308 | auto NegateEitherOperand = [&](MachineOperand &X, MachineOperand &Y) { |
| 309 | Register XReg = X.getReg(); |
| 310 | Register YReg = Y.getReg(); |
| 311 | if (mi_match(R: XReg, MRI, P: m_GFNeg(Src: m_Reg(R&: XReg)))) |
| 312 | replaceRegOpWith(MRI, FromRegOp&: X, ToReg: XReg); |
| 313 | else if (mi_match(R: YReg, MRI, P: m_GFNeg(Src: m_Reg(R&: YReg)))) |
| 314 | replaceRegOpWith(MRI, FromRegOp&: Y, ToReg: YReg); |
| 315 | else { |
| 316 | YReg = Builder.buildFNeg(Dst: MRI.getType(Reg: YReg), Src0: YReg).getReg(Idx: 0); |
| 317 | replaceRegOpWith(MRI, FromRegOp&: Y, ToReg: YReg); |
| 318 | } |
| 319 | }; |
| 320 | |
| 321 | Builder.setInstrAndDebugLoc(*MatchInfo); |
| 322 | |
| 323 | // Negate appropriate operands so that resulting value of MatchInfo is |
| 324 | // negated. |
| 325 | switch (MatchInfo->getOpcode()) { |
| 326 | case AMDGPU::G_FADD: |
| 327 | case AMDGPU::G_FSUB: |
| 328 | NegateOperand(MatchInfo->getOperand(i: 1)); |
| 329 | NegateOperand(MatchInfo->getOperand(i: 2)); |
| 330 | break; |
| 331 | case AMDGPU::G_FMUL: |
| 332 | NegateEitherOperand(MatchInfo->getOperand(i: 1), MatchInfo->getOperand(i: 2)); |
| 333 | break; |
| 334 | case AMDGPU::G_FMINNUM: |
| 335 | case AMDGPU::G_FMAXNUM: |
| 336 | case AMDGPU::G_FMINNUM_IEEE: |
| 337 | case AMDGPU::G_FMAXNUM_IEEE: |
| 338 | case AMDGPU::G_FMINIMUM: |
| 339 | case AMDGPU::G_FMAXIMUM: |
| 340 | case AMDGPU::G_AMDGPU_FMIN_LEGACY: |
| 341 | case AMDGPU::G_AMDGPU_FMAX_LEGACY: { |
| 342 | NegateOperand(MatchInfo->getOperand(i: 1)); |
| 343 | NegateOperand(MatchInfo->getOperand(i: 2)); |
| 344 | unsigned Opposite = inverseMinMax(Opc: MatchInfo->getOpcode()); |
| 345 | replaceOpcodeWith(FromMI&: *MatchInfo, ToOpcode: Opposite); |
| 346 | break; |
| 347 | } |
| 348 | case AMDGPU::G_FMA: |
| 349 | case AMDGPU::G_FMAD: |
| 350 | NegateEitherOperand(MatchInfo->getOperand(i: 1), MatchInfo->getOperand(i: 2)); |
| 351 | NegateOperand(MatchInfo->getOperand(i: 3)); |
| 352 | break; |
| 353 | case AMDGPU::G_FPEXT: |
| 354 | case AMDGPU::G_INTRINSIC_TRUNC: |
| 355 | case AMDGPU::G_FRINT: |
| 356 | case AMDGPU::G_FNEARBYINT: |
| 357 | case AMDGPU::G_INTRINSIC_ROUND: |
| 358 | case AMDGPU::G_INTRINSIC_ROUNDEVEN: |
| 359 | case AMDGPU::G_FSIN: |
| 360 | case AMDGPU::G_FCANONICALIZE: |
| 361 | case AMDGPU::G_AMDGPU_RCP_IFLAG: |
| 362 | case AMDGPU::G_FPTRUNC: |
| 363 | NegateOperand(MatchInfo->getOperand(i: 1)); |
| 364 | break; |
| 365 | case AMDGPU::G_INTRINSIC: |
| 366 | case AMDGPU::G_INTRINSIC_CONVERGENT: { |
| 367 | Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: MatchInfo)->getIntrinsicID(); |
| 368 | switch (IntrinsicID) { |
| 369 | case Intrinsic::amdgcn_rcp: |
| 370 | case Intrinsic::amdgcn_rcp_legacy: |
| 371 | case Intrinsic::amdgcn_sin: |
| 372 | NegateOperand(MatchInfo->getOperand(i: 2)); |
| 373 | break; |
| 374 | case Intrinsic::amdgcn_fmul_legacy: |
| 375 | NegateEitherOperand(MatchInfo->getOperand(i: 2), MatchInfo->getOperand(i: 3)); |
| 376 | break; |
| 377 | case Intrinsic::amdgcn_fmed3: |
| 378 | NegateOperand(MatchInfo->getOperand(i: 2)); |
| 379 | NegateOperand(MatchInfo->getOperand(i: 3)); |
| 380 | NegateOperand(MatchInfo->getOperand(i: 4)); |
| 381 | break; |
| 382 | case Intrinsic::amdgcn_fma_legacy: |
| 383 | NegateEitherOperand(MatchInfo->getOperand(i: 2), MatchInfo->getOperand(i: 3)); |
| 384 | NegateOperand(MatchInfo->getOperand(i: 4)); |
| 385 | break; |
| 386 | default: |
| 387 | llvm_unreachable("folding fneg not supported for this intrinsic" ); |
| 388 | } |
| 389 | break; |
| 390 | } |
| 391 | default: |
| 392 | llvm_unreachable("folding fneg not supported for this instruction" ); |
| 393 | } |
| 394 | |
| 395 | Register Dst = MI.getOperand(i: 0).getReg(); |
| 396 | Register MatchInfoDst = MatchInfo->getOperand(i: 0).getReg(); |
| 397 | |
| 398 | if (MRI.hasOneNonDBGUse(RegNo: MatchInfoDst)) { |
| 399 | // MatchInfo now has negated value so use that instead of old Dst. |
| 400 | replaceRegWith(MRI, FromReg: Dst, ToReg: MatchInfoDst); |
| 401 | } else { |
| 402 | // We want to swap all uses of Dst with uses of MatchInfoDst and vice versa |
| 403 | // but replaceRegWith will replace defs as well. It is easier to replace one |
| 404 | // def with a new register. |
| 405 | LLT Type = MRI.getType(Reg: Dst); |
| 406 | Register NegatedMatchInfo = MRI.createGenericVirtualRegister(Ty: Type); |
| 407 | replaceRegOpWith(MRI, FromRegOp&: MatchInfo->getOperand(i: 0), ToReg: NegatedMatchInfo); |
| 408 | |
| 409 | // MatchInfo now has negated value so use that instead of old Dst. |
| 410 | replaceRegWith(MRI, FromReg: Dst, ToReg: NegatedMatchInfo); |
| 411 | |
| 412 | // Recreate non negated value for other uses of old MatchInfoDst |
| 413 | auto NextInst = ++MatchInfo->getIterator(); |
| 414 | Builder.setInstrAndDebugLoc(*NextInst); |
| 415 | Builder.buildFNeg(Dst: MatchInfoDst, Src0: NegatedMatchInfo, Flags: MI.getFlags()); |
| 416 | } |
| 417 | |
| 418 | MI.eraseFromParent(); |
| 419 | } |
| 420 | |
| 421 | bool AMDGPUCombinerHelper::matchFoldFAbsFptrunc(MachineInstr &Fabs, |
| 422 | MachineInstr &Fptrunc) const { |
| 423 | Register Round = Fptrunc.getOperand(i: 0).getReg(); |
| 424 | if (!MRI.hasOneNonDBGUse(RegNo: Round)) |
| 425 | return false; |
| 426 | |
| 427 | LLT SrcTy = MRI.getType(Reg: Fptrunc.getOperand(i: 1).getReg()); |
| 428 | return isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_FABS, {SrcTy}}); |
| 429 | } |
| 430 | |
| 431 | void AMDGPUCombinerHelper::applyFoldFAbsFptrunc(MachineInstr &Fabs, |
| 432 | MachineInstr &Fptrunc) const { |
| 433 | // fabs (fptrunc x) -> fptrunc (fabs x) |
| 434 | Register Dst = Fabs.getOperand(i: 0).getReg(); |
| 435 | Register Src = Fptrunc.getOperand(i: 1).getReg(); |
| 436 | Builder.setInstrAndDebugLoc(Fabs); |
| 437 | Register Abs = |
| 438 | Builder.buildFAbs(Dst: MRI.getType(Reg: Src), Src0: Src, Flags: Fabs.getFlags()).getReg(Idx: 0); |
| 439 | Builder.buildFPTrunc(Res: Dst, Op: Abs, Flags: Fptrunc.getFlags()); |
| 440 | Fabs.eraseFromParent(); |
| 441 | } |
| 442 | |
| 443 | // TODO: Should return converted value / extension source and avoid introducing |
| 444 | // intermediate fptruncs in the apply function. |
| 445 | static bool isFPExtFromF16OrConst(const MachineRegisterInfo &MRI, |
| 446 | Register Reg) { |
| 447 | Register SrcReg; |
| 448 | if (mi_match(R: Reg, MRI, P: m_GFPExt(Src: m_Reg(R&: SrcReg)))) |
| 449 | return MRI.getType(Reg: SrcReg) == LLT::float16(); |
| 450 | |
| 451 | const ConstantFP *FPImm; |
| 452 | if (mi_match(R: Reg, MRI, P: m_GFCst(C&: FPImm))) { |
| 453 | APFloat Val = FPImm->getValueAPF(); |
| 454 | bool LosesInfo = true; |
| 455 | Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo); |
| 456 | return !LosesInfo; |
| 457 | } |
| 458 | |
| 459 | return false; |
| 460 | } |
| 461 | |
| 462 | bool AMDGPUCombinerHelper::matchExpandPromotedF16FMed3(MachineInstr &MI, |
| 463 | Register Src0, |
| 464 | Register Src1, |
| 465 | Register Src2) const { |
| 466 | assert(MI.getOpcode() == TargetOpcode::G_FPTRUNC); |
| 467 | Register SrcReg = MI.getOperand(i: 1).getReg(); |
| 468 | if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()) != LLT::float16()) |
| 469 | return false; |
| 470 | if (!MRI.hasOneNonDBGUse(RegNo: SrcReg) || MRI.getType(Reg: SrcReg) != LLT::float32()) |
| 471 | return false; |
| 472 | |
| 473 | return isFPExtFromF16OrConst(MRI, Reg: Src0) && isFPExtFromF16OrConst(MRI, Reg: Src1) && |
| 474 | isFPExtFromF16OrConst(MRI, Reg: Src2); |
| 475 | } |
| 476 | |
| 477 | void AMDGPUCombinerHelper::applyExpandPromotedF16FMed3(MachineInstr &MI, |
| 478 | Register Src0, |
| 479 | Register Src1, |
| 480 | Register Src2) const { |
| 481 | // We expect fptrunc (fpext x) to fold out, and to constant fold any constant |
| 482 | // sources. |
| 483 | Src0 = Builder.buildFPTrunc(Res: LLT::float16(), Op: Src0).getReg(Idx: 0); |
| 484 | Src1 = Builder.buildFPTrunc(Res: LLT::float16(), Op: Src1).getReg(Idx: 0); |
| 485 | Src2 = Builder.buildFPTrunc(Res: LLT::float16(), Op: Src2).getReg(Idx: 0); |
| 486 | |
| 487 | LLT Ty = MRI.getType(Reg: Src0); |
| 488 | auto A1 = Builder.buildFMinNumIEEE(Dst: Ty, Src0, Src1); |
| 489 | auto B1 = Builder.buildFMaxNumIEEE(Dst: Ty, Src0, Src1); |
| 490 | auto C1 = Builder.buildFMaxNumIEEE(Dst: Ty, Src0: A1, Src1: Src2); |
| 491 | Builder.buildFMinNumIEEE(Dst: MI.getOperand(i: 0), Src0: B1, Src1: C1); |
| 492 | MI.eraseFromParent(); |
| 493 | } |
| 494 | |
| 495 | bool AMDGPUCombinerHelper::matchCombineFmulWithSelectToFldexp( |
| 496 | MachineInstr &MI, MachineInstr &Sel, |
| 497 | std::function<void(MachineIRBuilder &)> &MatchInfo) const { |
| 498 | assert(MI.getOpcode() == TargetOpcode::G_FMUL); |
| 499 | assert(Sel.getOpcode() == TargetOpcode::G_SELECT); |
| 500 | assert(MI.getOperand(2).getReg() == Sel.getOperand(0).getReg()); |
| 501 | |
| 502 | Register Dst = MI.getOperand(i: 0).getReg(); |
| 503 | LLT DestTy = MRI.getType(Reg: Dst); |
| 504 | LLT ScalarDestTy = DestTy.getScalarType(); |
| 505 | |
| 506 | // TODO: Expected float type in ScalarDestTy |
| 507 | if ((ScalarDestTy != LLT::float64() && ScalarDestTy != LLT::float32() && |
| 508 | ScalarDestTy != LLT::float16()) || |
| 509 | !MRI.hasOneNonDBGUse(RegNo: Sel.getOperand(i: 0).getReg())) |
| 510 | return false; |
| 511 | |
| 512 | Register SelectCondReg = Sel.getOperand(i: 1).getReg(); |
| 513 | Register SelectTrueReg = Sel.getOperand(i: 2).getReg(); |
| 514 | Register SelectFalseReg = Sel.getOperand(i: 3).getReg(); |
| 515 | |
| 516 | const auto SelectTrueVal = |
| 517 | isConstantOrConstantSplatVectorFP(Def: SelectTrueReg, MRI); |
| 518 | if (!SelectTrueVal) |
| 519 | return false; |
| 520 | const auto SelectFalseVal = |
| 521 | isConstantOrConstantSplatVectorFP(Def: SelectFalseReg, MRI); |
| 522 | if (!SelectFalseVal) |
| 523 | return false; |
| 524 | |
| 525 | if (SelectTrueVal->isNegative() != SelectFalseVal->isNegative()) |
| 526 | return false; |
| 527 | |
| 528 | // For f32, only non-inline constants should be transformed. |
| 529 | // TODO: Expected float32 |
| 530 | if (ScalarDestTy == LLT::float32() && TII.isInlineConstant(Imm: *SelectTrueVal) && |
| 531 | TII.isInlineConstant(Imm: *SelectFalseVal)) |
| 532 | return false; |
| 533 | |
| 534 | int SelectTrueLog2Val = SelectTrueVal->getExactLog2Abs(); |
| 535 | if (SelectTrueLog2Val == INT_MIN) |
| 536 | return false; |
| 537 | int SelectFalseLog2Val = SelectFalseVal->getExactLog2Abs(); |
| 538 | if (SelectFalseLog2Val == INT_MIN) |
| 539 | return false; |
| 540 | |
| 541 | MatchInfo = [=, &MI](MachineIRBuilder &Builder) { |
| 542 | LLT IntDestTy = DestTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 32)); |
| 543 | auto NewSel = Builder.buildSelect( |
| 544 | Res: IntDestTy, Tst: SelectCondReg, |
| 545 | Op0: Builder.buildConstant(Res: IntDestTy, Val: SelectTrueLog2Val), |
| 546 | Op1: Builder.buildConstant(Res: IntDestTy, Val: SelectFalseLog2Val)); |
| 547 | |
| 548 | Register XReg = MI.getOperand(i: 1).getReg(); |
| 549 | if (SelectTrueVal->isNegative()) { |
| 550 | auto NegX = |
| 551 | Builder.buildFNeg(Dst: DestTy, Src0: XReg, Flags: MRI.getVRegDef(Reg: XReg)->getFlags()); |
| 552 | Builder.buildFLdexp(Dst, Src0: NegX, Src1: NewSel, Flags: MI.getFlags()); |
| 553 | } else { |
| 554 | Builder.buildFLdexp(Dst, Src0: XReg, Src1: NewSel, Flags: MI.getFlags()); |
| 555 | } |
| 556 | }; |
| 557 | |
| 558 | return true; |
| 559 | } |
| 560 | |
| 561 | bool AMDGPUCombinerHelper::matchConstantIs32BitMask(Register Reg) const { |
| 562 | auto Res = getIConstantVRegValWithLookThrough(VReg: Reg, MRI); |
| 563 | if (!Res) |
| 564 | return false; |
| 565 | |
| 566 | const uint64_t Val = Res->Value.getZExtValue(); |
| 567 | unsigned MaskIdx = 0; |
| 568 | unsigned MaskLen = 0; |
| 569 | if (!isShiftedMask_64(Value: Val, MaskIdx, MaskLen)) |
| 570 | return false; |
| 571 | |
| 572 | // Check if low 32 bits or high 32 bits are all ones. |
| 573 | return MaskLen >= 32 && ((MaskIdx == 0) || (MaskIdx == 64 - MaskLen)); |
| 574 | } |
| 575 | |
| 576 | bool AMDGPUCombinerHelper::matchBinopWith32BitMask(Register X, |
| 577 | Register Y) const { |
| 578 | // If both operands are constants, constant_fold_binop can fold the whole |
| 579 | // binop to a single constant, which is strictly better than narrowing it |
| 580 | // into two s32 ops. Bail so the fold is not preempted. |
| 581 | if (getIConstantVRegValWithLookThrough(VReg: X, MRI) && |
| 582 | getIConstantVRegValWithLookThrough(VReg: Y, MRI)) |
| 583 | return false; |
| 584 | return matchConstantIs32BitMask(Reg: X) || matchConstantIs32BitMask(Reg: Y); |
| 585 | } |
| 586 | |
| 587 | bool AMDGPUCombinerHelper::isCarryLikeBool(Register Reg, unsigned Depth) const { |
| 588 | constexpr unsigned MaxDepth = 6; |
| 589 | const MachineInstr *Def = MRI.getVRegDef(Reg); |
| 590 | if (!Def || Depth > MaxDepth) |
| 591 | return false; |
| 592 | |
| 593 | switch (Def->getOpcode()) { |
| 594 | case TargetOpcode::G_ICMP: |
| 595 | case TargetOpcode::G_FCMP: |
| 596 | case TargetOpcode::G_IS_FPCLASS: |
| 597 | return true; |
| 598 | case TargetOpcode::G_INTRINSIC: |
| 599 | return cast<GIntrinsic>(Val: Def)->is(ID: Intrinsic::amdgcn_class); |
| 600 | case TargetOpcode::G_UADDO: |
| 601 | case TargetOpcode::G_UADDE: |
| 602 | case TargetOpcode::G_USUBO: |
| 603 | case TargetOpcode::G_USUBE: |
| 604 | return Def->getOperand(i: 1).getReg() == Reg; |
| 605 | case TargetOpcode::G_XOR: |
| 606 | // A not of a carry-like boolean. |
| 607 | if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI, P: m_AllOnesInt())) |
| 608 | return isCarryLikeBool(Reg: Def->getOperand(i: 1).getReg(), Depth: Depth + 1); |
| 609 | [[fallthrough]]; |
| 610 | case TargetOpcode::G_AND: |
| 611 | case TargetOpcode::G_OR: |
| 612 | return isCarryLikeBool(Reg: Def->getOperand(i: 1).getReg(), Depth: Depth + 1) && |
| 613 | isCarryLikeBool(Reg: Def->getOperand(i: 2).getReg(), Depth: Depth + 1); |
| 614 | default: |
| 615 | return false; |
| 616 | } |
| 617 | } |
| 618 | |
| 619 | static bool isBoolType(LLT Ty) { return Ty.isScalar(Size: 1); } |
| 620 | |
| 621 | bool AMDGPUCombinerHelper::isFoldableCarryExt(Register Reg, Register *Carry, |
| 622 | bool *IsSExt) const { |
| 623 | Register Src; |
| 624 | bool SExt; |
| 625 | // The high bits of an anyext are undefined, so it can be treated as a zext. |
| 626 | if (mi_match(R: Reg, MRI, P: m_any_of(preds: m_GZExt(Src: m_Reg(R&: Src)), preds: m_GAnyExt(Src: m_Reg(R&: Src))))) |
| 627 | SExt = false; |
| 628 | else if (mi_match(R: Reg, MRI, P: m_GSExt(Src: m_Reg(R&: Src)))) |
| 629 | SExt = true; |
| 630 | else |
| 631 | return false; |
| 632 | |
| 633 | // Any other boolean must first be copied to SCC or VCC, which costs about |
| 634 | // as much as the extension. |
| 635 | if (!isBoolType(Ty: MRI.getType(Reg: Src)) || !MRI.hasOneNonDBGUse(RegNo: Reg) || |
| 636 | !isCarryLikeBool(Reg: Src)) |
| 637 | return false; |
| 638 | if (Carry) |
| 639 | *Carry = Src; |
| 640 | if (IsSExt) |
| 641 | *IsSExt = SExt; |
| 642 | return true; |
| 643 | } |
| 644 | |
| 645 | bool AMDGPUCombinerHelper::matchConditionalCarryOp( |
| 646 | MachineInstr &MI, ConditionalCarryOp &MatchInfo) const { |
| 647 | LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg()); |
| 648 | if (!Ty.isScalar() || Ty.isFloat()) |
| 649 | return false; |
| 650 | unsigned Size = Ty.getSizeInBits(); |
| 651 | // Most 16/32-bit cases come from div/rem legalization, and the legalizer |
| 652 | // splits a 64-bit add on most subtargets, hiding the x +/- 1 operand. |
| 653 | if (isPreLegalize() ? Size != 64 : (Size != 16 && Size != 32)) |
| 654 | return false; |
| 655 | |
| 656 | // A uniform 64-bit add or select does not clobber SCC on a subtarget with |
| 657 | // 64-bit scalar adds, but the two carry ops that replace it do, so a |
| 658 | // condition with other users would have to be saved and restored. |
| 659 | // Uniformity is not known before legalization, so a condition with other |
| 660 | // users is rejected for all 64-bit types on such a subtarget. |
| 661 | bool NeedsSingleUseCarry = Size == 64 && STI.hasScalarAddSub64(); |
| 662 | |
| 663 | // The base is folded into the add/sub-with-carry, so a known zero base is |
| 664 | // better left to the identity combines. |
| 665 | auto IsZero = [&](Register Reg) { |
| 666 | return mi_match(R: Reg, MRI, P: m_SpecificICst(RequestedValue: 0)); |
| 667 | }; |
| 668 | |
| 669 | // Ext is a zext or sext of a boolean that is used as an addend of an add |
| 670 | // (IsAddRoot) or as the subtrahend of a sub. |
| 671 | auto MatchExt = [&](Register Ext, Register Base, bool IsAddRoot) { |
| 672 | Register Carry; |
| 673 | bool IsSExt; |
| 674 | if (IsZero(Base) || !isFoldableCarryExt(Reg: Ext, Carry: &Carry, IsSExt: &IsSExt) || |
| 675 | (NeedsSingleUseCarry && !MRI.hasOneNonDBGUse(RegNo: Carry))) |
| 676 | return false; |
| 677 | MatchInfo = {.IsAdd: IsAddRoot != IsSExt, /*K=*/0, .Base: Base, .Carry: Carry}; |
| 678 | return true; |
| 679 | }; |
| 680 | |
| 681 | switch (MI.getOpcode()) { |
| 682 | case TargetOpcode::G_ADD: { |
| 683 | Register LHS = MI.getOperand(i: 1).getReg(); |
| 684 | Register RHS = MI.getOperand(i: 2).getReg(); |
| 685 | // When both operands are carries, fold only the right one, so that the |
| 686 | // two carries do not have to be live at the same time. |
| 687 | return MatchExt(RHS, LHS, /*IsAddRoot=*/true) || |
| 688 | MatchExt(LHS, RHS, /*IsAddRoot=*/true); |
| 689 | } |
| 690 | case TargetOpcode::G_SUB: |
| 691 | return MatchExt(MI.getOperand(i: 2).getReg(), MI.getOperand(i: 1).getReg(), |
| 692 | /*IsAddRoot=*/false); |
| 693 | case TargetOpcode::G_SELECT: { |
| 694 | Register Cond = MI.getOperand(i: 1).getReg(); |
| 695 | Register TrueReg = MI.getOperand(i: 2).getReg(); |
| 696 | Register FalseReg = MI.getOperand(i: 3).getReg(); |
| 697 | if (!isBoolType(Ty: MRI.getType(Reg: Cond)) || |
| 698 | (NeedsSingleUseCarry && !MRI.hasOneNonDBGUse(RegNo: Cond))) |
| 699 | return false; |
| 700 | |
| 701 | // The select already needs its condition in SCC or VCC, which is the |
| 702 | // form that the carry-in operand needs, so any condition is accepted. |
| 703 | // Unlike in SelectionDAG, a uniform condition with other users is |
| 704 | // already saved in an SGPR and copied back to SCC for each user, so a |
| 705 | // 16-bit or 32-bit carry op does not add a save and restore. |
| 706 | auto MatchStep = [&](Register Step, Register Base, bool StepIfTrue) { |
| 707 | if (!MRI.hasOneNonDBGUse(RegNo: Step)) |
| 708 | return false; |
| 709 | int64_t Imm; |
| 710 | Register Narrow; |
| 711 | if (mi_match(R: Step, MRI, P: m_GAnyExt(Src: m_Reg(R&: Narrow)))) { |
| 712 | // A narrow select that was widened by the legalizer: |
| 713 | // (select c, (anyext (add (trunc x), Imm)), x) |
| 714 | // The high bits of the anyext are undefined, so x + Imm can replace |
| 715 | // the step. |
| 716 | if (!MRI.hasOneNonDBGUse(RegNo: Narrow) || |
| 717 | !mi_match(R: Narrow, MRI, |
| 718 | P: m_GAdd(L: m_GTrunc(Src: m_SpecificReg(RequestedReg: Base)), R: m_ICst(Cst&: Imm)))) |
| 719 | return false; |
| 720 | } else if (!mi_match(R: Step, MRI, |
| 721 | P: m_GAdd(L: m_SpecificReg(RequestedReg: Base), R: m_ICst(Cst&: Imm)))) { |
| 722 | return false; |
| 723 | } |
| 724 | if (Imm != 1 && Imm != -1) |
| 725 | return false; |
| 726 | // (select c, x + Imm, x) = x + Imm * c |
| 727 | // (select c, x, x + Imm) = x + Imm - Imm * c |
| 728 | if (StepIfTrue) |
| 729 | MatchInfo = {.IsAdd: Imm == 1, /*K=*/0, .Base: Base, .Carry: Cond}; |
| 730 | else |
| 731 | MatchInfo = {.IsAdd: Imm == -1, /*K=*/-1, .Base: Base, .Carry: Cond}; |
| 732 | return true; |
| 733 | }; |
| 734 | return MatchStep(TrueReg, FalseReg, /*StepIfTrue=*/true) || |
| 735 | MatchStep(FalseReg, TrueReg, /*StepIfTrue=*/false); |
| 736 | } |
| 737 | default: |
| 738 | return false; |
| 739 | } |
| 740 | } |
| 741 | |
| 742 | void AMDGPUCombinerHelper::applyConditionalCarryOp( |
| 743 | MachineInstr &MI, const ConditionalCarryOp &MatchInfo) const { |
| 744 | assert((MatchInfo.K == 0 || MatchInfo.K == -1) && |
| 745 | "unexpected conditional carry op constant" ); |
| 746 | Builder.setInstrAndDebugLoc(MI); |
| 747 | Register Dst = MI.getOperand(i: 0).getReg(); |
| 748 | LLT Ty = MRI.getType(Reg: Dst); |
| 749 | LLT Ty32 = Ty.changeElementSize(NewEltSize: 32); |
| 750 | LLT CarryTy = MRI.getType(Reg: MatchInfo.Carry); |
| 751 | unsigned Opc = |
| 752 | MatchInfo.IsAdd ? TargetOpcode::G_UADDE : TargetOpcode::G_USUBE; |
| 753 | Register K = Builder.buildConstant(Res: Ty32, Val: MatchInfo.K).getReg(Idx: 0); |
| 754 | |
| 755 | switch (Ty.getSizeInBits()) { |
| 756 | case 16: { |
| 757 | // Only the low 16 bits of the 32-bit result are used. |
| 758 | auto Base = Builder.buildAnyExt(Res: Ty32, Op: MatchInfo.Base); |
| 759 | auto Op = |
| 760 | Builder.buildInstr(Opc, DstOps: {Ty32, CarryTy}, SrcOps: {Base, K, MatchInfo.Carry}); |
| 761 | Builder.buildTrunc(Res: Dst, Op: Op.getReg(Idx: 0)); |
| 762 | break; |
| 763 | } |
| 764 | case 32: |
| 765 | Builder.buildInstr(Opc, DstOps: {Dst, CarryTy}, |
| 766 | SrcOps: {MatchInfo.Base, K, MatchInfo.Carry}); |
| 767 | break; |
| 768 | case 64: { |
| 769 | // K is 0 or -1, so both halves of the 64-bit constant are K. |
| 770 | auto Base = Builder.buildUnmerge(Res: Ty32, Op: MatchInfo.Base); |
| 771 | auto Lo = Builder.buildInstr(Opc, DstOps: {Ty32, CarryTy}, |
| 772 | SrcOps: {Base.getReg(Idx: 0), K, MatchInfo.Carry}); |
| 773 | auto Hi = Builder.buildInstr(Opc, DstOps: {Ty32, CarryTy}, |
| 774 | SrcOps: {Base.getReg(Idx: 1), K, Lo.getReg(Idx: 1)}); |
| 775 | Builder.buildMergeLikeInstr(Res: Dst, Ops: {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)}); |
| 776 | break; |
| 777 | } |
| 778 | default: |
| 779 | llvm_unreachable("unexpected conditional carry op type" ); |
| 780 | } |
| 781 | MI.eraseFromParent(); |
| 782 | } |
| 783 | |