1//===-- AMDGPUCodeGenPrepare.cpp ------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass does misc. AMDGPU optimizations on IR before instruction
11/// selection.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPU.h"
16#include "AMDGPUMemoryUtils.h"
17#include "AMDGPUTargetMachine.h"
18#include "SIModeRegisterDefaults.h"
19#include "llvm/ADT/SetVector.h"
20#include "llvm/Analysis/AssumptionCache.h"
21#include "llvm/Analysis/ConstantFolding.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/TargetTransformInfo.h"
24#include "llvm/Analysis/UniformityAnalysis.h"
25#include "llvm/Analysis/ValueTracking.h"
26#include "llvm/CodeGen/TargetPassConfig.h"
27#include "llvm/IR/Dominators.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/InstVisitor.h"
30#include "llvm/IR/IntrinsicsAMDGPU.h"
31#include "llvm/IR/PatternMatch.h"
32#include "llvm/IR/ValueHandle.h"
33#include "llvm/InitializePasses.h"
34#include "llvm/Pass.h"
35#include "llvm/Support/KnownBits.h"
36#include "llvm/Support/KnownFPClass.h"
37#include "llvm/Transforms/Utils/BasicBlockUtils.h"
38#include "llvm/Transforms/Utils/IntegerDivision.h"
39#include "llvm/Transforms/Utils/Local.h"
40
41#define DEBUG_TYPE "amdgpu-codegenprepare"
42
43using namespace llvm;
44using namespace llvm::PatternMatch;
45
46namespace {
47
48static cl::opt<bool> WidenLoads(
49 "amdgpu-codegenprepare-widen-constant-loads",
50 cl::desc("Widen sub-dword constant address space loads in AMDGPUCodeGenPrepare"),
51 cl::ReallyHidden,
52 cl::init(Val: false));
53
54static cl::opt<bool>
55 BreakLargePHIs("amdgpu-codegenprepare-break-large-phis",
56 cl::desc("Break large PHI nodes for DAGISel"),
57 cl::ReallyHidden, cl::init(Val: true));
58
59static cl::opt<bool>
60 ForceBreakLargePHIs("amdgpu-codegenprepare-force-break-large-phis",
61 cl::desc("For testing purposes, always break large "
62 "PHIs even if it isn't profitable."),
63 cl::ReallyHidden, cl::init(Val: false));
64
65static cl::opt<unsigned> BreakLargePHIsThreshold(
66 "amdgpu-codegenprepare-break-large-phis-threshold",
67 cl::desc("Minimum type size in bits for breaking large PHI nodes"),
68 cl::ReallyHidden, cl::init(Val: 32));
69
70static cl::opt<bool> UseMul24Intrin(
71 "amdgpu-codegenprepare-mul24",
72 cl::desc("Introduce mul24 intrinsics in AMDGPUCodeGenPrepare"),
73 cl::ReallyHidden,
74 cl::init(Val: true));
75
76// Legalize 64-bit division by using the generic IR expansion.
77static cl::opt<bool> ExpandDiv64InIR(
78 "amdgpu-codegenprepare-expand-div64",
79 cl::desc("Expand 64-bit division in AMDGPUCodeGenPrepare"),
80 cl::ReallyHidden,
81 cl::init(Val: false));
82
83// Leave all division operations as they are. This supersedes ExpandDiv64InIR
84// and is used for testing the legalizer.
85static cl::opt<bool> DisableIDivExpand(
86 "amdgpu-codegenprepare-disable-idiv-expansion",
87 cl::desc("Prevent expanding integer division in AMDGPUCodeGenPrepare"),
88 cl::ReallyHidden,
89 cl::init(Val: false));
90
91// Disable processing of fdiv so we can better test the backend implementations.
92static cl::opt<bool> DisableFDivExpand(
93 "amdgpu-codegenprepare-disable-fdiv-expansion",
94 cl::desc("Prevent expanding floating point division in AMDGPUCodeGenPrepare"),
95 cl::ReallyHidden,
96 cl::init(Val: false));
97
98class AMDGPUCodeGenPrepareImpl
99 : public InstVisitor<AMDGPUCodeGenPrepareImpl, bool> {
100public:
101 Function &F;
102 const GCNSubtarget &ST;
103 const AMDGPUTargetMachine &TM;
104 const TargetTransformInfo &TTI;
105 const TargetLibraryInfo *TLI;
106 const UniformityInfo &UA;
107 const DataLayout &DL;
108 SimplifyQuery SQ;
109 const bool HasFP32DenormalFlush;
110 bool FlowChanged = false;
111 mutable Function *SqrtF32 = nullptr;
112 mutable Function *LdexpF32 = nullptr;
113 mutable SmallVector<WeakVH> DeadVals;
114
115 DenseMap<const PHINode *, bool> BreakPhiNodesCache;
116
117 AMDGPUCodeGenPrepareImpl(Function &F, const AMDGPUTargetMachine &TM,
118 const TargetTransformInfo &TTI,
119 const TargetLibraryInfo *TLI, AssumptionCache *AC,
120 const DominatorTree *DT, const UniformityInfo &UA)
121 : F(F), ST(TM.getSubtarget<GCNSubtarget>(F)), TM(TM), TTI(TTI), TLI(TLI),
122 UA(UA), DL(F.getDataLayout()), SQ(DL, TLI, DT, AC),
123 HasFP32DenormalFlush(SIModeRegisterDefaults(F, ST).FP32Denormals ==
124 DenormalMode::getPreserveSign()) {}
125
126 Function *getSqrtF32() const {
127 if (SqrtF32)
128 return SqrtF32;
129
130 LLVMContext &Ctx = F.getContext();
131 SqrtF32 = Intrinsic::getOrInsertDeclaration(
132 M: F.getParent(), id: Intrinsic::amdgcn_sqrt, OverloadTys: {Type::getFloatTy(C&: Ctx)});
133 return SqrtF32;
134 }
135
136 Function *getLdexpF32() const {
137 if (LdexpF32)
138 return LdexpF32;
139
140 LLVMContext &Ctx = F.getContext();
141 LdexpF32 = Intrinsic::getOrInsertDeclaration(
142 M: F.getParent(), id: Intrinsic::ldexp,
143 OverloadTys: {Type::getFloatTy(C&: Ctx), Type::getInt32Ty(C&: Ctx)});
144 return LdexpF32;
145 }
146
147 bool canBreakPHINode(const PHINode &I);
148
149 /// Return true if \p T is a legal scalar floating point type.
150 bool isLegalFloatingTy(const Type *T) const;
151
152 /// Wrapper to pass all the arguments to computeKnownFPClass
153 KnownFPClass computeKnownFPClass(const Value *V, FPClassTest Interested,
154 const Instruction *CtxI) const {
155 return llvm::computeKnownFPClass(V, InterestedClasses: Interested,
156 SQ: SQ.getWithInstruction(I: CtxI));
157 }
158
159 bool canIgnoreDenormalInput(const Value *V, const Instruction *CtxI) const {
160 return HasFP32DenormalFlush ||
161 computeKnownFPClass(V, Interested: fcSubnormal, CtxI).isKnownNeverSubnormal();
162 }
163
164 /// \returns The minimum number of bits needed to store the value of \Op as an
165 /// unsigned integer. Truncating to this size and then zero-extending to
166 /// the original will not change the value.
167 unsigned numBitsUnsigned(Value *Op, const Instruction *CtxI) const;
168
169 /// \returns The minimum number of bits needed to store the value of \Op as a
170 /// signed integer. Truncating to this size and then sign-extending to
171 /// the original size will not change the value.
172 unsigned numBitsSigned(Value *Op, const Instruction *CtxI) const;
173
174 /// Replace mul instructions with llvm.amdgcn.mul.u24 or llvm.amdgcn.mul.s24.
175 /// SelectionDAG has an issue where an and asserting the bits are known
176 bool replaceMulWithMul24(BinaryOperator &I) const;
177
178 /// Perform same function as equivalently named function in DAGCombiner. Since
179 /// we expand some divisions here, we need to perform this before obscuring.
180 bool foldBinOpIntoSelect(BinaryOperator &I) const;
181
182 bool divHasSpecialOptimization(BinaryOperator &I,
183 Value *Num, Value *Den) const;
184 unsigned getDivNumBits(BinaryOperator &I, Value *Num, Value *Den,
185 unsigned MaxDivBits, bool Signed) const;
186
187 /// Expands div or rem by using floating-point operations.
188 /// Operands must be in the range [-0x400000,0x3FFFFF]
189 Value *expandDivRemToFloat(IRBuilder<> &Builder, BinaryOperator &I,
190 Value *Num, Value *Den, bool IsDiv,
191 bool IsSigned) const;
192
193 Value *expandDivRemToFloatImpl(IRBuilder<> &Builder, BinaryOperator &I,
194 Value *Num, Value *Den, unsigned NumBits,
195 bool IsDiv, bool IsSigned) const;
196
197 /// Expands 32 bit div or rem.
198 Value* expandDivRem32(IRBuilder<> &Builder, BinaryOperator &I,
199 Value *Num, Value *Den) const;
200
201 Value *shrinkDivRem64(IRBuilder<> &Builder, BinaryOperator &I,
202 Value *Num, Value *Den) const;
203 void expandDivRem64(BinaryOperator &I) const;
204
205 /// Widen a scalar load.
206 ///
207 /// \details \p Widen scalar load for uniform, small type loads from constant
208 // memory / to a full 32-bits and then truncate the input to allow a scalar
209 // load instead of a vector load.
210 //
211 /// \returns True.
212
213 bool canWidenScalarExtLoad(LoadInst &I) const;
214
215 Value *matchFractPatImpl(Value &V, const APFloat &C) const;
216 Value *matchFractPatNanAvoidant(Value &V);
217 Value *applyFractPat(IRBuilder<> &Builder, Value *FractArg);
218
219 bool canOptimizeWithRsq(FastMathFlags DivFMF, FastMathFlags SqrtFMF) const;
220
221 Value *optimizeWithRsq(IRBuilder<> &Builder, Value *Num, Value *Den,
222 FastMathFlags DivFMF, FastMathFlags SqrtFMF,
223 const Instruction *CtxI) const;
224
225 Value *optimizeWithRcp(IRBuilder<> &Builder, Value *Num, Value *Den,
226 FastMathFlags FMF, const Instruction *CtxI) const;
227 Value *optimizeWithFDivFast(IRBuilder<> &Builder, Value *Num, Value *Den,
228 float ReqdAccuracy) const;
229
230 Value *visitFDivElement(IRBuilder<> &Builder, Value *Num, Value *Den,
231 FastMathFlags DivFMF, FastMathFlags SqrtFMF,
232 Value *RsqOp, const Instruction *FDiv,
233 float ReqdAccuracy) const;
234
235 std::pair<Value *, Value *> getFrexpResults(IRBuilder<> &Builder,
236 Value *Src) const;
237
238 Value *emitRcpIEEE1ULP(IRBuilder<> &Builder, Value *Src,
239 bool IsNegative) const;
240 Value *emitFrexpDiv(IRBuilder<> &Builder, Value *LHS, Value *RHS,
241 FastMathFlags FMF) const;
242 Value *emitSqrtIEEE2ULP(IRBuilder<> &Builder, Value *Src,
243 FastMathFlags FMF) const;
244 Value *emitRsqF64(IRBuilder<> &Builder, Value *X, FastMathFlags SqrtFMF,
245 FastMathFlags DivFMF, const Instruction *CtxI,
246 bool IsNegative) const;
247
248 CallInst *createWorkitemIdX(IRBuilder<> &B) const;
249 void replaceWithWorkitemIdX(Instruction &I) const;
250 void replaceWithMaskedWorkitemIdX(Instruction &I, unsigned WaveSize) const;
251 bool tryReplaceWithWorkitemId(Instruction &I, unsigned Wave) const;
252
253 bool tryNarrowMathIfNoOverflow(Instruction *I);
254
255public:
256 bool visitFDiv(BinaryOperator &I);
257
258 bool visitInstruction(Instruction &I) { return false; }
259 bool visitBinaryOperator(BinaryOperator &I);
260 bool visitLoadInst(LoadInst &I);
261 bool visitSelectInst(SelectInst &I);
262 bool visitPHINode(PHINode &I);
263 bool visitAddrSpaceCastInst(AddrSpaceCastInst &I);
264
265 bool visitIntrinsicInst(IntrinsicInst &I);
266 bool visitFMinLike(IntrinsicInst &I);
267 bool visitSqrt(IntrinsicInst &I);
268 bool visitLog(FPMathOperator &Log, Intrinsic::ID IID);
269 bool visitMbcntLo(IntrinsicInst &I) const;
270 bool visitMbcntHi(IntrinsicInst &I) const;
271 bool visitVectorReduceAdd(IntrinsicInst &I);
272 bool visitSaturatingAdd(IntrinsicInst &I);
273 bool run();
274};
275
276class AMDGPUCodeGenPrepare : public FunctionPass {
277public:
278 static char ID;
279 AMDGPUCodeGenPrepare() : FunctionPass(ID) {}
280 void getAnalysisUsage(AnalysisUsage &AU) const override {
281 AU.addRequired<AssumptionCacheTracker>();
282 AU.addRequired<UniformityInfoWrapperPass>();
283 AU.addRequired<TargetLibraryInfoWrapperPass>();
284 AU.addRequired<TargetTransformInfoWrapperPass>();
285
286 // FIXME: Division expansion needs to preserve the dominator tree.
287 if (!ExpandDiv64InIR)
288 AU.setPreservesAll();
289 }
290 bool runOnFunction(Function &F) override;
291 StringRef getPassName() const override { return "AMDGPU IR optimizations"; }
292};
293
294} // end anonymous namespace
295
296bool AMDGPUCodeGenPrepareImpl::run() {
297 BreakPhiNodesCache.clear();
298 bool MadeChange = false;
299
300 // Need to use make_early_inc_range because integer division expansion is
301 // handled by Transform/Utils, and it can delete instructions such as the
302 // terminator of the BB.
303 for (BasicBlock &BB : reverse(C&: F)) {
304 for (Instruction &I : make_early_inc_range(Range: reverse(C&: BB))) {
305 if (!isInstructionTriviallyDead(I: &I, TLI))
306 MadeChange |= visit(I);
307 }
308 }
309
310 while (!DeadVals.empty()) {
311 if (auto *I = dyn_cast_or_null<Instruction>(Val: DeadVals.pop_back_val()))
312 RecursivelyDeleteTriviallyDeadInstructions(V: I, TLI);
313 }
314
315 return MadeChange;
316}
317
318bool AMDGPUCodeGenPrepareImpl::isLegalFloatingTy(const Type *Ty) const {
319 return Ty->isFloatTy() || Ty->isDoubleTy() ||
320 (Ty->isHalfTy() && ST.has16BitInsts());
321}
322
323bool AMDGPUCodeGenPrepareImpl::canWidenScalarExtLoad(LoadInst &I) const {
324 Type *Ty = I.getType();
325 int TySize = DL.getTypeSizeInBits(Ty);
326 Align Alignment = DL.getValueOrABITypeAlignment(Alignment: I.getAlign(), Ty);
327
328 return I.isSimple() && TySize < 32 && Alignment >= 4 && UA.isUniformAtDef(V: &I);
329}
330
331unsigned
332AMDGPUCodeGenPrepareImpl::numBitsUnsigned(Value *Op,
333 const Instruction *CtxI) const {
334 return computeKnownBits(V: Op, Q: SQ.getWithInstruction(I: CtxI)).countMaxActiveBits();
335}
336
337unsigned
338AMDGPUCodeGenPrepareImpl::numBitsSigned(Value *Op,
339 const Instruction *CtxI) const {
340 return ComputeMaxSignificantBits(Op, DL: SQ.DL, AC: SQ.AC, CxtI: CtxI, DT: SQ.DT);
341}
342
343static void extractValues(IRBuilder<> &Builder,
344 SmallVectorImpl<Value *> &Values, Value *V) {
345 auto *VT = dyn_cast<FixedVectorType>(Val: V->getType());
346 if (!VT) {
347 Values.push_back(Elt: V);
348 return;
349 }
350
351 for (int I = 0, E = VT->getNumElements(); I != E; ++I)
352 Values.push_back(Elt: Builder.CreateExtractElement(Vec: V, Idx: I));
353}
354
355static Value *insertValues(IRBuilder<> &Builder,
356 Type *Ty,
357 SmallVectorImpl<Value *> &Values) {
358 if (!Ty->isVectorTy()) {
359 assert(Values.size() == 1);
360 return Values[0];
361 }
362
363 Value *NewVal = PoisonValue::get(T: Ty);
364 for (int I = 0, E = Values.size(); I != E; ++I)
365 NewVal = Builder.CreateInsertElement(Vec: NewVal, NewElt: Values[I], Idx: I);
366
367 return NewVal;
368}
369
370bool AMDGPUCodeGenPrepareImpl::replaceMulWithMul24(BinaryOperator &I) const {
371 if (I.getOpcode() != Instruction::Mul)
372 return false;
373
374 Type *Ty = I.getType();
375 unsigned Size = Ty->getScalarSizeInBits();
376 if (Size <= 16 && ST.has16BitInsts())
377 return false;
378
379 // Prefer scalar if this could be s_mul_i32
380 if (UA.isUniformAtDef(V: &I))
381 return false;
382
383 Value *LHS = I.getOperand(i_nocapture: 0);
384 Value *RHS = I.getOperand(i_nocapture: 1);
385 IRBuilder<> Builder(&I);
386 Builder.SetCurrentDebugLocation(I.getDebugLoc());
387
388 unsigned LHSBits = 0, RHSBits = 0;
389 bool IsSigned = false;
390
391 if (ST.hasMulU24() && (LHSBits = numBitsUnsigned(Op: LHS, CtxI: &I)) <= 24 &&
392 (RHSBits = numBitsUnsigned(Op: RHS, CtxI: &I)) <= 24) {
393 IsSigned = false;
394
395 } else if (ST.hasMulI24() && (LHSBits = numBitsSigned(Op: LHS, CtxI: &I)) <= 24 &&
396 (RHSBits = numBitsSigned(Op: RHS, CtxI: &I)) <= 24) {
397 IsSigned = true;
398
399 } else
400 return false;
401
402 SmallVector<Value *, 4> LHSVals;
403 SmallVector<Value *, 4> RHSVals;
404 SmallVector<Value *, 4> ResultVals;
405 extractValues(Builder, Values&: LHSVals, V: LHS);
406 extractValues(Builder, Values&: RHSVals, V: RHS);
407
408 IntegerType *I32Ty = Builder.getInt32Ty();
409 IntegerType *IntrinTy = Size > 32 ? Builder.getInt64Ty() : I32Ty;
410 Type *DstTy = LHSVals[0]->getType();
411
412 for (int I = 0, E = LHSVals.size(); I != E; ++I) {
413 Value *LHS = IsSigned ? Builder.CreateSExtOrTrunc(V: LHSVals[I], DestTy: I32Ty)
414 : Builder.CreateZExtOrTrunc(V: LHSVals[I], DestTy: I32Ty);
415 Value *RHS = IsSigned ? Builder.CreateSExtOrTrunc(V: RHSVals[I], DestTy: I32Ty)
416 : Builder.CreateZExtOrTrunc(V: RHSVals[I], DestTy: I32Ty);
417 Intrinsic::ID ID =
418 IsSigned ? Intrinsic::amdgcn_mul_i24 : Intrinsic::amdgcn_mul_u24;
419 Value *Result = Builder.CreateIntrinsic(ID, OverloadTypes: {IntrinTy}, Args: {LHS, RHS});
420 Result = IsSigned ? Builder.CreateSExtOrTrunc(V: Result, DestTy: DstTy)
421 : Builder.CreateZExtOrTrunc(V: Result, DestTy: DstTy);
422 ResultVals.push_back(Elt: Result);
423 }
424
425 Value *NewVal = insertValues(Builder, Ty, Values&: ResultVals);
426 NewVal->takeName(V: &I);
427 I.replaceAllUsesWith(V: NewVal);
428 DeadVals.push_back(Elt: &I);
429
430 return true;
431}
432
433// Find a select instruction, which may have been casted. This is mostly to deal
434// with cases where i16 selects were promoted here to i32.
435static SelectInst *findSelectThroughCast(Value *V, CastInst *&Cast) {
436 Cast = nullptr;
437 if (SelectInst *Sel = dyn_cast<SelectInst>(Val: V))
438 return Sel;
439
440 if ((Cast = dyn_cast<CastInst>(Val: V))) {
441 if (SelectInst *Sel = dyn_cast<SelectInst>(Val: Cast->getOperand(i_nocapture: 0)))
442 return Sel;
443 }
444
445 return nullptr;
446}
447
448bool AMDGPUCodeGenPrepareImpl::foldBinOpIntoSelect(BinaryOperator &BO) const {
449 // Don't do this unless the old select is going away. We want to eliminate the
450 // binary operator, not replace a binop with a select.
451 int SelOpNo = 0;
452
453 CastInst *CastOp;
454
455 // TODO: Should probably try to handle some cases with multiple
456 // users. Duplicating the select may be profitable for division.
457 SelectInst *Sel = findSelectThroughCast(V: BO.getOperand(i_nocapture: 0), Cast&: CastOp);
458 if (!Sel || !Sel->hasOneUse()) {
459 SelOpNo = 1;
460 Sel = findSelectThroughCast(V: BO.getOperand(i_nocapture: 1), Cast&: CastOp);
461 }
462
463 if (!Sel || !Sel->hasOneUse())
464 return false;
465
466 Constant *CT = dyn_cast<Constant>(Val: Sel->getTrueValue());
467 Constant *CF = dyn_cast<Constant>(Val: Sel->getFalseValue());
468 Constant *CBO = dyn_cast<Constant>(Val: BO.getOperand(i_nocapture: SelOpNo ^ 1));
469 if (!CBO || !CT || !CF)
470 return false;
471
472 if (CastOp) {
473 if (!CastOp->hasOneUse())
474 return false;
475 CT = ConstantFoldCastOperand(Opcode: CastOp->getOpcode(), C: CT, DestTy: BO.getType(), DL);
476 CF = ConstantFoldCastOperand(Opcode: CastOp->getOpcode(), C: CF, DestTy: BO.getType(), DL);
477 }
478
479 // TODO: Handle special 0/-1 cases DAG combine does, although we only really
480 // need to handle divisions here.
481 Constant *FoldedT =
482 SelOpNo ? ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CBO, RHS: CT, DL)
483 : ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CT, RHS: CBO, DL);
484 if (!FoldedT || isa<ConstantExpr>(Val: FoldedT))
485 return false;
486
487 Constant *FoldedF =
488 SelOpNo ? ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CBO, RHS: CF, DL)
489 : ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CF, RHS: CBO, DL);
490 if (!FoldedF || isa<ConstantExpr>(Val: FoldedF))
491 return false;
492
493 IRBuilder<> Builder(&BO);
494 Builder.SetCurrentDebugLocation(BO.getDebugLoc());
495 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Val: &BO))
496 Builder.setFastMathFlags(FPOp->getFastMathFlags());
497
498 Value *NewSelect = Builder.CreateSelect(C: Sel->getCondition(),
499 True: FoldedT, False: FoldedF);
500 NewSelect->takeName(V: &BO);
501 BO.replaceAllUsesWith(V: NewSelect);
502 DeadVals.push_back(Elt: &BO);
503 if (CastOp)
504 DeadVals.push_back(Elt: CastOp);
505 DeadVals.push_back(Elt: Sel);
506 return true;
507}
508
509std::pair<Value *, Value *>
510AMDGPUCodeGenPrepareImpl::getFrexpResults(IRBuilder<> &Builder,
511 Value *Src) const {
512 Type *Ty = Src->getType();
513 Value *Frexp = Builder.CreateIntrinsic(ID: Intrinsic::frexp,
514 OverloadTypes: {Ty, Builder.getInt32Ty()}, Args: Src);
515 Value *FrexpMant = Builder.CreateExtractValue(Agg: Frexp, Idxs: {0});
516
517 // Bypass the bug workaround for the exponent result since it doesn't matter.
518 // TODO: Does the bug workaround even really need to consider the exponent
519 // result? It's unspecified by the spec.
520
521 Value *FrexpExp =
522 ST.hasFractBug()
523 ? Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_frexp_exp,
524 OverloadTypes: {Builder.getInt32Ty(), Ty}, Args: Src)
525 : Builder.CreateExtractValue(Agg: Frexp, Idxs: {1});
526 return {FrexpMant, FrexpExp};
527}
528
529/// Emit an expansion of 1.0 / Src good for 1ulp that supports denormals.
530Value *AMDGPUCodeGenPrepareImpl::emitRcpIEEE1ULP(IRBuilder<> &Builder,
531 Value *Src,
532 bool IsNegative) const {
533 // Same as for 1.0, but expand the sign out of the constant.
534 // -1.0 / x -> rcp (fneg x)
535 if (IsNegative)
536 Src = Builder.CreateFNeg(V: Src);
537
538 // The rcp instruction doesn't support denormals, so scale the input
539 // out of the denormal range and convert at the end.
540 //
541 // Expand as 2^-n * (1.0 / (x * 2^n))
542
543 // TODO: Skip scaling if input is known never denormal and the input
544 // range won't underflow to denormal. The hard part is knowing the
545 // result. We need a range check, the result could be denormal for
546 // 0x1p+126 < den <= 0x1p+127.
547 auto [FrexpMant, FrexpExp] = getFrexpResults(Builder, Src);
548 Value *ScaleFactor = Builder.CreateNeg(V: FrexpExp);
549 Value *Rcp = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: FrexpMant);
550 return Builder.CreateCall(Callee: getLdexpF32(), Args: {Rcp, ScaleFactor});
551}
552
553/// Emit a 2ulp expansion for fdiv by using frexp for input scaling.
554Value *AMDGPUCodeGenPrepareImpl::emitFrexpDiv(IRBuilder<> &Builder, Value *LHS,
555 Value *RHS,
556 FastMathFlags FMF) const {
557 // If we have have to work around the fract/frexp bug, we're worse off than
558 // using the fdiv.fast expansion. The full safe expansion is faster if we have
559 // fast FMA.
560 if (HasFP32DenormalFlush && ST.hasFractBug() && !ST.hasFastFMAF32() &&
561 (!FMF.noNaNs() || !FMF.noInfs()))
562 return nullptr;
563
564 // We're scaling the LHS to avoid a denormal input, and scale the denominator
565 // to avoid large values underflowing the result.
566 auto [FrexpMantRHS, FrexpExpRHS] = getFrexpResults(Builder, Src: RHS);
567
568 Value *Rcp =
569 Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: FrexpMantRHS);
570
571 auto [FrexpMantLHS, FrexpExpLHS] = getFrexpResults(Builder, Src: LHS);
572 Value *Mul = Builder.CreateFMul(L: FrexpMantLHS, R: Rcp);
573
574 // We multiplied by 2^N/2^M, so we need to multiply by 2^(N-M) to scale the
575 // result.
576 Value *ExpDiff = Builder.CreateSub(LHS: FrexpExpLHS, RHS: FrexpExpRHS);
577 return Builder.CreateCall(Callee: getLdexpF32(), Args: {Mul, ExpDiff});
578}
579
580/// Emit a sqrt that handles denormals and is accurate to 2ulp.
581Value *AMDGPUCodeGenPrepareImpl::emitSqrtIEEE2ULP(IRBuilder<> &Builder,
582 Value *Src,
583 FastMathFlags FMF) const {
584 Type *Ty = Src->getType();
585 APFloat SmallestNormal =
586 APFloat::getSmallestNormalized(Sem: Ty->getFltSemantics());
587 Value *NeedScale =
588 Builder.CreateFCmpOLT(LHS: Src, RHS: ConstantFP::get(Ty, V: SmallestNormal));
589
590 ConstantInt *Zero = Builder.getInt32(C: 0);
591 Value *InputScaleFactor =
592 Builder.CreateSelect(C: NeedScale, True: Builder.getInt32(C: 32), False: Zero);
593
594 Value *Scaled = Builder.CreateCall(Callee: getLdexpF32(), Args: {Src, InputScaleFactor});
595
596 Value *Sqrt = Builder.CreateCall(Callee: getSqrtF32(), Args: Scaled);
597
598 Value *OutputScaleFactor =
599 Builder.CreateSelect(C: NeedScale, True: Builder.getInt32(C: -16), False: Zero);
600 return Builder.CreateCall(Callee: getLdexpF32(), Args: {Sqrt, OutputScaleFactor});
601}
602
603/// Emit an expansion of 1.0 / sqrt(Src) good for 1ulp that supports denormals.
604static Value *emitRsqIEEE1ULP(IRBuilder<> &Builder, Value *Src,
605 bool IsNegative) {
606 // bool need_scale = x < 0x1p-126f;
607 // float input_scale = need_scale ? 0x1.0p+24f : 1.0f;
608 // float output_scale = need_scale ? 0x1.0p+12f : 1.0f;
609 // rsq(x * input_scale) * output_scale;
610
611 Type *Ty = Src->getType();
612 APFloat SmallestNormal =
613 APFloat::getSmallestNormalized(Sem: Ty->getFltSemantics());
614 Value *NeedScale =
615 Builder.CreateFCmpOLT(LHS: Src, RHS: ConstantFP::get(Ty, V: SmallestNormal));
616 Constant *One = ConstantFP::get(Ty, V: 1.0);
617 Constant *InputScale = ConstantFP::get(Ty, V: 0x1.0p+24);
618 Constant *OutputScale =
619 ConstantFP::get(Ty, V: IsNegative ? -0x1.0p+12 : 0x1.0p+12);
620
621 Value *InputScaleFactor = Builder.CreateSelect(C: NeedScale, True: InputScale, False: One);
622
623 Value *ScaledInput = Builder.CreateFMul(L: Src, R: InputScaleFactor);
624 Value *Rsq = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rsq, Op: ScaledInput);
625 Value *OutputScaleFactor = Builder.CreateSelect(
626 C: NeedScale, True: OutputScale, False: IsNegative ? ConstantFP::get(Ty, V: -1.0) : One);
627
628 return Builder.CreateFMul(L: Rsq, R: OutputScaleFactor);
629}
630
631/// Emit inverse sqrt expansion for f64 with a correction sequence on top of
632/// v_rsq_f64. This should give a 1ulp result.
633Value *AMDGPUCodeGenPrepareImpl::emitRsqF64(IRBuilder<> &Builder, Value *X,
634 FastMathFlags SqrtFMF,
635 FastMathFlags DivFMF,
636 const Instruction *CtxI,
637 bool IsNegative) const {
638 // rsq(x):
639 // double y0 = BUILTIN_AMDGPU_RSQRT_F64(x);
640 // double e = MATH_MAD(-y0 * (x == PINF_F64 || x == 0.0 ? y0 : x), y0, 1.0);
641 // return MATH_MAD(y0*e, MATH_MAD(e, 0.375, 0.5), y0);
642 //
643 // -rsq(x):
644 // double y0 = BUILTIN_AMDGPU_RSQRT_F64(x);
645 // double e = MATH_MAD(-y0 * (x == PINF_F64 || x == 0.0 ? y0 : x), y0, 1.0);
646 // return MATH_MAD(-y0*e, MATH_MAD(e, 0.375, 0.5), -y0);
647 //
648 // The rsq instruction handles the special cases correctly. We need to check
649 // for the edge case conditions to ensure the special case propagates through
650 // the later instructions.
651
652 Value *Y0 = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rsq, Op: X);
653
654 // Try to elide the edge case check.
655 //
656 // Fast math flags imply:
657 // sqrt ninf => !isinf(x)
658 // fdiv ninf => x != 0, !isinf(x)
659 bool MaybePosInf = !SqrtFMF.noInfs() && !DivFMF.noInfs();
660 bool MaybeZero = !DivFMF.noInfs();
661
662 DenormalMode DenormMode;
663 FPClassTest Interested = fcNone;
664 if (MaybePosInf)
665 Interested = fcPosInf;
666 if (MaybeZero)
667 Interested |= fcZero;
668
669 if (Interested != fcNone) {
670 KnownFPClass KnownSrc = computeKnownFPClass(V: X, Interested, CtxI);
671 if (KnownSrc.isKnownNeverPosInfinity())
672 MaybePosInf = false;
673
674 DenormMode = F.getDenormalMode(FPType: X->getType()->getFltSemantics());
675 if (KnownSrc.isKnownNeverLogicalZero(Mode: DenormMode))
676 MaybeZero = false;
677 }
678
679 Value *SpecialOrRsq = X;
680 if (MaybeZero || MaybePosInf) {
681 Value *Cond;
682 if (MaybePosInf && MaybeZero) {
683 if (DenormMode.Input != DenormalMode::DenormalModeKind::Dynamic) {
684 FPClassTest TestMask = fcPosInf | fcZero;
685 if (DenormMode.inputsAreZero())
686 TestMask |= fcSubnormal;
687
688 Cond = Builder.createIsFPClass(FPNum: X, Test: TestMask);
689 } else {
690 // Avoid using llvm.is.fpclass for dynamic denormal mode, since it
691 // doesn't respect the floating-point environment.
692 Value *IsZero =
693 Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getZero(Ty: X->getType()));
694 Value *IsInf =
695 Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getInfinity(Ty: X->getType()));
696 Cond = Builder.CreateOr(LHS: IsZero, RHS: IsInf);
697 }
698 } else if (MaybeZero) {
699 Cond = Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getZero(Ty: X->getType()));
700 } else {
701 Cond = Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getInfinity(Ty: X->getType()));
702 }
703
704 SpecialOrRsq = Builder.CreateSelect(C: Cond, True: Y0, False: X);
705 }
706
707 Value *NegY0 = Builder.CreateFNeg(V: Y0);
708 Value *NegXY0 = Builder.CreateFMul(L: SpecialOrRsq, R: NegY0);
709
710 // Could be fmuladd, but isFMAFasterThanFMulAndFAdd is always true for f64.
711 Value *E = Builder.CreateFMA(Factor1: NegXY0, Factor2: Y0, Summand: ConstantFP::get(Ty: X->getType(), V: 1.0));
712
713 Value *Y0E = Builder.CreateFMul(L: E, R: IsNegative ? NegY0 : Y0);
714
715 Value *EFMA = Builder.CreateFMA(Factor1: E, Factor2: ConstantFP::get(Ty: X->getType(), V: 0.375),
716 Summand: ConstantFP::get(Ty: X->getType(), V: 0.5));
717
718 return Builder.CreateFMA(Factor1: Y0E, Factor2: EFMA, Summand: IsNegative ? NegY0 : Y0);
719}
720
721bool AMDGPUCodeGenPrepareImpl::canOptimizeWithRsq(FastMathFlags DivFMF,
722 FastMathFlags SqrtFMF) const {
723 // The rsqrt contraction increases accuracy from ~2ulp to ~1ulp for f32 and
724 // f64.
725 return DivFMF.allowContract() && SqrtFMF.allowContract();
726}
727
728Value *AMDGPUCodeGenPrepareImpl::optimizeWithRsq(
729 IRBuilder<> &Builder, Value *Num, Value *Den, const FastMathFlags DivFMF,
730 const FastMathFlags SqrtFMF, const Instruction *CtxI) const {
731 // The rsqrt contraction increases accuracy from ~2ulp to ~1ulp.
732 assert(DivFMF.allowContract() && SqrtFMF.allowContract());
733
734 // rsq_f16 is accurate to 0.51 ulp.
735 // rsq_f32 is accurate for !fpmath >= 1.0ulp and denormals are flushed.
736 // rsq_f64 is never accurate.
737 const ConstantFP *CLHS = dyn_cast<ConstantFP>(Val: Num);
738 if (!CLHS)
739 return nullptr;
740
741 bool IsNegative = false;
742
743 // TODO: Handle other numerator values with arcp.
744 if (CLHS->isOne() || (IsNegative = CLHS->isMinusOne())) {
745 // Add sqrt flags, but require both ninf and nsz from the div and the
746 // sqrt: sqrt's ninf/nsz don't say anything about the quotient.
747 IRBuilder<>::FastMathFlagGuard Guard(Builder);
748 FastMathFlags NewFMF = DivFMF | SqrtFMF;
749 NewFMF.setNoInfs(DivFMF.noInfs() && SqrtFMF.noInfs());
750 NewFMF.setNoSignedZeros(DivFMF.noSignedZeros() && SqrtFMF.noSignedZeros());
751 Builder.setFastMathFlags(NewFMF);
752
753 if (Den->getType()->isFloatTy()) {
754 if ((DivFMF.approxFunc() && SqrtFMF.approxFunc()) ||
755 canIgnoreDenormalInput(V: Den, CtxI)) {
756 Value *Result =
757 Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rsq, Op: Den);
758 // -1.0 / sqrt(x) -> fneg(rsq(x))
759 return IsNegative ? Builder.CreateFNeg(V: Result) : Result;
760 }
761
762 return emitRsqIEEE1ULP(Builder, Src: Den, IsNegative);
763 }
764
765 if (Den->getType()->isDoubleTy())
766 return emitRsqF64(Builder, X: Den, SqrtFMF, DivFMF, CtxI, IsNegative);
767 }
768
769 return nullptr;
770}
771
772// Optimize fdiv with rcp:
773//
774// 1/x -> rcp(x) when rcp is sufficiently accurate or inaccurate rcp is
775// allowed with afn.
776//
777// a/b -> a*rcp(b) when arcp is allowed, and we only need provide ULP 1.0
778Value *
779AMDGPUCodeGenPrepareImpl::optimizeWithRcp(IRBuilder<> &Builder, Value *Num,
780 Value *Den, FastMathFlags FMF,
781 const Instruction *CtxI) const {
782 // rcp_f16 is accurate to 0.51 ulp.
783 // rcp_f32 is accurate for !fpmath >= 1.0ulp and denormals are flushed.
784 // rcp_f64 is never accurate.
785 assert(Den->getType()->isFloatTy());
786
787 if (const ConstantFP *CLHS = dyn_cast<ConstantFP>(Val: Num)) {
788 bool IsNegative = false;
789 if (CLHS->isOne() || (IsNegative = CLHS->isMinusOne())) {
790 Value *Src = Den;
791
792 if (HasFP32DenormalFlush || FMF.approxFunc()) {
793 // -1.0 / x -> 1.0 / fneg(x)
794 if (IsNegative)
795 Src = Builder.CreateFNeg(V: Src);
796
797 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
798 // the CI documentation has a worst case error of 1 ulp.
799 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK
800 // to use it as long as we aren't trying to use denormals.
801 //
802 // v_rcp_f16 and v_rsq_f16 DO support denormals.
803
804 // NOTE: v_sqrt and v_rcp will be combined to v_rsq later. So we don't
805 // insert rsq intrinsic here.
806
807 // 1.0 / x -> rcp(x)
808 return Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: Src);
809 }
810
811 // TODO: If the input isn't denormal, and we know the input exponent isn't
812 // big enough to introduce a denormal we can avoid the scaling.
813 return emitRcpIEEE1ULP(Builder, Src, IsNegative);
814 }
815 }
816
817 if (FMF.allowReciprocal()) {
818 // x / y -> x * (1.0 / y)
819
820 // TODO: Could avoid denormal scaling and use raw rcp if we knew the output
821 // will never underflow.
822 if (HasFP32DenormalFlush || FMF.approxFunc()) {
823 Value *Recip = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: Den);
824 return Builder.CreateFMul(L: Num, R: Recip);
825 }
826
827 Value *Recip = emitRcpIEEE1ULP(Builder, Src: Den, IsNegative: false);
828 return Builder.CreateFMul(L: Num, R: Recip);
829 }
830
831 return nullptr;
832}
833
834// optimize with fdiv.fast:
835//
836// a/b -> fdiv.fast(a, b) when !fpmath >= 2.5ulp with denormals flushed.
837//
838// 1/x -> fdiv.fast(1,x) when !fpmath >= 2.5ulp.
839//
840// NOTE: optimizeWithRcp should be tried first because rcp is the preference.
841Value *AMDGPUCodeGenPrepareImpl::optimizeWithFDivFast(
842 IRBuilder<> &Builder, Value *Num, Value *Den, float ReqdAccuracy) const {
843 // fdiv.fast can achieve 2.5 ULP accuracy.
844 if (ReqdAccuracy < 2.5f)
845 return nullptr;
846
847 // Only have fdiv.fast for f32.
848 assert(Den->getType()->isFloatTy());
849
850 bool NumIsOne = false;
851 if (const ConstantFP *CNum = dyn_cast<ConstantFP>(Val: Num)) {
852 if (CNum->isOne() || CNum->isMinusOne())
853 NumIsOne = true;
854 }
855
856 // fdiv does not support denormals. But 1.0/x is always fine to use it.
857 //
858 // TODO: This works for any value with a specific known exponent range, don't
859 // just limit to constant 1.
860 if (!HasFP32DenormalFlush && !NumIsOne)
861 return nullptr;
862
863 return Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_fdiv_fast, Args: {Num, Den});
864}
865
866Value *AMDGPUCodeGenPrepareImpl::visitFDivElement(
867 IRBuilder<> &Builder, Value *Num, Value *Den, FastMathFlags DivFMF,
868 FastMathFlags SqrtFMF, Value *RsqOp, const Instruction *FDivInst,
869 float ReqdDivAccuracy) const {
870 if (RsqOp) {
871 Value *Rsq =
872 optimizeWithRsq(Builder, Num, Den: RsqOp, DivFMF, SqrtFMF, CtxI: FDivInst);
873 if (Rsq)
874 return Rsq;
875 }
876
877 if (!Num->getType()->isFloatTy())
878 return nullptr;
879
880 Value *Rcp = optimizeWithRcp(Builder, Num, Den, FMF: DivFMF, CtxI: FDivInst);
881 if (Rcp)
882 return Rcp;
883
884 // In the basic case fdiv_fast has the same instruction count as the frexp div
885 // expansion. Slightly prefer fdiv_fast since it ends in an fmul that can
886 // potentially be fused into a user. Also, materialization of the constants
887 // can be reused for multiple instances.
888 Value *FDivFast = optimizeWithFDivFast(Builder, Num, Den, ReqdAccuracy: ReqdDivAccuracy);
889 if (FDivFast)
890 return FDivFast;
891
892 return emitFrexpDiv(Builder, LHS: Num, RHS: Den, FMF: DivFMF);
893}
894
895// Optimizations is performed based on fpmath, fast math flags as well as
896// denormals to optimize fdiv with either rcp or fdiv.fast.
897//
898// With rcp:
899// 1/x -> rcp(x) when rcp is sufficiently accurate or inaccurate rcp is
900// allowed with afn.
901//
902// a/b -> a*rcp(b) when inaccurate rcp is allowed with afn.
903//
904// With fdiv.fast:
905// a/b -> fdiv.fast(a, b) when !fpmath >= 2.5ulp with denormals flushed.
906//
907// 1/x -> fdiv.fast(1,x) when !fpmath >= 2.5ulp.
908//
909// NOTE: rcp is the preference in cases that both are legal.
910bool AMDGPUCodeGenPrepareImpl::visitFDiv(BinaryOperator &FDiv) {
911 if (DisableFDivExpand)
912 return false;
913
914 Type *Ty = FDiv.getType()->getScalarType();
915 const bool IsFloat = Ty->isFloatTy();
916 if (!IsFloat && !Ty->isDoubleTy())
917 return false;
918
919 // The f64 rcp/rsq approximations are pretty inaccurate. We can do an
920 // expansion around them in codegen. f16 is good enough to always use.
921
922 const FPMathOperator *FPOp = cast<const FPMathOperator>(Val: &FDiv);
923 const FastMathFlags DivFMF = FPOp->getFastMathFlags();
924 const float ReqdAccuracy = FPOp->getFPAccuracy();
925
926 FastMathFlags SqrtFMF;
927
928 Value *Num = FDiv.getOperand(i_nocapture: 0);
929 Value *Den = FDiv.getOperand(i_nocapture: 1);
930
931 Value *RsqOp = nullptr;
932 auto *DenII = dyn_cast<IntrinsicInst>(Val: Den);
933 if (DenII && DenII->getIntrinsicID() == Intrinsic::sqrt &&
934 DenII->hasOneUse()) {
935 const auto *SqrtOp = cast<FPMathOperator>(Val: DenII);
936 SqrtFMF = SqrtOp->getFastMathFlags();
937 if (canOptimizeWithRsq(DivFMF, SqrtFMF))
938 RsqOp = SqrtOp->getOperand(i: 0);
939 }
940
941 // rcp path not yet implemented for f64.
942 if (!IsFloat && !RsqOp)
943 return false;
944
945 // Inaccurate rcp is allowed with afn.
946 //
947 // Defer to codegen to handle this.
948 //
949 // TODO: Decide on an interpretation for interactions between afn + arcp +
950 // !fpmath, and make it consistent between here and codegen. For now, defer
951 // expansion of afn to codegen. The current interpretation is so aggressive we
952 // don't need any pre-consideration here when we have better information. A
953 // more conservative interpretation could use handling here.
954 const bool AllowInaccurateRcp = DivFMF.approxFunc();
955 if (!RsqOp && AllowInaccurateRcp)
956 return false;
957
958 // Defer the correct implementations to codegen.
959 if (IsFloat && ReqdAccuracy < 1.0f)
960 return false;
961
962 IRBuilder<> Builder(FDiv.getParent(), std::next(x: FDiv.getIterator()));
963 Builder.setFastMathFlags(DivFMF);
964 Builder.SetCurrentDebugLocation(FDiv.getDebugLoc());
965
966 SmallVector<Value *, 4> NumVals;
967 SmallVector<Value *, 4> DenVals;
968 SmallVector<Value *, 4> RsqDenVals;
969 extractValues(Builder, Values&: NumVals, V: Num);
970 extractValues(Builder, Values&: DenVals, V: Den);
971
972 if (RsqOp)
973 extractValues(Builder, Values&: RsqDenVals, V: RsqOp);
974
975 SmallVector<Value *, 4> ResultVals(NumVals.size());
976 for (int I = 0, E = NumVals.size(); I != E; ++I) {
977 Value *NumElt = NumVals[I];
978 Value *DenElt = DenVals[I];
979 Value *RsqDenElt = RsqOp ? RsqDenVals[I] : nullptr;
980
981 Value *NewElt =
982 visitFDivElement(Builder, Num: NumElt, Den: DenElt, DivFMF, SqrtFMF, RsqOp: RsqDenElt,
983 FDivInst: cast<Instruction>(Val: FPOp), ReqdDivAccuracy: ReqdAccuracy);
984 if (!NewElt) {
985 // Keep the original, but scalarized.
986
987 // This has the unfortunate side effect of sometimes scalarizing when
988 // we're not going to do anything.
989 NewElt = Builder.CreateFDiv(L: NumElt, R: DenElt);
990 if (auto *NewEltInst = dyn_cast<Instruction>(Val: NewElt))
991 NewEltInst->copyMetadata(SrcInst: FDiv);
992 }
993
994 ResultVals[I] = NewElt;
995 }
996
997 Value *NewVal = insertValues(Builder, Ty: FDiv.getType(), Values&: ResultVals);
998
999 if (NewVal) {
1000 FDiv.replaceAllUsesWith(V: NewVal);
1001 NewVal->takeName(V: &FDiv);
1002 DeadVals.push_back(Elt: &FDiv);
1003 }
1004
1005 return true;
1006}
1007
1008static std::pair<Value*, Value*> getMul64(IRBuilder<> &Builder,
1009 Value *LHS, Value *RHS) {
1010 Type *I32Ty = Builder.getInt32Ty();
1011 Type *I64Ty = Builder.getInt64Ty();
1012
1013 Value *LHS_EXT64 = Builder.CreateZExt(V: LHS, DestTy: I64Ty);
1014 Value *RHS_EXT64 = Builder.CreateZExt(V: RHS, DestTy: I64Ty);
1015 Value *MUL64 = Builder.CreateMul(LHS: LHS_EXT64, RHS: RHS_EXT64);
1016 Value *Lo = Builder.CreateTrunc(V: MUL64, DestTy: I32Ty);
1017 Value *Hi = Builder.CreateLShr(LHS: MUL64, RHS: Builder.getInt64(C: 32));
1018 Hi = Builder.CreateTrunc(V: Hi, DestTy: I32Ty);
1019 return std::pair(Lo, Hi);
1020}
1021
1022static Value* getMulHu(IRBuilder<> &Builder, Value *LHS, Value *RHS) {
1023 return getMul64(Builder, LHS, RHS).second;
1024}
1025
1026/// Figure out how many bits are really needed for this division.
1027/// \p MaxDivBits is an optimization hint to bypass the second
1028/// ComputeNumSignBits/computeKnownBits call if the first one is
1029/// insufficient.
1030unsigned AMDGPUCodeGenPrepareImpl::getDivNumBits(BinaryOperator &I, Value *Num,
1031 Value *Den,
1032 unsigned MaxDivBits,
1033 bool IsSigned) const {
1034 assert(Num->getType()->getScalarSizeInBits() ==
1035 Den->getType()->getScalarSizeInBits());
1036 unsigned SSBits = Num->getType()->getScalarSizeInBits();
1037 if (IsSigned) {
1038 unsigned RHSSignBits = ComputeNumSignBits(Op: Den, DL: SQ.DL, AC: SQ.AC, CxtI: &I, DT: SQ.DT);
1039 // A sign bit needs to be reserved for shrinking.
1040 unsigned DivBits = SSBits - RHSSignBits + 1;
1041 if (DivBits > MaxDivBits)
1042 return SSBits;
1043
1044 unsigned LHSSignBits = ComputeNumSignBits(Op: Num, DL: SQ.DL, AC: SQ.AC, CxtI: &I);
1045
1046 unsigned SignBits = std::min(a: LHSSignBits, b: RHSSignBits);
1047 DivBits = SSBits - SignBits + 1;
1048 return DivBits;
1049 }
1050
1051 // All bits are used for unsigned division for Num or Den in range
1052 // (SignedMax, UnsignedMax].
1053 KnownBits Known = computeKnownBits(V: Den, Q: SQ.getWithInstruction(I: &I));
1054 unsigned RHSBits = Known.countMaxActiveBits();
1055 if (RHSBits > MaxDivBits)
1056 return SSBits;
1057
1058 Known = computeKnownBits(V: Num, Q: SQ.getWithInstruction(I: &I));
1059 unsigned LHSBits = Known.countMaxActiveBits();
1060
1061 unsigned DivBits = std::max(a: LHSBits, b: RHSBits);
1062 return DivBits;
1063}
1064
1065Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloat(IRBuilder<> &Builder,
1066 BinaryOperator &I,
1067 Value *Num, Value *Den,
1068 bool IsDiv,
1069 bool IsSigned) const {
1070 unsigned DivBits = getDivNumBits(I, Num, Den, MaxDivBits: 23, IsSigned);
1071
1072 if (DivBits > (IsSigned ? 23 : 22))
1073 return nullptr;
1074 return expandDivRemToFloatImpl(Builder, I, Num, Den, NumBits: DivBits, IsDiv,
1075 IsSigned);
1076}
1077
1078Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
1079 IRBuilder<> &Builder, BinaryOperator &I, Value *Num, Value *Den,
1080 unsigned DivBits, bool IsDiv, bool IsSigned) const {
1081
1082 // v_rcp_f32(float(X)) can have an error of 1 ulp.
1083 // This would cause incorrect calculation of Y/X if:
1084 // Y = (0x7FFFFF/X)*(X-0)-1
1085 // were allowed.
1086 //
1087 // For example,
1088 // (0x7FF6D3/0x000FE7) would erroneously produce 2060 instead of 2059.
1089 // (0x7FF8F5/0x007EFB) would erroneously produce 258 instead of 257.
1090 //
1091 // Thus, we conservatively restrict expandDivRemToFloatImpl to
1092 // [-0x400000,0x3FFFFF] for IsSigned
1093 // [ 0x000000,0x3FFFFF] for !IsSigned.
1094 assert(0 < DivBits && DivBits <= (IsSigned ? 23 : 22) &&
1095 "abs(Num) must be <= 0x400000 for expandDivRemToFloatImpl to work "
1096 "correctly");
1097
1098 Type *I32Ty = Builder.getInt32Ty();
1099 Num = Builder.CreateTrunc(V: Num, DestTy: I32Ty);
1100 Den = Builder.CreateTrunc(V: Den, DestTy: I32Ty);
1101
1102 Type *F32Ty = Builder.getFloatTy();
1103 ConstantInt *One = Builder.getInt32(C: 1);
1104
1105 // int ia = (int)LHS;
1106 Value *IA = Num;
1107
1108 // int ib, (int)RHS;
1109 Value *IB = Den;
1110
1111 // float fa = (float)ia;
1112 Value *FA = IsSigned ? Builder.CreateSIToFP(V: IA, DestTy: F32Ty)
1113 : Builder.CreateUIToFP(V: IA, DestTy: F32Ty);
1114
1115 // float fb = (float)ib;
1116 Value *FB = IsSigned ? Builder.CreateSIToFP(V: IB, DestTy: F32Ty)
1117 : Builder.CreateUIToFP(V: IB, DestTy: F32Ty);
1118
1119 Value *RCP = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_rcp,
1120 OverloadTypes: Builder.getFloatTy(), Args: {FB});
1121
1122 // The calculation:
1123 // fq = fa*recip(fb)
1124 // may be too small due to the 1ulp accuracy in the recip
1125 // operation and rounding issues. Since fq is truncated to produce
1126 // an integer value it may be too small by one. This is
1127 // dealt with by incrementing fa by 1ulp:
1128 // fq = (fa+1ulp)*recip(fb)
1129 // This will increase fa's magnitude by at most 0.5
1130 // (i.e. when fabs(fa)==0x400000 the LSB of the mantissa represents 0.5).
1131 // Thus, this method is safe since fa must be incremented by at least 1.0
1132 // for the quotient to increase by one.
1133
1134 Value *FABits = Builder.CreateBitCast(V: FA, DestTy: I32Ty);
1135 Value *FABitsInc = Builder.CreateAdd(LHS: FABits, RHS: One);
1136 FA = Builder.CreateBitCast(V: FABitsInc, DestTy: F32Ty);
1137
1138 Value *FQM = Builder.CreateFMul(L: FA, R: RCP);
1139
1140 // fq = trunc(fqm);
1141 Value *FQ = Builder.CreateUnaryIntrinsic(ID: Intrinsic::trunc, Op: FQM);
1142
1143 // int iq = (int)fq;
1144 Value *IQ = IsSigned ? Builder.CreateFPToSI(V: FQ, DestTy: I32Ty)
1145 : Builder.CreateFPToUI(V: FQ, DestTy: I32Ty);
1146
1147 Value *Res = IQ;
1148 if (!IsDiv) {
1149 // Rem needs compensation, it's easier to recompute it
1150 Value *Rem = Builder.CreateMul(LHS: IQ, RHS: Den);
1151 Res = Builder.CreateSub(LHS: Num, RHS: Rem);
1152 }
1153
1154 return Res;
1155}
1156
1157// Try to recognize special cases the DAG will emit special, better expansions
1158// than the general expansion we do here.
1159
1160// TODO: It would be better to just directly handle those optimizations here.
1161bool AMDGPUCodeGenPrepareImpl::divHasSpecialOptimization(BinaryOperator &I,
1162 Value *Num,
1163 Value *Den) const {
1164 if (Constant *C = dyn_cast<Constant>(Val: Den)) {
1165 // Arbitrary constants get a better expansion as long as a wider mulhi is
1166 // legal.
1167 if (C->getType()->getScalarSizeInBits() <= 32)
1168 return true;
1169
1170 // TODO: Sdiv check for not exact for some reason.
1171
1172 // If there's no wider mulhi, there's only a better expansion for powers of
1173 // two.
1174 // TODO: Should really know for each vector element.
1175 if (isKnownToBeAPowerOfTwo(V: C, OrZero: true, Q: SQ.getWithInstruction(I: &I)))
1176 return true;
1177
1178 return false;
1179 }
1180
1181 if (BinaryOperator *BinOpDen = dyn_cast<BinaryOperator>(Val: Den)) {
1182 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
1183 if (BinOpDen->getOpcode() == Instruction::Shl &&
1184 isa<Constant>(Val: BinOpDen->getOperand(i_nocapture: 0)) &&
1185 isKnownToBeAPowerOfTwo(V: BinOpDen->getOperand(i_nocapture: 0), OrZero: true,
1186 Q: SQ.getWithInstruction(I: &I))) {
1187 return true;
1188 }
1189 }
1190
1191 return false;
1192}
1193
1194static Value *getSign32(Value *V, IRBuilder<> &Builder, const DataLayout DL) {
1195 // Check whether the sign can be determined statically.
1196 KnownBits Known = computeKnownBits(V, DL);
1197 if (Known.isNegative())
1198 return Constant::getAllOnesValue(Ty: V->getType());
1199 if (Known.isNonNegative())
1200 return Constant::getNullValue(Ty: V->getType());
1201 return Builder.CreateAShr(LHS: V, RHS: Builder.getInt32(C: 31));
1202}
1203
1204Value *AMDGPUCodeGenPrepareImpl::expandDivRem32(IRBuilder<> &Builder,
1205 BinaryOperator &I, Value *X,
1206 Value *Y) const {
1207 Instruction::BinaryOps Opc = I.getOpcode();
1208 assert(Opc == Instruction::URem || Opc == Instruction::UDiv ||
1209 Opc == Instruction::SRem || Opc == Instruction::SDiv);
1210
1211 FastMathFlags FMF;
1212 FMF.setFast();
1213 Builder.setFastMathFlags(FMF);
1214
1215 if (divHasSpecialOptimization(I, Num: X, Den: Y))
1216 return nullptr; // Keep it for later optimization.
1217
1218 bool IsDiv = Opc == Instruction::UDiv || Opc == Instruction::SDiv;
1219 bool IsSigned = Opc == Instruction::SRem || Opc == Instruction::SDiv;
1220
1221 Type *Ty = X->getType();
1222 Type *I32Ty = Builder.getInt32Ty();
1223 Type *F32Ty = Builder.getFloatTy();
1224
1225 if (Ty->getScalarSizeInBits() != 32) {
1226 if (IsSigned) {
1227 X = Builder.CreateSExtOrTrunc(V: X, DestTy: I32Ty);
1228 Y = Builder.CreateSExtOrTrunc(V: Y, DestTy: I32Ty);
1229 } else {
1230 X = Builder.CreateZExtOrTrunc(V: X, DestTy: I32Ty);
1231 Y = Builder.CreateZExtOrTrunc(V: Y, DestTy: I32Ty);
1232 }
1233 }
1234
1235 if (Value *Res = expandDivRemToFloat(Builder, I, Num: X, Den: Y, IsDiv, IsSigned)) {
1236 return IsSigned ? Builder.CreateSExtOrTrunc(V: Res, DestTy: Ty) :
1237 Builder.CreateZExtOrTrunc(V: Res, DestTy: Ty);
1238 }
1239
1240 ConstantInt *Zero = Builder.getInt32(C: 0);
1241 ConstantInt *One = Builder.getInt32(C: 1);
1242
1243 Value *Sign = nullptr;
1244 if (IsSigned) {
1245 Value *SignX = getSign32(V: X, Builder, DL);
1246 Value *SignY = getSign32(V: Y, Builder, DL);
1247 // Remainder sign is the same as LHS
1248 Sign = IsDiv ? Builder.CreateXor(LHS: SignX, RHS: SignY) : SignX;
1249
1250 X = Builder.CreateAdd(LHS: X, RHS: SignX);
1251 Y = Builder.CreateAdd(LHS: Y, RHS: SignY);
1252
1253 X = Builder.CreateXor(LHS: X, RHS: SignX);
1254 Y = Builder.CreateXor(LHS: Y, RHS: SignY);
1255 }
1256
1257 // The algorithm here is based on ideas from "Software Integer Division", Tom
1258 // Rodeheffer, August 2008.
1259 //
1260 // unsigned udiv(unsigned x, unsigned y) {
1261 // // Initial estimate of inv(y). The constant is less than 2^32 to ensure
1262 // // that this is a lower bound on inv(y), even if some of the calculations
1263 // // round up.
1264 // unsigned z = (unsigned)((4294967296.0 - 512.0) * v_rcp_f32((float)y));
1265 //
1266 // // One round of UNR (Unsigned integer Newton-Raphson) to improve z.
1267 // // Empirically this is guaranteed to give a "two-y" lower bound on
1268 // // inv(y).
1269 // z += umulh(z, -y * z);
1270 //
1271 // // Quotient/remainder estimate.
1272 // unsigned q = umulh(x, z);
1273 // unsigned r = x - q * y;
1274 //
1275 // // Two rounds of quotient/remainder refinement.
1276 // if (r >= y) {
1277 // ++q;
1278 // r -= y;
1279 // }
1280 // if (r >= y) {
1281 // ++q;
1282 // r -= y;
1283 // }
1284 //
1285 // return q;
1286 // }
1287
1288 // Initial estimate of inv(y).
1289 Value *FloatY = Builder.CreateUIToFP(V: Y, DestTy: F32Ty);
1290 Value *RcpY = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_rcp, OverloadTypes: F32Ty, Args: {FloatY});
1291 Constant *Scale = ConstantFP::get(Ty: F32Ty, V: llvm::bit_cast<float>(from: 0x4F7FFFFE));
1292 Value *ScaledY = Builder.CreateFMul(L: RcpY, R: Scale);
1293 Value *Z = Builder.CreateFPToUI(V: ScaledY, DestTy: I32Ty);
1294
1295 // One round of UNR.
1296 Value *NegY = Builder.CreateSub(LHS: Zero, RHS: Y);
1297 Value *NegYZ = Builder.CreateMul(LHS: NegY, RHS: Z);
1298 Z = Builder.CreateAdd(LHS: Z, RHS: getMulHu(Builder, LHS: Z, RHS: NegYZ));
1299
1300 // Quotient/remainder estimate.
1301 Value *Q = getMulHu(Builder, LHS: X, RHS: Z);
1302 Value *R = Builder.CreateSub(LHS: X, RHS: Builder.CreateMul(LHS: Q, RHS: Y));
1303
1304 // First quotient/remainder refinement.
1305 Value *Cond = Builder.CreateICmpUGE(LHS: R, RHS: Y);
1306 if (IsDiv)
1307 Q = Builder.CreateSelect(C: Cond, True: Builder.CreateAdd(LHS: Q, RHS: One), False: Q);
1308 R = Builder.CreateSelect(C: Cond, True: Builder.CreateSub(LHS: R, RHS: Y), False: R);
1309
1310 // Second quotient/remainder refinement.
1311 Cond = Builder.CreateICmpUGE(LHS: R, RHS: Y);
1312 Value *Res;
1313 if (IsDiv)
1314 Res = Builder.CreateSelect(C: Cond, True: Builder.CreateAdd(LHS: Q, RHS: One), False: Q);
1315 else
1316 Res = Builder.CreateSelect(C: Cond, True: Builder.CreateSub(LHS: R, RHS: Y), False: R);
1317
1318 if (IsSigned) {
1319 Res = Builder.CreateXor(LHS: Res, RHS: Sign);
1320 Res = Builder.CreateSub(LHS: Res, RHS: Sign);
1321 Res = Builder.CreateSExtOrTrunc(V: Res, DestTy: Ty);
1322 } else {
1323 Res = Builder.CreateZExtOrTrunc(V: Res, DestTy: Ty);
1324 }
1325 return Res;
1326}
1327
1328Value *AMDGPUCodeGenPrepareImpl::shrinkDivRem64(IRBuilder<> &Builder,
1329 BinaryOperator &I, Value *Num,
1330 Value *Den) const {
1331 if (!ExpandDiv64InIR && divHasSpecialOptimization(I, Num, Den))
1332 return nullptr; // Keep it for later optimization.
1333
1334 Instruction::BinaryOps Opc = I.getOpcode();
1335
1336 bool IsDiv = Opc == Instruction::SDiv || Opc == Instruction::UDiv;
1337 bool IsSigned = Opc == Instruction::SDiv || Opc == Instruction::SRem;
1338
1339 unsigned NumDivBits = getDivNumBits(I, Num, Den, MaxDivBits: 32, IsSigned);
1340 if (NumDivBits > 32)
1341 return nullptr;
1342
1343 Value *Narrowed = nullptr;
1344 if (NumDivBits <= (IsSigned ? 23 : 22)) {
1345 Narrowed = expandDivRemToFloatImpl(Builder, I, Num, Den, DivBits: NumDivBits, IsDiv,
1346 IsSigned);
1347 } else if (NumDivBits <= (IsSigned ? 31 : 32)) {
1348 // Do not use 32-bit division if dividend may be -2147483648.
1349 // Otherwise 32-bit division cannot be used safely.
1350 // -2147483648/1 and -2147483648/-1 are not equal,
1351 // but they produce the same lower 32-bit result.
1352 Narrowed = expandDivRem32(Builder, I, X: Num, Y: Den);
1353 }
1354
1355 if (Narrowed) {
1356 return IsSigned ? Builder.CreateSExt(V: Narrowed, DestTy: Num->getType()) :
1357 Builder.CreateZExt(V: Narrowed, DestTy: Num->getType());
1358 }
1359
1360 return nullptr;
1361}
1362
1363void AMDGPUCodeGenPrepareImpl::expandDivRem64(BinaryOperator &I) const {
1364 Instruction::BinaryOps Opc = I.getOpcode();
1365 // Do the general expansion.
1366 if (Opc == Instruction::UDiv || Opc == Instruction::SDiv) {
1367 expandDivisionUpTo64Bits(Div: &I);
1368 return;
1369 }
1370
1371 if (Opc == Instruction::URem || Opc == Instruction::SRem) {
1372 expandRemainderUpTo64Bits(Rem: &I);
1373 return;
1374 }
1375
1376 llvm_unreachable("not a division");
1377}
1378
1379/*
1380This will cause non-byte load in consistency, for example:
1381```
1382 %load = load i1, ptr addrspace(4) %arg, align 4
1383 %zext = zext i1 %load to
1384 i64 %add = add i64 %zext
1385```
1386Instead of creating `s_and_b32 s0, s0, 1`,
1387it will create `s_and_b32 s0, s0, 0xff`.
1388We accept this change since the non-byte load assumes the upper bits
1389within the byte are all 0.
1390*/
1391bool AMDGPUCodeGenPrepareImpl::tryNarrowMathIfNoOverflow(Instruction *I) {
1392 unsigned Opc = I->getOpcode();
1393 Type *OldType = I->getType();
1394
1395 if (Opc != Instruction::Add && Opc != Instruction::Mul)
1396 return false;
1397
1398 unsigned OrigBit = OldType->getScalarSizeInBits();
1399
1400 if (Opc != Instruction::Add && Opc != Instruction::Mul)
1401 llvm_unreachable("Unexpected opcode, only valid for Instruction::Add and "
1402 "Instruction::Mul.");
1403
1404 unsigned MaxBitsNeeded = computeKnownBits(V: I, DL).countMaxActiveBits();
1405
1406 MaxBitsNeeded = std::max<unsigned>(a: bit_ceil(Value: MaxBitsNeeded), b: 8);
1407 Type *NewType = DL.getSmallestLegalIntType(C&: I->getContext(), Width: MaxBitsNeeded);
1408 if (!NewType)
1409 return false;
1410 unsigned NewBit = NewType->getIntegerBitWidth();
1411 if (NewBit >= OrigBit)
1412 return false;
1413 NewType = I->getType()->getWithNewBitWidth(NewBitWidth: NewBit);
1414
1415 // Old cost
1416 InstructionCost OldCost =
1417 TTI.getArithmeticInstrCost(Opcode: Opc, Ty: OldType, CostKind: TTI::TCK_RecipThroughput);
1418 // New cost of new op
1419 InstructionCost NewCost =
1420 TTI.getArithmeticInstrCost(Opcode: Opc, Ty: NewType, CostKind: TTI::TCK_RecipThroughput);
1421 // New cost of narrowing 2 operands (use trunc)
1422 int NumOfNonConstOps = 2;
1423 if (isa<Constant>(Val: I->getOperand(i: 0)) || isa<Constant>(Val: I->getOperand(i: 1))) {
1424 // Cannot be both constant, should be propagated
1425 NumOfNonConstOps = 1;
1426 }
1427 NewCost += NumOfNonConstOps * TTI.getCastInstrCost(Opcode: Instruction::Trunc,
1428 Dst: NewType, Src: OldType,
1429 CCH: TTI.getCastContextHint(I),
1430 CostKind: TTI::TCK_RecipThroughput);
1431 // New cost of zext narrowed result to original type
1432 NewCost +=
1433 TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: OldType, Src: NewType,
1434 CCH: TTI.getCastContextHint(I), CostKind: TTI::TCK_RecipThroughput);
1435 if (NewCost >= OldCost)
1436 return false;
1437
1438 IRBuilder<> Builder(I);
1439 Value *Trunc0 = Builder.CreateTrunc(V: I->getOperand(i: 0), DestTy: NewType);
1440 Value *Trunc1 = Builder.CreateTrunc(V: I->getOperand(i: 1), DestTy: NewType);
1441 Value *Arith =
1442 Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Opc, LHS: Trunc0, RHS: Trunc1);
1443
1444 Value *Zext = Builder.CreateZExt(V: Arith, DestTy: OldType);
1445 I->replaceAllUsesWith(V: Zext);
1446 DeadVals.push_back(Elt: I);
1447 return true;
1448}
1449
1450bool AMDGPUCodeGenPrepareImpl::visitBinaryOperator(BinaryOperator &I) {
1451 if (foldBinOpIntoSelect(BO&: I))
1452 return true;
1453
1454 if (UseMul24Intrin && replaceMulWithMul24(I))
1455 return true;
1456 if (tryNarrowMathIfNoOverflow(I: &I))
1457 return true;
1458
1459 bool Changed = false;
1460 Instruction::BinaryOps Opc = I.getOpcode();
1461 Type *Ty = I.getType();
1462 Value *NewDiv = nullptr;
1463 unsigned ScalarSize = Ty->getScalarSizeInBits();
1464
1465 SmallVector<BinaryOperator *, 8> Div64ToExpand;
1466
1467 if ((Opc == Instruction::URem || Opc == Instruction::UDiv ||
1468 Opc == Instruction::SRem || Opc == Instruction::SDiv) &&
1469 ScalarSize <= 64 &&
1470 !DisableIDivExpand) {
1471 Value *Num = I.getOperand(i_nocapture: 0);
1472 Value *Den = I.getOperand(i_nocapture: 1);
1473 IRBuilder<> Builder(&I);
1474 Builder.SetCurrentDebugLocation(I.getDebugLoc());
1475
1476 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1477 NewDiv = PoisonValue::get(T: VT);
1478
1479 for (unsigned N = 0, E = VT->getNumElements(); N != E; ++N) {
1480 Value *NumEltN = Builder.CreateExtractElement(Vec: Num, Idx: N);
1481 Value *DenEltN = Builder.CreateExtractElement(Vec: Den, Idx: N);
1482
1483 Value *NewElt;
1484 if (ScalarSize <= 32) {
1485 NewElt = expandDivRem32(Builder, I, X: NumEltN, Y: DenEltN);
1486 if (!NewElt)
1487 NewElt = Builder.CreateBinOp(Opc, LHS: NumEltN, RHS: DenEltN);
1488 } else {
1489 // See if this 64-bit division can be shrunk to 32/24-bits before
1490 // producing the general expansion.
1491 NewElt = shrinkDivRem64(Builder, I, Num: NumEltN, Den: DenEltN);
1492 if (!NewElt) {
1493 // The general 64-bit expansion introduces control flow and doesn't
1494 // return the new value. Just insert a scalar copy and defer
1495 // expanding it.
1496 NewElt = Builder.CreateBinOp(Opc, LHS: NumEltN, RHS: DenEltN);
1497 // CreateBinOp does constant folding. If the operands are constant,
1498 // it will return a Constant instead of a BinaryOperator.
1499 if (auto *NewEltBO = dyn_cast<BinaryOperator>(Val: NewElt))
1500 Div64ToExpand.push_back(Elt: NewEltBO);
1501 }
1502 }
1503
1504 if (auto *NewEltI = dyn_cast<Instruction>(Val: NewElt))
1505 NewEltI->copyIRFlags(V: &I);
1506
1507 NewDiv = Builder.CreateInsertElement(Vec: NewDiv, NewElt, Idx: N);
1508 }
1509 } else {
1510 if (ScalarSize <= 32)
1511 NewDiv = expandDivRem32(Builder, I, X: Num, Y: Den);
1512 else {
1513 NewDiv = shrinkDivRem64(Builder, I, Num, Den);
1514 if (!NewDiv)
1515 Div64ToExpand.push_back(Elt: &I);
1516 }
1517 }
1518
1519 if (NewDiv) {
1520 I.replaceAllUsesWith(V: NewDiv);
1521 DeadVals.push_back(Elt: &I);
1522 Changed = true;
1523 }
1524 }
1525
1526 if (ExpandDiv64InIR) {
1527 // TODO: We get much worse code in specially handled constant cases.
1528 for (BinaryOperator *Div : Div64ToExpand) {
1529 expandDivRem64(I&: *Div);
1530 FlowChanged = true;
1531 Changed = true;
1532 }
1533 }
1534
1535 return Changed;
1536}
1537
1538bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) {
1539 if (!WidenLoads)
1540 return false;
1541
1542 if ((I.getPointerAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
1543 I.getPointerAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
1544 canWidenScalarExtLoad(I)) {
1545 IRBuilder<> Builder(&I);
1546 Builder.SetCurrentDebugLocation(I.getDebugLoc());
1547
1548 Type *I32Ty = Builder.getInt32Ty();
1549 LoadInst *WidenLoad = Builder.CreateLoad(Ty: I32Ty, Ptr: I.getPointerOperand());
1550 AMDGPU::copyMetadataForWidenedLoad(Dest&: *WidenLoad, Source: I);
1551
1552 // The widened load reads the original bytes in the low bits, so a !range
1553 // lower bound still holds. Convert it to the new type and don't make
1554 // assumptions about the high bits.
1555 if (auto *Range = I.getMetadata(KindID: LLVMContext::MD_range)) {
1556 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD: Range->getOperand(I: 0));
1557
1558 if (!Lower->isNullValue()) {
1559 Metadata *LowAndHigh[] = {
1560 ConstantAsMetadata::get(C: ConstantInt::get(Ty: I32Ty, V: Lower->getValue().zext(width: 32))),
1561 // Don't make assumptions about the high bits.
1562 ConstantAsMetadata::get(C: ConstantInt::get(Ty: I32Ty, V: 0))
1563 };
1564
1565 WidenLoad->setMetadata(KindID: LLVMContext::MD_range,
1566 Node: MDNode::get(Context&: F.getContext(), MDs: LowAndHigh));
1567 }
1568 }
1569
1570 int TySize = DL.getTypeSizeInBits(Ty: I.getType());
1571 Type *IntNTy = Builder.getIntNTy(N: TySize);
1572 Value *ValTrunc = Builder.CreateTrunc(V: WidenLoad, DestTy: IntNTy);
1573 Value *ValOrig = Builder.CreateBitCast(V: ValTrunc, DestTy: I.getType());
1574 I.replaceAllUsesWith(V: ValOrig);
1575 DeadVals.push_back(Elt: &I);
1576 return true;
1577 }
1578
1579 return false;
1580}
1581
1582bool AMDGPUCodeGenPrepareImpl::visitSelectInst(SelectInst &I) {
1583 FPMathOperator *FPOp = dyn_cast<FPMathOperator>(Val: &I);
1584 if (!FPOp)
1585 return false;
1586
1587 Value *X;
1588 Value *Fract = nullptr;
1589
1590 // Match:
1591 // (x - floor(x)) >= MIN_CONSTANT ? MIN_CONSTANT : (x - floor(x))
1592 //
1593 // This is the preferred way to implement fract.
1594 // TODO: Could also match with compare against 1.0
1595 const APFloat *C;
1596 if (match(V: &I, P: m_UnordFMin(L: m_Value(V&: X), R: m_APFloatAllowPoison(Res&: C)))) {
1597 Value *FractSrc = matchFractPatImpl(V&: *X, C: *C);
1598 if (!FractSrc)
1599 return false;
1600 IRBuilder<> Builder(&I);
1601 Builder.setFastMathFlags(FPOp->getFastMathFlags());
1602 Fract = applyFractPat(Builder, FractArg: FractSrc);
1603 } else {
1604 // Match patterns which may appear in legacy implementations of the fract()
1605 // function, built around the nan-avoidant minnum intrinsic. These are the
1606 // core pattern plus additional clamping of inf and nan values on the
1607 // result.
1608 Value *Cond = I.getCondition();
1609 Value *TrueVal = I.getTrueValue();
1610 Value *FalseVal = I.getFalseValue();
1611 Value *CmpVal;
1612 CmpPredicate IsNanPred;
1613
1614 // Match fract pattern with nan check.
1615 if (!match(V: Cond, P: m_FCmp(Pred&: IsNanPred, L: m_Value(V&: CmpVal), R: m_NonNaN())))
1616 return false;
1617
1618 IRBuilder<> Builder(&I);
1619 Builder.setFastMathFlags(FPOp->getFastMathFlags());
1620
1621 if (IsNanPred == FCmpInst::FCMP_UNO && TrueVal == CmpVal &&
1622 CmpVal == matchFractPatNanAvoidant(V&: *FalseVal)) {
1623 // isnan(x) ? x : fract(x)
1624 Fract = applyFractPat(Builder, FractArg: CmpVal);
1625 } else if (IsNanPred == FCmpInst::FCMP_ORD && FalseVal == CmpVal) {
1626 if (CmpVal == matchFractPatNanAvoidant(V&: *TrueVal)) {
1627 // !isnan(x) ? fract(x) : x
1628 Fract = applyFractPat(Builder, FractArg: CmpVal);
1629 } else {
1630 // Match an intermediate clamp infinity to 0 pattern. i.e.
1631 // !isnan(x) ? (!isinf(x) ? fract(x) : 0.0) : x
1632 CmpPredicate PredInf;
1633 Value *IfNotInf;
1634
1635 if (!match(V: TrueVal, P: m_Select(C: m_FCmp(Pred&: PredInf, L: m_FAbs(Op0: m_Specific(V: CmpVal)),
1636 R: m_PosInf()),
1637 L: m_Value(V&: IfNotInf), R: m_PosZeroFP())) ||
1638 PredInf != FCmpInst::FCMP_UNE ||
1639 CmpVal != matchFractPatNanAvoidant(V&: *IfNotInf))
1640 return false;
1641
1642 SelectInst *ClampInfSelect = cast<SelectInst>(Val: TrueVal);
1643
1644 // Insert before the fabs
1645 Value *InsertPt =
1646 cast<Instruction>(Val: ClampInfSelect->getCondition())->getOperand(i: 0);
1647
1648 Builder.SetInsertPoint(cast<Instruction>(Val: InsertPt));
1649 Value *NewFract = applyFractPat(Builder, FractArg: CmpVal);
1650 NewFract->takeName(V: TrueVal);
1651
1652 // Thread the new fract into the inf clamping sequence.
1653 DeadVals.push_back(Elt: ClampInfSelect->getOperand(i_nocapture: 1));
1654 ClampInfSelect->setOperand(i_nocapture: 1, Val_nocapture: NewFract);
1655
1656 // The outer select nan handling is also absorbed into the fract.
1657 Fract = ClampInfSelect;
1658 }
1659 } else
1660 return false;
1661 }
1662
1663 Fract->takeName(V: &I);
1664 I.replaceAllUsesWith(V: Fract);
1665 DeadVals.push_back(Elt: &I);
1666 return true;
1667}
1668
1669static bool areInSameBB(const Value *A, const Value *B) {
1670 const auto *IA = dyn_cast<Instruction>(Val: A);
1671 const auto *IB = dyn_cast<Instruction>(Val: B);
1672 return IA && IB && IA->getParent() == IB->getParent();
1673}
1674
1675// Helper for breaking large PHIs that returns true when an extractelement on V
1676// is likely to be folded away by the DAG combiner.
1677static bool isInterestingPHIIncomingValue(const Value *V) {
1678 const auto *FVT = dyn_cast<FixedVectorType>(Val: V->getType());
1679 if (!FVT)
1680 return false;
1681
1682 const Value *CurVal = V;
1683
1684 // Check for insertelements, keeping track of the elements covered.
1685 BitVector EltsCovered(FVT->getNumElements());
1686 while (const auto *IE = dyn_cast<InsertElementInst>(Val: CurVal)) {
1687 const auto *Idx = dyn_cast<ConstantInt>(Val: IE->getOperand(i_nocapture: 2));
1688
1689 // Non constant index/out of bounds index -> folding is unlikely.
1690 // The latter is more of a sanity check because canonical IR should just
1691 // have replaced those with poison.
1692 if (!Idx || Idx->getZExtValue() >= FVT->getNumElements())
1693 return false;
1694
1695 const auto *VecSrc = IE->getOperand(i_nocapture: 0);
1696
1697 // If the vector source is another instruction, it must be in the same basic
1698 // block. Otherwise, the DAGCombiner won't see the whole thing and is
1699 // unlikely to be able to do anything interesting here.
1700 if (isa<Instruction>(Val: VecSrc) && !areInSameBB(A: VecSrc, B: IE))
1701 return false;
1702
1703 CurVal = VecSrc;
1704 EltsCovered.set(Idx->getZExtValue());
1705
1706 // All elements covered.
1707 if (EltsCovered.all())
1708 return true;
1709 }
1710
1711 // We either didn't find a single insertelement, or the insertelement chain
1712 // ended before all elements were covered. Check for other interesting values.
1713
1714 // Constants are always interesting because we can just constant fold the
1715 // extractelements.
1716 if (isa<Constant>(Val: CurVal))
1717 return true;
1718
1719 // shufflevector is likely to be profitable if either operand is a constant,
1720 // or if either source is in the same block.
1721 // This is because shufflevector is most often lowered as a series of
1722 // insert/extract elements anyway.
1723 if (const auto *SV = dyn_cast<ShuffleVectorInst>(Val: CurVal)) {
1724 return isa<Constant>(Val: SV->getOperand(i_nocapture: 1)) ||
1725 areInSameBB(A: SV, B: SV->getOperand(i_nocapture: 0)) ||
1726 areInSameBB(A: SV, B: SV->getOperand(i_nocapture: 1));
1727 }
1728
1729 return false;
1730}
1731
1732static void collectPHINodes(const PHINode &I,
1733 SmallPtrSet<const PHINode *, 8> &SeenPHIs) {
1734 const auto [It, Inserted] = SeenPHIs.insert(Ptr: &I);
1735 if (!Inserted)
1736 return;
1737
1738 for (const Value *Inc : I.incoming_values()) {
1739 if (const auto *PhiInc = dyn_cast<PHINode>(Val: Inc))
1740 collectPHINodes(I: *PhiInc, SeenPHIs);
1741 }
1742
1743 for (const User *U : I.users()) {
1744 if (const auto *PhiU = dyn_cast<PHINode>(Val: U))
1745 collectPHINodes(I: *PhiU, SeenPHIs);
1746 }
1747}
1748
1749bool AMDGPUCodeGenPrepareImpl::canBreakPHINode(const PHINode &I) {
1750 // Check in the cache first.
1751 if (const auto It = BreakPhiNodesCache.find(Val: &I);
1752 It != BreakPhiNodesCache.end())
1753 return It->second;
1754
1755 // We consider PHI nodes as part of "chains", so given a PHI node I, we
1756 // recursively consider all its users and incoming values that are also PHI
1757 // nodes. We then make a decision about all of those PHIs at once. Either they
1758 // all get broken up, or none of them do. That way, we avoid cases where a
1759 // single PHI is/is not broken and we end up reforming/exploding a vector
1760 // multiple times, or even worse, doing it in a loop.
1761 SmallPtrSet<const PHINode *, 8> WorkList;
1762 collectPHINodes(I, SeenPHIs&: WorkList);
1763
1764#ifndef NDEBUG
1765 // Check that none of the PHI nodes in the worklist are in the map. If some of
1766 // them are, it means we're not good enough at collecting related PHIs.
1767 for (const PHINode *WLP : WorkList) {
1768 assert(BreakPhiNodesCache.count(WLP) == 0);
1769 }
1770#endif
1771
1772 // To consider a PHI profitable to break, we need to see some interesting
1773 // incoming values. At least 2/3rd (rounded up) of all PHIs in the worklist
1774 // must have one to consider all PHIs breakable.
1775 //
1776 // This threshold has been determined through performance testing.
1777 //
1778 // Note that the computation below is equivalent to
1779 //
1780 // (unsigned)ceil((K / 3.0) * 2)
1781 //
1782 // It's simply written this way to avoid mixing integral/FP arithmetic.
1783 const auto Threshold = (alignTo(Value: WorkList.size() * 2, Align: 3) / 3);
1784 unsigned NumBreakablePHIs = 0;
1785 bool CanBreak = false;
1786 for (const PHINode *Cur : WorkList) {
1787 // Don't break PHIs that have no interesting incoming values. That is, where
1788 // there is no clear opportunity to fold the "extractelement" instructions
1789 // we would add.
1790 //
1791 // Note: IC does not run after this pass, so we're only interested in the
1792 // foldings that the DAG combiner can do.
1793 if (any_of(Range: Cur->incoming_values(), P: isInterestingPHIIncomingValue)) {
1794 if (++NumBreakablePHIs >= Threshold) {
1795 CanBreak = true;
1796 break;
1797 }
1798 }
1799 }
1800
1801 for (const PHINode *Cur : WorkList)
1802 BreakPhiNodesCache[Cur] = CanBreak;
1803
1804 return CanBreak;
1805}
1806
1807/// Helper class for "break large PHIs" (visitPHINode).
1808///
1809/// This represents a slice of a PHI's incoming value, which is made up of:
1810/// - The type of the slice (Ty)
1811/// - The index in the incoming value's vector where the slice starts (Idx)
1812/// - The number of elements in the slice (NumElts).
1813/// It also keeps track of the NewPHI node inserted for this particular slice.
1814///
1815/// Slice examples:
1816/// <4 x i64> -> Split into four i64 slices.
1817/// -> [i64, 0, 1], [i64, 1, 1], [i64, 2, 1], [i64, 3, 1]
1818/// <5 x i16> -> Split into 2 <2 x i16> slices + a i16 tail.
1819/// -> [<2 x i16>, 0, 2], [<2 x i16>, 2, 2], [i16, 4, 1]
1820class VectorSlice {
1821public:
1822 VectorSlice(Type *Ty, unsigned Idx, unsigned NumElts)
1823 : Ty(Ty), Idx(Idx), NumElts(NumElts) {}
1824
1825 Type *Ty = nullptr;
1826 unsigned Idx = 0;
1827 unsigned NumElts = 0;
1828 PHINode *NewPHI = nullptr;
1829
1830 /// Slice \p Inc according to the information contained within this slice.
1831 /// This is cached, so if called multiple times for the same \p BB & \p Inc
1832 /// pair, it returns the same Sliced value as well.
1833 ///
1834 /// Note this *intentionally* does not return the same value for, say,
1835 /// [%bb.0, %0] & [%bb.1, %0] as:
1836 /// - It could cause issues with dominance (e.g. if bb.1 is seen first, then
1837 /// the value in bb.1 may not be reachable from bb.0 if it's its
1838 /// predecessor.)
1839 /// - We also want to make our extract instructions as local as possible so
1840 /// the DAG has better chances of folding them out. Duplicating them like
1841 /// that is beneficial in that regard.
1842 ///
1843 /// This is both a minor optimization to avoid creating duplicate
1844 /// instructions, but also a requirement for correctness. It is not forbidden
1845 /// for a PHI node to have the same [BB, Val] pair multiple times. If we
1846 /// returned a new value each time, those previously identical pairs would all
1847 /// have different incoming values (from the same block) and it'd cause a "PHI
1848 /// node has multiple entries for the same basic block with different incoming
1849 /// values!" verifier error.
1850 Value *getSlicedVal(BasicBlock *BB, Value *Inc, StringRef NewValName) {
1851 Value *&Res = SlicedVals[{BB, Inc}];
1852 if (Res)
1853 return Res;
1854
1855 IRBuilder<> B(BB->getTerminator());
1856 if (Instruction *IncInst = dyn_cast<Instruction>(Val: Inc))
1857 B.SetCurrentDebugLocation(IncInst->getDebugLoc());
1858
1859 if (NumElts > 1) {
1860 SmallVector<int, 4> Mask;
1861 for (unsigned K = Idx; K < (Idx + NumElts); ++K)
1862 Mask.push_back(Elt: K);
1863 Res = B.CreateShuffleVector(V: Inc, Mask, Name: NewValName);
1864 } else
1865 Res = B.CreateExtractElement(Vec: Inc, Idx, Name: NewValName);
1866
1867 return Res;
1868 }
1869
1870private:
1871 SmallDenseMap<std::pair<BasicBlock *, Value *>, Value *> SlicedVals;
1872};
1873
1874bool AMDGPUCodeGenPrepareImpl::visitPHINode(PHINode &I) {
1875 // Break-up fixed-vector PHIs into smaller pieces.
1876 // Default threshold is 32, so it breaks up any vector that's >32 bits into
1877 // its elements, or into 32-bit pieces (for 8/16 bit elts).
1878 //
1879 // This is only helpful for DAGISel because it doesn't handle large PHIs as
1880 // well as GlobalISel. DAGISel lowers PHIs by using CopyToReg/CopyFromReg.
1881 // With large, odd-sized PHIs we may end up needing many `build_vector`
1882 // operations with most elements being "undef". This inhibits a lot of
1883 // optimization opportunities and can result in unreasonably high register
1884 // pressure and the inevitable stack spilling.
1885 if (!BreakLargePHIs || getCGPassBuilderOption().EnableGlobalISelOption ==
1886 cl::boolOrDefault::BOU_TRUE)
1887 return false;
1888
1889 FixedVectorType *FVT = dyn_cast<FixedVectorType>(Val: I.getType());
1890 if (!FVT || FVT->getNumElements() == 1 ||
1891 DL.getTypeSizeInBits(Ty: FVT) <= BreakLargePHIsThreshold)
1892 return false;
1893
1894 if (!ForceBreakLargePHIs && !canBreakPHINode(I))
1895 return false;
1896
1897 std::vector<VectorSlice> Slices;
1898
1899 Type *EltTy = FVT->getElementType();
1900 {
1901 unsigned Idx = 0;
1902 // For 8/16 bits type, don't scalarize fully but break it up into as many
1903 // 32-bit slices as we can, and scalarize the tail.
1904 const unsigned EltSize = DL.getTypeSizeInBits(Ty: EltTy);
1905 const unsigned NumElts = FVT->getNumElements();
1906 if (EltSize == 8 || EltSize == 16) {
1907 const unsigned SubVecSize = (32 / EltSize);
1908 Type *SubVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: SubVecSize);
1909 for (unsigned End = alignDown(Value: NumElts, Align: SubVecSize); Idx < End;
1910 Idx += SubVecSize)
1911 Slices.emplace_back(args&: SubVecTy, args&: Idx, args: SubVecSize);
1912 }
1913
1914 // Scalarize all remaining elements.
1915 for (; Idx < NumElts; ++Idx)
1916 Slices.emplace_back(args&: EltTy, args&: Idx, args: 1);
1917 }
1918
1919 assert(Slices.size() > 1);
1920
1921 // Create one PHI per vector piece. The "VectorSlice" class takes care of
1922 // creating the necessary instruction to extract the relevant slices of each
1923 // incoming value.
1924 IRBuilder<> B(I.getParent());
1925 B.SetCurrentDebugLocation(I.getDebugLoc());
1926
1927 unsigned IncNameSuffix = 0;
1928 for (VectorSlice &S : Slices) {
1929 // We need to reset the build on each iteration, because getSlicedVal may
1930 // have inserted something into I's BB.
1931 B.SetInsertPoint(I.getParent()->getFirstNonPHIIt());
1932 S.NewPHI = B.CreatePHI(Ty: S.Ty, NumReservedValues: I.getNumIncomingValues());
1933
1934 for (const auto &[Idx, BB] : enumerate(First: I.blocks())) {
1935 S.NewPHI->addIncoming(V: S.getSlicedVal(BB, Inc: I.getIncomingValue(i: Idx),
1936 NewValName: "largephi.extractslice" +
1937 std::to_string(val: IncNameSuffix++)),
1938 BB);
1939 }
1940 }
1941
1942 // And replace this PHI with a vector of all the previous PHI values.
1943 Value *Vec = PoisonValue::get(T: FVT);
1944 unsigned NameSuffix = 0;
1945 for (VectorSlice &S : Slices) {
1946 const auto ValName = "largephi.insertslice" + std::to_string(val: NameSuffix++);
1947 if (S.NumElts > 1)
1948 Vec = B.CreateInsertVector(DstType: FVT, SrcVec: Vec, SubVec: S.NewPHI, Idx: S.Idx, Name: ValName);
1949 else
1950 Vec = B.CreateInsertElement(Vec, NewElt: S.NewPHI, Idx: S.Idx, Name: ValName);
1951 }
1952
1953 I.replaceAllUsesWith(V: Vec);
1954 DeadVals.push_back(Elt: &I);
1955 return true;
1956}
1957
1958/// \param V Value to check
1959/// \param DL DataLayout
1960/// \param TM TargetMachine (TODO: remove once DL contains nullptr values)
1961/// \param AS Target Address Space
1962/// \return true if \p V cannot be the null value of \p AS, false otherwise.
1963static bool isPtrKnownNeverNull(const Value *V, const DataLayout &DL,
1964 const AMDGPUTargetMachine &TM, unsigned AS) {
1965 // Pointer cannot be null if it's a block address, GV or alloca.
1966 // NOTE: We don't support extern_weak, but if we did, we'd need to check for
1967 // it as the symbol could be null in such cases.
1968 if (isa<BlockAddress, GlobalValue, AllocaInst>(Val: V))
1969 return true;
1970
1971 // Check nonnull arguments.
1972 if (const auto *Arg = dyn_cast<Argument>(Val: V); Arg && Arg->hasNonNullAttr())
1973 return true;
1974
1975 // Check nonnull loads.
1976 if (const auto *Load = dyn_cast<LoadInst>(Val: V);
1977 Load && Load->hasMetadata(KindID: LLVMContext::MD_nonnull))
1978 return true;
1979
1980 // getUnderlyingObject may have looked through another addrspacecast, although
1981 // the optimizable situations most likely folded out by now.
1982 if (AS != cast<PointerType>(Val: V->getType())->getAddressSpace())
1983 return false;
1984
1985 // TODO: Calls that return nonnull?
1986
1987 // For all other things, use KnownBits.
1988 // We either use 0 or all bits set to indicate null, so check whether the
1989 // value can be zero or all ones.
1990 //
1991 // TODO: Use ValueTracking's isKnownNeverNull if it becomes aware that some
1992 // address spaces have non-zero null values.
1993 auto SrcPtrKB = computeKnownBits(V, DL);
1994 const auto NullVal = AMDGPU::getNullPointerValue(AS);
1995
1996 assert(SrcPtrKB.getBitWidth() == DL.getPointerSizeInBits(AS));
1997 assert((NullVal == 0 || NullVal == -1) &&
1998 "don't know how to check for this null value!");
1999 return NullVal ? !SrcPtrKB.getMaxValue().isAllOnes() : SrcPtrKB.isNonZero();
2000}
2001
2002bool AMDGPUCodeGenPrepareImpl::visitAddrSpaceCastInst(AddrSpaceCastInst &I) {
2003 // Intrinsic doesn't support vectors, also it seems that it's often difficult
2004 // to prove that a vector cannot have any nulls in it so it's unclear if it's
2005 // worth supporting.
2006 if (I.getType()->isVectorTy())
2007 return false;
2008
2009 // Check if this can be lowered to a amdgcn.addrspacecast.nonnull.
2010 // This is only worthwhile for casts from/to priv/local to flat.
2011 const unsigned SrcAS = I.getSrcAddressSpace();
2012 const unsigned DstAS = I.getDestAddressSpace();
2013
2014 bool CanLower = false;
2015 if (SrcAS == AMDGPUAS::FLAT_ADDRESS)
2016 CanLower = (DstAS == AMDGPUAS::LOCAL_ADDRESS ||
2017 DstAS == AMDGPUAS::PRIVATE_ADDRESS);
2018 else if (DstAS == AMDGPUAS::FLAT_ADDRESS)
2019 CanLower = (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
2020 SrcAS == AMDGPUAS::PRIVATE_ADDRESS);
2021 if (!CanLower)
2022 return false;
2023
2024 SmallVector<const Value *, 4> WorkList;
2025 getUnderlyingObjects(V: I.getOperand(i_nocapture: 0), Objects&: WorkList);
2026 if (!all_of(Range&: WorkList, P: [&](const Value *V) {
2027 return isPtrKnownNeverNull(V, DL, TM, AS: SrcAS);
2028 }))
2029 return false;
2030
2031 IRBuilder<> B(&I);
2032 auto *Intrin = B.CreateIntrinsic(
2033 RetTy: I.getType(), ID: Intrinsic::amdgcn_addrspacecast_nonnull, Args: {I.getOperand(i_nocapture: 0)});
2034 I.replaceAllUsesWith(V: Intrin);
2035 DeadVals.push_back(Elt: &I);
2036 return true;
2037}
2038
2039bool AMDGPUCodeGenPrepareImpl::visitIntrinsicInst(IntrinsicInst &I) {
2040 Intrinsic::ID IID = I.getIntrinsicID();
2041 switch (IID) {
2042 case Intrinsic::minnum:
2043 case Intrinsic::minimumnum:
2044 case Intrinsic::minimum:
2045 return visitFMinLike(I);
2046 case Intrinsic::sqrt:
2047 return visitSqrt(I);
2048 case Intrinsic::log:
2049 case Intrinsic::log10:
2050 return visitLog(Log&: cast<FPMathOperator>(Val&: I), IID);
2051 case Intrinsic::log2:
2052 // No reason to handle log2.
2053 return false;
2054 case Intrinsic::amdgcn_mbcnt_lo:
2055 return visitMbcntLo(I);
2056 case Intrinsic::amdgcn_mbcnt_hi:
2057 return visitMbcntHi(I);
2058 case Intrinsic::vector_reduce_add:
2059 return visitVectorReduceAdd(I);
2060 case Intrinsic::uadd_sat:
2061 case Intrinsic::sadd_sat:
2062 return visitSaturatingAdd(I);
2063 default:
2064 return false;
2065 }
2066}
2067
2068/// Match the core sequence in the fract pattern (x - floor(x), which doesn't
2069/// need to consider edge case handling.
2070Value *AMDGPUCodeGenPrepareImpl::matchFractPatImpl(Value &FractSrc,
2071 const APFloat &C) const {
2072 if (ST.hasFractBug())
2073 return nullptr;
2074
2075 Type *Ty = FractSrc.getType();
2076 if (!isLegalFloatingTy(Ty: Ty->getScalarType()))
2077 return nullptr;
2078
2079 APFloat OneNextDown = APFloat::getOne(Sem: C.getSemantics());
2080 OneNextDown.next(nextDown: true);
2081
2082 // Match nextafter(1.0, -1)
2083 if (OneNextDown != C)
2084 return nullptr;
2085
2086 Value *FloorSrc;
2087 if (match(V: &FractSrc, P: m_FSub(L: m_Value(V&: FloorSrc), R: m_Intrinsic<Intrinsic::floor>(
2088 Ops: m_Deferred(V: FloorSrc)))))
2089 return FloorSrc;
2090 return nullptr;
2091}
2092
2093/// Match non-nan fract pattern.
2094// MIN_CONSTANT = nextafter(1.0, -1.0)
2095/// minnum(fsub(x, floor(x)), MIN_CONSTANT)
2096/// minimumnum(fsub(x, floor(x)), MIN_CONSTANT)
2097/// minimum(fsub(x, floor(x)), MIN_CONSTANT)
2098
2099// x_sub_floor >= MIN_CONSTANT ? MIN_CONSTANT : x_sub_floor;
2100///
2101/// If fract is a useful instruction for the subtarget. Does not account for the
2102/// nan handling; the instruction has a nan check on the input value.
2103Value *AMDGPUCodeGenPrepareImpl::matchFractPatNanAvoidant(Value &V) {
2104 Value *Arg0;
2105 const APFloat *C;
2106
2107 // The value is only used in contexts where we know the input isn't a nan, so
2108 // any of the fmin variants are fine.
2109 if (!match(V: &V,
2110 P: m_CombineOr(Ps: m_FMinNum_or_FMinimumNum(Op0: m_Value(V&: Arg0),
2111 Op1: m_APFloatAllowPoison(Res&: C)),
2112 Ps: m_FMinimum(Op0: m_Value(V&: Arg0), Op1: m_APFloatAllowPoison(Res&: C)))))
2113 return nullptr;
2114
2115 return matchFractPatImpl(FractSrc&: *Arg0, C: *C);
2116}
2117
2118Value *AMDGPUCodeGenPrepareImpl::applyFractPat(IRBuilder<> &Builder,
2119 Value *FractArg) {
2120 SmallVector<Value *, 4> FractVals;
2121 extractValues(Builder, Values&: FractVals, V: FractArg);
2122
2123 SmallVector<Value *, 4> ResultVals(FractVals.size());
2124
2125 Type *Ty = FractArg->getType()->getScalarType();
2126 for (unsigned I = 0, E = FractVals.size(); I != E; ++I) {
2127 ResultVals[I] =
2128 Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_fract, OverloadTypes: {Ty}, Args: {FractVals[I]});
2129 }
2130
2131 return insertValues(Builder, Ty: FractArg->getType(), Values&: ResultVals);
2132}
2133
2134bool AMDGPUCodeGenPrepareImpl::visitFMinLike(IntrinsicInst &I) {
2135 const APFloat *C;
2136 Value *FractArg;
2137
2138 // minimum(x - floor(x), MIN_CONSTANT)
2139 Value *X;
2140 if (!ST.hasFractBug() &&
2141 match(V: &I, P: m_FMinimum(Op0: m_Value(V&: X), Op1: m_APFloatAllowPoison(Res&: C)))) {
2142 FractArg = matchFractPatImpl(FractSrc&: *X, C: *C);
2143 if (!FractArg)
2144 return false;
2145 } else {
2146 // minnum(x - floor(x), MIN_CONSTANT)
2147 FractArg = matchFractPatNanAvoidant(V&: I);
2148 if (!FractArg)
2149 return false;
2150
2151 // Match pattern for fract intrinsic in contexts where the nan check has
2152 // been optimized out (and hope the knowledge the source can't be nan wasn't
2153 // lost).
2154 if (!I.hasNoNaNs() && !isKnownNeverNaN(V: FractArg, SQ: SQ.getWithInstruction(I: &I)))
2155 return false;
2156 }
2157
2158 IRBuilder<> Builder(&I);
2159 FastMathFlags FMF = I.getFastMathFlags();
2160 FMF.setNoNaNs();
2161 Builder.setFastMathFlags(FMF);
2162
2163 Value *Fract = applyFractPat(Builder, FractArg);
2164 Fract->takeName(V: &I);
2165 I.replaceAllUsesWith(V: Fract);
2166 DeadVals.push_back(Elt: &I);
2167 return true;
2168}
2169
2170// Expand llvm.sqrt.f32 calls with !fpmath metadata in a semi-fast way.
2171bool AMDGPUCodeGenPrepareImpl::visitSqrt(IntrinsicInst &Sqrt) {
2172 Type *Ty = Sqrt.getType()->getScalarType();
2173 if (!Ty->isFloatTy())
2174 return false;
2175
2176 const FPMathOperator *FPOp = cast<const FPMathOperator>(Val: &Sqrt);
2177 FastMathFlags SqrtFMF = FPOp->getFastMathFlags();
2178
2179 // We're trying to handle the fast-but-not-that-fast case only. The lowering
2180 // of fast llvm.sqrt will give the raw instruction anyway.
2181 if (SqrtFMF.approxFunc())
2182 return false;
2183
2184 const float ReqdAccuracy = FPOp->getFPAccuracy();
2185
2186 // Defer correctly rounded expansion to codegen.
2187 if (ReqdAccuracy < 1.0f)
2188 return false;
2189
2190 Value *SrcVal = Sqrt.getOperand(i_nocapture: 0);
2191 bool CanTreatAsDAZ = canIgnoreDenormalInput(V: SrcVal, CtxI: &Sqrt);
2192
2193 // The raw instruction is 1 ulp, but the correction for denormal handling
2194 // brings it to 2.
2195 if (!CanTreatAsDAZ && ReqdAccuracy < 2.0f)
2196 return false;
2197
2198 IRBuilder<> Builder(&Sqrt);
2199 SmallVector<Value *, 4> SrcVals;
2200 extractValues(Builder, Values&: SrcVals, V: SrcVal);
2201
2202 SmallVector<Value *, 4> ResultVals(SrcVals.size());
2203 for (int I = 0, E = SrcVals.size(); I != E; ++I) {
2204 if (CanTreatAsDAZ)
2205 ResultVals[I] = Builder.CreateCall(Callee: getSqrtF32(), Args: SrcVals[I]);
2206 else
2207 ResultVals[I] = emitSqrtIEEE2ULP(Builder, Src: SrcVals[I], FMF: SqrtFMF);
2208 }
2209
2210 Value *NewSqrt = insertValues(Builder, Ty: Sqrt.getType(), Values&: ResultVals);
2211 NewSqrt->takeName(V: &Sqrt);
2212 Sqrt.replaceAllUsesWith(V: NewSqrt);
2213 DeadVals.push_back(Elt: &Sqrt);
2214 return true;
2215}
2216
2217/// Replace log and log10 intrinsic calls based on fpmath metadata.
2218bool AMDGPUCodeGenPrepareImpl::visitLog(FPMathOperator &Log,
2219 Intrinsic::ID IID) {
2220 Type *Ty = Log.getType();
2221 if (!Ty->getScalarType()->isHalfTy() || !ST.has16BitInsts())
2222 return false;
2223
2224 FastMathFlags FMF = Log.getFastMathFlags();
2225
2226 // Defer fast math cases to codegen.
2227 if (FMF.approxFunc())
2228 return false;
2229
2230 // Limit experimentally determined from OpenCL conformance test (1.79)
2231 if (Log.getFPAccuracy() < 1.80f)
2232 return false;
2233
2234 IRBuilder<> Builder(&cast<CallInst>(Val&: Log));
2235
2236 // Use the generic intrinsic for convenience in the vector case. Codegen will
2237 // recognize the denormal handling is not necessary from the fpext.
2238 // TODO: Move to generic code
2239 Value *Log2 =
2240 Builder.CreateUnaryIntrinsic(ID: Intrinsic::log2, Op: Log.getOperand(i: 0), FMFSource: FMF);
2241
2242 double Log2BaseInverted =
2243 IID == Intrinsic::log10 ? numbers::ln2 / numbers::ln10 : numbers::ln2;
2244 Value *Mul =
2245 Builder.CreateFMulFMF(L: Log2, R: ConstantFP::get(Ty, V: Log2BaseInverted), FMFSource: FMF);
2246
2247 Mul->takeName(V: &Log);
2248
2249 Log.replaceAllUsesWith(V: Mul);
2250 DeadVals.push_back(Elt: &Log);
2251 return true;
2252}
2253
2254bool AMDGPUCodeGenPrepare::runOnFunction(Function &F) {
2255 if (skipFunction(F))
2256 return false;
2257
2258 auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
2259 if (!TPC)
2260 return false;
2261
2262 const AMDGPUTargetMachine &TM = TPC->getTM<AMDGPUTargetMachine>();
2263 const TargetTransformInfo &TTI =
2264 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
2265 const TargetLibraryInfo *TLI =
2266 &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
2267 AssumptionCache *AC =
2268 &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
2269 auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
2270 const DominatorTree *DT = DTWP ? &DTWP->getDomTree() : nullptr;
2271 const UniformityInfo &UA =
2272 getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
2273 return AMDGPUCodeGenPrepareImpl(F, TM, TTI, TLI, AC, DT, UA).run();
2274}
2275
2276PreservedAnalyses AMDGPUCodeGenPreparePass::run(Function &F,
2277 FunctionAnalysisManager &FAM) {
2278 const AMDGPUTargetMachine &ATM = static_cast<const AMDGPUTargetMachine &>(TM);
2279 const TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(IR&: F);
2280 const TargetLibraryInfo *TLI = &FAM.getResult<TargetLibraryAnalysis>(IR&: F);
2281 AssumptionCache *AC = &FAM.getResult<AssumptionAnalysis>(IR&: F);
2282 const DominatorTree *DT = FAM.getCachedResult<DominatorTreeAnalysis>(IR&: F);
2283 const UniformityInfo &UA = FAM.getResult<UniformityInfoAnalysis>(IR&: F);
2284 AMDGPUCodeGenPrepareImpl Impl(F, ATM, TTI, TLI, AC, DT, UA);
2285 if (!Impl.run())
2286 return PreservedAnalyses::all();
2287 PreservedAnalyses PA = PreservedAnalyses::none();
2288 if (!Impl.FlowChanged)
2289 PA.preserveSet<CFGAnalyses>();
2290 return PA;
2291}
2292
2293INITIALIZE_PASS_BEGIN(AMDGPUCodeGenPrepare, DEBUG_TYPE,
2294 "AMDGPU IR optimizations", false, false)
2295INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
2296INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
2297INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
2298INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
2299INITIALIZE_PASS_END(AMDGPUCodeGenPrepare, DEBUG_TYPE, "AMDGPU IR optimizations",
2300 false, false)
2301
2302/// Create a workitem.id.x intrinsic call with range metadata.
2303CallInst *AMDGPUCodeGenPrepareImpl::createWorkitemIdX(IRBuilder<> &B) const {
2304 CallInst *Tid =
2305 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::amdgcn_workitem_id_x, Args: {});
2306 ST.makeLIDRangeMetadata(I: Tid);
2307 return Tid;
2308}
2309
2310/// Replace the instruction with a direct workitem.id.x call.
2311void AMDGPUCodeGenPrepareImpl::replaceWithWorkitemIdX(Instruction &I) const {
2312 IRBuilder<> B(&I);
2313 CallInst *Tid = createWorkitemIdX(B);
2314 BasicBlock::iterator BI(&I);
2315 ReplaceInstWithValue(BI, V: Tid);
2316}
2317
2318/// Replace the instruction with (workitem.id.x & mask).
2319void AMDGPUCodeGenPrepareImpl::replaceWithMaskedWorkitemIdX(
2320 Instruction &I, unsigned WaveSize) const {
2321 IRBuilder<> B(&I);
2322 CallInst *Tid = createWorkitemIdX(B);
2323 Constant *Mask = ConstantInt::get(Ty: Tid->getType(), V: WaveSize - 1);
2324 Value *AndInst = B.CreateAnd(LHS: Tid, RHS: Mask);
2325 BasicBlock::iterator BI(&I);
2326 ReplaceInstWithValue(BI, V: AndInst);
2327}
2328
2329/// Try to optimize mbcnt instruction by replacing with workitem.id.x when
2330/// work group size allows direct computation of lane ID.
2331/// Returns true if optimization was applied, false otherwise.
2332bool AMDGPUCodeGenPrepareImpl::tryReplaceWithWorkitemId(Instruction &I,
2333 unsigned Wave) const {
2334 std::optional<unsigned> MaybeX = ST.getReqdWorkGroupSize(F, Dim: 0);
2335 if (!MaybeX)
2336 return false;
2337
2338 // When work group size == wave_size, each work group contains exactly one
2339 // wave, so the instruction can be replaced with workitem.id.x directly.
2340 if (*MaybeX == Wave) {
2341 replaceWithWorkitemIdX(I);
2342 return true;
2343 }
2344
2345 // When work group evenly splits into waves, compute lane ID within wave
2346 // using bit masking: lane_id = workitem.id.x & (wave_size - 1).
2347 if (ST.hasWavefrontsEvenlySplittingXDim(F, /*RequiresUniformYZ=*/REquiresUniformYZ: true)) {
2348 replaceWithMaskedWorkitemIdX(I, WaveSize: Wave);
2349 return true;
2350 }
2351
2352 return false;
2353}
2354
2355/// Optimize mbcnt.lo calls on wave32 architectures for lane ID computation.
2356bool AMDGPUCodeGenPrepareImpl::visitMbcntLo(IntrinsicInst &I) const {
2357 // This optimization only applies to wave32 targets where mbcnt.lo operates on
2358 // the full execution mask.
2359 if (!ST.isWave32())
2360 return false;
2361
2362 // Only optimize the pattern mbcnt.lo(~0, 0) which counts active lanes with
2363 // lower IDs.
2364 if (!match(V: &I,
2365 P: m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(Ops: m_AllOnes(), Ops: m_Zero())))
2366 return false;
2367
2368 return tryReplaceWithWorkitemId(I, Wave: ST.getWavefrontSize());
2369}
2370
2371/// Optimize mbcnt.hi calls for lane ID computation.
2372bool AMDGPUCodeGenPrepareImpl::visitMbcntHi(IntrinsicInst &I) const {
2373 // Abort if wave size is not known at compile time.
2374 if (!ST.isWaveSizeKnown())
2375 return false;
2376
2377 unsigned Wave = ST.getWavefrontSize();
2378
2379 // On wave32, the upper 32 bits of execution mask are always 0, so
2380 // mbcnt.hi(mask, val) always returns val unchanged.
2381 if (ST.isWave32()) {
2382 if (auto MaybeX = ST.getReqdWorkGroupSize(F, Dim: 0)) {
2383 // Replace mbcnt.hi(mask, val) with val only when work group size matches
2384 // wave size (single wave per work group).
2385 if (*MaybeX == Wave) {
2386 BasicBlock::iterator BI(&I);
2387 ReplaceInstWithValue(BI, V: I.getArgOperand(i: 1));
2388 return true;
2389 }
2390 }
2391 }
2392
2393 // Optimize the complete lane ID computation pattern:
2394 // mbcnt.hi(~0, mbcnt.lo(~0, 0)) which counts all active lanes with lower IDs
2395 // across the full execution mask.
2396 using namespace PatternMatch;
2397
2398 // Check for pattern: mbcnt.hi(~0, mbcnt.lo(~0, 0))
2399 if (!match(V: &I, P: m_Intrinsic<Intrinsic::amdgcn_mbcnt_hi>(
2400 Ops: m_AllOnes(), Ops: m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(
2401 Ops: m_AllOnes(), Ops: m_Zero()))))
2402 return false;
2403
2404 return tryReplaceWithWorkitemId(I, Wave);
2405}
2406
2407/// Check if type is <4 x i8>.
2408static bool isV4I8(Type *Ty) {
2409 FixedVectorType *VTy = dyn_cast<FixedVectorType>(Val: Ty);
2410 return VTy && VTy->getNumElements() == 4 &&
2411 VTy->getElementType()->isIntegerTy(BitWidth: 8);
2412}
2413
2414/// Helper to match the dot4 pattern: mul(zext/sext <4 x i8>, zext/sext <4 x
2415/// i8>) Returns true if pattern matches and signedness matches IsSigned.
2416/// Sets A, B to the <4 x i8> sources.
2417static bool matchDot4Pattern(Value *MulOp, Value *&A, Value *&B,
2418 bool IsSigned) {
2419 Value *Src0, *Src1;
2420 if (!match(V: MulOp, P: m_Mul(L: m_Value(V&: Src0), R: m_Value(V&: Src1))))
2421 return false;
2422
2423 // Check that result type is <4 x i32>
2424 FixedVectorType *MulTy = dyn_cast<FixedVectorType>(Val: MulOp->getType());
2425 if (!MulTy || MulTy->getNumElements() != 4 ||
2426 !MulTy->getElementType()->isIntegerTy(BitWidth: 32))
2427 return false;
2428
2429 // Match zext or sext based on IsSigned
2430 Value *ExtSrc0, *ExtSrc1;
2431 if (IsSigned) {
2432 if (!match(V: Src0, P: m_SExt(Op: m_Value(V&: ExtSrc0))) || !isV4I8(Ty: ExtSrc0->getType()))
2433 return false;
2434 if (!match(V: Src1, P: m_SExt(Op: m_Value(V&: ExtSrc1))) || !isV4I8(Ty: ExtSrc1->getType()))
2435 return false;
2436 } else {
2437 if (!match(V: Src0, P: m_ZExt(Op: m_Value(V&: ExtSrc0))) || !isV4I8(Ty: ExtSrc0->getType()))
2438 return false;
2439 if (!match(V: Src1, P: m_ZExt(Op: m_Value(V&: ExtSrc1))) || !isV4I8(Ty: ExtSrc1->getType()))
2440 return false;
2441 }
2442
2443 A = ExtSrc0;
2444 B = ExtSrc1;
2445 return true;
2446}
2447
2448/// Try to convert vector.reduce.add(mul(zext/sext <4 x i8>, zext/sext <4 x
2449/// i8>)) to a dot4 intrinsic call (non-saturating case only).
2450bool AMDGPUCodeGenPrepareImpl::visitVectorReduceAdd(IntrinsicInst &I) {
2451 // Check if we have dot4 instructions available
2452 if (!ST.hasDot7Insts() || (!ST.hasDot1Insts() && !ST.hasDot8Insts()))
2453 return false;
2454
2455 Value *A = nullptr, *B = nullptr;
2456
2457 // Try unsigned first, then signed
2458 bool IsSigned = false;
2459 if (!matchDot4Pattern(MulOp: I.getArgOperand(i: 0), A, B, /*IsSigned=*/false)) {
2460 if (!matchDot4Pattern(MulOp: I.getArgOperand(i: 0), A, B, /*IsSigned=*/true))
2461 return false;
2462 IsSigned = true;
2463 }
2464
2465 LLVMContext &Ctx = I.getContext();
2466 Type *I32Ty = Type::getInt32Ty(C&: Ctx);
2467 IRBuilder<> Builder(&I);
2468
2469 // Bitcast <4 x i8> to i32
2470 Value *ASrc = Builder.CreateBitCast(V: A, DestTy: I32Ty);
2471 Value *BSrc = Builder.CreateBitCast(V: B, DestTy: I32Ty);
2472
2473 // Non-saturating case: accumulator is 0, clamp is false
2474 Value *Acc = ConstantInt::get(Ty: I32Ty, V: 0);
2475 Value *Clamp = ConstantInt::getFalse(Context&: Ctx);
2476
2477 Intrinsic::ID DotIID =
2478 IsSigned ? Intrinsic::amdgcn_sdot4 : Intrinsic::amdgcn_udot4;
2479
2480 Value *Dot = Builder.CreateIntrinsic(ID: DotIID, OverloadTypes: {}, Args: {ASrc, BSrc, Acc, Clamp});
2481 Dot->takeName(V: &I);
2482
2483 I.replaceAllUsesWith(V: Dot);
2484 DeadVals.push_back(Elt: &I);
2485
2486 return true;
2487}
2488
2489/// Try to convert uadd.sat/sadd.sat(vector.reduce.add(mul(...)), c) to a
2490/// saturating dot4 intrinsic. This combine starts at the root (saturating add)
2491/// and looks at its operands.
2492bool AMDGPUCodeGenPrepareImpl::visitSaturatingAdd(IntrinsicInst &I) {
2493 // Check if we have dot4 instructions available
2494 if (!ST.hasDot7Insts() || (!ST.hasDot1Insts() && !ST.hasDot8Insts()))
2495 return false;
2496
2497 Intrinsic::ID IID = I.getIntrinsicID();
2498 bool IsSigned = (IID == Intrinsic::sadd_sat);
2499
2500 // Look for vector.reduce.add as one of the operands (commutative match)
2501 Value *Op0 = I.getArgOperand(i: 0);
2502 Value *Op1 = I.getArgOperand(i: 1);
2503 Value *MulOp = nullptr;
2504 Value *Accum = nullptr;
2505 IntrinsicInst *ReduceInst = nullptr;
2506
2507 if (match(V: Op0, P: m_Intrinsic<Intrinsic::vector_reduce_add>(Ops: m_Value(V&: MulOp)))) {
2508 ReduceInst = cast<IntrinsicInst>(Val: Op0);
2509 Accum = Op1;
2510 } else if (match(V: Op1,
2511 P: m_Intrinsic<Intrinsic::vector_reduce_add>(Ops: m_Value(V&: MulOp)))) {
2512 ReduceInst = cast<IntrinsicInst>(Val: Op1);
2513 Accum = Op0;
2514 } else {
2515 return false;
2516 }
2517
2518 Value *A = nullptr, *B = nullptr;
2519
2520 if (!matchDot4Pattern(MulOp, A, B, IsSigned))
2521 return false;
2522
2523 LLVMContext &Ctx = I.getContext();
2524 Type *I32Ty = Type::getInt32Ty(C&: Ctx);
2525 IRBuilder<> Builder(&I);
2526
2527 // Bitcast <4 x i8> to i32
2528 Value *ASrc = Builder.CreateBitCast(V: A, DestTy: I32Ty);
2529 Value *BSrc = Builder.CreateBitCast(V: B, DestTy: I32Ty);
2530
2531 // Saturating case: use the accumulator and set clamp to true
2532 Value *Clamp = ConstantInt::getTrue(Context&: Ctx);
2533
2534 Intrinsic::ID DotIID =
2535 IsSigned ? Intrinsic::amdgcn_sdot4 : Intrinsic::amdgcn_udot4;
2536
2537 Value *Dot = Builder.CreateIntrinsic(ID: DotIID, OverloadTypes: {}, Args: {ASrc, BSrc, Accum, Clamp});
2538 Dot->takeName(V: &I);
2539
2540 I.replaceAllUsesWith(V: Dot);
2541 DeadVals.push_back(Elt: &I);
2542 // The reduce.add will be dead after this and cleaned up later
2543 if (ReduceInst->use_empty())
2544 DeadVals.push_back(Elt: ReduceInst);
2545
2546 return true;
2547}
2548
2549char AMDGPUCodeGenPrepare::ID = 0;
2550
2551FunctionPass *llvm::createAMDGPUCodeGenPreparePass() {
2552 return new AMDGPUCodeGenPrepare();
2553}
2554