1//===-- AMDGPUCodeGenPrepare.cpp ------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass does misc. AMDGPU optimizations on IR before instruction
11/// selection.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPU.h"
16#include "AMDGPUMemoryUtils.h"
17#include "AMDGPUTargetMachine.h"
18#include "SIModeRegisterDefaults.h"
19#include "llvm/Analysis/AssumptionCache.h"
20#include "llvm/Analysis/ConstantFolding.h"
21#include "llvm/Analysis/TargetLibraryInfo.h"
22#include "llvm/Analysis/TargetTransformInfo.h"
23#include "llvm/Analysis/UniformityAnalysis.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/CodeGen/TargetPassConfig.h"
26#include "llvm/IR/Dominators.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/InstVisitor.h"
29#include "llvm/IR/IntrinsicsAMDGPU.h"
30#include "llvm/IR/PatternMatch.h"
31#include "llvm/InitializePasses.h"
32#include "llvm/Pass.h"
33#include "llvm/Support/KnownBits.h"
34#include "llvm/Support/KnownFPClass.h"
35#include "llvm/Transforms/Utils/BasicBlockUtils.h"
36#include "llvm/Transforms/Utils/IntegerDivision.h"
37#include "llvm/Transforms/Utils/Local.h"
38
39#define DEBUG_TYPE "amdgpu-codegenprepare"
40
41using namespace llvm;
42using namespace llvm::PatternMatch;
43
44namespace {
45
46static cl::opt<bool> WidenLoads(
47 "amdgpu-codegenprepare-widen-constant-loads",
48 cl::desc("Widen sub-dword constant address space loads in AMDGPUCodeGenPrepare"),
49 cl::ReallyHidden,
50 cl::init(Val: false));
51
52static cl::opt<bool>
53 BreakLargePHIs("amdgpu-codegenprepare-break-large-phis",
54 cl::desc("Break large PHI nodes for DAGISel"),
55 cl::ReallyHidden, cl::init(Val: true));
56
57static cl::opt<bool>
58 ForceBreakLargePHIs("amdgpu-codegenprepare-force-break-large-phis",
59 cl::desc("For testing purposes, always break large "
60 "PHIs even if it isn't profitable."),
61 cl::ReallyHidden, cl::init(Val: false));
62
63static cl::opt<unsigned> BreakLargePHIsThreshold(
64 "amdgpu-codegenprepare-break-large-phis-threshold",
65 cl::desc("Minimum type size in bits for breaking large PHI nodes"),
66 cl::ReallyHidden, cl::init(Val: 32));
67
68static cl::opt<bool> UseMul24Intrin(
69 "amdgpu-codegenprepare-mul24",
70 cl::desc("Introduce mul24 intrinsics in AMDGPUCodeGenPrepare"),
71 cl::ReallyHidden,
72 cl::init(Val: true));
73
74// Legalize 64-bit division by using the generic IR expansion.
75static cl::opt<bool> ExpandDiv64InIR(
76 "amdgpu-codegenprepare-expand-div64",
77 cl::desc("Expand 64-bit division in AMDGPUCodeGenPrepare"),
78 cl::ReallyHidden,
79 cl::init(Val: false));
80
81// Leave all division operations as they are. This supersedes ExpandDiv64InIR
82// and is used for testing the legalizer.
83static cl::opt<bool> DisableIDivExpand(
84 "amdgpu-codegenprepare-disable-idiv-expansion",
85 cl::desc("Prevent expanding integer division in AMDGPUCodeGenPrepare"),
86 cl::ReallyHidden,
87 cl::init(Val: false));
88
89// Disable processing of fdiv so we can better test the backend implementations.
90static cl::opt<bool> DisableFDivExpand(
91 "amdgpu-codegenprepare-disable-fdiv-expansion",
92 cl::desc("Prevent expanding floating point division in AMDGPUCodeGenPrepare"),
93 cl::ReallyHidden,
94 cl::init(Val: false));
95
96class AMDGPUCodeGenPrepareImpl
97 : public InstVisitor<AMDGPUCodeGenPrepareImpl, bool> {
98public:
99 Function &F;
100 const GCNSubtarget &ST;
101 const AMDGPUTargetMachine &TM;
102 const TargetTransformInfo &TTI;
103 const TargetLibraryInfo *TLI;
104 const UniformityInfo &UA;
105 const DataLayout &DL;
106 SimplifyQuery SQ;
107 const bool HasFP32DenormalFlush;
108 bool FlowChanged = false;
109 mutable Function *SqrtF32 = nullptr;
110 mutable Function *LdexpF32 = nullptr;
111 mutable SmallVector<WeakVH> DeadVals;
112
113 DenseMap<const PHINode *, bool> BreakPhiNodesCache;
114
115 AMDGPUCodeGenPrepareImpl(Function &F, const AMDGPUTargetMachine &TM,
116 const TargetTransformInfo &TTI,
117 const TargetLibraryInfo *TLI, AssumptionCache *AC,
118 const DominatorTree *DT, const UniformityInfo &UA)
119 : F(F), ST(TM.getSubtarget<GCNSubtarget>(F)), TM(TM), TTI(TTI), TLI(TLI),
120 UA(UA), DL(F.getDataLayout()), SQ(DL, TLI, DT, AC),
121 HasFP32DenormalFlush(SIModeRegisterDefaults(F, ST).FP32Denormals ==
122 DenormalMode::getPreserveSign()) {}
123
124 Function *getSqrtF32() const {
125 if (SqrtF32)
126 return SqrtF32;
127
128 LLVMContext &Ctx = F.getContext();
129 SqrtF32 = Intrinsic::getOrInsertDeclaration(
130 M: F.getParent(), id: Intrinsic::amdgcn_sqrt, OverloadTys: {Type::getFloatTy(C&: Ctx)});
131 return SqrtF32;
132 }
133
134 Function *getLdexpF32() const {
135 if (LdexpF32)
136 return LdexpF32;
137
138 LLVMContext &Ctx = F.getContext();
139 LdexpF32 = Intrinsic::getOrInsertDeclaration(
140 M: F.getParent(), id: Intrinsic::ldexp,
141 OverloadTys: {Type::getFloatTy(C&: Ctx), Type::getInt32Ty(C&: Ctx)});
142 return LdexpF32;
143 }
144
145 bool canBreakPHINode(const PHINode &I);
146
147 /// Return true if \p T is a legal scalar floating point type.
148 bool isLegalFloatingTy(const Type *T) const;
149
150 /// Wrapper to pass all the arguments to computeKnownFPClass
151 KnownFPClass computeKnownFPClass(const Value *V, FPClassTest Interested,
152 const Instruction *CtxI) const {
153 return llvm::computeKnownFPClass(V, InterestedClasses: Interested,
154 SQ: SQ.getWithInstruction(I: CtxI));
155 }
156
157 bool canIgnoreDenormalInput(const Value *V, const Instruction *CtxI) const {
158 return HasFP32DenormalFlush ||
159 computeKnownFPClass(V, Interested: fcSubnormal, CtxI).isKnownNeverSubnormal();
160 }
161
162 /// \returns The minimum number of bits needed to store the value of \Op as an
163 /// unsigned integer. Truncating to this size and then zero-extending to
164 /// the original will not change the value.
165 unsigned numBitsUnsigned(Value *Op, const Instruction *CtxI) const;
166
167 /// \returns The minimum number of bits needed to store the value of \Op as a
168 /// signed integer. Truncating to this size and then sign-extending to
169 /// the original size will not change the value.
170 unsigned numBitsSigned(Value *Op, const Instruction *CtxI) const;
171
172 /// Replace mul instructions with llvm.amdgcn.mul.u24 or llvm.amdgcn.mul.s24.
173 /// SelectionDAG has an issue where an and asserting the bits are known
174 bool replaceMulWithMul24(BinaryOperator &I) const;
175
176 /// Perform same function as equivalently named function in DAGCombiner. Since
177 /// we expand some divisions here, we need to perform this before obscuring.
178 bool foldBinOpIntoSelect(BinaryOperator &I) const;
179
180 bool divHasSpecialOptimization(BinaryOperator &I,
181 Value *Num, Value *Den) const;
182 unsigned getDivNumBits(BinaryOperator &I, Value *Num, Value *Den,
183 unsigned MaxDivBits, bool Signed) const;
184
185 /// Expands div or rem by using floating-point operations.
186 /// Operands must be in the range [-0x400000,0x3FFFFF]
187 Value *expandDivRemToFloat(IRBuilder<> &Builder, BinaryOperator &I,
188 Value *Num, Value *Den, bool IsDiv,
189 bool IsSigned) const;
190
191 Value *expandDivRemToFloatImpl(IRBuilder<> &Builder, BinaryOperator &I,
192 Value *Num, Value *Den, unsigned NumBits,
193 bool IsDiv, bool IsSigned) const;
194
195 /// Expands 32 bit div or rem.
196 Value* expandDivRem32(IRBuilder<> &Builder, BinaryOperator &I,
197 Value *Num, Value *Den) const;
198
199 Value *shrinkDivRem64(IRBuilder<> &Builder, BinaryOperator &I,
200 Value *Num, Value *Den) const;
201 void expandDivRem64(BinaryOperator &I) const;
202
203 /// Widen a scalar load.
204 ///
205 /// \details \p Widen scalar load for uniform, small type loads from constant
206 // memory / to a full 32-bits and then truncate the input to allow a scalar
207 // load instead of a vector load.
208 //
209 /// \returns True.
210
211 bool canWidenScalarExtLoad(LoadInst &I) const;
212
213 Value *matchFractPatImpl(Value &V, const APFloat &C) const;
214 Value *matchFractPatNanAvoidant(Value &V);
215 Value *applyFractPat(IRBuilder<> &Builder, Value *FractArg);
216
217 bool canOptimizeWithRsq(FastMathFlags DivFMF, FastMathFlags SqrtFMF) const;
218
219 Value *optimizeWithRsq(IRBuilder<> &Builder, Value *Num, Value *Den,
220 FastMathFlags DivFMF, FastMathFlags SqrtFMF,
221 const Instruction *CtxI) const;
222
223 Value *optimizeWithRcp(IRBuilder<> &Builder, Value *Num, Value *Den,
224 FastMathFlags FMF, const Instruction *CtxI) const;
225 Value *optimizeWithFDivFast(IRBuilder<> &Builder, Value *Num, Value *Den,
226 float ReqdAccuracy) const;
227
228 Value *visitFDivElement(IRBuilder<> &Builder, Value *Num, Value *Den,
229 FastMathFlags DivFMF, FastMathFlags SqrtFMF,
230 Value *RsqOp, const Instruction *FDiv,
231 float ReqdAccuracy) const;
232
233 std::pair<Value *, Value *> getFrexpResults(IRBuilder<> &Builder,
234 Value *Src) const;
235
236 Value *emitRcpIEEE1ULP(IRBuilder<> &Builder, Value *Src,
237 bool IsNegative) const;
238 Value *emitFrexpDiv(IRBuilder<> &Builder, Value *LHS, Value *RHS,
239 FastMathFlags FMF) const;
240 Value *emitSqrtIEEE2ULP(IRBuilder<> &Builder, Value *Src,
241 FastMathFlags FMF) const;
242 Value *emitRsqF64(IRBuilder<> &Builder, Value *X, FastMathFlags SqrtFMF,
243 FastMathFlags DivFMF, const Instruction *CtxI,
244 bool IsNegative) const;
245
246 CallInst *createWorkitemIdX(IRBuilder<> &B) const;
247 void replaceWithWorkitemIdX(Instruction &I) const;
248 void replaceWithMaskedWorkitemIdX(Instruction &I, unsigned WaveSize) const;
249 bool tryReplaceWithWorkitemId(Instruction &I, unsigned Wave) const;
250
251 bool tryNarrowMathIfNoOverflow(Instruction *I);
252
253public:
254 bool visitFDiv(BinaryOperator &I);
255
256 bool visitInstruction(Instruction &I) { return false; }
257 bool visitBinaryOperator(BinaryOperator &I);
258 bool visitLoadInst(LoadInst &I);
259 bool visitSelectInst(SelectInst &I);
260 bool visitPHINode(PHINode &I);
261 bool visitAddrSpaceCastInst(AddrSpaceCastInst &I);
262
263 bool visitIntrinsicInst(IntrinsicInst &I);
264 bool visitFMinLike(IntrinsicInst &I);
265 bool visitSqrt(IntrinsicInst &I);
266 bool visitLog(FPMathOperator &Log, Intrinsic::ID IID);
267 bool visitMbcntLo(IntrinsicInst &I) const;
268 bool visitMbcntHi(IntrinsicInst &I) const;
269 bool visitVectorReduceAdd(IntrinsicInst &I);
270 bool visitSaturatingAdd(IntrinsicInst &I);
271 bool run();
272};
273
274class AMDGPUCodeGenPrepare : public FunctionPass {
275public:
276 static char ID;
277 AMDGPUCodeGenPrepare() : FunctionPass(ID) {}
278 void getAnalysisUsage(AnalysisUsage &AU) const override {
279 AU.addRequired<AssumptionCacheTracker>();
280 AU.addRequired<UniformityInfoWrapperPass>();
281 AU.addRequired<TargetLibraryInfoWrapperPass>();
282 AU.addRequired<TargetTransformInfoWrapperPass>();
283
284 // FIXME: Division expansion needs to preserve the dominator tree.
285 if (!ExpandDiv64InIR)
286 AU.setPreservesAll();
287 }
288 bool runOnFunction(Function &F) override;
289 StringRef getPassName() const override { return "AMDGPU IR optimizations"; }
290};
291
292} // end anonymous namespace
293
294bool AMDGPUCodeGenPrepareImpl::run() {
295 BreakPhiNodesCache.clear();
296 bool MadeChange = false;
297
298 // Need to use make_early_inc_range because integer division expansion is
299 // handled by Transform/Utils, and it can delete instructions such as the
300 // terminator of the BB.
301 for (BasicBlock &BB : reverse(C&: F)) {
302 for (Instruction &I : make_early_inc_range(Range: reverse(C&: BB))) {
303 if (!isInstructionTriviallyDead(I: &I, TLI))
304 MadeChange |= visit(I);
305 }
306 }
307
308 while (!DeadVals.empty()) {
309 if (auto *I = dyn_cast_or_null<Instruction>(Val: DeadVals.pop_back_val()))
310 RecursivelyDeleteTriviallyDeadInstructions(V: I, TLI);
311 }
312
313 return MadeChange;
314}
315
316bool AMDGPUCodeGenPrepareImpl::isLegalFloatingTy(const Type *Ty) const {
317 return Ty->isFloatTy() || Ty->isDoubleTy() ||
318 (Ty->isHalfTy() && ST.has16BitInsts());
319}
320
321bool AMDGPUCodeGenPrepareImpl::canWidenScalarExtLoad(LoadInst &I) const {
322 Type *Ty = I.getType();
323 int TySize = DL.getTypeSizeInBits(Ty);
324 Align Alignment = DL.getValueOrABITypeAlignment(Alignment: I.getAlign(), Ty);
325
326 return I.isSimple() && TySize < 32 && Alignment >= 4 && UA.isUniformAtDef(V: &I);
327}
328
329unsigned
330AMDGPUCodeGenPrepareImpl::numBitsUnsigned(Value *Op,
331 const Instruction *CtxI) const {
332 return computeKnownBits(V: Op, Q: SQ.getWithInstruction(I: CtxI)).countMaxActiveBits();
333}
334
335unsigned
336AMDGPUCodeGenPrepareImpl::numBitsSigned(Value *Op,
337 const Instruction *CtxI) const {
338 return ComputeMaxSignificantBits(Op, DL: SQ.DL, AC: SQ.AC, CxtI: CtxI, DT: SQ.DT);
339}
340
341static void extractValues(IRBuilder<> &Builder,
342 SmallVectorImpl<Value *> &Values, Value *V) {
343 auto *VT = dyn_cast<FixedVectorType>(Val: V->getType());
344 if (!VT) {
345 Values.push_back(Elt: V);
346 return;
347 }
348
349 for (int I = 0, E = VT->getNumElements(); I != E; ++I)
350 Values.push_back(Elt: Builder.CreateExtractElement(Vec: V, Idx: I));
351}
352
353static Value *insertValues(IRBuilder<> &Builder,
354 Type *Ty,
355 SmallVectorImpl<Value *> &Values) {
356 if (!Ty->isVectorTy()) {
357 assert(Values.size() == 1);
358 return Values[0];
359 }
360
361 Value *NewVal = PoisonValue::get(T: Ty);
362 for (int I = 0, E = Values.size(); I != E; ++I)
363 NewVal = Builder.CreateInsertElement(Vec: NewVal, NewElt: Values[I], Idx: I);
364
365 return NewVal;
366}
367
368bool AMDGPUCodeGenPrepareImpl::replaceMulWithMul24(BinaryOperator &I) const {
369 if (I.getOpcode() != Instruction::Mul)
370 return false;
371
372 Type *Ty = I.getType();
373 unsigned Size = Ty->getScalarSizeInBits();
374 if (Size <= 16 && ST.has16BitInsts())
375 return false;
376
377 // Prefer scalar if this could be s_mul_i32
378 if (UA.isUniformAtDef(V: &I))
379 return false;
380
381 Value *LHS = I.getOperand(i_nocapture: 0);
382 Value *RHS = I.getOperand(i_nocapture: 1);
383 IRBuilder<> Builder(&I);
384 Builder.SetCurrentDebugLocation(I.getDebugLoc());
385
386 unsigned LHSBits = 0, RHSBits = 0;
387 bool IsSigned = false;
388
389 if (ST.hasMulU24() && (LHSBits = numBitsUnsigned(Op: LHS, CtxI: &I)) <= 24 &&
390 (RHSBits = numBitsUnsigned(Op: RHS, CtxI: &I)) <= 24) {
391 IsSigned = false;
392
393 } else if (ST.hasMulI24() && (LHSBits = numBitsSigned(Op: LHS, CtxI: &I)) <= 24 &&
394 (RHSBits = numBitsSigned(Op: RHS, CtxI: &I)) <= 24) {
395 IsSigned = true;
396
397 } else
398 return false;
399
400 SmallVector<Value *, 4> LHSVals;
401 SmallVector<Value *, 4> RHSVals;
402 SmallVector<Value *, 4> ResultVals;
403 extractValues(Builder, Values&: LHSVals, V: LHS);
404 extractValues(Builder, Values&: RHSVals, V: RHS);
405
406 IntegerType *I32Ty = Builder.getInt32Ty();
407 IntegerType *IntrinTy = Size > 32 ? Builder.getInt64Ty() : I32Ty;
408 Type *DstTy = LHSVals[0]->getType();
409
410 for (int I = 0, E = LHSVals.size(); I != E; ++I) {
411 Value *LHS = IsSigned ? Builder.CreateSExtOrTrunc(V: LHSVals[I], DestTy: I32Ty)
412 : Builder.CreateZExtOrTrunc(V: LHSVals[I], DestTy: I32Ty);
413 Value *RHS = IsSigned ? Builder.CreateSExtOrTrunc(V: RHSVals[I], DestTy: I32Ty)
414 : Builder.CreateZExtOrTrunc(V: RHSVals[I], DestTy: I32Ty);
415 Intrinsic::ID ID =
416 IsSigned ? Intrinsic::amdgcn_mul_i24 : Intrinsic::amdgcn_mul_u24;
417 Value *Result = Builder.CreateIntrinsic(ID, OverloadTypes: {IntrinTy}, Args: {LHS, RHS});
418 Result = IsSigned ? Builder.CreateSExtOrTrunc(V: Result, DestTy: DstTy)
419 : Builder.CreateZExtOrTrunc(V: Result, DestTy: DstTy);
420 ResultVals.push_back(Elt: Result);
421 }
422
423 Value *NewVal = insertValues(Builder, Ty, Values&: ResultVals);
424 NewVal->takeName(V: &I);
425 I.replaceAllUsesWith(V: NewVal);
426 DeadVals.push_back(Elt: &I);
427
428 return true;
429}
430
431// Find a select instruction, which may have been casted. This is mostly to deal
432// with cases where i16 selects were promoted here to i32.
433static SelectInst *findSelectThroughCast(Value *V, CastInst *&Cast) {
434 Cast = nullptr;
435 if (SelectInst *Sel = dyn_cast<SelectInst>(Val: V))
436 return Sel;
437
438 if ((Cast = dyn_cast<CastInst>(Val: V))) {
439 if (SelectInst *Sel = dyn_cast<SelectInst>(Val: Cast->getOperand(i_nocapture: 0)))
440 return Sel;
441 }
442
443 return nullptr;
444}
445
446bool AMDGPUCodeGenPrepareImpl::foldBinOpIntoSelect(BinaryOperator &BO) const {
447 // Don't do this unless the old select is going away. We want to eliminate the
448 // binary operator, not replace a binop with a select.
449 int SelOpNo = 0;
450
451 CastInst *CastOp;
452
453 // TODO: Should probably try to handle some cases with multiple
454 // users. Duplicating the select may be profitable for division.
455 SelectInst *Sel = findSelectThroughCast(V: BO.getOperand(i_nocapture: 0), Cast&: CastOp);
456 if (!Sel || !Sel->hasOneUse()) {
457 SelOpNo = 1;
458 Sel = findSelectThroughCast(V: BO.getOperand(i_nocapture: 1), Cast&: CastOp);
459 }
460
461 if (!Sel || !Sel->hasOneUse())
462 return false;
463
464 Constant *CT = dyn_cast<Constant>(Val: Sel->getTrueValue());
465 Constant *CF = dyn_cast<Constant>(Val: Sel->getFalseValue());
466 Constant *CBO = dyn_cast<Constant>(Val: BO.getOperand(i_nocapture: SelOpNo ^ 1));
467 if (!CBO || !CT || !CF)
468 return false;
469
470 if (CastOp) {
471 if (!CastOp->hasOneUse())
472 return false;
473 CT = ConstantFoldCastOperand(Opcode: CastOp->getOpcode(), C: CT, DestTy: BO.getType(), DL);
474 CF = ConstantFoldCastOperand(Opcode: CastOp->getOpcode(), C: CF, DestTy: BO.getType(), DL);
475 }
476
477 // TODO: Handle special 0/-1 cases DAG combine does, although we only really
478 // need to handle divisions here.
479 Constant *FoldedT =
480 SelOpNo ? ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CBO, RHS: CT, DL)
481 : ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CT, RHS: CBO, DL);
482 if (!FoldedT || isa<ConstantExpr>(Val: FoldedT))
483 return false;
484
485 Constant *FoldedF =
486 SelOpNo ? ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CBO, RHS: CF, DL)
487 : ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CF, RHS: CBO, DL);
488 if (!FoldedF || isa<ConstantExpr>(Val: FoldedF))
489 return false;
490
491 IRBuilder<> Builder(&BO);
492 Builder.SetCurrentDebugLocation(BO.getDebugLoc());
493 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Val: &BO))
494 Builder.setFastMathFlags(FPOp->getFastMathFlags());
495
496 Value *NewSelect = Builder.CreateSelect(C: Sel->getCondition(),
497 True: FoldedT, False: FoldedF);
498 NewSelect->takeName(V: &BO);
499 BO.replaceAllUsesWith(V: NewSelect);
500 DeadVals.push_back(Elt: &BO);
501 if (CastOp)
502 DeadVals.push_back(Elt: CastOp);
503 DeadVals.push_back(Elt: Sel);
504 return true;
505}
506
507std::pair<Value *, Value *>
508AMDGPUCodeGenPrepareImpl::getFrexpResults(IRBuilder<> &Builder,
509 Value *Src) const {
510 Type *Ty = Src->getType();
511 Value *Frexp = Builder.CreateIntrinsic(ID: Intrinsic::frexp,
512 OverloadTypes: {Ty, Builder.getInt32Ty()}, Args: Src);
513 Value *FrexpMant = Builder.CreateExtractValue(Agg: Frexp, Idxs: {0});
514
515 // Bypass the bug workaround for the exponent result since it doesn't matter.
516 // TODO: Does the bug workaround even really need to consider the exponent
517 // result? It's unspecified by the spec.
518
519 Value *FrexpExp =
520 ST.hasFractBug()
521 ? Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_frexp_exp,
522 OverloadTypes: {Builder.getInt32Ty(), Ty}, Args: Src)
523 : Builder.CreateExtractValue(Agg: Frexp, Idxs: {1});
524 return {FrexpMant, FrexpExp};
525}
526
527/// Emit an expansion of 1.0 / Src good for 1ulp that supports denormals.
528Value *AMDGPUCodeGenPrepareImpl::emitRcpIEEE1ULP(IRBuilder<> &Builder,
529 Value *Src,
530 bool IsNegative) const {
531 // Same as for 1.0, but expand the sign out of the constant.
532 // -1.0 / x -> rcp (fneg x)
533 if (IsNegative)
534 Src = Builder.CreateFNeg(V: Src);
535
536 // The rcp instruction doesn't support denormals, so scale the input
537 // out of the denormal range and convert at the end.
538 //
539 // Expand as 2^-n * (1.0 / (x * 2^n))
540
541 // TODO: Skip scaling if input is known never denormal and the input
542 // range won't underflow to denormal. The hard part is knowing the
543 // result. We need a range check, the result could be denormal for
544 // 0x1p+126 < den <= 0x1p+127.
545 auto [FrexpMant, FrexpExp] = getFrexpResults(Builder, Src);
546 Value *ScaleFactor = Builder.CreateNeg(V: FrexpExp);
547 Value *Rcp = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: FrexpMant);
548 return Builder.CreateCall(Callee: getLdexpF32(), Args: {Rcp, ScaleFactor});
549}
550
551/// Emit a 2ulp expansion for fdiv by using frexp for input scaling.
552Value *AMDGPUCodeGenPrepareImpl::emitFrexpDiv(IRBuilder<> &Builder, Value *LHS,
553 Value *RHS,
554 FastMathFlags FMF) const {
555 // If we have have to work around the fract/frexp bug, we're worse off than
556 // using the fdiv.fast expansion. The full safe expansion is faster if we have
557 // fast FMA.
558 if (HasFP32DenormalFlush && ST.hasFractBug() && !ST.hasFastFMAF32() &&
559 (!FMF.noNaNs() || !FMF.noInfs()))
560 return nullptr;
561
562 // We're scaling the LHS to avoid a denormal input, and scale the denominator
563 // to avoid large values underflowing the result.
564 auto [FrexpMantRHS, FrexpExpRHS] = getFrexpResults(Builder, Src: RHS);
565
566 Value *Rcp =
567 Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: FrexpMantRHS);
568
569 auto [FrexpMantLHS, FrexpExpLHS] = getFrexpResults(Builder, Src: LHS);
570 Value *Mul = Builder.CreateFMul(L: FrexpMantLHS, R: Rcp);
571
572 // We multiplied by 2^N/2^M, so we need to multiply by 2^(N-M) to scale the
573 // result.
574 Value *ExpDiff = Builder.CreateSub(LHS: FrexpExpLHS, RHS: FrexpExpRHS);
575 return Builder.CreateCall(Callee: getLdexpF32(), Args: {Mul, ExpDiff});
576}
577
578/// Emit a sqrt that handles denormals and is accurate to 2ulp.
579Value *AMDGPUCodeGenPrepareImpl::emitSqrtIEEE2ULP(IRBuilder<> &Builder,
580 Value *Src,
581 FastMathFlags FMF) const {
582 Type *Ty = Src->getType();
583 APFloat SmallestNormal =
584 APFloat::getSmallestNormalized(Sem: Ty->getFltSemantics());
585 Value *NeedScale =
586 Builder.CreateFCmpOLT(LHS: Src, RHS: ConstantFP::get(Ty, V: SmallestNormal));
587
588 ConstantInt *Zero = Builder.getInt32(C: 0);
589 Value *InputScaleFactor =
590 Builder.CreateSelect(C: NeedScale, True: Builder.getInt32(C: 32), False: Zero);
591
592 Value *Scaled = Builder.CreateCall(Callee: getLdexpF32(), Args: {Src, InputScaleFactor});
593
594 Value *Sqrt = Builder.CreateCall(Callee: getSqrtF32(), Args: Scaled);
595
596 Value *OutputScaleFactor =
597 Builder.CreateSelect(C: NeedScale, True: Builder.getInt32(C: -16), False: Zero);
598 return Builder.CreateCall(Callee: getLdexpF32(), Args: {Sqrt, OutputScaleFactor});
599}
600
601/// Emit an expansion of 1.0 / sqrt(Src) good for 1ulp that supports denormals.
602static Value *emitRsqIEEE1ULP(IRBuilder<> &Builder, Value *Src,
603 bool IsNegative) {
604 // bool need_scale = x < 0x1p-126f;
605 // float input_scale = need_scale ? 0x1.0p+24f : 1.0f;
606 // float output_scale = need_scale ? 0x1.0p+12f : 1.0f;
607 // rsq(x * input_scale) * output_scale;
608
609 Type *Ty = Src->getType();
610 APFloat SmallestNormal =
611 APFloat::getSmallestNormalized(Sem: Ty->getFltSemantics());
612 Value *NeedScale =
613 Builder.CreateFCmpOLT(LHS: Src, RHS: ConstantFP::get(Ty, V: SmallestNormal));
614 Constant *One = ConstantFP::get(Ty, V: 1.0);
615 Constant *InputScale = ConstantFP::get(Ty, V: 0x1.0p+24);
616 Constant *OutputScale =
617 ConstantFP::get(Ty, V: IsNegative ? -0x1.0p+12 : 0x1.0p+12);
618
619 Value *InputScaleFactor = Builder.CreateSelect(C: NeedScale, True: InputScale, False: One);
620
621 Value *ScaledInput = Builder.CreateFMul(L: Src, R: InputScaleFactor);
622 Value *Rsq = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rsq, Op: ScaledInput);
623 Value *OutputScaleFactor = Builder.CreateSelect(
624 C: NeedScale, True: OutputScale, False: IsNegative ? ConstantFP::get(Ty, V: -1.0) : One);
625
626 return Builder.CreateFMul(L: Rsq, R: OutputScaleFactor);
627}
628
629/// Emit inverse sqrt expansion for f64 with a correction sequence on top of
630/// v_rsq_f64. This should give a 1ulp result.
631Value *AMDGPUCodeGenPrepareImpl::emitRsqF64(IRBuilder<> &Builder, Value *X,
632 FastMathFlags SqrtFMF,
633 FastMathFlags DivFMF,
634 const Instruction *CtxI,
635 bool IsNegative) const {
636 // rsq(x):
637 // double y0 = BUILTIN_AMDGPU_RSQRT_F64(x);
638 // double e = MATH_MAD(-y0 * (x == PINF_F64 || x == 0.0 ? y0 : x), y0, 1.0);
639 // return MATH_MAD(y0*e, MATH_MAD(e, 0.375, 0.5), y0);
640 //
641 // -rsq(x):
642 // double y0 = BUILTIN_AMDGPU_RSQRT_F64(x);
643 // double e = MATH_MAD(-y0 * (x == PINF_F64 || x == 0.0 ? y0 : x), y0, 1.0);
644 // return MATH_MAD(-y0*e, MATH_MAD(e, 0.375, 0.5), -y0);
645 //
646 // The rsq instruction handles the special cases correctly. We need to check
647 // for the edge case conditions to ensure the special case propagates through
648 // the later instructions.
649
650 Value *Y0 = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rsq, Op: X);
651
652 // Try to elide the edge case check.
653 //
654 // Fast math flags imply:
655 // sqrt ninf => !isinf(x)
656 // fdiv ninf => x != 0, !isinf(x)
657 bool MaybePosInf = !SqrtFMF.noInfs() && !DivFMF.noInfs();
658 bool MaybeZero = !DivFMF.noInfs();
659
660 DenormalMode DenormMode;
661 FPClassTest Interested = fcNone;
662 if (MaybePosInf)
663 Interested = fcPosInf;
664 if (MaybeZero)
665 Interested |= fcZero;
666
667 if (Interested != fcNone) {
668 KnownFPClass KnownSrc = computeKnownFPClass(V: X, Interested, CtxI);
669 if (KnownSrc.isKnownNeverPosInfinity())
670 MaybePosInf = false;
671
672 DenormMode = F.getDenormalMode(FPType: X->getType()->getFltSemantics());
673 if (KnownSrc.isKnownNeverLogicalZero(Mode: DenormMode))
674 MaybeZero = false;
675 }
676
677 Value *SpecialOrRsq = X;
678 if (MaybeZero || MaybePosInf) {
679 Value *Cond;
680 if (MaybePosInf && MaybeZero) {
681 if (DenormMode.Input != DenormalMode::DenormalModeKind::Dynamic) {
682 FPClassTest TestMask = fcPosInf | fcZero;
683 if (DenormMode.inputsAreZero())
684 TestMask |= fcSubnormal;
685
686 Cond = Builder.createIsFPClass(FPNum: X, Test: TestMask);
687 } else {
688 // Avoid using llvm.is.fpclass for dynamic denormal mode, since it
689 // doesn't respect the floating-point environment.
690 Value *IsZero =
691 Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getZero(Ty: X->getType()));
692 Value *IsInf =
693 Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getInfinity(Ty: X->getType()));
694 Cond = Builder.CreateOr(LHS: IsZero, RHS: IsInf);
695 }
696 } else if (MaybeZero) {
697 Cond = Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getZero(Ty: X->getType()));
698 } else {
699 Cond = Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getInfinity(Ty: X->getType()));
700 }
701
702 SpecialOrRsq = Builder.CreateSelect(C: Cond, True: Y0, False: X);
703 }
704
705 Value *NegY0 = Builder.CreateFNeg(V: Y0);
706 Value *NegXY0 = Builder.CreateFMul(L: SpecialOrRsq, R: NegY0);
707
708 // Could be fmuladd, but isFMAFasterThanFMulAndFAdd is always true for f64.
709 Value *E = Builder.CreateFMA(Factor1: NegXY0, Factor2: Y0, Summand: ConstantFP::get(Ty: X->getType(), V: 1.0));
710
711 Value *Y0E = Builder.CreateFMul(L: E, R: IsNegative ? NegY0 : Y0);
712
713 Value *EFMA = Builder.CreateFMA(Factor1: E, Factor2: ConstantFP::get(Ty: X->getType(), V: 0.375),
714 Summand: ConstantFP::get(Ty: X->getType(), V: 0.5));
715
716 return Builder.CreateFMA(Factor1: Y0E, Factor2: EFMA, Summand: IsNegative ? NegY0 : Y0);
717}
718
719bool AMDGPUCodeGenPrepareImpl::canOptimizeWithRsq(FastMathFlags DivFMF,
720 FastMathFlags SqrtFMF) const {
721 // The rsqrt contraction increases accuracy from ~2ulp to ~1ulp for f32 and
722 // f64.
723 return DivFMF.allowContract() && SqrtFMF.allowContract();
724}
725
726Value *AMDGPUCodeGenPrepareImpl::optimizeWithRsq(
727 IRBuilder<> &Builder, Value *Num, Value *Den, const FastMathFlags DivFMF,
728 const FastMathFlags SqrtFMF, const Instruction *CtxI) const {
729 // The rsqrt contraction increases accuracy from ~2ulp to ~1ulp.
730 assert(DivFMF.allowContract() && SqrtFMF.allowContract());
731
732 // rsq_f16 is accurate to 0.51 ulp.
733 // rsq_f32 is accurate for !fpmath >= 1.0ulp and denormals are flushed.
734 // rsq_f64 is never accurate.
735 const ConstantFP *CLHS = dyn_cast<ConstantFP>(Val: Num);
736 if (!CLHS)
737 return nullptr;
738
739 bool IsNegative = false;
740
741 // TODO: Handle other numerator values with arcp.
742 if (CLHS->isOne() || (IsNegative = CLHS->isMinusOne())) {
743 // Add sqrt flags, but require both ninf and nsz from the div and the
744 // sqrt: sqrt's ninf/nsz don't say anything about the quotient.
745 IRBuilder<>::FastMathFlagGuard Guard(Builder);
746 FastMathFlags NewFMF = DivFMF | SqrtFMF;
747 FastMathFlags ValueFMF = FastMathFlags::intersectValue(LHS: DivFMF, RHS: SqrtFMF);
748 NewFMF.setNoInfs(ValueFMF.noInfs());
749 NewFMF.setNoSignedZeros(ValueFMF.noSignedZeros());
750 Builder.setFastMathFlags(NewFMF);
751
752 if (Den->getType()->isFloatTy()) {
753 if ((DivFMF.approxFunc() && SqrtFMF.approxFunc()) ||
754 canIgnoreDenormalInput(V: Den, CtxI)) {
755 Value *Result =
756 Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rsq, Op: Den);
757 // -1.0 / sqrt(x) -> fneg(rsq(x))
758 return IsNegative ? Builder.CreateFNeg(V: Result) : Result;
759 }
760
761 return emitRsqIEEE1ULP(Builder, Src: Den, IsNegative);
762 }
763
764 if (Den->getType()->isDoubleTy())
765 return emitRsqF64(Builder, X: Den, SqrtFMF, DivFMF, CtxI, IsNegative);
766 }
767
768 return nullptr;
769}
770
771// Optimize fdiv with rcp:
772//
773// 1/x -> rcp(x) when rcp is sufficiently accurate or inaccurate rcp is
774// allowed with afn.
775//
776// a/b -> a*rcp(b) when arcp is allowed, and we only need provide ULP 1.0
777Value *
778AMDGPUCodeGenPrepareImpl::optimizeWithRcp(IRBuilder<> &Builder, Value *Num,
779 Value *Den, FastMathFlags FMF,
780 const Instruction *CtxI) const {
781 // rcp_f16 is accurate to 0.51 ulp.
782 // rcp_f32 is accurate for !fpmath >= 1.0ulp and denormals are flushed.
783 // rcp_f64 is never accurate.
784 assert(Den->getType()->isFloatTy());
785
786 if (const ConstantFP *CLHS = dyn_cast<ConstantFP>(Val: Num)) {
787 bool IsNegative = false;
788 if (CLHS->isOne() || (IsNegative = CLHS->isMinusOne())) {
789 Value *Src = Den;
790
791 if (HasFP32DenormalFlush || FMF.approxFunc()) {
792 // -1.0 / x -> 1.0 / fneg(x)
793 if (IsNegative)
794 Src = Builder.CreateFNeg(V: Src);
795
796 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
797 // the CI documentation has a worst case error of 1 ulp.
798 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK
799 // to use it as long as we aren't trying to use denormals.
800 //
801 // v_rcp_f16 and v_rsq_f16 DO support denormals.
802
803 // NOTE: v_sqrt and v_rcp will be combined to v_rsq later. So we don't
804 // insert rsq intrinsic here.
805
806 // 1.0 / x -> rcp(x)
807 return Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: Src);
808 }
809
810 // TODO: If the input isn't denormal, and we know the input exponent isn't
811 // big enough to introduce a denormal we can avoid the scaling.
812 return emitRcpIEEE1ULP(Builder, Src, IsNegative);
813 }
814 }
815
816 if (FMF.allowReciprocal()) {
817 // x / y -> x * (1.0 / y)
818
819 // TODO: Could avoid denormal scaling and use raw rcp if we knew the output
820 // will never underflow.
821 if (HasFP32DenormalFlush || FMF.approxFunc()) {
822 Value *Recip = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: Den);
823 return Builder.CreateFMul(L: Num, R: Recip);
824 }
825
826 Value *Recip = emitRcpIEEE1ULP(Builder, Src: Den, IsNegative: false);
827 return Builder.CreateFMul(L: Num, R: Recip);
828 }
829
830 return nullptr;
831}
832
833// optimize with fdiv.fast:
834//
835// a/b -> fdiv.fast(a, b) when !fpmath >= 2.5ulp with denormals flushed.
836//
837// 1/x -> fdiv.fast(1,x) when !fpmath >= 2.5ulp.
838//
839// NOTE: optimizeWithRcp should be tried first because rcp is the preference.
840Value *AMDGPUCodeGenPrepareImpl::optimizeWithFDivFast(
841 IRBuilder<> &Builder, Value *Num, Value *Den, float ReqdAccuracy) const {
842 // fdiv.fast can achieve 2.5 ULP accuracy.
843 if (ReqdAccuracy < 2.5f)
844 return nullptr;
845
846 // Only have fdiv.fast for f32.
847 assert(Den->getType()->isFloatTy());
848
849 bool NumIsOne = false;
850 if (const ConstantFP *CNum = dyn_cast<ConstantFP>(Val: Num)) {
851 if (CNum->isOne() || CNum->isMinusOne())
852 NumIsOne = true;
853 }
854
855 // fdiv does not support denormals. But 1.0/x is always fine to use it.
856 //
857 // TODO: This works for any value with a specific known exponent range, don't
858 // just limit to constant 1.
859 if (!HasFP32DenormalFlush && !NumIsOne)
860 return nullptr;
861
862 return Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_fdiv_fast, Args: {Num, Den});
863}
864
865Value *AMDGPUCodeGenPrepareImpl::visitFDivElement(
866 IRBuilder<> &Builder, Value *Num, Value *Den, FastMathFlags DivFMF,
867 FastMathFlags SqrtFMF, Value *RsqOp, const Instruction *FDivInst,
868 float ReqdDivAccuracy) const {
869 if (RsqOp) {
870 Value *Rsq =
871 optimizeWithRsq(Builder, Num, Den: RsqOp, DivFMF, SqrtFMF, CtxI: FDivInst);
872 if (Rsq)
873 return Rsq;
874 }
875
876 if (!Num->getType()->isFloatTy())
877 return nullptr;
878
879 Value *Rcp = optimizeWithRcp(Builder, Num, Den, FMF: DivFMF, CtxI: FDivInst);
880 if (Rcp)
881 return Rcp;
882
883 // In the basic case fdiv_fast has the same instruction count as the frexp div
884 // expansion. Slightly prefer fdiv_fast since it ends in an fmul that can
885 // potentially be fused into a user. Also, materialization of the constants
886 // can be reused for multiple instances.
887 Value *FDivFast = optimizeWithFDivFast(Builder, Num, Den, ReqdAccuracy: ReqdDivAccuracy);
888 if (FDivFast)
889 return FDivFast;
890
891 return emitFrexpDiv(Builder, LHS: Num, RHS: Den, FMF: DivFMF);
892}
893
894// Optimizations is performed based on fpmath, fast math flags as well as
895// denormals to optimize fdiv with either rcp or fdiv.fast.
896//
897// With rcp:
898// 1/x -> rcp(x) when rcp is sufficiently accurate or inaccurate rcp is
899// allowed with afn.
900//
901// a/b -> a*rcp(b) when inaccurate rcp is allowed with afn.
902//
903// With fdiv.fast:
904// a/b -> fdiv.fast(a, b) when !fpmath >= 2.5ulp with denormals flushed.
905//
906// 1/x -> fdiv.fast(1,x) when !fpmath >= 2.5ulp.
907//
908// NOTE: rcp is the preference in cases that both are legal.
909bool AMDGPUCodeGenPrepareImpl::visitFDiv(BinaryOperator &FDiv) {
910 if (DisableFDivExpand)
911 return false;
912
913 Type *Ty = FDiv.getType()->getScalarType();
914 const bool IsFloat = Ty->isFloatTy();
915 if (!IsFloat && !Ty->isDoubleTy())
916 return false;
917
918 // The f64 rcp/rsq approximations are pretty inaccurate. We can do an
919 // expansion around them in codegen. f16 is good enough to always use.
920
921 const FPMathOperator *FPOp = cast<const FPMathOperator>(Val: &FDiv);
922 const FastMathFlags DivFMF = FPOp->getFastMathFlags();
923 const float ReqdAccuracy = FPOp->getFPAccuracy();
924
925 FastMathFlags SqrtFMF;
926
927 Value *Num = FDiv.getOperand(i_nocapture: 0);
928 Value *Den = FDiv.getOperand(i_nocapture: 1);
929
930 Value *RsqOp = nullptr;
931 auto *DenII = dyn_cast<IntrinsicInst>(Val: Den);
932 if (DenII && DenII->getIntrinsicID() == Intrinsic::sqrt &&
933 DenII->hasOneUse()) {
934 const auto *SqrtOp = cast<FPMathOperator>(Val: DenII);
935 SqrtFMF = SqrtOp->getFastMathFlags();
936 if (canOptimizeWithRsq(DivFMF, SqrtFMF))
937 RsqOp = SqrtOp->getOperand(i: 0);
938 }
939
940 // rcp path not yet implemented for f64.
941 if (!IsFloat && !RsqOp)
942 return false;
943
944 // Inaccurate rcp is allowed with afn.
945 //
946 // Defer to codegen to handle this.
947 //
948 // TODO: Decide on an interpretation for interactions between afn + arcp +
949 // !fpmath, and make it consistent between here and codegen. For now, defer
950 // expansion of afn to codegen. The current interpretation is so aggressive we
951 // don't need any pre-consideration here when we have better information. A
952 // more conservative interpretation could use handling here.
953 const bool AllowInaccurateRcp = DivFMF.approxFunc();
954 if (!RsqOp && AllowInaccurateRcp)
955 return false;
956
957 // Defer the correct implementations to codegen.
958 if (IsFloat && ReqdAccuracy < 1.0f)
959 return false;
960
961 IRBuilder<> Builder(FDiv.getParent(), std::next(x: FDiv.getIterator()));
962 Builder.setFastMathFlags(DivFMF);
963 Builder.SetCurrentDebugLocation(FDiv.getDebugLoc());
964
965 SmallVector<Value *, 4> NumVals;
966 SmallVector<Value *, 4> DenVals;
967 SmallVector<Value *, 4> RsqDenVals;
968 extractValues(Builder, Values&: NumVals, V: Num);
969 extractValues(Builder, Values&: DenVals, V: Den);
970
971 if (RsqOp)
972 extractValues(Builder, Values&: RsqDenVals, V: RsqOp);
973
974 SmallVector<Value *, 4> ResultVals(NumVals.size());
975 for (int I = 0, E = NumVals.size(); I != E; ++I) {
976 Value *NumElt = NumVals[I];
977 Value *DenElt = DenVals[I];
978 Value *RsqDenElt = RsqOp ? RsqDenVals[I] : nullptr;
979
980 Value *NewElt =
981 visitFDivElement(Builder, Num: NumElt, Den: DenElt, DivFMF, SqrtFMF, RsqOp: RsqDenElt,
982 FDivInst: cast<Instruction>(Val: FPOp), ReqdDivAccuracy: ReqdAccuracy);
983 if (!NewElt) {
984 // Keep the original, but scalarized.
985
986 // This has the unfortunate side effect of sometimes scalarizing when
987 // we're not going to do anything.
988 NewElt = Builder.CreateFDiv(L: NumElt, R: DenElt);
989 if (auto *NewEltInst = dyn_cast<Instruction>(Val: NewElt))
990 NewEltInst->copyMetadata(SrcInst: FDiv);
991 }
992
993 ResultVals[I] = NewElt;
994 }
995
996 Value *NewVal = insertValues(Builder, Ty: FDiv.getType(), Values&: ResultVals);
997
998 if (NewVal) {
999 FDiv.replaceAllUsesWith(V: NewVal);
1000 NewVal->takeName(V: &FDiv);
1001 DeadVals.push_back(Elt: &FDiv);
1002 }
1003
1004 return true;
1005}
1006
1007static std::pair<Value*, Value*> getMul64(IRBuilder<> &Builder,
1008 Value *LHS, Value *RHS) {
1009 Type *I32Ty = Builder.getInt32Ty();
1010 Type *I64Ty = Builder.getInt64Ty();
1011
1012 Value *LHS_EXT64 = Builder.CreateZExt(V: LHS, DestTy: I64Ty);
1013 Value *RHS_EXT64 = Builder.CreateZExt(V: RHS, DestTy: I64Ty);
1014 Value *MUL64 = Builder.CreateMul(LHS: LHS_EXT64, RHS: RHS_EXT64);
1015 Value *Lo = Builder.CreateTrunc(V: MUL64, DestTy: I32Ty);
1016 Value *Hi = Builder.CreateLShr(LHS: MUL64, RHS: Builder.getInt64(C: 32));
1017 Hi = Builder.CreateTrunc(V: Hi, DestTy: I32Ty);
1018 return std::pair(Lo, Hi);
1019}
1020
1021static Value* getMulHu(IRBuilder<> &Builder, Value *LHS, Value *RHS) {
1022 return getMul64(Builder, LHS, RHS).second;
1023}
1024
1025/// Figure out how many bits are really needed for this division.
1026/// \p MaxDivBits is an optimization hint to bypass the second
1027/// ComputeNumSignBits/computeKnownBits call if the first one is
1028/// insufficient.
1029unsigned AMDGPUCodeGenPrepareImpl::getDivNumBits(BinaryOperator &I, Value *Num,
1030 Value *Den,
1031 unsigned MaxDivBits,
1032 bool IsSigned) const {
1033 assert(Num->getType()->getScalarSizeInBits() ==
1034 Den->getType()->getScalarSizeInBits());
1035 unsigned SSBits = Num->getType()->getScalarSizeInBits();
1036 if (IsSigned) {
1037 unsigned RHSSignBits = ComputeNumSignBits(Op: Den, DL: SQ.DL, AC: SQ.AC, CxtI: &I, DT: SQ.DT);
1038 // A sign bit needs to be reserved for shrinking.
1039 unsigned DivBits = SSBits - RHSSignBits + 1;
1040 if (DivBits > MaxDivBits)
1041 return SSBits;
1042
1043 unsigned LHSSignBits = ComputeNumSignBits(Op: Num, DL: SQ.DL, AC: SQ.AC, CxtI: &I);
1044
1045 unsigned SignBits = std::min(a: LHSSignBits, b: RHSSignBits);
1046 DivBits = SSBits - SignBits + 1;
1047 return DivBits;
1048 }
1049
1050 // All bits are used for unsigned division for Num or Den in range
1051 // (SignedMax, UnsignedMax].
1052 KnownBits Known = computeKnownBits(V: Den, Q: SQ.getWithInstruction(I: &I));
1053 unsigned RHSBits = Known.countMaxActiveBits();
1054 if (RHSBits > MaxDivBits)
1055 return SSBits;
1056
1057 Known = computeKnownBits(V: Num, Q: SQ.getWithInstruction(I: &I));
1058 unsigned LHSBits = Known.countMaxActiveBits();
1059
1060 unsigned DivBits = std::max(a: LHSBits, b: RHSBits);
1061 return DivBits;
1062}
1063
1064Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloat(IRBuilder<> &Builder,
1065 BinaryOperator &I,
1066 Value *Num, Value *Den,
1067 bool IsDiv,
1068 bool IsSigned) const {
1069 unsigned DivBits = getDivNumBits(I, Num, Den, MaxDivBits: 23, IsSigned);
1070
1071 if (DivBits > (IsSigned ? 23 : 22))
1072 return nullptr;
1073 return expandDivRemToFloatImpl(Builder, I, Num, Den, NumBits: DivBits, IsDiv,
1074 IsSigned);
1075}
1076
1077Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
1078 IRBuilder<> &Builder, BinaryOperator &I, Value *Num, Value *Den,
1079 unsigned DivBits, bool IsDiv, bool IsSigned) const {
1080
1081 // v_rcp_f32(float(X)) can have an error of 1 ulp.
1082 // This would cause incorrect calculation of Y/X if:
1083 // Y = (0x7FFFFF/X)*(X-0)-1
1084 // were allowed.
1085 //
1086 // For example,
1087 // (0x7FF6D3/0x000FE7) would erroneously produce 2060 instead of 2059.
1088 // (0x7FF8F5/0x007EFB) would erroneously produce 258 instead of 257.
1089 //
1090 // Thus, we conservatively restrict expandDivRemToFloatImpl to
1091 // [-0x400000,0x3FFFFF] for IsSigned
1092 // [ 0x000000,0x3FFFFF] for !IsSigned.
1093 assert(0 < DivBits && DivBits <= (IsSigned ? 23 : 22) &&
1094 "abs(Num) must be <= 0x400000 for expandDivRemToFloatImpl to work "
1095 "correctly");
1096
1097 Type *I32Ty = Builder.getInt32Ty();
1098 Num = Builder.CreateTrunc(V: Num, DestTy: I32Ty);
1099 Den = Builder.CreateTrunc(V: Den, DestTy: I32Ty);
1100
1101 Type *F32Ty = Builder.getFloatTy();
1102 ConstantInt *One = Builder.getInt32(C: 1);
1103
1104 // int ia = (int)LHS;
1105 Value *IA = Num;
1106
1107 // int ib, (int)RHS;
1108 Value *IB = Den;
1109
1110 // float fa = (float)ia;
1111 Value *FA = IsSigned ? Builder.CreateSIToFP(V: IA, DestTy: F32Ty)
1112 : Builder.CreateUIToFP(V: IA, DestTy: F32Ty);
1113
1114 // float fb = (float)ib;
1115 Value *FB = IsSigned ? Builder.CreateSIToFP(V: IB, DestTy: F32Ty)
1116 : Builder.CreateUIToFP(V: IB, DestTy: F32Ty);
1117
1118 Value *RCP = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_rcp,
1119 OverloadTypes: Builder.getFloatTy(), Args: {FB});
1120
1121 // The calculation:
1122 // fq = fa*recip(fb)
1123 // may be too small due to the 1ulp accuracy in the recip
1124 // operation and rounding issues. Since fq is truncated to produce
1125 // an integer value it may be too small by one. This is
1126 // dealt with by incrementing fa by 1ulp:
1127 // fq = (fa+1ulp)*recip(fb)
1128 // This will increase fa's magnitude by at most 0.5
1129 // (i.e. when fabs(fa)==0x400000 the LSB of the mantissa represents 0.5).
1130 // Thus, this method is safe since fa must be incremented by at least 1.0
1131 // for the quotient to increase by one.
1132
1133 Value *FABits = Builder.CreateBitCast(V: FA, DestTy: I32Ty);
1134 Value *FABitsInc = Builder.CreateAdd(LHS: FABits, RHS: One);
1135 FA = Builder.CreateBitCast(V: FABitsInc, DestTy: F32Ty);
1136
1137 Value *FQM = Builder.CreateFMul(L: FA, R: RCP);
1138
1139 // fq = trunc(fqm);
1140 Value *FQ = Builder.CreateUnaryIntrinsic(ID: Intrinsic::trunc, Op: FQM);
1141
1142 // int iq = (int)fq;
1143 Value *IQ = IsSigned ? Builder.CreateFPToSI(V: FQ, DestTy: I32Ty)
1144 : Builder.CreateFPToUI(V: FQ, DestTy: I32Ty);
1145
1146 Value *Res = IQ;
1147 if (!IsDiv) {
1148 // Rem needs compensation, it's easier to recompute it
1149 Value *Rem = Builder.CreateMul(LHS: IQ, RHS: Den);
1150 Res = Builder.CreateSub(LHS: Num, RHS: Rem);
1151 }
1152
1153 return Res;
1154}
1155
1156// Try to recognize special cases the DAG will emit special, better expansions
1157// than the general expansion we do here.
1158
1159// TODO: It would be better to just directly handle those optimizations here.
1160bool AMDGPUCodeGenPrepareImpl::divHasSpecialOptimization(BinaryOperator &I,
1161 Value *Num,
1162 Value *Den) const {
1163 if (Constant *C = dyn_cast<Constant>(Val: Den)) {
1164 // Arbitrary constants get a better expansion as long as a wider mulhi is
1165 // legal.
1166 if (C->getType()->getScalarSizeInBits() <= 32)
1167 return true;
1168
1169 // TODO: Sdiv check for not exact for some reason.
1170
1171 // If there's no wider mulhi, there's only a better expansion for powers of
1172 // two.
1173 // TODO: Should really know for each vector element.
1174 if (isKnownToBeAPowerOfTwo(V: C, OrZero: true, Q: SQ.getWithInstruction(I: &I)))
1175 return true;
1176
1177 return false;
1178 }
1179
1180 if (BinaryOperator *BinOpDen = dyn_cast<BinaryOperator>(Val: Den)) {
1181 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
1182 if (BinOpDen->getOpcode() == Instruction::Shl &&
1183 isa<Constant>(Val: BinOpDen->getOperand(i_nocapture: 0)) &&
1184 isKnownToBeAPowerOfTwo(V: BinOpDen->getOperand(i_nocapture: 0), OrZero: true,
1185 Q: SQ.getWithInstruction(I: &I))) {
1186 return true;
1187 }
1188 }
1189
1190 return false;
1191}
1192
1193static Value *getSign32(Value *V, IRBuilder<> &Builder, const DataLayout DL) {
1194 // Check whether the sign can be determined statically.
1195 KnownBits Known = computeKnownBits(V, DL);
1196 if (Known.isNegative())
1197 return Constant::getAllOnesValue(Ty: V->getType());
1198 if (Known.isNonNegative())
1199 return Constant::getNullValue(Ty: V->getType());
1200 return Builder.CreateAShr(LHS: V, RHS: Builder.getInt32(C: 31));
1201}
1202
1203Value *AMDGPUCodeGenPrepareImpl::expandDivRem32(IRBuilder<> &Builder,
1204 BinaryOperator &I, Value *X,
1205 Value *Y) const {
1206 Instruction::BinaryOps Opc = I.getOpcode();
1207 assert(Opc == Instruction::URem || Opc == Instruction::UDiv ||
1208 Opc == Instruction::SRem || Opc == Instruction::SDiv);
1209
1210 FastMathFlags FMF;
1211 FMF.setFast();
1212 Builder.setFastMathFlags(FMF);
1213
1214 if (divHasSpecialOptimization(I, Num: X, Den: Y))
1215 return nullptr; // Keep it for later optimization.
1216
1217 bool IsDiv = Opc == Instruction::UDiv || Opc == Instruction::SDiv;
1218 bool IsSigned = Opc == Instruction::SRem || Opc == Instruction::SDiv;
1219
1220 Type *Ty = X->getType();
1221 Type *I32Ty = Builder.getInt32Ty();
1222 Type *F32Ty = Builder.getFloatTy();
1223
1224 if (Ty->getScalarSizeInBits() != 32) {
1225 if (IsSigned) {
1226 X = Builder.CreateSExtOrTrunc(V: X, DestTy: I32Ty);
1227 Y = Builder.CreateSExtOrTrunc(V: Y, DestTy: I32Ty);
1228 } else {
1229 X = Builder.CreateZExtOrTrunc(V: X, DestTy: I32Ty);
1230 Y = Builder.CreateZExtOrTrunc(V: Y, DestTy: I32Ty);
1231 }
1232 }
1233
1234 if (Value *Res = expandDivRemToFloat(Builder, I, Num: X, Den: Y, IsDiv, IsSigned)) {
1235 return IsSigned ? Builder.CreateSExtOrTrunc(V: Res, DestTy: Ty) :
1236 Builder.CreateZExtOrTrunc(V: Res, DestTy: Ty);
1237 }
1238
1239 ConstantInt *Zero = Builder.getInt32(C: 0);
1240 ConstantInt *One = Builder.getInt32(C: 1);
1241
1242 Value *Sign = nullptr;
1243 if (IsSigned) {
1244 Value *SignX = getSign32(V: X, Builder, DL);
1245 Value *SignY = getSign32(V: Y, Builder, DL);
1246 // Remainder sign is the same as LHS
1247 Sign = IsDiv ? Builder.CreateXor(LHS: SignX, RHS: SignY) : SignX;
1248
1249 X = Builder.CreateAdd(LHS: X, RHS: SignX);
1250 Y = Builder.CreateAdd(LHS: Y, RHS: SignY);
1251
1252 X = Builder.CreateXor(LHS: X, RHS: SignX);
1253 Y = Builder.CreateXor(LHS: Y, RHS: SignY);
1254 }
1255
1256 // The algorithm here is based on ideas from "Software Integer Division", Tom
1257 // Rodeheffer, August 2008.
1258 //
1259 // unsigned udiv(unsigned x, unsigned y) {
1260 // // Initial estimate of inv(y). The constant is less than 2^32 to ensure
1261 // // that this is a lower bound on inv(y), even if some of the calculations
1262 // // round up.
1263 // unsigned z = (unsigned)((4294967296.0 - 512.0) * v_rcp_f32((float)y));
1264 //
1265 // // One round of UNR (Unsigned integer Newton-Raphson) to improve z.
1266 // // Empirically this is guaranteed to give a "two-y" lower bound on
1267 // // inv(y).
1268 // z += umulh(z, -y * z);
1269 //
1270 // // Quotient/remainder estimate.
1271 // unsigned q = umulh(x, z);
1272 // unsigned r = x - q * y;
1273 //
1274 // // Two rounds of quotient/remainder refinement.
1275 // if (r >= y) {
1276 // ++q;
1277 // r -= y;
1278 // }
1279 // if (r >= y) {
1280 // ++q;
1281 // r -= y;
1282 // }
1283 //
1284 // return q;
1285 // }
1286
1287 // Initial estimate of inv(y).
1288 Value *FloatY = Builder.CreateUIToFP(V: Y, DestTy: F32Ty);
1289 Value *RcpY = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_rcp, OverloadTypes: F32Ty, Args: {FloatY});
1290 Constant *Scale = ConstantFP::get(Ty: F32Ty, V: llvm::bit_cast<float>(from: 0x4F7FFFFE));
1291 Value *ScaledY = Builder.CreateFMul(L: RcpY, R: Scale);
1292 Value *Z = Builder.CreateFPToUI(V: ScaledY, DestTy: I32Ty);
1293
1294 // One round of UNR.
1295 Value *NegY = Builder.CreateSub(LHS: Zero, RHS: Y);
1296 Value *NegYZ = Builder.CreateMul(LHS: NegY, RHS: Z);
1297 Z = Builder.CreateAdd(LHS: Z, RHS: getMulHu(Builder, LHS: Z, RHS: NegYZ));
1298
1299 // Quotient/remainder estimate.
1300 Value *Q = getMulHu(Builder, LHS: X, RHS: Z);
1301 Value *R = Builder.CreateSub(LHS: X, RHS: Builder.CreateMul(LHS: Q, RHS: Y));
1302
1303 // First quotient/remainder refinement.
1304 Value *Cond = Builder.CreateICmpUGE(LHS: R, RHS: Y);
1305 if (IsDiv)
1306 Q = Builder.CreateSelect(C: Cond, True: Builder.CreateAdd(LHS: Q, RHS: One), False: Q);
1307 R = Builder.CreateSelect(C: Cond, True: Builder.CreateSub(LHS: R, RHS: Y), False: R);
1308
1309 // Second quotient/remainder refinement.
1310 Cond = Builder.CreateICmpUGE(LHS: R, RHS: Y);
1311 Value *Res;
1312 if (IsDiv)
1313 Res = Builder.CreateSelect(C: Cond, True: Builder.CreateAdd(LHS: Q, RHS: One), False: Q);
1314 else
1315 Res = Builder.CreateSelect(C: Cond, True: Builder.CreateSub(LHS: R, RHS: Y), False: R);
1316
1317 if (IsSigned) {
1318 Res = Builder.CreateXor(LHS: Res, RHS: Sign);
1319 Res = Builder.CreateSub(LHS: Res, RHS: Sign);
1320 Res = Builder.CreateSExtOrTrunc(V: Res, DestTy: Ty);
1321 } else {
1322 Res = Builder.CreateZExtOrTrunc(V: Res, DestTy: Ty);
1323 }
1324 return Res;
1325}
1326
1327Value *AMDGPUCodeGenPrepareImpl::shrinkDivRem64(IRBuilder<> &Builder,
1328 BinaryOperator &I, Value *Num,
1329 Value *Den) const {
1330 if (!ExpandDiv64InIR && divHasSpecialOptimization(I, Num, Den))
1331 return nullptr; // Keep it for later optimization.
1332
1333 Instruction::BinaryOps Opc = I.getOpcode();
1334
1335 bool IsDiv = Opc == Instruction::SDiv || Opc == Instruction::UDiv;
1336 bool IsSigned = Opc == Instruction::SDiv || Opc == Instruction::SRem;
1337
1338 unsigned NumDivBits = getDivNumBits(I, Num, Den, MaxDivBits: 32, IsSigned);
1339 if (NumDivBits > 32)
1340 return nullptr;
1341
1342 Value *Narrowed = nullptr;
1343 if (NumDivBits <= (IsSigned ? 23 : 22)) {
1344 Narrowed = expandDivRemToFloatImpl(Builder, I, Num, Den, DivBits: NumDivBits, IsDiv,
1345 IsSigned);
1346 } else if (NumDivBits <= (IsSigned ? 31 : 32)) {
1347 // Do not use 32-bit division if dividend may be -2147483648.
1348 // Otherwise 32-bit division cannot be used safely.
1349 // -2147483648/1 and -2147483648/-1 are not equal,
1350 // but they produce the same lower 32-bit result.
1351 Narrowed = expandDivRem32(Builder, I, X: Num, Y: Den);
1352 }
1353
1354 if (Narrowed) {
1355 return IsSigned ? Builder.CreateSExt(V: Narrowed, DestTy: Num->getType()) :
1356 Builder.CreateZExt(V: Narrowed, DestTy: Num->getType());
1357 }
1358
1359 return nullptr;
1360}
1361
1362void AMDGPUCodeGenPrepareImpl::expandDivRem64(BinaryOperator &I) const {
1363 Instruction::BinaryOps Opc = I.getOpcode();
1364 // Do the general expansion.
1365 if (Opc == Instruction::UDiv || Opc == Instruction::SDiv) {
1366 expandDivisionUpTo64Bits(Div: &I);
1367 return;
1368 }
1369
1370 if (Opc == Instruction::URem || Opc == Instruction::SRem) {
1371 expandRemainderUpTo64Bits(Rem: &I);
1372 return;
1373 }
1374
1375 llvm_unreachable("not a division");
1376}
1377
1378/*
1379This will cause non-byte load in consistency, for example:
1380```
1381 %load = load i1, ptr addrspace(4) %arg, align 4
1382 %zext = zext i1 %load to
1383 i64 %add = add i64 %zext
1384```
1385Instead of creating `s_and_b32 s0, s0, 1`,
1386it will create `s_and_b32 s0, s0, 0xff`.
1387We accept this change since the non-byte load assumes the upper bits
1388within the byte are all 0.
1389*/
1390bool AMDGPUCodeGenPrepareImpl::tryNarrowMathIfNoOverflow(Instruction *I) {
1391 unsigned Opc = I->getOpcode();
1392 Type *OldType = I->getType();
1393
1394 if (Opc != Instruction::Add && Opc != Instruction::Mul)
1395 return false;
1396
1397 unsigned OrigBit = OldType->getScalarSizeInBits();
1398
1399 if (Opc != Instruction::Add && Opc != Instruction::Mul)
1400 llvm_unreachable("Unexpected opcode, only valid for Instruction::Add and "
1401 "Instruction::Mul.");
1402
1403 unsigned MaxBitsNeeded = computeKnownBits(V: I, DL).countMaxActiveBits();
1404
1405 MaxBitsNeeded = std::max<unsigned>(a: bit_ceil(Value: MaxBitsNeeded), b: 8);
1406 Type *NewType = DL.getSmallestLegalIntType(C&: I->getContext(), Width: MaxBitsNeeded);
1407 if (!NewType)
1408 return false;
1409 unsigned NewBit = NewType->getIntegerBitWidth();
1410 if (NewBit >= OrigBit)
1411 return false;
1412 NewType = I->getType()->getWithNewBitWidth(NewBitWidth: NewBit);
1413
1414 // Old cost
1415 InstructionCost OldCost =
1416 TTI.getArithmeticInstrCost(Opcode: Opc, Ty: OldType, CostKind: TTI::TCK_RecipThroughput);
1417 // New cost of new op
1418 InstructionCost NewCost =
1419 TTI.getArithmeticInstrCost(Opcode: Opc, Ty: NewType, CostKind: TTI::TCK_RecipThroughput);
1420 // New cost of narrowing 2 operands (use trunc)
1421 int NumOfNonConstOps = 2;
1422 if (isa<Constant>(Val: I->getOperand(i: 0)) || isa<Constant>(Val: I->getOperand(i: 1))) {
1423 // Cannot be both constant, should be propagated
1424 NumOfNonConstOps = 1;
1425 }
1426 NewCost += NumOfNonConstOps * TTI.getCastInstrCost(Opcode: Instruction::Trunc,
1427 Dst: NewType, Src: OldType,
1428 CCH: TTI.getCastContextHint(I),
1429 CostKind: TTI::TCK_RecipThroughput);
1430 // New cost of zext narrowed result to original type
1431 NewCost +=
1432 TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: OldType, Src: NewType,
1433 CCH: TTI.getCastContextHint(I), CostKind: TTI::TCK_RecipThroughput);
1434 if (NewCost >= OldCost)
1435 return false;
1436
1437 IRBuilder<> Builder(I);
1438 Value *Trunc0 = Builder.CreateTrunc(V: I->getOperand(i: 0), DestTy: NewType);
1439 Value *Trunc1 = Builder.CreateTrunc(V: I->getOperand(i: 1), DestTy: NewType);
1440 Value *Arith =
1441 Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Opc, LHS: Trunc0, RHS: Trunc1);
1442
1443 Value *Zext = Builder.CreateZExt(V: Arith, DestTy: OldType);
1444 I->replaceAllUsesWith(V: Zext);
1445 DeadVals.push_back(Elt: I);
1446 return true;
1447}
1448
1449bool AMDGPUCodeGenPrepareImpl::visitBinaryOperator(BinaryOperator &I) {
1450 if (foldBinOpIntoSelect(BO&: I))
1451 return true;
1452
1453 if (UseMul24Intrin && replaceMulWithMul24(I))
1454 return true;
1455 if (tryNarrowMathIfNoOverflow(I: &I))
1456 return true;
1457
1458 bool Changed = false;
1459 Instruction::BinaryOps Opc = I.getOpcode();
1460 Type *Ty = I.getType();
1461 Value *NewDiv = nullptr;
1462 unsigned ScalarSize = Ty->getScalarSizeInBits();
1463
1464 SmallVector<BinaryOperator *, 8> Div64ToExpand;
1465
1466 if ((Opc == Instruction::URem || Opc == Instruction::UDiv ||
1467 Opc == Instruction::SRem || Opc == Instruction::SDiv) &&
1468 ScalarSize <= 64 &&
1469 !DisableIDivExpand) {
1470 Value *Num = I.getOperand(i_nocapture: 0);
1471 Value *Den = I.getOperand(i_nocapture: 1);
1472 IRBuilder<> Builder(&I);
1473 Builder.SetCurrentDebugLocation(I.getDebugLoc());
1474
1475 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1476 NewDiv = PoisonValue::get(T: VT);
1477
1478 for (unsigned N = 0, E = VT->getNumElements(); N != E; ++N) {
1479 Value *NumEltN = Builder.CreateExtractElement(Vec: Num, Idx: N);
1480 Value *DenEltN = Builder.CreateExtractElement(Vec: Den, Idx: N);
1481
1482 Value *NewElt;
1483 if (ScalarSize <= 32) {
1484 NewElt = expandDivRem32(Builder, I, X: NumEltN, Y: DenEltN);
1485 if (!NewElt)
1486 NewElt = Builder.CreateBinOp(Opc, LHS: NumEltN, RHS: DenEltN);
1487 } else {
1488 // See if this 64-bit division can be shrunk to 32/24-bits before
1489 // producing the general expansion.
1490 NewElt = shrinkDivRem64(Builder, I, Num: NumEltN, Den: DenEltN);
1491 if (!NewElt) {
1492 // The general 64-bit expansion introduces control flow and doesn't
1493 // return the new value. Just insert a scalar copy and defer
1494 // expanding it.
1495 NewElt = Builder.CreateBinOp(Opc, LHS: NumEltN, RHS: DenEltN);
1496 // CreateBinOp does constant folding. If the operands are constant,
1497 // it will return a Constant instead of a BinaryOperator.
1498 if (auto *NewEltBO = dyn_cast<BinaryOperator>(Val: NewElt))
1499 Div64ToExpand.push_back(Elt: NewEltBO);
1500 }
1501 }
1502
1503 if (auto *NewEltI = dyn_cast<Instruction>(Val: NewElt))
1504 NewEltI->copyIRFlags(V: &I);
1505
1506 NewDiv = Builder.CreateInsertElement(Vec: NewDiv, NewElt, Idx: N);
1507 }
1508 } else {
1509 if (ScalarSize <= 32)
1510 NewDiv = expandDivRem32(Builder, I, X: Num, Y: Den);
1511 else {
1512 NewDiv = shrinkDivRem64(Builder, I, Num, Den);
1513 if (!NewDiv)
1514 Div64ToExpand.push_back(Elt: &I);
1515 }
1516 }
1517
1518 if (NewDiv) {
1519 I.replaceAllUsesWith(V: NewDiv);
1520 DeadVals.push_back(Elt: &I);
1521 Changed = true;
1522 }
1523 }
1524
1525 if (ExpandDiv64InIR) {
1526 // TODO: We get much worse code in specially handled constant cases.
1527 for (BinaryOperator *Div : Div64ToExpand) {
1528 expandDivRem64(I&: *Div);
1529 FlowChanged = true;
1530 Changed = true;
1531 }
1532 }
1533
1534 return Changed;
1535}
1536
1537bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) {
1538 if (!WidenLoads)
1539 return false;
1540
1541 if ((I.getPointerAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
1542 I.getPointerAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
1543 canWidenScalarExtLoad(I)) {
1544 IRBuilder<> Builder(&I);
1545 Builder.SetCurrentDebugLocation(I.getDebugLoc());
1546
1547 Type *I32Ty = Builder.getInt32Ty();
1548 LoadInst *WidenLoad = Builder.CreateLoad(Ty: I32Ty, Ptr: I.getPointerOperand());
1549 AMDGPU::copyMetadataForWidenedLoad(Dest&: *WidenLoad, Source: I);
1550
1551 // The widened load reads the original bytes in the low bits, so a !range
1552 // lower bound still holds. Convert it to the new type and don't make
1553 // assumptions about the high bits.
1554 if (auto *Range = I.getMetadata(KindID: LLVMContext::MD_range)) {
1555 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD: Range->getOperand(I: 0));
1556
1557 if (!Lower->isNullValue()) {
1558 Metadata *LowAndHigh[] = {
1559 ConstantAsMetadata::get(C: ConstantInt::get(Ty: I32Ty, V: Lower->getValue().zext(width: 32))),
1560 // Don't make assumptions about the high bits.
1561 ConstantAsMetadata::get(C: ConstantInt::get(Ty: I32Ty, V: 0))
1562 };
1563
1564 WidenLoad->setMetadata(KindID: LLVMContext::MD_range,
1565 Node: MDNode::get(Context&: F.getContext(), MDs: LowAndHigh));
1566 }
1567 }
1568
1569 int TySize = DL.getTypeSizeInBits(Ty: I.getType());
1570 Type *IntNTy = Builder.getIntNTy(N: TySize);
1571 Value *ValTrunc = Builder.CreateTrunc(V: WidenLoad, DestTy: IntNTy);
1572 Value *ValOrig = Builder.CreateBitCast(V: ValTrunc, DestTy: I.getType());
1573 I.replaceAllUsesWith(V: ValOrig);
1574 DeadVals.push_back(Elt: &I);
1575 return true;
1576 }
1577
1578 return false;
1579}
1580
1581bool AMDGPUCodeGenPrepareImpl::visitSelectInst(SelectInst &I) {
1582 FPMathOperator *FPOp = dyn_cast<FPMathOperator>(Val: &I);
1583 if (!FPOp)
1584 return false;
1585
1586 Value *X;
1587 Value *Fract = nullptr;
1588
1589 // Match:
1590 // (x - floor(x)) >= MIN_CONSTANT ? MIN_CONSTANT : (x - floor(x))
1591 //
1592 // This is the preferred way to implement fract.
1593 // TODO: Could also match with compare against 1.0
1594 const APFloat *C;
1595 if (match(V: &I, P: m_UnordFMin(L: m_Value(V&: X), R: m_APFloatAllowPoison(Res&: C)))) {
1596 Value *FractSrc = matchFractPatImpl(V&: *X, C: *C);
1597 if (!FractSrc)
1598 return false;
1599 IRBuilder<> Builder(&I);
1600 Builder.setFastMathFlags(FPOp->getFastMathFlags());
1601 Fract = applyFractPat(Builder, FractArg: FractSrc);
1602 } else {
1603 // Match patterns which may appear in legacy implementations of the fract()
1604 // function, built around the nan-avoidant minnum intrinsic. These are the
1605 // core pattern plus additional clamping of inf and nan values on the
1606 // result.
1607 Value *Cond = I.getCondition();
1608 Value *TrueVal = I.getTrueValue();
1609 Value *FalseVal = I.getFalseValue();
1610 Value *CmpVal;
1611 CmpPredicate IsNanPred;
1612
1613 // Match fract pattern with nan check.
1614 if (!match(V: Cond, P: m_FCmp(Pred&: IsNanPred, L: m_Value(V&: CmpVal), R: m_NonNaN())))
1615 return false;
1616
1617 IRBuilder<> Builder(&I);
1618 Builder.setFastMathFlags(FPOp->getFastMathFlags());
1619
1620 if (IsNanPred == FCmpInst::FCMP_UNO && TrueVal == CmpVal &&
1621 CmpVal == matchFractPatNanAvoidant(V&: *FalseVal)) {
1622 // isnan(x) ? x : fract(x)
1623 Fract = applyFractPat(Builder, FractArg: CmpVal);
1624 } else if (IsNanPred == FCmpInst::FCMP_ORD && FalseVal == CmpVal) {
1625 if (CmpVal == matchFractPatNanAvoidant(V&: *TrueVal)) {
1626 // !isnan(x) ? fract(x) : x
1627 Fract = applyFractPat(Builder, FractArg: CmpVal);
1628 } else {
1629 // Match an intermediate clamp infinity to 0 pattern. i.e.
1630 // !isnan(x) ? (!isinf(x) ? fract(x) : 0.0) : x
1631 CmpPredicate PredInf;
1632 Value *IfNotInf;
1633
1634 if (!match(V: TrueVal, P: m_Select(C: m_FCmp(Pred&: PredInf, L: m_FAbs(Op0: m_Specific(V: CmpVal)),
1635 R: m_PosInf()),
1636 L: m_Value(V&: IfNotInf), R: m_PosZeroFP())) ||
1637 PredInf != FCmpInst::FCMP_UNE ||
1638 CmpVal != matchFractPatNanAvoidant(V&: *IfNotInf))
1639 return false;
1640
1641 SelectInst *ClampInfSelect = cast<SelectInst>(Val: TrueVal);
1642
1643 // Insert before the fabs
1644 Value *InsertPt =
1645 cast<Instruction>(Val: ClampInfSelect->getCondition())->getOperand(i: 0);
1646
1647 Builder.SetInsertPoint(cast<Instruction>(Val: InsertPt));
1648 Value *NewFract = applyFractPat(Builder, FractArg: CmpVal);
1649 NewFract->takeName(V: TrueVal);
1650
1651 // Thread the new fract into the inf clamping sequence.
1652 DeadVals.push_back(Elt: ClampInfSelect->getOperand(i_nocapture: 1));
1653 ClampInfSelect->setOperand(i_nocapture: 1, Val_nocapture: NewFract);
1654
1655 // The outer select nan handling is also absorbed into the fract.
1656 Fract = ClampInfSelect;
1657 }
1658 } else
1659 return false;
1660 }
1661
1662 Fract->takeName(V: &I);
1663 I.replaceAllUsesWith(V: Fract);
1664 DeadVals.push_back(Elt: &I);
1665 return true;
1666}
1667
1668static bool areInSameBB(const Value *A, const Value *B) {
1669 const auto *IA = dyn_cast<Instruction>(Val: A);
1670 const auto *IB = dyn_cast<Instruction>(Val: B);
1671 return IA && IB && IA->getParent() == IB->getParent();
1672}
1673
1674// Helper for breaking large PHIs that returns true when an extractelement on V
1675// is likely to be folded away by the DAG combiner.
1676static bool isInterestingPHIIncomingValue(const Value *V) {
1677 const auto *FVT = dyn_cast<FixedVectorType>(Val: V->getType());
1678 if (!FVT)
1679 return false;
1680
1681 const Value *CurVal = V;
1682
1683 // Check for insertelements, keeping track of the elements covered.
1684 BitVector EltsCovered(FVT->getNumElements());
1685 while (const auto *IE = dyn_cast<InsertElementInst>(Val: CurVal)) {
1686 const auto *Idx = dyn_cast<ConstantInt>(Val: IE->getOperand(i_nocapture: 2));
1687
1688 // Non constant index/out of bounds index -> folding is unlikely.
1689 // The latter is more of a sanity check because canonical IR should just
1690 // have replaced those with poison.
1691 if (!Idx || Idx->getZExtValue() >= FVT->getNumElements())
1692 return false;
1693
1694 const auto *VecSrc = IE->getOperand(i_nocapture: 0);
1695
1696 // If the vector source is another instruction, it must be in the same basic
1697 // block. Otherwise, the DAGCombiner won't see the whole thing and is
1698 // unlikely to be able to do anything interesting here.
1699 if (isa<Instruction>(Val: VecSrc) && !areInSameBB(A: VecSrc, B: IE))
1700 return false;
1701
1702 CurVal = VecSrc;
1703 EltsCovered.set(Idx->getZExtValue());
1704
1705 // All elements covered.
1706 if (EltsCovered.all())
1707 return true;
1708 }
1709
1710 // We either didn't find a single insertelement, or the insertelement chain
1711 // ended before all elements were covered. Check for other interesting values.
1712
1713 // Constants are always interesting because we can just constant fold the
1714 // extractelements.
1715 if (isa<Constant>(Val: CurVal))
1716 return true;
1717
1718 // shufflevector is likely to be profitable if either operand is a constant,
1719 // or if either source is in the same block.
1720 // This is because shufflevector is most often lowered as a series of
1721 // insert/extract elements anyway.
1722 if (const auto *SV = dyn_cast<ShuffleVectorInst>(Val: CurVal)) {
1723 return isa<Constant>(Val: SV->getOperand(i_nocapture: 1)) ||
1724 areInSameBB(A: SV, B: SV->getOperand(i_nocapture: 0)) ||
1725 areInSameBB(A: SV, B: SV->getOperand(i_nocapture: 1));
1726 }
1727
1728 return false;
1729}
1730
1731static void collectPHINodes(const PHINode &I,
1732 SmallPtrSet<const PHINode *, 8> &SeenPHIs) {
1733 const auto [It, Inserted] = SeenPHIs.insert(Ptr: &I);
1734 if (!Inserted)
1735 return;
1736
1737 for (const Value *Inc : I.incoming_values()) {
1738 if (const auto *PhiInc = dyn_cast<PHINode>(Val: Inc))
1739 collectPHINodes(I: *PhiInc, SeenPHIs);
1740 }
1741
1742 for (const User *U : I.users()) {
1743 if (const auto *PhiU = dyn_cast<PHINode>(Val: U))
1744 collectPHINodes(I: *PhiU, SeenPHIs);
1745 }
1746}
1747
1748bool AMDGPUCodeGenPrepareImpl::canBreakPHINode(const PHINode &I) {
1749 // Check in the cache first.
1750 if (const auto It = BreakPhiNodesCache.find(Val: &I);
1751 It != BreakPhiNodesCache.end())
1752 return It->second;
1753
1754 // We consider PHI nodes as part of "chains", so given a PHI node I, we
1755 // recursively consider all its users and incoming values that are also PHI
1756 // nodes. We then make a decision about all of those PHIs at once. Either they
1757 // all get broken up, or none of them do. That way, we avoid cases where a
1758 // single PHI is/is not broken and we end up reforming/exploding a vector
1759 // multiple times, or even worse, doing it in a loop.
1760 SmallPtrSet<const PHINode *, 8> WorkList;
1761 collectPHINodes(I, SeenPHIs&: WorkList);
1762
1763#ifndef NDEBUG
1764 // Check that none of the PHI nodes in the worklist are in the map. If some of
1765 // them are, it means we're not good enough at collecting related PHIs.
1766 for (const PHINode *WLP : WorkList) {
1767 assert(BreakPhiNodesCache.count(WLP) == 0);
1768 }
1769#endif
1770
1771 // To consider a PHI profitable to break, we need to see some interesting
1772 // incoming values. At least 2/3rd (rounded up) of all PHIs in the worklist
1773 // must have one to consider all PHIs breakable.
1774 //
1775 // This threshold has been determined through performance testing.
1776 //
1777 // Note that the computation below is equivalent to
1778 //
1779 // (unsigned)ceil((K / 3.0) * 2)
1780 //
1781 // It's simply written this way to avoid mixing integral/FP arithmetic.
1782 const auto Threshold = (alignTo(Value: WorkList.size() * 2, Align: 3) / 3);
1783 unsigned NumBreakablePHIs = 0;
1784 bool CanBreak = false;
1785 for (const PHINode *Cur : WorkList) {
1786 // Don't break PHIs that have no interesting incoming values. That is, where
1787 // there is no clear opportunity to fold the "extractelement" instructions
1788 // we would add.
1789 //
1790 // Note: IC does not run after this pass, so we're only interested in the
1791 // foldings that the DAG combiner can do.
1792 if (any_of(Range: Cur->incoming_values(), P: isInterestingPHIIncomingValue)) {
1793 if (++NumBreakablePHIs >= Threshold) {
1794 CanBreak = true;
1795 break;
1796 }
1797 }
1798 }
1799
1800 for (const PHINode *Cur : WorkList)
1801 BreakPhiNodesCache[Cur] = CanBreak;
1802
1803 return CanBreak;
1804}
1805
1806/// Helper class for "break large PHIs" (visitPHINode).
1807///
1808/// This represents a slice of a PHI's incoming value, which is made up of:
1809/// - The type of the slice (Ty)
1810/// - The index in the incoming value's vector where the slice starts (Idx)
1811/// - The number of elements in the slice (NumElts).
1812/// It also keeps track of the NewPHI node inserted for this particular slice.
1813///
1814/// Slice examples:
1815/// <4 x i64> -> Split into four i64 slices.
1816/// -> [i64, 0, 1], [i64, 1, 1], [i64, 2, 1], [i64, 3, 1]
1817/// <5 x i16> -> Split into 2 <2 x i16> slices + a i16 tail.
1818/// -> [<2 x i16>, 0, 2], [<2 x i16>, 2, 2], [i16, 4, 1]
1819class VectorSlice {
1820public:
1821 VectorSlice(Type *Ty, unsigned Idx, unsigned NumElts)
1822 : Ty(Ty), Idx(Idx), NumElts(NumElts) {}
1823
1824 Type *Ty = nullptr;
1825 unsigned Idx = 0;
1826 unsigned NumElts = 0;
1827 PHINode *NewPHI = nullptr;
1828
1829 /// Slice \p Inc according to the information contained within this slice.
1830 /// This is cached, so if called multiple times for the same \p BB & \p Inc
1831 /// pair, it returns the same Sliced value as well.
1832 ///
1833 /// Note this *intentionally* does not return the same value for, say,
1834 /// [%bb.0, %0] & [%bb.1, %0] as:
1835 /// - It could cause issues with dominance (e.g. if bb.1 is seen first, then
1836 /// the value in bb.1 may not be reachable from bb.0 if it's its
1837 /// predecessor.)
1838 /// - We also want to make our extract instructions as local as possible so
1839 /// the DAG has better chances of folding them out. Duplicating them like
1840 /// that is beneficial in that regard.
1841 ///
1842 /// This is both a minor optimization to avoid creating duplicate
1843 /// instructions, but also a requirement for correctness. It is not forbidden
1844 /// for a PHI node to have the same [BB, Val] pair multiple times. If we
1845 /// returned a new value each time, those previously identical pairs would all
1846 /// have different incoming values (from the same block) and it'd cause a "PHI
1847 /// node has multiple entries for the same basic block with different incoming
1848 /// values!" verifier error.
1849 Value *getSlicedVal(BasicBlock *BB, Value *Inc, StringRef NewValName) {
1850 Value *&Res = SlicedVals[{BB, Inc}];
1851 if (Res)
1852 return Res;
1853
1854 IRBuilder<> B(BB->getTerminator());
1855 if (Instruction *IncInst = dyn_cast<Instruction>(Val: Inc))
1856 B.SetCurrentDebugLocation(IncInst->getDebugLoc());
1857
1858 if (NumElts > 1) {
1859 SmallVector<int, 4> Mask;
1860 for (unsigned K = Idx; K < (Idx + NumElts); ++K)
1861 Mask.push_back(Elt: K);
1862 Res = B.CreateShuffleVector(V: Inc, Mask, Name: NewValName);
1863 } else
1864 Res = B.CreateExtractElement(Vec: Inc, Idx, Name: NewValName);
1865
1866 return Res;
1867 }
1868
1869private:
1870 SmallDenseMap<std::pair<BasicBlock *, Value *>, Value *> SlicedVals;
1871};
1872
1873bool AMDGPUCodeGenPrepareImpl::visitPHINode(PHINode &I) {
1874 // Break-up fixed-vector PHIs into smaller pieces.
1875 // Default threshold is 32, so it breaks up any vector that's >32 bits into
1876 // its elements, or into 32-bit pieces (for 8/16 bit elts).
1877 //
1878 // This is only helpful for DAGISel because it doesn't handle large PHIs as
1879 // well as GlobalISel. DAGISel lowers PHIs by using CopyToReg/CopyFromReg.
1880 // With large, odd-sized PHIs we may end up needing many `build_vector`
1881 // operations with most elements being "undef". This inhibits a lot of
1882 // optimization opportunities and can result in unreasonably high register
1883 // pressure and the inevitable stack spilling.
1884 if (!BreakLargePHIs || getCGPassBuilderOption().EnableGlobalISelOption ==
1885 cl::boolOrDefault::BOU_TRUE)
1886 return false;
1887
1888 FixedVectorType *FVT = dyn_cast<FixedVectorType>(Val: I.getType());
1889 if (!FVT || FVT->getNumElements() == 1 ||
1890 DL.getTypeSizeInBits(Ty: FVT) <= BreakLargePHIsThreshold)
1891 return false;
1892
1893 if (!ForceBreakLargePHIs && !canBreakPHINode(I))
1894 return false;
1895
1896 std::vector<VectorSlice> Slices;
1897
1898 Type *EltTy = FVT->getElementType();
1899 {
1900 unsigned Idx = 0;
1901 // For 8/16 bits type, don't scalarize fully but break it up into as many
1902 // 32-bit slices as we can, and scalarize the tail.
1903 const unsigned EltSize = DL.getTypeSizeInBits(Ty: EltTy);
1904 const unsigned NumElts = FVT->getNumElements();
1905 if (EltSize == 8 || EltSize == 16) {
1906 const unsigned SubVecSize = (32 / EltSize);
1907 Type *SubVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: SubVecSize);
1908 for (unsigned End = alignDown(Value: NumElts, Align: SubVecSize); Idx < End;
1909 Idx += SubVecSize)
1910 Slices.emplace_back(args&: SubVecTy, args&: Idx, args: SubVecSize);
1911 }
1912
1913 // Scalarize all remaining elements.
1914 for (; Idx < NumElts; ++Idx)
1915 Slices.emplace_back(args&: EltTy, args&: Idx, args: 1);
1916 }
1917
1918 assert(Slices.size() > 1);
1919
1920 // Create one PHI per vector piece. The "VectorSlice" class takes care of
1921 // creating the necessary instruction to extract the relevant slices of each
1922 // incoming value.
1923 IRBuilder<> B(I.getParent());
1924 B.SetCurrentDebugLocation(I.getDebugLoc());
1925
1926 unsigned IncNameSuffix = 0;
1927 for (VectorSlice &S : Slices) {
1928 // We need to reset the build on each iteration, because getSlicedVal may
1929 // have inserted something into I's BB.
1930 B.SetInsertPoint(I.getParent()->getFirstNonPHIIt());
1931 S.NewPHI = B.CreatePHI(Ty: S.Ty, NumReservedValues: I.getNumIncomingValues());
1932
1933 for (const auto &[Idx, BB] : enumerate(First: I.blocks())) {
1934 S.NewPHI->addIncoming(V: S.getSlicedVal(BB, Inc: I.getIncomingValue(i: Idx),
1935 NewValName: "largephi.extractslice" +
1936 std::to_string(val: IncNameSuffix++)),
1937 BB);
1938 }
1939 }
1940
1941 // And replace this PHI with a vector of all the previous PHI values.
1942 Value *Vec = PoisonValue::get(T: FVT);
1943 unsigned NameSuffix = 0;
1944 for (VectorSlice &S : Slices) {
1945 const auto ValName = "largephi.insertslice" + std::to_string(val: NameSuffix++);
1946 if (S.NumElts > 1)
1947 Vec = B.CreateInsertVector(DstType: FVT, SrcVec: Vec, SubVec: S.NewPHI, Idx: S.Idx, Name: ValName);
1948 else
1949 Vec = B.CreateInsertElement(Vec, NewElt: S.NewPHI, Idx: S.Idx, Name: ValName);
1950 }
1951
1952 I.replaceAllUsesWith(V: Vec);
1953 DeadVals.push_back(Elt: &I);
1954 return true;
1955}
1956
1957/// \param V Value to check
1958/// \param DL DataLayout
1959/// \param TM TargetMachine (TODO: remove once DL contains nullptr values)
1960/// \param AS Target Address Space
1961/// \return true if \p V cannot be the null value of \p AS, false otherwise.
1962static bool isPtrKnownNeverNull(const Value *V, const DataLayout &DL,
1963 const AMDGPUTargetMachine &TM, unsigned AS) {
1964 // Pointer cannot be null if it's a block address, GV or alloca.
1965 // NOTE: We don't support extern_weak, but if we did, we'd need to check for
1966 // it as the symbol could be null in such cases.
1967 if (isa<BlockAddress, GlobalValue, AllocaInst>(Val: V))
1968 return true;
1969
1970 // Check nonnull arguments.
1971 if (const auto *Arg = dyn_cast<Argument>(Val: V); Arg && Arg->hasNonNullAttr())
1972 return true;
1973
1974 // Check nonnull loads.
1975 if (const auto *Load = dyn_cast<LoadInst>(Val: V);
1976 Load && Load->hasMetadata(KindID: LLVMContext::MD_nonnull))
1977 return true;
1978
1979 // getUnderlyingObject may have looked through another addrspacecast, although
1980 // the optimizable situations most likely folded out by now.
1981 if (AS != cast<PointerType>(Val: V->getType())->getAddressSpace())
1982 return false;
1983
1984 // TODO: Calls that return nonnull?
1985
1986 // For all other things, use KnownBits.
1987 // We either use 0 or all bits set to indicate null, so check whether the
1988 // value can be zero or all ones.
1989 //
1990 // TODO: Use ValueTracking's isKnownNeverNull if it becomes aware that some
1991 // address spaces have non-zero null values.
1992 auto SrcPtrKB = computeKnownBits(V, DL);
1993 const auto NullVal = AMDGPU::getNullPointerValue(AS);
1994
1995 assert(SrcPtrKB.getBitWidth() == DL.getPointerSizeInBits(AS));
1996 assert((NullVal == 0 || NullVal == -1) &&
1997 "don't know how to check for this null value!");
1998 return NullVal ? !SrcPtrKB.getMaxValue().isAllOnes() : SrcPtrKB.isNonZero();
1999}
2000
2001bool AMDGPUCodeGenPrepareImpl::visitAddrSpaceCastInst(AddrSpaceCastInst &I) {
2002 // TODO: This is target-independent reasoning about the source pointer being
2003 // non-null, and would fit better in a generic pass such as
2004 // AggressiveInstCombine. It lives here for now because proving the source is
2005 // not the null value requires knowing the numeric null pointer value of the
2006 // source address space, which is not yet a first-class IR concept.
2007
2008 // If the flag is already set there is nothing to do.
2009 if (I.hasNonNull())
2010 return false;
2011
2012 // It is often difficult to prove that a vector of pointers cannot have any
2013 // nulls in it, so it's unclear if it's worth supporting.
2014 if (I.getType()->isVectorTy())
2015 return false;
2016
2017 // The nonnull flag only affects the lowering of casts from/to priv/local to
2018 // flat, so only bother proving non-null for those.
2019 const unsigned SrcAS = I.getSrcAddressSpace();
2020 const unsigned DstAS = I.getDestAddressSpace();
2021
2022 bool CanLower = false;
2023 if (SrcAS == AMDGPUAS::FLAT_ADDRESS)
2024 CanLower = (DstAS == AMDGPUAS::LOCAL_ADDRESS ||
2025 DstAS == AMDGPUAS::PRIVATE_ADDRESS);
2026 else if (DstAS == AMDGPUAS::FLAT_ADDRESS)
2027 CanLower = (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
2028 SrcAS == AMDGPUAS::PRIVATE_ADDRESS);
2029 if (!CanLower)
2030 return false;
2031
2032 SmallVector<const Value *, 4> WorkList;
2033 getUnderlyingObjects(V: I.getOperand(i_nocapture: 0), Objects&: WorkList);
2034 if (!all_of(Range&: WorkList, P: [&](const Value *V) {
2035 return isPtrKnownNeverNull(V, DL, TM, AS: SrcAS);
2036 }))
2037 return false;
2038
2039 I.setNonNull();
2040 return true;
2041}
2042
2043bool AMDGPUCodeGenPrepareImpl::visitIntrinsicInst(IntrinsicInst &I) {
2044 Intrinsic::ID IID = I.getIntrinsicID();
2045 switch (IID) {
2046 case Intrinsic::minnum:
2047 case Intrinsic::minimumnum:
2048 case Intrinsic::minimum:
2049 return visitFMinLike(I);
2050 case Intrinsic::sqrt:
2051 return visitSqrt(I);
2052 case Intrinsic::log:
2053 case Intrinsic::log10:
2054 return visitLog(Log&: cast<FPMathOperator>(Val&: I), IID);
2055 case Intrinsic::log2:
2056 // No reason to handle log2.
2057 return false;
2058 case Intrinsic::amdgcn_mbcnt_lo:
2059 return visitMbcntLo(I);
2060 case Intrinsic::amdgcn_mbcnt_hi:
2061 return visitMbcntHi(I);
2062 case Intrinsic::vector_reduce_add:
2063 return visitVectorReduceAdd(I);
2064 case Intrinsic::uadd_sat:
2065 case Intrinsic::sadd_sat:
2066 return visitSaturatingAdd(I);
2067 default:
2068 return false;
2069 }
2070}
2071
2072/// Match the core sequence in the fract pattern (x - floor(x), which doesn't
2073/// need to consider edge case handling.
2074Value *AMDGPUCodeGenPrepareImpl::matchFractPatImpl(Value &FractSrc,
2075 const APFloat &C) const {
2076 if (ST.hasFractBug())
2077 return nullptr;
2078
2079 Type *Ty = FractSrc.getType();
2080 if (!isLegalFloatingTy(Ty: Ty->getScalarType()))
2081 return nullptr;
2082
2083 APFloat OneNextDown = APFloat::getOne(Sem: C.getSemantics());
2084 OneNextDown.next(nextDown: true);
2085
2086 // Match nextafter(1.0, -1)
2087 if (OneNextDown != C)
2088 return nullptr;
2089
2090 Value *FloorSrc;
2091 if (match(V: &FractSrc, P: m_FSub(L: m_Value(V&: FloorSrc), R: m_Intrinsic<Intrinsic::floor>(
2092 Ops: m_Deferred(V: FloorSrc)))))
2093 return FloorSrc;
2094 return nullptr;
2095}
2096
2097/// Match non-nan fract pattern.
2098// MIN_CONSTANT = nextafter(1.0, -1.0)
2099/// minnum(fsub(x, floor(x)), MIN_CONSTANT)
2100/// minimumnum(fsub(x, floor(x)), MIN_CONSTANT)
2101/// minimum(fsub(x, floor(x)), MIN_CONSTANT)
2102
2103// x_sub_floor >= MIN_CONSTANT ? MIN_CONSTANT : x_sub_floor;
2104///
2105/// If fract is a useful instruction for the subtarget. Does not account for the
2106/// nan handling; the instruction has a nan check on the input value.
2107Value *AMDGPUCodeGenPrepareImpl::matchFractPatNanAvoidant(Value &V) {
2108 Value *Arg0;
2109 const APFloat *C;
2110
2111 // The value is only used in contexts where we know the input isn't a nan, so
2112 // any of the fmin variants are fine.
2113 if (!match(V: &V,
2114 P: m_CombineOr(Ps: m_FMinNum_or_FMinimumNum(Op0: m_Value(V&: Arg0),
2115 Op1: m_APFloatAllowPoison(Res&: C)),
2116 Ps: m_FMinimum(Op0: m_Value(V&: Arg0), Op1: m_APFloatAllowPoison(Res&: C)))))
2117 return nullptr;
2118
2119 return matchFractPatImpl(FractSrc&: *Arg0, C: *C);
2120}
2121
2122Value *AMDGPUCodeGenPrepareImpl::applyFractPat(IRBuilder<> &Builder,
2123 Value *FractArg) {
2124 SmallVector<Value *, 4> FractVals;
2125 extractValues(Builder, Values&: FractVals, V: FractArg);
2126
2127 SmallVector<Value *, 4> ResultVals(FractVals.size());
2128
2129 Type *Ty = FractArg->getType()->getScalarType();
2130 for (unsigned I = 0, E = FractVals.size(); I != E; ++I) {
2131 ResultVals[I] =
2132 Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_fract, OverloadTypes: {Ty}, Args: {FractVals[I]});
2133 }
2134
2135 return insertValues(Builder, Ty: FractArg->getType(), Values&: ResultVals);
2136}
2137
2138bool AMDGPUCodeGenPrepareImpl::visitFMinLike(IntrinsicInst &I) {
2139 const APFloat *C;
2140 Value *FractArg;
2141
2142 // minimum(x - floor(x), MIN_CONSTANT)
2143 Value *X;
2144 if (!ST.hasFractBug() &&
2145 match(V: &I, P: m_FMinimum(Op0: m_Value(V&: X), Op1: m_APFloatAllowPoison(Res&: C)))) {
2146 FractArg = matchFractPatImpl(FractSrc&: *X, C: *C);
2147 if (!FractArg)
2148 return false;
2149 } else {
2150 // minnum(x - floor(x), MIN_CONSTANT)
2151 FractArg = matchFractPatNanAvoidant(V&: I);
2152 if (!FractArg)
2153 return false;
2154
2155 // Match pattern for fract intrinsic in contexts where the nan check has
2156 // been optimized out (and hope the knowledge the source can't be nan wasn't
2157 // lost).
2158 if (!I.hasNoNaNs() && !isKnownNeverNaN(V: FractArg, SQ: SQ.getWithInstruction(I: &I)))
2159 return false;
2160 }
2161
2162 IRBuilder<> Builder(&I);
2163 FastMathFlags FMF = I.getFastMathFlags();
2164 FMF.setNoNaNs();
2165 Builder.setFastMathFlags(FMF);
2166
2167 Value *Fract = applyFractPat(Builder, FractArg);
2168 Fract->takeName(V: &I);
2169 I.replaceAllUsesWith(V: Fract);
2170 DeadVals.push_back(Elt: &I);
2171 return true;
2172}
2173
2174// Expand llvm.sqrt.f32 calls with !fpmath metadata in a semi-fast way.
2175bool AMDGPUCodeGenPrepareImpl::visitSqrt(IntrinsicInst &Sqrt) {
2176 Type *Ty = Sqrt.getType()->getScalarType();
2177 if (!Ty->isFloatTy())
2178 return false;
2179
2180 const FPMathOperator *FPOp = cast<const FPMathOperator>(Val: &Sqrt);
2181 FastMathFlags SqrtFMF = FPOp->getFastMathFlags();
2182
2183 // We're trying to handle the fast-but-not-that-fast case only. The lowering
2184 // of fast llvm.sqrt will give the raw instruction anyway.
2185 if (SqrtFMF.approxFunc())
2186 return false;
2187
2188 const float ReqdAccuracy = FPOp->getFPAccuracy();
2189
2190 // Defer correctly rounded expansion to codegen.
2191 if (ReqdAccuracy < 1.0f)
2192 return false;
2193
2194 Value *SrcVal = Sqrt.getOperand(i_nocapture: 0);
2195 bool CanTreatAsDAZ = canIgnoreDenormalInput(V: SrcVal, CtxI: &Sqrt);
2196
2197 // The raw instruction is 1 ulp, but the correction for denormal handling
2198 // brings it to 2.
2199 if (!CanTreatAsDAZ && ReqdAccuracy < 2.0f)
2200 return false;
2201
2202 IRBuilder<> Builder(&Sqrt);
2203 SmallVector<Value *, 4> SrcVals;
2204 extractValues(Builder, Values&: SrcVals, V: SrcVal);
2205
2206 SmallVector<Value *, 4> ResultVals(SrcVals.size());
2207 for (int I = 0, E = SrcVals.size(); I != E; ++I) {
2208 if (CanTreatAsDAZ)
2209 ResultVals[I] = Builder.CreateCall(Callee: getSqrtF32(), Args: SrcVals[I]);
2210 else
2211 ResultVals[I] = emitSqrtIEEE2ULP(Builder, Src: SrcVals[I], FMF: SqrtFMF);
2212 }
2213
2214 Value *NewSqrt = insertValues(Builder, Ty: Sqrt.getType(), Values&: ResultVals);
2215 NewSqrt->takeName(V: &Sqrt);
2216 Sqrt.replaceAllUsesWith(V: NewSqrt);
2217 DeadVals.push_back(Elt: &Sqrt);
2218 return true;
2219}
2220
2221/// Replace log and log10 intrinsic calls based on fpmath metadata.
2222bool AMDGPUCodeGenPrepareImpl::visitLog(FPMathOperator &Log,
2223 Intrinsic::ID IID) {
2224 Type *Ty = Log.getType();
2225 if (!Ty->getScalarType()->isHalfTy() || !ST.has16BitInsts())
2226 return false;
2227
2228 FastMathFlags FMF = Log.getFastMathFlags();
2229
2230 // Defer fast math cases to codegen.
2231 if (FMF.approxFunc())
2232 return false;
2233
2234 // Limit experimentally determined from OpenCL conformance test (1.79)
2235 if (Log.getFPAccuracy() < 1.80f)
2236 return false;
2237
2238 IRBuilder<> Builder(&cast<CallInst>(Val&: Log));
2239
2240 // Use the generic intrinsic for convenience in the vector case. Codegen will
2241 // recognize the denormal handling is not necessary from the fpext.
2242 // TODO: Move to generic code
2243 Value *Log2 =
2244 Builder.CreateUnaryIntrinsic(ID: Intrinsic::log2, Op: Log.getOperand(i: 0), FMFSource: FMF);
2245
2246 double Log2BaseInverted =
2247 IID == Intrinsic::log10 ? numbers::ln2 / numbers::ln10 : numbers::ln2;
2248 Value *Mul =
2249 Builder.CreateFMulFMF(L: Log2, R: ConstantFP::get(Ty, V: Log2BaseInverted), FMFSource: FMF);
2250
2251 Mul->takeName(V: &Log);
2252
2253 Log.replaceAllUsesWith(V: Mul);
2254 DeadVals.push_back(Elt: &Log);
2255 return true;
2256}
2257
2258bool AMDGPUCodeGenPrepare::runOnFunction(Function &F) {
2259 if (skipFunction(F))
2260 return false;
2261
2262 auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
2263 if (!TPC)
2264 return false;
2265
2266 const AMDGPUTargetMachine &TM = TPC->getTM<AMDGPUTargetMachine>();
2267 const TargetTransformInfo &TTI =
2268 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
2269 const TargetLibraryInfo *TLI =
2270 &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
2271 AssumptionCache *AC =
2272 &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
2273 auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
2274 const DominatorTree *DT = DTWP ? &DTWP->getDomTree() : nullptr;
2275 const UniformityInfo &UA =
2276 getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
2277 return AMDGPUCodeGenPrepareImpl(F, TM, TTI, TLI, AC, DT, UA).run();
2278}
2279
2280PreservedAnalyses AMDGPUCodeGenPreparePass::run(Function &F,
2281 FunctionAnalysisManager &FAM) {
2282 const AMDGPUTargetMachine &ATM = static_cast<const AMDGPUTargetMachine &>(TM);
2283 const TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(IR&: F);
2284 const TargetLibraryInfo *TLI = &FAM.getResult<TargetLibraryAnalysis>(IR&: F);
2285 AssumptionCache *AC = &FAM.getResult<AssumptionAnalysis>(IR&: F);
2286 const DominatorTree *DT = FAM.getCachedResult<DominatorTreeAnalysis>(IR&: F);
2287 const UniformityInfo &UA = FAM.getResult<UniformityInfoAnalysis>(IR&: F);
2288 AMDGPUCodeGenPrepareImpl Impl(F, ATM, TTI, TLI, AC, DT, UA);
2289 if (!Impl.run())
2290 return PreservedAnalyses::all();
2291 PreservedAnalyses PA = PreservedAnalyses::none();
2292 if (!Impl.FlowChanged)
2293 PA.preserveSet<CFGAnalyses>();
2294 return PA;
2295}
2296
2297INITIALIZE_PASS_BEGIN(AMDGPUCodeGenPrepare, DEBUG_TYPE,
2298 "AMDGPU IR optimizations", false, false)
2299INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
2300INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
2301INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
2302INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
2303INITIALIZE_PASS_END(AMDGPUCodeGenPrepare, DEBUG_TYPE, "AMDGPU IR optimizations",
2304 false, false)
2305
2306/// Create a workitem.id.x intrinsic call with range metadata.
2307CallInst *AMDGPUCodeGenPrepareImpl::createWorkitemIdX(IRBuilder<> &B) const {
2308 CallInst *Tid =
2309 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::amdgcn_workitem_id_x, Args: {});
2310 ST.makeLIDRangeMetadata(I: Tid);
2311 return Tid;
2312}
2313
2314/// Replace the instruction with a direct workitem.id.x call.
2315void AMDGPUCodeGenPrepareImpl::replaceWithWorkitemIdX(Instruction &I) const {
2316 IRBuilder<> B(&I);
2317 CallInst *Tid = createWorkitemIdX(B);
2318 BasicBlock::iterator BI(&I);
2319 ReplaceInstWithValue(BI, V: Tid);
2320}
2321
2322/// Replace the instruction with (workitem.id.x & mask).
2323void AMDGPUCodeGenPrepareImpl::replaceWithMaskedWorkitemIdX(
2324 Instruction &I, unsigned WaveSize) const {
2325 IRBuilder<> B(&I);
2326 CallInst *Tid = createWorkitemIdX(B);
2327 Constant *Mask = ConstantInt::get(Ty: Tid->getType(), V: WaveSize - 1);
2328 Value *AndInst = B.CreateAnd(LHS: Tid, RHS: Mask);
2329 BasicBlock::iterator BI(&I);
2330 ReplaceInstWithValue(BI, V: AndInst);
2331}
2332
2333/// Try to optimize mbcnt instruction by replacing with workitem.id.x when
2334/// work group size allows direct computation of lane ID.
2335/// Returns true if optimization was applied, false otherwise.
2336bool AMDGPUCodeGenPrepareImpl::tryReplaceWithWorkitemId(Instruction &I,
2337 unsigned Wave) const {
2338 std::optional<unsigned> MaybeX = ST.getReqdWorkGroupSize(F, Dim: 0);
2339 if (!MaybeX)
2340 return false;
2341
2342 // When work group size == wave_size, each work group contains exactly one
2343 // wave, so the instruction can be replaced with workitem.id.x directly.
2344 if (*MaybeX == Wave) {
2345 replaceWithWorkitemIdX(I);
2346 return true;
2347 }
2348
2349 // When work group evenly splits into waves, compute lane ID within wave
2350 // using bit masking: lane_id = workitem.id.x & (wave_size - 1).
2351 if (ST.hasWavefrontsEvenlySplittingXDim(F, /*RequiresUniformYZ=*/REquiresUniformYZ: true)) {
2352 replaceWithMaskedWorkitemIdX(I, WaveSize: Wave);
2353 return true;
2354 }
2355
2356 return false;
2357}
2358
2359/// Optimize mbcnt.lo calls on wave32 architectures for lane ID computation.
2360bool AMDGPUCodeGenPrepareImpl::visitMbcntLo(IntrinsicInst &I) const {
2361 // This optimization only applies to wave32 targets where mbcnt.lo operates on
2362 // the full execution mask.
2363 if (!ST.isWave32())
2364 return false;
2365
2366 // Only optimize the pattern mbcnt.lo(~0, 0) which counts active lanes with
2367 // lower IDs.
2368 if (!match(V: &I,
2369 P: m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(Ops: m_AllOnes(), Ops: m_Zero())))
2370 return false;
2371
2372 return tryReplaceWithWorkitemId(I, Wave: ST.getWavefrontSize());
2373}
2374
2375/// Optimize mbcnt.hi calls for lane ID computation.
2376bool AMDGPUCodeGenPrepareImpl::visitMbcntHi(IntrinsicInst &I) const {
2377 // Abort if wave size is not known at compile time.
2378 if (!ST.isWaveSizeKnown())
2379 return false;
2380
2381 unsigned Wave = ST.getWavefrontSize();
2382
2383 // On wave32, the upper 32 bits of execution mask are always 0, so
2384 // mbcnt.hi(mask, val) always returns val unchanged.
2385 if (ST.isWave32()) {
2386 BasicBlock::iterator BI(&I);
2387 ReplaceInstWithValue(BI, V: I.getArgOperand(i: 1));
2388 return true;
2389 }
2390
2391 // Optimize the complete lane ID computation pattern:
2392 // mbcnt.hi(~0, mbcnt.lo(~0, 0)) which counts all active lanes with lower IDs
2393 // across the full execution mask.
2394 using namespace PatternMatch;
2395
2396 // Check for pattern: mbcnt.hi(~0, mbcnt.lo(~0, 0))
2397 if (!match(V: &I, P: m_Intrinsic<Intrinsic::amdgcn_mbcnt_hi>(
2398 Ops: m_AllOnes(), Ops: m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(
2399 Ops: m_AllOnes(), Ops: m_Zero()))))
2400 return false;
2401
2402 return tryReplaceWithWorkitemId(I, Wave);
2403}
2404
2405/// Check if type is <4 x i8>.
2406static bool isV4I8(Type *Ty) {
2407 FixedVectorType *VTy = dyn_cast<FixedVectorType>(Val: Ty);
2408 return VTy && VTy->getNumElements() == 4 &&
2409 VTy->getElementType()->isIntegerTy(BitWidth: 8);
2410}
2411
2412/// Helper to match the dot4 pattern: mul(zext/sext <4 x i8>, zext/sext <4 x
2413/// i8>) Returns true if pattern matches and signedness matches IsSigned.
2414/// Sets A, B to the <4 x i8> sources.
2415static bool matchDot4Pattern(Value *MulOp, Value *&A, Value *&B,
2416 bool IsSigned) {
2417 Value *Src0, *Src1;
2418 if (!match(V: MulOp, P: m_Mul(L: m_Value(V&: Src0), R: m_Value(V&: Src1))))
2419 return false;
2420
2421 // Check that result type is <4 x i32>
2422 FixedVectorType *MulTy = dyn_cast<FixedVectorType>(Val: MulOp->getType());
2423 if (!MulTy || MulTy->getNumElements() != 4 ||
2424 !MulTy->getElementType()->isIntegerTy(BitWidth: 32))
2425 return false;
2426
2427 // Match zext or sext based on IsSigned
2428 Value *ExtSrc0, *ExtSrc1;
2429 if (IsSigned) {
2430 if (!match(V: Src0, P: m_SExt(Op: m_Value(V&: ExtSrc0))) || !isV4I8(Ty: ExtSrc0->getType()))
2431 return false;
2432 if (!match(V: Src1, P: m_SExt(Op: m_Value(V&: ExtSrc1))) || !isV4I8(Ty: ExtSrc1->getType()))
2433 return false;
2434 } else {
2435 if (!match(V: Src0, P: m_ZExt(Op: m_Value(V&: ExtSrc0))) || !isV4I8(Ty: ExtSrc0->getType()))
2436 return false;
2437 if (!match(V: Src1, P: m_ZExt(Op: m_Value(V&: ExtSrc1))) || !isV4I8(Ty: ExtSrc1->getType()))
2438 return false;
2439 }
2440
2441 A = ExtSrc0;
2442 B = ExtSrc1;
2443 return true;
2444}
2445
2446/// Try to convert vector.reduce.add(mul(zext/sext <4 x i8>, zext/sext <4 x
2447/// i8>)) to a dot4 intrinsic call (non-saturating case only).
2448bool AMDGPUCodeGenPrepareImpl::visitVectorReduceAdd(IntrinsicInst &I) {
2449 // Check if we have dot4 instructions available
2450 if (!ST.hasDot7Insts() || (!ST.hasDot1Insts() && !ST.hasDot8Insts()))
2451 return false;
2452
2453 Value *A = nullptr, *B = nullptr;
2454
2455 // Try unsigned first, then signed
2456 bool IsSigned = false;
2457 if (!matchDot4Pattern(MulOp: I.getArgOperand(i: 0), A, B, /*IsSigned=*/false)) {
2458 if (!matchDot4Pattern(MulOp: I.getArgOperand(i: 0), A, B, /*IsSigned=*/true))
2459 return false;
2460 IsSigned = true;
2461 }
2462
2463 LLVMContext &Ctx = I.getContext();
2464 Type *I32Ty = Type::getInt32Ty(C&: Ctx);
2465 IRBuilder<> Builder(&I);
2466
2467 // Bitcast <4 x i8> to i32
2468 Value *ASrc = Builder.CreateBitCast(V: A, DestTy: I32Ty);
2469 Value *BSrc = Builder.CreateBitCast(V: B, DestTy: I32Ty);
2470
2471 // Non-saturating case: accumulator is 0, clamp is false
2472 Value *Acc = ConstantInt::get(Ty: I32Ty, V: 0);
2473 Value *Clamp = ConstantInt::getFalse(Context&: Ctx);
2474
2475 Intrinsic::ID DotIID =
2476 IsSigned ? Intrinsic::amdgcn_sdot4 : Intrinsic::amdgcn_udot4;
2477
2478 Value *Dot = Builder.CreateIntrinsic(ID: DotIID, OverloadTypes: {}, Args: {ASrc, BSrc, Acc, Clamp});
2479 Dot->takeName(V: &I);
2480
2481 I.replaceAllUsesWith(V: Dot);
2482 DeadVals.push_back(Elt: &I);
2483
2484 return true;
2485}
2486
2487/// Try to convert uadd.sat/sadd.sat(vector.reduce.add(mul(...)), c) to a
2488/// saturating dot4 intrinsic. This combine starts at the root (saturating add)
2489/// and looks at its operands.
2490bool AMDGPUCodeGenPrepareImpl::visitSaturatingAdd(IntrinsicInst &I) {
2491 // Check if we have dot4 instructions available
2492 if (!ST.hasDot7Insts() || (!ST.hasDot1Insts() && !ST.hasDot8Insts()))
2493 return false;
2494
2495 Intrinsic::ID IID = I.getIntrinsicID();
2496 bool IsSigned = (IID == Intrinsic::sadd_sat);
2497
2498 // Look for vector.reduce.add as one of the operands (commutative match)
2499 Value *Op0 = I.getArgOperand(i: 0);
2500 Value *Op1 = I.getArgOperand(i: 1);
2501 Value *MulOp = nullptr;
2502 Value *Accum = nullptr;
2503 IntrinsicInst *ReduceInst = nullptr;
2504
2505 if (match(V: Op0, P: m_Intrinsic<Intrinsic::vector_reduce_add>(Ops: m_Value(V&: MulOp)))) {
2506 ReduceInst = cast<IntrinsicInst>(Val: Op0);
2507 Accum = Op1;
2508 } else if (match(V: Op1,
2509 P: m_Intrinsic<Intrinsic::vector_reduce_add>(Ops: m_Value(V&: MulOp)))) {
2510 ReduceInst = cast<IntrinsicInst>(Val: Op1);
2511 Accum = Op0;
2512 } else {
2513 return false;
2514 }
2515
2516 Value *A = nullptr, *B = nullptr;
2517
2518 if (!matchDot4Pattern(MulOp, A, B, IsSigned))
2519 return false;
2520
2521 LLVMContext &Ctx = I.getContext();
2522 Type *I32Ty = Type::getInt32Ty(C&: Ctx);
2523 IRBuilder<> Builder(&I);
2524
2525 // Bitcast <4 x i8> to i32
2526 Value *ASrc = Builder.CreateBitCast(V: A, DestTy: I32Ty);
2527 Value *BSrc = Builder.CreateBitCast(V: B, DestTy: I32Ty);
2528
2529 // Saturating case: use the accumulator and set clamp to true
2530 Value *Clamp = ConstantInt::getTrue(Context&: Ctx);
2531
2532 Intrinsic::ID DotIID =
2533 IsSigned ? Intrinsic::amdgcn_sdot4 : Intrinsic::amdgcn_udot4;
2534
2535 Value *Dot = Builder.CreateIntrinsic(ID: DotIID, OverloadTypes: {}, Args: {ASrc, BSrc, Accum, Clamp});
2536 Dot->takeName(V: &I);
2537
2538 I.replaceAllUsesWith(V: Dot);
2539 DeadVals.push_back(Elt: &I);
2540 // The reduce.add will be dead after this and cleaned up later
2541 if (ReduceInst->use_empty())
2542 DeadVals.push_back(Elt: ReduceInst);
2543
2544 return true;
2545}
2546
2547char AMDGPUCodeGenPrepare::ID = 0;
2548
2549FunctionPass *llvm::createAMDGPUCodeGenPreparePass() {
2550 return new AMDGPUCodeGenPrepare();
2551}
2552