1//===-- AMDGPUCodeGenPrepare.cpp ------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass does misc. AMDGPU optimizations on IR before instruction
11/// selection.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPU.h"
16#include "AMDGPUMemoryUtils.h"
17#include "AMDGPUTargetMachine.h"
18#include "SIModeRegisterDefaults.h"
19#include "llvm/ADT/SetVector.h"
20#include "llvm/Analysis/AssumptionCache.h"
21#include "llvm/Analysis/ConstantFolding.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Analysis/TargetTransformInfo.h"
24#include "llvm/Analysis/UniformityAnalysis.h"
25#include "llvm/Analysis/ValueTracking.h"
26#include "llvm/CodeGen/TargetPassConfig.h"
27#include "llvm/IR/Dominators.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/InstVisitor.h"
30#include "llvm/IR/IntrinsicsAMDGPU.h"
31#include "llvm/IR/PatternMatch.h"
32#include "llvm/IR/ValueHandle.h"
33#include "llvm/InitializePasses.h"
34#include "llvm/Pass.h"
35#include "llvm/Support/KnownBits.h"
36#include "llvm/Support/KnownFPClass.h"
37#include "llvm/Transforms/Utils/BasicBlockUtils.h"
38#include "llvm/Transforms/Utils/IntegerDivision.h"
39#include "llvm/Transforms/Utils/Local.h"
40
41#define DEBUG_TYPE "amdgpu-codegenprepare"
42
43using namespace llvm;
44using namespace llvm::PatternMatch;
45
46namespace {
47
48static cl::opt<bool> WidenLoads(
49 "amdgpu-codegenprepare-widen-constant-loads",
50 cl::desc("Widen sub-dword constant address space loads in AMDGPUCodeGenPrepare"),
51 cl::ReallyHidden,
52 cl::init(Val: false));
53
54static cl::opt<bool>
55 BreakLargePHIs("amdgpu-codegenprepare-break-large-phis",
56 cl::desc("Break large PHI nodes for DAGISel"),
57 cl::ReallyHidden, cl::init(Val: true));
58
59static cl::opt<bool>
60 ForceBreakLargePHIs("amdgpu-codegenprepare-force-break-large-phis",
61 cl::desc("For testing purposes, always break large "
62 "PHIs even if it isn't profitable."),
63 cl::ReallyHidden, cl::init(Val: false));
64
65static cl::opt<unsigned> BreakLargePHIsThreshold(
66 "amdgpu-codegenprepare-break-large-phis-threshold",
67 cl::desc("Minimum type size in bits for breaking large PHI nodes"),
68 cl::ReallyHidden, cl::init(Val: 32));
69
70static cl::opt<bool> UseMul24Intrin(
71 "amdgpu-codegenprepare-mul24",
72 cl::desc("Introduce mul24 intrinsics in AMDGPUCodeGenPrepare"),
73 cl::ReallyHidden,
74 cl::init(Val: true));
75
76// Legalize 64-bit division by using the generic IR expansion.
77static cl::opt<bool> ExpandDiv64InIR(
78 "amdgpu-codegenprepare-expand-div64",
79 cl::desc("Expand 64-bit division in AMDGPUCodeGenPrepare"),
80 cl::ReallyHidden,
81 cl::init(Val: false));
82
83// Leave all division operations as they are. This supersedes ExpandDiv64InIR
84// and is used for testing the legalizer.
85static cl::opt<bool> DisableIDivExpand(
86 "amdgpu-codegenprepare-disable-idiv-expansion",
87 cl::desc("Prevent expanding integer division in AMDGPUCodeGenPrepare"),
88 cl::ReallyHidden,
89 cl::init(Val: false));
90
91// Disable processing of fdiv so we can better test the backend implementations.
92static cl::opt<bool> DisableFDivExpand(
93 "amdgpu-codegenprepare-disable-fdiv-expansion",
94 cl::desc("Prevent expanding floating point division in AMDGPUCodeGenPrepare"),
95 cl::ReallyHidden,
96 cl::init(Val: false));
97
98class AMDGPUCodeGenPrepareImpl
99 : public InstVisitor<AMDGPUCodeGenPrepareImpl, bool> {
100public:
101 Function &F;
102 const GCNSubtarget &ST;
103 const AMDGPUTargetMachine &TM;
104 const TargetTransformInfo &TTI;
105 const TargetLibraryInfo *TLI;
106 const UniformityInfo &UA;
107 const DataLayout &DL;
108 SimplifyQuery SQ;
109 const bool HasFP32DenormalFlush;
110 bool FlowChanged = false;
111 mutable Function *SqrtF32 = nullptr;
112 mutable Function *LdexpF32 = nullptr;
113 mutable SmallVector<WeakVH> DeadVals;
114
115 DenseMap<const PHINode *, bool> BreakPhiNodesCache;
116
117 AMDGPUCodeGenPrepareImpl(Function &F, const AMDGPUTargetMachine &TM,
118 const TargetTransformInfo &TTI,
119 const TargetLibraryInfo *TLI, AssumptionCache *AC,
120 const DominatorTree *DT, const UniformityInfo &UA)
121 : F(F), ST(TM.getSubtarget<GCNSubtarget>(F)), TM(TM), TTI(TTI), TLI(TLI),
122 UA(UA), DL(F.getDataLayout()), SQ(DL, TLI, DT, AC),
123 HasFP32DenormalFlush(SIModeRegisterDefaults(F, ST).FP32Denormals ==
124 DenormalMode::getPreserveSign()) {}
125
126 Function *getSqrtF32() const {
127 if (SqrtF32)
128 return SqrtF32;
129
130 LLVMContext &Ctx = F.getContext();
131 SqrtF32 = Intrinsic::getOrInsertDeclaration(
132 M: F.getParent(), id: Intrinsic::amdgcn_sqrt, OverloadTys: {Type::getFloatTy(C&: Ctx)});
133 return SqrtF32;
134 }
135
136 Function *getLdexpF32() const {
137 if (LdexpF32)
138 return LdexpF32;
139
140 LLVMContext &Ctx = F.getContext();
141 LdexpF32 = Intrinsic::getOrInsertDeclaration(
142 M: F.getParent(), id: Intrinsic::ldexp,
143 OverloadTys: {Type::getFloatTy(C&: Ctx), Type::getInt32Ty(C&: Ctx)});
144 return LdexpF32;
145 }
146
147 bool canBreakPHINode(const PHINode &I);
148
149 /// Return true if \p T is a legal scalar floating point type.
150 bool isLegalFloatingTy(const Type *T) const;
151
152 /// Wrapper to pass all the arguments to computeKnownFPClass
153 KnownFPClass computeKnownFPClass(const Value *V, FPClassTest Interested,
154 const Instruction *CtxI) const {
155 return llvm::computeKnownFPClass(V, InterestedClasses: Interested,
156 SQ: SQ.getWithInstruction(I: CtxI));
157 }
158
159 bool canIgnoreDenormalInput(const Value *V, const Instruction *CtxI) const {
160 return HasFP32DenormalFlush ||
161 computeKnownFPClass(V, Interested: fcSubnormal, CtxI).isKnownNeverSubnormal();
162 }
163
164 /// \returns The minimum number of bits needed to store the value of \Op as an
165 /// unsigned integer. Truncating to this size and then zero-extending to
166 /// the original will not change the value.
167 unsigned numBitsUnsigned(Value *Op, const Instruction *CtxI) const;
168
169 /// \returns The minimum number of bits needed to store the value of \Op as a
170 /// signed integer. Truncating to this size and then sign-extending to
171 /// the original size will not change the value.
172 unsigned numBitsSigned(Value *Op, const Instruction *CtxI) const;
173
174 /// Replace mul instructions with llvm.amdgcn.mul.u24 or llvm.amdgcn.mul.s24.
175 /// SelectionDAG has an issue where an and asserting the bits are known
176 bool replaceMulWithMul24(BinaryOperator &I) const;
177
178 /// Perform same function as equivalently named function in DAGCombiner. Since
179 /// we expand some divisions here, we need to perform this before obscuring.
180 bool foldBinOpIntoSelect(BinaryOperator &I) const;
181
182 bool divHasSpecialOptimization(BinaryOperator &I,
183 Value *Num, Value *Den) const;
184 unsigned getDivNumBits(BinaryOperator &I, Value *Num, Value *Den,
185 unsigned MaxDivBits, bool Signed) const;
186
187 /// Expands div or rem by using floating-point operations.
188 /// Operands must be in the range [-0x400000,0x3FFFFF]
189 Value *expandDivRemToFloat(IRBuilder<> &Builder, BinaryOperator &I,
190 Value *Num, Value *Den, bool IsDiv,
191 bool IsSigned) const;
192
193 Value *expandDivRemToFloatImpl(IRBuilder<> &Builder, BinaryOperator &I,
194 Value *Num, Value *Den, unsigned NumBits,
195 bool IsDiv, bool IsSigned) const;
196
197 /// Expands 32 bit div or rem.
198 Value* expandDivRem32(IRBuilder<> &Builder, BinaryOperator &I,
199 Value *Num, Value *Den) const;
200
201 Value *shrinkDivRem64(IRBuilder<> &Builder, BinaryOperator &I,
202 Value *Num, Value *Den) const;
203 void expandDivRem64(BinaryOperator &I) const;
204
205 /// Widen a scalar load.
206 ///
207 /// \details \p Widen scalar load for uniform, small type loads from constant
208 // memory / to a full 32-bits and then truncate the input to allow a scalar
209 // load instead of a vector load.
210 //
211 /// \returns True.
212
213 bool canWidenScalarExtLoad(LoadInst &I) const;
214
215 Value *matchFractPatImpl(Value &V, const APFloat &C) const;
216 Value *matchFractPatNanAvoidant(Value &V);
217 Value *applyFractPat(IRBuilder<> &Builder, Value *FractArg);
218
219 bool canOptimizeWithRsq(FastMathFlags DivFMF, FastMathFlags SqrtFMF) const;
220
221 Value *optimizeWithRsq(IRBuilder<> &Builder, Value *Num, Value *Den,
222 FastMathFlags DivFMF, FastMathFlags SqrtFMF,
223 const Instruction *CtxI) const;
224
225 Value *optimizeWithRcp(IRBuilder<> &Builder, Value *Num, Value *Den,
226 FastMathFlags FMF, const Instruction *CtxI) const;
227 Value *optimizeWithFDivFast(IRBuilder<> &Builder, Value *Num, Value *Den,
228 float ReqdAccuracy) const;
229
230 Value *visitFDivElement(IRBuilder<> &Builder, Value *Num, Value *Den,
231 FastMathFlags DivFMF, FastMathFlags SqrtFMF,
232 Value *RsqOp, const Instruction *FDiv,
233 float ReqdAccuracy) const;
234
235 std::pair<Value *, Value *> getFrexpResults(IRBuilder<> &Builder,
236 Value *Src) const;
237
238 Value *emitRcpIEEE1ULP(IRBuilder<> &Builder, Value *Src,
239 bool IsNegative) const;
240 Value *emitFrexpDiv(IRBuilder<> &Builder, Value *LHS, Value *RHS,
241 FastMathFlags FMF) const;
242 Value *emitSqrtIEEE2ULP(IRBuilder<> &Builder, Value *Src,
243 FastMathFlags FMF) const;
244 Value *emitRsqF64(IRBuilder<> &Builder, Value *X, FastMathFlags SqrtFMF,
245 FastMathFlags DivFMF, const Instruction *CtxI,
246 bool IsNegative) const;
247
248 CallInst *createWorkitemIdX(IRBuilder<> &B) const;
249 void replaceWithWorkitemIdX(Instruction &I) const;
250 void replaceWithMaskedWorkitemIdX(Instruction &I, unsigned WaveSize) const;
251 bool tryReplaceWithWorkitemId(Instruction &I, unsigned Wave) const;
252
253 bool tryNarrowMathIfNoOverflow(Instruction *I);
254
255public:
256 bool visitFDiv(BinaryOperator &I);
257
258 bool visitInstruction(Instruction &I) { return false; }
259 bool visitBinaryOperator(BinaryOperator &I);
260 bool visitLoadInst(LoadInst &I);
261 bool visitSelectInst(SelectInst &I);
262 bool visitPHINode(PHINode &I);
263 bool visitAddrSpaceCastInst(AddrSpaceCastInst &I);
264
265 bool visitIntrinsicInst(IntrinsicInst &I);
266 bool visitFMinLike(IntrinsicInst &I);
267 bool visitSqrt(IntrinsicInst &I);
268 bool visitLog(FPMathOperator &Log, Intrinsic::ID IID);
269 bool visitMbcntLo(IntrinsicInst &I) const;
270 bool visitMbcntHi(IntrinsicInst &I) const;
271 bool visitVectorReduceAdd(IntrinsicInst &I);
272 bool visitSaturatingAdd(IntrinsicInst &I);
273 bool run();
274};
275
276class AMDGPUCodeGenPrepare : public FunctionPass {
277public:
278 static char ID;
279 AMDGPUCodeGenPrepare() : FunctionPass(ID) {}
280 void getAnalysisUsage(AnalysisUsage &AU) const override {
281 AU.addRequired<AssumptionCacheTracker>();
282 AU.addRequired<UniformityInfoWrapperPass>();
283 AU.addRequired<TargetLibraryInfoWrapperPass>();
284 AU.addRequired<TargetTransformInfoWrapperPass>();
285
286 // FIXME: Division expansion needs to preserve the dominator tree.
287 if (!ExpandDiv64InIR)
288 AU.setPreservesAll();
289 }
290 bool runOnFunction(Function &F) override;
291 StringRef getPassName() const override { return "AMDGPU IR optimizations"; }
292};
293
294} // end anonymous namespace
295
296bool AMDGPUCodeGenPrepareImpl::run() {
297 BreakPhiNodesCache.clear();
298 bool MadeChange = false;
299
300 // Need to use make_early_inc_range because integer division expansion is
301 // handled by Transform/Utils, and it can delete instructions such as the
302 // terminator of the BB.
303 for (BasicBlock &BB : reverse(C&: F)) {
304 for (Instruction &I : make_early_inc_range(Range: reverse(C&: BB))) {
305 if (!isInstructionTriviallyDead(I: &I, TLI))
306 MadeChange |= visit(I);
307 }
308 }
309
310 while (!DeadVals.empty()) {
311 if (auto *I = dyn_cast_or_null<Instruction>(Val: DeadVals.pop_back_val()))
312 RecursivelyDeleteTriviallyDeadInstructions(V: I, TLI);
313 }
314
315 return MadeChange;
316}
317
318bool AMDGPUCodeGenPrepareImpl::isLegalFloatingTy(const Type *Ty) const {
319 return Ty->isFloatTy() || Ty->isDoubleTy() ||
320 (Ty->isHalfTy() && ST.has16BitInsts());
321}
322
323bool AMDGPUCodeGenPrepareImpl::canWidenScalarExtLoad(LoadInst &I) const {
324 Type *Ty = I.getType();
325 int TySize = DL.getTypeSizeInBits(Ty);
326 Align Alignment = DL.getValueOrABITypeAlignment(Alignment: I.getAlign(), Ty);
327
328 return I.isSimple() && TySize < 32 && Alignment >= 4 && UA.isUniformAtDef(V: &I);
329}
330
331unsigned
332AMDGPUCodeGenPrepareImpl::numBitsUnsigned(Value *Op,
333 const Instruction *CtxI) const {
334 return computeKnownBits(V: Op, Q: SQ.getWithInstruction(I: CtxI)).countMaxActiveBits();
335}
336
337unsigned
338AMDGPUCodeGenPrepareImpl::numBitsSigned(Value *Op,
339 const Instruction *CtxI) const {
340 return ComputeMaxSignificantBits(Op, DL: SQ.DL, AC: SQ.AC, CxtI: CtxI, DT: SQ.DT);
341}
342
343static void extractValues(IRBuilder<> &Builder,
344 SmallVectorImpl<Value *> &Values, Value *V) {
345 auto *VT = dyn_cast<FixedVectorType>(Val: V->getType());
346 if (!VT) {
347 Values.push_back(Elt: V);
348 return;
349 }
350
351 for (int I = 0, E = VT->getNumElements(); I != E; ++I)
352 Values.push_back(Elt: Builder.CreateExtractElement(Vec: V, Idx: I));
353}
354
355static Value *insertValues(IRBuilder<> &Builder,
356 Type *Ty,
357 SmallVectorImpl<Value *> &Values) {
358 if (!Ty->isVectorTy()) {
359 assert(Values.size() == 1);
360 return Values[0];
361 }
362
363 Value *NewVal = PoisonValue::get(T: Ty);
364 for (int I = 0, E = Values.size(); I != E; ++I)
365 NewVal = Builder.CreateInsertElement(Vec: NewVal, NewElt: Values[I], Idx: I);
366
367 return NewVal;
368}
369
370bool AMDGPUCodeGenPrepareImpl::replaceMulWithMul24(BinaryOperator &I) const {
371 if (I.getOpcode() != Instruction::Mul)
372 return false;
373
374 Type *Ty = I.getType();
375 unsigned Size = Ty->getScalarSizeInBits();
376 if (Size <= 16 && ST.has16BitInsts())
377 return false;
378
379 // Prefer scalar if this could be s_mul_i32
380 if (UA.isUniformAtDef(V: &I))
381 return false;
382
383 Value *LHS = I.getOperand(i_nocapture: 0);
384 Value *RHS = I.getOperand(i_nocapture: 1);
385 IRBuilder<> Builder(&I);
386 Builder.SetCurrentDebugLocation(I.getDebugLoc());
387
388 unsigned LHSBits = 0, RHSBits = 0;
389 bool IsSigned = false;
390
391 if (ST.hasMulU24() && (LHSBits = numBitsUnsigned(Op: LHS, CtxI: &I)) <= 24 &&
392 (RHSBits = numBitsUnsigned(Op: RHS, CtxI: &I)) <= 24) {
393 IsSigned = false;
394
395 } else if (ST.hasMulI24() && (LHSBits = numBitsSigned(Op: LHS, CtxI: &I)) <= 24 &&
396 (RHSBits = numBitsSigned(Op: RHS, CtxI: &I)) <= 24) {
397 IsSigned = true;
398
399 } else
400 return false;
401
402 SmallVector<Value *, 4> LHSVals;
403 SmallVector<Value *, 4> RHSVals;
404 SmallVector<Value *, 4> ResultVals;
405 extractValues(Builder, Values&: LHSVals, V: LHS);
406 extractValues(Builder, Values&: RHSVals, V: RHS);
407
408 IntegerType *I32Ty = Builder.getInt32Ty();
409 IntegerType *IntrinTy = Size > 32 ? Builder.getInt64Ty() : I32Ty;
410 Type *DstTy = LHSVals[0]->getType();
411
412 for (int I = 0, E = LHSVals.size(); I != E; ++I) {
413 Value *LHS = IsSigned ? Builder.CreateSExtOrTrunc(V: LHSVals[I], DestTy: I32Ty)
414 : Builder.CreateZExtOrTrunc(V: LHSVals[I], DestTy: I32Ty);
415 Value *RHS = IsSigned ? Builder.CreateSExtOrTrunc(V: RHSVals[I], DestTy: I32Ty)
416 : Builder.CreateZExtOrTrunc(V: RHSVals[I], DestTy: I32Ty);
417 Intrinsic::ID ID =
418 IsSigned ? Intrinsic::amdgcn_mul_i24 : Intrinsic::amdgcn_mul_u24;
419 Value *Result = Builder.CreateIntrinsic(ID, OverloadTypes: {IntrinTy}, Args: {LHS, RHS});
420 Result = IsSigned ? Builder.CreateSExtOrTrunc(V: Result, DestTy: DstTy)
421 : Builder.CreateZExtOrTrunc(V: Result, DestTy: DstTy);
422 ResultVals.push_back(Elt: Result);
423 }
424
425 Value *NewVal = insertValues(Builder, Ty, Values&: ResultVals);
426 NewVal->takeName(V: &I);
427 I.replaceAllUsesWith(V: NewVal);
428 DeadVals.push_back(Elt: &I);
429
430 return true;
431}
432
433// Find a select instruction, which may have been casted. This is mostly to deal
434// with cases where i16 selects were promoted here to i32.
435static SelectInst *findSelectThroughCast(Value *V, CastInst *&Cast) {
436 Cast = nullptr;
437 if (SelectInst *Sel = dyn_cast<SelectInst>(Val: V))
438 return Sel;
439
440 if ((Cast = dyn_cast<CastInst>(Val: V))) {
441 if (SelectInst *Sel = dyn_cast<SelectInst>(Val: Cast->getOperand(i_nocapture: 0)))
442 return Sel;
443 }
444
445 return nullptr;
446}
447
448bool AMDGPUCodeGenPrepareImpl::foldBinOpIntoSelect(BinaryOperator &BO) const {
449 // Don't do this unless the old select is going away. We want to eliminate the
450 // binary operator, not replace a binop with a select.
451 int SelOpNo = 0;
452
453 CastInst *CastOp;
454
455 // TODO: Should probably try to handle some cases with multiple
456 // users. Duplicating the select may be profitable for division.
457 SelectInst *Sel = findSelectThroughCast(V: BO.getOperand(i_nocapture: 0), Cast&: CastOp);
458 if (!Sel || !Sel->hasOneUse()) {
459 SelOpNo = 1;
460 Sel = findSelectThroughCast(V: BO.getOperand(i_nocapture: 1), Cast&: CastOp);
461 }
462
463 if (!Sel || !Sel->hasOneUse())
464 return false;
465
466 Constant *CT = dyn_cast<Constant>(Val: Sel->getTrueValue());
467 Constant *CF = dyn_cast<Constant>(Val: Sel->getFalseValue());
468 Constant *CBO = dyn_cast<Constant>(Val: BO.getOperand(i_nocapture: SelOpNo ^ 1));
469 if (!CBO || !CT || !CF)
470 return false;
471
472 if (CastOp) {
473 if (!CastOp->hasOneUse())
474 return false;
475 CT = ConstantFoldCastOperand(Opcode: CastOp->getOpcode(), C: CT, DestTy: BO.getType(), DL);
476 CF = ConstantFoldCastOperand(Opcode: CastOp->getOpcode(), C: CF, DestTy: BO.getType(), DL);
477 }
478
479 // TODO: Handle special 0/-1 cases DAG combine does, although we only really
480 // need to handle divisions here.
481 Constant *FoldedT =
482 SelOpNo ? ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CBO, RHS: CT, DL)
483 : ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CT, RHS: CBO, DL);
484 if (!FoldedT || isa<ConstantExpr>(Val: FoldedT))
485 return false;
486
487 Constant *FoldedF =
488 SelOpNo ? ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CBO, RHS: CF, DL)
489 : ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CF, RHS: CBO, DL);
490 if (!FoldedF || isa<ConstantExpr>(Val: FoldedF))
491 return false;
492
493 IRBuilder<> Builder(&BO);
494 Builder.SetCurrentDebugLocation(BO.getDebugLoc());
495 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Val: &BO))
496 Builder.setFastMathFlags(FPOp->getFastMathFlags());
497
498 Value *NewSelect = Builder.CreateSelect(C: Sel->getCondition(),
499 True: FoldedT, False: FoldedF);
500 NewSelect->takeName(V: &BO);
501 BO.replaceAllUsesWith(V: NewSelect);
502 DeadVals.push_back(Elt: &BO);
503 if (CastOp)
504 DeadVals.push_back(Elt: CastOp);
505 DeadVals.push_back(Elt: Sel);
506 return true;
507}
508
509std::pair<Value *, Value *>
510AMDGPUCodeGenPrepareImpl::getFrexpResults(IRBuilder<> &Builder,
511 Value *Src) const {
512 Type *Ty = Src->getType();
513 Value *Frexp = Builder.CreateIntrinsic(ID: Intrinsic::frexp,
514 OverloadTypes: {Ty, Builder.getInt32Ty()}, Args: Src);
515 Value *FrexpMant = Builder.CreateExtractValue(Agg: Frexp, Idxs: {0});
516
517 // Bypass the bug workaround for the exponent result since it doesn't matter.
518 // TODO: Does the bug workaround even really need to consider the exponent
519 // result? It's unspecified by the spec.
520
521 Value *FrexpExp =
522 ST.hasFractBug()
523 ? Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_frexp_exp,
524 OverloadTypes: {Builder.getInt32Ty(), Ty}, Args: Src)
525 : Builder.CreateExtractValue(Agg: Frexp, Idxs: {1});
526 return {FrexpMant, FrexpExp};
527}
528
529/// Emit an expansion of 1.0 / Src good for 1ulp that supports denormals.
530Value *AMDGPUCodeGenPrepareImpl::emitRcpIEEE1ULP(IRBuilder<> &Builder,
531 Value *Src,
532 bool IsNegative) const {
533 // Same as for 1.0, but expand the sign out of the constant.
534 // -1.0 / x -> rcp (fneg x)
535 if (IsNegative)
536 Src = Builder.CreateFNeg(V: Src);
537
538 // The rcp instruction doesn't support denormals, so scale the input
539 // out of the denormal range and convert at the end.
540 //
541 // Expand as 2^-n * (1.0 / (x * 2^n))
542
543 // TODO: Skip scaling if input is known never denormal and the input
544 // range won't underflow to denormal. The hard part is knowing the
545 // result. We need a range check, the result could be denormal for
546 // 0x1p+126 < den <= 0x1p+127.
547 auto [FrexpMant, FrexpExp] = getFrexpResults(Builder, Src);
548 Value *ScaleFactor = Builder.CreateNeg(V: FrexpExp);
549 Value *Rcp = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: FrexpMant);
550 return Builder.CreateCall(Callee: getLdexpF32(), Args: {Rcp, ScaleFactor});
551}
552
553/// Emit a 2ulp expansion for fdiv by using frexp for input scaling.
554Value *AMDGPUCodeGenPrepareImpl::emitFrexpDiv(IRBuilder<> &Builder, Value *LHS,
555 Value *RHS,
556 FastMathFlags FMF) const {
557 // If we have have to work around the fract/frexp bug, we're worse off than
558 // using the fdiv.fast expansion. The full safe expansion is faster if we have
559 // fast FMA.
560 if (HasFP32DenormalFlush && ST.hasFractBug() && !ST.hasFastFMAF32() &&
561 (!FMF.noNaNs() || !FMF.noInfs()))
562 return nullptr;
563
564 // We're scaling the LHS to avoid a denormal input, and scale the denominator
565 // to avoid large values underflowing the result.
566 auto [FrexpMantRHS, FrexpExpRHS] = getFrexpResults(Builder, Src: RHS);
567
568 Value *Rcp =
569 Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: FrexpMantRHS);
570
571 auto [FrexpMantLHS, FrexpExpLHS] = getFrexpResults(Builder, Src: LHS);
572 Value *Mul = Builder.CreateFMul(L: FrexpMantLHS, R: Rcp);
573
574 // We multiplied by 2^N/2^M, so we need to multiply by 2^(N-M) to scale the
575 // result.
576 Value *ExpDiff = Builder.CreateSub(LHS: FrexpExpLHS, RHS: FrexpExpRHS);
577 return Builder.CreateCall(Callee: getLdexpF32(), Args: {Mul, ExpDiff});
578}
579
580/// Emit a sqrt that handles denormals and is accurate to 2ulp.
581Value *AMDGPUCodeGenPrepareImpl::emitSqrtIEEE2ULP(IRBuilder<> &Builder,
582 Value *Src,
583 FastMathFlags FMF) const {
584 Type *Ty = Src->getType();
585 APFloat SmallestNormal =
586 APFloat::getSmallestNormalized(Sem: Ty->getFltSemantics());
587 Value *NeedScale =
588 Builder.CreateFCmpOLT(LHS: Src, RHS: ConstantFP::get(Ty, V: SmallestNormal));
589
590 ConstantInt *Zero = Builder.getInt32(C: 0);
591 Value *InputScaleFactor =
592 Builder.CreateSelect(C: NeedScale, True: Builder.getInt32(C: 32), False: Zero);
593
594 Value *Scaled = Builder.CreateCall(Callee: getLdexpF32(), Args: {Src, InputScaleFactor});
595
596 Value *Sqrt = Builder.CreateCall(Callee: getSqrtF32(), Args: Scaled);
597
598 Value *OutputScaleFactor =
599 Builder.CreateSelect(C: NeedScale, True: Builder.getInt32(C: -16), False: Zero);
600 return Builder.CreateCall(Callee: getLdexpF32(), Args: {Sqrt, OutputScaleFactor});
601}
602
603/// Emit an expansion of 1.0 / sqrt(Src) good for 1ulp that supports denormals.
604static Value *emitRsqIEEE1ULP(IRBuilder<> &Builder, Value *Src,
605 bool IsNegative) {
606 // bool need_scale = x < 0x1p-126f;
607 // float input_scale = need_scale ? 0x1.0p+24f : 1.0f;
608 // float output_scale = need_scale ? 0x1.0p+12f : 1.0f;
609 // rsq(x * input_scale) * output_scale;
610
611 Type *Ty = Src->getType();
612 APFloat SmallestNormal =
613 APFloat::getSmallestNormalized(Sem: Ty->getFltSemantics());
614 Value *NeedScale =
615 Builder.CreateFCmpOLT(LHS: Src, RHS: ConstantFP::get(Ty, V: SmallestNormal));
616 Constant *One = ConstantFP::get(Ty, V: 1.0);
617 Constant *InputScale = ConstantFP::get(Ty, V: 0x1.0p+24);
618 Constant *OutputScale =
619 ConstantFP::get(Ty, V: IsNegative ? -0x1.0p+12 : 0x1.0p+12);
620
621 Value *InputScaleFactor = Builder.CreateSelect(C: NeedScale, True: InputScale, False: One);
622
623 Value *ScaledInput = Builder.CreateFMul(L: Src, R: InputScaleFactor);
624 Value *Rsq = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rsq, Op: ScaledInput);
625 Value *OutputScaleFactor = Builder.CreateSelect(
626 C: NeedScale, True: OutputScale, False: IsNegative ? ConstantFP::get(Ty, V: -1.0) : One);
627
628 return Builder.CreateFMul(L: Rsq, R: OutputScaleFactor);
629}
630
631/// Emit inverse sqrt expansion for f64 with a correction sequence on top of
632/// v_rsq_f64. This should give a 1ulp result.
633Value *AMDGPUCodeGenPrepareImpl::emitRsqF64(IRBuilder<> &Builder, Value *X,
634 FastMathFlags SqrtFMF,
635 FastMathFlags DivFMF,
636 const Instruction *CtxI,
637 bool IsNegative) const {
638 // rsq(x):
639 // double y0 = BUILTIN_AMDGPU_RSQRT_F64(x);
640 // double e = MATH_MAD(-y0 * (x == PINF_F64 || x == 0.0 ? y0 : x), y0, 1.0);
641 // return MATH_MAD(y0*e, MATH_MAD(e, 0.375, 0.5), y0);
642 //
643 // -rsq(x):
644 // double y0 = BUILTIN_AMDGPU_RSQRT_F64(x);
645 // double e = MATH_MAD(-y0 * (x == PINF_F64 || x == 0.0 ? y0 : x), y0, 1.0);
646 // return MATH_MAD(-y0*e, MATH_MAD(e, 0.375, 0.5), -y0);
647 //
648 // The rsq instruction handles the special cases correctly. We need to check
649 // for the edge case conditions to ensure the special case propagates through
650 // the later instructions.
651
652 Value *Y0 = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rsq, Op: X);
653
654 // Try to elide the edge case check.
655 //
656 // Fast math flags imply:
657 // sqrt ninf => !isinf(x)
658 // fdiv ninf => x != 0, !isinf(x)
659 bool MaybePosInf = !SqrtFMF.noInfs() && !DivFMF.noInfs();
660 bool MaybeZero = !DivFMF.noInfs();
661
662 DenormalMode DenormMode;
663 FPClassTest Interested = fcNone;
664 if (MaybePosInf)
665 Interested = fcPosInf;
666 if (MaybeZero)
667 Interested |= fcZero;
668
669 if (Interested != fcNone) {
670 KnownFPClass KnownSrc = computeKnownFPClass(V: X, Interested, CtxI);
671 if (KnownSrc.isKnownNeverPosInfinity())
672 MaybePosInf = false;
673
674 DenormMode = F.getDenormalMode(FPType: X->getType()->getFltSemantics());
675 if (KnownSrc.isKnownNeverLogicalZero(Mode: DenormMode))
676 MaybeZero = false;
677 }
678
679 Value *SpecialOrRsq = X;
680 if (MaybeZero || MaybePosInf) {
681 Value *Cond;
682 if (MaybePosInf && MaybeZero) {
683 if (DenormMode.Input != DenormalMode::DenormalModeKind::Dynamic) {
684 FPClassTest TestMask = fcPosInf | fcZero;
685 if (DenormMode.inputsAreZero())
686 TestMask |= fcSubnormal;
687
688 Cond = Builder.createIsFPClass(FPNum: X, Test: TestMask);
689 } else {
690 // Avoid using llvm.is.fpclass for dynamic denormal mode, since it
691 // doesn't respect the floating-point environment.
692 Value *IsZero =
693 Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getZero(Ty: X->getType()));
694 Value *IsInf =
695 Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getInfinity(Ty: X->getType()));
696 Cond = Builder.CreateOr(LHS: IsZero, RHS: IsInf);
697 }
698 } else if (MaybeZero) {
699 Cond = Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getZero(Ty: X->getType()));
700 } else {
701 Cond = Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getInfinity(Ty: X->getType()));
702 }
703
704 SpecialOrRsq = Builder.CreateSelect(C: Cond, True: Y0, False: X);
705 }
706
707 Value *NegY0 = Builder.CreateFNeg(V: Y0);
708 Value *NegXY0 = Builder.CreateFMul(L: SpecialOrRsq, R: NegY0);
709
710 // Could be fmuladd, but isFMAFasterThanFMulAndFAdd is always true for f64.
711 Value *E = Builder.CreateFMA(Factor1: NegXY0, Factor2: Y0, Summand: ConstantFP::get(Ty: X->getType(), V: 1.0));
712
713 Value *Y0E = Builder.CreateFMul(L: E, R: IsNegative ? NegY0 : Y0);
714
715 Value *EFMA = Builder.CreateFMA(Factor1: E, Factor2: ConstantFP::get(Ty: X->getType(), V: 0.375),
716 Summand: ConstantFP::get(Ty: X->getType(), V: 0.5));
717
718 return Builder.CreateFMA(Factor1: Y0E, Factor2: EFMA, Summand: IsNegative ? NegY0 : Y0);
719}
720
721bool AMDGPUCodeGenPrepareImpl::canOptimizeWithRsq(FastMathFlags DivFMF,
722 FastMathFlags SqrtFMF) const {
723 // The rsqrt contraction increases accuracy from ~2ulp to ~1ulp for f32 and
724 // f64.
725 return DivFMF.allowContract() && SqrtFMF.allowContract();
726}
727
728Value *AMDGPUCodeGenPrepareImpl::optimizeWithRsq(
729 IRBuilder<> &Builder, Value *Num, Value *Den, const FastMathFlags DivFMF,
730 const FastMathFlags SqrtFMF, const Instruction *CtxI) const {
731 // The rsqrt contraction increases accuracy from ~2ulp to ~1ulp.
732 assert(DivFMF.allowContract() && SqrtFMF.allowContract());
733
734 // rsq_f16 is accurate to 0.51 ulp.
735 // rsq_f32 is accurate for !fpmath >= 1.0ulp and denormals are flushed.
736 // rsq_f64 is never accurate.
737 const ConstantFP *CLHS = dyn_cast<ConstantFP>(Val: Num);
738 if (!CLHS)
739 return nullptr;
740
741 bool IsNegative = false;
742
743 // TODO: Handle other numerator values with arcp.
744 if (CLHS->isOne() || (IsNegative = CLHS->isMinusOne())) {
745 // Add in the sqrt flags.
746 IRBuilder<>::FastMathFlagGuard Guard(Builder);
747 Builder.setFastMathFlags(DivFMF | SqrtFMF);
748
749 if (Den->getType()->isFloatTy()) {
750 if ((DivFMF.approxFunc() && SqrtFMF.approxFunc()) ||
751 canIgnoreDenormalInput(V: Den, CtxI)) {
752 Value *Result =
753 Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rsq, Op: Den);
754 // -1.0 / sqrt(x) -> fneg(rsq(x))
755 return IsNegative ? Builder.CreateFNeg(V: Result) : Result;
756 }
757
758 return emitRsqIEEE1ULP(Builder, Src: Den, IsNegative);
759 }
760
761 if (Den->getType()->isDoubleTy())
762 return emitRsqF64(Builder, X: Den, SqrtFMF, DivFMF, CtxI, IsNegative);
763 }
764
765 return nullptr;
766}
767
768// Optimize fdiv with rcp:
769//
770// 1/x -> rcp(x) when rcp is sufficiently accurate or inaccurate rcp is
771// allowed with afn.
772//
773// a/b -> a*rcp(b) when arcp is allowed, and we only need provide ULP 1.0
774Value *
775AMDGPUCodeGenPrepareImpl::optimizeWithRcp(IRBuilder<> &Builder, Value *Num,
776 Value *Den, FastMathFlags FMF,
777 const Instruction *CtxI) const {
778 // rcp_f16 is accurate to 0.51 ulp.
779 // rcp_f32 is accurate for !fpmath >= 1.0ulp and denormals are flushed.
780 // rcp_f64 is never accurate.
781 assert(Den->getType()->isFloatTy());
782
783 if (const ConstantFP *CLHS = dyn_cast<ConstantFP>(Val: Num)) {
784 bool IsNegative = false;
785 if (CLHS->isOne() || (IsNegative = CLHS->isMinusOne())) {
786 Value *Src = Den;
787
788 if (HasFP32DenormalFlush || FMF.approxFunc()) {
789 // -1.0 / x -> 1.0 / fneg(x)
790 if (IsNegative)
791 Src = Builder.CreateFNeg(V: Src);
792
793 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
794 // the CI documentation has a worst case error of 1 ulp.
795 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK
796 // to use it as long as we aren't trying to use denormals.
797 //
798 // v_rcp_f16 and v_rsq_f16 DO support denormals.
799
800 // NOTE: v_sqrt and v_rcp will be combined to v_rsq later. So we don't
801 // insert rsq intrinsic here.
802
803 // 1.0 / x -> rcp(x)
804 return Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: Src);
805 }
806
807 // TODO: If the input isn't denormal, and we know the input exponent isn't
808 // big enough to introduce a denormal we can avoid the scaling.
809 return emitRcpIEEE1ULP(Builder, Src, IsNegative);
810 }
811 }
812
813 if (FMF.allowReciprocal()) {
814 // x / y -> x * (1.0 / y)
815
816 // TODO: Could avoid denormal scaling and use raw rcp if we knew the output
817 // will never underflow.
818 if (HasFP32DenormalFlush || FMF.approxFunc()) {
819 Value *Recip = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: Den);
820 return Builder.CreateFMul(L: Num, R: Recip);
821 }
822
823 Value *Recip = emitRcpIEEE1ULP(Builder, Src: Den, IsNegative: false);
824 return Builder.CreateFMul(L: Num, R: Recip);
825 }
826
827 return nullptr;
828}
829
830// optimize with fdiv.fast:
831//
832// a/b -> fdiv.fast(a, b) when !fpmath >= 2.5ulp with denormals flushed.
833//
834// 1/x -> fdiv.fast(1,x) when !fpmath >= 2.5ulp.
835//
836// NOTE: optimizeWithRcp should be tried first because rcp is the preference.
837Value *AMDGPUCodeGenPrepareImpl::optimizeWithFDivFast(
838 IRBuilder<> &Builder, Value *Num, Value *Den, float ReqdAccuracy) const {
839 // fdiv.fast can achieve 2.5 ULP accuracy.
840 if (ReqdAccuracy < 2.5f)
841 return nullptr;
842
843 // Only have fdiv.fast for f32.
844 assert(Den->getType()->isFloatTy());
845
846 bool NumIsOne = false;
847 if (const ConstantFP *CNum = dyn_cast<ConstantFP>(Val: Num)) {
848 if (CNum->isOne() || CNum->isMinusOne())
849 NumIsOne = true;
850 }
851
852 // fdiv does not support denormals. But 1.0/x is always fine to use it.
853 //
854 // TODO: This works for any value with a specific known exponent range, don't
855 // just limit to constant 1.
856 if (!HasFP32DenormalFlush && !NumIsOne)
857 return nullptr;
858
859 return Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_fdiv_fast, Args: {Num, Den});
860}
861
862Value *AMDGPUCodeGenPrepareImpl::visitFDivElement(
863 IRBuilder<> &Builder, Value *Num, Value *Den, FastMathFlags DivFMF,
864 FastMathFlags SqrtFMF, Value *RsqOp, const Instruction *FDivInst,
865 float ReqdDivAccuracy) const {
866 if (RsqOp) {
867 Value *Rsq =
868 optimizeWithRsq(Builder, Num, Den: RsqOp, DivFMF, SqrtFMF, CtxI: FDivInst);
869 if (Rsq)
870 return Rsq;
871 }
872
873 if (!Num->getType()->isFloatTy())
874 return nullptr;
875
876 Value *Rcp = optimizeWithRcp(Builder, Num, Den, FMF: DivFMF, CtxI: FDivInst);
877 if (Rcp)
878 return Rcp;
879
880 // In the basic case fdiv_fast has the same instruction count as the frexp div
881 // expansion. Slightly prefer fdiv_fast since it ends in an fmul that can
882 // potentially be fused into a user. Also, materialization of the constants
883 // can be reused for multiple instances.
884 Value *FDivFast = optimizeWithFDivFast(Builder, Num, Den, ReqdAccuracy: ReqdDivAccuracy);
885 if (FDivFast)
886 return FDivFast;
887
888 return emitFrexpDiv(Builder, LHS: Num, RHS: Den, FMF: DivFMF);
889}
890
891// Optimizations is performed based on fpmath, fast math flags as well as
892// denormals to optimize fdiv with either rcp or fdiv.fast.
893//
894// With rcp:
895// 1/x -> rcp(x) when rcp is sufficiently accurate or inaccurate rcp is
896// allowed with afn.
897//
898// a/b -> a*rcp(b) when inaccurate rcp is allowed with afn.
899//
900// With fdiv.fast:
901// a/b -> fdiv.fast(a, b) when !fpmath >= 2.5ulp with denormals flushed.
902//
903// 1/x -> fdiv.fast(1,x) when !fpmath >= 2.5ulp.
904//
905// NOTE: rcp is the preference in cases that both are legal.
906bool AMDGPUCodeGenPrepareImpl::visitFDiv(BinaryOperator &FDiv) {
907 if (DisableFDivExpand)
908 return false;
909
910 Type *Ty = FDiv.getType()->getScalarType();
911 const bool IsFloat = Ty->isFloatTy();
912 if (!IsFloat && !Ty->isDoubleTy())
913 return false;
914
915 // The f64 rcp/rsq approximations are pretty inaccurate. We can do an
916 // expansion around them in codegen. f16 is good enough to always use.
917
918 const FPMathOperator *FPOp = cast<const FPMathOperator>(Val: &FDiv);
919 const FastMathFlags DivFMF = FPOp->getFastMathFlags();
920 const float ReqdAccuracy = FPOp->getFPAccuracy();
921
922 FastMathFlags SqrtFMF;
923
924 Value *Num = FDiv.getOperand(i_nocapture: 0);
925 Value *Den = FDiv.getOperand(i_nocapture: 1);
926
927 Value *RsqOp = nullptr;
928 auto *DenII = dyn_cast<IntrinsicInst>(Val: Den);
929 if (DenII && DenII->getIntrinsicID() == Intrinsic::sqrt &&
930 DenII->hasOneUse()) {
931 const auto *SqrtOp = cast<FPMathOperator>(Val: DenII);
932 SqrtFMF = SqrtOp->getFastMathFlags();
933 if (canOptimizeWithRsq(DivFMF, SqrtFMF))
934 RsqOp = SqrtOp->getOperand(i: 0);
935 }
936
937 // rcp path not yet implemented for f64.
938 if (!IsFloat && !RsqOp)
939 return false;
940
941 // Inaccurate rcp is allowed with afn.
942 //
943 // Defer to codegen to handle this.
944 //
945 // TODO: Decide on an interpretation for interactions between afn + arcp +
946 // !fpmath, and make it consistent between here and codegen. For now, defer
947 // expansion of afn to codegen. The current interpretation is so aggressive we
948 // don't need any pre-consideration here when we have better information. A
949 // more conservative interpretation could use handling here.
950 const bool AllowInaccurateRcp = DivFMF.approxFunc();
951 if (!RsqOp && AllowInaccurateRcp)
952 return false;
953
954 // Defer the correct implementations to codegen.
955 if (IsFloat && ReqdAccuracy < 1.0f)
956 return false;
957
958 IRBuilder<> Builder(FDiv.getParent(), std::next(x: FDiv.getIterator()));
959 Builder.setFastMathFlags(DivFMF);
960 Builder.SetCurrentDebugLocation(FDiv.getDebugLoc());
961
962 SmallVector<Value *, 4> NumVals;
963 SmallVector<Value *, 4> DenVals;
964 SmallVector<Value *, 4> RsqDenVals;
965 extractValues(Builder, Values&: NumVals, V: Num);
966 extractValues(Builder, Values&: DenVals, V: Den);
967
968 if (RsqOp)
969 extractValues(Builder, Values&: RsqDenVals, V: RsqOp);
970
971 SmallVector<Value *, 4> ResultVals(NumVals.size());
972 for (int I = 0, E = NumVals.size(); I != E; ++I) {
973 Value *NumElt = NumVals[I];
974 Value *DenElt = DenVals[I];
975 Value *RsqDenElt = RsqOp ? RsqDenVals[I] : nullptr;
976
977 Value *NewElt =
978 visitFDivElement(Builder, Num: NumElt, Den: DenElt, DivFMF, SqrtFMF, RsqOp: RsqDenElt,
979 FDivInst: cast<Instruction>(Val: FPOp), ReqdDivAccuracy: ReqdAccuracy);
980 if (!NewElt) {
981 // Keep the original, but scalarized.
982
983 // This has the unfortunate side effect of sometimes scalarizing when
984 // we're not going to do anything.
985 NewElt = Builder.CreateFDiv(L: NumElt, R: DenElt);
986 if (auto *NewEltInst = dyn_cast<Instruction>(Val: NewElt))
987 NewEltInst->copyMetadata(SrcInst: FDiv);
988 }
989
990 ResultVals[I] = NewElt;
991 }
992
993 Value *NewVal = insertValues(Builder, Ty: FDiv.getType(), Values&: ResultVals);
994
995 if (NewVal) {
996 FDiv.replaceAllUsesWith(V: NewVal);
997 NewVal->takeName(V: &FDiv);
998 DeadVals.push_back(Elt: &FDiv);
999 }
1000
1001 return true;
1002}
1003
1004static std::pair<Value*, Value*> getMul64(IRBuilder<> &Builder,
1005 Value *LHS, Value *RHS) {
1006 Type *I32Ty = Builder.getInt32Ty();
1007 Type *I64Ty = Builder.getInt64Ty();
1008
1009 Value *LHS_EXT64 = Builder.CreateZExt(V: LHS, DestTy: I64Ty);
1010 Value *RHS_EXT64 = Builder.CreateZExt(V: RHS, DestTy: I64Ty);
1011 Value *MUL64 = Builder.CreateMul(LHS: LHS_EXT64, RHS: RHS_EXT64);
1012 Value *Lo = Builder.CreateTrunc(V: MUL64, DestTy: I32Ty);
1013 Value *Hi = Builder.CreateLShr(LHS: MUL64, RHS: Builder.getInt64(C: 32));
1014 Hi = Builder.CreateTrunc(V: Hi, DestTy: I32Ty);
1015 return std::pair(Lo, Hi);
1016}
1017
1018static Value* getMulHu(IRBuilder<> &Builder, Value *LHS, Value *RHS) {
1019 return getMul64(Builder, LHS, RHS).second;
1020}
1021
1022/// Figure out how many bits are really needed for this division.
1023/// \p MaxDivBits is an optimization hint to bypass the second
1024/// ComputeNumSignBits/computeKnownBits call if the first one is
1025/// insufficient.
1026unsigned AMDGPUCodeGenPrepareImpl::getDivNumBits(BinaryOperator &I, Value *Num,
1027 Value *Den,
1028 unsigned MaxDivBits,
1029 bool IsSigned) const {
1030 assert(Num->getType()->getScalarSizeInBits() ==
1031 Den->getType()->getScalarSizeInBits());
1032 unsigned SSBits = Num->getType()->getScalarSizeInBits();
1033 if (IsSigned) {
1034 unsigned RHSSignBits = ComputeNumSignBits(Op: Den, DL: SQ.DL, AC: SQ.AC, CxtI: &I, DT: SQ.DT);
1035 // A sign bit needs to be reserved for shrinking.
1036 unsigned DivBits = SSBits - RHSSignBits + 1;
1037 if (DivBits > MaxDivBits)
1038 return SSBits;
1039
1040 unsigned LHSSignBits = ComputeNumSignBits(Op: Num, DL: SQ.DL, AC: SQ.AC, CxtI: &I);
1041
1042 unsigned SignBits = std::min(a: LHSSignBits, b: RHSSignBits);
1043 DivBits = SSBits - SignBits + 1;
1044 return DivBits;
1045 }
1046
1047 // All bits are used for unsigned division for Num or Den in range
1048 // (SignedMax, UnsignedMax].
1049 KnownBits Known = computeKnownBits(V: Den, Q: SQ.getWithInstruction(I: &I));
1050 unsigned RHSBits = Known.countMaxActiveBits();
1051 if (RHSBits > MaxDivBits)
1052 return SSBits;
1053
1054 Known = computeKnownBits(V: Num, Q: SQ.getWithInstruction(I: &I));
1055 unsigned LHSBits = Known.countMaxActiveBits();
1056
1057 unsigned DivBits = std::max(a: LHSBits, b: RHSBits);
1058 return DivBits;
1059}
1060
1061Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloat(IRBuilder<> &Builder,
1062 BinaryOperator &I,
1063 Value *Num, Value *Den,
1064 bool IsDiv,
1065 bool IsSigned) const {
1066 unsigned DivBits = getDivNumBits(I, Num, Den, MaxDivBits: 23, IsSigned);
1067
1068 if (DivBits > (IsSigned ? 23 : 22))
1069 return nullptr;
1070 return expandDivRemToFloatImpl(Builder, I, Num, Den, NumBits: DivBits, IsDiv,
1071 IsSigned);
1072}
1073
1074Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
1075 IRBuilder<> &Builder, BinaryOperator &I, Value *Num, Value *Den,
1076 unsigned DivBits, bool IsDiv, bool IsSigned) const {
1077
1078 // v_rcp_f32(float(X)) can have an error of 1 ulp.
1079 // This would cause incorrect calculation of Y/X if:
1080 // Y = (0x7FFFFF/X)*(X-0)-1
1081 // were allowed.
1082 //
1083 // For example,
1084 // (0x7FF6D3/0x000FE7) would erroneously produce 2060 instead of 2059.
1085 // (0x7FF8F5/0x007EFB) would erroneously produce 258 instead of 257.
1086 //
1087 // Thus, we conservatively restrict expandDivRemToFloatImpl to
1088 // [-0x400000,0x3FFFFF] for IsSigned
1089 // [ 0x000000,0x3FFFFF] for !IsSigned.
1090 assert(0 < DivBits && DivBits <= (IsSigned ? 23 : 22) &&
1091 "abs(Num) must be <= 0x400000 for expandDivRemToFloatImpl to work "
1092 "correctly");
1093
1094 Type *I32Ty = Builder.getInt32Ty();
1095 Num = Builder.CreateTrunc(V: Num, DestTy: I32Ty);
1096 Den = Builder.CreateTrunc(V: Den, DestTy: I32Ty);
1097
1098 Type *F32Ty = Builder.getFloatTy();
1099 ConstantInt *One = Builder.getInt32(C: 1);
1100
1101 // int ia = (int)LHS;
1102 Value *IA = Num;
1103
1104 // int ib, (int)RHS;
1105 Value *IB = Den;
1106
1107 // float fa = (float)ia;
1108 Value *FA = IsSigned ? Builder.CreateSIToFP(V: IA, DestTy: F32Ty)
1109 : Builder.CreateUIToFP(V: IA, DestTy: F32Ty);
1110
1111 // float fb = (float)ib;
1112 Value *FB = IsSigned ? Builder.CreateSIToFP(V: IB, DestTy: F32Ty)
1113 : Builder.CreateUIToFP(V: IB, DestTy: F32Ty);
1114
1115 Value *RCP = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_rcp,
1116 OverloadTypes: Builder.getFloatTy(), Args: {FB});
1117
1118 // The calculation:
1119 // fq = fa*recip(fb)
1120 // may be too small due to the 1ulp accuracy in the recip
1121 // operation and rounding issues. Since fq is truncated to produce
1122 // an integer value it may be too small by one. This is
1123 // dealt with by incrementing fa by 1ulp:
1124 // fq = (fa+1ulp)*recip(fb)
1125 // This will increase fa's magnitude by at most 0.5
1126 // (i.e. when fabs(fa)==0x400000 the LSB of the mantissa represents 0.5).
1127 // Thus, this method is safe since fa must be incremented by at least 1.0
1128 // for the quotient to increase by one.
1129
1130 Value *FABits = Builder.CreateBitCast(V: FA, DestTy: I32Ty);
1131 Value *FABitsInc = Builder.CreateAdd(LHS: FABits, RHS: One);
1132 FA = Builder.CreateBitCast(V: FABitsInc, DestTy: F32Ty);
1133
1134 Value *FQM = Builder.CreateFMul(L: FA, R: RCP);
1135
1136 // fq = trunc(fqm);
1137 Value *FQ = Builder.CreateUnaryIntrinsic(ID: Intrinsic::trunc, Op: FQM);
1138
1139 // int iq = (int)fq;
1140 Value *IQ = IsSigned ? Builder.CreateFPToSI(V: FQ, DestTy: I32Ty)
1141 : Builder.CreateFPToUI(V: FQ, DestTy: I32Ty);
1142
1143 Value *Res = IQ;
1144 if (!IsDiv) {
1145 // Rem needs compensation, it's easier to recompute it
1146 Value *Rem = Builder.CreateMul(LHS: IQ, RHS: Den);
1147 Res = Builder.CreateSub(LHS: Num, RHS: Rem);
1148 }
1149
1150 return Res;
1151}
1152
1153// Try to recognize special cases the DAG will emit special, better expansions
1154// than the general expansion we do here.
1155
1156// TODO: It would be better to just directly handle those optimizations here.
1157bool AMDGPUCodeGenPrepareImpl::divHasSpecialOptimization(BinaryOperator &I,
1158 Value *Num,
1159 Value *Den) const {
1160 if (Constant *C = dyn_cast<Constant>(Val: Den)) {
1161 // Arbitrary constants get a better expansion as long as a wider mulhi is
1162 // legal.
1163 if (C->getType()->getScalarSizeInBits() <= 32)
1164 return true;
1165
1166 // TODO: Sdiv check for not exact for some reason.
1167
1168 // If there's no wider mulhi, there's only a better expansion for powers of
1169 // two.
1170 // TODO: Should really know for each vector element.
1171 if (isKnownToBeAPowerOfTwo(V: C, OrZero: true, Q: SQ.getWithInstruction(I: &I)))
1172 return true;
1173
1174 return false;
1175 }
1176
1177 if (BinaryOperator *BinOpDen = dyn_cast<BinaryOperator>(Val: Den)) {
1178 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
1179 if (BinOpDen->getOpcode() == Instruction::Shl &&
1180 isa<Constant>(Val: BinOpDen->getOperand(i_nocapture: 0)) &&
1181 isKnownToBeAPowerOfTwo(V: BinOpDen->getOperand(i_nocapture: 0), OrZero: true,
1182 Q: SQ.getWithInstruction(I: &I))) {
1183 return true;
1184 }
1185 }
1186
1187 return false;
1188}
1189
1190static Value *getSign32(Value *V, IRBuilder<> &Builder, const DataLayout DL) {
1191 // Check whether the sign can be determined statically.
1192 KnownBits Known = computeKnownBits(V, DL);
1193 if (Known.isNegative())
1194 return Constant::getAllOnesValue(Ty: V->getType());
1195 if (Known.isNonNegative())
1196 return Constant::getNullValue(Ty: V->getType());
1197 return Builder.CreateAShr(LHS: V, RHS: Builder.getInt32(C: 31));
1198}
1199
1200Value *AMDGPUCodeGenPrepareImpl::expandDivRem32(IRBuilder<> &Builder,
1201 BinaryOperator &I, Value *X,
1202 Value *Y) const {
1203 Instruction::BinaryOps Opc = I.getOpcode();
1204 assert(Opc == Instruction::URem || Opc == Instruction::UDiv ||
1205 Opc == Instruction::SRem || Opc == Instruction::SDiv);
1206
1207 FastMathFlags FMF;
1208 FMF.setFast();
1209 Builder.setFastMathFlags(FMF);
1210
1211 if (divHasSpecialOptimization(I, Num: X, Den: Y))
1212 return nullptr; // Keep it for later optimization.
1213
1214 bool IsDiv = Opc == Instruction::UDiv || Opc == Instruction::SDiv;
1215 bool IsSigned = Opc == Instruction::SRem || Opc == Instruction::SDiv;
1216
1217 Type *Ty = X->getType();
1218 Type *I32Ty = Builder.getInt32Ty();
1219 Type *F32Ty = Builder.getFloatTy();
1220
1221 if (Ty->getScalarSizeInBits() != 32) {
1222 if (IsSigned) {
1223 X = Builder.CreateSExtOrTrunc(V: X, DestTy: I32Ty);
1224 Y = Builder.CreateSExtOrTrunc(V: Y, DestTy: I32Ty);
1225 } else {
1226 X = Builder.CreateZExtOrTrunc(V: X, DestTy: I32Ty);
1227 Y = Builder.CreateZExtOrTrunc(V: Y, DestTy: I32Ty);
1228 }
1229 }
1230
1231 if (Value *Res = expandDivRemToFloat(Builder, I, Num: X, Den: Y, IsDiv, IsSigned)) {
1232 return IsSigned ? Builder.CreateSExtOrTrunc(V: Res, DestTy: Ty) :
1233 Builder.CreateZExtOrTrunc(V: Res, DestTy: Ty);
1234 }
1235
1236 ConstantInt *Zero = Builder.getInt32(C: 0);
1237 ConstantInt *One = Builder.getInt32(C: 1);
1238
1239 Value *Sign = nullptr;
1240 if (IsSigned) {
1241 Value *SignX = getSign32(V: X, Builder, DL);
1242 Value *SignY = getSign32(V: Y, Builder, DL);
1243 // Remainder sign is the same as LHS
1244 Sign = IsDiv ? Builder.CreateXor(LHS: SignX, RHS: SignY) : SignX;
1245
1246 X = Builder.CreateAdd(LHS: X, RHS: SignX);
1247 Y = Builder.CreateAdd(LHS: Y, RHS: SignY);
1248
1249 X = Builder.CreateXor(LHS: X, RHS: SignX);
1250 Y = Builder.CreateXor(LHS: Y, RHS: SignY);
1251 }
1252
1253 // The algorithm here is based on ideas from "Software Integer Division", Tom
1254 // Rodeheffer, August 2008.
1255 //
1256 // unsigned udiv(unsigned x, unsigned y) {
1257 // // Initial estimate of inv(y). The constant is less than 2^32 to ensure
1258 // // that this is a lower bound on inv(y), even if some of the calculations
1259 // // round up.
1260 // unsigned z = (unsigned)((4294967296.0 - 512.0) * v_rcp_f32((float)y));
1261 //
1262 // // One round of UNR (Unsigned integer Newton-Raphson) to improve z.
1263 // // Empirically this is guaranteed to give a "two-y" lower bound on
1264 // // inv(y).
1265 // z += umulh(z, -y * z);
1266 //
1267 // // Quotient/remainder estimate.
1268 // unsigned q = umulh(x, z);
1269 // unsigned r = x - q * y;
1270 //
1271 // // Two rounds of quotient/remainder refinement.
1272 // if (r >= y) {
1273 // ++q;
1274 // r -= y;
1275 // }
1276 // if (r >= y) {
1277 // ++q;
1278 // r -= y;
1279 // }
1280 //
1281 // return q;
1282 // }
1283
1284 // Initial estimate of inv(y).
1285 Value *FloatY = Builder.CreateUIToFP(V: Y, DestTy: F32Ty);
1286 Value *RcpY = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_rcp, OverloadTypes: F32Ty, Args: {FloatY});
1287 Constant *Scale = ConstantFP::get(Ty: F32Ty, V: llvm::bit_cast<float>(from: 0x4F7FFFFE));
1288 Value *ScaledY = Builder.CreateFMul(L: RcpY, R: Scale);
1289 Value *Z = Builder.CreateFPToUI(V: ScaledY, DestTy: I32Ty);
1290
1291 // One round of UNR.
1292 Value *NegY = Builder.CreateSub(LHS: Zero, RHS: Y);
1293 Value *NegYZ = Builder.CreateMul(LHS: NegY, RHS: Z);
1294 Z = Builder.CreateAdd(LHS: Z, RHS: getMulHu(Builder, LHS: Z, RHS: NegYZ));
1295
1296 // Quotient/remainder estimate.
1297 Value *Q = getMulHu(Builder, LHS: X, RHS: Z);
1298 Value *R = Builder.CreateSub(LHS: X, RHS: Builder.CreateMul(LHS: Q, RHS: Y));
1299
1300 // First quotient/remainder refinement.
1301 Value *Cond = Builder.CreateICmpUGE(LHS: R, RHS: Y);
1302 if (IsDiv)
1303 Q = Builder.CreateSelect(C: Cond, True: Builder.CreateAdd(LHS: Q, RHS: One), False: Q);
1304 R = Builder.CreateSelect(C: Cond, True: Builder.CreateSub(LHS: R, RHS: Y), False: R);
1305
1306 // Second quotient/remainder refinement.
1307 Cond = Builder.CreateICmpUGE(LHS: R, RHS: Y);
1308 Value *Res;
1309 if (IsDiv)
1310 Res = Builder.CreateSelect(C: Cond, True: Builder.CreateAdd(LHS: Q, RHS: One), False: Q);
1311 else
1312 Res = Builder.CreateSelect(C: Cond, True: Builder.CreateSub(LHS: R, RHS: Y), False: R);
1313
1314 if (IsSigned) {
1315 Res = Builder.CreateXor(LHS: Res, RHS: Sign);
1316 Res = Builder.CreateSub(LHS: Res, RHS: Sign);
1317 Res = Builder.CreateSExtOrTrunc(V: Res, DestTy: Ty);
1318 } else {
1319 Res = Builder.CreateZExtOrTrunc(V: Res, DestTy: Ty);
1320 }
1321 return Res;
1322}
1323
1324Value *AMDGPUCodeGenPrepareImpl::shrinkDivRem64(IRBuilder<> &Builder,
1325 BinaryOperator &I, Value *Num,
1326 Value *Den) const {
1327 if (!ExpandDiv64InIR && divHasSpecialOptimization(I, Num, Den))
1328 return nullptr; // Keep it for later optimization.
1329
1330 Instruction::BinaryOps Opc = I.getOpcode();
1331
1332 bool IsDiv = Opc == Instruction::SDiv || Opc == Instruction::UDiv;
1333 bool IsSigned = Opc == Instruction::SDiv || Opc == Instruction::SRem;
1334
1335 unsigned NumDivBits = getDivNumBits(I, Num, Den, MaxDivBits: 32, IsSigned);
1336 if (NumDivBits > 32)
1337 return nullptr;
1338
1339 Value *Narrowed = nullptr;
1340 if (NumDivBits <= (IsSigned ? 23 : 22)) {
1341 Narrowed = expandDivRemToFloatImpl(Builder, I, Num, Den, DivBits: NumDivBits, IsDiv,
1342 IsSigned);
1343 } else if (NumDivBits <= (IsSigned ? 31 : 32)) {
1344 // Do not use 32-bit division if dividend may be -2147483648.
1345 // Otherwise 32-bit division cannot be used safely.
1346 // -2147483648/1 and -2147483648/-1 are not equal,
1347 // but they produce the same lower 32-bit result.
1348 Narrowed = expandDivRem32(Builder, I, X: Num, Y: Den);
1349 }
1350
1351 if (Narrowed) {
1352 return IsSigned ? Builder.CreateSExt(V: Narrowed, DestTy: Num->getType()) :
1353 Builder.CreateZExt(V: Narrowed, DestTy: Num->getType());
1354 }
1355
1356 return nullptr;
1357}
1358
1359void AMDGPUCodeGenPrepareImpl::expandDivRem64(BinaryOperator &I) const {
1360 Instruction::BinaryOps Opc = I.getOpcode();
1361 // Do the general expansion.
1362 if (Opc == Instruction::UDiv || Opc == Instruction::SDiv) {
1363 expandDivisionUpTo64Bits(Div: &I);
1364 return;
1365 }
1366
1367 if (Opc == Instruction::URem || Opc == Instruction::SRem) {
1368 expandRemainderUpTo64Bits(Rem: &I);
1369 return;
1370 }
1371
1372 llvm_unreachable("not a division");
1373}
1374
1375/*
1376This will cause non-byte load in consistency, for example:
1377```
1378 %load = load i1, ptr addrspace(4) %arg, align 4
1379 %zext = zext i1 %load to
1380 i64 %add = add i64 %zext
1381```
1382Instead of creating `s_and_b32 s0, s0, 1`,
1383it will create `s_and_b32 s0, s0, 0xff`.
1384We accept this change since the non-byte load assumes the upper bits
1385within the byte are all 0.
1386*/
1387bool AMDGPUCodeGenPrepareImpl::tryNarrowMathIfNoOverflow(Instruction *I) {
1388 unsigned Opc = I->getOpcode();
1389 Type *OldType = I->getType();
1390
1391 if (Opc != Instruction::Add && Opc != Instruction::Mul)
1392 return false;
1393
1394 unsigned OrigBit = OldType->getScalarSizeInBits();
1395
1396 if (Opc != Instruction::Add && Opc != Instruction::Mul)
1397 llvm_unreachable("Unexpected opcode, only valid for Instruction::Add and "
1398 "Instruction::Mul.");
1399
1400 unsigned MaxBitsNeeded = computeKnownBits(V: I, DL).countMaxActiveBits();
1401
1402 MaxBitsNeeded = std::max<unsigned>(a: bit_ceil(Value: MaxBitsNeeded), b: 8);
1403 Type *NewType = DL.getSmallestLegalIntType(C&: I->getContext(), Width: MaxBitsNeeded);
1404 if (!NewType)
1405 return false;
1406 unsigned NewBit = NewType->getIntegerBitWidth();
1407 if (NewBit >= OrigBit)
1408 return false;
1409 NewType = I->getType()->getWithNewBitWidth(NewBitWidth: NewBit);
1410
1411 // Old cost
1412 InstructionCost OldCost =
1413 TTI.getArithmeticInstrCost(Opcode: Opc, Ty: OldType, CostKind: TTI::TCK_RecipThroughput);
1414 // New cost of new op
1415 InstructionCost NewCost =
1416 TTI.getArithmeticInstrCost(Opcode: Opc, Ty: NewType, CostKind: TTI::TCK_RecipThroughput);
1417 // New cost of narrowing 2 operands (use trunc)
1418 int NumOfNonConstOps = 2;
1419 if (isa<Constant>(Val: I->getOperand(i: 0)) || isa<Constant>(Val: I->getOperand(i: 1))) {
1420 // Cannot be both constant, should be propagated
1421 NumOfNonConstOps = 1;
1422 }
1423 NewCost += NumOfNonConstOps * TTI.getCastInstrCost(Opcode: Instruction::Trunc,
1424 Dst: NewType, Src: OldType,
1425 CCH: TTI.getCastContextHint(I),
1426 CostKind: TTI::TCK_RecipThroughput);
1427 // New cost of zext narrowed result to original type
1428 NewCost +=
1429 TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: OldType, Src: NewType,
1430 CCH: TTI.getCastContextHint(I), CostKind: TTI::TCK_RecipThroughput);
1431 if (NewCost >= OldCost)
1432 return false;
1433
1434 IRBuilder<> Builder(I);
1435 Value *Trunc0 = Builder.CreateTrunc(V: I->getOperand(i: 0), DestTy: NewType);
1436 Value *Trunc1 = Builder.CreateTrunc(V: I->getOperand(i: 1), DestTy: NewType);
1437 Value *Arith =
1438 Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Opc, LHS: Trunc0, RHS: Trunc1);
1439
1440 Value *Zext = Builder.CreateZExt(V: Arith, DestTy: OldType);
1441 I->replaceAllUsesWith(V: Zext);
1442 DeadVals.push_back(Elt: I);
1443 return true;
1444}
1445
1446bool AMDGPUCodeGenPrepareImpl::visitBinaryOperator(BinaryOperator &I) {
1447 if (foldBinOpIntoSelect(BO&: I))
1448 return true;
1449
1450 if (UseMul24Intrin && replaceMulWithMul24(I))
1451 return true;
1452 if (tryNarrowMathIfNoOverflow(I: &I))
1453 return true;
1454
1455 bool Changed = false;
1456 Instruction::BinaryOps Opc = I.getOpcode();
1457 Type *Ty = I.getType();
1458 Value *NewDiv = nullptr;
1459 unsigned ScalarSize = Ty->getScalarSizeInBits();
1460
1461 SmallVector<BinaryOperator *, 8> Div64ToExpand;
1462
1463 if ((Opc == Instruction::URem || Opc == Instruction::UDiv ||
1464 Opc == Instruction::SRem || Opc == Instruction::SDiv) &&
1465 ScalarSize <= 64 &&
1466 !DisableIDivExpand) {
1467 Value *Num = I.getOperand(i_nocapture: 0);
1468 Value *Den = I.getOperand(i_nocapture: 1);
1469 IRBuilder<> Builder(&I);
1470 Builder.SetCurrentDebugLocation(I.getDebugLoc());
1471
1472 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1473 NewDiv = PoisonValue::get(T: VT);
1474
1475 for (unsigned N = 0, E = VT->getNumElements(); N != E; ++N) {
1476 Value *NumEltN = Builder.CreateExtractElement(Vec: Num, Idx: N);
1477 Value *DenEltN = Builder.CreateExtractElement(Vec: Den, Idx: N);
1478
1479 Value *NewElt;
1480 if (ScalarSize <= 32) {
1481 NewElt = expandDivRem32(Builder, I, X: NumEltN, Y: DenEltN);
1482 if (!NewElt)
1483 NewElt = Builder.CreateBinOp(Opc, LHS: NumEltN, RHS: DenEltN);
1484 } else {
1485 // See if this 64-bit division can be shrunk to 32/24-bits before
1486 // producing the general expansion.
1487 NewElt = shrinkDivRem64(Builder, I, Num: NumEltN, Den: DenEltN);
1488 if (!NewElt) {
1489 // The general 64-bit expansion introduces control flow and doesn't
1490 // return the new value. Just insert a scalar copy and defer
1491 // expanding it.
1492 NewElt = Builder.CreateBinOp(Opc, LHS: NumEltN, RHS: DenEltN);
1493 // CreateBinOp does constant folding. If the operands are constant,
1494 // it will return a Constant instead of a BinaryOperator.
1495 if (auto *NewEltBO = dyn_cast<BinaryOperator>(Val: NewElt))
1496 Div64ToExpand.push_back(Elt: NewEltBO);
1497 }
1498 }
1499
1500 if (auto *NewEltI = dyn_cast<Instruction>(Val: NewElt))
1501 NewEltI->copyIRFlags(V: &I);
1502
1503 NewDiv = Builder.CreateInsertElement(Vec: NewDiv, NewElt, Idx: N);
1504 }
1505 } else {
1506 if (ScalarSize <= 32)
1507 NewDiv = expandDivRem32(Builder, I, X: Num, Y: Den);
1508 else {
1509 NewDiv = shrinkDivRem64(Builder, I, Num, Den);
1510 if (!NewDiv)
1511 Div64ToExpand.push_back(Elt: &I);
1512 }
1513 }
1514
1515 if (NewDiv) {
1516 I.replaceAllUsesWith(V: NewDiv);
1517 DeadVals.push_back(Elt: &I);
1518 Changed = true;
1519 }
1520 }
1521
1522 if (ExpandDiv64InIR) {
1523 // TODO: We get much worse code in specially handled constant cases.
1524 for (BinaryOperator *Div : Div64ToExpand) {
1525 expandDivRem64(I&: *Div);
1526 FlowChanged = true;
1527 Changed = true;
1528 }
1529 }
1530
1531 return Changed;
1532}
1533
1534bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) {
1535 if (!WidenLoads)
1536 return false;
1537
1538 if ((I.getPointerAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
1539 I.getPointerAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
1540 canWidenScalarExtLoad(I)) {
1541 IRBuilder<> Builder(&I);
1542 Builder.SetCurrentDebugLocation(I.getDebugLoc());
1543
1544 Type *I32Ty = Builder.getInt32Ty();
1545 LoadInst *WidenLoad = Builder.CreateLoad(Ty: I32Ty, Ptr: I.getPointerOperand());
1546 AMDGPU::copyMetadataForWidenedLoad(Dest&: *WidenLoad, Source: I);
1547
1548 // The widened load reads the original bytes in the low bits, so a !range
1549 // lower bound still holds. Convert it to the new type and don't make
1550 // assumptions about the high bits.
1551 if (auto *Range = I.getMetadata(KindID: LLVMContext::MD_range)) {
1552 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD: Range->getOperand(I: 0));
1553
1554 if (!Lower->isNullValue()) {
1555 Metadata *LowAndHigh[] = {
1556 ConstantAsMetadata::get(C: ConstantInt::get(Ty: I32Ty, V: Lower->getValue().zext(width: 32))),
1557 // Don't make assumptions about the high bits.
1558 ConstantAsMetadata::get(C: ConstantInt::get(Ty: I32Ty, V: 0))
1559 };
1560
1561 WidenLoad->setMetadata(KindID: LLVMContext::MD_range,
1562 Node: MDNode::get(Context&: F.getContext(), MDs: LowAndHigh));
1563 }
1564 }
1565
1566 int TySize = DL.getTypeSizeInBits(Ty: I.getType());
1567 Type *IntNTy = Builder.getIntNTy(N: TySize);
1568 Value *ValTrunc = Builder.CreateTrunc(V: WidenLoad, DestTy: IntNTy);
1569 Value *ValOrig = Builder.CreateBitCast(V: ValTrunc, DestTy: I.getType());
1570 I.replaceAllUsesWith(V: ValOrig);
1571 DeadVals.push_back(Elt: &I);
1572 return true;
1573 }
1574
1575 return false;
1576}
1577
1578bool AMDGPUCodeGenPrepareImpl::visitSelectInst(SelectInst &I) {
1579 FPMathOperator *FPOp = dyn_cast<FPMathOperator>(Val: &I);
1580 if (!FPOp)
1581 return false;
1582
1583 Value *X;
1584 Value *Fract = nullptr;
1585
1586 // Match:
1587 // (x - floor(x)) >= MIN_CONSTANT ? MIN_CONSTANT : (x - floor(x))
1588 //
1589 // This is the preferred way to implement fract.
1590 // TODO: Could also match with compare against 1.0
1591 const APFloat *C;
1592 if (match(V: &I, P: m_UnordFMin(L: m_Value(V&: X), R: m_APFloatAllowPoison(Res&: C)))) {
1593 Value *FractSrc = matchFractPatImpl(V&: *X, C: *C);
1594 if (!FractSrc)
1595 return false;
1596 IRBuilder<> Builder(&I);
1597 Builder.setFastMathFlags(FPOp->getFastMathFlags());
1598 Fract = applyFractPat(Builder, FractArg: FractSrc);
1599 } else {
1600 // Match patterns which may appear in legacy implementations of the fract()
1601 // function, built around the nan-avoidant minnum intrinsic. These are the
1602 // core pattern plus additional clamping of inf and nan values on the
1603 // result.
1604 Value *Cond = I.getCondition();
1605 Value *TrueVal = I.getTrueValue();
1606 Value *FalseVal = I.getFalseValue();
1607 Value *CmpVal;
1608 CmpPredicate IsNanPred;
1609
1610 // Match fract pattern with nan check.
1611 if (!match(V: Cond, P: m_FCmp(Pred&: IsNanPred, L: m_Value(V&: CmpVal), R: m_NonNaN())))
1612 return false;
1613
1614 IRBuilder<> Builder(&I);
1615 Builder.setFastMathFlags(FPOp->getFastMathFlags());
1616
1617 if (IsNanPred == FCmpInst::FCMP_UNO && TrueVal == CmpVal &&
1618 CmpVal == matchFractPatNanAvoidant(V&: *FalseVal)) {
1619 // isnan(x) ? x : fract(x)
1620 Fract = applyFractPat(Builder, FractArg: CmpVal);
1621 } else if (IsNanPred == FCmpInst::FCMP_ORD && FalseVal == CmpVal) {
1622 if (CmpVal == matchFractPatNanAvoidant(V&: *TrueVal)) {
1623 // !isnan(x) ? fract(x) : x
1624 Fract = applyFractPat(Builder, FractArg: CmpVal);
1625 } else {
1626 // Match an intermediate clamp infinity to 0 pattern. i.e.
1627 // !isnan(x) ? (!isinf(x) ? fract(x) : 0.0) : x
1628 CmpPredicate PredInf;
1629 Value *IfNotInf;
1630
1631 if (!match(V: TrueVal, P: m_Select(C: m_FCmp(Pred&: PredInf, L: m_FAbs(Op0: m_Specific(V: CmpVal)),
1632 R: m_PosInf()),
1633 L: m_Value(V&: IfNotInf), R: m_PosZeroFP())) ||
1634 PredInf != FCmpInst::FCMP_UNE ||
1635 CmpVal != matchFractPatNanAvoidant(V&: *IfNotInf))
1636 return false;
1637
1638 SelectInst *ClampInfSelect = cast<SelectInst>(Val: TrueVal);
1639
1640 // Insert before the fabs
1641 Value *InsertPt =
1642 cast<Instruction>(Val: ClampInfSelect->getCondition())->getOperand(i: 0);
1643
1644 Builder.SetInsertPoint(cast<Instruction>(Val: InsertPt));
1645 Value *NewFract = applyFractPat(Builder, FractArg: CmpVal);
1646 NewFract->takeName(V: TrueVal);
1647
1648 // Thread the new fract into the inf clamping sequence.
1649 DeadVals.push_back(Elt: ClampInfSelect->getOperand(i_nocapture: 1));
1650 ClampInfSelect->setOperand(i_nocapture: 1, Val_nocapture: NewFract);
1651
1652 // The outer select nan handling is also absorbed into the fract.
1653 Fract = ClampInfSelect;
1654 }
1655 } else
1656 return false;
1657 }
1658
1659 Fract->takeName(V: &I);
1660 I.replaceAllUsesWith(V: Fract);
1661 DeadVals.push_back(Elt: &I);
1662 return true;
1663}
1664
1665static bool areInSameBB(const Value *A, const Value *B) {
1666 const auto *IA = dyn_cast<Instruction>(Val: A);
1667 const auto *IB = dyn_cast<Instruction>(Val: B);
1668 return IA && IB && IA->getParent() == IB->getParent();
1669}
1670
1671// Helper for breaking large PHIs that returns true when an extractelement on V
1672// is likely to be folded away by the DAG combiner.
1673static bool isInterestingPHIIncomingValue(const Value *V) {
1674 const auto *FVT = dyn_cast<FixedVectorType>(Val: V->getType());
1675 if (!FVT)
1676 return false;
1677
1678 const Value *CurVal = V;
1679
1680 // Check for insertelements, keeping track of the elements covered.
1681 BitVector EltsCovered(FVT->getNumElements());
1682 while (const auto *IE = dyn_cast<InsertElementInst>(Val: CurVal)) {
1683 const auto *Idx = dyn_cast<ConstantInt>(Val: IE->getOperand(i_nocapture: 2));
1684
1685 // Non constant index/out of bounds index -> folding is unlikely.
1686 // The latter is more of a sanity check because canonical IR should just
1687 // have replaced those with poison.
1688 if (!Idx || Idx->getZExtValue() >= FVT->getNumElements())
1689 return false;
1690
1691 const auto *VecSrc = IE->getOperand(i_nocapture: 0);
1692
1693 // If the vector source is another instruction, it must be in the same basic
1694 // block. Otherwise, the DAGCombiner won't see the whole thing and is
1695 // unlikely to be able to do anything interesting here.
1696 if (isa<Instruction>(Val: VecSrc) && !areInSameBB(A: VecSrc, B: IE))
1697 return false;
1698
1699 CurVal = VecSrc;
1700 EltsCovered.set(Idx->getZExtValue());
1701
1702 // All elements covered.
1703 if (EltsCovered.all())
1704 return true;
1705 }
1706
1707 // We either didn't find a single insertelement, or the insertelement chain
1708 // ended before all elements were covered. Check for other interesting values.
1709
1710 // Constants are always interesting because we can just constant fold the
1711 // extractelements.
1712 if (isa<Constant>(Val: CurVal))
1713 return true;
1714
1715 // shufflevector is likely to be profitable if either operand is a constant,
1716 // or if either source is in the same block.
1717 // This is because shufflevector is most often lowered as a series of
1718 // insert/extract elements anyway.
1719 if (const auto *SV = dyn_cast<ShuffleVectorInst>(Val: CurVal)) {
1720 return isa<Constant>(Val: SV->getOperand(i_nocapture: 1)) ||
1721 areInSameBB(A: SV, B: SV->getOperand(i_nocapture: 0)) ||
1722 areInSameBB(A: SV, B: SV->getOperand(i_nocapture: 1));
1723 }
1724
1725 return false;
1726}
1727
1728static void collectPHINodes(const PHINode &I,
1729 SmallPtrSet<const PHINode *, 8> &SeenPHIs) {
1730 const auto [It, Inserted] = SeenPHIs.insert(Ptr: &I);
1731 if (!Inserted)
1732 return;
1733
1734 for (const Value *Inc : I.incoming_values()) {
1735 if (const auto *PhiInc = dyn_cast<PHINode>(Val: Inc))
1736 collectPHINodes(I: *PhiInc, SeenPHIs);
1737 }
1738
1739 for (const User *U : I.users()) {
1740 if (const auto *PhiU = dyn_cast<PHINode>(Val: U))
1741 collectPHINodes(I: *PhiU, SeenPHIs);
1742 }
1743}
1744
1745bool AMDGPUCodeGenPrepareImpl::canBreakPHINode(const PHINode &I) {
1746 // Check in the cache first.
1747 if (const auto It = BreakPhiNodesCache.find(Val: &I);
1748 It != BreakPhiNodesCache.end())
1749 return It->second;
1750
1751 // We consider PHI nodes as part of "chains", so given a PHI node I, we
1752 // recursively consider all its users and incoming values that are also PHI
1753 // nodes. We then make a decision about all of those PHIs at once. Either they
1754 // all get broken up, or none of them do. That way, we avoid cases where a
1755 // single PHI is/is not broken and we end up reforming/exploding a vector
1756 // multiple times, or even worse, doing it in a loop.
1757 SmallPtrSet<const PHINode *, 8> WorkList;
1758 collectPHINodes(I, SeenPHIs&: WorkList);
1759
1760#ifndef NDEBUG
1761 // Check that none of the PHI nodes in the worklist are in the map. If some of
1762 // them are, it means we're not good enough at collecting related PHIs.
1763 for (const PHINode *WLP : WorkList) {
1764 assert(BreakPhiNodesCache.count(WLP) == 0);
1765 }
1766#endif
1767
1768 // To consider a PHI profitable to break, we need to see some interesting
1769 // incoming values. At least 2/3rd (rounded up) of all PHIs in the worklist
1770 // must have one to consider all PHIs breakable.
1771 //
1772 // This threshold has been determined through performance testing.
1773 //
1774 // Note that the computation below is equivalent to
1775 //
1776 // (unsigned)ceil((K / 3.0) * 2)
1777 //
1778 // It's simply written this way to avoid mixing integral/FP arithmetic.
1779 const auto Threshold = (alignTo(Value: WorkList.size() * 2, Align: 3) / 3);
1780 unsigned NumBreakablePHIs = 0;
1781 bool CanBreak = false;
1782 for (const PHINode *Cur : WorkList) {
1783 // Don't break PHIs that have no interesting incoming values. That is, where
1784 // there is no clear opportunity to fold the "extractelement" instructions
1785 // we would add.
1786 //
1787 // Note: IC does not run after this pass, so we're only interested in the
1788 // foldings that the DAG combiner can do.
1789 if (any_of(Range: Cur->incoming_values(), P: isInterestingPHIIncomingValue)) {
1790 if (++NumBreakablePHIs >= Threshold) {
1791 CanBreak = true;
1792 break;
1793 }
1794 }
1795 }
1796
1797 for (const PHINode *Cur : WorkList)
1798 BreakPhiNodesCache[Cur] = CanBreak;
1799
1800 return CanBreak;
1801}
1802
1803/// Helper class for "break large PHIs" (visitPHINode).
1804///
1805/// This represents a slice of a PHI's incoming value, which is made up of:
1806/// - The type of the slice (Ty)
1807/// - The index in the incoming value's vector where the slice starts (Idx)
1808/// - The number of elements in the slice (NumElts).
1809/// It also keeps track of the NewPHI node inserted for this particular slice.
1810///
1811/// Slice examples:
1812/// <4 x i64> -> Split into four i64 slices.
1813/// -> [i64, 0, 1], [i64, 1, 1], [i64, 2, 1], [i64, 3, 1]
1814/// <5 x i16> -> Split into 2 <2 x i16> slices + a i16 tail.
1815/// -> [<2 x i16>, 0, 2], [<2 x i16>, 2, 2], [i16, 4, 1]
1816class VectorSlice {
1817public:
1818 VectorSlice(Type *Ty, unsigned Idx, unsigned NumElts)
1819 : Ty(Ty), Idx(Idx), NumElts(NumElts) {}
1820
1821 Type *Ty = nullptr;
1822 unsigned Idx = 0;
1823 unsigned NumElts = 0;
1824 PHINode *NewPHI = nullptr;
1825
1826 /// Slice \p Inc according to the information contained within this slice.
1827 /// This is cached, so if called multiple times for the same \p BB & \p Inc
1828 /// pair, it returns the same Sliced value as well.
1829 ///
1830 /// Note this *intentionally* does not return the same value for, say,
1831 /// [%bb.0, %0] & [%bb.1, %0] as:
1832 /// - It could cause issues with dominance (e.g. if bb.1 is seen first, then
1833 /// the value in bb.1 may not be reachable from bb.0 if it's its
1834 /// predecessor.)
1835 /// - We also want to make our extract instructions as local as possible so
1836 /// the DAG has better chances of folding them out. Duplicating them like
1837 /// that is beneficial in that regard.
1838 ///
1839 /// This is both a minor optimization to avoid creating duplicate
1840 /// instructions, but also a requirement for correctness. It is not forbidden
1841 /// for a PHI node to have the same [BB, Val] pair multiple times. If we
1842 /// returned a new value each time, those previously identical pairs would all
1843 /// have different incoming values (from the same block) and it'd cause a "PHI
1844 /// node has multiple entries for the same basic block with different incoming
1845 /// values!" verifier error.
1846 Value *getSlicedVal(BasicBlock *BB, Value *Inc, StringRef NewValName) {
1847 Value *&Res = SlicedVals[{BB, Inc}];
1848 if (Res)
1849 return Res;
1850
1851 IRBuilder<> B(BB->getTerminator());
1852 if (Instruction *IncInst = dyn_cast<Instruction>(Val: Inc))
1853 B.SetCurrentDebugLocation(IncInst->getDebugLoc());
1854
1855 if (NumElts > 1) {
1856 SmallVector<int, 4> Mask;
1857 for (unsigned K = Idx; K < (Idx + NumElts); ++K)
1858 Mask.push_back(Elt: K);
1859 Res = B.CreateShuffleVector(V: Inc, Mask, Name: NewValName);
1860 } else
1861 Res = B.CreateExtractElement(Vec: Inc, Idx, Name: NewValName);
1862
1863 return Res;
1864 }
1865
1866private:
1867 SmallDenseMap<std::pair<BasicBlock *, Value *>, Value *> SlicedVals;
1868};
1869
1870bool AMDGPUCodeGenPrepareImpl::visitPHINode(PHINode &I) {
1871 // Break-up fixed-vector PHIs into smaller pieces.
1872 // Default threshold is 32, so it breaks up any vector that's >32 bits into
1873 // its elements, or into 32-bit pieces (for 8/16 bit elts).
1874 //
1875 // This is only helpful for DAGISel because it doesn't handle large PHIs as
1876 // well as GlobalISel. DAGISel lowers PHIs by using CopyToReg/CopyFromReg.
1877 // With large, odd-sized PHIs we may end up needing many `build_vector`
1878 // operations with most elements being "undef". This inhibits a lot of
1879 // optimization opportunities and can result in unreasonably high register
1880 // pressure and the inevitable stack spilling.
1881 if (!BreakLargePHIs || getCGPassBuilderOption().EnableGlobalISelOption ==
1882 cl::boolOrDefault::BOU_TRUE)
1883 return false;
1884
1885 FixedVectorType *FVT = dyn_cast<FixedVectorType>(Val: I.getType());
1886 if (!FVT || FVT->getNumElements() == 1 ||
1887 DL.getTypeSizeInBits(Ty: FVT) <= BreakLargePHIsThreshold)
1888 return false;
1889
1890 if (!ForceBreakLargePHIs && !canBreakPHINode(I))
1891 return false;
1892
1893 std::vector<VectorSlice> Slices;
1894
1895 Type *EltTy = FVT->getElementType();
1896 {
1897 unsigned Idx = 0;
1898 // For 8/16 bits type, don't scalarize fully but break it up into as many
1899 // 32-bit slices as we can, and scalarize the tail.
1900 const unsigned EltSize = DL.getTypeSizeInBits(Ty: EltTy);
1901 const unsigned NumElts = FVT->getNumElements();
1902 if (EltSize == 8 || EltSize == 16) {
1903 const unsigned SubVecSize = (32 / EltSize);
1904 Type *SubVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: SubVecSize);
1905 for (unsigned End = alignDown(Value: NumElts, Align: SubVecSize); Idx < End;
1906 Idx += SubVecSize)
1907 Slices.emplace_back(args&: SubVecTy, args&: Idx, args: SubVecSize);
1908 }
1909
1910 // Scalarize all remaining elements.
1911 for (; Idx < NumElts; ++Idx)
1912 Slices.emplace_back(args&: EltTy, args&: Idx, args: 1);
1913 }
1914
1915 assert(Slices.size() > 1);
1916
1917 // Create one PHI per vector piece. The "VectorSlice" class takes care of
1918 // creating the necessary instruction to extract the relevant slices of each
1919 // incoming value.
1920 IRBuilder<> B(I.getParent());
1921 B.SetCurrentDebugLocation(I.getDebugLoc());
1922
1923 unsigned IncNameSuffix = 0;
1924 for (VectorSlice &S : Slices) {
1925 // We need to reset the build on each iteration, because getSlicedVal may
1926 // have inserted something into I's BB.
1927 B.SetInsertPoint(I.getParent()->getFirstNonPHIIt());
1928 S.NewPHI = B.CreatePHI(Ty: S.Ty, NumReservedValues: I.getNumIncomingValues());
1929
1930 for (const auto &[Idx, BB] : enumerate(First: I.blocks())) {
1931 S.NewPHI->addIncoming(V: S.getSlicedVal(BB, Inc: I.getIncomingValue(i: Idx),
1932 NewValName: "largephi.extractslice" +
1933 std::to_string(val: IncNameSuffix++)),
1934 BB);
1935 }
1936 }
1937
1938 // And replace this PHI with a vector of all the previous PHI values.
1939 Value *Vec = PoisonValue::get(T: FVT);
1940 unsigned NameSuffix = 0;
1941 for (VectorSlice &S : Slices) {
1942 const auto ValName = "largephi.insertslice" + std::to_string(val: NameSuffix++);
1943 if (S.NumElts > 1)
1944 Vec = B.CreateInsertVector(DstType: FVT, SrcVec: Vec, SubVec: S.NewPHI, Idx: S.Idx, Name: ValName);
1945 else
1946 Vec = B.CreateInsertElement(Vec, NewElt: S.NewPHI, Idx: S.Idx, Name: ValName);
1947 }
1948
1949 I.replaceAllUsesWith(V: Vec);
1950 DeadVals.push_back(Elt: &I);
1951 return true;
1952}
1953
1954/// \param V Value to check
1955/// \param DL DataLayout
1956/// \param TM TargetMachine (TODO: remove once DL contains nullptr values)
1957/// \param AS Target Address Space
1958/// \return true if \p V cannot be the null value of \p AS, false otherwise.
1959static bool isPtrKnownNeverNull(const Value *V, const DataLayout &DL,
1960 const AMDGPUTargetMachine &TM, unsigned AS) {
1961 // Pointer cannot be null if it's a block address, GV or alloca.
1962 // NOTE: We don't support extern_weak, but if we did, we'd need to check for
1963 // it as the symbol could be null in such cases.
1964 if (isa<BlockAddress, GlobalValue, AllocaInst>(Val: V))
1965 return true;
1966
1967 // Check nonnull arguments.
1968 if (const auto *Arg = dyn_cast<Argument>(Val: V); Arg && Arg->hasNonNullAttr())
1969 return true;
1970
1971 // Check nonnull loads.
1972 if (const auto *Load = dyn_cast<LoadInst>(Val: V);
1973 Load && Load->hasMetadata(KindID: LLVMContext::MD_nonnull))
1974 return true;
1975
1976 // getUnderlyingObject may have looked through another addrspacecast, although
1977 // the optimizable situations most likely folded out by now.
1978 if (AS != cast<PointerType>(Val: V->getType())->getAddressSpace())
1979 return false;
1980
1981 // TODO: Calls that return nonnull?
1982
1983 // For all other things, use KnownBits.
1984 // We either use 0 or all bits set to indicate null, so check whether the
1985 // value can be zero or all ones.
1986 //
1987 // TODO: Use ValueTracking's isKnownNeverNull if it becomes aware that some
1988 // address spaces have non-zero null values.
1989 auto SrcPtrKB = computeKnownBits(V, DL);
1990 const auto NullVal = AMDGPU::getNullPointerValue(AS);
1991
1992 assert(SrcPtrKB.getBitWidth() == DL.getPointerSizeInBits(AS));
1993 assert((NullVal == 0 || NullVal == -1) &&
1994 "don't know how to check for this null value!");
1995 return NullVal ? !SrcPtrKB.getMaxValue().isAllOnes() : SrcPtrKB.isNonZero();
1996}
1997
1998bool AMDGPUCodeGenPrepareImpl::visitAddrSpaceCastInst(AddrSpaceCastInst &I) {
1999 // Intrinsic doesn't support vectors, also it seems that it's often difficult
2000 // to prove that a vector cannot have any nulls in it so it's unclear if it's
2001 // worth supporting.
2002 if (I.getType()->isVectorTy())
2003 return false;
2004
2005 // Check if this can be lowered to a amdgcn.addrspacecast.nonnull.
2006 // This is only worthwhile for casts from/to priv/local to flat.
2007 const unsigned SrcAS = I.getSrcAddressSpace();
2008 const unsigned DstAS = I.getDestAddressSpace();
2009
2010 bool CanLower = false;
2011 if (SrcAS == AMDGPUAS::FLAT_ADDRESS)
2012 CanLower = (DstAS == AMDGPUAS::LOCAL_ADDRESS ||
2013 DstAS == AMDGPUAS::PRIVATE_ADDRESS);
2014 else if (DstAS == AMDGPUAS::FLAT_ADDRESS)
2015 CanLower = (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
2016 SrcAS == AMDGPUAS::PRIVATE_ADDRESS);
2017 if (!CanLower)
2018 return false;
2019
2020 SmallVector<const Value *, 4> WorkList;
2021 getUnderlyingObjects(V: I.getOperand(i_nocapture: 0), Objects&: WorkList);
2022 if (!all_of(Range&: WorkList, P: [&](const Value *V) {
2023 return isPtrKnownNeverNull(V, DL, TM, AS: SrcAS);
2024 }))
2025 return false;
2026
2027 IRBuilder<> B(&I);
2028 auto *Intrin = B.CreateIntrinsic(
2029 RetTy: I.getType(), ID: Intrinsic::amdgcn_addrspacecast_nonnull, Args: {I.getOperand(i_nocapture: 0)});
2030 I.replaceAllUsesWith(V: Intrin);
2031 DeadVals.push_back(Elt: &I);
2032 return true;
2033}
2034
2035bool AMDGPUCodeGenPrepareImpl::visitIntrinsicInst(IntrinsicInst &I) {
2036 Intrinsic::ID IID = I.getIntrinsicID();
2037 switch (IID) {
2038 case Intrinsic::minnum:
2039 case Intrinsic::minimumnum:
2040 case Intrinsic::minimum:
2041 return visitFMinLike(I);
2042 case Intrinsic::sqrt:
2043 return visitSqrt(I);
2044 case Intrinsic::log:
2045 case Intrinsic::log10:
2046 return visitLog(Log&: cast<FPMathOperator>(Val&: I), IID);
2047 case Intrinsic::log2:
2048 // No reason to handle log2.
2049 return false;
2050 case Intrinsic::amdgcn_mbcnt_lo:
2051 return visitMbcntLo(I);
2052 case Intrinsic::amdgcn_mbcnt_hi:
2053 return visitMbcntHi(I);
2054 case Intrinsic::vector_reduce_add:
2055 return visitVectorReduceAdd(I);
2056 case Intrinsic::uadd_sat:
2057 case Intrinsic::sadd_sat:
2058 return visitSaturatingAdd(I);
2059 default:
2060 return false;
2061 }
2062}
2063
2064/// Match the core sequence in the fract pattern (x - floor(x), which doesn't
2065/// need to consider edge case handling.
2066Value *AMDGPUCodeGenPrepareImpl::matchFractPatImpl(Value &FractSrc,
2067 const APFloat &C) const {
2068 if (ST.hasFractBug())
2069 return nullptr;
2070
2071 Type *Ty = FractSrc.getType();
2072 if (!isLegalFloatingTy(Ty: Ty->getScalarType()))
2073 return nullptr;
2074
2075 APFloat OneNextDown = APFloat::getOne(Sem: C.getSemantics());
2076 OneNextDown.next(nextDown: true);
2077
2078 // Match nextafter(1.0, -1)
2079 if (OneNextDown != C)
2080 return nullptr;
2081
2082 Value *FloorSrc;
2083 if (match(V: &FractSrc, P: m_FSub(L: m_Value(V&: FloorSrc), R: m_Intrinsic<Intrinsic::floor>(
2084 Ops: m_Deferred(V: FloorSrc)))))
2085 return FloorSrc;
2086 return nullptr;
2087}
2088
2089/// Match non-nan fract pattern.
2090// MIN_CONSTANT = nextafter(1.0, -1.0)
2091/// minnum(fsub(x, floor(x)), MIN_CONSTANT)
2092/// minimumnum(fsub(x, floor(x)), MIN_CONSTANT)
2093/// minimum(fsub(x, floor(x)), MIN_CONSTANT)
2094
2095// x_sub_floor >= MIN_CONSTANT ? MIN_CONSTANT : x_sub_floor;
2096///
2097/// If fract is a useful instruction for the subtarget. Does not account for the
2098/// nan handling; the instruction has a nan check on the input value.
2099Value *AMDGPUCodeGenPrepareImpl::matchFractPatNanAvoidant(Value &V) {
2100 Value *Arg0;
2101 const APFloat *C;
2102
2103 // The value is only used in contexts where we know the input isn't a nan, so
2104 // any of the fmin variants are fine.
2105 if (!match(V: &V,
2106 P: m_CombineOr(Ps: m_FMinNum_or_FMinimumNum(Op0: m_Value(V&: Arg0),
2107 Op1: m_APFloatAllowPoison(Res&: C)),
2108 Ps: m_FMinimum(Op0: m_Value(V&: Arg0), Op1: m_APFloatAllowPoison(Res&: C)))))
2109 return nullptr;
2110
2111 return matchFractPatImpl(FractSrc&: *Arg0, C: *C);
2112}
2113
2114Value *AMDGPUCodeGenPrepareImpl::applyFractPat(IRBuilder<> &Builder,
2115 Value *FractArg) {
2116 SmallVector<Value *, 4> FractVals;
2117 extractValues(Builder, Values&: FractVals, V: FractArg);
2118
2119 SmallVector<Value *, 4> ResultVals(FractVals.size());
2120
2121 Type *Ty = FractArg->getType()->getScalarType();
2122 for (unsigned I = 0, E = FractVals.size(); I != E; ++I) {
2123 ResultVals[I] =
2124 Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_fract, OverloadTypes: {Ty}, Args: {FractVals[I]});
2125 }
2126
2127 return insertValues(Builder, Ty: FractArg->getType(), Values&: ResultVals);
2128}
2129
2130bool AMDGPUCodeGenPrepareImpl::visitFMinLike(IntrinsicInst &I) {
2131 const APFloat *C;
2132 Value *FractArg;
2133
2134 // minimum(x - floor(x), MIN_CONSTANT)
2135 Value *X;
2136 if (!ST.hasFractBug() &&
2137 match(V: &I, P: m_FMinimum(Op0: m_Value(V&: X), Op1: m_APFloatAllowPoison(Res&: C)))) {
2138 FractArg = matchFractPatImpl(FractSrc&: *X, C: *C);
2139 if (!FractArg)
2140 return false;
2141 } else {
2142 // minnum(x - floor(x), MIN_CONSTANT)
2143 FractArg = matchFractPatNanAvoidant(V&: I);
2144 if (!FractArg)
2145 return false;
2146
2147 // Match pattern for fract intrinsic in contexts where the nan check has
2148 // been optimized out (and hope the knowledge the source can't be nan wasn't
2149 // lost).
2150 if (!I.hasNoNaNs() && !isKnownNeverNaN(V: FractArg, SQ: SQ.getWithInstruction(I: &I)))
2151 return false;
2152 }
2153
2154 IRBuilder<> Builder(&I);
2155 FastMathFlags FMF = I.getFastMathFlags();
2156 FMF.setNoNaNs();
2157 Builder.setFastMathFlags(FMF);
2158
2159 Value *Fract = applyFractPat(Builder, FractArg);
2160 Fract->takeName(V: &I);
2161 I.replaceAllUsesWith(V: Fract);
2162 DeadVals.push_back(Elt: &I);
2163 return true;
2164}
2165
2166// Expand llvm.sqrt.f32 calls with !fpmath metadata in a semi-fast way.
2167bool AMDGPUCodeGenPrepareImpl::visitSqrt(IntrinsicInst &Sqrt) {
2168 Type *Ty = Sqrt.getType()->getScalarType();
2169 if (!Ty->isFloatTy() && (!Ty->isHalfTy() || ST.has16BitInsts()))
2170 return false;
2171
2172 const FPMathOperator *FPOp = cast<const FPMathOperator>(Val: &Sqrt);
2173 FastMathFlags SqrtFMF = FPOp->getFastMathFlags();
2174
2175 // We're trying to handle the fast-but-not-that-fast case only. The lowering
2176 // of fast llvm.sqrt will give the raw instruction anyway.
2177 if (SqrtFMF.approxFunc())
2178 return false;
2179
2180 const float ReqdAccuracy = FPOp->getFPAccuracy();
2181
2182 // Defer correctly rounded expansion to codegen.
2183 if (ReqdAccuracy < 1.0f)
2184 return false;
2185
2186 Value *SrcVal = Sqrt.getOperand(i_nocapture: 0);
2187 bool CanTreatAsDAZ = canIgnoreDenormalInput(V: SrcVal, CtxI: &Sqrt);
2188
2189 // The raw instruction is 1 ulp, but the correction for denormal handling
2190 // brings it to 2.
2191 if (!CanTreatAsDAZ && ReqdAccuracy < 2.0f)
2192 return false;
2193
2194 IRBuilder<> Builder(&Sqrt);
2195 SmallVector<Value *, 4> SrcVals;
2196 extractValues(Builder, Values&: SrcVals, V: SrcVal);
2197
2198 SmallVector<Value *, 4> ResultVals(SrcVals.size());
2199 for (int I = 0, E = SrcVals.size(); I != E; ++I) {
2200 if (CanTreatAsDAZ)
2201 ResultVals[I] = Builder.CreateCall(Callee: getSqrtF32(), Args: SrcVals[I]);
2202 else
2203 ResultVals[I] = emitSqrtIEEE2ULP(Builder, Src: SrcVals[I], FMF: SqrtFMF);
2204 }
2205
2206 Value *NewSqrt = insertValues(Builder, Ty: Sqrt.getType(), Values&: ResultVals);
2207 NewSqrt->takeName(V: &Sqrt);
2208 Sqrt.replaceAllUsesWith(V: NewSqrt);
2209 DeadVals.push_back(Elt: &Sqrt);
2210 return true;
2211}
2212
2213/// Replace log and log10 intrinsic calls based on fpmath metadata.
2214bool AMDGPUCodeGenPrepareImpl::visitLog(FPMathOperator &Log,
2215 Intrinsic::ID IID) {
2216 Type *Ty = Log.getType();
2217 if (!Ty->getScalarType()->isHalfTy() || !ST.has16BitInsts())
2218 return false;
2219
2220 FastMathFlags FMF = Log.getFastMathFlags();
2221
2222 // Defer fast math cases to codegen.
2223 if (FMF.approxFunc())
2224 return false;
2225
2226 // Limit experimentally determined from OpenCL conformance test (1.79)
2227 if (Log.getFPAccuracy() < 1.80f)
2228 return false;
2229
2230 IRBuilder<> Builder(&cast<CallInst>(Val&: Log));
2231
2232 // Use the generic intrinsic for convenience in the vector case. Codegen will
2233 // recognize the denormal handling is not necessary from the fpext.
2234 // TODO: Move to generic code
2235 Value *Log2 =
2236 Builder.CreateUnaryIntrinsic(ID: Intrinsic::log2, Op: Log.getOperand(i: 0), FMFSource: FMF);
2237
2238 double Log2BaseInverted =
2239 IID == Intrinsic::log10 ? numbers::ln2 / numbers::ln10 : numbers::ln2;
2240 Value *Mul =
2241 Builder.CreateFMulFMF(L: Log2, R: ConstantFP::get(Ty, V: Log2BaseInverted), FMFSource: FMF);
2242
2243 Mul->takeName(V: &Log);
2244
2245 Log.replaceAllUsesWith(V: Mul);
2246 DeadVals.push_back(Elt: &Log);
2247 return true;
2248}
2249
2250bool AMDGPUCodeGenPrepare::runOnFunction(Function &F) {
2251 if (skipFunction(F))
2252 return false;
2253
2254 auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
2255 if (!TPC)
2256 return false;
2257
2258 const AMDGPUTargetMachine &TM = TPC->getTM<AMDGPUTargetMachine>();
2259 const TargetTransformInfo &TTI =
2260 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
2261 const TargetLibraryInfo *TLI =
2262 &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
2263 AssumptionCache *AC =
2264 &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
2265 auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
2266 const DominatorTree *DT = DTWP ? &DTWP->getDomTree() : nullptr;
2267 const UniformityInfo &UA =
2268 getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
2269 return AMDGPUCodeGenPrepareImpl(F, TM, TTI, TLI, AC, DT, UA).run();
2270}
2271
2272PreservedAnalyses AMDGPUCodeGenPreparePass::run(Function &F,
2273 FunctionAnalysisManager &FAM) {
2274 const AMDGPUTargetMachine &ATM = static_cast<const AMDGPUTargetMachine &>(TM);
2275 const TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(IR&: F);
2276 const TargetLibraryInfo *TLI = &FAM.getResult<TargetLibraryAnalysis>(IR&: F);
2277 AssumptionCache *AC = &FAM.getResult<AssumptionAnalysis>(IR&: F);
2278 const DominatorTree *DT = FAM.getCachedResult<DominatorTreeAnalysis>(IR&: F);
2279 const UniformityInfo &UA = FAM.getResult<UniformityInfoAnalysis>(IR&: F);
2280 AMDGPUCodeGenPrepareImpl Impl(F, ATM, TTI, TLI, AC, DT, UA);
2281 if (!Impl.run())
2282 return PreservedAnalyses::all();
2283 PreservedAnalyses PA = PreservedAnalyses::none();
2284 if (!Impl.FlowChanged)
2285 PA.preserveSet<CFGAnalyses>();
2286 return PA;
2287}
2288
2289INITIALIZE_PASS_BEGIN(AMDGPUCodeGenPrepare, DEBUG_TYPE,
2290 "AMDGPU IR optimizations", false, false)
2291INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
2292INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
2293INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
2294INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
2295INITIALIZE_PASS_END(AMDGPUCodeGenPrepare, DEBUG_TYPE, "AMDGPU IR optimizations",
2296 false, false)
2297
2298/// Create a workitem.id.x intrinsic call with range metadata.
2299CallInst *AMDGPUCodeGenPrepareImpl::createWorkitemIdX(IRBuilder<> &B) const {
2300 CallInst *Tid =
2301 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::amdgcn_workitem_id_x, Args: {});
2302 ST.makeLIDRangeMetadata(I: Tid);
2303 return Tid;
2304}
2305
2306/// Replace the instruction with a direct workitem.id.x call.
2307void AMDGPUCodeGenPrepareImpl::replaceWithWorkitemIdX(Instruction &I) const {
2308 IRBuilder<> B(&I);
2309 CallInst *Tid = createWorkitemIdX(B);
2310 BasicBlock::iterator BI(&I);
2311 ReplaceInstWithValue(BI, V: Tid);
2312}
2313
2314/// Replace the instruction with (workitem.id.x & mask).
2315void AMDGPUCodeGenPrepareImpl::replaceWithMaskedWorkitemIdX(
2316 Instruction &I, unsigned WaveSize) const {
2317 IRBuilder<> B(&I);
2318 CallInst *Tid = createWorkitemIdX(B);
2319 Constant *Mask = ConstantInt::get(Ty: Tid->getType(), V: WaveSize - 1);
2320 Value *AndInst = B.CreateAnd(LHS: Tid, RHS: Mask);
2321 BasicBlock::iterator BI(&I);
2322 ReplaceInstWithValue(BI, V: AndInst);
2323}
2324
2325/// Try to optimize mbcnt instruction by replacing with workitem.id.x when
2326/// work group size allows direct computation of lane ID.
2327/// Returns true if optimization was applied, false otherwise.
2328bool AMDGPUCodeGenPrepareImpl::tryReplaceWithWorkitemId(Instruction &I,
2329 unsigned Wave) const {
2330 std::optional<unsigned> MaybeX = ST.getReqdWorkGroupSize(F, Dim: 0);
2331 if (!MaybeX)
2332 return false;
2333
2334 // When work group size == wave_size, each work group contains exactly one
2335 // wave, so the instruction can be replaced with workitem.id.x directly.
2336 if (*MaybeX == Wave) {
2337 replaceWithWorkitemIdX(I);
2338 return true;
2339 }
2340
2341 // When work group evenly splits into waves, compute lane ID within wave
2342 // using bit masking: lane_id = workitem.id.x & (wave_size - 1).
2343 if (ST.hasWavefrontsEvenlySplittingXDim(F, /*RequiresUniformYZ=*/REquiresUniformYZ: true)) {
2344 replaceWithMaskedWorkitemIdX(I, WaveSize: Wave);
2345 return true;
2346 }
2347
2348 return false;
2349}
2350
2351/// Optimize mbcnt.lo calls on wave32 architectures for lane ID computation.
2352bool AMDGPUCodeGenPrepareImpl::visitMbcntLo(IntrinsicInst &I) const {
2353 // This optimization only applies to wave32 targets where mbcnt.lo operates on
2354 // the full execution mask.
2355 if (!ST.isWave32())
2356 return false;
2357
2358 // Only optimize the pattern mbcnt.lo(~0, 0) which counts active lanes with
2359 // lower IDs.
2360 if (!match(V: &I,
2361 P: m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(Ops: m_AllOnes(), Ops: m_Zero())))
2362 return false;
2363
2364 return tryReplaceWithWorkitemId(I, Wave: ST.getWavefrontSize());
2365}
2366
2367/// Optimize mbcnt.hi calls for lane ID computation.
2368bool AMDGPUCodeGenPrepareImpl::visitMbcntHi(IntrinsicInst &I) const {
2369 // Abort if wave size is not known at compile time.
2370 if (!ST.isWaveSizeKnown())
2371 return false;
2372
2373 unsigned Wave = ST.getWavefrontSize();
2374
2375 // On wave32, the upper 32 bits of execution mask are always 0, so
2376 // mbcnt.hi(mask, val) always returns val unchanged.
2377 if (ST.isWave32()) {
2378 if (auto MaybeX = ST.getReqdWorkGroupSize(F, Dim: 0)) {
2379 // Replace mbcnt.hi(mask, val) with val only when work group size matches
2380 // wave size (single wave per work group).
2381 if (*MaybeX == Wave) {
2382 BasicBlock::iterator BI(&I);
2383 ReplaceInstWithValue(BI, V: I.getArgOperand(i: 1));
2384 return true;
2385 }
2386 }
2387 }
2388
2389 // Optimize the complete lane ID computation pattern:
2390 // mbcnt.hi(~0, mbcnt.lo(~0, 0)) which counts all active lanes with lower IDs
2391 // across the full execution mask.
2392 using namespace PatternMatch;
2393
2394 // Check for pattern: mbcnt.hi(~0, mbcnt.lo(~0, 0))
2395 if (!match(V: &I, P: m_Intrinsic<Intrinsic::amdgcn_mbcnt_hi>(
2396 Ops: m_AllOnes(), Ops: m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(
2397 Ops: m_AllOnes(), Ops: m_Zero()))))
2398 return false;
2399
2400 return tryReplaceWithWorkitemId(I, Wave);
2401}
2402
2403/// Check if type is <4 x i8>.
2404static bool isV4I8(Type *Ty) {
2405 FixedVectorType *VTy = dyn_cast<FixedVectorType>(Val: Ty);
2406 return VTy && VTy->getNumElements() == 4 &&
2407 VTy->getElementType()->isIntegerTy(BitWidth: 8);
2408}
2409
2410/// Helper to match the dot4 pattern: mul(zext/sext <4 x i8>, zext/sext <4 x
2411/// i8>) Returns true if pattern matches and signedness matches IsSigned.
2412/// Sets A, B to the <4 x i8> sources.
2413static bool matchDot4Pattern(Value *MulOp, Value *&A, Value *&B,
2414 bool IsSigned) {
2415 Value *Src0, *Src1;
2416 if (!match(V: MulOp, P: m_Mul(L: m_Value(V&: Src0), R: m_Value(V&: Src1))))
2417 return false;
2418
2419 // Check that result type is <4 x i32>
2420 FixedVectorType *MulTy = dyn_cast<FixedVectorType>(Val: MulOp->getType());
2421 if (!MulTy || MulTy->getNumElements() != 4 ||
2422 !MulTy->getElementType()->isIntegerTy(BitWidth: 32))
2423 return false;
2424
2425 // Match zext or sext based on IsSigned
2426 Value *ExtSrc0, *ExtSrc1;
2427 if (IsSigned) {
2428 if (!match(V: Src0, P: m_SExt(Op: m_Value(V&: ExtSrc0))) || !isV4I8(Ty: ExtSrc0->getType()))
2429 return false;
2430 if (!match(V: Src1, P: m_SExt(Op: m_Value(V&: ExtSrc1))) || !isV4I8(Ty: ExtSrc1->getType()))
2431 return false;
2432 } else {
2433 if (!match(V: Src0, P: m_ZExt(Op: m_Value(V&: ExtSrc0))) || !isV4I8(Ty: ExtSrc0->getType()))
2434 return false;
2435 if (!match(V: Src1, P: m_ZExt(Op: m_Value(V&: ExtSrc1))) || !isV4I8(Ty: ExtSrc1->getType()))
2436 return false;
2437 }
2438
2439 A = ExtSrc0;
2440 B = ExtSrc1;
2441 return true;
2442}
2443
2444/// Try to convert vector.reduce.add(mul(zext/sext <4 x i8>, zext/sext <4 x
2445/// i8>)) to a dot4 intrinsic call (non-saturating case only).
2446bool AMDGPUCodeGenPrepareImpl::visitVectorReduceAdd(IntrinsicInst &I) {
2447 // Check if we have dot4 instructions available
2448 if (!ST.hasDot7Insts() || (!ST.hasDot1Insts() && !ST.hasDot8Insts()))
2449 return false;
2450
2451 Value *A = nullptr, *B = nullptr;
2452
2453 // Try unsigned first, then signed
2454 bool IsSigned = false;
2455 if (!matchDot4Pattern(MulOp: I.getArgOperand(i: 0), A, B, /*IsSigned=*/false)) {
2456 if (!matchDot4Pattern(MulOp: I.getArgOperand(i: 0), A, B, /*IsSigned=*/true))
2457 return false;
2458 IsSigned = true;
2459 }
2460
2461 LLVMContext &Ctx = I.getContext();
2462 Type *I32Ty = Type::getInt32Ty(C&: Ctx);
2463 IRBuilder<> Builder(&I);
2464
2465 // Bitcast <4 x i8> to i32
2466 Value *ASrc = Builder.CreateBitCast(V: A, DestTy: I32Ty);
2467 Value *BSrc = Builder.CreateBitCast(V: B, DestTy: I32Ty);
2468
2469 // Non-saturating case: accumulator is 0, clamp is false
2470 Value *Acc = ConstantInt::get(Ty: I32Ty, V: 0);
2471 Value *Clamp = ConstantInt::getFalse(Context&: Ctx);
2472
2473 Intrinsic::ID DotIID =
2474 IsSigned ? Intrinsic::amdgcn_sdot4 : Intrinsic::amdgcn_udot4;
2475
2476 Value *Dot = Builder.CreateIntrinsic(ID: DotIID, OverloadTypes: {}, Args: {ASrc, BSrc, Acc, Clamp});
2477 Dot->takeName(V: &I);
2478
2479 I.replaceAllUsesWith(V: Dot);
2480 DeadVals.push_back(Elt: &I);
2481
2482 return true;
2483}
2484
2485/// Try to convert uadd.sat/sadd.sat(vector.reduce.add(mul(...)), c) to a
2486/// saturating dot4 intrinsic. This combine starts at the root (saturating add)
2487/// and looks at its operands.
2488bool AMDGPUCodeGenPrepareImpl::visitSaturatingAdd(IntrinsicInst &I) {
2489 // Check if we have dot4 instructions available
2490 if (!ST.hasDot7Insts() || (!ST.hasDot1Insts() && !ST.hasDot8Insts()))
2491 return false;
2492
2493 Intrinsic::ID IID = I.getIntrinsicID();
2494 bool IsSigned = (IID == Intrinsic::sadd_sat);
2495
2496 // Look for vector.reduce.add as one of the operands (commutative match)
2497 Value *Op0 = I.getArgOperand(i: 0);
2498 Value *Op1 = I.getArgOperand(i: 1);
2499 Value *MulOp = nullptr;
2500 Value *Accum = nullptr;
2501 IntrinsicInst *ReduceInst = nullptr;
2502
2503 if (match(V: Op0, P: m_Intrinsic<Intrinsic::vector_reduce_add>(Ops: m_Value(V&: MulOp)))) {
2504 ReduceInst = cast<IntrinsicInst>(Val: Op0);
2505 Accum = Op1;
2506 } else if (match(V: Op1,
2507 P: m_Intrinsic<Intrinsic::vector_reduce_add>(Ops: m_Value(V&: MulOp)))) {
2508 ReduceInst = cast<IntrinsicInst>(Val: Op1);
2509 Accum = Op0;
2510 } else {
2511 return false;
2512 }
2513
2514 Value *A = nullptr, *B = nullptr;
2515
2516 if (!matchDot4Pattern(MulOp, A, B, IsSigned))
2517 return false;
2518
2519 LLVMContext &Ctx = I.getContext();
2520 Type *I32Ty = Type::getInt32Ty(C&: Ctx);
2521 IRBuilder<> Builder(&I);
2522
2523 // Bitcast <4 x i8> to i32
2524 Value *ASrc = Builder.CreateBitCast(V: A, DestTy: I32Ty);
2525 Value *BSrc = Builder.CreateBitCast(V: B, DestTy: I32Ty);
2526
2527 // Saturating case: use the accumulator and set clamp to true
2528 Value *Clamp = ConstantInt::getTrue(Context&: Ctx);
2529
2530 Intrinsic::ID DotIID =
2531 IsSigned ? Intrinsic::amdgcn_sdot4 : Intrinsic::amdgcn_udot4;
2532
2533 Value *Dot = Builder.CreateIntrinsic(ID: DotIID, OverloadTypes: {}, Args: {ASrc, BSrc, Accum, Clamp});
2534 Dot->takeName(V: &I);
2535
2536 I.replaceAllUsesWith(V: Dot);
2537 DeadVals.push_back(Elt: &I);
2538 // The reduce.add will be dead after this and cleaned up later
2539 if (ReduceInst->use_empty())
2540 DeadVals.push_back(Elt: ReduceInst);
2541
2542 return true;
2543}
2544
2545char AMDGPUCodeGenPrepare::ID = 0;
2546
2547FunctionPass *llvm::createAMDGPUCodeGenPreparePass() {
2548 return new AMDGPUCodeGenPrepare();
2549}
2550