1//===-- AMDGPUCodeGenPrepare.cpp ------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass does misc. AMDGPU optimizations on IR before instruction
11/// selection.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPU.h"
16#include "AMDGPUMemoryUtils.h"
17#include "AMDGPUTargetMachine.h"
18#include "SIModeRegisterDefaults.h"
19#include "llvm/Analysis/AssumptionCache.h"
20#include "llvm/Analysis/ConstantFolding.h"
21#include "llvm/Analysis/TargetLibraryInfo.h"
22#include "llvm/Analysis/TargetTransformInfo.h"
23#include "llvm/Analysis/UniformityAnalysis.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/CodeGen/TargetPassConfig.h"
26#include "llvm/IR/Dominators.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/InstVisitor.h"
29#include "llvm/IR/IntrinsicsAMDGPU.h"
30#include "llvm/IR/PatternMatch.h"
31#include "llvm/InitializePasses.h"
32#include "llvm/Pass.h"
33#include "llvm/Support/KnownBits.h"
34#include "llvm/Support/KnownFPClass.h"
35#include "llvm/Transforms/Utils/BasicBlockUtils.h"
36#include "llvm/Transforms/Utils/IntegerDivision.h"
37#include "llvm/Transforms/Utils/Local.h"
38
39#define DEBUG_TYPE "amdgpu-codegenprepare"
40
41using namespace llvm;
42using namespace llvm::PatternMatch;
43
44namespace {
45
46class AMDGPUCodeGenPrepareImpl
47 : public InstVisitor<AMDGPUCodeGenPrepareImpl, bool> {
48public:
49 const AMDGPUOptions &CLOpts;
50 Function &F;
51 const GCNSubtarget &ST;
52 const AMDGPUTargetMachine &TM;
53 const TargetTransformInfo &TTI;
54 const TargetLibraryInfo *TLI;
55 const UniformityInfo &UA;
56 const DataLayout &DL;
57 SimplifyQuery SQ;
58 const bool HasFP32DenormalFlush;
59 const bool ExpandDiv64InIR;
60 bool FlowChanged = false;
61 mutable Function *SqrtF32 = nullptr;
62 mutable Function *LdexpF32 = nullptr;
63 mutable SmallVector<WeakVH> DeadVals;
64
65 DenseMap<const PHINode *, bool> BreakPhiNodesCache;
66
67 AMDGPUCodeGenPrepareImpl(Function &F, const AMDGPUTargetMachine &TM,
68 const TargetTransformInfo &TTI,
69 const TargetLibraryInfo *TLI, AssumptionCache *AC,
70 const DominatorTree *DT, const UniformityInfo &UA,
71 bool ExpandDiv64InIR)
72 : CLOpts(TM.getCLOpts()), F(F), ST(TM.getSubtarget<GCNSubtarget>(F)),
73 TM(TM), TTI(TTI), TLI(TLI), UA(UA), DL(F.getDataLayout()),
74 SQ(DL, TLI, DT, AC),
75 HasFP32DenormalFlush(SIModeRegisterDefaults(F, ST).FP32Denormals ==
76 DenormalMode::getPreserveSign()),
77 ExpandDiv64InIR(ExpandDiv64InIR) {}
78
79 Function *getSqrtF32() const {
80 if (SqrtF32)
81 return SqrtF32;
82
83 LLVMContext &Ctx = F.getContext();
84 SqrtF32 = Intrinsic::getOrInsertDeclaration(
85 M: F.getParent(), id: Intrinsic::amdgcn_sqrt, OverloadTys: {Type::getFloatTy(C&: Ctx)});
86 return SqrtF32;
87 }
88
89 Function *getLdexpF32() const {
90 if (LdexpF32)
91 return LdexpF32;
92
93 LLVMContext &Ctx = F.getContext();
94 LdexpF32 = Intrinsic::getOrInsertDeclaration(
95 M: F.getParent(), id: Intrinsic::ldexp,
96 OverloadTys: {Type::getFloatTy(C&: Ctx), Type::getInt32Ty(C&: Ctx)});
97 return LdexpF32;
98 }
99
100 bool canBreakPHINode(const PHINode &I);
101
102 /// Return true if \p T is a legal scalar floating point type.
103 bool isLegalFloatingTy(const Type *T) const;
104
105 /// Wrapper to pass all the arguments to computeKnownFPClass
106 KnownFPClass computeKnownFPClass(const Value *V, FPClassTest Interested,
107 const Instruction *CtxI) const {
108 return llvm::computeKnownFPClass(V, InterestedClasses: Interested,
109 SQ: SQ.getWithInstruction(I: CtxI));
110 }
111
112 bool canIgnoreDenormalInput(const Value *V, const Instruction *CtxI) const {
113 return HasFP32DenormalFlush ||
114 computeKnownFPClass(V, Interested: fcSubnormal, CtxI).isKnownNeverSubnormal();
115 }
116
117 /// \returns The minimum number of bits needed to store the value of \Op as an
118 /// unsigned integer. Truncating to this size and then zero-extending to
119 /// the original will not change the value.
120 unsigned numBitsUnsigned(Value *Op, const Instruction *CtxI) const;
121
122 /// \returns The minimum number of bits needed to store the value of \Op as a
123 /// signed integer. Truncating to this size and then sign-extending to
124 /// the original size will not change the value.
125 unsigned numBitsSigned(Value *Op, const Instruction *CtxI) const;
126
127 /// Replace mul instructions with llvm.amdgcn.mul.u24 or llvm.amdgcn.mul.s24.
128 /// SelectionDAG has an issue where an and asserting the bits are known
129 bool replaceMulWithMul24(BinaryOperator &I) const;
130
131 /// Perform same function as equivalently named function in DAGCombiner. Since
132 /// we expand some divisions here, we need to perform this before obscuring.
133 bool foldBinOpIntoSelect(BinaryOperator &I) const;
134
135 bool divHasSpecialOptimization(BinaryOperator &I,
136 Value *Num, Value *Den) const;
137 unsigned getDivNumBits(BinaryOperator &I, Value *Num, Value *Den,
138 unsigned MaxDivBits, bool Signed) const;
139
140 /// Expands div or rem by using floating-point operations.
141 /// Operands must be in the range [-0x400000,0x3FFFFF]
142 Value *expandDivRemToFloat(IRBuilder<> &Builder, BinaryOperator &I,
143 Value *Num, Value *Den, bool IsDiv,
144 bool IsSigned) const;
145
146 Value *expandDivRemToFloatImpl(IRBuilder<> &Builder, BinaryOperator &I,
147 Value *Num, Value *Den, unsigned NumBits,
148 bool IsDiv, bool IsSigned) const;
149
150 /// Expands 32 bit div or rem.
151 Value* expandDivRem32(IRBuilder<> &Builder, BinaryOperator &I,
152 Value *Num, Value *Den) const;
153
154 Value *shrinkDivRem64(IRBuilder<> &Builder, BinaryOperator &I,
155 Value *Num, Value *Den) const;
156 void expandDivRem64(BinaryOperator &I) const;
157
158 /// Widen a scalar load.
159 ///
160 /// \details \p Widen scalar load for uniform, small type loads from constant
161 // memory / to a full 32-bits and then truncate the input to allow a scalar
162 // load instead of a vector load.
163 //
164 /// \returns True.
165
166 bool canWidenScalarExtLoad(LoadInst &I) const;
167
168 Value *matchFractPatImpl(Value &V, const APFloat &C) const;
169 Value *matchFractPatNanAvoidant(Value &V);
170 Value *applyFractPat(IRBuilder<> &Builder, Value *FractArg);
171
172 bool canOptimizeWithRsq(FastMathFlags DivFMF, FastMathFlags SqrtFMF) const;
173
174 Value *optimizeWithRsq(IRBuilder<> &Builder, Value *Num, Value *Den,
175 FastMathFlags DivFMF, FastMathFlags SqrtFMF,
176 const Instruction *CtxI) const;
177
178 Value *optimizeWithRcp(IRBuilder<> &Builder, Value *Num, Value *Den,
179 FastMathFlags FMF, const Instruction *CtxI) const;
180 Value *optimizeWithFDivFast(IRBuilder<> &Builder, Value *Num, Value *Den,
181 float ReqdAccuracy) const;
182
183 Value *visitFDivElement(IRBuilder<> &Builder, Value *Num, Value *Den,
184 FastMathFlags DivFMF, FastMathFlags SqrtFMF,
185 Value *RsqOp, const Instruction *FDiv,
186 float ReqdAccuracy) const;
187
188 std::pair<Value *, Value *> getFrexpResults(IRBuilder<> &Builder,
189 Value *Src) const;
190
191 Value *emitRcpIEEE1ULP(IRBuilder<> &Builder, Value *Src,
192 bool IsNegative) const;
193 Value *emitFrexpDiv(IRBuilder<> &Builder, Value *LHS, Value *RHS,
194 FastMathFlags FMF) const;
195 Value *emitSqrtIEEE2ULP(IRBuilder<> &Builder, Value *Src,
196 FastMathFlags FMF) const;
197 Value *emitRsqF64(IRBuilder<> &Builder, Value *X, FastMathFlags SqrtFMF,
198 FastMathFlags DivFMF, const Instruction *CtxI,
199 bool IsNegative) const;
200
201 CallInst *createWorkitemIdX(IRBuilder<> &B) const;
202 void replaceWithWorkitemIdX(Instruction &I) const;
203 void replaceWithMaskedWorkitemIdX(Instruction &I, unsigned WaveSize) const;
204 bool tryReplaceWithWorkitemId(Instruction &I, unsigned Wave) const;
205
206 bool tryNarrowMathIfNoOverflow(Instruction *I);
207
208public:
209 bool visitFDiv(BinaryOperator &I);
210
211 bool visitInstruction(Instruction &I) { return false; }
212 bool visitBinaryOperator(BinaryOperator &I);
213 bool visitLoadInst(LoadInst &I);
214 bool visitSelectInst(SelectInst &I);
215 bool visitPHINode(PHINode &I);
216 bool visitAddrSpaceCastInst(AddrSpaceCastInst &I);
217
218 bool visitIntrinsicInst(IntrinsicInst &I);
219 bool visitFMinLike(IntrinsicInst &I);
220 bool visitSqrt(IntrinsicInst &I);
221 bool visitLog(FPMathOperator &Log, Intrinsic::ID IID);
222 bool visitMbcntLo(IntrinsicInst &I) const;
223 bool visitMbcntHi(IntrinsicInst &I) const;
224 bool visitVectorReduceAdd(IntrinsicInst &I);
225 bool visitSaturatingAdd(IntrinsicInst &I);
226 bool run();
227};
228
229class AMDGPUCodeGenPrepare : public FunctionPass {
230public:
231 static char ID;
232 explicit AMDGPUCodeGenPrepare(bool ExpandDiv64InIR = false)
233 : FunctionPass(ID), ExpandDiv64InIR(ExpandDiv64InIR) {}
234 void getAnalysisUsage(AnalysisUsage &AU) const override {
235 AU.addRequired<AssumptionCacheTracker>();
236 AU.addRequired<UniformityInfoWrapperPass>();
237 AU.addRequired<TargetLibraryInfoWrapperPass>();
238 AU.addRequired<TargetTransformInfoWrapperPass>();
239
240 // FIXME: Division expansion needs to preserve the dominator tree.
241 if (!ExpandDiv64InIR)
242 AU.setPreservesAll();
243 }
244 bool runOnFunction(Function &F) override;
245 StringRef getPassName() const override { return "AMDGPU IR optimizations"; }
246
247private:
248 bool ExpandDiv64InIR;
249};
250
251} // end anonymous namespace
252
253bool AMDGPUCodeGenPrepareImpl::run() {
254 BreakPhiNodesCache.clear();
255 bool MadeChange = false;
256
257 // Need to use make_early_inc_range because integer division expansion is
258 // handled by Transform/Utils, and it can delete instructions such as the
259 // terminator of the BB.
260 for (BasicBlock &BB : reverse(C&: F)) {
261 for (Instruction &I : make_early_inc_range(Range: reverse(C&: BB))) {
262 if (!isInstructionTriviallyDead(I: &I, TLI))
263 MadeChange |= visit(I);
264 }
265 }
266
267 while (!DeadVals.empty()) {
268 if (auto *I = dyn_cast_or_null<Instruction>(Val: DeadVals.pop_back_val()))
269 RecursivelyDeleteTriviallyDeadInstructions(V: I, TLI);
270 }
271
272 return MadeChange;
273}
274
275bool AMDGPUCodeGenPrepareImpl::isLegalFloatingTy(const Type *Ty) const {
276 return Ty->isFloatTy() || Ty->isDoubleTy() ||
277 (Ty->isHalfTy() && ST.has16BitInsts());
278}
279
280bool AMDGPUCodeGenPrepareImpl::canWidenScalarExtLoad(LoadInst &I) const {
281 Type *Ty = I.getType();
282 int TySize = DL.getTypeSizeInBits(Ty);
283 Align Alignment = DL.getValueOrABITypeAlignment(Alignment: I.getAlign(), Ty);
284
285 return I.isSimple() && TySize < 32 && Alignment >= 4 && UA.isUniformAtDef(V: &I);
286}
287
288unsigned
289AMDGPUCodeGenPrepareImpl::numBitsUnsigned(Value *Op,
290 const Instruction *CtxI) const {
291 return computeKnownBits(V: Op, Q: SQ.getWithInstruction(I: CtxI)).countMaxActiveBits();
292}
293
294unsigned
295AMDGPUCodeGenPrepareImpl::numBitsSigned(Value *Op,
296 const Instruction *CtxI) const {
297 return ComputeMaxSignificantBits(Op, DL: SQ.DL, AC: SQ.AC, CtxI, DT: SQ.DT);
298}
299
300static void extractValues(IRBuilder<> &Builder,
301 SmallVectorImpl<Value *> &Values, Value *V) {
302 auto *VT = dyn_cast<FixedVectorType>(Val: V->getType());
303 if (!VT) {
304 Values.push_back(Elt: V);
305 return;
306 }
307
308 for (int I = 0, E = VT->getNumElements(); I != E; ++I)
309 Values.push_back(Elt: Builder.CreateExtractElement(Vec: V, Idx: I));
310}
311
312static Value *insertValues(IRBuilder<> &Builder,
313 Type *Ty,
314 SmallVectorImpl<Value *> &Values) {
315 if (!Ty->isVectorTy()) {
316 assert(Values.size() == 1);
317 return Values[0];
318 }
319
320 Value *NewVal = PoisonValue::get(T: Ty);
321 for (int I = 0, E = Values.size(); I != E; ++I)
322 NewVal = Builder.CreateInsertElement(Vec: NewVal, NewElt: Values[I], Idx: I);
323
324 return NewVal;
325}
326
327bool AMDGPUCodeGenPrepareImpl::replaceMulWithMul24(BinaryOperator &I) const {
328 if (I.getOpcode() != Instruction::Mul)
329 return false;
330
331 Type *Ty = I.getType();
332 unsigned Size = Ty->getScalarSizeInBits();
333 if (Size <= 16 && ST.has16BitInsts())
334 return false;
335
336 // Prefer scalar if this could be s_mul_i32
337 if (UA.isUniformAtDef(V: &I))
338 return false;
339
340 Value *LHS = I.getOperand(i_nocapture: 0);
341 Value *RHS = I.getOperand(i_nocapture: 1);
342 IRBuilder<> Builder(&I);
343 Builder.SetCurrentDebugLocation(I.getDebugLoc());
344
345 unsigned LHSBits = 0, RHSBits = 0;
346 bool IsSigned = false;
347
348 if (ST.hasMulU24() && (LHSBits = numBitsUnsigned(Op: LHS, CtxI: &I)) <= 24 &&
349 (RHSBits = numBitsUnsigned(Op: RHS, CtxI: &I)) <= 24) {
350 IsSigned = false;
351
352 } else if (ST.hasMulI24() && (LHSBits = numBitsSigned(Op: LHS, CtxI: &I)) <= 24 &&
353 (RHSBits = numBitsSigned(Op: RHS, CtxI: &I)) <= 24) {
354 IsSigned = true;
355
356 } else
357 return false;
358
359 SmallVector<Value *, 4> LHSVals;
360 SmallVector<Value *, 4> RHSVals;
361 SmallVector<Value *, 4> ResultVals;
362 extractValues(Builder, Values&: LHSVals, V: LHS);
363 extractValues(Builder, Values&: RHSVals, V: RHS);
364
365 IntegerType *I32Ty = Builder.getInt32Ty();
366 IntegerType *IntrinTy = Size > 32 ? Builder.getInt64Ty() : I32Ty;
367 Type *DstTy = LHSVals[0]->getType();
368
369 for (int I = 0, E = LHSVals.size(); I != E; ++I) {
370 Value *LHS = IsSigned ? Builder.CreateSExtOrTrunc(V: LHSVals[I], DestTy: I32Ty)
371 : Builder.CreateZExtOrTrunc(V: LHSVals[I], DestTy: I32Ty);
372 Value *RHS = IsSigned ? Builder.CreateSExtOrTrunc(V: RHSVals[I], DestTy: I32Ty)
373 : Builder.CreateZExtOrTrunc(V: RHSVals[I], DestTy: I32Ty);
374 Intrinsic::ID ID =
375 IsSigned ? Intrinsic::amdgcn_mul_i24 : Intrinsic::amdgcn_mul_u24;
376 Value *Result = Builder.CreateIntrinsic(ID, OverloadTypes: {IntrinTy}, Args: {LHS, RHS});
377 Result = IsSigned ? Builder.CreateSExtOrTrunc(V: Result, DestTy: DstTy)
378 : Builder.CreateZExtOrTrunc(V: Result, DestTy: DstTy);
379 ResultVals.push_back(Elt: Result);
380 }
381
382 Value *NewVal = insertValues(Builder, Ty, Values&: ResultVals);
383 NewVal->takeName(V: &I);
384 I.replaceAllUsesWith(V: NewVal);
385 DeadVals.push_back(Elt: &I);
386
387 return true;
388}
389
390// Find a select instruction, which may have been casted. This is mostly to deal
391// with cases where i16 selects were promoted here to i32.
392static SelectInst *findSelectThroughCast(Value *V, CastInst *&Cast) {
393 Cast = nullptr;
394 if (SelectInst *Sel = dyn_cast<SelectInst>(Val: V))
395 return Sel;
396
397 if ((Cast = dyn_cast<CastInst>(Val: V))) {
398 if (SelectInst *Sel = dyn_cast<SelectInst>(Val: Cast->getOperand(i_nocapture: 0)))
399 return Sel;
400 }
401
402 return nullptr;
403}
404
405bool AMDGPUCodeGenPrepareImpl::foldBinOpIntoSelect(BinaryOperator &BO) const {
406 // Don't do this unless the old select is going away. We want to eliminate the
407 // binary operator, not replace a binop with a select.
408 int SelOpNo = 0;
409
410 CastInst *CastOp;
411
412 // TODO: Should probably try to handle some cases with multiple
413 // users. Duplicating the select may be profitable for division.
414 SelectInst *Sel = findSelectThroughCast(V: BO.getOperand(i_nocapture: 0), Cast&: CastOp);
415 if (!Sel || !Sel->hasOneUse()) {
416 SelOpNo = 1;
417 Sel = findSelectThroughCast(V: BO.getOperand(i_nocapture: 1), Cast&: CastOp);
418 }
419
420 if (!Sel || !Sel->hasOneUse())
421 return false;
422
423 Constant *CT = dyn_cast<Constant>(Val: Sel->getTrueValue());
424 Constant *CF = dyn_cast<Constant>(Val: Sel->getFalseValue());
425 Constant *CBO = dyn_cast<Constant>(Val: BO.getOperand(i_nocapture: SelOpNo ^ 1));
426 if (!CBO || !CT || !CF)
427 return false;
428
429 if (CastOp) {
430 if (!CastOp->hasOneUse())
431 return false;
432 CT = ConstantFoldCastOperand(Opcode: CastOp->getOpcode(), C: CT, DestTy: BO.getType(), DL);
433 CF = ConstantFoldCastOperand(Opcode: CastOp->getOpcode(), C: CF, DestTy: BO.getType(), DL);
434 }
435
436 // TODO: Handle special 0/-1 cases DAG combine does, although we only really
437 // need to handle divisions here.
438 Constant *FoldedT =
439 SelOpNo ? ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CBO, RHS: CT, DL)
440 : ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CT, RHS: CBO, DL);
441 if (!FoldedT || isa<ConstantExpr>(Val: FoldedT))
442 return false;
443
444 Constant *FoldedF =
445 SelOpNo ? ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CBO, RHS: CF, DL)
446 : ConstantFoldBinaryOpOperands(Opcode: BO.getOpcode(), LHS: CF, RHS: CBO, DL);
447 if (!FoldedF || isa<ConstantExpr>(Val: FoldedF))
448 return false;
449
450 IRBuilder<> Builder(&BO);
451 Builder.SetCurrentDebugLocation(BO.getDebugLoc());
452 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Val: &BO))
453 Builder.setFastMathFlags(FPOp->getFastMathFlags());
454
455 Value *NewSelect = Builder.CreateSelect(C: Sel->getCondition(),
456 True: FoldedT, False: FoldedF);
457 NewSelect->takeName(V: &BO);
458 BO.replaceAllUsesWith(V: NewSelect);
459 DeadVals.push_back(Elt: &BO);
460 if (CastOp)
461 DeadVals.push_back(Elt: CastOp);
462 DeadVals.push_back(Elt: Sel);
463 return true;
464}
465
466std::pair<Value *, Value *>
467AMDGPUCodeGenPrepareImpl::getFrexpResults(IRBuilder<> &Builder,
468 Value *Src) const {
469 Type *Ty = Src->getType();
470 Value *Frexp = Builder.CreateIntrinsic(ID: Intrinsic::frexp,
471 OverloadTypes: {Ty, Builder.getInt32Ty()}, Args: Src);
472 Value *FrexpMant = Builder.CreateExtractValue(Agg: Frexp, Idxs: {0});
473
474 // Bypass the bug workaround for the exponent result since it doesn't matter.
475 // TODO: Does the bug workaround even really need to consider the exponent
476 // result? It's unspecified by the spec.
477
478 Value *FrexpExp =
479 ST.hasFractBug()
480 ? Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_frexp_exp,
481 OverloadTypes: {Builder.getInt32Ty(), Ty}, Args: Src)
482 : Builder.CreateExtractValue(Agg: Frexp, Idxs: {1});
483 return {FrexpMant, FrexpExp};
484}
485
486/// Emit an expansion of 1.0 / Src good for 1ulp that supports denormals.
487Value *AMDGPUCodeGenPrepareImpl::emitRcpIEEE1ULP(IRBuilder<> &Builder,
488 Value *Src,
489 bool IsNegative) const {
490 // Same as for 1.0, but expand the sign out of the constant.
491 // -1.0 / x -> rcp (fneg x)
492 if (IsNegative)
493 Src = Builder.CreateFNeg(V: Src);
494
495 // The rcp instruction doesn't support denormals, so scale the input
496 // out of the denormal range and convert at the end.
497 //
498 // Expand as 2^-n * (1.0 / (x * 2^n))
499
500 // TODO: Skip scaling if input is known never denormal and the input
501 // range won't underflow to denormal. The hard part is knowing the
502 // result. We need a range check, the result could be denormal for
503 // 0x1p+126 < den <= 0x1p+127.
504 auto [FrexpMant, FrexpExp] = getFrexpResults(Builder, Src);
505 Value *ScaleFactor = Builder.CreateNeg(V: FrexpExp);
506 Value *Rcp = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: FrexpMant);
507 return Builder.CreateCall(Callee: getLdexpF32(), Args: {Rcp, ScaleFactor});
508}
509
510/// Emit a 2ulp expansion for fdiv by using frexp for input scaling.
511Value *AMDGPUCodeGenPrepareImpl::emitFrexpDiv(IRBuilder<> &Builder, Value *LHS,
512 Value *RHS,
513 FastMathFlags FMF) const {
514 // If we have have to work around the fract/frexp bug, we're worse off than
515 // using the fdiv.fast expansion. The full safe expansion is faster if we have
516 // fast FMA.
517 if (HasFP32DenormalFlush && ST.hasFractBug() && !ST.hasFastFMAF32() &&
518 (!FMF.noNaNs() || !FMF.noInfs()))
519 return nullptr;
520
521 // We're scaling the LHS to avoid a denormal input, and scale the denominator
522 // to avoid large values underflowing the result.
523 auto [FrexpMantRHS, FrexpExpRHS] = getFrexpResults(Builder, Src: RHS);
524
525 Value *Rcp =
526 Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: FrexpMantRHS);
527
528 auto [FrexpMantLHS, FrexpExpLHS] = getFrexpResults(Builder, Src: LHS);
529 Value *Mul = Builder.CreateFMul(L: FrexpMantLHS, R: Rcp);
530
531 // We multiplied by 2^N/2^M, so we need to multiply by 2^(N-M) to scale the
532 // result.
533 Value *ExpDiff = Builder.CreateSub(LHS: FrexpExpLHS, RHS: FrexpExpRHS);
534 return Builder.CreateCall(Callee: getLdexpF32(), Args: {Mul, ExpDiff});
535}
536
537/// Emit a sqrt that handles denormals and is accurate to 2ulp.
538Value *AMDGPUCodeGenPrepareImpl::emitSqrtIEEE2ULP(IRBuilder<> &Builder,
539 Value *Src,
540 FastMathFlags FMF) const {
541 Type *Ty = Src->getType();
542 APFloat SmallestNormal =
543 APFloat::getSmallestNormalized(Sem: Ty->getFltSemantics());
544 Value *NeedScale =
545 Builder.CreateFCmpOLT(LHS: Src, RHS: ConstantFP::get(Ty, V: SmallestNormal));
546
547 ConstantInt *Zero = Builder.getInt32(C: 0);
548 Value *InputScaleFactor =
549 Builder.CreateSelect(C: NeedScale, True: Builder.getInt32(C: 32), False: Zero);
550
551 Value *Scaled = Builder.CreateCall(Callee: getLdexpF32(), Args: {Src, InputScaleFactor});
552
553 Value *Sqrt = Builder.CreateCall(Callee: getSqrtF32(), Args: Scaled);
554
555 Value *OutputScaleFactor =
556 Builder.CreateSelect(C: NeedScale, True: Builder.getInt32(C: -16), False: Zero);
557 return Builder.CreateCall(Callee: getLdexpF32(), Args: {Sqrt, OutputScaleFactor});
558}
559
560/// Emit an expansion of 1.0 / sqrt(Src) good for 1ulp that supports denormals.
561static Value *emitRsqIEEE1ULP(IRBuilder<> &Builder, Value *Src,
562 bool IsNegative) {
563 // bool need_scale = x < 0x1p-126f;
564 // float input_scale = need_scale ? 0x1.0p+24f : 1.0f;
565 // float output_scale = need_scale ? 0x1.0p+12f : 1.0f;
566 // rsq(x * input_scale) * output_scale;
567
568 Type *Ty = Src->getType();
569 APFloat SmallestNormal =
570 APFloat::getSmallestNormalized(Sem: Ty->getFltSemantics());
571 Value *NeedScale =
572 Builder.CreateFCmpOLT(LHS: Src, RHS: ConstantFP::get(Ty, V: SmallestNormal));
573 Constant *One = ConstantFP::get(Ty, V: 1.0);
574 Constant *InputScale = ConstantFP::get(Ty, V: 0x1.0p+24);
575 Constant *OutputScale =
576 ConstantFP::get(Ty, V: IsNegative ? -0x1.0p+12 : 0x1.0p+12);
577
578 Value *InputScaleFactor = Builder.CreateSelect(C: NeedScale, True: InputScale, False: One);
579
580 Value *ScaledInput = Builder.CreateFMul(L: Src, R: InputScaleFactor);
581 Value *Rsq = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rsq, Op: ScaledInput);
582 Value *OutputScaleFactor = Builder.CreateSelect(
583 C: NeedScale, True: OutputScale, False: IsNegative ? ConstantFP::get(Ty, V: -1.0) : One);
584
585 return Builder.CreateFMul(L: Rsq, R: OutputScaleFactor);
586}
587
588/// Emit inverse sqrt expansion for f64 with a correction sequence on top of
589/// v_rsq_f64. This should give a 1ulp result.
590Value *AMDGPUCodeGenPrepareImpl::emitRsqF64(IRBuilder<> &Builder, Value *X,
591 FastMathFlags SqrtFMF,
592 FastMathFlags DivFMF,
593 const Instruction *CtxI,
594 bool IsNegative) const {
595 // rsq(x):
596 // double y0 = BUILTIN_AMDGPU_RSQRT_F64(x);
597 // double e = MATH_MAD(-y0 * (x == PINF_F64 || x == 0.0 ? y0 : x), y0, 1.0);
598 // return MATH_MAD(y0*e, MATH_MAD(e, 0.375, 0.5), y0);
599 //
600 // -rsq(x):
601 // double y0 = BUILTIN_AMDGPU_RSQRT_F64(x);
602 // double e = MATH_MAD(-y0 * (x == PINF_F64 || x == 0.0 ? y0 : x), y0, 1.0);
603 // return MATH_MAD(-y0*e, MATH_MAD(e, 0.375, 0.5), -y0);
604 //
605 // The rsq instruction handles the special cases correctly. We need to check
606 // for the edge case conditions to ensure the special case propagates through
607 // the later instructions.
608
609 Value *Y0 = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rsq, Op: X);
610
611 // Try to elide the edge case check.
612 //
613 // Fast math flags imply:
614 // sqrt ninf => !isinf(x)
615 // fdiv ninf => x != 0, !isinf(x)
616 bool MaybePosInf = !SqrtFMF.noInfs() && !DivFMF.noInfs();
617 bool MaybeZero = !DivFMF.noInfs();
618
619 DenormalMode DenormMode;
620 FPClassTest Interested = fcNone;
621 if (MaybePosInf)
622 Interested = fcPosInf;
623 if (MaybeZero)
624 Interested |= fcZero;
625
626 if (Interested != fcNone) {
627 KnownFPClass KnownSrc = computeKnownFPClass(V: X, Interested, CtxI);
628 if (KnownSrc.isKnownNeverPosInfinity())
629 MaybePosInf = false;
630
631 DenormMode = F.getDenormalMode(FPType: X->getType()->getFltSemantics());
632 if (KnownSrc.isKnownNeverLogicalZero(Mode: DenormMode))
633 MaybeZero = false;
634 }
635
636 Value *SpecialOrRsq = X;
637 if (MaybeZero || MaybePosInf) {
638 Value *Cond;
639 if (MaybePosInf && MaybeZero) {
640 if (DenormMode.Input != DenormalMode::DenormalModeKind::Dynamic) {
641 FPClassTest TestMask = fcPosInf | fcZero;
642 if (DenormMode.inputsAreZero())
643 TestMask |= fcSubnormal;
644
645 Cond = Builder.createIsFPClass(FPNum: X, Test: TestMask);
646 } else {
647 // Avoid using llvm.is.fpclass for dynamic denormal mode, since it
648 // doesn't respect the floating-point environment.
649 Value *IsZero =
650 Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getZero(Ty: X->getType()));
651 Value *IsInf =
652 Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getInfinity(Ty: X->getType()));
653 Cond = Builder.CreateOr(LHS: IsZero, RHS: IsInf);
654 }
655 } else if (MaybeZero) {
656 Cond = Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getZero(Ty: X->getType()));
657 } else {
658 Cond = Builder.CreateFCmpOEQ(LHS: X, RHS: ConstantFP::getInfinity(Ty: X->getType()));
659 }
660
661 SpecialOrRsq = Builder.CreateSelect(C: Cond, True: Y0, False: X);
662 }
663
664 Value *NegY0 = Builder.CreateFNeg(V: Y0);
665 Value *NegXY0 = Builder.CreateFMul(L: SpecialOrRsq, R: NegY0);
666
667 // Could be fmuladd, but isFMAFasterThanFMulAndFAdd is always true for f64.
668 Value *E = Builder.CreateFMA(Factor1: NegXY0, Factor2: Y0, Summand: ConstantFP::get(Ty: X->getType(), V: 1.0));
669
670 Value *Y0E = Builder.CreateFMul(L: E, R: IsNegative ? NegY0 : Y0);
671
672 Value *EFMA = Builder.CreateFMA(Factor1: E, Factor2: ConstantFP::get(Ty: X->getType(), V: 0.375),
673 Summand: ConstantFP::get(Ty: X->getType(), V: 0.5));
674
675 return Builder.CreateFMA(Factor1: Y0E, Factor2: EFMA, Summand: IsNegative ? NegY0 : Y0);
676}
677
678bool AMDGPUCodeGenPrepareImpl::canOptimizeWithRsq(FastMathFlags DivFMF,
679 FastMathFlags SqrtFMF) const {
680 // The rsqrt contraction increases accuracy from ~2ulp to ~1ulp for f32 and
681 // f64.
682 return DivFMF.allowContract() && SqrtFMF.allowContract();
683}
684
685Value *AMDGPUCodeGenPrepareImpl::optimizeWithRsq(
686 IRBuilder<> &Builder, Value *Num, Value *Den, const FastMathFlags DivFMF,
687 const FastMathFlags SqrtFMF, const Instruction *CtxI) const {
688 // The rsqrt contraction increases accuracy from ~2ulp to ~1ulp.
689 assert(DivFMF.allowContract() && SqrtFMF.allowContract());
690
691 // rsq_f16 is accurate to 0.51 ulp.
692 // rsq_f32 is accurate for !fpmath >= 1.0ulp and denormals are flushed.
693 // rsq_f64 is never accurate.
694 const ConstantFP *CLHS = dyn_cast<ConstantFP>(Val: Num);
695 if (!CLHS)
696 return nullptr;
697
698 bool IsNegative = false;
699
700 // TODO: Handle other numerator values with arcp.
701 if (CLHS->isOne() || (IsNegative = CLHS->isMinusOne())) {
702 // Add sqrt flags, but require both ninf and nsz from the div and the
703 // sqrt: sqrt's ninf/nsz don't say anything about the quotient.
704 IRBuilder<>::FastMathFlagGuard Guard(Builder);
705 FastMathFlags NewFMF = DivFMF | SqrtFMF;
706 FastMathFlags ValueFMF = FastMathFlags::intersectValue(LHS: DivFMF, RHS: SqrtFMF);
707 NewFMF.setNoInfs(ValueFMF.noInfs());
708 NewFMF.setNoSignedZeros(ValueFMF.noSignedZeros());
709 Builder.setFastMathFlags(NewFMF);
710
711 if (Den->getType()->isFloatTy()) {
712 if ((DivFMF.approxFunc() && SqrtFMF.approxFunc()) ||
713 canIgnoreDenormalInput(V: Den, CtxI)) {
714 Value *Result =
715 Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rsq, Op: Den);
716 // -1.0 / sqrt(x) -> fneg(rsq(x))
717 return IsNegative ? Builder.CreateFNeg(V: Result) : Result;
718 }
719
720 return emitRsqIEEE1ULP(Builder, Src: Den, IsNegative);
721 }
722
723 if (Den->getType()->isDoubleTy())
724 return emitRsqF64(Builder, X: Den, SqrtFMF, DivFMF, CtxI, IsNegative);
725 }
726
727 return nullptr;
728}
729
730// Optimize fdiv with rcp:
731//
732// 1/x -> rcp(x) when rcp is sufficiently accurate or inaccurate rcp is
733// allowed with afn.
734//
735// a/b -> a*rcp(b) when arcp is allowed, and we only need provide ULP 1.0
736Value *
737AMDGPUCodeGenPrepareImpl::optimizeWithRcp(IRBuilder<> &Builder, Value *Num,
738 Value *Den, FastMathFlags FMF,
739 const Instruction *CtxI) const {
740 // rcp_f16 is accurate to 0.51 ulp.
741 // rcp_f32 is accurate for !fpmath >= 1.0ulp and denormals are flushed.
742 // rcp_f64 is never accurate.
743 assert(Den->getType()->isFloatTy());
744
745 if (const ConstantFP *CLHS = dyn_cast<ConstantFP>(Val: Num)) {
746 bool IsNegative = false;
747 if (CLHS->isOne() || (IsNegative = CLHS->isMinusOne())) {
748 Value *Src = Den;
749
750 if (HasFP32DenormalFlush || FMF.approxFunc()) {
751 // -1.0 / x -> 1.0 / fneg(x)
752 if (IsNegative)
753 Src = Builder.CreateFNeg(V: Src);
754
755 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
756 // the CI documentation has a worst case error of 1 ulp.
757 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK
758 // to use it as long as we aren't trying to use denormals.
759 //
760 // v_rcp_f16 and v_rsq_f16 DO support denormals.
761
762 // NOTE: v_sqrt and v_rcp will be combined to v_rsq later. So we don't
763 // insert rsq intrinsic here.
764
765 // 1.0 / x -> rcp(x)
766 return Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: Src);
767 }
768
769 // TODO: If the input isn't denormal, and we know the input exponent isn't
770 // big enough to introduce a denormal we can avoid the scaling.
771 return emitRcpIEEE1ULP(Builder, Src, IsNegative);
772 }
773 }
774
775 if (FMF.allowReciprocal()) {
776 // x / y -> x * (1.0 / y)
777
778 // TODO: Could avoid denormal scaling and use raw rcp if we knew the output
779 // will never underflow.
780 if (HasFP32DenormalFlush || FMF.approxFunc()) {
781 Value *Recip = Builder.CreateUnaryIntrinsic(ID: Intrinsic::amdgcn_rcp, Op: Den);
782 return Builder.CreateFMul(L: Num, R: Recip);
783 }
784
785 Value *Recip = emitRcpIEEE1ULP(Builder, Src: Den, IsNegative: false);
786 return Builder.CreateFMul(L: Num, R: Recip);
787 }
788
789 return nullptr;
790}
791
792// optimize with fdiv.fast:
793//
794// a/b -> fdiv.fast(a, b) when !fpmath >= 2.5ulp with denormals flushed.
795//
796// 1/x -> fdiv.fast(1,x) when !fpmath >= 2.5ulp.
797//
798// NOTE: optimizeWithRcp should be tried first because rcp is the preference.
799Value *AMDGPUCodeGenPrepareImpl::optimizeWithFDivFast(
800 IRBuilder<> &Builder, Value *Num, Value *Den, float ReqdAccuracy) const {
801 // fdiv.fast can achieve 2.5 ULP accuracy.
802 if (ReqdAccuracy < 2.5f)
803 return nullptr;
804
805 // Only have fdiv.fast for f32.
806 assert(Den->getType()->isFloatTy());
807
808 bool NumIsOne = false;
809 if (const ConstantFP *CNum = dyn_cast<ConstantFP>(Val: Num)) {
810 if (CNum->isOne() || CNum->isMinusOne())
811 NumIsOne = true;
812 }
813
814 // fdiv does not support denormals. But 1.0/x is always fine to use it.
815 //
816 // TODO: This works for any value with a specific known exponent range, don't
817 // just limit to constant 1.
818 if (!HasFP32DenormalFlush && !NumIsOne)
819 return nullptr;
820
821 return Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_fdiv_fast, Args: {Num, Den});
822}
823
824Value *AMDGPUCodeGenPrepareImpl::visitFDivElement(
825 IRBuilder<> &Builder, Value *Num, Value *Den, FastMathFlags DivFMF,
826 FastMathFlags SqrtFMF, Value *RsqOp, const Instruction *FDivInst,
827 float ReqdDivAccuracy) const {
828 if (RsqOp) {
829 Value *Rsq =
830 optimizeWithRsq(Builder, Num, Den: RsqOp, DivFMF, SqrtFMF, CtxI: FDivInst);
831 if (Rsq)
832 return Rsq;
833 }
834
835 if (!Num->getType()->isFloatTy())
836 return nullptr;
837
838 Value *Rcp = optimizeWithRcp(Builder, Num, Den, FMF: DivFMF, CtxI: FDivInst);
839 if (Rcp)
840 return Rcp;
841
842 // In the basic case fdiv_fast has the same instruction count as the frexp div
843 // expansion. Slightly prefer fdiv_fast since it ends in an fmul that can
844 // potentially be fused into a user. Also, materialization of the constants
845 // can be reused for multiple instances.
846 Value *FDivFast = optimizeWithFDivFast(Builder, Num, Den, ReqdAccuracy: ReqdDivAccuracy);
847 if (FDivFast)
848 return FDivFast;
849
850 return emitFrexpDiv(Builder, LHS: Num, RHS: Den, FMF: DivFMF);
851}
852
853// Optimizations is performed based on fpmath, fast math flags as well as
854// denormals to optimize fdiv with either rcp or fdiv.fast.
855//
856// With rcp:
857// 1/x -> rcp(x) when rcp is sufficiently accurate or inaccurate rcp is
858// allowed with afn.
859//
860// a/b -> a*rcp(b) when inaccurate rcp is allowed with afn.
861//
862// With fdiv.fast:
863// a/b -> fdiv.fast(a, b) when !fpmath >= 2.5ulp with denormals flushed.
864//
865// 1/x -> fdiv.fast(1,x) when !fpmath >= 2.5ulp.
866//
867// NOTE: rcp is the preference in cases that both are legal.
868bool AMDGPUCodeGenPrepareImpl::visitFDiv(BinaryOperator &FDiv) {
869 if (!CLOpts.codegenprepare_fdiv_expansion)
870 return false;
871
872 Type *Ty = FDiv.getType()->getScalarType();
873 const bool IsFloat = Ty->isFloatTy();
874 if (!IsFloat && !Ty->isDoubleTy())
875 return false;
876
877 // The f64 rcp/rsq approximations are pretty inaccurate. We can do an
878 // expansion around them in codegen. f16 is good enough to always use.
879
880 const FPMathOperator *FPOp = cast<const FPMathOperator>(Val: &FDiv);
881 const FastMathFlags DivFMF = FPOp->getFastMathFlags();
882 const float ReqdAccuracy = FPOp->getFPAccuracy();
883
884 FastMathFlags SqrtFMF;
885
886 Value *Num = FDiv.getOperand(i_nocapture: 0);
887 Value *Den = FDiv.getOperand(i_nocapture: 1);
888
889 Value *RsqOp = nullptr;
890 auto *DenII = dyn_cast<IntrinsicInst>(Val: Den);
891 if (DenII && DenII->getIntrinsicID() == Intrinsic::sqrt &&
892 DenII->hasOneUse()) {
893 const auto *SqrtOp = cast<FPMathOperator>(Val: DenII);
894 SqrtFMF = SqrtOp->getFastMathFlags();
895 if (canOptimizeWithRsq(DivFMF, SqrtFMF))
896 RsqOp = SqrtOp->getOperand(i: 0);
897 }
898
899 // rcp path not yet implemented for f64.
900 if (!IsFloat && !RsqOp)
901 return false;
902
903 // Inaccurate rcp is allowed with afn.
904 //
905 // Defer to codegen to handle this.
906 //
907 // TODO: Decide on an interpretation for interactions between afn + arcp +
908 // !fpmath, and make it consistent between here and codegen. For now, defer
909 // expansion of afn to codegen. The current interpretation is so aggressive we
910 // don't need any pre-consideration here when we have better information. A
911 // more conservative interpretation could use handling here.
912 const bool AllowInaccurateRcp = DivFMF.approxFunc();
913 if (!RsqOp && AllowInaccurateRcp)
914 return false;
915
916 // Defer the correct implementations to codegen.
917 if (IsFloat && ReqdAccuracy < 1.0f)
918 return false;
919
920 IRBuilder<> Builder(std::next(x: FDiv.getIterator()));
921 Builder.setFastMathFlags(DivFMF);
922 Builder.SetCurrentDebugLocation(FDiv.getDebugLoc());
923
924 SmallVector<Value *, 4> NumVals;
925 SmallVector<Value *, 4> DenVals;
926 SmallVector<Value *, 4> RsqDenVals;
927 extractValues(Builder, Values&: NumVals, V: Num);
928 extractValues(Builder, Values&: DenVals, V: Den);
929
930 if (RsqOp)
931 extractValues(Builder, Values&: RsqDenVals, V: RsqOp);
932
933 SmallVector<Value *, 4> ResultVals(NumVals.size());
934 for (int I = 0, E = NumVals.size(); I != E; ++I) {
935 Value *NumElt = NumVals[I];
936 Value *DenElt = DenVals[I];
937 Value *RsqDenElt = RsqOp ? RsqDenVals[I] : nullptr;
938
939 Value *NewElt =
940 visitFDivElement(Builder, Num: NumElt, Den: DenElt, DivFMF, SqrtFMF, RsqOp: RsqDenElt,
941 FDivInst: cast<Instruction>(Val: FPOp), ReqdDivAccuracy: ReqdAccuracy);
942 if (!NewElt) {
943 // Keep the original, but scalarized.
944
945 // This has the unfortunate side effect of sometimes scalarizing when
946 // we're not going to do anything.
947 NewElt = Builder.CreateFDiv(L: NumElt, R: DenElt);
948 if (auto *NewEltInst = dyn_cast<Instruction>(Val: NewElt))
949 NewEltInst->copyMetadata(SrcInst: FDiv);
950 }
951
952 ResultVals[I] = NewElt;
953 }
954
955 Value *NewVal = insertValues(Builder, Ty: FDiv.getType(), Values&: ResultVals);
956
957 if (NewVal) {
958 FDiv.replaceAllUsesWith(V: NewVal);
959 NewVal->takeName(V: &FDiv);
960 DeadVals.push_back(Elt: &FDiv);
961 }
962
963 return true;
964}
965
966static std::pair<Value*, Value*> getMul64(IRBuilder<> &Builder,
967 Value *LHS, Value *RHS) {
968 Type *I32Ty = Builder.getInt32Ty();
969 Type *I64Ty = Builder.getInt64Ty();
970
971 Value *LHS_EXT64 = Builder.CreateZExt(V: LHS, DestTy: I64Ty);
972 Value *RHS_EXT64 = Builder.CreateZExt(V: RHS, DestTy: I64Ty);
973 Value *MUL64 = Builder.CreateMul(LHS: LHS_EXT64, RHS: RHS_EXT64);
974 Value *Lo = Builder.CreateTrunc(V: MUL64, DestTy: I32Ty);
975 Value *Hi = Builder.CreateLShr(LHS: MUL64, RHS: Builder.getInt64(C: 32));
976 Hi = Builder.CreateTrunc(V: Hi, DestTy: I32Ty);
977 return std::pair(Lo, Hi);
978}
979
980static Value* getMulHu(IRBuilder<> &Builder, Value *LHS, Value *RHS) {
981 return getMul64(Builder, LHS, RHS).second;
982}
983
984/// Figure out how many bits are really needed for this division.
985/// \p MaxDivBits is an optimization hint to bypass the second
986/// ComputeNumSignBits/computeKnownBits call if the first one is
987/// insufficient.
988unsigned AMDGPUCodeGenPrepareImpl::getDivNumBits(BinaryOperator &I, Value *Num,
989 Value *Den,
990 unsigned MaxDivBits,
991 bool IsSigned) const {
992 assert(Num->getType()->getScalarSizeInBits() ==
993 Den->getType()->getScalarSizeInBits());
994 unsigned SSBits = Num->getType()->getScalarSizeInBits();
995 if (IsSigned) {
996 unsigned RHSSignBits = ComputeNumSignBits(Op: Den, DL: SQ.DL, AC: SQ.AC, CtxI: &I, DT: SQ.DT);
997 // A sign bit needs to be reserved for shrinking.
998 unsigned DivBits = SSBits - RHSSignBits + 1;
999 if (DivBits > MaxDivBits)
1000 return SSBits;
1001
1002 unsigned LHSSignBits = ComputeNumSignBits(Op: Num, DL: SQ.DL, AC: SQ.AC, CtxI: &I);
1003
1004 unsigned SignBits = std::min(a: LHSSignBits, b: RHSSignBits);
1005 DivBits = SSBits - SignBits + 1;
1006 return DivBits;
1007 }
1008
1009 // All bits are used for unsigned division for Num or Den in range
1010 // (SignedMax, UnsignedMax].
1011 KnownBits Known = computeKnownBits(V: Den, Q: SQ.getWithInstruction(I: &I));
1012 unsigned RHSBits = Known.countMaxActiveBits();
1013 if (RHSBits > MaxDivBits)
1014 return SSBits;
1015
1016 Known = computeKnownBits(V: Num, Q: SQ.getWithInstruction(I: &I));
1017 unsigned LHSBits = Known.countMaxActiveBits();
1018
1019 unsigned DivBits = std::max(a: LHSBits, b: RHSBits);
1020 return DivBits;
1021}
1022
1023Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloat(IRBuilder<> &Builder,
1024 BinaryOperator &I,
1025 Value *Num, Value *Den,
1026 bool IsDiv,
1027 bool IsSigned) const {
1028 unsigned DivBits = getDivNumBits(I, Num, Den, MaxDivBits: 23, IsSigned);
1029
1030 if (DivBits > (IsSigned ? 23u : 22u))
1031 return nullptr;
1032 return expandDivRemToFloatImpl(Builder, I, Num, Den, NumBits: DivBits, IsDiv,
1033 IsSigned);
1034}
1035
1036Value *AMDGPUCodeGenPrepareImpl::expandDivRemToFloatImpl(
1037 IRBuilder<> &Builder, BinaryOperator &I, Value *Num, Value *Den,
1038 unsigned DivBits, bool IsDiv, bool IsSigned) const {
1039
1040 // v_rcp_f32(float(X)) can have an error of 1 ulp.
1041 // This would cause incorrect calculation of Y/X if:
1042 // Y = (0x7FFFFF/X)*(X-0)-1
1043 // were allowed.
1044 //
1045 // For example,
1046 // (0x7FF6D3/0x000FE7) would erroneously produce 2060 instead of 2059.
1047 // (0x7FF8F5/0x007EFB) would erroneously produce 258 instead of 257.
1048 //
1049 // Thus, we conservatively restrict expandDivRemToFloatImpl to
1050 // [-0x400000,0x3FFFFF] for IsSigned
1051 // [ 0x000000,0x3FFFFF] for !IsSigned.
1052 assert(0 < DivBits && DivBits <= (IsSigned ? 23u : 22u) &&
1053 "abs(Num) must be <= 0x400000 for expandDivRemToFloatImpl to work "
1054 "correctly");
1055
1056 Type *I32Ty = Builder.getInt32Ty();
1057 Num = Builder.CreateTrunc(V: Num, DestTy: I32Ty);
1058 Den = Builder.CreateTrunc(V: Den, DestTy: I32Ty);
1059
1060 Type *F32Ty = Builder.getFloatTy();
1061 ConstantInt *One = Builder.getInt32(C: 1);
1062
1063 // int ia = (int)LHS;
1064 Value *IA = Num;
1065
1066 // int ib, (int)RHS;
1067 Value *IB = Den;
1068
1069 // float fa = (float)ia;
1070 Value *FA = IsSigned ? Builder.CreateSIToFP(V: IA, DestTy: F32Ty)
1071 : Builder.CreateUIToFP(V: IA, DestTy: F32Ty);
1072
1073 // float fb = (float)ib;
1074 Value *FB = IsSigned ? Builder.CreateSIToFP(V: IB, DestTy: F32Ty)
1075 : Builder.CreateUIToFP(V: IB, DestTy: F32Ty);
1076
1077 Value *RCP = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_rcp,
1078 OverloadTypes: Builder.getFloatTy(), Args: {FB});
1079
1080 // The calculation:
1081 // fq = fa*recip(fb)
1082 // may be too small due to the 1ulp accuracy in the recip
1083 // operation and rounding issues. Since fq is truncated to produce
1084 // an integer value it may be too small by one. This is
1085 // dealt with by incrementing fa by 1ulp:
1086 // fq = (fa+1ulp)*recip(fb)
1087 // This will increase fa's magnitude by at most 0.5
1088 // (i.e. when fabs(fa)==0x400000 the LSB of the mantissa represents 0.5).
1089 // Thus, this method is safe since fa must be incremented by at least 1.0
1090 // for the quotient to increase by one.
1091
1092 Value *FABits = Builder.CreateBitCast(V: FA, DestTy: I32Ty);
1093 Value *FABitsInc = Builder.CreateAdd(LHS: FABits, RHS: One);
1094 FA = Builder.CreateBitCast(V: FABitsInc, DestTy: F32Ty);
1095
1096 Value *FQM = Builder.CreateFMul(L: FA, R: RCP);
1097
1098 // fq = trunc(fqm);
1099 Value *FQ = Builder.CreateUnaryIntrinsic(ID: Intrinsic::trunc, Op: FQM);
1100
1101 // int iq = (int)fq;
1102 Value *IQ = IsSigned ? Builder.CreateFPToSI(V: FQ, DestTy: I32Ty)
1103 : Builder.CreateFPToUI(V: FQ, DestTy: I32Ty);
1104
1105 Value *Res = IQ;
1106 if (!IsDiv) {
1107 // Rem needs compensation, it's easier to recompute it
1108 Value *Rem = Builder.CreateMul(LHS: IQ, RHS: Den);
1109 Res = Builder.CreateSub(LHS: Num, RHS: Rem);
1110 }
1111
1112 return Res;
1113}
1114
1115// Try to recognize special cases the DAG will emit special, better expansions
1116// than the general expansion we do here.
1117
1118// TODO: It would be better to just directly handle those optimizations here.
1119bool AMDGPUCodeGenPrepareImpl::divHasSpecialOptimization(BinaryOperator &I,
1120 Value *Num,
1121 Value *Den) const {
1122 if (Constant *C = dyn_cast<Constant>(Val: Den)) {
1123 // Arbitrary constants get a better expansion as long as a wider mulhi is
1124 // legal.
1125 if (C->getType()->getScalarSizeInBits() <= 32)
1126 return true;
1127
1128 // TODO: Sdiv check for not exact for some reason.
1129
1130 // If there's no wider mulhi, there's only a better expansion for powers of
1131 // two.
1132 // TODO: Should really know for each vector element.
1133 if (isKnownToBeAPowerOfTwo(V: C, OrZero: true, Q: SQ.getWithInstruction(I: &I)))
1134 return true;
1135
1136 return false;
1137 }
1138
1139 if (BinaryOperator *BinOpDen = dyn_cast<BinaryOperator>(Val: Den)) {
1140 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
1141 if (BinOpDen->getOpcode() == Instruction::Shl &&
1142 isa<Constant>(Val: BinOpDen->getOperand(i_nocapture: 0)) &&
1143 isKnownToBeAPowerOfTwo(V: BinOpDen->getOperand(i_nocapture: 0), OrZero: true,
1144 Q: SQ.getWithInstruction(I: &I))) {
1145 return true;
1146 }
1147 }
1148
1149 return false;
1150}
1151
1152static Value *getSign32(Value *V, IRBuilder<> &Builder, const DataLayout DL) {
1153 // Check whether the sign can be determined statically.
1154 KnownBits Known = computeKnownBits(V, DL);
1155 if (Known.isNegative())
1156 return Constant::getAllOnesValue(Ty: V->getType());
1157 if (Known.isNonNegative())
1158 return Constant::getNullValue(Ty: V->getType());
1159 return Builder.CreateAShr(LHS: V, RHS: Builder.getInt32(C: 31));
1160}
1161
1162Value *AMDGPUCodeGenPrepareImpl::expandDivRem32(IRBuilder<> &Builder,
1163 BinaryOperator &I, Value *X,
1164 Value *Y) const {
1165 Instruction::BinaryOps Opc = I.getOpcode();
1166 assert(Opc == Instruction::URem || Opc == Instruction::UDiv ||
1167 Opc == Instruction::SRem || Opc == Instruction::SDiv);
1168
1169 FastMathFlags FMF;
1170 FMF.setFast();
1171 Builder.setFastMathFlags(FMF);
1172
1173 if (divHasSpecialOptimization(I, Num: X, Den: Y))
1174 return nullptr; // Keep it for later optimization.
1175
1176 bool IsDiv = Opc == Instruction::UDiv || Opc == Instruction::SDiv;
1177 bool IsSigned = Opc == Instruction::SRem || Opc == Instruction::SDiv;
1178
1179 Type *Ty = X->getType();
1180 Type *I32Ty = Builder.getInt32Ty();
1181 Type *F32Ty = Builder.getFloatTy();
1182
1183 if (Ty->getScalarSizeInBits() != 32) {
1184 if (IsSigned) {
1185 X = Builder.CreateSExtOrTrunc(V: X, DestTy: I32Ty);
1186 Y = Builder.CreateSExtOrTrunc(V: Y, DestTy: I32Ty);
1187 } else {
1188 X = Builder.CreateZExtOrTrunc(V: X, DestTy: I32Ty);
1189 Y = Builder.CreateZExtOrTrunc(V: Y, DestTy: I32Ty);
1190 }
1191 }
1192
1193 if (Value *Res = expandDivRemToFloat(Builder, I, Num: X, Den: Y, IsDiv, IsSigned)) {
1194 return IsSigned ? Builder.CreateSExtOrTrunc(V: Res, DestTy: Ty) :
1195 Builder.CreateZExtOrTrunc(V: Res, DestTy: Ty);
1196 }
1197
1198 ConstantInt *Zero = Builder.getInt32(C: 0);
1199 ConstantInt *One = Builder.getInt32(C: 1);
1200
1201 Value *Sign = nullptr;
1202 if (IsSigned) {
1203 Value *SignX = getSign32(V: X, Builder, DL);
1204 Value *SignY = getSign32(V: Y, Builder, DL);
1205 // Remainder sign is the same as LHS
1206 Sign = IsDiv ? Builder.CreateXor(LHS: SignX, RHS: SignY) : SignX;
1207
1208 X = Builder.CreateAdd(LHS: X, RHS: SignX);
1209 Y = Builder.CreateAdd(LHS: Y, RHS: SignY);
1210
1211 X = Builder.CreateXor(LHS: X, RHS: SignX);
1212 Y = Builder.CreateXor(LHS: Y, RHS: SignY);
1213 }
1214
1215 // The algorithm here is based on ideas from "Software Integer Division", Tom
1216 // Rodeheffer, August 2008.
1217 //
1218 // unsigned udiv(unsigned x, unsigned y) {
1219 // // Initial estimate of inv(y). The constant is less than 2^32 to ensure
1220 // // that this is a lower bound on inv(y), even if some of the calculations
1221 // // round up.
1222 // unsigned z = (unsigned)((4294967296.0 - 512.0) * v_rcp_f32((float)y));
1223 //
1224 // // One round of UNR (Unsigned integer Newton-Raphson) to improve z.
1225 // // Empirically this is guaranteed to give a "two-y" lower bound on
1226 // // inv(y).
1227 // z += umulh(z, -y * z);
1228 //
1229 // // Quotient/remainder estimate.
1230 // unsigned q = umulh(x, z);
1231 // unsigned r = x - q * y;
1232 //
1233 // // Two rounds of quotient/remainder refinement.
1234 // if (r >= y) {
1235 // ++q;
1236 // r -= y;
1237 // }
1238 // if (r >= y) {
1239 // ++q;
1240 // r -= y;
1241 // }
1242 //
1243 // return q;
1244 // }
1245
1246 // Initial estimate of inv(y).
1247 Value *FloatY = Builder.CreateUIToFP(V: Y, DestTy: F32Ty);
1248 Value *RcpY = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_rcp, OverloadTypes: F32Ty, Args: {FloatY});
1249 Constant *Scale = ConstantFP::get(Ty: F32Ty, V: llvm::bit_cast<float>(from: 0x4F7FFFFE));
1250 Value *ScaledY = Builder.CreateFMul(L: RcpY, R: Scale);
1251 Value *Z = Builder.CreateFPToUI(V: ScaledY, DestTy: I32Ty);
1252
1253 // One round of UNR.
1254 Value *NegY = Builder.CreateSub(LHS: Zero, RHS: Y);
1255 Value *NegYZ = Builder.CreateMul(LHS: NegY, RHS: Z);
1256 Z = Builder.CreateAdd(LHS: Z, RHS: getMulHu(Builder, LHS: Z, RHS: NegYZ));
1257
1258 // Quotient/remainder estimate.
1259 Value *Q = getMulHu(Builder, LHS: X, RHS: Z);
1260 Value *R = Builder.CreateSub(LHS: X, RHS: Builder.CreateMul(LHS: Q, RHS: Y));
1261
1262 // First quotient/remainder refinement.
1263 Value *Cond = Builder.CreateICmpUGE(LHS: R, RHS: Y);
1264 if (IsDiv)
1265 Q = Builder.CreateSelect(C: Cond, True: Builder.CreateAdd(LHS: Q, RHS: One), False: Q);
1266 R = Builder.CreateSelect(C: Cond, True: Builder.CreateSub(LHS: R, RHS: Y), False: R);
1267
1268 // Second quotient/remainder refinement.
1269 Cond = Builder.CreateICmpUGE(LHS: R, RHS: Y);
1270 Value *Res;
1271 if (IsDiv)
1272 Res = Builder.CreateSelect(C: Cond, True: Builder.CreateAdd(LHS: Q, RHS: One), False: Q);
1273 else
1274 Res = Builder.CreateSelect(C: Cond, True: Builder.CreateSub(LHS: R, RHS: Y), False: R);
1275
1276 if (IsSigned) {
1277 Res = Builder.CreateXor(LHS: Res, RHS: Sign);
1278 Res = Builder.CreateSub(LHS: Res, RHS: Sign);
1279 Res = Builder.CreateSExtOrTrunc(V: Res, DestTy: Ty);
1280 } else {
1281 Res = Builder.CreateZExtOrTrunc(V: Res, DestTy: Ty);
1282 }
1283 return Res;
1284}
1285
1286Value *AMDGPUCodeGenPrepareImpl::shrinkDivRem64(IRBuilder<> &Builder,
1287 BinaryOperator &I, Value *Num,
1288 Value *Den) const {
1289 if (!ExpandDiv64InIR && divHasSpecialOptimization(I, Num, Den))
1290 return nullptr; // Keep it for later optimization.
1291
1292 Instruction::BinaryOps Opc = I.getOpcode();
1293
1294 bool IsDiv = Opc == Instruction::SDiv || Opc == Instruction::UDiv;
1295 bool IsSigned = Opc == Instruction::SDiv || Opc == Instruction::SRem;
1296
1297 unsigned NumDivBits = getDivNumBits(I, Num, Den, MaxDivBits: 32, IsSigned);
1298 if (NumDivBits > 32)
1299 return nullptr;
1300
1301 Value *Narrowed = nullptr;
1302 if (NumDivBits <= (IsSigned ? 23u : 22u)) {
1303 Narrowed = expandDivRemToFloatImpl(Builder, I, Num, Den, DivBits: NumDivBits, IsDiv,
1304 IsSigned);
1305 } else if (NumDivBits <= (IsSigned ? 31u : 32u)) {
1306 // Do not use 32-bit division if dividend may be -2147483648.
1307 // Otherwise 32-bit division cannot be used safely.
1308 // -2147483648/1 and -2147483648/-1 are not equal,
1309 // but they produce the same lower 32-bit result.
1310 Narrowed = expandDivRem32(Builder, I, X: Num, Y: Den);
1311 }
1312
1313 if (Narrowed) {
1314 return IsSigned ? Builder.CreateSExt(V: Narrowed, DestTy: Num->getType()) :
1315 Builder.CreateZExt(V: Narrowed, DestTy: Num->getType());
1316 }
1317
1318 return nullptr;
1319}
1320
1321void AMDGPUCodeGenPrepareImpl::expandDivRem64(BinaryOperator &I) const {
1322 Instruction::BinaryOps Opc = I.getOpcode();
1323 // Do the general expansion.
1324 if (Opc == Instruction::UDiv || Opc == Instruction::SDiv) {
1325 expandDivisionUpTo64Bits(Div: &I);
1326 return;
1327 }
1328
1329 if (Opc == Instruction::URem || Opc == Instruction::SRem) {
1330 expandRemainderUpTo64Bits(Rem: &I);
1331 return;
1332 }
1333
1334 llvm_unreachable("not a division");
1335}
1336
1337/*
1338This will cause non-byte load in consistency, for example:
1339```
1340 %load = load i1, ptr addrspace(4) %arg, align 4
1341 %zext = zext i1 %load to
1342 i64 %add = add i64 %zext
1343```
1344Instead of creating `s_and_b32 s0, s0, 1`,
1345it will create `s_and_b32 s0, s0, 0xff`.
1346We accept this change since the non-byte load assumes the upper bits
1347within the byte are all 0.
1348*/
1349bool AMDGPUCodeGenPrepareImpl::tryNarrowMathIfNoOverflow(Instruction *I) {
1350 unsigned Opc = I->getOpcode();
1351
1352 if (Opc != Instruction::Add && Opc != Instruction::Mul)
1353 return false;
1354
1355 Type *OldType = I->getType();
1356 unsigned OrigBit = OldType->getScalarSizeInBits();
1357 unsigned MaxBitsNeeded = computeKnownBits(V: I, DL).countMaxActiveBits();
1358
1359 MaxBitsNeeded = std::max<unsigned>(a: bit_ceil(Value: MaxBitsNeeded), b: 8);
1360 Type *NewType = DL.getSmallestLegalIntType(C&: I->getContext(), Width: MaxBitsNeeded);
1361 if (!NewType)
1362 return false;
1363 unsigned NewBit = NewType->getIntegerBitWidth();
1364 if (NewBit >= OrigBit)
1365 return false;
1366 NewType = I->getType()->getWithNewBitWidth(NewBitWidth: NewBit);
1367
1368 // Old cost
1369 InstructionCost OldCost =
1370 TTI.getArithmeticInstrCost(Opcode: Opc, Ty: OldType, CostKind: TTI::TCK_RecipThroughput);
1371 // New cost of new op
1372 InstructionCost NewCost =
1373 TTI.getArithmeticInstrCost(Opcode: Opc, Ty: NewType, CostKind: TTI::TCK_RecipThroughput);
1374 // New cost of narrowing 2 operands (use trunc)
1375 int NumOfNonConstOps = 2;
1376 if (isa<Constant>(Val: I->getOperand(i: 0)) || isa<Constant>(Val: I->getOperand(i: 1))) {
1377 // Cannot be both constant, should be propagated
1378 NumOfNonConstOps = 1;
1379 }
1380 NewCost += NumOfNonConstOps * TTI.getCastInstrCost(Opcode: Instruction::Trunc,
1381 Dst: NewType, Src: OldType,
1382 CCH: TTI.getCastContextHint(I),
1383 CostKind: TTI::TCK_RecipThroughput);
1384 // New cost of zext narrowed result to original type
1385 NewCost +=
1386 TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: OldType, Src: NewType,
1387 CCH: TTI.getCastContextHint(I), CostKind: TTI::TCK_RecipThroughput);
1388 if (NewCost >= OldCost)
1389 return false;
1390
1391 IRBuilder<> Builder(I);
1392 Value *Trunc0 = Builder.CreateTrunc(V: I->getOperand(i: 0), DestTy: NewType);
1393 Value *Trunc1 = Builder.CreateTrunc(V: I->getOperand(i: 1), DestTy: NewType);
1394 Value *Arith =
1395 Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Opc, LHS: Trunc0, RHS: Trunc1);
1396
1397 Value *Zext = Builder.CreateZExt(V: Arith, DestTy: OldType);
1398 I->replaceAllUsesWith(V: Zext);
1399 DeadVals.push_back(Elt: I);
1400 return true;
1401}
1402
1403bool AMDGPUCodeGenPrepareImpl::visitBinaryOperator(BinaryOperator &I) {
1404 if (foldBinOpIntoSelect(BO&: I))
1405 return true;
1406
1407 if (CLOpts.codegenprepare_mul24 && replaceMulWithMul24(I))
1408 return true;
1409 if (tryNarrowMathIfNoOverflow(I: &I))
1410 return true;
1411
1412 bool Changed = false;
1413 Instruction::BinaryOps Opc = I.getOpcode();
1414 Type *Ty = I.getType();
1415 Value *NewDiv = nullptr;
1416 unsigned ScalarSize = Ty->getScalarSizeInBits();
1417
1418 SmallVector<BinaryOperator *, 8> Div64ToExpand;
1419
1420 if ((Opc == Instruction::URem || Opc == Instruction::UDiv ||
1421 Opc == Instruction::SRem || Opc == Instruction::SDiv) &&
1422 ScalarSize <= 64 && CLOpts.codegenprepare_idiv_expansion) {
1423 Value *Num = I.getOperand(i_nocapture: 0);
1424 Value *Den = I.getOperand(i_nocapture: 1);
1425 IRBuilder<> Builder(&I);
1426 Builder.SetCurrentDebugLocation(I.getDebugLoc());
1427
1428 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1429 NewDiv = PoisonValue::get(T: VT);
1430
1431 for (unsigned N = 0, E = VT->getNumElements(); N != E; ++N) {
1432 Value *NumEltN = Builder.CreateExtractElement(Vec: Num, Idx: N);
1433 Value *DenEltN = Builder.CreateExtractElement(Vec: Den, Idx: N);
1434
1435 Value *NewElt;
1436 if (ScalarSize <= 32) {
1437 NewElt = expandDivRem32(Builder, I, X: NumEltN, Y: DenEltN);
1438 if (!NewElt)
1439 NewElt = Builder.CreateBinOp(Opc, LHS: NumEltN, RHS: DenEltN);
1440 } else {
1441 // See if this 64-bit division can be shrunk to 32/24-bits before
1442 // producing the general expansion.
1443 NewElt = shrinkDivRem64(Builder, I, Num: NumEltN, Den: DenEltN);
1444 if (!NewElt) {
1445 // The general 64-bit expansion introduces control flow and doesn't
1446 // return the new value. Just insert a scalar copy and defer
1447 // expanding it.
1448 NewElt = Builder.CreateBinOp(Opc, LHS: NumEltN, RHS: DenEltN);
1449 // CreateBinOp does constant folding. If the operands are constant,
1450 // it will return a Constant instead of a BinaryOperator.
1451 if (auto *NewEltBO = dyn_cast<BinaryOperator>(Val: NewElt))
1452 Div64ToExpand.push_back(Elt: NewEltBO);
1453 }
1454 }
1455
1456 if (auto *NewEltI = dyn_cast<Instruction>(Val: NewElt))
1457 NewEltI->copyIRFlags(V: &I);
1458
1459 NewDiv = Builder.CreateInsertElement(Vec: NewDiv, NewElt, Idx: N);
1460 }
1461 } else {
1462 if (ScalarSize <= 32)
1463 NewDiv = expandDivRem32(Builder, I, X: Num, Y: Den);
1464 else {
1465 NewDiv = shrinkDivRem64(Builder, I, Num, Den);
1466 if (!NewDiv)
1467 Div64ToExpand.push_back(Elt: &I);
1468 }
1469 }
1470
1471 if (NewDiv) {
1472 I.replaceAllUsesWith(V: NewDiv);
1473 DeadVals.push_back(Elt: &I);
1474 Changed = true;
1475 }
1476 }
1477
1478 if (ExpandDiv64InIR) {
1479 // TODO: We get much worse code in specially handled constant cases.
1480 for (BinaryOperator *Div : Div64ToExpand) {
1481 expandDivRem64(I&: *Div);
1482 FlowChanged = true;
1483 Changed = true;
1484 }
1485 }
1486
1487 return Changed;
1488}
1489
1490bool AMDGPUCodeGenPrepareImpl::visitLoadInst(LoadInst &I) {
1491 if (!CLOpts.codegenprepare_widen_constant_loads)
1492 return false;
1493
1494 if ((I.getPointerAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
1495 I.getPointerAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
1496 canWidenScalarExtLoad(I)) {
1497 IRBuilder<> Builder(&I);
1498 Builder.SetCurrentDebugLocation(I.getDebugLoc());
1499
1500 Type *I32Ty = Builder.getInt32Ty();
1501 LoadInst *WidenLoad = Builder.CreateLoad(Ty: I32Ty, Ptr: I.getPointerOperand());
1502 AMDGPU::copyMetadataForWidenedLoad(Dest&: *WidenLoad, Source: I);
1503
1504 // The widened load reads the original bytes in the low bits, so a !range
1505 // lower bound still holds. Convert it to the new type and don't make
1506 // assumptions about the high bits.
1507 if (auto *Range = I.getMetadata(KindID: LLVMContext::MD_range)) {
1508 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD: Range->getOperand(I: 0));
1509
1510 if (!Lower->isNullValue()) {
1511 Metadata *LowAndHigh[] = {
1512 ConstantAsMetadata::get(C: ConstantInt::get(Ty: I32Ty, V: Lower->getValue().zext(width: 32))),
1513 // Don't make assumptions about the high bits.
1514 ConstantAsMetadata::get(C: ConstantInt::get(Ty: I32Ty, V: 0))
1515 };
1516
1517 WidenLoad->setMetadata(KindID: LLVMContext::MD_range,
1518 Node: MDNode::get(Context&: F.getContext(), MDs: LowAndHigh));
1519 }
1520 }
1521
1522 int TySize = DL.getTypeSizeInBits(Ty: I.getType());
1523 Type *IntNTy = Builder.getIntNTy(N: TySize);
1524 Value *ValTrunc = Builder.CreateTrunc(V: WidenLoad, DestTy: IntNTy);
1525 Value *ValOrig = Builder.CreateBitCast(V: ValTrunc, DestTy: I.getType());
1526 I.replaceAllUsesWith(V: ValOrig);
1527 DeadVals.push_back(Elt: &I);
1528 return true;
1529 }
1530
1531 return false;
1532}
1533
1534bool AMDGPUCodeGenPrepareImpl::visitSelectInst(SelectInst &I) {
1535 FPMathOperator *FPOp = dyn_cast<FPMathOperator>(Val: &I);
1536 if (!FPOp)
1537 return false;
1538
1539 Value *X;
1540 Value *Fract = nullptr;
1541
1542 // Match:
1543 // (x - floor(x)) >= MIN_CONSTANT ? MIN_CONSTANT : (x - floor(x))
1544 //
1545 // This is the preferred way to implement fract.
1546 // TODO: Could also match with compare against 1.0
1547 const APFloat *C;
1548 if (match(V: &I, P: m_UnordFMin(L: m_Value(V&: X), R: m_APFloatAllowPoison(Res&: C)))) {
1549 Value *FractSrc = matchFractPatImpl(V&: *X, C: *C);
1550 if (!FractSrc)
1551 return false;
1552 IRBuilder<> Builder(&I);
1553 Builder.setFastMathFlags(FPOp->getFastMathFlags());
1554 Fract = applyFractPat(Builder, FractArg: FractSrc);
1555 } else {
1556 // Match patterns which may appear in legacy implementations of the fract()
1557 // function, built around the nan-avoidant minnum intrinsic. These are the
1558 // core pattern plus additional clamping of inf and nan values on the
1559 // result.
1560 Value *Cond = I.getCondition();
1561 Value *TrueVal = I.getTrueValue();
1562 Value *FalseVal = I.getFalseValue();
1563 Value *CmpVal;
1564 CmpPredicate IsNanPred;
1565
1566 // Match fract pattern with nan check.
1567 if (!match(V: Cond, P: m_FCmp(Pred&: IsNanPred, L: m_Value(V&: CmpVal), R: m_NonNaN())))
1568 return false;
1569
1570 IRBuilder<> Builder(&I);
1571 Builder.setFastMathFlags(FPOp->getFastMathFlags());
1572
1573 if (IsNanPred == FCmpInst::FCMP_UNO && TrueVal == CmpVal &&
1574 CmpVal == matchFractPatNanAvoidant(V&: *FalseVal)) {
1575 // isnan(x) ? x : fract(x)
1576 Fract = applyFractPat(Builder, FractArg: CmpVal);
1577 } else if (IsNanPred == FCmpInst::FCMP_ORD && FalseVal == CmpVal) {
1578 if (CmpVal == matchFractPatNanAvoidant(V&: *TrueVal)) {
1579 // !isnan(x) ? fract(x) : x
1580 Fract = applyFractPat(Builder, FractArg: CmpVal);
1581 } else {
1582 // Match an intermediate clamp infinity to 0 pattern. i.e.
1583 // !isnan(x) ? (!isinf(x) ? fract(x) : 0.0) : x
1584 CmpPredicate PredInf;
1585 Value *IfNotInf;
1586
1587 if (!match(V: TrueVal, P: m_Select(C: m_FCmp(Pred&: PredInf, L: m_FAbs(Op0: m_Specific(V: CmpVal)),
1588 R: m_PosInf()),
1589 L: m_Value(V&: IfNotInf), R: m_PosZeroFP())) ||
1590 PredInf != FCmpInst::FCMP_UNE ||
1591 CmpVal != matchFractPatNanAvoidant(V&: *IfNotInf))
1592 return false;
1593
1594 SelectInst *ClampInfSelect = cast<SelectInst>(Val: TrueVal);
1595
1596 // Insert before the fabs
1597 Value *InsertPt =
1598 cast<Instruction>(Val: ClampInfSelect->getCondition())->getOperand(i: 0);
1599
1600 Builder.SetInsertPoint(cast<Instruction>(Val: InsertPt));
1601 Value *NewFract = applyFractPat(Builder, FractArg: CmpVal);
1602 NewFract->takeName(V: TrueVal);
1603
1604 // Thread the new fract into the inf clamping sequence.
1605 DeadVals.push_back(Elt: ClampInfSelect->getOperand(i_nocapture: 1));
1606 ClampInfSelect->setOperand(i_nocapture: 1, Val_nocapture: NewFract);
1607
1608 // The outer select nan handling is also absorbed into the fract.
1609 Fract = ClampInfSelect;
1610 }
1611 } else
1612 return false;
1613 }
1614
1615 Fract->takeName(V: &I);
1616 I.replaceAllUsesWith(V: Fract);
1617 DeadVals.push_back(Elt: &I);
1618 return true;
1619}
1620
1621static bool areInSameBB(const Value *A, const Value *B) {
1622 const auto *IA = dyn_cast<Instruction>(Val: A);
1623 const auto *IB = dyn_cast<Instruction>(Val: B);
1624 return IA && IB && IA->getParent() == IB->getParent();
1625}
1626
1627// Helper for breaking large PHIs that returns true when an extractelement on V
1628// is likely to be folded away by the DAG combiner.
1629static bool isInterestingPHIIncomingValue(const Value *V) {
1630 const auto *FVT = dyn_cast<FixedVectorType>(Val: V->getType());
1631 if (!FVT)
1632 return false;
1633
1634 const Value *CurVal = V;
1635
1636 // Check for insertelements, keeping track of the elements covered.
1637 BitVector EltsCovered(FVT->getNumElements());
1638 while (const auto *IE = dyn_cast<InsertElementInst>(Val: CurVal)) {
1639 const auto *Idx = dyn_cast<ConstantInt>(Val: IE->getOperand(i_nocapture: 2));
1640
1641 // Non constant index/out of bounds index -> folding is unlikely.
1642 // The latter is more of a sanity check because canonical IR should just
1643 // have replaced those with poison.
1644 if (!Idx || Idx->getZExtValue() >= FVT->getNumElements())
1645 return false;
1646
1647 const auto *VecSrc = IE->getOperand(i_nocapture: 0);
1648
1649 // If the vector source is another instruction, it must be in the same basic
1650 // block. Otherwise, the DAGCombiner won't see the whole thing and is
1651 // unlikely to be able to do anything interesting here.
1652 if (isa<Instruction>(Val: VecSrc) && !areInSameBB(A: VecSrc, B: IE))
1653 return false;
1654
1655 CurVal = VecSrc;
1656 EltsCovered.set(Idx->getZExtValue());
1657
1658 // All elements covered.
1659 if (EltsCovered.all())
1660 return true;
1661 }
1662
1663 // We either didn't find a single insertelement, or the insertelement chain
1664 // ended before all elements were covered. Check for other interesting values.
1665
1666 // Constants are always interesting because we can just constant fold the
1667 // extractelements.
1668 if (isa<Constant>(Val: CurVal))
1669 return true;
1670
1671 // shufflevector is likely to be profitable if either operand is a constant,
1672 // or if either source is in the same block.
1673 // This is because shufflevector is most often lowered as a series of
1674 // insert/extract elements anyway.
1675 if (const auto *SV = dyn_cast<ShuffleVectorInst>(Val: CurVal)) {
1676 return isa<Constant>(Val: SV->getOperand(i_nocapture: 1)) ||
1677 areInSameBB(A: SV, B: SV->getOperand(i_nocapture: 0)) ||
1678 areInSameBB(A: SV, B: SV->getOperand(i_nocapture: 1));
1679 }
1680
1681 return false;
1682}
1683
1684static void collectPHINodes(const PHINode &I,
1685 SmallPtrSet<const PHINode *, 8> &SeenPHIs) {
1686 const auto [It, Inserted] = SeenPHIs.insert(Ptr: &I);
1687 if (!Inserted)
1688 return;
1689
1690 for (const Value *Inc : I.incoming_values()) {
1691 if (const auto *PhiInc = dyn_cast<PHINode>(Val: Inc))
1692 collectPHINodes(I: *PhiInc, SeenPHIs);
1693 }
1694
1695 for (const User *U : I.users()) {
1696 if (const auto *PhiU = dyn_cast<PHINode>(Val: U))
1697 collectPHINodes(I: *PhiU, SeenPHIs);
1698 }
1699}
1700
1701bool AMDGPUCodeGenPrepareImpl::canBreakPHINode(const PHINode &I) {
1702 // Check in the cache first.
1703 if (const auto It = BreakPhiNodesCache.find(Val: &I);
1704 It != BreakPhiNodesCache.end())
1705 return It->second;
1706
1707 // We consider PHI nodes as part of "chains", so given a PHI node I, we
1708 // recursively consider all its users and incoming values that are also PHI
1709 // nodes. We then make a decision about all of those PHIs at once. Either they
1710 // all get broken up, or none of them do. That way, we avoid cases where a
1711 // single PHI is/is not broken and we end up reforming/exploding a vector
1712 // multiple times, or even worse, doing it in a loop.
1713 SmallPtrSet<const PHINode *, 8> WorkList;
1714 collectPHINodes(I, SeenPHIs&: WorkList);
1715
1716#ifndef NDEBUG
1717 // Check that none of the PHI nodes in the worklist are in the map. If some of
1718 // them are, it means we're not good enough at collecting related PHIs.
1719 for (const PHINode *WLP : WorkList) {
1720 assert(BreakPhiNodesCache.count(WLP) == 0);
1721 }
1722#endif
1723
1724 // To consider a PHI profitable to break, we need to see some interesting
1725 // incoming values. At least 2/3rd (rounded up) of all PHIs in the worklist
1726 // must have one to consider all PHIs breakable.
1727 //
1728 // This threshold has been determined through performance testing.
1729 //
1730 // Note that the computation below is equivalent to
1731 //
1732 // (unsigned)ceil((K / 3.0) * 2)
1733 //
1734 // It's simply written this way to avoid mixing integral/FP arithmetic.
1735 const auto Threshold = (alignTo(Value: WorkList.size() * 2, Align: 3) / 3);
1736 unsigned NumBreakablePHIs = 0;
1737 bool CanBreak = false;
1738 for (const PHINode *Cur : WorkList) {
1739 // Don't break PHIs that have no interesting incoming values. That is, where
1740 // there is no clear opportunity to fold the "extractelement" instructions
1741 // we would add.
1742 //
1743 // Note: IC does not run after this pass, so we're only interested in the
1744 // foldings that the DAG combiner can do.
1745 if (any_of(Range: Cur->incoming_values(), P: isInterestingPHIIncomingValue)) {
1746 if (++NumBreakablePHIs >= Threshold) {
1747 CanBreak = true;
1748 break;
1749 }
1750 }
1751 }
1752
1753 for (const PHINode *Cur : WorkList)
1754 BreakPhiNodesCache[Cur] = CanBreak;
1755
1756 return CanBreak;
1757}
1758
1759/// Helper class for "break large PHIs" (visitPHINode).
1760///
1761/// This represents a slice of a PHI's incoming value, which is made up of:
1762/// - The type of the slice (Ty)
1763/// - The index in the incoming value's vector where the slice starts (Idx)
1764/// - The number of elements in the slice (NumElts).
1765/// It also keeps track of the NewPHI node inserted for this particular slice.
1766///
1767/// Slice examples:
1768/// <4 x i64> -> Split into four i64 slices.
1769/// -> [i64, 0, 1], [i64, 1, 1], [i64, 2, 1], [i64, 3, 1]
1770/// <5 x i16> -> Split into 2 <2 x i16> slices + a i16 tail.
1771/// -> [<2 x i16>, 0, 2], [<2 x i16>, 2, 2], [i16, 4, 1]
1772class VectorSlice {
1773public:
1774 VectorSlice(Type *Ty, unsigned Idx, unsigned NumElts)
1775 : Ty(Ty), Idx(Idx), NumElts(NumElts) {}
1776
1777 Type *Ty = nullptr;
1778 unsigned Idx = 0;
1779 unsigned NumElts = 0;
1780 PHINode *NewPHI = nullptr;
1781
1782 /// Slice \p Inc according to the information contained within this slice.
1783 /// This is cached, so if called multiple times for the same \p BB & \p Inc
1784 /// pair, it returns the same Sliced value as well.
1785 ///
1786 /// Note this *intentionally* does not return the same value for, say,
1787 /// [%bb.0, %0] & [%bb.1, %0] as:
1788 /// - It could cause issues with dominance (e.g. if bb.1 is seen first, then
1789 /// the value in bb.1 may not be reachable from bb.0 if it's its
1790 /// predecessor.)
1791 /// - We also want to make our extract instructions as local as possible so
1792 /// the DAG has better chances of folding them out. Duplicating them like
1793 /// that is beneficial in that regard.
1794 ///
1795 /// This is both a minor optimization to avoid creating duplicate
1796 /// instructions, but also a requirement for correctness. It is not forbidden
1797 /// for a PHI node to have the same [BB, Val] pair multiple times. If we
1798 /// returned a new value each time, those previously identical pairs would all
1799 /// have different incoming values (from the same block) and it'd cause a "PHI
1800 /// node has multiple entries for the same basic block with different incoming
1801 /// values!" verifier error.
1802 Value *getSlicedVal(BasicBlock *BB, Value *Inc, StringRef NewValName) {
1803 Value *&Res = SlicedVals[{BB, Inc}];
1804 if (Res)
1805 return Res;
1806
1807 IRBuilder<> B(BB->getTerminator());
1808 if (Instruction *IncInst = dyn_cast<Instruction>(Val: Inc))
1809 B.SetCurrentDebugLocation(IncInst->getDebugLoc());
1810
1811 if (NumElts > 1) {
1812 SmallVector<int, 4> Mask;
1813 for (unsigned K = Idx; K < (Idx + NumElts); ++K)
1814 Mask.push_back(Elt: K);
1815 Res = B.CreateShuffleVector(V: Inc, Mask, Name: NewValName);
1816 } else
1817 Res = B.CreateExtractElement(Vec: Inc, Idx, Name: NewValName);
1818
1819 return Res;
1820 }
1821
1822private:
1823 SmallDenseMap<std::pair<BasicBlock *, Value *>, Value *> SlicedVals;
1824};
1825
1826bool AMDGPUCodeGenPrepareImpl::visitPHINode(PHINode &I) {
1827 // Break-up fixed-vector PHIs into smaller pieces.
1828 // Default threshold is 32, so it breaks up any vector that's >32 bits into
1829 // its elements, or into 32-bit pieces (for 8/16 bit elts).
1830 //
1831 // This is only helpful for DAGISel because it doesn't handle large PHIs as
1832 // well as GlobalISel. DAGISel lowers PHIs by using CopyToReg/CopyFromReg.
1833 // With large, odd-sized PHIs we may end up needing many `build_vector`
1834 // operations with most elements being "undef". This inhibits a lot of
1835 // optimization opportunities and can result in unreasonably high register
1836 // pressure and the inevitable stack spilling.
1837 if (!CLOpts.codegenprepare_break_large_phis ||
1838 getCGPassBuilderOption(TM).EnableGlobalISelOption ==
1839 cl::boolOrDefault::BOU_TRUE)
1840 return false;
1841
1842 FixedVectorType *FVT = dyn_cast<FixedVectorType>(Val: I.getType());
1843 if (!FVT || FVT->getNumElements() == 1 ||
1844 DL.getTypeSizeInBits(Ty: FVT) <=
1845 CLOpts.codegenprepare_break_large_phis_threshold)
1846 return false;
1847
1848 if (!CLOpts.codegenprepare_force_break_large_phis && !canBreakPHINode(I))
1849 return false;
1850
1851 std::vector<VectorSlice> Slices;
1852
1853 Type *EltTy = FVT->getElementType();
1854 {
1855 unsigned Idx = 0;
1856 // For 8/16 bits type, don't scalarize fully but break it up into as many
1857 // 32-bit slices as we can, and scalarize the tail.
1858 const unsigned EltSize = DL.getTypeSizeInBits(Ty: EltTy);
1859 const unsigned NumElts = FVT->getNumElements();
1860 if (EltSize == 8 || EltSize == 16) {
1861 const unsigned SubVecSize = (32 / EltSize);
1862 Type *SubVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: SubVecSize);
1863 for (unsigned End = alignDown(Value: NumElts, Align: SubVecSize); Idx < End;
1864 Idx += SubVecSize)
1865 Slices.emplace_back(args&: SubVecTy, args&: Idx, args: SubVecSize);
1866 }
1867
1868 // Scalarize all remaining elements.
1869 for (; Idx < NumElts; ++Idx)
1870 Slices.emplace_back(args&: EltTy, args&: Idx, args: 1);
1871 }
1872
1873 assert(Slices.size() > 1);
1874
1875 // Create one PHI per vector piece. The "VectorSlice" class takes care of
1876 // creating the necessary instruction to extract the relevant slices of each
1877 // incoming value.
1878 IRBuilder<> B(I.getParent());
1879 B.SetCurrentDebugLocation(I.getDebugLoc());
1880
1881 unsigned IncNameSuffix = 0;
1882 for (VectorSlice &S : Slices) {
1883 // We need to reset the build on each iteration, because getSlicedVal may
1884 // have inserted something into I's BB.
1885 B.SetInsertPoint(I.getParent()->getFirstNonPHIIt());
1886 S.NewPHI = B.CreatePHI(Ty: S.Ty, NumReservedValues: I.getNumIncomingValues());
1887
1888 for (const auto &[Idx, BB] : enumerate(First: I.blocks())) {
1889 S.NewPHI->addIncoming(V: S.getSlicedVal(BB, Inc: I.getIncomingValue(i: Idx),
1890 NewValName: "largephi.extractslice" +
1891 std::to_string(val: IncNameSuffix++)),
1892 BB);
1893 }
1894 }
1895
1896 // And replace this PHI with a vector of all the previous PHI values.
1897 Value *Vec = PoisonValue::get(T: FVT);
1898 unsigned NameSuffix = 0;
1899 for (VectorSlice &S : Slices) {
1900 const auto ValName = "largephi.insertslice" + std::to_string(val: NameSuffix++);
1901 if (S.NumElts > 1)
1902 Vec = B.CreateInsertVector(DstType: FVT, SrcVec: Vec, SubVec: S.NewPHI, Idx: S.Idx, Name: ValName);
1903 else
1904 Vec = B.CreateInsertElement(Vec, NewElt: S.NewPHI, Idx: S.Idx, Name: ValName);
1905 }
1906
1907 I.replaceAllUsesWith(V: Vec);
1908 DeadVals.push_back(Elt: &I);
1909 return true;
1910}
1911
1912/// \param V Value to check
1913/// \param DL DataLayout
1914/// \param TM TargetMachine (TODO: remove once DL contains nullptr values)
1915/// \param AS Target Address Space
1916/// \return true if \p V cannot be the null value of \p AS, false otherwise.
1917static bool isPtrKnownNeverNull(const Value *V, const DataLayout &DL,
1918 const AMDGPUTargetMachine &TM, unsigned AS) {
1919 // Pointer cannot be null if it's a block address, GV or alloca.
1920 // NOTE: We don't support extern_weak, but if we did, we'd need to check for
1921 // it as the symbol could be null in such cases.
1922 if (isa<BlockAddress, GlobalValue, AllocaInst>(Val: V))
1923 return true;
1924
1925 // Check nonnull arguments.
1926 if (const auto *Arg = dyn_cast<Argument>(Val: V); Arg && Arg->hasNonNullAttr())
1927 return true;
1928
1929 // Check nonnull loads.
1930 if (const auto *Load = dyn_cast<LoadInst>(Val: V);
1931 Load && Load->hasMetadata(KindID: LLVMContext::MD_nonnull))
1932 return true;
1933
1934 // getUnderlyingObject may have looked through another addrspacecast, although
1935 // the optimizable situations most likely folded out by now.
1936 if (AS != cast<PointerType>(Val: V->getType())->getAddressSpace())
1937 return false;
1938
1939 // TODO: Calls that return nonnull?
1940
1941 // For all other things, use KnownBits.
1942 // We either use 0 or all bits set to indicate null, so check whether the
1943 // value can be zero or all ones.
1944 //
1945 // TODO: Use ValueTracking's isKnownNeverNull if it becomes aware that some
1946 // address spaces have non-zero null values.
1947 auto SrcPtrKB = computeKnownBits(V, DL);
1948 const auto NullVal = AMDGPU::getNullPointerValue(AS);
1949
1950 assert(SrcPtrKB.getBitWidth() == DL.getPointerSizeInBits(AS));
1951 assert((NullVal == 0 || NullVal == -1) &&
1952 "don't know how to check for this null value!");
1953 return NullVal ? !SrcPtrKB.getMaxValue().isAllOnes() : SrcPtrKB.isNonZero();
1954}
1955
1956bool AMDGPUCodeGenPrepareImpl::visitAddrSpaceCastInst(AddrSpaceCastInst &I) {
1957 // TODO: This is target-independent reasoning about the source pointer being
1958 // non-null, and would fit better in a generic pass such as
1959 // AggressiveInstCombine. It lives here for now because proving the source is
1960 // not the null value requires knowing the numeric null pointer value of the
1961 // source address space, which is not yet a first-class IR concept.
1962
1963 // If the flag is already set there is nothing to do.
1964 if (I.hasNonNull())
1965 return false;
1966
1967 // It is often difficult to prove that a vector of pointers cannot have any
1968 // nulls in it, so it's unclear if it's worth supporting.
1969 if (I.getType()->isVectorTy())
1970 return false;
1971
1972 // The nonnull flag only affects the lowering of casts from/to priv/local to
1973 // flat, so only bother proving non-null for those.
1974 const unsigned SrcAS = I.getSrcAddressSpace();
1975 const unsigned DstAS = I.getDestAddressSpace();
1976
1977 bool CanLower = false;
1978 if (SrcAS == AMDGPUAS::FLAT_ADDRESS)
1979 CanLower = (DstAS == AMDGPUAS::LOCAL_ADDRESS ||
1980 DstAS == AMDGPUAS::PRIVATE_ADDRESS);
1981 else if (DstAS == AMDGPUAS::FLAT_ADDRESS)
1982 CanLower = (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
1983 SrcAS == AMDGPUAS::PRIVATE_ADDRESS);
1984 if (!CanLower)
1985 return false;
1986
1987 SmallVector<const Value *, 4> WorkList;
1988 getUnderlyingObjects(V: I.getOperand(i_nocapture: 0), Objects&: WorkList);
1989 if (!all_of(Range&: WorkList, P: [&](const Value *V) {
1990 return isPtrKnownNeverNull(V, DL, TM, AS: SrcAS);
1991 }))
1992 return false;
1993
1994 I.setNonNull();
1995 return true;
1996}
1997
1998bool AMDGPUCodeGenPrepareImpl::visitIntrinsicInst(IntrinsicInst &I) {
1999 Intrinsic::ID IID = I.getIntrinsicID();
2000 switch (IID) {
2001 case Intrinsic::minnum:
2002 case Intrinsic::minimumnum:
2003 case Intrinsic::minimum:
2004 return visitFMinLike(I);
2005 case Intrinsic::sqrt:
2006 return visitSqrt(I);
2007 case Intrinsic::log:
2008 case Intrinsic::log10:
2009 return visitLog(Log&: cast<FPMathOperator>(Val&: I), IID);
2010 case Intrinsic::log2:
2011 // No reason to handle log2.
2012 return false;
2013 case Intrinsic::amdgcn_mbcnt_lo:
2014 return visitMbcntLo(I);
2015 case Intrinsic::amdgcn_mbcnt_hi:
2016 return visitMbcntHi(I);
2017 case Intrinsic::vector_reduce_add:
2018 return visitVectorReduceAdd(I);
2019 case Intrinsic::uadd_sat:
2020 case Intrinsic::sadd_sat:
2021 return visitSaturatingAdd(I);
2022 default:
2023 return false;
2024 }
2025}
2026
2027/// Match the core sequence in the fract pattern (x - floor(x), which doesn't
2028/// need to consider edge case handling.
2029Value *AMDGPUCodeGenPrepareImpl::matchFractPatImpl(Value &FractSrc,
2030 const APFloat &C) const {
2031 if (ST.hasFractBug())
2032 return nullptr;
2033
2034 Type *Ty = FractSrc.getType();
2035 if (!isLegalFloatingTy(Ty: Ty->getScalarType()))
2036 return nullptr;
2037
2038 APFloat OneNextDown = APFloat::getOne(Sem: C.getSemantics());
2039 OneNextDown.next(nextDown: true);
2040
2041 // Match nextafter(1.0, -1)
2042 if (OneNextDown != C)
2043 return nullptr;
2044
2045 Value *FloorSrc;
2046 if (match(V: &FractSrc, P: m_FSub(L: m_Value(V&: FloorSrc), R: m_Intrinsic<Intrinsic::floor>(
2047 Ops: m_Deferred(V: FloorSrc)))))
2048 return FloorSrc;
2049 return nullptr;
2050}
2051
2052/// Match non-nan fract pattern.
2053// MIN_CONSTANT = nextafter(1.0, -1.0)
2054/// minnum(fsub(x, floor(x)), MIN_CONSTANT)
2055/// minimumnum(fsub(x, floor(x)), MIN_CONSTANT)
2056/// minimum(fsub(x, floor(x)), MIN_CONSTANT)
2057
2058// x_sub_floor >= MIN_CONSTANT ? MIN_CONSTANT : x_sub_floor;
2059///
2060/// If fract is a useful instruction for the subtarget. Does not account for the
2061/// nan handling; the instruction has a nan check on the input value.
2062Value *AMDGPUCodeGenPrepareImpl::matchFractPatNanAvoidant(Value &V) {
2063 Value *Arg0;
2064 const APFloat *C;
2065
2066 // The value is only used in contexts where we know the input isn't a nan, so
2067 // any of the fmin variants are fine.
2068 if (!match(V: &V,
2069 P: m_CombineOr(Ps: m_FMinNum_or_FMinimumNum(Op0: m_Value(V&: Arg0),
2070 Op1: m_APFloatAllowPoison(Res&: C)),
2071 Ps: m_FMinimum(Op0: m_Value(V&: Arg0), Op1: m_APFloatAllowPoison(Res&: C)))))
2072 return nullptr;
2073
2074 return matchFractPatImpl(FractSrc&: *Arg0, C: *C);
2075}
2076
2077Value *AMDGPUCodeGenPrepareImpl::applyFractPat(IRBuilder<> &Builder,
2078 Value *FractArg) {
2079 SmallVector<Value *, 4> FractVals;
2080 extractValues(Builder, Values&: FractVals, V: FractArg);
2081
2082 SmallVector<Value *, 4> ResultVals(FractVals.size());
2083
2084 Type *Ty = FractArg->getType()->getScalarType();
2085 for (unsigned I = 0, E = FractVals.size(); I != E; ++I) {
2086 ResultVals[I] =
2087 Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_fract, OverloadTypes: {Ty}, Args: {FractVals[I]});
2088 }
2089
2090 return insertValues(Builder, Ty: FractArg->getType(), Values&: ResultVals);
2091}
2092
2093bool AMDGPUCodeGenPrepareImpl::visitFMinLike(IntrinsicInst &I) {
2094 const APFloat *C;
2095 Value *FractArg;
2096
2097 // minimum(x - floor(x), MIN_CONSTANT)
2098 Value *X;
2099 if (!ST.hasFractBug() &&
2100 match(V: &I, P: m_FMinimum(Op0: m_Value(V&: X), Op1: m_APFloatAllowPoison(Res&: C)))) {
2101 FractArg = matchFractPatImpl(FractSrc&: *X, C: *C);
2102 if (!FractArg)
2103 return false;
2104 } else {
2105 // minnum(x - floor(x), MIN_CONSTANT)
2106 FractArg = matchFractPatNanAvoidant(V&: I);
2107 if (!FractArg)
2108 return false;
2109
2110 // Match pattern for fract intrinsic in contexts where the nan check has
2111 // been optimized out (and hope the knowledge the source can't be nan wasn't
2112 // lost).
2113 if (!I.hasNoNaNs() && !isKnownNeverNaN(V: FractArg, SQ: SQ.getWithInstruction(I: &I)))
2114 return false;
2115 }
2116
2117 IRBuilder<> Builder(&I);
2118 FastMathFlags FMF = I.getFastMathFlags();
2119 FMF.setNoNaNs();
2120 Builder.setFastMathFlags(FMF);
2121
2122 Value *Fract = applyFractPat(Builder, FractArg);
2123 Fract->takeName(V: &I);
2124 I.replaceAllUsesWith(V: Fract);
2125 DeadVals.push_back(Elt: &I);
2126 return true;
2127}
2128
2129// Expand llvm.sqrt.f32 calls with !fpmath metadata in a semi-fast way.
2130bool AMDGPUCodeGenPrepareImpl::visitSqrt(IntrinsicInst &Sqrt) {
2131 Type *Ty = Sqrt.getType()->getScalarType();
2132 if (!Ty->isFloatTy())
2133 return false;
2134
2135 const FPMathOperator *FPOp = cast<const FPMathOperator>(Val: &Sqrt);
2136 FastMathFlags SqrtFMF = FPOp->getFastMathFlags();
2137
2138 // We're trying to handle the fast-but-not-that-fast case only. The lowering
2139 // of fast llvm.sqrt will give the raw instruction anyway.
2140 if (SqrtFMF.approxFunc())
2141 return false;
2142
2143 const float ReqdAccuracy = FPOp->getFPAccuracy();
2144
2145 // Defer correctly rounded expansion to codegen.
2146 if (ReqdAccuracy < 1.0f)
2147 return false;
2148
2149 Value *SrcVal = Sqrt.getOperand(i_nocapture: 0);
2150 bool CanTreatAsDAZ = canIgnoreDenormalInput(V: SrcVal, CtxI: &Sqrt);
2151
2152 // The raw instruction is 1 ulp, but the correction for denormal handling
2153 // brings it to 2.
2154 if (!CanTreatAsDAZ && ReqdAccuracy < 2.0f)
2155 return false;
2156
2157 IRBuilder<> Builder(&Sqrt);
2158 SmallVector<Value *, 4> SrcVals;
2159 extractValues(Builder, Values&: SrcVals, V: SrcVal);
2160
2161 SmallVector<Value *, 4> ResultVals(SrcVals.size());
2162 for (int I = 0, E = SrcVals.size(); I != E; ++I) {
2163 if (CanTreatAsDAZ)
2164 ResultVals[I] = Builder.CreateCall(Callee: getSqrtF32(), Args: SrcVals[I]);
2165 else
2166 ResultVals[I] = emitSqrtIEEE2ULP(Builder, Src: SrcVals[I], FMF: SqrtFMF);
2167 }
2168
2169 Value *NewSqrt = insertValues(Builder, Ty: Sqrt.getType(), Values&: ResultVals);
2170 NewSqrt->takeName(V: &Sqrt);
2171 Sqrt.replaceAllUsesWith(V: NewSqrt);
2172 DeadVals.push_back(Elt: &Sqrt);
2173 return true;
2174}
2175
2176/// Replace log and log10 intrinsic calls based on fpmath metadata.
2177bool AMDGPUCodeGenPrepareImpl::visitLog(FPMathOperator &Log,
2178 Intrinsic::ID IID) {
2179 Type *Ty = Log.getType();
2180 if (!Ty->getScalarType()->isHalfTy() || !ST.has16BitInsts())
2181 return false;
2182
2183 FastMathFlags FMF = Log.getFastMathFlags();
2184
2185 // Defer fast math cases to codegen.
2186 if (FMF.approxFunc())
2187 return false;
2188
2189 // Limit experimentally determined from OpenCL conformance test (1.79)
2190 if (Log.getFPAccuracy() < 1.80f)
2191 return false;
2192
2193 IRBuilder<> Builder(&cast<CallInst>(Val&: Log));
2194
2195 // Use the generic intrinsic for convenience in the vector case. Codegen will
2196 // recognize the denormal handling is not necessary from the fpext.
2197 // TODO: Move to generic code
2198 Value *Log2 =
2199 Builder.CreateUnaryIntrinsic(ID: Intrinsic::log2, Op: Log.getOperand(i: 0), FMFSource: FMF);
2200
2201 double Log2BaseInverted =
2202 IID == Intrinsic::log10 ? numbers::ln2 / numbers::ln10 : numbers::ln2;
2203 Value *Mul =
2204 Builder.CreateFMulFMF(L: Log2, R: ConstantFP::get(Ty, V: Log2BaseInverted), FMFSource: FMF);
2205
2206 Mul->takeName(V: &Log);
2207
2208 Log.replaceAllUsesWith(V: Mul);
2209 DeadVals.push_back(Elt: &Log);
2210 return true;
2211}
2212
2213bool AMDGPUCodeGenPrepare::runOnFunction(Function &F) {
2214 if (skipFunction(F))
2215 return false;
2216
2217 auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
2218 if (!TPC)
2219 return false;
2220
2221 const AMDGPUTargetMachine &TM = TPC->getTM<AMDGPUTargetMachine>();
2222 const TargetTransformInfo &TTI =
2223 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
2224 const TargetLibraryInfo *TLI =
2225 &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
2226 AssumptionCache *AC =
2227 &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
2228 auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
2229 const DominatorTree *DT = DTWP ? &DTWP->getDomTree() : nullptr;
2230 const UniformityInfo &UA =
2231 getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
2232 return AMDGPUCodeGenPrepareImpl(F, TM, TTI, TLI, AC, DT, UA, ExpandDiv64InIR)
2233 .run();
2234}
2235
2236PreservedAnalyses AMDGPUCodeGenPreparePass::run(Function &F,
2237 FunctionAnalysisManager &FAM) {
2238 const AMDGPUTargetMachine &ATM = static_cast<const AMDGPUTargetMachine &>(TM);
2239 const TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(IR&: F);
2240 const TargetLibraryInfo *TLI = &FAM.getResult<TargetLibraryAnalysis>(IR&: F);
2241 AssumptionCache *AC = &FAM.getResult<AssumptionAnalysis>(IR&: F);
2242 const DominatorTree *DT = FAM.getCachedResult<DominatorTreeAnalysis>(IR&: F);
2243 const UniformityInfo &UA = FAM.getResult<UniformityInfoAnalysis>(IR&: F);
2244 AMDGPUCodeGenPrepareImpl Impl(F, ATM, TTI, TLI, AC, DT, UA,
2245 ATM.getCLOpts().codegenprepare_expand_div64);
2246 if (!Impl.run())
2247 return PreservedAnalyses::all();
2248 PreservedAnalyses PA = PreservedAnalyses::none();
2249 if (!Impl.FlowChanged)
2250 PA.preserveSet<CFGAnalyses>();
2251 return PA;
2252}
2253
2254INITIALIZE_PASS_BEGIN(AMDGPUCodeGenPrepare, DEBUG_TYPE,
2255 "AMDGPU IR optimizations", false, false)
2256INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
2257INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
2258INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
2259INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
2260INITIALIZE_PASS_END(AMDGPUCodeGenPrepare, DEBUG_TYPE, "AMDGPU IR optimizations",
2261 false, false)
2262
2263/// Create a workitem.id.x intrinsic call with range metadata.
2264CallInst *AMDGPUCodeGenPrepareImpl::createWorkitemIdX(IRBuilder<> &B) const {
2265 CallInst *Tid =
2266 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::amdgcn_workitem_id_x, Args: {});
2267 ST.makeLIDRangeMetadata(I: Tid);
2268 return Tid;
2269}
2270
2271/// Replace the instruction with a direct workitem.id.x call.
2272void AMDGPUCodeGenPrepareImpl::replaceWithWorkitemIdX(Instruction &I) const {
2273 IRBuilder<> B(&I);
2274 CallInst *Tid = createWorkitemIdX(B);
2275 BasicBlock::iterator BI(&I);
2276 ReplaceInstWithValue(BI, V: Tid);
2277}
2278
2279/// Replace the instruction with (workitem.id.x & mask).
2280void AMDGPUCodeGenPrepareImpl::replaceWithMaskedWorkitemIdX(
2281 Instruction &I, unsigned WaveSize) const {
2282 IRBuilder<> B(&I);
2283 CallInst *Tid = createWorkitemIdX(B);
2284 Constant *Mask = ConstantInt::get(Ty: Tid->getType(), V: WaveSize - 1);
2285 Value *AndInst = B.CreateAnd(LHS: Tid, RHS: Mask);
2286 BasicBlock::iterator BI(&I);
2287 ReplaceInstWithValue(BI, V: AndInst);
2288}
2289
2290/// Try to optimize mbcnt instruction by replacing with workitem.id.x when
2291/// work group size allows direct computation of lane ID.
2292/// Returns true if optimization was applied, false otherwise.
2293bool AMDGPUCodeGenPrepareImpl::tryReplaceWithWorkitemId(Instruction &I,
2294 unsigned Wave) const {
2295 std::optional<unsigned> MaybeX = ST.getReqdWorkGroupSize(F, Dim: 0);
2296 if (!MaybeX)
2297 return false;
2298
2299 // When work group size == wave_size, each work group contains exactly one
2300 // wave, so the instruction can be replaced with workitem.id.x directly.
2301 if (*MaybeX == Wave) {
2302 replaceWithWorkitemIdX(I);
2303 return true;
2304 }
2305
2306 // When work group evenly splits into waves, compute lane ID within wave
2307 // using bit masking: lane_id = workitem.id.x & (wave_size - 1).
2308 if (ST.hasWavefrontsEvenlySplittingXDim(F, /*RequiresUniformYZ=*/true)) {
2309 replaceWithMaskedWorkitemIdX(I, WaveSize: Wave);
2310 return true;
2311 }
2312
2313 return false;
2314}
2315
2316/// Optimize mbcnt.lo calls on wave32 architectures for lane ID computation.
2317bool AMDGPUCodeGenPrepareImpl::visitMbcntLo(IntrinsicInst &I) const {
2318 // This optimization only applies to wave32 targets where mbcnt.lo operates on
2319 // the full execution mask.
2320 if (!ST.isWave32())
2321 return false;
2322
2323 // Only optimize the pattern mbcnt.lo(~0, 0) which counts active lanes with
2324 // lower IDs.
2325 if (!match(V: &I,
2326 P: m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(Ops: m_AllOnes(), Ops: m_Zero())))
2327 return false;
2328
2329 return tryReplaceWithWorkitemId(I, Wave: ST.getWavefrontSize());
2330}
2331
2332/// Optimize mbcnt.hi calls for lane ID computation.
2333bool AMDGPUCodeGenPrepareImpl::visitMbcntHi(IntrinsicInst &I) const {
2334 // Abort if wave size is not known at compile time.
2335 if (!ST.isWaveSizeKnown())
2336 return false;
2337
2338 unsigned Wave = ST.getWavefrontSize();
2339
2340 // On wave32, the upper 32 bits of execution mask are always 0, so
2341 // mbcnt.hi(mask, val) always returns val unchanged.
2342 if (ST.isWave32()) {
2343 BasicBlock::iterator BI(&I);
2344 ReplaceInstWithValue(BI, V: I.getArgOperand(i: 1));
2345 return true;
2346 }
2347
2348 // Optimize the complete lane ID computation pattern:
2349 // mbcnt.hi(~0, mbcnt.lo(~0, 0)) which counts all active lanes with lower IDs
2350 // across the full execution mask.
2351 using namespace PatternMatch;
2352
2353 // Check for pattern: mbcnt.hi(~0, mbcnt.lo(~0, 0))
2354 if (!match(V: &I, P: m_Intrinsic<Intrinsic::amdgcn_mbcnt_hi>(
2355 Ops: m_AllOnes(), Ops: m_Intrinsic<Intrinsic::amdgcn_mbcnt_lo>(
2356 Ops: m_AllOnes(), Ops: m_Zero()))))
2357 return false;
2358
2359 return tryReplaceWithWorkitemId(I, Wave);
2360}
2361
2362/// Check if type is <4 x i8>.
2363static bool isV4I8(Type *Ty) {
2364 FixedVectorType *VTy = dyn_cast<FixedVectorType>(Val: Ty);
2365 return VTy && VTy->getNumElements() == 4 &&
2366 VTy->getElementType()->isIntegerTy(BitWidth: 8);
2367}
2368
2369/// Helper to match the dot4 pattern: mul(zext/sext <4 x i8>, zext/sext <4 x
2370/// i8>) Returns true if pattern matches and signedness matches IsSigned.
2371/// Sets A, B to the <4 x i8> sources.
2372static bool matchDot4Pattern(Value *MulOp, Value *&A, Value *&B,
2373 bool IsSigned) {
2374 Value *Src0, *Src1;
2375 if (!match(V: MulOp, P: m_Mul(L: m_Value(V&: Src0), R: m_Value(V&: Src1))))
2376 return false;
2377
2378 // Check that result type is <4 x i32>
2379 FixedVectorType *MulTy = dyn_cast<FixedVectorType>(Val: MulOp->getType());
2380 if (!MulTy || MulTy->getNumElements() != 4 ||
2381 !MulTy->getElementType()->isIntegerTy(BitWidth: 32))
2382 return false;
2383
2384 // Match zext or sext based on IsSigned
2385 Value *ExtSrc0, *ExtSrc1;
2386 if (IsSigned) {
2387 if (!match(V: Src0, P: m_SExt(Op: m_Value(V&: ExtSrc0))) || !isV4I8(Ty: ExtSrc0->getType()))
2388 return false;
2389 if (!match(V: Src1, P: m_SExt(Op: m_Value(V&: ExtSrc1))) || !isV4I8(Ty: ExtSrc1->getType()))
2390 return false;
2391 } else {
2392 if (!match(V: Src0, P: m_ZExt(Op: m_Value(V&: ExtSrc0))) || !isV4I8(Ty: ExtSrc0->getType()))
2393 return false;
2394 if (!match(V: Src1, P: m_ZExt(Op: m_Value(V&: ExtSrc1))) || !isV4I8(Ty: ExtSrc1->getType()))
2395 return false;
2396 }
2397
2398 A = ExtSrc0;
2399 B = ExtSrc1;
2400 return true;
2401}
2402
2403/// Try to convert vector.reduce.add(mul(zext/sext <4 x i8>, zext/sext <4 x
2404/// i8>)) to a dot4 intrinsic call (non-saturating case only).
2405bool AMDGPUCodeGenPrepareImpl::visitVectorReduceAdd(IntrinsicInst &I) {
2406 // Check if we have dot4 instructions available
2407 if (!ST.hasDot7Insts() || (!ST.hasDot1Insts() && !ST.hasDot8Insts()))
2408 return false;
2409
2410 Value *A = nullptr, *B = nullptr;
2411
2412 // Try unsigned first, then signed
2413 bool IsSigned = false;
2414 if (!matchDot4Pattern(MulOp: I.getArgOperand(i: 0), A, B, /*IsSigned=*/false)) {
2415 if (!matchDot4Pattern(MulOp: I.getArgOperand(i: 0), A, B, /*IsSigned=*/true))
2416 return false;
2417 IsSigned = true;
2418 }
2419
2420 LLVMContext &Ctx = I.getContext();
2421 Type *I32Ty = Type::getInt32Ty(C&: Ctx);
2422 IRBuilder<> Builder(&I);
2423
2424 // Bitcast <4 x i8> to i32
2425 Value *ASrc = Builder.CreateBitCast(V: A, DestTy: I32Ty);
2426 Value *BSrc = Builder.CreateBitCast(V: B, DestTy: I32Ty);
2427
2428 // Non-saturating case: accumulator is 0, clamp is false
2429 Value *Acc = ConstantInt::get(Ty: I32Ty, V: 0);
2430 Value *Clamp = ConstantInt::getFalse(Context&: Ctx);
2431
2432 Intrinsic::ID DotIID =
2433 IsSigned ? Intrinsic::amdgcn_sdot4 : Intrinsic::amdgcn_udot4;
2434
2435 Value *Dot = Builder.CreateIntrinsic(ID: DotIID, OverloadTypes: {}, Args: {ASrc, BSrc, Acc, Clamp});
2436 Dot->takeName(V: &I);
2437
2438 I.replaceAllUsesWith(V: Dot);
2439 DeadVals.push_back(Elt: &I);
2440
2441 return true;
2442}
2443
2444/// Try to convert uadd.sat/sadd.sat(vector.reduce.add(mul(...)), c) to a
2445/// saturating dot4 intrinsic. This combine starts at the root (saturating add)
2446/// and looks at its operands.
2447bool AMDGPUCodeGenPrepareImpl::visitSaturatingAdd(IntrinsicInst &I) {
2448 // Check if we have dot4 instructions available
2449 if (!ST.hasDot7Insts() || (!ST.hasDot1Insts() && !ST.hasDot8Insts()))
2450 return false;
2451
2452 Intrinsic::ID IID = I.getIntrinsicID();
2453 bool IsSigned = (IID == Intrinsic::sadd_sat);
2454
2455 // Look for vector.reduce.add as one of the operands (commutative match)
2456 Value *Op0 = I.getArgOperand(i: 0);
2457 Value *Op1 = I.getArgOperand(i: 1);
2458 Value *MulOp = nullptr;
2459 Value *Accum = nullptr;
2460 IntrinsicInst *ReduceInst = nullptr;
2461
2462 if (match(V: Op0, P: m_Intrinsic<Intrinsic::vector_reduce_add>(Ops: m_Value(V&: MulOp)))) {
2463 ReduceInst = cast<IntrinsicInst>(Val: Op0);
2464 Accum = Op1;
2465 } else if (match(V: Op1,
2466 P: m_Intrinsic<Intrinsic::vector_reduce_add>(Ops: m_Value(V&: MulOp)))) {
2467 ReduceInst = cast<IntrinsicInst>(Val: Op1);
2468 Accum = Op0;
2469 } else {
2470 return false;
2471 }
2472
2473 Value *A = nullptr, *B = nullptr;
2474
2475 if (!matchDot4Pattern(MulOp, A, B, IsSigned))
2476 return false;
2477
2478 LLVMContext &Ctx = I.getContext();
2479 Type *I32Ty = Type::getInt32Ty(C&: Ctx);
2480 IRBuilder<> Builder(&I);
2481
2482 // Bitcast <4 x i8> to i32
2483 Value *ASrc = Builder.CreateBitCast(V: A, DestTy: I32Ty);
2484 Value *BSrc = Builder.CreateBitCast(V: B, DestTy: I32Ty);
2485
2486 // Saturating case: use the accumulator and set clamp to true
2487 Value *Clamp = ConstantInt::getTrue(Context&: Ctx);
2488
2489 Intrinsic::ID DotIID =
2490 IsSigned ? Intrinsic::amdgcn_sdot4 : Intrinsic::amdgcn_udot4;
2491
2492 Value *Dot = Builder.CreateIntrinsic(ID: DotIID, OverloadTypes: {}, Args: {ASrc, BSrc, Accum, Clamp});
2493 Dot->takeName(V: &I);
2494
2495 I.replaceAllUsesWith(V: Dot);
2496 DeadVals.push_back(Elt: &I);
2497 // The reduce.add will be dead after this and cleaned up later
2498 if (ReduceInst->use_empty())
2499 DeadVals.push_back(Elt: ReduceInst);
2500
2501 return true;
2502}
2503
2504char AMDGPUCodeGenPrepare::ID = 0;
2505
2506FunctionPass *llvm::createAMDGPUCodeGenPreparePass(bool ExpandDiv64InIR) {
2507 return new AMDGPUCodeGenPrepare(ExpandDiv64InIR);
2508}
2509