1//===- AMDGPUTargetTransformInfo.h - AMDGPU specific TTI --------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file a TargetTransformInfoImplBase conforming object specific to the
11/// AMDGPU target machine. It uses the target's detailed information to
12/// provide more precise answers to certain TTI queries, while letting the
13/// target independent and default TTI implementations handle the rest.
14//
15//===----------------------------------------------------------------------===//
16
17#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETTRANSFORMINFO_H
18#define LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETTRANSFORMINFO_H
19
20#include "AMDGPU.h"
21#include "llvm/CodeGen/BasicTTIImpl.h"
22#include "llvm/Support/AMDGPUAddrSpace.h"
23#include <optional>
24
25namespace llvm {
26
27class AMDGPUTargetMachine;
28class GCNSubtarget;
29class InstCombiner;
30class Loop;
31class ScalarEvolution;
32class SITargetLowering;
33class Type;
34class Value;
35
36class AMDGPUTTIImpl final : public BasicTTIImplBase<AMDGPUTTIImpl> {
37 using BaseT = BasicTTIImplBase<AMDGPUTTIImpl>;
38 using TTI = TargetTransformInfo;
39
40 friend BaseT;
41
42 Triple TargetTriple;
43
44 const TargetSubtargetInfo *ST;
45 const TargetLoweringBase *TLI;
46
47 const TargetSubtargetInfo *getST() const { return ST; }
48 const TargetLoweringBase *getTLI() const { return TLI; }
49
50public:
51 explicit AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const Function &F);
52
53 void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
54 TTI::UnrollingPreferences &UP,
55 OptimizationRemarkEmitter *ORE) const override;
56
57 void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
58 TTI::PeelingPreferences &PP) const override;
59
60 uint64_t getMaxMemIntrinsicInlineSizeThreshold() const override;
61};
62
63class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
64 using BaseT = BasicTTIImplBase<GCNTTIImpl>;
65 using TTI = TargetTransformInfo;
66
67 friend BaseT;
68
69 const GCNSubtarget *ST;
70 const SITargetLowering *TLI;
71 AMDGPUTTIImpl CommonTTI;
72 bool IsGraphics;
73 bool HasFP32Denormals;
74 static constexpr bool InlinerVectorBonusPercent = 0;
75
76 static const FeatureBitset InlineFeatureIgnoreList;
77
78 const GCNSubtarget *getST() const { return ST; }
79 const SITargetLowering *getTLI() const { return TLI; }
80
81 static inline int getFullRateInstrCost() {
82 return TargetTransformInfo::TCC_Basic;
83 }
84
85 static inline int getHalfRateInstrCost(TTI::TargetCostKind CostKind) {
86 return CostKind == TTI::TCK_CodeSize ? 2
87 : 2 * TargetTransformInfo::TCC_Basic;
88 }
89
90 // TODO: The size is usually 8 bytes, but takes 4x as many cycles. Maybe
91 // should be 2 or 4.
92 static inline int getQuarterRateInstrCost(TTI::TargetCostKind CostKind) {
93 return CostKind == TTI::TCK_CodeSize ? 2
94 : 4 * TargetTransformInfo::TCC_Basic;
95 }
96
97 int getTransInstrCost(TTI::TargetCostKind CostKind) const;
98
99 // On some parts, normal fp64 operations are half rate, and others
100 // quarter. This also applies to some integer operations.
101 int get64BitInstrCost(TTI::TargetCostKind CostKind) const;
102
103 std::pair<InstructionCost, MVT> getTypeLegalizationCost(Type *Ty) const;
104
105 /// \returns true if V might be divergent even when all of its operands
106 /// are uniform.
107 bool isSourceOfDivergence(const Value *V) const;
108
109 /// Returns true for the target specific set of operations which produce
110 /// uniform result even taking non-uniform arguments.
111 bool isAlwaysUniform(const Value *V) const;
112
113public:
114 explicit GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F);
115
116 bool hasBranchDivergence(const Function *F = nullptr) const override;
117
118 void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
119 TTI::UnrollingPreferences &UP,
120 OptimizationRemarkEmitter *ORE) const override;
121
122 void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
123 TTI::PeelingPreferences &PP) const override;
124
125 TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const override {
126 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
127 return TTI::PSK_FastHardware;
128 }
129
130 unsigned getNumberOfRegisters(unsigned RCID) const override;
131 TypeSize
132 getRegisterBitWidth(TargetTransformInfo::RegisterKind Vector) const override;
133 unsigned getMinVectorRegisterBitWidth() const override;
134 unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const override;
135 bool preferSLPInstCountCheck() const override;
136 unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize,
137 unsigned ChainSizeInBytes,
138 VectorType *VecTy) const override;
139 unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize,
140 unsigned ChainSizeInBytes,
141 VectorType *VecTy) const override;
142 unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const override;
143
144 bool isLegalToVectorizeMemChain(unsigned ChainSizeInBytes, Align Alignment,
145 unsigned AddrSpace) const;
146 bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, Align Alignment,
147 unsigned AddrSpace) const override;
148 bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment,
149 unsigned AddrSpace) const override;
150
151 uint64_t getMaxMemIntrinsicInlineSizeThreshold() const override;
152 Type *getMemcpyLoopLoweringType(
153 LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
154 unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
155 std::optional<uint32_t> AtomicElementSize) const override;
156
157 void getMemcpyLoopResidualLoweringType(
158 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
159 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
160 Align SrcAlign, Align DestAlign,
161 std::optional<uint32_t> AtomicCpySize) const override;
162 unsigned getMaxInterleaveFactor(ElementCount VF,
163 bool HasUnorderedReductions) const override;
164
165 bool getTgtMemIntrinsic(IntrinsicInst *Inst,
166 MemIntrinsicInfo &Info) const override;
167
168 InstructionCost getArithmeticInstrCost(
169 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
170 TTI::OperandValueInfo Op1Info = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
171 TTI::OperandValueInfo Op2Info = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
172 ArrayRef<const Value *> Args = {},
173 const Instruction *CtxI = nullptr) const override;
174
175 InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
176 const Instruction *I = nullptr) const override;
177
178 InstructionCost getCmpSelInstrCost(
179 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
180 TTI::TargetCostKind CostKind,
181 TTI::OperandValueInfo Op1Info = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
182 TTI::OperandValueInfo Op2Info = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
183 const Instruction *I = nullptr) const override;
184
185 bool isInlineAsmSourceOfDivergence(const CallInst *CI,
186 ArrayRef<unsigned> Indices = {}) const;
187
188 using BaseT::getVectorInstrCost;
189 InstructionCost
190 getVectorInstrCost(unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind,
191 unsigned Index, const Value *Op0, const Value *Op1,
192 TTI::VectorInstrContext VIC =
193 TTI::VectorInstrContext::None) const override;
194
195 bool isReadRegisterSourceOfDivergence(const IntrinsicInst *ReadReg) const;
196
197 bool isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const override {
198 // Address space casts must cast between different address spaces.
199 if (FromAS == ToAS)
200 return false;
201
202 // Casts between any aliasing address spaces are valid.
203 return AMDGPU::addrspacesMayAlias(AS1: FromAS, AS2: ToAS);
204 }
205
206 bool addrspacesMayAlias(unsigned AS0, unsigned AS1) const override {
207 return AMDGPU::addrspacesMayAlias(AS1: AS0, AS2: AS1);
208 }
209
210 unsigned getFlatAddressSpace() const override {
211 // Don't bother running InferAddressSpaces pass on graphics shaders which
212 // don't use flat addressing.
213 if (IsGraphics)
214 return -1;
215 return AMDGPUAS::FLAT_ADDRESS;
216 }
217
218 bool collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,
219 Intrinsic::ID IID) const override;
220
221 bool
222 canHaveNonUndefGlobalInitializerInAddressSpace(unsigned AS) const override {
223 return AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS &&
224 AS != AMDGPUAS::PRIVATE_ADDRESS;
225 }
226
227 Value *rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, Value *OldV,
228 Value *NewV) const override;
229
230 bool canSimplifyLegacyMulToMul(const Instruction &I, const Value *Op0,
231 const Value *Op1, InstCombiner &IC) const;
232
233 bool simplifyDemandedLaneMaskArg(InstCombiner &IC, IntrinsicInst &II,
234 unsigned LaneAgIdx) const;
235
236 std::optional<Instruction *>
237 instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const override;
238
239 Value *simplifyAMDGCNLaneIntrinsicDemanded(InstCombiner &IC,
240 IntrinsicInst &II,
241 const APInt &DemandedElts,
242 APInt &UndefElts) const;
243
244 Instruction *hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
245 IntrinsicInst &II) const;
246
247 std::optional<Value *> simplifyDemandedVectorEltsIntrinsic(
248 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
249 APInt &UndefElts2, APInt &UndefElts3,
250 std::function<void(Instruction *, unsigned, APInt, APInt &)>
251 SimplifyAndSetOp) const override;
252
253 InstructionCost getVectorSplitCost() const { return 0; }
254
255 InstructionCost
256 getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
257 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
258 VectorType *SubTp, ArrayRef<const Value *> Args = {},
259 const Instruction *CtxI = nullptr,
260 TTI::VectorInstrContext VIC =
261 TTI::VectorInstrContext::None) const override;
262
263 bool isProfitableToSinkOperands(Instruction *I,
264 SmallVectorImpl<Use *> &Ops) const override;
265
266 bool areInlineCompatible(const Function *Caller,
267 const Function *Callee) const override;
268
269 int getInliningLastCallToStaticBonus() const override;
270 unsigned getInliningThresholdMultiplier() const override { return 11; }
271 unsigned adjustInliningThreshold(const CallBase *CB) const override;
272 unsigned getCallerAllocaCost(const CallBase *CB,
273 const AllocaInst *AI) const override;
274
275 int getInlinerVectorBonusPercent() const override {
276 return InlinerVectorBonusPercent;
277 }
278
279 InstructionCost
280 getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
281 TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
282 const Instruction *I = nullptr) const override;
283
284 InstructionCost
285 getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
286 std::optional<FastMathFlags> FMF,
287 TTI::TargetCostKind CostKind) const override;
288
289 InstructionCost getPartialReductionCost(
290 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
291 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
292 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
293 TTI::TargetCostKind CostKind,
294 std::optional<FastMathFlags> FMF) const override {
295 return InstructionCost::getInvalid();
296 }
297
298 InstructionCost
299 getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
300 TTI::TargetCostKind CostKind) const override;
301 InstructionCost
302 getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,
303 TTI::TargetCostKind CostKind) const override;
304
305 /// Data cache line size for LoopDataPrefetch pass. Has no use before GFX12.
306 unsigned getCacheLineSize() const override;
307
308 /// How much before a load we should place the prefetch instruction.
309 /// This is currently measured in number of IR instructions.
310 unsigned getPrefetchDistance() const override;
311
312 /// \return if target want to issue a prefetch in address space \p AS.
313 bool shouldPrefetchAddressSpace(unsigned AS) const override;
314 void collectKernelLaunchBounds(
315 const Function &F,
316 SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const override;
317
318 enum class KnownIEEEMode { Unknown, On, Off };
319
320 /// Return KnownIEEEMode::On if we know if the use context can assume
321 /// "amdgpu-ieee"="true" and KnownIEEEMode::Off if we can assume
322 /// "amdgpu-ieee"="false".
323 KnownIEEEMode fpenvIEEEMode(const Instruction &I) const;
324
325 /// Account for loads of i8 vector types to have reduced cost. For
326 /// example the cost of load 4 i8s values is one is the cost of loading
327 /// a single i32 value.
328 InstructionCost getMemoryOpCost(
329 unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
330 TTI::TargetCostKind CostKind,
331 TTI::OperandValueInfo OpInfo = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
332 const Instruction *I = nullptr) const override;
333
334 /// When counting parts on AMD GPUs, account for i8s being grouped
335 /// together under a single i32 value. Otherwise fall back to base
336 /// implementation.
337 unsigned getNumberOfParts(Type *Tp) const override;
338
339 ValueUniformity getValueUniformity(const Value *V) const override;
340
341 InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
342 StackOffset BaseOffset, bool HasBaseReg,
343 int64_t Scale,
344 unsigned AddrSpace) const override;
345
346 bool isLSRCostLess(const TTI::LSRCost &A,
347 const TTI::LSRCost &B) const override;
348 bool isNumRegsMajorCostOfLSR() const override;
349 bool shouldDropLSRSolutionIfLessProfitable() const override;
350
351 bool isUniform(const Instruction *I,
352 const SmallBitVector &UniformArgs) const override;
353};
354
355} // end namespace llvm
356
357#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETTRANSFORMINFO_H
358