1//===- AMDGPUTargetTransformInfo.h - AMDGPU specific TTI --------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file a TargetTransformInfoImplBase conforming object specific to the
11/// AMDGPU target machine. It uses the target's detailed information to
12/// provide more precise answers to certain TTI queries, while letting the
13/// target independent and default TTI implementations handle the rest.
14//
15//===----------------------------------------------------------------------===//
16
17#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETTRANSFORMINFO_H
18#define LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETTRANSFORMINFO_H
19
20#include "AMDGPU.h"
21#include "llvm/CodeGen/BasicTTIImpl.h"
22#include "llvm/Support/AMDGPUAddrSpace.h"
23#include <optional>
24
25namespace llvm {
26
27class AMDGPUTargetMachine;
28class GCNSubtarget;
29class InstCombiner;
30class Loop;
31class ScalarEvolution;
32class SITargetLowering;
33class Type;
34class Value;
35
36class AMDGPUTTIImpl final : public BasicTTIImplBase<AMDGPUTTIImpl> {
37 using BaseT = BasicTTIImplBase<AMDGPUTTIImpl>;
38 using TTI = TargetTransformInfo;
39
40 friend BaseT;
41
42 Triple TargetTriple;
43
44 const TargetSubtargetInfo *ST;
45 const TargetLoweringBase *TLI;
46
47 const TargetSubtargetInfo *getST() const { return ST; }
48 const TargetLoweringBase *getTLI() const { return TLI; }
49
50public:
51 explicit AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const Function &F);
52
53 void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
54 TTI::UnrollingPreferences &UP,
55 OptimizationRemarkEmitter *ORE) const override;
56
57 void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
58 TTI::PeelingPreferences &PP) const override;
59
60 uint64_t getMaxMemIntrinsicInlineSizeThreshold() const override;
61};
62
63class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
64 using BaseT = BasicTTIImplBase<GCNTTIImpl>;
65 using TTI = TargetTransformInfo;
66
67 friend BaseT;
68
69 const GCNSubtarget *ST;
70 const SITargetLowering *TLI;
71 AMDGPUTTIImpl CommonTTI;
72 bool IsGraphics;
73 bool HasFP32Denormals;
74 static constexpr bool InlinerVectorBonusPercent = 0;
75
76 static const FeatureBitset InlineFeatureIgnoreList;
77
78 const GCNSubtarget *getST() const { return ST; }
79 const SITargetLowering *getTLI() const { return TLI; }
80
81 static inline int getFullRateInstrCost() {
82 return TargetTransformInfo::TCC_Basic;
83 }
84
85 static inline int getHalfRateInstrCost(TTI::TargetCostKind CostKind) {
86 return CostKind == TTI::TCK_CodeSize ? 2
87 : 2 * TargetTransformInfo::TCC_Basic;
88 }
89
90 // TODO: The size is usually 8 bytes, but takes 4x as many cycles. Maybe
91 // should be 2 or 4.
92 static inline int getQuarterRateInstrCost(TTI::TargetCostKind CostKind) {
93 return CostKind == TTI::TCK_CodeSize ? 2
94 : 4 * TargetTransformInfo::TCC_Basic;
95 }
96
97 int getTransInstrCost(TTI::TargetCostKind CostKind) const;
98
99 // On some parts, normal fp64 operations are half rate, and others
100 // quarter. This also applies to some integer operations.
101 int get64BitInstrCost(TTI::TargetCostKind CostKind) const;
102
103 std::pair<InstructionCost, MVT> getTypeLegalizationCost(Type *Ty) const;
104
105 /// \returns true if V might be divergent even when all of its operands
106 /// are uniform.
107 bool isSourceOfDivergence(const Value *V) const;
108
109 /// Returns true for the target specific set of operations which produce
110 /// uniform result even taking non-uniform arguments.
111 bool isAlwaysUniform(const Value *V) const;
112
113public:
114 explicit GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F);
115
116 bool hasBranchDivergence(const Function *F = nullptr) const override;
117
118 void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
119 TTI::UnrollingPreferences &UP,
120 OptimizationRemarkEmitter *ORE) const override;
121
122 void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
123 TTI::PeelingPreferences &PP) const override;
124
125 TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const override {
126 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
127 return TTI::PSK_FastHardware;
128 }
129
130 unsigned getNumberOfRegisters(unsigned RCID) const override;
131 TypeSize
132 getRegisterBitWidth(TargetTransformInfo::RegisterKind Vector) const override;
133 unsigned getMinVectorRegisterBitWidth() const override;
134 unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const override;
135 bool preferSLPInstCountCheck() const override;
136 unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize,
137 unsigned ChainSizeInBytes,
138 VectorType *VecTy) const override;
139 unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize,
140 unsigned ChainSizeInBytes,
141 VectorType *VecTy) const override;
142 unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const override;
143
144 bool isLegalToVectorizeMemChain(unsigned ChainSizeInBytes, Align Alignment,
145 unsigned AddrSpace) const;
146 bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, Align Alignment,
147 unsigned AddrSpace) const override;
148 bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment,
149 unsigned AddrSpace) const override;
150
151 uint64_t getMaxMemIntrinsicInlineSizeThreshold() const override;
152 Type *getMemcpyLoopLoweringType(
153 LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
154 unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
155 std::optional<uint32_t> AtomicElementSize) const override;
156
157 void getMemcpyLoopResidualLoweringType(
158 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
159 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
160 Align SrcAlign, Align DestAlign,
161 std::optional<uint32_t> AtomicCpySize) const override;
162 unsigned getMaxInterleaveFactor(ElementCount VF,
163 bool HasUnorderedReductions) const override;
164
165 bool getTgtMemIntrinsic(IntrinsicInst *Inst,
166 MemIntrinsicInfo &Info) const override;
167
168 InstructionCost getArithmeticInstrCost(
169 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
170 TTI::OperandValueInfo Op1Info = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
171 TTI::OperandValueInfo Op2Info = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
172 ArrayRef<const Value *> Args = {},
173 const Instruction *CxtI = nullptr) const override;
174
175 InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
176 const Instruction *I = nullptr) const override;
177
178 bool isInlineAsmSourceOfDivergence(const CallInst *CI,
179 ArrayRef<unsigned> Indices = {}) const;
180
181 using BaseT::getVectorInstrCost;
182 InstructionCost
183 getVectorInstrCost(unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind,
184 unsigned Index, const Value *Op0, const Value *Op1,
185 TTI::VectorInstrContext VIC =
186 TTI::VectorInstrContext::None) const override;
187
188 bool isReadRegisterSourceOfDivergence(const IntrinsicInst *ReadReg) const;
189
190 bool isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const override {
191 // Address space casts must cast between different address spaces.
192 if (FromAS == ToAS)
193 return false;
194
195 // Casts between any aliasing address spaces are valid.
196 return AMDGPU::addrspacesMayAlias(AS1: FromAS, AS2: ToAS);
197 }
198
199 bool addrspacesMayAlias(unsigned AS0, unsigned AS1) const override {
200 return AMDGPU::addrspacesMayAlias(AS1: AS0, AS2: AS1);
201 }
202
203 unsigned getFlatAddressSpace() const override {
204 // Don't bother running InferAddressSpaces pass on graphics shaders which
205 // don't use flat addressing.
206 if (IsGraphics)
207 return -1;
208 return AMDGPUAS::FLAT_ADDRESS;
209 }
210
211 bool collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,
212 Intrinsic::ID IID) const override;
213
214 bool
215 canHaveNonUndefGlobalInitializerInAddressSpace(unsigned AS) const override {
216 return AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS &&
217 AS != AMDGPUAS::PRIVATE_ADDRESS;
218 }
219
220 Value *rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, Value *OldV,
221 Value *NewV) const override;
222
223 bool canSimplifyLegacyMulToMul(const Instruction &I, const Value *Op0,
224 const Value *Op1, InstCombiner &IC) const;
225
226 bool simplifyDemandedLaneMaskArg(InstCombiner &IC, IntrinsicInst &II,
227 unsigned LaneAgIdx) const;
228
229 std::optional<Instruction *>
230 instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const override;
231
232 Value *simplifyAMDGCNLaneIntrinsicDemanded(InstCombiner &IC,
233 IntrinsicInst &II,
234 const APInt &DemandedElts,
235 APInt &UndefElts) const;
236
237 Instruction *hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
238 IntrinsicInst &II) const;
239
240 std::optional<Value *> simplifyDemandedVectorEltsIntrinsic(
241 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
242 APInt &UndefElts2, APInt &UndefElts3,
243 std::function<void(Instruction *, unsigned, APInt, APInt &)>
244 SimplifyAndSetOp) const override;
245
246 InstructionCost getVectorSplitCost() const { return 0; }
247
248 InstructionCost
249 getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
250 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
251 VectorType *SubTp, ArrayRef<const Value *> Args = {},
252 const Instruction *CxtI = nullptr) const override;
253
254 bool isProfitableToSinkOperands(Instruction *I,
255 SmallVectorImpl<Use *> &Ops) const override;
256
257 bool areInlineCompatible(const Function *Caller,
258 const Function *Callee) const override;
259
260 int getInliningLastCallToStaticBonus() const override;
261 unsigned getInliningThresholdMultiplier() const override { return 11; }
262 unsigned adjustInliningThreshold(const CallBase *CB) const override;
263 unsigned getCallerAllocaCost(const CallBase *CB,
264 const AllocaInst *AI) const override;
265
266 int getInlinerVectorBonusPercent() const override {
267 return InlinerVectorBonusPercent;
268 }
269
270 InstructionCost
271 getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
272 TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
273 const Instruction *I = nullptr) const override;
274
275 InstructionCost
276 getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
277 std::optional<FastMathFlags> FMF,
278 TTI::TargetCostKind CostKind) const override;
279
280 InstructionCost getPartialReductionCost(
281 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
282 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
283 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
284 TTI::TargetCostKind CostKind,
285 std::optional<FastMathFlags> FMF) const override {
286 return InstructionCost::getInvalid();
287 }
288
289 InstructionCost
290 getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
291 TTI::TargetCostKind CostKind) const override;
292 InstructionCost
293 getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,
294 TTI::TargetCostKind CostKind) const override;
295
296 /// Data cache line size for LoopDataPrefetch pass. Has no use before GFX12.
297 unsigned getCacheLineSize() const override;
298
299 /// How much before a load we should place the prefetch instruction.
300 /// This is currently measured in number of IR instructions.
301 unsigned getPrefetchDistance() const override;
302
303 /// \return if target want to issue a prefetch in address space \p AS.
304 bool shouldPrefetchAddressSpace(unsigned AS) const override;
305 void collectKernelLaunchBounds(
306 const Function &F,
307 SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const override;
308
309 enum class KnownIEEEMode { Unknown, On, Off };
310
311 /// Return KnownIEEEMode::On if we know if the use context can assume
312 /// "amdgpu-ieee"="true" and KnownIEEEMode::Off if we can assume
313 /// "amdgpu-ieee"="false".
314 KnownIEEEMode fpenvIEEEMode(const Instruction &I) const;
315
316 /// Account for loads of i8 vector types to have reduced cost. For
317 /// example the cost of load 4 i8s values is one is the cost of loading
318 /// a single i32 value.
319 InstructionCost getMemoryOpCost(
320 unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
321 TTI::TargetCostKind CostKind,
322 TTI::OperandValueInfo OpInfo = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
323 const Instruction *I = nullptr) const override;
324
325 /// When counting parts on AMD GPUs, account for i8s being grouped
326 /// together under a single i32 value. Otherwise fall back to base
327 /// implementation.
328 unsigned getNumberOfParts(Type *Tp) const override;
329
330 ValueUniformity getValueUniformity(const Value *V) const override;
331
332 InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
333 StackOffset BaseOffset, bool HasBaseReg,
334 int64_t Scale,
335 unsigned AddrSpace) const override;
336
337 bool isLSRCostLess(const TTI::LSRCost &A,
338 const TTI::LSRCost &B) const override;
339 bool isNumRegsMajorCostOfLSR() const override;
340 bool shouldDropLSRSolutionIfLessProfitable() const override;
341
342 bool isUniform(const Instruction *I,
343 const SmallBitVector &UniformArgs) const override;
344};
345
346} // end namespace llvm
347
348#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETTRANSFORMINFO_H
349