1//===- AMDGPUTargetTransformInfo.h - AMDGPU specific TTI --------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file a TargetTransformInfoImplBase conforming object specific to the
11/// AMDGPU target machine. It uses the target's detailed information to
12/// provide more precise answers to certain TTI queries, while letting the
13/// target independent and default TTI implementations handle the rest.
14//
15//===----------------------------------------------------------------------===//
16
17#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETTRANSFORMINFO_H
18#define LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETTRANSFORMINFO_H
19
20#include "AMDGPU.h"
21#include "llvm/CodeGen/BasicTTIImpl.h"
22#include "llvm/Support/AMDGPUAddrSpace.h"
23#include <optional>
24
25namespace llvm {
26
27class AMDGPUTargetMachine;
28struct AMDGPUOptions;
29class GCNSubtarget;
30class InstCombiner;
31class Loop;
32class ScalarEvolution;
33class SITargetLowering;
34class Type;
35class Value;
36
37class AMDGPUTTIImpl final : public BasicTTIImplBase<AMDGPUTTIImpl> {
38 using BaseT = BasicTTIImplBase<AMDGPUTTIImpl>;
39 using TTI = TargetTransformInfo;
40
41 friend BaseT;
42
43 const AMDGPUOptions &CLOpts;
44 Triple TargetTriple;
45
46 const TargetSubtargetInfo *ST;
47 const TargetLoweringBase *TLI;
48
49 const TargetSubtargetInfo *getST() const { return ST; }
50 const TargetLoweringBase *getTLI() const { return TLI; }
51
52public:
53 explicit AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const Function &F);
54
55 void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
56 TTI::UnrollingPreferences &UP,
57 OptimizationRemarkEmitter *ORE) const override;
58
59 void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
60 TTI::PeelingPreferences &PP) const override;
61
62 uint64_t getMaxMemIntrinsicInlineSizeThreshold() const override;
63};
64
65class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
66 using BaseT = BasicTTIImplBase<GCNTTIImpl>;
67 using TTI = TargetTransformInfo;
68
69 friend BaseT;
70
71 const AMDGPUOptions &CLOpts;
72 const GCNSubtarget *ST;
73 const SITargetLowering *TLI;
74 AMDGPUTTIImpl CommonTTI;
75 bool IsGraphics;
76 bool HasFP32Denormals;
77 static constexpr bool InlinerVectorBonusPercent = 0;
78
79 static const FeatureBitset InlineFeatureIgnoreList;
80
81 const GCNSubtarget *getST() const { return ST; }
82 const SITargetLowering *getTLI() const { return TLI; }
83
84 static inline int getFullRateInstrCost() {
85 return TargetTransformInfo::TCC_Basic;
86 }
87
88 static inline int getHalfRateInstrCost(TTI::TargetCostKind CostKind) {
89 return CostKind == TTI::TCK_CodeSize ? 2
90 : 2 * TargetTransformInfo::TCC_Basic;
91 }
92
93 // TODO: The size is usually 8 bytes, but takes 4x as many cycles. Maybe
94 // should be 2 or 4.
95 static inline int getQuarterRateInstrCost(TTI::TargetCostKind CostKind) {
96 return CostKind == TTI::TCK_CodeSize ? 2
97 : 4 * TargetTransformInfo::TCC_Basic;
98 }
99
100 int getTransInstrCost(TTI::TargetCostKind CostKind) const;
101
102 // On some parts, normal fp64 operations are half rate, and others
103 // quarter. This also applies to some integer operations.
104 int get64BitInstrCost(TTI::TargetCostKind CostKind) const;
105
106 std::pair<InstructionCost, MVT> getTypeLegalizationCost(Type *Ty) const;
107
108 /// \returns true if V might be divergent even when all of its operands
109 /// are uniform.
110 bool isSourceOfDivergence(const Value *V) const;
111
112 /// Returns true for the target specific set of operations which produce
113 /// uniform result even taking non-uniform arguments.
114 bool isAlwaysUniform(const Value *V) const;
115
116public:
117 explicit GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F);
118
119 bool hasBranchDivergence(const Function *F = nullptr) const override;
120
121 void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
122 TTI::UnrollingPreferences &UP,
123 OptimizationRemarkEmitter *ORE) const override;
124
125 void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
126 TTI::PeelingPreferences &PP) const override;
127
128 TTI::PopcntSupportKind getPopcntSupport(unsigned TyWidth) const override {
129 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
130 return TTI::PSK_FastHardware;
131 }
132
133 unsigned getNumberOfRegisters(unsigned RCID) const override;
134 TypeSize
135 getRegisterBitWidth(TargetTransformInfo::RegisterKind Vector) const override;
136 unsigned getMinVectorRegisterBitWidth() const override;
137 unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const override;
138 bool preferSLPInstCountCheck() const override;
139 unsigned getLoadVectorFactor(unsigned VF, unsigned LoadSize,
140 unsigned ChainSizeInBytes,
141 VectorType *VecTy) const override;
142 unsigned getStoreVectorFactor(unsigned VF, unsigned StoreSize,
143 unsigned ChainSizeInBytes,
144 VectorType *VecTy) const override;
145 unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const override;
146
147 bool isLegalToVectorizeMemChain(unsigned ChainSizeInBytes, Align Alignment,
148 unsigned AddrSpace) const;
149 bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, Align Alignment,
150 unsigned AddrSpace) const override;
151 bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment,
152 unsigned AddrSpace) const override;
153
154 uint64_t getMaxMemIntrinsicInlineSizeThreshold() const override;
155 Type *getMemcpyLoopLoweringType(
156 LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
157 unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
158 std::optional<uint32_t> AtomicElementSize) const override;
159
160 void getMemcpyLoopResidualLoweringType(
161 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
162 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
163 Align SrcAlign, Align DestAlign,
164 std::optional<uint32_t> AtomicCpySize) const override;
165 unsigned getMaxInterleaveFactor(ElementCount VF,
166 bool HasUnorderedReductions) const override;
167
168 bool getTgtMemIntrinsic(IntrinsicInst *Inst,
169 MemIntrinsicInfo &Info) const override;
170
171 InstructionCost getArithmeticInstrCost(
172 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
173 TTI::OperandValueInfo Op1Info = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
174 TTI::OperandValueInfo Op2Info = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
175 ArrayRef<const Value *> Args = {},
176 const Instruction *CtxI = nullptr) const override;
177
178 InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
179 const Instruction *I = nullptr) const override;
180
181 InstructionCost getCmpSelInstrCost(
182 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
183 TTI::TargetCostKind CostKind,
184 TTI::OperandValueInfo Op1Info = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
185 TTI::OperandValueInfo Op2Info = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
186 const Instruction *I = nullptr) const override;
187
188 bool isInlineAsmSourceOfDivergence(const CallBase *CB,
189 ArrayRef<unsigned> Indices = {}) const;
190
191 using BaseT::getVectorInstrCost;
192 InstructionCost
193 getVectorInstrCost(unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind,
194 unsigned Index, const Value *Op0, const Value *Op1,
195 TTI::VectorInstrContext VIC =
196 TTI::VectorInstrContext::None) const override;
197
198 bool isReadRegisterSourceOfDivergence(const IntrinsicInst *ReadReg) const;
199
200 bool isValidAddrSpaceCast(unsigned FromAS, unsigned ToAS) const override {
201 // Address space casts must cast between different address spaces.
202 if (FromAS == ToAS)
203 return false;
204
205 // Casts between any aliasing address spaces are valid.
206 return AMDGPU::addrspacesMayAlias(AS1: FromAS, AS2: ToAS);
207 }
208
209 bool addrspacesMayAlias(unsigned AS0, unsigned AS1) const override {
210 return AMDGPU::addrspacesMayAlias(AS1: AS0, AS2: AS1);
211 }
212
213 unsigned getFlatAddressSpace() const override {
214 // Don't bother running InferAddressSpaces pass on graphics shaders which
215 // don't use flat addressing.
216 if (IsGraphics)
217 return -1;
218 return AMDGPUAS::FLAT_ADDRESS;
219 }
220
221 bool collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,
222 Intrinsic::ID IID) const override;
223
224 bool
225 canHaveNonUndefGlobalInitializerInAddressSpace(unsigned AS) const override {
226 return AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS &&
227 AS != AMDGPUAS::PRIVATE_ADDRESS;
228 }
229
230 Value *rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, Value *OldV,
231 Value *NewV) const override;
232
233 bool canSimplifyLegacyMulToMul(const Instruction &I, const Value *Op0,
234 const Value *Op1, InstCombiner &IC) const;
235
236 bool simplifyDemandedLaneMaskArg(InstCombiner &IC, IntrinsicInst &II,
237 unsigned LaneAgIdx) const;
238
239 std::optional<Instruction *>
240 instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const override;
241
242 Value *simplifyAMDGCNLaneIntrinsicDemanded(InstCombiner &IC,
243 IntrinsicInst &II,
244 const APInt &DemandedElts,
245 APInt &UndefElts) const;
246
247 Value *simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC,
248 IntrinsicInst &II,
249 APInt DemandedElts) const;
250
251 Instruction *hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
252 IntrinsicInst &II) const;
253
254 std::optional<Value *> simplifyDemandedVectorEltsIntrinsic(
255 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
256 APInt &UndefElts2, APInt &UndefElts3,
257 std::function<void(Instruction *, unsigned, APInt, APInt &)>
258 SimplifyAndSetOp) const override;
259
260 InstructionCost getVectorSplitCost() const { return 0; }
261
262 InstructionCost
263 getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
264 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
265 VectorType *SubTp, ArrayRef<const Value *> Args = {},
266 const Instruction *CtxI = nullptr,
267 TTI::VectorInstrContext VIC =
268 TTI::VectorInstrContext::None) const override;
269
270 bool isProfitableToSinkOperands(Instruction *I,
271 SmallVectorImpl<Use *> &Ops) const override;
272
273 bool areInlineCompatible(const Function *Caller,
274 const Function *Callee) const override;
275
276 int getInliningLastCallToStaticBonus() const override;
277 unsigned getInliningThresholdMultiplier() const override { return 11; }
278 unsigned adjustInliningThreshold(const CallBase *CB) const override;
279 unsigned getCallerAllocaCost(const CallBase *CB,
280 const AllocaInst *AI) const override;
281
282 int getInlinerVectorBonusPercent() const override {
283 return InlinerVectorBonusPercent;
284 }
285
286 InstructionCost
287 getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
288 TTI::CastContextHint CCH, TTI::TargetCostKind CostKind,
289 const Instruction *I = nullptr) const override;
290
291 InstructionCost
292 getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
293 std::optional<FastMathFlags> FMF,
294 TTI::TargetCostKind CostKind) const override;
295
296 InstructionCost getPartialReductionCost(
297 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
298 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
299 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
300 TTI::TargetCostKind CostKind,
301 std::optional<FastMathFlags> FMF) const override {
302 return InstructionCost::getInvalid();
303 }
304
305 InstructionCost
306 getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
307 TTI::TargetCostKind CostKind) const override;
308 InstructionCost
309 getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,
310 TTI::TargetCostKind CostKind) const override;
311
312 /// Data cache line size for LoopDataPrefetch pass. Has no use before GFX12.
313 unsigned getCacheLineSize() const override;
314
315 /// How much before a load we should place the prefetch instruction.
316 /// This is currently measured in number of IR instructions.
317 unsigned getPrefetchDistance() const override;
318
319 /// \return if target want to issue a prefetch in address space \p AS.
320 bool shouldPrefetchAddressSpace(unsigned AS) const override;
321 void collectKernelLaunchBounds(
322 const Function &F,
323 SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const override;
324
325 enum class KnownIEEEMode { Unknown, On, Off };
326
327 /// Return KnownIEEEMode::On if we know if the use context can assume
328 /// "amdgpu-ieee"="true" and KnownIEEEMode::Off if we can assume
329 /// "amdgpu-ieee"="false".
330 KnownIEEEMode fpenvIEEEMode(const Instruction &I) const;
331
332 /// Account for loads of i8 vector types to have reduced cost. For
333 /// example the cost of load 4 i8s values is one is the cost of loading
334 /// a single i32 value.
335 InstructionCost getMemoryOpCost(
336 unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
337 TTI::TargetCostKind CostKind,
338 TTI::OperandValueInfo OpInfo = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
339 const Instruction *I = nullptr) const override;
340
341 /// When counting parts on AMD GPUs, account for i8s being grouped
342 /// together under a single i32 value. Otherwise fall back to base
343 /// implementation.
344 unsigned getNumberOfParts(Type *Tp) const override;
345
346 ValueUniformity getValueUniformity(const Value *V) const override;
347
348 InstructionCost getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
349 StackOffset BaseOffset, bool HasBaseReg,
350 int64_t Scale,
351 unsigned AddrSpace) const override;
352
353 bool isLSRCostLess(const TTI::LSRCost &A,
354 const TTI::LSRCost &B) const override;
355 bool isNumRegsMajorCostOfLSR() const override;
356 bool shouldDropLSRSolutionIfLessProfitable() const override;
357
358 bool isUniform(const Instruction *I,
359 const SmallBitVector &UniformArgs) const override;
360};
361
362} // end namespace llvm
363
364#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETTRANSFORMINFO_H
365