1//===-- NVPTXTargetTransformInfo.h - NVPTX specific TTI ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file a TargetTransformInfoImplBase conforming object specific to the
10/// NVPTX target machine. It uses the target's detailed information to
11/// provide more precise answers to certain TTI queries, while letting the
12/// target independent and default TTI implementations handle the rest.
13///
14//===----------------------------------------------------------------------===//
15
16#ifndef LLVM_LIB_TARGET_NVPTX_NVPTXTARGETTRANSFORMINFO_H
17#define LLVM_LIB_TARGET_NVPTX_NVPTXTARGETTRANSFORMINFO_H
18
19#include "MCTargetDesc/NVPTXBaseInfo.h"
20#include "NVPTXTargetMachine.h"
21#include "NVPTXUtilities.h"
22#include "llvm/Analysis/TargetTransformInfo.h"
23#include "llvm/CodeGen/BasicTTIImpl.h"
24#include "llvm/CodeGen/TargetLowering.h"
25#include <optional>
26
27namespace llvm {
28
29class NVPTXTTIImpl final : public BasicTTIImplBase<NVPTXTTIImpl> {
30 typedef BasicTTIImplBase<NVPTXTTIImpl> BaseT;
31 typedef TargetTransformInfo TTI;
32 friend BaseT;
33
34 const NVPTXSubtarget *ST;
35 const NVPTXTargetLowering *TLI;
36
37 const NVPTXSubtarget *getST() const { return ST; };
38 const NVPTXTargetLowering *getTLI() const { return TLI; };
39
40 /// \returns true if the result of the value could potentially be
41 /// different across threads in a warp.
42 bool isSourceOfDivergence(const Value *V) const;
43
44public:
45 explicit NVPTXTTIImpl(const NVPTXTargetMachine *TM, const Function &F)
46 : BaseT(TM, F.getDataLayout()), ST(TM->getSubtargetImpl()),
47 TLI(ST->getTargetLowering()) {}
48
49 bool hasBranchDivergence(const Function *F = nullptr) const override {
50 return true;
51 }
52
53 unsigned getFlatAddressSpace() const override {
54 return AddressSpace::ADDRESS_SPACE_GENERIC;
55 }
56
57 unsigned getAddressSpaceJoin(unsigned AS1, unsigned AS2) const override {
58 if ((AS1 == AddressSpace::ADDRESS_SPACE_SHARED &&
59 AS2 == AddressSpace::ADDRESS_SPACE_SHARED_CLUSTER) ||
60 (AS2 == AddressSpace::ADDRESS_SPACE_SHARED &&
61 AS1 == AddressSpace::ADDRESS_SPACE_SHARED_CLUSTER))
62 return AddressSpace::ADDRESS_SPACE_SHARED_CLUSTER;
63 return AddressSpace::ADDRESS_SPACE_GENERIC;
64 }
65
66 bool
67 canHaveNonUndefGlobalInitializerInAddressSpace(unsigned AS) const override {
68 return AS != AddressSpace::ADDRESS_SPACE_SHARED &&
69 AS != AddressSpace::ADDRESS_SPACE_LOCAL &&
70 AS != AddressSpace::ADDRESS_SPACE_ENTRY_PARAM;
71 }
72
73 std::optional<Instruction *>
74 instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const override;
75
76 // Loads and stores can be vectorized if the alignment is at least as big as
77 // the load/store we want to vectorize.
78 bool isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes, Align Alignment,
79 unsigned AddrSpace) const override {
80 return Alignment >= ChainSizeInBytes;
81 }
82 bool isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes, Align Alignment,
83 unsigned AddrSpace) const override {
84 return isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment, AddrSpace);
85 }
86
87 // NVPTX has infinite registers of all kinds, but the actual machine doesn't.
88 // We conservatively return 1 here which is just enough to enable the
89 // vectorizers but disables heuristics based on the number of registers.
90 // FIXME: Return a more reasonable number, while keeping an eye on
91 // LoopVectorizer's unrolling heuristics.
92 unsigned getNumberOfRegisters(unsigned ClassID) const override { return 1; }
93
94 // Only <2 x half> should be vectorized, so always return 32 for the vector
95 // register size.
96 TypeSize
97 getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const override {
98 return TypeSize::getFixed(ExactSize: 32);
99 }
100 unsigned getMinVectorRegisterBitWidth() const override { return 32; }
101
102 bool shouldExpandReduction(const IntrinsicInst *II) const override {
103 // Turn off ExpandReductions pass for NVPTX, which doesn't have advanced
104 // swizzling operations. Our backend/Selection DAG can expand these
105 // reductions with less movs.
106 return false;
107 }
108
109 // We don't want to prevent inlining because of target-cpu and -features
110 // attributes that were added to newer versions of LLVM/Clang: There are
111 // no incompatible functions in PTX, ptxas will throw errors in such cases.
112 bool areInlineCompatible(const Function *Caller,
113 const Function *Callee) const override {
114 return true;
115 }
116
117 // Increase the inlining cost threshold by a factor of 11, reflecting that
118 // calls are particularly expensive in NVPTX.
119 unsigned getInliningThresholdMultiplier() const override { return 11; }
120
121 InstructionCost
122 getInstructionCost(const User *U, ArrayRef<const Value *> Operands,
123 TTI::TargetCostKind CostKind) const override;
124
125 InstructionCost getArithmeticInstrCost(
126 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
127 TTI::OperandValueInfo Op1Info = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
128 TTI::OperandValueInfo Op2Info = {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
129 ArrayRef<const Value *> Args = {},
130 const Instruction *CtxI = nullptr) const override;
131
132 InstructionCost
133 getScalarizationOverhead(VectorType *InTy, const APInt &DemandedElts,
134 bool Insert, bool Extract,
135 TTI::TargetCostKind CostKind,
136 bool ForPoisonSrc = true, ArrayRef<Value *> VL = {},
137 TTI::VectorInstrContext VIC =
138 TTI::VectorInstrContext::None) const override {
139 if (!InTy->getElementCount().isFixed())
140 return InstructionCost::getInvalid();
141
142 auto VT = getTLI()->getValueType(DL, Ty: InTy);
143 auto NumElements = InTy->getElementCount().getFixedValue();
144 InstructionCost Cost = 0;
145 if (Insert && !VL.empty()) {
146 bool AllConstant = all_of(Range: seq(Size: NumElements), P: [&](int Idx) {
147 return !DemandedElts[Idx] || isa<Constant>(Val: VL[Idx]);
148 });
149 if (AllConstant) {
150 Cost += TTI::TCC_Free;
151 Insert = false;
152 }
153 }
154 if (Insert && VT == MVT::v4i8) {
155 Cost += 3; // 3 x PRMT
156 for (auto Idx : seq(Size: NumElements))
157 if (DemandedElts[Idx])
158 Cost += 1; // zext operand to i32
159 Insert = false;
160 } else if (Insert && NVPTX::isPackedVectorTy(VT) && VT.is32BitVector()) {
161 // Can be built in a single 32-bit mov (64-bit regs are emulated in SASS
162 // with 2x 32-bit regs)
163 Cost += 1;
164 Insert = false;
165 }
166 return Cost + BaseT::getScalarizationOverhead(InTy, DemandedElts, Insert,
167 Extract, CostKind,
168 ForPoisonSrc, VL);
169 }
170
171 void getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
172 TTI::UnrollingPreferences &UP,
173 OptimizationRemarkEmitter *ORE) const override;
174
175 void getPeelingPreferences(Loop *L, ScalarEvolution &SE,
176 TTI::PeelingPreferences &PP) const override;
177
178 bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) const override {
179 if (!isa<LoadInst, StoreInst>(Val: I))
180 return false;
181
182 switch (AddrSpace) {
183 default:
184 return false;
185 case ADDRESS_SPACE_GENERIC:
186 case ADDRESS_SPACE_GLOBAL:
187 case ADDRESS_SPACE_SHARED:
188 case ADDRESS_SPACE_SHARED_CLUSTER:
189 return true;
190 case ADDRESS_SPACE_LOCAL:
191 return ST->hasLocalVolatile();
192 }
193 }
194
195 APInt getAddrSpaceCastPreservedPtrMask(unsigned SrcAS,
196 unsigned DstAS) const override {
197 if (SrcAS != llvm::ADDRESS_SPACE_GENERIC)
198 return BaseT::getAddrSpaceCastPreservedPtrMask(SrcAS, DstAS);
199 if (DstAS != llvm::ADDRESS_SPACE_GLOBAL &&
200 DstAS != llvm::ADDRESS_SPACE_SHARED)
201 return BaseT::getAddrSpaceCastPreservedPtrMask(SrcAS, DstAS);
202
203 // Address change within 4K size does not change the original address space
204 // and is safe to perform address cast form SrcAS to DstAS.
205 APInt PtrMask(DL.getPointerSizeInBits(AS: llvm::ADDRESS_SPACE_GENERIC), 0xfff);
206 return PtrMask;
207 }
208
209 bool collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,
210 Intrinsic::ID IID) const override;
211
212 bool isLegalMaskedStore(Type *DataType, Align Alignment, unsigned AddrSpace,
213 TTI::MaskKind MaskKind) const override;
214
215 bool isLegalMaskedLoad(Type *DataType, Align Alignment, unsigned AddrSpace,
216 TTI::MaskKind MaskKind) const override;
217
218 unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const override;
219
220 Value *rewriteIntrinsicWithAddressSpace(IntrinsicInst *II, Value *OldV,
221 Value *NewV) const override;
222 unsigned getAssumedAddrSpace(const Value *V) const override;
223
224 void collectKernelLaunchBounds(
225 const Function &F,
226 SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const override;
227
228 bool shouldBuildRelLookupTables() const override {
229 // Self-referential globals are not supported.
230 return false;
231 }
232
233 bool shouldBuildLookupTablesForConstant(Constant *C) const override;
234
235 InstructionCost getPartialReductionCost(
236 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
237 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
238 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
239 TTI::TargetCostKind CostKind,
240 std::optional<FastMathFlags> FMF) const override {
241 return InstructionCost::getInvalid();
242 }
243
244 ValueUniformity getValueUniformity(const Value *V) const override;
245};
246
247} // end namespace llvm
248
249#endif
250