1//===- SLPCostAnalysis.cpp - SLP Vectorizer free cost helpers -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "SLPCostAnalysis.h"
10
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/ADT/SmallVector.h"
13#include "llvm/IR/DerivedTypes.h"
14#include "llvm/IR/Instructions.h"
15#include "llvm/IR/Operator.h"
16#include "llvm/IR/Value.h"
17#include "llvm/Support/Casting.h"
18
19#include <utility>
20
21using namespace llvm;
22
23namespace llvm::slpvectorizer {
24
25InstructionCost getShuffleCost(const TargetTransformInfo &TTI,
26 TTI::ShuffleKind Kind, VectorType *Tp,
27 const TTI::TargetCostKind CostKind,
28 ArrayRef<int> Mask, int Index, VectorType *SubTp,
29 ArrayRef<const Value *> Args) {
30 VectorType *DstTy = Tp;
31 if (!Mask.empty())
32 DstTy = FixedVectorType::get(ElementType: Tp->getScalarType(), NumElts: Mask.size());
33
34 if (Kind != TTI::SK_PermuteTwoSrc)
35 return TTI.getShuffleCost(Kind, DstTy, SrcTy: Tp, CostKind, Mask, Index, SubTp,
36 Args);
37 int NumSrcElts = Tp->getElementCount().getKnownMinValue();
38 int NumSubElts;
39 if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask(
40 Mask, NumSrcElts, NumSubElts, Index)) {
41 if (Index + NumSubElts > NumSrcElts &&
42 Index + NumSrcElts <= static_cast<int>(Mask.size()))
43 return TTI.getShuffleCost(Kind: TTI::SK_InsertSubvector, DstTy, SrcTy: Tp, CostKind,
44 Mask, Index, SubTp: Tp);
45 }
46 return TTI.getShuffleCost(Kind, DstTy, SrcTy: Tp, CostKind, Mask, Index, SubTp,
47 Args);
48}
49
50std::pair<InstructionCost, InstructionCost>
51getGEPCosts(const TargetTransformInfo &TTI, ArrayRef<Value *> Ptrs,
52 Value *BasePtr, unsigned Opcode, const TTI::TargetCostKind CostKind,
53 Type *ScalarTy, VectorType *VecTy) {
54 InstructionCost ScalarCost = 0;
55 InstructionCost VecCost = 0;
56 // Here we differentiate two cases: (1) when Ptrs represent a regular
57 // vectorization tree node (as they are pointer arguments of scattered
58 // loads) or (2) when Ptrs are the arguments of loads or stores being
59 // vectorized as plane wide unit-stride load/store since all the
60 // loads/stores are known to be from/to adjacent locations.
61 if (Opcode == Instruction::Load || Opcode == Instruction::Store) {
62 // Case 2: estimate costs for pointer related costs when vectorizing to
63 // a wide load/store.
64 // Scalar cost is estimated as a set of pointers with known relationship
65 // between them.
66 // For vector code we will use BasePtr as argument for the wide load/store
67 // but we also need to account all the instructions which are going to
68 // stay in vectorized code due to uses outside of these scalar
69 // loads/stores.
70 ScalarCost = TTI.getPointersChainCost(
71 Ptrs, Base: BasePtr, Info: TTI::PointersChainInfo::getUnitStride(), AccessTy: ScalarTy,
72 CostKind);
73
74 SmallVector<const Value *> PtrsRetainedInVecCode;
75 for (Value *V : Ptrs) {
76 if (V == BasePtr) {
77 PtrsRetainedInVecCode.push_back(Elt: V);
78 continue;
79 }
80 auto *Ptr = dyn_cast<GetElementPtrInst>(Val: V);
81 // For simplicity assume Ptr to stay in vectorized code if it's not a
82 // GEP instruction. We don't care since it's cost considered free.
83 // TODO: We should check for any uses outside of vectorizable tree
84 // rather than just single use.
85 if (!Ptr || !Ptr->hasOneUse())
86 PtrsRetainedInVecCode.push_back(Elt: V);
87 }
88
89 if (PtrsRetainedInVecCode.size() == Ptrs.size()) {
90 // If all pointers stay in vectorized code then we don't have
91 // any savings on that.
92 return std::make_pair(x: TTI::TCC_Free, y: TTI::TCC_Free);
93 }
94 VecCost = TTI.getPointersChainCost(Ptrs: PtrsRetainedInVecCode, Base: BasePtr,
95 Info: TTI::PointersChainInfo::getKnownStride(),
96 AccessTy: VecTy, CostKind);
97 } else {
98 // Case 1: Ptrs are the arguments of loads that we are going to transform
99 // into masked gather load intrinsic.
100 // All the scalar GEPs will be removed as a result of vectorization.
101 // For any external uses of some lanes extract element instructions will
102 // be generated (which cost is estimated separately).
103 TTI::PointersChainInfo PtrsInfo =
104 all_of(Range&: Ptrs,
105 P: [](const Value *V) {
106 auto *Ptr = dyn_cast<GetElementPtrInst>(Val: V);
107 return Ptr && !Ptr->hasAllConstantIndices();
108 })
109 ? TTI::PointersChainInfo::getUnknownStride()
110 : TTI::PointersChainInfo::getKnownStride();
111
112 ScalarCost =
113 TTI.getPointersChainCost(Ptrs, Base: BasePtr, Info: PtrsInfo, AccessTy: ScalarTy, CostKind);
114 auto *BaseGEP = dyn_cast<GEPOperator>(Val: BasePtr);
115 if (!BaseGEP) {
116 auto *It = find_if(Range&: Ptrs, P: IsaPred<GEPOperator>);
117 if (It != Ptrs.end())
118 BaseGEP = cast<GEPOperator>(Val: *It);
119 }
120 if (BaseGEP) {
121 SmallVector<const Value *> Indices(BaseGEP->indices());
122 VecCost = TTI.getGEPCost(PointeeType: BaseGEP->getSourceElementType(),
123 Ptr: BaseGEP->getPointerOperand(), Operands: Indices, CostKind,
124 AccessType: VecTy);
125 }
126 }
127
128 return std::make_pair(x&: ScalarCost, y&: VecCost);
129}
130
131InstructionCost getBlendedLoadCost(const TargetTransformInfo &TTI, Type *VecTy,
132 Align Alignment, unsigned AddressSpace,
133 const TTI::TargetCostKind CostKind) {
134 Type *CmpTy = CmpInst::makeCmpResultType(opnd_type: VecTy);
135 return 2 * TTI.getMemIntrinsicInstrCost(
136 MICA: MemIntrinsicCostAttributes(Intrinsic::masked_load, VecTy,
137 Alignment, AddressSpace),
138 CostKind) +
139 TTI.getArithmeticInstrCost(Opcode: Instruction::Xor, Ty: CmpTy, CostKind) +
140 TTI.getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: VecTy, CondTy: CmpTy,
141 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
142}
143
144} // namespace llvm::slpvectorizer
145