1//=== ReplaceWithVeclib.cpp - Replace vector intrinsics with veclib calls -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Replaces calls to LLVM Intrinsics with matching calls to functions from a
10// vector library (e.g libmvec, SVML) using TargetLibraryInfo interface.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/ReplaceWithVeclib.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/Statistic.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/Analysis/DemandedBits.h"
19#include "llvm/Analysis/GlobalsModRef.h"
20#include "llvm/Analysis/OptimizationRemarkEmitter.h"
21#include "llvm/Analysis/TargetLibraryInfo.h"
22#include "llvm/Analysis/VectorUtils.h"
23#include "llvm/CodeGen/Passes.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/IRBuilder.h"
26#include "llvm/IR/InstIterator.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/VFABIDemangler.h"
29#include "llvm/InitializePasses.h"
30#include "llvm/Support/TypeSize.h"
31#include "llvm/Transforms/Utils/ModuleUtils.h"
32
33using namespace llvm;
34
35#define DEBUG_TYPE "replace-with-veclib"
36
37STATISTIC(NumCallsReplaced,
38 "Number of calls to intrinsics that have been replaced.");
39
40STATISTIC(NumTLIFuncDeclAdded,
41 "Number of vector library function declarations added.");
42
43STATISTIC(NumFuncUsedAdded,
44 "Number of functions added to `llvm.compiler.used`");
45
46/// Returns a vector Function that it adds to the Module \p M. When an \p
47/// ScalarFunc is not null, it copies its attributes to the newly created
48/// Function.
49Function *getTLIFunction(Module *M, FunctionType *VectorFTy,
50 const StringRef TLIName,
51 std::optional<CallingConv::ID> CC,
52 Function *ScalarFunc = nullptr) {
53 Function *TLIFunc = M->getFunction(Name: TLIName);
54 if (!TLIFunc) {
55 TLIFunc =
56 Function::Create(Ty: VectorFTy, Linkage: Function::ExternalLinkage, N: TLIName, M&: *M);
57 if (ScalarFunc)
58 TLIFunc->copyAttributesFrom(Src: ScalarFunc);
59 if (CC)
60 TLIFunc->setCallingConv(*CC);
61
62 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added vector library function `"
63 << TLIName << "` of type `" << *(TLIFunc->getType())
64 << "` to module.\n");
65
66 ++NumTLIFuncDeclAdded;
67 // Add the freshly created function to llvm.compiler.used, similar to as it
68 // is done in InjectTLIMappings.
69 appendToCompilerUsed(M&: *M, Values: {TLIFunc});
70 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << TLIName
71 << "` to `@llvm.compiler.used`.\n");
72 ++NumFuncUsedAdded;
73 }
74 return TLIFunc;
75}
76
77/// Replace the intrinsic call \p II to \p TLIVecFunc, which is the
78/// corresponding function from the vector library.
79static void replaceWithTLIFunction(IntrinsicInst *II, VFInfo &Info,
80 Function *TLIVecFunc) {
81 IRBuilder<> IRBuilder(II);
82 SmallVector<Value *> Args(II->args());
83 if (Info.isMasked()) {
84 auto *MaskTy =
85 VectorType::get(ElementType: Type::getInt1Ty(C&: II->getContext()), EC: Info.Shape.VF);
86 Args.push_back(Elt: Constant::getAllOnesValue(Ty: MaskTy));
87 }
88
89 // Preserve the operand bundles.
90 SmallVector<OperandBundleDef, 1> OpBundles;
91 II->getOperandBundlesAsDefs(Defs&: OpBundles);
92
93 // Preserve fast math flags for FP math (getFastMathFlagsOrNone keeps this
94 // safe for non-FP intrinsics, whose flags are simply empty).
95 auto *Replacement = IRBuilder.CreateCall(
96 Callee: TLIVecFunc, Args, OpBundles, /*FMFSource=*/II->getFastMathFlagsOrNone());
97 // Preserve fpmath for FP math
98 if (isa<FPMathOperator>(Val: Replacement))
99 Replacement->copyMetadata(SrcInst: *II, WL: {LLVMContext::MD_fpmath});
100 II->replaceAllUsesWith(V: Replacement);
101 Replacement->setCallingConv(TLIVecFunc->getCallingConv());
102}
103
104/// Returns true when successfully replaced \p II, which is a call to a
105/// vectorized intrinsic, with a suitable function taking vector arguments,
106/// based on available mappings in the \p TLI.
107static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI,
108 IntrinsicInst *II) {
109 assert(II != nullptr && "Intrinsic cannot be null");
110 Intrinsic::ID IID = II->getIntrinsicID();
111 Type *RetTy = II->getType();
112 Type *ScalarRetTy = RetTy->getScalarType();
113 // At the moment VFABI assumes the return type is always widened unless it is
114 // a void type.
115 auto *VTy = dyn_cast<VectorType>(Val: RetTy);
116 ElementCount EC(VTy ? VTy->getElementCount() : ElementCount::getFixed(MinVal: 0));
117
118 // OloadTys collects types used in scalar intrinsic overload name.
119 SmallVector<Type *, 3> OloadTys;
120 if (!RetTy->isVoidTy() &&
121 isVectorIntrinsicWithOverloadTypeAtArg(ID: IID, OpdIdx: -1, /*TTI=*/nullptr))
122 OloadTys.push_back(Elt: ScalarRetTy);
123
124 // Compute the argument types of the corresponding scalar call and check that
125 // all vector operands match the previously found EC.
126 SmallVector<Type *, 8> ScalarArgTypes;
127 for (auto Arg : enumerate(First: II->args())) {
128 auto *ArgTy = Arg.value()->getType();
129 bool IsOloadTy = isVectorIntrinsicWithOverloadTypeAtArg(ID: IID, OpdIdx: Arg.index(),
130 /*TTI=*/nullptr);
131 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: Arg.index(), /*TTI=*/nullptr)) {
132 ScalarArgTypes.push_back(Elt: ArgTy);
133 if (IsOloadTy)
134 OloadTys.push_back(Elt: ArgTy);
135 } else if (auto *VectorArgTy = dyn_cast<VectorType>(Val: ArgTy)) {
136 auto *ScalarArgTy = VectorArgTy->getElementType();
137 ScalarArgTypes.push_back(Elt: ScalarArgTy);
138 if (IsOloadTy)
139 OloadTys.push_back(Elt: ScalarArgTy);
140 // When return type is void, set EC to the first vector argument, and
141 // disallow vector arguments with different ECs.
142 if (EC.isZero())
143 EC = VectorArgTy->getElementCount();
144 else if (EC != VectorArgTy->getElementCount())
145 return false;
146 } else
147 // Exit when it is supposed to be a vector argument but it isn't.
148 return false;
149 }
150
151 // Try to reconstruct the name for the scalar version of the instruction,
152 // using scalar argument types.
153 std::string ScalarName =
154 Intrinsic::isOverloaded(id: IID)
155 ? Intrinsic::getName(Id: IID, OverloadTys: OloadTys, M: II->getModule())
156 : Intrinsic::getName(id: IID).str();
157
158 // Try to find the mapping for the scalar version of this intrinsic and the
159 // exact vector width of the call operands in the TargetLibraryInfo. First,
160 // check with a non-masked variant, and if that fails try with a masked one.
161 const VecDesc *VD =
162 TLI.getVectorMappingInfo(F: ScalarName, VF: EC, /*Masked*/ false);
163 if (!VD && !(VD = TLI.getVectorMappingInfo(F: ScalarName, VF: EC, /*Masked*/ true)))
164 return false;
165
166 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Found TLI mapping from: `" << ScalarName
167 << "` and vector width " << EC << " to: `"
168 << VD->getVectorFnName() << "`.\n");
169
170 // Replace the call to the intrinsic with a call to the vector library
171 // function.
172 FunctionType *ScalarFTy =
173 FunctionType::get(Result: ScalarRetTy, Params: ScalarArgTypes, /*isVarArg*/ false);
174 const std::string MangledName = VD->getVectorFunctionABIVariantString();
175 auto OptInfo = VFABI::tryDemangleForVFABI(MangledName, FTy: ScalarFTy);
176 if (!OptInfo)
177 return false;
178
179 // There is no guarantee that the vectorized instructions followed the VFABI
180 // specification when being created, this is why we need to add extra check to
181 // make sure that the operands of the vector function obtained via VFABI match
182 // the operands of the original vector instruction.
183 for (auto &VFParam : OptInfo->Shape.Parameters) {
184 if (VFParam.ParamKind == VFParamKind::GlobalPredicate)
185 continue;
186
187 // tryDemangleForVFABI must return valid ParamPos, otherwise it could be
188 // a bug in the VFABI parser.
189 assert(VFParam.ParamPos < II->arg_size() && "ParamPos has invalid range");
190 Type *OrigTy = II->getArgOperand(i: VFParam.ParamPos)->getType();
191 if (OrigTy->isVectorTy() != (VFParam.ParamKind == VFParamKind::Vector)) {
192 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Will not replace: " << ScalarName
193 << ". Wrong type at index " << VFParam.ParamPos << ": "
194 << *OrigTy << "\n");
195 return false;
196 }
197 }
198
199 FunctionType *VectorFTy = VFABI::createFunctionType(Info: *OptInfo, ScalarFTy);
200 if (!VectorFTy)
201 return false;
202
203 Function *TLIFunc =
204 getTLIFunction(M: II->getModule(), VectorFTy, TLIName: VD->getVectorFnName(),
205 CC: VD->getCallingConv(), ScalarFunc: II->getCalledFunction());
206 replaceWithTLIFunction(II, Info&: *OptInfo, TLIVecFunc: TLIFunc);
207 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Replaced call to `" << ScalarName
208 << "` with call to `" << TLIFunc->getName() << "`.\n");
209 ++NumCallsReplaced;
210 return true;
211}
212
213/// Returns true when \p TLI has a vector mapping for the scalar function name
214/// \p Name at \p EC (matching either masked or unmasked variants).
215static bool hasVectorMapping(const TargetLibraryInfo &TLI, StringRef Name,
216 ElementCount EC) {
217 return TLI.getVectorMappingInfo(F: Name, VF: EC, /*Masked=*/false) ||
218 TLI.getVectorMappingInfo(F: Name, VF: EC, /*Masked=*/true);
219}
220
221/// Returns true when \p TLI has a vector mapping for \p IID at the given
222/// element type and \p EC.
223static bool hasIntrinsicVectorMapping(const TargetLibraryInfo &TLI,
224 Intrinsic::ID IID, Type *ScalarTy,
225 ElementCount EC, Module *M) {
226 std::string Name = Intrinsic::getName(Id: IID, OverloadTys: {ScalarTy}, M);
227 return hasVectorMapping(TLI, Name, EC);
228}
229
230/// If \p II is a vector llvm.sincos with no direct vector library mapping but
231/// the target does have vector mappings for both llvm.sin and llvm.cos at the
232/// same element count, replace it with separate llvm.sin and llvm.cos calls
233/// and run the standard veclib replacement on each.
234static bool trySplitVectorSinCos(const TargetLibraryInfo &TLI,
235 IntrinsicInst *II,
236 SmallVectorImpl<Instruction *> &Replaced) {
237 if (II->getIntrinsicID() != Intrinsic::sincos)
238 return false;
239 Value *Arg = II->getArgOperand(i: 0);
240 auto *VTy = dyn_cast<VectorType>(Val: Arg->getType());
241 if (!VTy)
242 return false;
243
244 ElementCount EC = VTy->getElementCount();
245 Type *ScalarTy = VTy->getElementType();
246 Module *M = II->getModule();
247
248 // If a vector sincos mapping exists for the intrinsic name (e.g.
249 // "llvm.sincos.f32") or for the scalar libcall name ("sincos"/"sincosf"),
250 // leave the call alone -- SelectionDAG legalization will handle it via
251 // expandMultipleResultFPLibCall when the runtime libcall impl is enabled.
252 if (hasIntrinsicVectorMapping(TLI, IID: Intrinsic::sincos, ScalarTy, EC, M))
253 return false;
254 LibFunc LF = NotLibFunc;
255 if (ScalarTy->isFloatTy())
256 LF = LibFunc_sincosf;
257 else if (ScalarTy->isDoubleTy())
258 LF = LibFunc_sincos;
259 if (LF != NotLibFunc && hasVectorMapping(TLI, Name: TLI.getName(F: LF), EC))
260 return false;
261
262 // Splitting is only worthwhile when both sin and cos have vector mappings.
263 if (!hasIntrinsicVectorMapping(TLI, IID: Intrinsic::sin, ScalarTy, EC, M) ||
264 !hasIntrinsicVectorMapping(TLI, IID: Intrinsic::cos, ScalarTy, EC, M))
265 return false;
266
267 // All users must be extractvalue.
268 for (User *U : II->users()) {
269 if (!isa<ExtractValueInst>(Val: U))
270 return false;
271 }
272
273 IRBuilder<> B(II);
274 Function *SinFn =
275 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::sin, OverloadTys: Arg->getType());
276 Function *CosFn =
277 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::cos, OverloadTys: Arg->getType());
278 CallInst *SinCall = B.CreateCall(Callee: SinFn, Args: {Arg}, /*FMFSource=*/II, Name: "sin");
279 CallInst *CosCall = B.CreateCall(Callee: CosFn, Args: {Arg}, /*FMFSource=*/II, Name: "cos");
280 SinCall->copyMetadata(SrcInst: *II, WL: {LLVMContext::MD_fpmath});
281 CosCall->copyMetadata(SrcInst: *II, WL: {LLVMContext::MD_fpmath});
282
283 // Forward extractvalue uses to the new calls.
284 for (User *U : make_early_inc_range(Range: II->users())) {
285 auto *EV = cast<ExtractValueInst>(Val: U);
286 EV->replaceAllUsesWith(V: EV->getIndices()[0] == 0 ? SinCall : CosCall);
287 EV->eraseFromParent();
288 }
289
290 // Replace each new call with the vector library function.
291 if (replaceWithCallToVeclib(TLI, II: cast<IntrinsicInst>(Val: SinCall)))
292 Replaced.push_back(Elt: SinCall);
293 if (replaceWithCallToVeclib(TLI, II: cast<IntrinsicInst>(Val: CosCall)))
294 Replaced.push_back(Elt: CosCall);
295
296 return true;
297}
298
299static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
300 SmallVector<Instruction *> ReplacedCalls;
301 for (auto &I : instructions(F)) {
302 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
303 if (!II)
304 continue;
305
306 // Vector llvm.sincos returns a struct so it does not fit the generic
307 // path below; try to split it into separate sin and cos calls when the
308 // target has vector mappings for them.
309 if (trySplitVectorSinCos(TLI, II, Replaced&: ReplacedCalls)) {
310 ReplacedCalls.push_back(Elt: &I);
311 continue;
312 }
313
314 // Process only intrinsic calls that return void or a vector.
315 if (!II->getType()->isVectorTy() && !II->getType()->isVoidTy())
316 continue;
317
318 if (replaceWithCallToVeclib(TLI, II))
319 ReplacedCalls.push_back(Elt: &I);
320 }
321 // Erase any intrinsic calls that were replaced with vector library calls.
322 for (auto *I : ReplacedCalls)
323 I->eraseFromParent();
324 return !ReplacedCalls.empty();
325}
326
327////////////////////////////////////////////////////////////////////////////////
328// New pass manager implementation.
329////////////////////////////////////////////////////////////////////////////////
330PreservedAnalyses ReplaceWithVeclib::run(Function &F,
331 FunctionAnalysisManager &AM) {
332 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(IR&: F);
333 auto Changed = runImpl(TLI, F);
334 if (Changed) {
335 LLVM_DEBUG(dbgs() << "Intrinsic calls replaced with vector libraries: "
336 << NumCallsReplaced << "\n");
337
338 PreservedAnalyses PA;
339 PA.preserveSet<CFGAnalyses>();
340 PA.preserve<TargetLibraryAnalysis>();
341 PA.preserve<ScalarEvolutionAnalysis>();
342 PA.preserve<LoopAccessAnalysis>();
343 PA.preserve<DemandedBitsAnalysis>();
344 PA.preserve<OptimizationRemarkEmitterAnalysis>();
345 return PA;
346 }
347
348 // The pass did not replace any calls, hence it preserves all analyses.
349 return PreservedAnalyses::all();
350}
351
352////////////////////////////////////////////////////////////////////////////////
353// Legacy PM Implementation.
354////////////////////////////////////////////////////////////////////////////////
355bool ReplaceWithVeclibLegacy::runOnFunction(Function &F) {
356 const TargetLibraryInfo &TLI =
357 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
358 return runImpl(TLI, F);
359}
360
361void ReplaceWithVeclibLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
362 AU.setPreservesCFG();
363 AU.addRequired<TargetLibraryInfoWrapperPass>();
364 AU.addPreserved<TargetLibraryInfoWrapperPass>();
365 AU.addPreserved<ScalarEvolutionWrapperPass>();
366 AU.addPreserved<AAResultsWrapperPass>();
367 AU.addPreserved<OptimizationRemarkEmitterWrapperPass>();
368 AU.addPreserved<GlobalsAAWrapperPass>();
369}
370
371////////////////////////////////////////////////////////////////////////////////
372// Legacy Pass manager initialization
373////////////////////////////////////////////////////////////////////////////////
374char ReplaceWithVeclibLegacy::ID = 0;
375
376INITIALIZE_PASS_BEGIN(ReplaceWithVeclibLegacy, DEBUG_TYPE,
377 "Replace intrinsics with calls to vector library", false,
378 false)
379INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
380INITIALIZE_PASS_END(ReplaceWithVeclibLegacy, DEBUG_TYPE,
381 "Replace intrinsics with calls to vector library", false,
382 false)
383
384FunctionPass *llvm::createReplaceWithVeclibLegacyPass() {
385 return new ReplaceWithVeclibLegacy();
386}
387