1//===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Populates the VFABI attribute with the scalar-to-vector mappings
10// from the TargetLibraryInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/InjectTLIMappings.h"
15#include "llvm/ADT/Statistic.h"
16#include "llvm/Analysis/DemandedBits.h"
17#include "llvm/Analysis/GlobalsModRef.h"
18#include "llvm/Analysis/TargetLibraryInfo.h"
19#include "llvm/Analysis/VectorUtils.h"
20#include "llvm/IR/InstIterator.h"
21#include "llvm/IR/VFABIDemangler.h"
22#include "llvm/Transforms/Utils/ModuleUtils.h"
23
24using namespace llvm;
25
26#define DEBUG_TYPE "inject-tli-mappings"
27
28STATISTIC(NumCallInjected,
29 "Number of calls in which the mappings have been injected.");
30
31STATISTIC(NumVFDeclAdded,
32 "Number of function declarations that have been added.");
33STATISTIC(NumCompUsedAdded,
34 "Number of `@llvm.compiler.used` operands that have been added.");
35
36/// A helper function that adds the vector variant declaration for vectorizing
37/// the CallInst \p CI with a vectorization factor of \p VF lanes. For each
38/// mapping, TLI provides a VABI prefix, which contains all information required
39/// to create vector function declaration.
40static void addVariantDeclaration(CallInst &CI, const ElementCount &VF,
41 const VecDesc *VD) {
42 Module *M = CI.getModule();
43 FunctionType *ScalarFTy = CI.getFunctionType();
44
45 assert(!ScalarFTy->isVarArg() && "VarArg functions are not supported.");
46
47 const std::optional<VFInfo> Info = VFABI::tryDemangleForVFABI(
48 MangledName: VD->getVectorFunctionABIVariantString(), FTy: ScalarFTy);
49
50 assert(Info && "Failed to demangle vector variant");
51 assert(Info->Shape.VF == VF && "Mangled name does not match VF");
52
53 const StringRef VFName = VD->getVectorFnName();
54 FunctionType *VectorFTy = VFABI::createFunctionType(Info: *Info, ScalarFTy);
55 Function *VecFunc =
56 Function::Create(Ty: VectorFTy, Linkage: Function::ExternalLinkage, N: VFName, M);
57 VecFunc->copyAttributesFrom(Src: CI.getCalledFunction());
58 if (auto CC = VD->getCallingConv())
59 VecFunc->setCallingConv(*CC);
60 ++NumVFDeclAdded;
61 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
62 << "` of type " << *VectorFTy << "\n");
63
64 // Make function declaration (without a body) "sticky" in the IR by
65 // listing it in the @llvm.compiler.used intrinsic.
66 assert(!VecFunc->size() && "VFABI attribute requires `@llvm.compiler.used` "
67 "only on declarations.");
68 appendToCompilerUsed(M&: *M, Values: {VecFunc});
69 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
70 << "` to `@llvm.compiler.used`.\n");
71 ++NumCompUsedAdded;
72}
73
74static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
75 // This is needed to make sure we don't query the TLI for calls to
76 // bitcast of function pointers, like `%call = call i32 (i32*, ...)
77 // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
78 // as such calls make the `isFunctionVectorizable` raise an
79 // exception.
80 if (CI.isNoBuiltin() || !CI.getCalledFunction())
81 return;
82
83 StringRef ScalarName = CI.getCalledFunction()->getName();
84
85 // Nothing to be done if the TLI thinks the function is not
86 // vectorizable.
87 if (!TLI.isFunctionVectorizable(F: ScalarName))
88 return;
89 SmallVector<std::string, 8> Mappings;
90 VFABI::getVectorVariantNames(CI, VariantMappings&: Mappings);
91 Module *M = CI.getModule();
92 const SetVector<StringRef> OriginalSetOfMappings(llvm::from_range, Mappings);
93
94 auto AddVariantDecl = [&](const ElementCount &VF, bool Predicate) {
95 const VecDesc *VD = TLI.getVectorMappingInfo(F: ScalarName, VF, Masked: Predicate);
96 if (VD && !VD->getVectorFnName().empty()) {
97 std::string MangledName = VD->getVectorFunctionABIVariantString();
98 if (!OriginalSetOfMappings.count(key: MangledName)) {
99 Mappings.push_back(Elt: MangledName);
100 ++NumCallInjected;
101 }
102 Function *VariantF = M->getFunction(Name: VD->getVectorFnName());
103 if (!VariantF)
104 addVariantDeclaration(CI, VF, VD);
105 }
106 };
107
108 // All VFs in the TLI are powers of 2.
109 ElementCount WidestFixedVF, WidestScalableVF;
110 TLI.getWidestVF(ScalarF: ScalarName, FixedVF&: WidestFixedVF, ScalableVF&: WidestScalableVF);
111
112 for (bool Predicated : {false, true}) {
113 for (ElementCount VF = ElementCount::getFixed(MinVal: 2);
114 ElementCount::isKnownLE(LHS: VF, RHS: WidestFixedVF); VF *= 2)
115 AddVariantDecl(VF, Predicated);
116
117 for (ElementCount VF = ElementCount::getScalable(MinVal: 2);
118 ElementCount::isKnownLE(LHS: VF, RHS: WidestScalableVF); VF *= 2)
119 AddVariantDecl(VF, Predicated);
120 }
121
122 VFABI::setVectorVariantNames(CI: &CI, VariantMappings: Mappings);
123}
124
125static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
126 for (auto &I : instructions(F))
127 if (auto CI = dyn_cast<CallInst>(Val: &I))
128 addMappingsFromTLI(TLI, CI&: *CI);
129 // Even if the pass adds IR attributes, the analyses are preserved.
130 return false;
131}
132
133////////////////////////////////////////////////////////////////////////////////
134// New pass manager implementation.
135////////////////////////////////////////////////////////////////////////////////
136PreservedAnalyses InjectTLIMappings::run(Function &F,
137 FunctionAnalysisManager &AM) {
138 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(IR&: F);
139 runImpl(TLI, F);
140 // Even if the pass adds IR attributes, the analyses are preserved.
141 return PreservedAnalyses::all();
142}
143