1//===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Populates the VFABI attribute with the scalar-to-vector mappings
10// from the TargetLibraryInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/InjectTLIMappings.h"
15#include "llvm/ADT/SetVector.h"
16#include "llvm/ADT/Statistic.h"
17#include "llvm/Analysis/DemandedBits.h"
18#include "llvm/Analysis/GlobalsModRef.h"
19#include "llvm/Analysis/TargetLibraryInfo.h"
20#include "llvm/Analysis/VectorUtils.h"
21#include "llvm/IR/AttributeMask.h"
22#include "llvm/IR/Attributes.h"
23#include "llvm/IR/InstIterator.h"
24#include "llvm/IR/VFABIDemangler.h"
25#include "llvm/Transforms/Utils/ModuleUtils.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE "inject-tli-mappings"
30
31STATISTIC(NumCallInjected,
32 "Number of calls in which the mappings have been injected.");
33
34STATISTIC(NumVFDeclAdded,
35 "Number of function declarations that have been added.");
36STATISTIC(NumCompUsedAdded,
37 "Number of `@llvm.compiler.used` operands that have been added.");
38
39/// A helper function that adds the vector variant declaration for vectorizing
40/// the CallInst \p CI with a vectorization factor of \p VF lanes. For each
41/// mapping, TLI provides a VABI prefix, which contains all information required
42/// to create vector function declaration.
43static void addVariantDeclaration(CallInst &CI, const ElementCount &VF,
44 const VecDesc *VD) {
45 Module *M = CI.getModule();
46 FunctionType *ScalarFTy = CI.getFunctionType();
47
48 assert(!ScalarFTy->isVarArg() && "VarArg functions are not supported.");
49
50 const std::optional<VFInfo> Info = VFABI::tryDemangleForVFABI(
51 MangledName: VD->getVectorFunctionABIVariantString(), FTy: ScalarFTy);
52
53 assert(Info && "Failed to demangle vector variant");
54 assert(Info->Shape.VF == VF && "Mangled name does not match VF");
55
56 const StringRef VFName = VD->getVectorFnName();
57 FunctionType *VectorFTy = VFABI::createFunctionType(Info: *Info, ScalarFTy);
58 Function *VecFunc =
59 Function::Create(Ty: VectorFTy, Linkage: Function::ExternalLinkage, N: VFName, M);
60 VecFunc->copyAttributesFrom(Src: CI.getCalledFunction());
61
62 // When mapping scalar functions to vector functions, some attributes
63 // (e.g. signext) are not valid on vector types. Remove attributes that are
64 // incompatible with the vectorized return type and arguments.
65 VecFunc->removeRetAttrs(Attrs: AttributeFuncs::typeIncompatible(
66 Ty: VecFunc->getReturnType(), AS: VecFunc->getAttributes().getRetAttrs()));
67 for (auto &Arg : VecFunc->args())
68 Arg.removeAttrs(
69 AM: AttributeFuncs::typeIncompatible(Ty: Arg.getType(), AS: Arg.getAttributes()));
70
71 if (auto CC = VD->getCallingConv())
72 VecFunc->setCallingConv(*CC);
73 ++NumVFDeclAdded;
74 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
75 << "` of type " << *VectorFTy << "\n");
76
77 // Make function declaration (without a body) "sticky" in the IR by
78 // listing it in the @llvm.compiler.used intrinsic.
79 assert(!VecFunc->size() && "VFABI attribute requires `@llvm.compiler.used` "
80 "only on declarations.");
81 appendToCompilerUsed(M&: *M, Values: {VecFunc});
82 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
83 << "` to `@llvm.compiler.used`.\n");
84 ++NumCompUsedAdded;
85}
86
87static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
88 // This is needed to make sure we don't query the TLI for calls to
89 // bitcast of function pointers, like `%call = call i32 (i32*, ...)
90 // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
91 // as such calls make the `isFunctionVectorizable` raise an
92 // exception.
93 if (CI.isNoBuiltin() || !CI.getCalledFunction())
94 return;
95
96 // Skip variadic function calls as they are not supported (like 'int pow();')
97 if (CI.getFunctionType()->isVarArg())
98 return;
99
100 StringRef ScalarName = CI.getCalledFunction()->getName();
101
102 // Nothing to be done if the TLI thinks the function is not
103 // vectorizable.
104 if (!TLI.isFunctionVectorizable(F: ScalarName))
105 return;
106 SmallVector<std::string, 8> Mappings;
107 VFABI::getVectorVariantNames(CI, VariantMappings&: Mappings);
108 Module *M = CI.getModule();
109 const SetVector<StringRef> OriginalSetOfMappings(llvm::from_range, Mappings);
110
111 auto AddVariantDecl = [&](const ElementCount &VF, bool Predicate) {
112 const VecDesc *VD = TLI.getVectorMappingInfo(F: ScalarName, VF, Masked: Predicate);
113 if (VD && !VD->getVectorFnName().empty()) {
114 std::string MangledName = VD->getVectorFunctionABIVariantString();
115 if (!OriginalSetOfMappings.count(key: MangledName)) {
116 Mappings.push_back(Elt: MangledName);
117 ++NumCallInjected;
118 }
119 Function *VariantF = M->getFunction(Name: VD->getVectorFnName());
120 if (!VariantF)
121 addVariantDeclaration(CI, VF, VD);
122 }
123 };
124
125 // All VFs in the TLI are powers of 2.
126 ElementCount WidestFixedVF, WidestScalableVF;
127 TLI.getWidestVF(ScalarF: ScalarName, FixedVF&: WidestFixedVF, ScalableVF&: WidestScalableVF);
128
129 for (bool Predicated : {false, true}) {
130 for (ElementCount VF = ElementCount::getFixed(MinVal: 2);
131 ElementCount::isKnownLE(LHS: VF, RHS: WidestFixedVF); VF *= 2)
132 AddVariantDecl(VF, Predicated);
133
134 for (ElementCount VF = ElementCount::getScalable(MinVal: 2);
135 ElementCount::isKnownLE(LHS: VF, RHS: WidestScalableVF); VF *= 2)
136 AddVariantDecl(VF, Predicated);
137 }
138
139 VFABI::setVectorVariantNames(CI: &CI, VariantMappings: Mappings);
140}
141
142static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
143 for (auto &I : instructions(F))
144 if (auto CI = dyn_cast<CallInst>(Val: &I))
145 addMappingsFromTLI(TLI, CI&: *CI);
146 // Even if the pass adds IR attributes, the analyses are preserved.
147 return false;
148}
149
150////////////////////////////////////////////////////////////////////////////////
151// New pass manager implementation.
152////////////////////////////////////////////////////////////////////////////////
153PreservedAnalyses InjectTLIMappings::run(Function &F,
154 FunctionAnalysisManager &AM) {
155 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(IR&: F);
156 runImpl(TLI, F);
157 // Even if the pass adds IR attributes, the analyses are preserved.
158 return PreservedAnalyses::all();
159}
160