1//===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Populates the VFABI attribute with the scalar-to-vector mappings
10// from the TargetLibraryInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/InjectTLIMappings.h"
15#include "llvm/ADT/Statistic.h"
16#include "llvm/Analysis/DemandedBits.h"
17#include "llvm/Analysis/GlobalsModRef.h"
18#include "llvm/Analysis/TargetLibraryInfo.h"
19#include "llvm/Analysis/VectorUtils.h"
20#include "llvm/IR/AttributeMask.h"
21#include "llvm/IR/Attributes.h"
22#include "llvm/IR/InstIterator.h"
23#include "llvm/IR/VFABIDemangler.h"
24#include "llvm/Transforms/Utils/ModuleUtils.h"
25
26using namespace llvm;
27
28#define DEBUG_TYPE "inject-tli-mappings"
29
30STATISTIC(NumCallInjected,
31 "Number of calls in which the mappings have been injected.");
32
33STATISTIC(NumVFDeclAdded,
34 "Number of function declarations that have been added.");
35STATISTIC(NumCompUsedAdded,
36 "Number of `@llvm.compiler.used` operands that have been added.");
37
38/// A helper function that adds the vector variant declaration for vectorizing
39/// the CallInst \p CI with a vectorization factor of \p VF lanes. For each
40/// mapping, TLI provides a VABI prefix, which contains all information required
41/// to create vector function declaration.
42static void addVariantDeclaration(CallInst &CI, const ElementCount &VF,
43 const VecDesc *VD) {
44 Module *M = CI.getModule();
45 FunctionType *ScalarFTy = CI.getFunctionType();
46
47 assert(!ScalarFTy->isVarArg() && "VarArg functions are not supported.");
48
49 const std::optional<VFInfo> Info = VFABI::tryDemangleForVFABI(
50 MangledName: VD->getVectorFunctionABIVariantString(), FTy: ScalarFTy);
51
52 assert(Info && "Failed to demangle vector variant");
53 assert(Info->Shape.VF == VF && "Mangled name does not match VF");
54
55 const StringRef VFName = VD->getVectorFnName();
56 FunctionType *VectorFTy = VFABI::createFunctionType(Info: *Info, ScalarFTy);
57 Function *VecFunc =
58 Function::Create(Ty: VectorFTy, Linkage: Function::ExternalLinkage, N: VFName, M);
59 VecFunc->copyAttributesFrom(Src: CI.getCalledFunction());
60
61 // When mapping scalar functions to vector functions, some attributes
62 // (e.g. signext) are not valid on vector types. Remove attributes that are
63 // incompatible with the vectorized return type and arguments.
64 VecFunc->removeRetAttrs(Attrs: AttributeFuncs::typeIncompatible(
65 Ty: VecFunc->getReturnType(), AS: VecFunc->getAttributes().getRetAttrs()));
66 for (auto &Arg : VecFunc->args())
67 Arg.removeAttrs(
68 AM: AttributeFuncs::typeIncompatible(Ty: Arg.getType(), AS: Arg.getAttributes()));
69
70 if (auto CC = VD->getCallingConv())
71 VecFunc->setCallingConv(*CC);
72 ++NumVFDeclAdded;
73 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
74 << "` of type " << *VectorFTy << "\n");
75
76 // Make function declaration (without a body) "sticky" in the IR by
77 // listing it in the @llvm.compiler.used intrinsic.
78 assert(!VecFunc->size() && "VFABI attribute requires `@llvm.compiler.used` "
79 "only on declarations.");
80 appendToCompilerUsed(M&: *M, Values: {VecFunc});
81 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
82 << "` to `@llvm.compiler.used`.\n");
83 ++NumCompUsedAdded;
84}
85
86static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
87 // This is needed to make sure we don't query the TLI for calls to
88 // bitcast of function pointers, like `%call = call i32 (i32*, ...)
89 // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
90 // as such calls make the `isFunctionVectorizable` raise an
91 // exception.
92 if (CI.isNoBuiltin() || !CI.getCalledFunction())
93 return;
94
95 StringRef ScalarName = CI.getCalledFunction()->getName();
96
97 // Nothing to be done if the TLI thinks the function is not
98 // vectorizable.
99 if (!TLI.isFunctionVectorizable(F: ScalarName))
100 return;
101 SmallVector<std::string, 8> Mappings;
102 VFABI::getVectorVariantNames(CI, VariantMappings&: Mappings);
103 Module *M = CI.getModule();
104 const SetVector<StringRef> OriginalSetOfMappings(llvm::from_range, Mappings);
105
106 auto AddVariantDecl = [&](const ElementCount &VF, bool Predicate) {
107 const VecDesc *VD = TLI.getVectorMappingInfo(F: ScalarName, VF, Masked: Predicate);
108 if (VD && !VD->getVectorFnName().empty()) {
109 std::string MangledName = VD->getVectorFunctionABIVariantString();
110 if (!OriginalSetOfMappings.count(key: MangledName)) {
111 Mappings.push_back(Elt: MangledName);
112 ++NumCallInjected;
113 }
114 Function *VariantF = M->getFunction(Name: VD->getVectorFnName());
115 if (!VariantF)
116 addVariantDeclaration(CI, VF, VD);
117 }
118 };
119
120 // All VFs in the TLI are powers of 2.
121 ElementCount WidestFixedVF, WidestScalableVF;
122 TLI.getWidestVF(ScalarF: ScalarName, FixedVF&: WidestFixedVF, ScalableVF&: WidestScalableVF);
123
124 for (bool Predicated : {false, true}) {
125 for (ElementCount VF = ElementCount::getFixed(MinVal: 2);
126 ElementCount::isKnownLE(LHS: VF, RHS: WidestFixedVF); VF *= 2)
127 AddVariantDecl(VF, Predicated);
128
129 for (ElementCount VF = ElementCount::getScalable(MinVal: 2);
130 ElementCount::isKnownLE(LHS: VF, RHS: WidestScalableVF); VF *= 2)
131 AddVariantDecl(VF, Predicated);
132 }
133
134 VFABI::setVectorVariantNames(CI: &CI, VariantMappings: Mappings);
135}
136
137static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
138 for (auto &I : instructions(F))
139 if (auto CI = dyn_cast<CallInst>(Val: &I))
140 addMappingsFromTLI(TLI, CI&: *CI);
141 // Even if the pass adds IR attributes, the analyses are preserved.
142 return false;
143}
144
145////////////////////////////////////////////////////////////////////////////////
146// New pass manager implementation.
147////////////////////////////////////////////////////////////////////////////////
148PreservedAnalyses InjectTLIMappings::run(Function &F,
149 FunctionAnalysisManager &AM) {
150 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(IR&: F);
151 runImpl(TLI, F);
152 // Even if the pass adds IR attributes, the analyses are preserved.
153 return PreservedAnalyses::all();
154}
155