1//===-- SPIRVPrepareFunctions.cpp - modify function signatures --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass modifies function signatures containing aggregate arguments
10// and/or return value before IRTranslator. Information about the original
11// signatures is stored in metadata. It is used during call lowering to
12// restore correct SPIR-V types of function arguments and return values.
13// This pass also substitutes some llvm intrinsic calls with calls to newly
14// generated functions (as the Khronos LLVM/SPIR-V Translator does).
15//
16// NOTE: this pass is a module-level one due to the necessity to modify
17// GVs/functions.
18//
19//===----------------------------------------------------------------------===//
20
21#include "SPIRV.h"
22#include "SPIRVBuiltins.h"
23#include "SPIRVSubtarget.h"
24#include "SPIRVTargetMachine.h"
25#include "SPIRVUtils.h"
26#include "llvm/ADT/StringExtras.h"
27#include "llvm/Analysis/TargetTransformInfo.h"
28#include "llvm/Analysis/ValueTracking.h"
29#include "llvm/CodeGen/IntrinsicLowering.h"
30#include "llvm/IR/DiagnosticInfo.h"
31#include "llvm/IR/IRBuilder.h"
32#include "llvm/IR/InstIterator.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/Intrinsics.h"
36#include "llvm/IR/IntrinsicsSPIRV.h"
37#include "llvm/InitializePasses.h"
38#include "llvm/Transforms/Utils/Cloning.h"
39#include "llvm/Transforms/Utils/Local.h"
40#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
41#include <regex>
42
43using namespace llvm;
44
45namespace {
46
47class SPIRVPrepareFunctionsImpl {
48 const SPIRVTargetMachine &TM;
49 function_ref<const TargetTransformInfo &(Function &)> GetTTI;
50 bool substituteIntrinsicCalls(Function *F);
51 bool substituteAbortKHRCalls(Function *F);
52 bool terminateBlocksAfterTrap(Module &M, Intrinsic::ID IID);
53 Function *removeAggregateTypesFromSignature(Function *F);
54 bool removeAggregateTypesFromCalls(Function *F);
55
56public:
57 SPIRVPrepareFunctionsImpl(
58 const SPIRVTargetMachine &TM,
59 function_ref<const TargetTransformInfo &(Function &)> GetTTI)
60 : TM(TM), GetTTI(GetTTI) {}
61 bool runOnModule(Module &M);
62};
63
64class SPIRVPrepareFunctionsLegacy : public ModulePass {
65 const SPIRVTargetMachine &TM;
66
67public:
68 static char ID;
69 SPIRVPrepareFunctionsLegacy(const SPIRVTargetMachine &TM)
70 : ModulePass(ID), TM(TM) {}
71
72 bool runOnModule(Module &M) override {
73 auto GetTTI = [this](Function &F) -> const TargetTransformInfo & {
74 return getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
75 };
76 return SPIRVPrepareFunctionsImpl(TM, GetTTI).runOnModule(M);
77 }
78
79 void getAnalysisUsage(AnalysisUsage &AU) const override {
80 AU.addRequired<TargetTransformInfoWrapperPass>();
81 }
82
83 StringRef getPassName() const override { return "SPIRV prepare functions"; }
84};
85
86static cl::list<std::string> SPVAllowUnknownIntrinsics(
87 "spv-allow-unknown-intrinsics", cl::CommaSeparated,
88 cl::desc("Emit unknown intrinsics as calls to external functions. A "
89 "comma-separated input list of intrinsic prefixes must be "
90 "provided, and only intrinsics carrying a listed prefix get "
91 "emitted as described."),
92 cl::value_desc("intrinsic_prefix_0,intrinsic_prefix_1"), cl::ValueOptional);
93} // namespace
94
95char SPIRVPrepareFunctionsLegacy::ID = 0;
96
97INITIALIZE_PASS_BEGIN(SPIRVPrepareFunctionsLegacy, "spirv-prepare-functions",
98 "SPIRV prepare functions", false, false)
99INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
100INITIALIZE_PASS_END(SPIRVPrepareFunctionsLegacy, "spirv-prepare-functions",
101 "SPIRV prepare functions", false, false)
102
103static std::string lowerLLVMIntrinsicName(IntrinsicInst *II) {
104 Function *IntrinsicFunc = II->getCalledFunction();
105 assert(IntrinsicFunc && "Missing function");
106 std::string FuncName = IntrinsicFunc->getName().str();
107 llvm::replace(Range&: FuncName, OldValue: '.', NewValue: '_');
108 FuncName = "spirv." + FuncName;
109 return FuncName;
110}
111
112static Function *getOrCreateFunction(Module *M, Type *RetTy,
113 ArrayRef<Type *> ArgTypes,
114 StringRef Name) {
115 FunctionType *FT = FunctionType::get(Result: RetTy, Params: ArgTypes, isVarArg: false);
116 Function *F = M->getFunction(Name);
117 if (F && F->getFunctionType() == FT)
118 return F;
119 Function *NewF = Function::Create(Ty: FT, Linkage: GlobalValue::ExternalLinkage, N: Name, M);
120 if (F)
121 NewF->setDSOLocal(F->isDSOLocal());
122 NewF->setCallingConv(CallingConv::SPIR_FUNC);
123 return NewF;
124}
125
126static bool lowerIntrinsicToFunction(IntrinsicInst *Intrinsic,
127 const TargetTransformInfo &TTI) {
128 // For @llvm.memset.* intrinsic cases with constant value and length arguments
129 // are emulated via "storing" a constant array to the destination. For other
130 // cases we wrap the intrinsic in @spirv.llvm_memset_* function and expand the
131 // intrinsic to a loop via expandMemSetAsLoop().
132 if (auto *MSI = dyn_cast<MemSetInst>(Val: Intrinsic))
133 if (isa<Constant>(Val: MSI->getValue()) && isa<ConstantInt>(Val: MSI->getLength()))
134 return false; // It is handled later using OpCopyMemorySized.
135
136 // An intrinsic with a metadata argument has no SPIR-V lowering and can't be
137 // turned into a function.
138 if (any_of(Range: Intrinsic->args(), P: IsaPred<MetadataAsValue>)) {
139 const Function *F = Intrinsic->getFunction();
140 F->getContext().diagnose(DI: DiagnosticInfoUnsupported(
141 *F,
142 "cannot lower the intrinsic '" +
143 Intrinsic->getCalledFunction()->getName() +
144 "' that takes a metadata argument",
145 Intrinsic->getDebugLoc()));
146 if (!Intrinsic->getType()->isVoidTy())
147 Intrinsic->replaceAllUsesWith(V: PoisonValue::get(T: Intrinsic->getType()));
148 Intrinsic->eraseFromParent();
149 return true;
150 }
151
152 Module *M = Intrinsic->getModule();
153 std::string FuncName = lowerLLVMIntrinsicName(II: Intrinsic);
154 if (Intrinsic->isVolatile())
155 FuncName += ".volatile";
156 // Redirect @llvm.intrinsic.* call to @spirv.llvm_intrinsic_*
157 Function *F = M->getFunction(Name: FuncName);
158 if (F) {
159 Intrinsic->setCalledFunction(F);
160 return true;
161 }
162 FunctionCallee FC =
163 M->getOrInsertFunction(Name: FuncName, T: Intrinsic->getFunctionType());
164 auto IntrinsicID = Intrinsic->getIntrinsicID();
165 Intrinsic->setCalledFunction(FC);
166 F = cast<Function>(Val: FC.getCallee());
167 F->setAttributes(Intrinsic->getAttributes());
168
169 switch (IntrinsicID) {
170 case Intrinsic::memset: {
171 auto *MSI = static_cast<MemSetInst *>(Intrinsic);
172 Argument *Dest = F->getArg(i: 0);
173 Argument *Val = F->getArg(i: 1);
174 Argument *Len = F->getArg(i: 2);
175 Argument *IsVolatile = F->getArg(i: 3);
176 Dest->setName("dest");
177 Val->setName("val");
178 Len->setName("len");
179 IsVolatile->setName("isvolatile");
180 BasicBlock *EntryBB = BasicBlock::Create(Context&: M->getContext(), Name: "entry", Parent: F);
181 IRBuilder<> IRB(EntryBB);
182 auto *MemSet = IRB.CreateMemSet(Ptr: Dest, Val, Size: Len, Align: MSI->getDestAlign(),
183 isVolatile: MSI->isVolatile());
184 IRB.CreateRetVoid();
185 expandMemSetAsLoop(MemSet: cast<MemSetInst>(Val: MemSet), TTI);
186 MemSet->eraseFromParent();
187 break;
188 }
189 case Intrinsic::bswap: {
190 BasicBlock *EntryBB = BasicBlock::Create(Context&: M->getContext(), Name: "entry", Parent: F);
191 IRBuilder<> IRB(EntryBB);
192 CallInst *BSwap = IRB.CreateIntrinsicWithoutFolding(
193 ID: Intrinsic::bswap, OverloadTypes: Intrinsic->getType(), Args: F->getArg(i: 0));
194 IRB.CreateRet(V: BSwap);
195 IntrinsicLowering IL(M->getDataLayout());
196 IL.LowerIntrinsicCall(CI: BSwap);
197 break;
198 }
199 default:
200 break;
201 }
202 return true;
203}
204
205static std::string getAnnotation(Value *AnnoVal, Value *OptAnnoVal) {
206 if (auto *Ref = dyn_cast_or_null<GetElementPtrInst>(Val: AnnoVal))
207 AnnoVal = Ref->getOperand(i_nocapture: 0);
208 if (auto *Ref = dyn_cast_or_null<BitCastInst>(Val: OptAnnoVal))
209 OptAnnoVal = Ref->getOperand(i_nocapture: 0);
210
211 std::string Anno;
212 if (auto *C = dyn_cast_or_null<Constant>(Val: AnnoVal)) {
213 StringRef Str;
214 if (getConstantStringInfo(V: C, Str))
215 Anno = Str;
216 }
217 // handle optional annotation parameter in a way that Khronos Translator do
218 // (collect integers wrapped in a struct)
219 if (auto *C = dyn_cast_or_null<Constant>(Val: OptAnnoVal);
220 C && C->getNumOperands()) {
221 Value *MaybeStruct = C->getOperand(i: 0);
222 if (auto *Struct = dyn_cast<ConstantStruct>(Val: MaybeStruct)) {
223 for (unsigned I = 0, E = Struct->getNumOperands(); I != E; ++I) {
224 if (auto *CInt = dyn_cast<ConstantInt>(Val: Struct->getOperand(i_nocapture: I)))
225 Anno += (I == 0 ? ": " : ", ") +
226 std::to_string(val: CInt->getType()->getIntegerBitWidth() == 1
227 ? CInt->getZExtValue()
228 : CInt->getSExtValue());
229 }
230 } else if (auto *Struct = dyn_cast<ConstantAggregateZero>(Val: MaybeStruct)) {
231 // { i32 i32 ... } zeroinitializer
232 for (unsigned I = 0, E = Struct->getType()->getStructNumElements();
233 I != E; ++I)
234 Anno += I == 0 ? ": 0" : ", 0";
235 }
236 }
237 return Anno;
238}
239
240static SmallVector<Metadata *> parseAnnotation(Value *I,
241 const std::string &Anno,
242 LLVMContext &Ctx,
243 Type *Int32Ty) {
244 // Try to parse the annotation string according to the following rules:
245 // annotation := ({kind} | {kind:value,value,...})+
246 // kind := number
247 // value := number | string
248 static const std::regex R(
249 "\\{(\\d+)(?:[:,](\\d+|\"[^\"]*\")(?:,(\\d+|\"[^\"]*\"))*)?\\}");
250 SmallVector<Metadata *> MDs;
251 int Pos = 0;
252 for (std::sregex_iterator
253 It = std::sregex_iterator(Anno.begin(), Anno.end(), R),
254 ItEnd = std::sregex_iterator();
255 It != ItEnd; ++It) {
256 if (It->position() != Pos)
257 return SmallVector<Metadata *>{};
258 Pos = It->position() + It->length();
259 std::smatch Match = *It;
260 SmallVector<Metadata *> MDsItem;
261 for (std::size_t i = 1; i < Match.size(); ++i) {
262 std::ssub_match SMatch = Match[i];
263 std::string Item = SMatch.str();
264 if (Item.length() == 0)
265 break;
266 if (Item[0] == '"') {
267 Item = Item.substr(pos: 1, n: Item.length() - 2);
268 // Acceptable format of the string snippet is:
269 static const std::regex RStr("^(\\d+)(?:,(\\d+))*$");
270 if (std::smatch MatchStr; std::regex_match(s: Item, m&: MatchStr, re: RStr)) {
271 for (std::size_t SubIdx = 1; SubIdx < MatchStr.size(); ++SubIdx)
272 if (std::string SubStr = MatchStr[SubIdx].str(); SubStr.length())
273 MDsItem.push_back(Elt: ConstantAsMetadata::get(
274 C: ConstantInt::get(Ty: Int32Ty, V: std::stoi(str: SubStr))));
275 } else {
276 MDsItem.push_back(Elt: MDString::get(Context&: Ctx, Str: Item));
277 }
278 } else if (int32_t Num; llvm::to_integer(S: StringRef(Item), Num, Base: 10)) {
279 MDsItem.push_back(
280 Elt: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: Num)));
281 } else {
282 MDsItem.push_back(Elt: MDString::get(Context&: Ctx, Str: Item));
283 }
284 }
285 if (MDsItem.size() == 0)
286 return SmallVector<Metadata *>{};
287 MDs.push_back(Elt: MDNode::get(Context&: Ctx, MDs: MDsItem));
288 }
289 return Pos == static_cast<int>(Anno.length()) ? std::move(MDs)
290 : SmallVector<Metadata *>{};
291}
292
293static void lowerPtrAnnotation(IntrinsicInst *II) {
294 LLVMContext &Ctx = II->getContext();
295 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
296
297 // Retrieve an annotation string from arguments.
298 Value *PtrArg = nullptr;
299 if (auto *BI = dyn_cast<BitCastInst>(Val: II->getArgOperand(i: 0)))
300 PtrArg = BI->getOperand(i_nocapture: 0);
301 else
302 PtrArg = II->getOperand(i_nocapture: 0);
303 std::string Anno =
304 getAnnotation(AnnoVal: II->getArgOperand(i: 1),
305 OptAnnoVal: 4 < II->arg_size() ? II->getArgOperand(i: 4) : nullptr);
306
307 // Parse the annotation.
308 SmallVector<Metadata *> MDs = parseAnnotation(I: II, Anno, Ctx, Int32Ty);
309
310 // If the annotation string is not parsed successfully we don't know the
311 // format used and output it as a general UserSemantic decoration.
312 // Otherwise MDs is a Metadata tuple (a decoration list) in the format
313 // expected by `spirv.Decorations`.
314 if (MDs.size() == 0) {
315 auto UserSemantic = ConstantAsMetadata::get(C: ConstantInt::get(
316 Ty: Int32Ty, V: static_cast<uint32_t>(SPIRV::Decoration::UserSemantic)));
317 MDs.push_back(Elt: MDNode::get(Context&: Ctx, MDs: {UserSemantic, MDString::get(Context&: Ctx, Str: Anno)}));
318 }
319
320 // Build the internal intrinsic function.
321 IRBuilder<> IRB(II->getParent());
322 IRB.SetInsertPoint(II);
323 IRB.CreateIntrinsic(
324 ID: Intrinsic::spv_assign_decoration, OverloadTypes: {PtrArg->getType()},
325 Args: {PtrArg, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs))});
326 II->replaceAllUsesWith(V: II->getOperand(i_nocapture: 0));
327}
328
329static void lowerFunnelShifts(IntrinsicInst *FSHIntrinsic) {
330 // Get a separate function - otherwise, we'd have to rework the CFG of the
331 // current one. Then simply replace the intrinsic uses with a call to the new
332 // function.
333 // Generate LLVM IR for i* @spirv.llvm_fsh?_i* (i* %a, i* %b, i* %c)
334 Module *M = FSHIntrinsic->getModule();
335 FunctionType *FSHFuncTy = FSHIntrinsic->getFunctionType();
336 Type *FSHRetTy = FSHFuncTy->getReturnType();
337 const std::string FuncName = lowerLLVMIntrinsicName(II: FSHIntrinsic);
338 Function *FSHFunc =
339 getOrCreateFunction(M, RetTy: FSHRetTy, ArgTypes: FSHFuncTy->params(), Name: FuncName);
340
341 if (!FSHFunc->empty()) {
342 FSHIntrinsic->setCalledFunction(FSHFunc);
343 return;
344 }
345 BasicBlock *RotateBB = BasicBlock::Create(Context&: M->getContext(), Name: "rotate", Parent: FSHFunc);
346 IRBuilder<> IRB(RotateBB);
347 Type *Ty = FSHFunc->getReturnType();
348 // Build the actual funnel shift rotate logic.
349 // In the comments, "int" is used interchangeably with "vector of int
350 // elements".
351 FixedVectorType *VectorTy = dyn_cast<FixedVectorType>(Val: Ty);
352 Type *IntTy = VectorTy ? VectorTy->getElementType() : Ty;
353 unsigned BitWidth = IntTy->getIntegerBitWidth();
354 ConstantInt *BitWidthConstant = IRB.getInt(AI: {BitWidth, BitWidth});
355 Value *BitWidthForInsts =
356 VectorTy
357 ? IRB.CreateVectorSplat(NumElts: VectorTy->getNumElements(), V: BitWidthConstant)
358 : BitWidthConstant;
359 Value *RotateModVal =
360 IRB.CreateURem(/*Rotate*/ LHS: FSHFunc->getArg(i: 2), RHS: BitWidthForInsts);
361 Value *FirstShift = nullptr, *SecShift = nullptr;
362 if (FSHIntrinsic->getIntrinsicID() == Intrinsic::fshr) {
363 // Shift the less significant number right, the "rotate" number of bits
364 // will be 0-filled on the left as a result of this regular shift.
365 FirstShift = IRB.CreateLShr(LHS: FSHFunc->getArg(i: 1), RHS: RotateModVal);
366 } else {
367 // Shift the more significant number left, the "rotate" number of bits
368 // will be 0-filled on the right as a result of this regular shift.
369 FirstShift = IRB.CreateShl(LHS: FSHFunc->getArg(i: 0), RHS: RotateModVal);
370 }
371 // We want the "rotate" number of the more significant int's LSBs (MSBs) to
372 // occupy the leftmost (rightmost) "0 space" left by the previous operation.
373 // Therefore, subtract the "rotate" number from the integer bitsize...
374 Value *SubRotateVal = IRB.CreateSub(LHS: BitWidthForInsts, RHS: RotateModVal);
375 if (FSHIntrinsic->getIntrinsicID() == Intrinsic::fshr) {
376 // ...and left-shift the more significant int by this number, zero-filling
377 // the LSBs.
378 SecShift = IRB.CreateShl(LHS: FSHFunc->getArg(i: 0), RHS: SubRotateVal);
379 } else {
380 // ...and right-shift the less significant int by this number, zero-filling
381 // the MSBs.
382 SecShift = IRB.CreateLShr(LHS: FSHFunc->getArg(i: 1), RHS: SubRotateVal);
383 }
384 // A simple binary addition of the shifted ints yields the final result.
385 IRB.CreateRet(V: IRB.CreateOr(LHS: FirstShift, RHS: SecShift));
386
387 FSHIntrinsic->setCalledFunction(FSHFunc);
388}
389
390static void lowerConstrainedFPCmpIntrinsic(
391 ConstrainedFPCmpIntrinsic *ConstrainedCmpIntrinsic,
392 SmallVector<Instruction *> &EraseFromParent) {
393 if (!ConstrainedCmpIntrinsic)
394 return;
395 // Extract the floating-point values being compared
396 Value *LHS = ConstrainedCmpIntrinsic->getArgOperand(i: 0);
397 Value *RHS = ConstrainedCmpIntrinsic->getArgOperand(i: 1);
398 FCmpInst::Predicate Pred = ConstrainedCmpIntrinsic->getPredicate();
399 IRBuilder<> Builder(ConstrainedCmpIntrinsic);
400 Value *FCmp = Builder.CreateFCmp(P: Pred, LHS, RHS);
401 ConstrainedCmpIntrinsic->replaceAllUsesWith(V: FCmp);
402 EraseFromParent.push_back(Elt: dyn_cast<Instruction>(Val: ConstrainedCmpIntrinsic));
403}
404
405static void lowerExpectAssume(IntrinsicInst *II) {
406 // If we cannot use the SPV_KHR_expect_assume extension, then we need to
407 // ignore the intrinsic and move on. It should be removed later on by LLVM.
408 // Otherwise we should lower the intrinsic to the corresponding SPIR-V
409 // instruction.
410 // For @llvm.assume we have OpAssumeTrueKHR.
411 // For @llvm.expect we have OpExpectKHR.
412 //
413 // We need to lower this into a builtin and then the builtin into a SPIR-V
414 // instruction.
415 if (II->getIntrinsicID() == Intrinsic::assume) {
416 Function *F = Intrinsic::getOrInsertDeclaration(
417 M: II->getModule(), id: Intrinsic::SPVIntrinsics::spv_assume);
418 II->setCalledFunction(F);
419 } else if (II->getIntrinsicID() == Intrinsic::expect) {
420 Function *F = Intrinsic::getOrInsertDeclaration(
421 M: II->getModule(), id: Intrinsic::SPVIntrinsics::spv_expect,
422 OverloadTys: {II->getOperand(i_nocapture: 0)->getType()});
423 II->setCalledFunction(F);
424 } else {
425 llvm_unreachable("Unknown intrinsic");
426 }
427}
428
429static bool toSpvLifetimeIntrinsic(IntrinsicInst *II, Intrinsic::ID NewID) {
430 auto *LifetimeArg0 = II->getArgOperand(i: 0);
431
432 // If the lifetime argument is a poison value, the intrinsic has no effect.
433 if (isa<PoisonValue>(Val: LifetimeArg0)) {
434 II->eraseFromParent();
435 return true;
436 }
437
438 IRBuilder<> Builder(II);
439 auto *Alloca = cast<AllocaInst>(Val: LifetimeArg0);
440 std::optional<TypeSize> Size =
441 Alloca->getAllocationSize(DL: Alloca->getDataLayout());
442 Value *SizeVal = Builder.getInt64(C: Size ? *Size : -1);
443 Builder.CreateIntrinsic(ID: NewID, OverloadTypes: Alloca->getType(), Args: {SizeVal, LifetimeArg0});
444 II->eraseFromParent();
445 return true;
446}
447
448static void
449lowerConstrainedFmuladd(IntrinsicInst *II,
450 SmallVector<Instruction *> &EraseFromParent) {
451 auto *FPI = cast<ConstrainedFPIntrinsic>(Val: II);
452 Value *A = FPI->getArgOperand(i: 0);
453 Value *Mul = FPI->getArgOperand(i: 1);
454 Value *Add = FPI->getArgOperand(i: 2);
455 IRBuilder<> Builder(II->getParent());
456 Builder.SetInsertPoint(II);
457 std::optional<RoundingMode> Rounding = FPI->getRoundingMode();
458 Value *Product = Builder.CreateFMul(L: A, R: Mul, Name: II->getName() + ".mul");
459 Value *Result = Builder.CreateConstrainedFPBinOp(
460 ID: Intrinsic::experimental_constrained_fadd, L: Product, R: Add, FMFSource: {},
461 Name: II->getName() + ".add", FPMathTag: nullptr, Rounding);
462 II->replaceAllUsesWith(V: Result);
463 EraseFromParent.push_back(Elt: II);
464}
465
466// Substitutes calls to LLVM intrinsics with either calls to SPIR-V intrinsics
467// or calls to proper generated functions. Returns True if F was modified.
468bool SPIRVPrepareFunctionsImpl::substituteIntrinsicCalls(Function *F) {
469 if (F->isDeclaration())
470 return false;
471
472 bool Changed = false;
473 const SPIRVSubtarget &STI = TM.getSubtarget<SPIRVSubtarget>(F: *F);
474 SmallVector<Instruction *> EraseFromParent;
475 const TargetTransformInfo &TTI = GetTTI(*F);
476 for (BasicBlock &BB : *F) {
477 for (Instruction &I : make_early_inc_range(Range&: BB)) {
478 auto Call = dyn_cast<CallInst>(Val: &I);
479 if (!Call)
480 continue;
481 Function *CF = Call->getCalledFunction();
482 if (!CF || !CF->isIntrinsic())
483 continue;
484 auto *II = cast<IntrinsicInst>(Val: Call);
485 if (Intrinsic::isTargetIntrinsic(IID: II->getIntrinsicID()) &&
486 II->getCalledOperand()->getName().starts_with(Prefix: "llvm.spv"))
487 continue;
488 switch (II->getIntrinsicID()) {
489 case Intrinsic::memset:
490 case Intrinsic::bswap:
491 Changed |= lowerIntrinsicToFunction(Intrinsic: II, TTI);
492 break;
493 case Intrinsic::fshl:
494 case Intrinsic::fshr:
495 lowerFunnelShifts(FSHIntrinsic: II);
496 Changed = true;
497 break;
498 case Intrinsic::assume:
499 case Intrinsic::expect:
500 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume))
501 lowerExpectAssume(II);
502 Changed = true;
503 break;
504 case Intrinsic::lifetime_start:
505 if (!STI.isShader()) {
506 Changed |= toSpvLifetimeIntrinsic(
507 II, NewID: Intrinsic::SPVIntrinsics::spv_lifetime_start);
508 } else {
509 II->eraseFromParent();
510 Changed = true;
511 }
512 break;
513 case Intrinsic::lifetime_end:
514 if (!STI.isShader()) {
515 Changed |= toSpvLifetimeIntrinsic(
516 II, NewID: Intrinsic::SPVIntrinsics::spv_lifetime_end);
517 } else {
518 II->eraseFromParent();
519 Changed = true;
520 }
521 break;
522 case Intrinsic::ptr_annotation:
523 lowerPtrAnnotation(II);
524 Changed = true;
525 break;
526 case Intrinsic::experimental_constrained_fmuladd:
527 lowerConstrainedFmuladd(II, EraseFromParent);
528 Changed = true;
529 break;
530 case Intrinsic::experimental_constrained_fcmp:
531 case Intrinsic::experimental_constrained_fcmps:
532 lowerConstrainedFPCmpIntrinsic(ConstrainedCmpIntrinsic: dyn_cast<ConstrainedFPCmpIntrinsic>(Val: II),
533 EraseFromParent);
534 Changed = true;
535 break;
536 default:
537 // Drop assume-like intrinsics that have no SPIR-V representation.
538 if (II->isAssumeLikeIntrinsic()) {
539 if (!II->getType()->isVoidTy())
540 II->replaceAllUsesWith(V: PoisonValue::get(T: II->getType()));
541 II->eraseFromParent();
542 Changed = true;
543 break;
544 }
545 if (TM.getTargetTriple().getVendor() == Triple::AMD ||
546 any_of(Range&: SPVAllowUnknownIntrinsics, P: [II](auto &&Prefix) {
547 if (Prefix.empty())
548 return false;
549 return II->getCalledFunction()->getName().starts_with(Prefix);
550 }))
551 Changed |= lowerIntrinsicToFunction(Intrinsic: II, TTI);
552 break;
553 }
554 }
555 }
556 for (auto *I : EraseFromParent)
557 I->eraseFromParent();
558 return Changed;
559}
560
561static void
562addFunctionTypeMutation(NamedMDNode *NMD,
563 SmallVector<std::pair<int, Type *>> ChangedTys,
564 StringRef Name, StringRef AsmConstraints = "") {
565
566 LLVMContext &Ctx = NMD->getParent()->getContext();
567 Type *I32Ty = IntegerType::getInt32Ty(C&: Ctx);
568
569 SmallVector<Metadata *> MDArgs;
570 MDArgs.push_back(Elt: MDString::get(Context&: Ctx, Str: Name));
571 transform(Range&: ChangedTys, d_first: std::back_inserter(x&: MDArgs), F: [=, &Ctx](auto &&CTy) {
572 return MDNode::get(
573 Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(I32Ty, CTy.first, true)),
574 ValueAsMetadata::get(V: Constant::getNullValue(Ty: CTy.second))});
575 });
576 if (!AsmConstraints.empty())
577 MDArgs.push_back(Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: AsmConstraints)));
578 NMD->addOperand(M: MDNode::get(Context&: Ctx, MDs: MDArgs));
579}
580
581// Returns F if aggregate argument/return types are not present or cloned F
582// function with the types replaced by i32 types. The change in types is
583// noted in 'spv.cloned_funcs' metadata for later restoration.
584Function *
585SPIRVPrepareFunctionsImpl::removeAggregateTypesFromSignature(Function *F) {
586 bool IsRetAggr = F->getReturnType()->isAggregateType();
587 // Allow intrinsics with aggregate return/argument types to reach GlobalISel.
588 // Renaming/mutating the signature of an intrinsic would desync its name from
589 // its argument types and break the IR verifier.
590 if (F->isIntrinsic())
591 return F;
592
593 IRBuilder<> B(F->getContext());
594
595 bool HasAggrArg = llvm::any_of(Range: F->args(), P: [](Argument &Arg) {
596 return Arg.getType()->isAggregateType();
597 });
598 bool DoClone = IsRetAggr || HasAggrArg;
599 if (!DoClone)
600 return F;
601 SmallVector<std::pair<int, Type *>, 4> ChangedTypes;
602 Type *RetType = IsRetAggr ? B.getInt32Ty() : F->getReturnType();
603 if (IsRetAggr)
604 ChangedTypes.push_back(Elt: std::pair<int, Type *>(-1, F->getReturnType()));
605 SmallVector<Type *, 4> ArgTypes;
606 for (const auto &Arg : F->args()) {
607 if (Arg.getType()->isAggregateType()) {
608 ArgTypes.push_back(Elt: B.getInt32Ty());
609 ChangedTypes.push_back(
610 Elt: std::pair<int, Type *>(Arg.getArgNo(), Arg.getType()));
611 } else
612 ArgTypes.push_back(Elt: Arg.getType());
613 }
614 FunctionType *NewFTy =
615 FunctionType::get(Result: RetType, Params: ArgTypes, isVarArg: F->getFunctionType()->isVarArg());
616 Function *NewF =
617 Function::Create(Ty: NewFTy, Linkage: F->getLinkage(), AddrSpace: F->getAddressSpace(),
618 N: F->getName(), M: F->getParent());
619
620 ValueToValueMapTy VMap;
621 auto NewFArgIt = NewF->arg_begin();
622 for (auto &Arg : F->args()) {
623 StringRef ArgName = Arg.getName();
624 NewFArgIt->setName(ArgName);
625 VMap[&Arg] = &(*NewFArgIt++);
626 }
627 SmallVector<ReturnInst *, 8> Returns;
628
629 CloneFunctionInto(NewFunc: NewF, OldFunc: F, VMap, Changes: CloneFunctionChangeType::LocalChangesOnly,
630 Returns);
631 NewF->takeName(V: F);
632 NewF->setComdat(F->getComdat());
633
634 addFunctionTypeMutation(
635 NMD: NewF->getParent()->getOrInsertNamedMetadata(Name: "spv.cloned_funcs"),
636 ChangedTys: std::move(ChangedTypes), Name: NewF->getName());
637
638 for (User *U : F->users()) {
639 if (auto *CB = dyn_cast<CallBase>(Val: U); CB && CB->getCalledFunction() == F)
640 CB->mutateFunctionType(FTy: NewF->getFunctionType());
641 }
642 // NewF keeps F's address space, so their pointer types match and
643 // RAUW is safe despite the differing signatures.
644 assert(F->getType() == NewF->getType() &&
645 "RAUW requires F and NewF to share the same pointer type");
646 F->replaceAllUsesWith(V: NewF);
647
648 // register the mutation
649 if (RetType != F->getReturnType())
650 TM.getSubtarget<SPIRVSubtarget>(F: *F).getSPIRVGlobalRegistry()->addMutated(
651 Val: NewF, Ty: F->getReturnType());
652 return NewF;
653}
654
655// Returns true iff `F`'s name resolves (after OpenCL/SPIR-V demangling and
656// builtin-name lookup) to the SPIR-V friendly built-in `__spirv_AbortKHR`.
657static bool isAbortKHRBuiltin(const Function &F) {
658 if (F.isIntrinsic())
659 return false;
660 StringRef Name = F.getName();
661 // Quick reject: the mangled or unmangled name must contain the substring.
662 if (!Name.contains(Other: "__spirv_AbortKHR"))
663 return false;
664 std::string Demangled = getOclOrSpirvBuiltinDemangledName(Name);
665 if (Demangled.empty())
666 return false;
667 return SPIRV::lookupBuiltinNameHelper(DemangledCall: Demangled) == "__spirv_AbortKHR";
668}
669
670// Rewrites a single call to `__spirv_AbortKHR` into a call to the
671// `llvm.spv.abort` target intrinsic, then re-terminates the block with
672// `unreachable`. OpAbortKHR is itself a SPIR-V function-termination
673// instruction and must be the last instruction in its block, so any trailing
674// stores/lifetime intrinsics/`ret` emitted by the OpenCL ABI are dropped.
675// `changeToUnreachable` cleans up any successor PHI predecessor entries.
676static void rewriteAbortKHRCall(CallInst *CI) {
677 IRBuilder<> B(CI);
678 Value *Msg = CI->getArgOperand(i: 0);
679 // The OpenCL C ABI may pass aggregate arguments by pointer (byval). In that
680 // case load the underlying value so that OpAbortKHR receives the composite
681 // itself, as required by the SPV_KHR_abort spec ("Message Type must be a
682 // concrete type").
683 if (CI->isByValArgument(ArgNo: 0)) {
684 Type *AggTy = CI->getParamByValType(ArgNo: 0);
685 Msg = B.CreateLoad(Ty: AggTy, Ptr: Msg);
686 }
687 B.CreateIntrinsic(ID: Intrinsic::spv_abort, OverloadTypes: {Msg->getType()}, Args: {Msg});
688 changeToUnreachable(I: CI);
689}
690
691// Replace OpenCL/SPIR-V style calls to `__spirv_AbortKHR(message)` (i.e.
692// calls to `F` when `F` is the `__spirv_AbortKHR` built-in) with calls to the
693// `llvm.spv.abort` target intrinsic.
694bool SPIRVPrepareFunctionsImpl::substituteAbortKHRCalls(Function *F) {
695 if (!isAbortKHRBuiltin(F: *F))
696 return false;
697
698 bool Changed = false;
699 for (User *U : make_early_inc_range(Range: F->users())) {
700 auto *CI = dyn_cast<CallInst>(Val: U);
701 if (!CI || CI->getCalledFunction() != F)
702 continue;
703 if (CI->arg_size() != 1)
704 continue;
705 rewriteAbortKHRCall(CI);
706 Changed = true;
707 }
708
709 return Changed;
710}
711
712// When the SPV_KHR_abort extension is enabled, `llvm.trap` and
713// `llvm.ubsantrap` are lowered to `OpAbortKHR` during instruction selection.
714// `OpAbortKHR` is itself a SPIR-V block terminator, so any instructions that
715// follow the trap call within the same basic block (e.g. `ret`, lifetime
716// markers) would produce SPIR-V ops after `OpAbortKHR` and break validation.
717// Terminate the block right after each call to the trap intrinsics by replacing
718// the next instruction with `unreachable`.
719bool SPIRVPrepareFunctionsImpl::terminateBlocksAfterTrap(Module &M,
720 Intrinsic::ID IID) {
721 assert((IID == Intrinsic::trap || IID == Intrinsic::ubsantrap) &&
722 "Expected trap intrinsic ID");
723
724 Function *F = Intrinsic::getDeclarationIfExists(M: &M, id: IID);
725 if (!F)
726 return false;
727
728 // If the target doesn't support SPV_KHR_abort, we won't be able to lower
729 // the trap intrinsic to OpAbortKHR, so we can skip the block-terminating
730 // transformation.
731 const auto &ST = TM.getSubtarget<SPIRVSubtarget>(F: *F);
732 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_abort))
733 return false;
734
735 bool Changed = false;
736 for (User *U : make_early_inc_range(Range: F->users())) {
737 auto *CI = dyn_cast<CallInst>(Val: U);
738 if (!CI || CI->getCalledFunction() != F)
739 continue;
740 Instruction *Next = CI->getNextNode();
741 if (!Next || isa<UnreachableInst>(Val: Next))
742 continue;
743 changeToUnreachable(I: Next);
744 Changed = true;
745 }
746 return Changed;
747}
748
749static std::string fixMultiOutputConstraintString(StringRef Constraints) {
750 // We should only have one =r return for the made up ASM type.
751 SmallVector<StringRef> Tmp;
752 SplitString(Source: Constraints, OutFragments&: Tmp, Delimiters: ",");
753 std::string SafeConstraints("=r,");
754 for (unsigned I = 0u; I != Tmp.size() - 1; ++I) {
755 if (Tmp[I].starts_with(Prefix: '=') && (Tmp[I][1] == '&' || isalnum(Tmp[I][1])))
756 continue;
757 SafeConstraints.append(svt: Tmp[I]).append(l: {','});
758 }
759 SafeConstraints.append(svt: Tmp.back());
760
761 return SafeConstraints;
762}
763
764// Mutates indirect and inline ASM callsites iff aggregate argument/return types
765// are present with the types replaced by i32 types. The change in types is
766// noted in 'spv.mutated_callsites' metadata for later restoration. For ASM we
767// also have to mutate the constraint string as IRTranslator tries to handle
768// multiple outputs and expects an aggregate return type in their presence.
769bool SPIRVPrepareFunctionsImpl::removeAggregateTypesFromCalls(Function *F) {
770 if (F->isDeclaration() || F->isIntrinsic())
771 return false;
772
773 SmallVector<std::pair<CallBase *, FunctionType *>> Calls;
774 for (auto &&I : instructions(F)) {
775 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
776 if (!CB->getCalledOperand() || CB->getCalledFunction())
777 continue;
778 if (CB->getType()->isAggregateType() ||
779 any_of(Range: CB->args(),
780 P: [](auto &&Arg) { return Arg->getType()->isAggregateType(); }))
781 Calls.emplace_back(Args&: CB, Args: nullptr);
782 }
783 }
784
785 if (Calls.empty())
786 return false;
787
788 IRBuilder<> B(F->getContext());
789
790 unsigned MutatedCallIdx = 0;
791 for (auto &&[CB, NewFnTy] : Calls) {
792 SmallVector<std::pair<int, Type *>> ChangedTypes;
793 SmallVector<Type *> NewArgTypes;
794
795 Type *RetTy = CB->getType();
796 if (RetTy->isAggregateType()) {
797 ChangedTypes.emplace_back(Args: -1, Args&: RetTy);
798 RetTy = B.getInt32Ty();
799 }
800
801 for (auto &&Arg : CB->args()) {
802 if (Arg->getType()->isAggregateType()) {
803 NewArgTypes.push_back(Elt: B.getInt32Ty());
804 ChangedTypes.emplace_back(Args: Arg.getOperandNo(), Args: Arg->getType());
805 } else {
806 NewArgTypes.push_back(Elt: Arg->getType());
807 }
808 }
809 NewFnTy = FunctionType::get(Result: RetTy, Params: NewArgTypes,
810 isVarArg: CB->getFunctionType()->isVarArg());
811
812 // Keyed via instruction metadata, not a name.
813 std::string Key =
814 ("spv.mutated_callsite." + F->getName() + "." + Twine(MutatedCallIdx++))
815 .str();
816 CB->setMetadata(
817 Kind: "spv.mutated_callsite",
818 Node: MDNode::get(Context&: F->getContext(), MDs: MDString::get(Context&: F->getContext(), Str: Key)));
819
820 std::string Constraints;
821 if (auto *ASM = dyn_cast<InlineAsm>(Val: CB->getCalledOperand())) {
822 Constraints = ASM->getConstraintString();
823
824 CB->setCalledOperand(InlineAsm::get(
825 Ty: NewFnTy, AsmString: ASM->getAsmString(),
826 Constraints: fixMultiOutputConstraintString(Constraints), hasSideEffects: ASM->hasSideEffects(),
827 isAlignStack: ASM->isAlignStack(), asmDialect: ASM->getDialect(), canThrow: ASM->canThrow()));
828 }
829
830 addFunctionTypeMutation(
831 NMD: F->getParent()->getOrInsertNamedMetadata(Name: "spv.mutated_callsites"),
832 ChangedTys: std::move(ChangedTypes), Name: Key, AsmConstraints: Constraints);
833 }
834
835 for (auto &&[CB, NewFTy] : Calls) {
836 if (NewFTy->getReturnType() != CB->getType())
837 TM.getSubtarget<SPIRVSubtarget>(F: *F).getSPIRVGlobalRegistry()->addMutated(
838 Val: CB, Ty: CB->getType());
839 CB->mutateFunctionType(FTy: NewFTy);
840 }
841
842 return true;
843}
844
845bool SPIRVPrepareFunctionsImpl::runOnModule(Module &M) {
846 // Resolve the SPIR-V environment from module content before any
847 // function-level processing. This must happen before legalization so that
848 // isShader()/isKernel() return correct values.
849 const_cast<SPIRVTargetMachine &>(TM)
850 .getMutableSubtargetImpl()
851 ->resolveEnvFromModule(M);
852
853 bool Changed = false;
854 if (M.getFunctionDefs().empty()) {
855 // If there are no function definitions, insert a service
856 // function so that the global/constant tracking intrinsics
857 // will be created. Without these intrinsics the generated SPIR-V
858 // will be empty. The service function itself is not emitted.
859 Function *SF = getOrCreateBackendServiceFunction(M);
860 BasicBlock *BB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: SF);
861 IRBuilder<> IRB(BB);
862 IRB.CreateRetVoid();
863 Changed = true;
864 }
865
866 Changed |= terminateBlocksAfterTrap(M, IID: Intrinsic::trap);
867 Changed |= terminateBlocksAfterTrap(M, IID: Intrinsic::ubsantrap);
868
869 for (GlobalVariable &GV : M.globals()) {
870 // Strip + tag available_externally globals so AuxData can re-emit the
871 // original linkage as NonSemantic.AuxData::Linkage.
872 if (GV.hasAvailableExternallyLinkage() && !GV.isDeclaration()) {
873 GV.addAttribute(SPIRV_WAS_AVAILABLE_EXTERNALLY_ATTR);
874 GV.setLinkage(GlobalValue::ExternalLinkage);
875 Changed = true;
876 }
877 }
878
879 std::vector<Function *> FuncsWorklist;
880 for (Function &F : M) {
881 // MachineFunctionPass skips available_externally; strip + tag so AuxData
882 // can re-emit the original linkage as NonSemantic.AuxData::Linkage.
883 if (F.hasAvailableExternallyLinkage() && !F.isDeclaration()) {
884 F.addFnAttr(SPIRV_WAS_AVAILABLE_EXTERNALLY_ATTR);
885 F.setLinkage(GlobalValue::ExternalLinkage);
886 Changed = true;
887 }
888 Changed |= substituteAbortKHRCalls(F: &F);
889 Changed |= substituteIntrinsicCalls(F: &F);
890 Changed |= sortBlocks(F);
891 Changed |= removeAggregateTypesFromCalls(F: &F);
892 FuncsWorklist.push_back(x: &F);
893 }
894
895 for (auto *F : FuncsWorklist) {
896 Function *NewF = removeAggregateTypesFromSignature(F);
897
898 if (NewF != F) {
899 F->eraseFromParent();
900 Changed = true;
901 }
902 }
903 return Changed;
904}
905
906PreservedAnalyses SPIRVPrepareFunctionsPass::run(Module &M,
907 ModuleAnalysisManager &AM) {
908 FunctionAnalysisManager &FAM =
909 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
910 auto GetTTI = [&FAM](Function &F) -> const TargetTransformInfo & {
911 return FAM.getResult<TargetIRAnalysis>(IR&: F);
912 };
913 return SPIRVPrepareFunctionsImpl(TM, GetTTI).runOnModule(M)
914 ? PreservedAnalyses::none()
915 : PreservedAnalyses::all();
916}
917
918ModulePass *
919llvm::createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM) {
920 return new SPIRVPrepareFunctionsLegacy(TM);
921}
922