1//===-- SPIRVPrepareFunctions.cpp - modify function signatures --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass modifies function signatures containing aggregate arguments
10// and/or return value before IRTranslator. Information about the original
11// signatures is stored in metadata. It is used during call lowering to
12// restore correct SPIR-V types of function arguments and return values.
13// This pass also substitutes some llvm intrinsic calls with calls to newly
14// generated functions (as the Khronos LLVM/SPIR-V Translator does).
15//
16// NOTE: this pass is a module-level one due to the necessity to modify
17// GVs/functions.
18//
19//===----------------------------------------------------------------------===//
20
21#include "SPIRVPrepareFunctions.h"
22#include "SPIRV.h"
23#include "SPIRVBuiltins.h"
24#include "SPIRVSubtarget.h"
25#include "SPIRVTargetMachine.h"
26#include "SPIRVUtils.h"
27#include "llvm/ADT/StringExtras.h"
28#include "llvm/Analysis/TargetTransformInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/CodeGen/IntrinsicLowering.h"
31#include "llvm/IR/DiagnosticInfo.h"
32#include "llvm/IR/IRBuilder.h"
33#include "llvm/IR/InstIterator.h"
34#include "llvm/IR/Instructions.h"
35#include "llvm/IR/IntrinsicInst.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/IntrinsicsSPIRV.h"
38#include "llvm/Transforms/Utils/Cloning.h"
39#include "llvm/Transforms/Utils/Local.h"
40#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
41#include <regex>
42
43using namespace llvm;
44
45namespace {
46
47class SPIRVPrepareFunctionsImpl {
48 const SPIRVTargetMachine &TM;
49 bool substituteIntrinsicCalls(Function *F);
50 bool substituteAbortKHRCalls(Function *F);
51 bool terminateBlocksAfterTrap(Module &M, Intrinsic::ID IID);
52 Function *removeAggregateTypesFromSignature(Function *F);
53 bool removeAggregateTypesFromCalls(Function *F);
54
55public:
56 SPIRVPrepareFunctionsImpl(const SPIRVTargetMachine &TM) : TM(TM) {}
57 bool runOnModule(Module &M);
58};
59
60class SPIRVPrepareFunctionsLegacy : public ModulePass {
61 const SPIRVTargetMachine &TM;
62
63public:
64 static char ID;
65 SPIRVPrepareFunctionsLegacy(const SPIRVTargetMachine &TM)
66 : ModulePass(ID), TM(TM) {}
67
68 bool runOnModule(Module &M) override {
69 return SPIRVPrepareFunctionsImpl(TM).runOnModule(M);
70 }
71
72 StringRef getPassName() const override { return "SPIRV prepare functions"; }
73};
74
75static cl::list<std::string> SPVAllowUnknownIntrinsics(
76 "spv-allow-unknown-intrinsics", cl::CommaSeparated,
77 cl::desc("Emit unknown intrinsics as calls to external functions. A "
78 "comma-separated input list of intrinsic prefixes must be "
79 "provided, and only intrinsics carrying a listed prefix get "
80 "emitted as described."),
81 cl::value_desc("intrinsic_prefix_0,intrinsic_prefix_1"), cl::ValueOptional);
82} // namespace
83
84char SPIRVPrepareFunctionsLegacy::ID = 0;
85
86INITIALIZE_PASS(SPIRVPrepareFunctionsLegacy, "spirv-prepare-functions",
87 "SPIRV prepare functions", false, false)
88
89static std::string lowerLLVMIntrinsicName(IntrinsicInst *II) {
90 Function *IntrinsicFunc = II->getCalledFunction();
91 assert(IntrinsicFunc && "Missing function");
92 std::string FuncName = IntrinsicFunc->getName().str();
93 llvm::replace(Range&: FuncName, OldValue: '.', NewValue: '_');
94 FuncName = "spirv." + FuncName;
95 return FuncName;
96}
97
98static Function *getOrCreateFunction(Module *M, Type *RetTy,
99 ArrayRef<Type *> ArgTypes,
100 StringRef Name) {
101 FunctionType *FT = FunctionType::get(Result: RetTy, Params: ArgTypes, isVarArg: false);
102 Function *F = M->getFunction(Name);
103 if (F && F->getFunctionType() == FT)
104 return F;
105 Function *NewF = Function::Create(Ty: FT, Linkage: GlobalValue::ExternalLinkage, N: Name, M);
106 if (F)
107 NewF->setDSOLocal(F->isDSOLocal());
108 NewF->setCallingConv(CallingConv::SPIR_FUNC);
109 return NewF;
110}
111
112static bool lowerIntrinsicToFunction(IntrinsicInst *Intrinsic,
113 const TargetTransformInfo &TTI) {
114 // For @llvm.memset.* intrinsic cases with constant value and length arguments
115 // are emulated via "storing" a constant array to the destination. For other
116 // cases we wrap the intrinsic in @spirv.llvm_memset_* function and expand the
117 // intrinsic to a loop via expandMemSetAsLoop().
118 if (auto *MSI = dyn_cast<MemSetInst>(Val: Intrinsic))
119 if (isa<Constant>(Val: MSI->getValue()) && isa<ConstantInt>(Val: MSI->getLength()))
120 return false; // It is handled later using OpCopyMemorySized.
121
122 // An intrinsic with a metadata argument has no SPIR-V lowering and can't be
123 // turned into a function.
124 if (any_of(Range: Intrinsic->args(), P: IsaPred<MetadataAsValue>)) {
125 const Function *F = Intrinsic->getFunction();
126 F->getContext().diagnose(DI: DiagnosticInfoUnsupported(
127 *F,
128 "cannot lower the intrinsic '" +
129 Intrinsic->getCalledFunction()->getName() +
130 "' that takes a metadata argument",
131 Intrinsic->getDebugLoc()));
132 if (!Intrinsic->getType()->isVoidTy())
133 Intrinsic->replaceAllUsesWith(V: PoisonValue::get(T: Intrinsic->getType()));
134 Intrinsic->eraseFromParent();
135 return true;
136 }
137
138 Module *M = Intrinsic->getModule();
139 std::string FuncName = lowerLLVMIntrinsicName(II: Intrinsic);
140 if (Intrinsic->isVolatile())
141 FuncName += ".volatile";
142 // Redirect @llvm.intrinsic.* call to @spirv.llvm_intrinsic_*
143 Function *F = M->getFunction(Name: FuncName);
144 if (F) {
145 Intrinsic->setCalledFunction(F);
146 return true;
147 }
148 FunctionCallee FC =
149 M->getOrInsertFunction(Name: FuncName, T: Intrinsic->getFunctionType());
150 auto IntrinsicID = Intrinsic->getIntrinsicID();
151 Intrinsic->setCalledFunction(FC);
152 F = cast<Function>(Val: FC.getCallee());
153 F->setAttributes(Intrinsic->getAttributes());
154
155 switch (IntrinsicID) {
156 case Intrinsic::memset: {
157 auto *MSI = static_cast<MemSetInst *>(Intrinsic);
158 Argument *Dest = F->getArg(i: 0);
159 Argument *Val = F->getArg(i: 1);
160 Argument *Len = F->getArg(i: 2);
161 Argument *IsVolatile = F->getArg(i: 3);
162 Dest->setName("dest");
163 Val->setName("val");
164 Len->setName("len");
165 IsVolatile->setName("isvolatile");
166 BasicBlock *EntryBB = BasicBlock::Create(Context&: M->getContext(), Name: "entry", Parent: F);
167 IRBuilder<> IRB(EntryBB);
168 auto *MemSet = IRB.CreateMemSet(Ptr: Dest, Val, Size: Len, Align: MSI->getDestAlign(),
169 isVolatile: MSI->isVolatile());
170 IRB.CreateRetVoid();
171 expandMemSetAsLoop(MemSet: cast<MemSetInst>(Val: MemSet), TTI);
172 MemSet->eraseFromParent();
173 break;
174 }
175 case Intrinsic::bswap: {
176 BasicBlock *EntryBB = BasicBlock::Create(Context&: M->getContext(), Name: "entry", Parent: F);
177 IRBuilder<> IRB(EntryBB);
178 CallInst *BSwap = IRB.CreateIntrinsicWithoutFolding(
179 ID: Intrinsic::bswap, OverloadTypes: Intrinsic->getType(), Args: F->getArg(i: 0));
180 IRB.CreateRet(V: BSwap);
181 IntrinsicLowering IL(M->getDataLayout());
182 IL.LowerIntrinsicCall(CI: BSwap);
183 break;
184 }
185 default:
186 break;
187 }
188 return true;
189}
190
191static std::string getAnnotation(Value *AnnoVal, Value *OptAnnoVal) {
192 if (auto *Ref = dyn_cast_or_null<GetElementPtrInst>(Val: AnnoVal))
193 AnnoVal = Ref->getOperand(i_nocapture: 0);
194 if (auto *Ref = dyn_cast_or_null<BitCastInst>(Val: OptAnnoVal))
195 OptAnnoVal = Ref->getOperand(i_nocapture: 0);
196
197 std::string Anno;
198 if (auto *C = dyn_cast_or_null<Constant>(Val: AnnoVal)) {
199 StringRef Str;
200 if (getConstantStringInfo(V: C, Str))
201 Anno = Str;
202 }
203 // handle optional annotation parameter in a way that Khronos Translator do
204 // (collect integers wrapped in a struct)
205 if (auto *C = dyn_cast_or_null<Constant>(Val: OptAnnoVal);
206 C && C->getNumOperands()) {
207 Value *MaybeStruct = C->getOperand(i: 0);
208 if (auto *Struct = dyn_cast<ConstantStruct>(Val: MaybeStruct)) {
209 for (unsigned I = 0, E = Struct->getNumOperands(); I != E; ++I) {
210 if (auto *CInt = dyn_cast<ConstantInt>(Val: Struct->getOperand(i_nocapture: I)))
211 Anno += (I == 0 ? ": " : ", ") +
212 std::to_string(val: CInt->getType()->getIntegerBitWidth() == 1
213 ? CInt->getZExtValue()
214 : CInt->getSExtValue());
215 }
216 } else if (auto *Struct = dyn_cast<ConstantAggregateZero>(Val: MaybeStruct)) {
217 // { i32 i32 ... } zeroinitializer
218 for (unsigned I = 0, E = Struct->getType()->getStructNumElements();
219 I != E; ++I)
220 Anno += I == 0 ? ": 0" : ", 0";
221 }
222 }
223 return Anno;
224}
225
226static SmallVector<Metadata *> parseAnnotation(Value *I,
227 const std::string &Anno,
228 LLVMContext &Ctx,
229 Type *Int32Ty) {
230 // Try to parse the annotation string according to the following rules:
231 // annotation := ({kind} | {kind:value,value,...})+
232 // kind := number
233 // value := number | string
234 static const std::regex R(
235 "\\{(\\d+)(?:[:,](\\d+|\"[^\"]*\")(?:,(\\d+|\"[^\"]*\"))*)?\\}");
236 SmallVector<Metadata *> MDs;
237 int Pos = 0;
238 for (std::sregex_iterator
239 It = std::sregex_iterator(Anno.begin(), Anno.end(), R),
240 ItEnd = std::sregex_iterator();
241 It != ItEnd; ++It) {
242 if (It->position() != Pos)
243 return SmallVector<Metadata *>{};
244 Pos = It->position() + It->length();
245 std::smatch Match = *It;
246 SmallVector<Metadata *> MDsItem;
247 for (std::size_t i = 1; i < Match.size(); ++i) {
248 std::ssub_match SMatch = Match[i];
249 std::string Item = SMatch.str();
250 if (Item.length() == 0)
251 break;
252 if (Item[0] == '"') {
253 Item = Item.substr(pos: 1, n: Item.length() - 2);
254 // Acceptable format of the string snippet is:
255 static const std::regex RStr("^(\\d+)(?:,(\\d+))*$");
256 if (std::smatch MatchStr; std::regex_match(s: Item, m&: MatchStr, re: RStr)) {
257 for (std::size_t SubIdx = 1; SubIdx < MatchStr.size(); ++SubIdx)
258 if (std::string SubStr = MatchStr[SubIdx].str(); SubStr.length())
259 MDsItem.push_back(Elt: ConstantAsMetadata::get(
260 C: ConstantInt::get(Ty: Int32Ty, V: std::stoi(str: SubStr))));
261 } else {
262 MDsItem.push_back(Elt: MDString::get(Context&: Ctx, Str: Item));
263 }
264 } else if (int32_t Num; llvm::to_integer(S: StringRef(Item), Num, Base: 10)) {
265 MDsItem.push_back(
266 Elt: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: Num)));
267 } else {
268 MDsItem.push_back(Elt: MDString::get(Context&: Ctx, Str: Item));
269 }
270 }
271 if (MDsItem.size() == 0)
272 return SmallVector<Metadata *>{};
273 MDs.push_back(Elt: MDNode::get(Context&: Ctx, MDs: MDsItem));
274 }
275 return Pos == static_cast<int>(Anno.length()) ? std::move(MDs)
276 : SmallVector<Metadata *>{};
277}
278
279static void lowerPtrAnnotation(IntrinsicInst *II) {
280 LLVMContext &Ctx = II->getContext();
281 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
282
283 // Retrieve an annotation string from arguments.
284 Value *PtrArg = nullptr;
285 if (auto *BI = dyn_cast<BitCastInst>(Val: II->getArgOperand(i: 0)))
286 PtrArg = BI->getOperand(i_nocapture: 0);
287 else
288 PtrArg = II->getOperand(i_nocapture: 0);
289 std::string Anno =
290 getAnnotation(AnnoVal: II->getArgOperand(i: 1),
291 OptAnnoVal: 4 < II->arg_size() ? II->getArgOperand(i: 4) : nullptr);
292
293 // Parse the annotation.
294 SmallVector<Metadata *> MDs = parseAnnotation(I: II, Anno, Ctx, Int32Ty);
295
296 // If the annotation string is not parsed successfully we don't know the
297 // format used and output it as a general UserSemantic decoration.
298 // Otherwise MDs is a Metadata tuple (a decoration list) in the format
299 // expected by `spirv.Decorations`.
300 if (MDs.size() == 0) {
301 auto UserSemantic = ConstantAsMetadata::get(C: ConstantInt::get(
302 Ty: Int32Ty, V: static_cast<uint32_t>(SPIRV::Decoration::UserSemantic)));
303 MDs.push_back(Elt: MDNode::get(Context&: Ctx, MDs: {UserSemantic, MDString::get(Context&: Ctx, Str: Anno)}));
304 }
305
306 // Build the internal intrinsic function.
307 IRBuilder<> IRB(II->getParent());
308 IRB.SetInsertPoint(II);
309 IRB.CreateIntrinsic(
310 ID: Intrinsic::spv_assign_decoration, OverloadTypes: {PtrArg->getType()},
311 Args: {PtrArg, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs))});
312 II->replaceAllUsesWith(V: II->getOperand(i_nocapture: 0));
313}
314
315static void lowerFunnelShifts(IntrinsicInst *FSHIntrinsic) {
316 // Get a separate function - otherwise, we'd have to rework the CFG of the
317 // current one. Then simply replace the intrinsic uses with a call to the new
318 // function.
319 // Generate LLVM IR for i* @spirv.llvm_fsh?_i* (i* %a, i* %b, i* %c)
320 Module *M = FSHIntrinsic->getModule();
321 FunctionType *FSHFuncTy = FSHIntrinsic->getFunctionType();
322 Type *FSHRetTy = FSHFuncTy->getReturnType();
323 const std::string FuncName = lowerLLVMIntrinsicName(II: FSHIntrinsic);
324 Function *FSHFunc =
325 getOrCreateFunction(M, RetTy: FSHRetTy, ArgTypes: FSHFuncTy->params(), Name: FuncName);
326
327 if (!FSHFunc->empty()) {
328 FSHIntrinsic->setCalledFunction(FSHFunc);
329 return;
330 }
331 BasicBlock *RotateBB = BasicBlock::Create(Context&: M->getContext(), Name: "rotate", Parent: FSHFunc);
332 IRBuilder<> IRB(RotateBB);
333 Type *Ty = FSHFunc->getReturnType();
334 // Build the actual funnel shift rotate logic.
335 // In the comments, "int" is used interchangeably with "vector of int
336 // elements".
337 FixedVectorType *VectorTy = dyn_cast<FixedVectorType>(Val: Ty);
338 Type *IntTy = VectorTy ? VectorTy->getElementType() : Ty;
339 unsigned BitWidth = IntTy->getIntegerBitWidth();
340 ConstantInt *BitWidthConstant = IRB.getInt(AI: {BitWidth, BitWidth});
341 Value *BitWidthForInsts =
342 VectorTy
343 ? IRB.CreateVectorSplat(NumElts: VectorTy->getNumElements(), V: BitWidthConstant)
344 : BitWidthConstant;
345 Value *RotateModVal =
346 IRB.CreateURem(/*Rotate*/ LHS: FSHFunc->getArg(i: 2), RHS: BitWidthForInsts);
347 Value *FirstShift = nullptr, *SecShift = nullptr;
348 if (FSHIntrinsic->getIntrinsicID() == Intrinsic::fshr) {
349 // Shift the less significant number right, the "rotate" number of bits
350 // will be 0-filled on the left as a result of this regular shift.
351 FirstShift = IRB.CreateLShr(LHS: FSHFunc->getArg(i: 1), RHS: RotateModVal);
352 } else {
353 // Shift the more significant number left, the "rotate" number of bits
354 // will be 0-filled on the right as a result of this regular shift.
355 FirstShift = IRB.CreateShl(LHS: FSHFunc->getArg(i: 0), RHS: RotateModVal);
356 }
357 // We want the "rotate" number of the more significant int's LSBs (MSBs) to
358 // occupy the leftmost (rightmost) "0 space" left by the previous operation.
359 // Therefore, subtract the "rotate" number from the integer bitsize...
360 Value *SubRotateVal = IRB.CreateSub(LHS: BitWidthForInsts, RHS: RotateModVal);
361 if (FSHIntrinsic->getIntrinsicID() == Intrinsic::fshr) {
362 // ...and left-shift the more significant int by this number, zero-filling
363 // the LSBs.
364 SecShift = IRB.CreateShl(LHS: FSHFunc->getArg(i: 0), RHS: SubRotateVal);
365 } else {
366 // ...and right-shift the less significant int by this number, zero-filling
367 // the MSBs.
368 SecShift = IRB.CreateLShr(LHS: FSHFunc->getArg(i: 1), RHS: SubRotateVal);
369 }
370 // A simple binary addition of the shifted ints yields the final result.
371 IRB.CreateRet(V: IRB.CreateOr(LHS: FirstShift, RHS: SecShift));
372
373 FSHIntrinsic->setCalledFunction(FSHFunc);
374}
375
376static void lowerConstrainedFPCmpIntrinsic(
377 ConstrainedFPCmpIntrinsic *ConstrainedCmpIntrinsic,
378 SmallVector<Instruction *> &EraseFromParent) {
379 if (!ConstrainedCmpIntrinsic)
380 return;
381 // Extract the floating-point values being compared
382 Value *LHS = ConstrainedCmpIntrinsic->getArgOperand(i: 0);
383 Value *RHS = ConstrainedCmpIntrinsic->getArgOperand(i: 1);
384 FCmpInst::Predicate Pred = ConstrainedCmpIntrinsic->getPredicate();
385 IRBuilder<> Builder(ConstrainedCmpIntrinsic);
386 Value *FCmp = Builder.CreateFCmp(P: Pred, LHS, RHS);
387 ConstrainedCmpIntrinsic->replaceAllUsesWith(V: FCmp);
388 EraseFromParent.push_back(Elt: dyn_cast<Instruction>(Val: ConstrainedCmpIntrinsic));
389}
390
391static void lowerExpectAssume(IntrinsicInst *II) {
392 // If we cannot use the SPV_KHR_expect_assume extension, then we need to
393 // ignore the intrinsic and move on. It should be removed later on by LLVM.
394 // Otherwise we should lower the intrinsic to the corresponding SPIR-V
395 // instruction.
396 // For @llvm.assume we have OpAssumeTrueKHR.
397 // For @llvm.expect we have OpExpectKHR.
398 //
399 // We need to lower this into a builtin and then the builtin into a SPIR-V
400 // instruction.
401 if (II->getIntrinsicID() == Intrinsic::assume) {
402 Function *F = Intrinsic::getOrInsertDeclaration(
403 M: II->getModule(), id: Intrinsic::SPVIntrinsics::spv_assume);
404 II->setCalledFunction(F);
405 } else if (II->getIntrinsicID() == Intrinsic::expect) {
406 Function *F = Intrinsic::getOrInsertDeclaration(
407 M: II->getModule(), id: Intrinsic::SPVIntrinsics::spv_expect,
408 OverloadTys: {II->getOperand(i_nocapture: 0)->getType()});
409 II->setCalledFunction(F);
410 } else {
411 llvm_unreachable("Unknown intrinsic");
412 }
413}
414
415static bool toSpvLifetimeIntrinsic(IntrinsicInst *II, Intrinsic::ID NewID) {
416 auto *LifetimeArg0 = II->getArgOperand(i: 0);
417
418 // If the lifetime argument is a poison value, the intrinsic has no effect.
419 if (isa<PoisonValue>(Val: LifetimeArg0)) {
420 II->eraseFromParent();
421 return true;
422 }
423
424 IRBuilder<> Builder(II);
425 auto *Alloca = cast<AllocaInst>(Val: LifetimeArg0);
426 std::optional<TypeSize> Size =
427 Alloca->getAllocationSize(DL: Alloca->getDataLayout());
428 Value *SizeVal = Builder.getInt64(C: Size ? *Size : -1);
429 Builder.CreateIntrinsic(ID: NewID, OverloadTypes: Alloca->getType(), Args: {SizeVal, LifetimeArg0});
430 II->eraseFromParent();
431 return true;
432}
433
434static void
435lowerConstrainedFmuladd(IntrinsicInst *II,
436 SmallVector<Instruction *> &EraseFromParent) {
437 auto *FPI = cast<ConstrainedFPIntrinsic>(Val: II);
438 Value *A = FPI->getArgOperand(i: 0);
439 Value *Mul = FPI->getArgOperand(i: 1);
440 Value *Add = FPI->getArgOperand(i: 2);
441 IRBuilder<> Builder(II->getParent());
442 Builder.SetInsertPoint(II);
443 std::optional<RoundingMode> Rounding = FPI->getRoundingMode();
444 Value *Product = Builder.CreateFMul(L: A, R: Mul, Name: II->getName() + ".mul");
445 Value *Result = Builder.CreateConstrainedFPBinOp(
446 ID: Intrinsic::experimental_constrained_fadd, L: Product, R: Add, FMFSource: {},
447 Name: II->getName() + ".add", FPMathTag: nullptr, Rounding);
448 II->replaceAllUsesWith(V: Result);
449 EraseFromParent.push_back(Elt: II);
450}
451
452// Substitutes calls to LLVM intrinsics with either calls to SPIR-V intrinsics
453// or calls to proper generated functions. Returns True if F was modified.
454bool SPIRVPrepareFunctionsImpl::substituteIntrinsicCalls(Function *F) {
455 bool Changed = false;
456 const SPIRVSubtarget &STI = TM.getSubtarget<SPIRVSubtarget>(F: *F);
457 SmallVector<Instruction *> EraseFromParent;
458 const TargetTransformInfo &TTI = TM.getTargetTransformInfo(F: *F);
459 for (BasicBlock &BB : *F) {
460 for (Instruction &I : make_early_inc_range(Range&: BB)) {
461 auto Call = dyn_cast<CallInst>(Val: &I);
462 if (!Call)
463 continue;
464 Function *CF = Call->getCalledFunction();
465 if (!CF || !CF->isIntrinsic())
466 continue;
467 auto *II = cast<IntrinsicInst>(Val: Call);
468 if (Intrinsic::isTargetIntrinsic(IID: II->getIntrinsicID()) &&
469 II->getCalledOperand()->getName().starts_with(Prefix: "llvm.spv"))
470 continue;
471 switch (II->getIntrinsicID()) {
472 case Intrinsic::memset:
473 case Intrinsic::bswap:
474 Changed |= lowerIntrinsicToFunction(Intrinsic: II, TTI);
475 break;
476 case Intrinsic::fshl:
477 case Intrinsic::fshr:
478 lowerFunnelShifts(FSHIntrinsic: II);
479 Changed = true;
480 break;
481 case Intrinsic::assume:
482 case Intrinsic::expect:
483 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume))
484 lowerExpectAssume(II);
485 Changed = true;
486 break;
487 case Intrinsic::lifetime_start:
488 if (!STI.isShader()) {
489 Changed |= toSpvLifetimeIntrinsic(
490 II, NewID: Intrinsic::SPVIntrinsics::spv_lifetime_start);
491 } else {
492 II->eraseFromParent();
493 Changed = true;
494 }
495 break;
496 case Intrinsic::lifetime_end:
497 if (!STI.isShader()) {
498 Changed |= toSpvLifetimeIntrinsic(
499 II, NewID: Intrinsic::SPVIntrinsics::spv_lifetime_end);
500 } else {
501 II->eraseFromParent();
502 Changed = true;
503 }
504 break;
505 case Intrinsic::ptr_annotation:
506 lowerPtrAnnotation(II);
507 Changed = true;
508 break;
509 case Intrinsic::experimental_constrained_fmuladd:
510 lowerConstrainedFmuladd(II, EraseFromParent);
511 Changed = true;
512 break;
513 case Intrinsic::experimental_constrained_fcmp:
514 case Intrinsic::experimental_constrained_fcmps:
515 lowerConstrainedFPCmpIntrinsic(ConstrainedCmpIntrinsic: dyn_cast<ConstrainedFPCmpIntrinsic>(Val: II),
516 EraseFromParent);
517 Changed = true;
518 break;
519 default:
520 // Drop assume-like intrinsics that have no SPIR-V representation.
521 if (II->isAssumeLikeIntrinsic()) {
522 if (!II->getType()->isVoidTy())
523 II->replaceAllUsesWith(V: PoisonValue::get(T: II->getType()));
524 II->eraseFromParent();
525 Changed = true;
526 break;
527 }
528 if (TM.getTargetTriple().getVendor() == Triple::AMD ||
529 any_of(Range&: SPVAllowUnknownIntrinsics, P: [II](auto &&Prefix) {
530 if (Prefix.empty())
531 return false;
532 return II->getCalledFunction()->getName().starts_with(Prefix);
533 }))
534 Changed |= lowerIntrinsicToFunction(Intrinsic: II, TTI);
535 break;
536 }
537 }
538 }
539 for (auto *I : EraseFromParent)
540 I->eraseFromParent();
541 return Changed;
542}
543
544static void
545addFunctionTypeMutation(NamedMDNode *NMD,
546 SmallVector<std::pair<int, Type *>> ChangedTys,
547 StringRef Name, StringRef AsmConstraints = "") {
548
549 LLVMContext &Ctx = NMD->getParent()->getContext();
550 Type *I32Ty = IntegerType::getInt32Ty(C&: Ctx);
551
552 SmallVector<Metadata *> MDArgs;
553 MDArgs.push_back(Elt: MDString::get(Context&: Ctx, Str: Name));
554 transform(Range&: ChangedTys, d_first: std::back_inserter(x&: MDArgs), F: [=, &Ctx](auto &&CTy) {
555 return MDNode::get(
556 Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(I32Ty, CTy.first, true)),
557 ValueAsMetadata::get(V: Constant::getNullValue(Ty: CTy.second))});
558 });
559 if (!AsmConstraints.empty())
560 MDArgs.push_back(Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: AsmConstraints)));
561 NMD->addOperand(M: MDNode::get(Context&: Ctx, MDs: MDArgs));
562}
563
564// Returns F if aggregate argument/return types are not present or cloned F
565// function with the types replaced by i32 types. The change in types is
566// noted in 'spv.cloned_funcs' metadata for later restoration.
567Function *
568SPIRVPrepareFunctionsImpl::removeAggregateTypesFromSignature(Function *F) {
569 bool IsRetAggr = F->getReturnType()->isAggregateType();
570 // Allow intrinsics with aggregate return/argument types to reach GlobalISel.
571 // Renaming/mutating the signature of an intrinsic would desync its name from
572 // its argument types and break the IR verifier.
573 if (F->isIntrinsic())
574 return F;
575
576 IRBuilder<> B(F->getContext());
577
578 bool HasAggrArg = llvm::any_of(Range: F->args(), P: [](Argument &Arg) {
579 return Arg.getType()->isAggregateType();
580 });
581 bool DoClone = IsRetAggr || HasAggrArg;
582 if (!DoClone)
583 return F;
584 SmallVector<std::pair<int, Type *>, 4> ChangedTypes;
585 Type *RetType = IsRetAggr ? B.getInt32Ty() : F->getReturnType();
586 if (IsRetAggr)
587 ChangedTypes.push_back(Elt: std::pair<int, Type *>(-1, F->getReturnType()));
588 SmallVector<Type *, 4> ArgTypes;
589 for (const auto &Arg : F->args()) {
590 if (Arg.getType()->isAggregateType()) {
591 ArgTypes.push_back(Elt: B.getInt32Ty());
592 ChangedTypes.push_back(
593 Elt: std::pair<int, Type *>(Arg.getArgNo(), Arg.getType()));
594 } else
595 ArgTypes.push_back(Elt: Arg.getType());
596 }
597 FunctionType *NewFTy =
598 FunctionType::get(Result: RetType, Params: ArgTypes, isVarArg: F->getFunctionType()->isVarArg());
599 Function *NewF =
600 Function::Create(Ty: NewFTy, Linkage: F->getLinkage(), AddrSpace: F->getAddressSpace(),
601 N: F->getName(), M: F->getParent());
602
603 ValueToValueMapTy VMap;
604 auto NewFArgIt = NewF->arg_begin();
605 for (auto &Arg : F->args()) {
606 StringRef ArgName = Arg.getName();
607 NewFArgIt->setName(ArgName);
608 VMap[&Arg] = &(*NewFArgIt++);
609 }
610 SmallVector<ReturnInst *, 8> Returns;
611
612 CloneFunctionInto(NewFunc: NewF, OldFunc: F, VMap, Changes: CloneFunctionChangeType::LocalChangesOnly,
613 Returns);
614 NewF->takeName(V: F);
615
616 addFunctionTypeMutation(
617 NMD: NewF->getParent()->getOrInsertNamedMetadata(Name: "spv.cloned_funcs"),
618 ChangedTys: std::move(ChangedTypes), Name: NewF->getName());
619
620 for (auto *U : make_early_inc_range(Range: F->users())) {
621 if (CallInst *CI;
622 (CI = dyn_cast<CallInst>(Val: U)) && CI->getCalledFunction() == F)
623 CI->mutateFunctionType(FTy: NewF->getFunctionType());
624 if (auto *C = dyn_cast<Constant>(Val: U))
625 C->handleOperandChange(F, NewF);
626 else
627 U->replaceUsesOfWith(From: F, To: NewF);
628 }
629
630 // register the mutation
631 if (RetType != F->getReturnType())
632 TM.getSubtarget<SPIRVSubtarget>(F: *F).getSPIRVGlobalRegistry()->addMutated(
633 Val: NewF, Ty: F->getReturnType());
634 return NewF;
635}
636
637// Returns true iff `F`'s name resolves (after OpenCL/SPIR-V demangling and
638// builtin-name lookup) to the SPIR-V friendly built-in `__spirv_AbortKHR`.
639static bool isAbortKHRBuiltin(const Function &F) {
640 if (F.isIntrinsic())
641 return false;
642 StringRef Name = F.getName();
643 // Quick reject: the mangled or unmangled name must contain the substring.
644 if (!Name.contains(Other: "__spirv_AbortKHR"))
645 return false;
646 std::string Demangled = getOclOrSpirvBuiltinDemangledName(Name);
647 if (Demangled.empty())
648 return false;
649 return SPIRV::lookupBuiltinNameHelper(DemangledCall: Demangled) == "__spirv_AbortKHR";
650}
651
652// Rewrites a single call to `__spirv_AbortKHR` into a call to the
653// `llvm.spv.abort` target intrinsic, then re-terminates the block with
654// `unreachable`. OpAbortKHR is itself a SPIR-V function-termination
655// instruction and must be the last instruction in its block, so any trailing
656// stores/lifetime intrinsics/`ret` emitted by the OpenCL ABI are dropped.
657// `changeToUnreachable` cleans up any successor PHI predecessor entries.
658static void rewriteAbortKHRCall(CallInst *CI) {
659 IRBuilder<> B(CI);
660 Value *Msg = CI->getArgOperand(i: 0);
661 // The OpenCL C ABI may pass aggregate arguments by pointer (byval). In that
662 // case load the underlying value so that OpAbortKHR receives the composite
663 // itself, as required by the SPV_KHR_abort spec ("Message Type must be a
664 // concrete type").
665 if (CI->isByValArgument(ArgNo: 0)) {
666 Type *AggTy = CI->getParamByValType(ArgNo: 0);
667 Msg = B.CreateLoad(Ty: AggTy, Ptr: Msg);
668 }
669 B.CreateIntrinsic(ID: Intrinsic::spv_abort, OverloadTypes: {Msg->getType()}, Args: {Msg});
670 changeToUnreachable(I: CI);
671}
672
673// Replace OpenCL/SPIR-V style calls to `__spirv_AbortKHR(message)` (i.e.
674// calls to `F` when `F` is the `__spirv_AbortKHR` built-in) with calls to the
675// `llvm.spv.abort` target intrinsic.
676bool SPIRVPrepareFunctionsImpl::substituteAbortKHRCalls(Function *F) {
677 if (!isAbortKHRBuiltin(F: *F))
678 return false;
679
680 SmallVector<CallInst *> Calls;
681 for (User *U : F->users()) {
682 auto *CI = dyn_cast<CallInst>(Val: U);
683 if (!CI || CI->getCalledFunction() != F)
684 continue;
685 if (CI->arg_size() != 1)
686 continue;
687 Calls.push_back(Elt: CI);
688 }
689
690 for (CallInst *CI : Calls)
691 rewriteAbortKHRCall(CI);
692
693 return !Calls.empty();
694}
695
696// When the SPV_KHR_abort extension is enabled, `llvm.trap` and
697// `llvm.ubsantrap` are lowered to `OpAbortKHR` during instruction selection.
698// `OpAbortKHR` is itself a SPIR-V block terminator, so any instructions that
699// follow the trap call within the same basic block (e.g. `ret`, lifetime
700// markers) would produce SPIR-V ops after `OpAbortKHR` and break validation.
701// Terminate the block right after each call to the trap intrinsics by replacing
702// the next instruction with `unreachable`.
703bool SPIRVPrepareFunctionsImpl::terminateBlocksAfterTrap(Module &M,
704 Intrinsic::ID IID) {
705 assert((IID == Intrinsic::trap || IID == Intrinsic::ubsantrap) &&
706 "Expected trap intrinsic ID");
707
708 Function *F = Intrinsic::getDeclarationIfExists(M: &M, id: IID);
709 if (!F)
710 return false;
711
712 // If the target doesn't support SPV_KHR_abort, we won't be able to lower
713 // the trap intrinsic to OpAbortKHR, so we can skip the block-terminating
714 // transformation.
715 const auto &ST = TM.getSubtarget<SPIRVSubtarget>(F: *F);
716 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_abort))
717 return false;
718
719 SmallVector<CallInst *> Calls;
720 for (User *U : F->users()) {
721 auto *CI = dyn_cast<CallInst>(Val: U);
722 if (!CI || CI->getCalledFunction() != F)
723 continue;
724 Calls.push_back(Elt: CI);
725 }
726
727 bool Changed = false;
728 for (CallInst *CI : Calls) {
729 Instruction *Next = CI->getNextNode();
730 if (!Next || isa<UnreachableInst>(Val: Next))
731 continue;
732 changeToUnreachable(I: Next);
733 Changed = true;
734 }
735 return Changed;
736}
737
738static std::string fixMultiOutputConstraintString(StringRef Constraints) {
739 // We should only have one =r return for the made up ASM type.
740 SmallVector<StringRef> Tmp;
741 SplitString(Source: Constraints, OutFragments&: Tmp, Delimiters: ",");
742 std::string SafeConstraints("=r,");
743 for (unsigned I = 0u; I != Tmp.size() - 1; ++I) {
744 if (Tmp[I].starts_with(Prefix: '=') && (Tmp[I][1] == '&' || isalnum(Tmp[I][1])))
745 continue;
746 SafeConstraints.append(svt: Tmp[I]).append(l: {','});
747 }
748 SafeConstraints.append(svt: Tmp.back());
749
750 return SafeConstraints;
751}
752
753// Mutates indirect and inline ASM callsites iff aggregate argument/return types
754// are present with the types replaced by i32 types. The change in types is
755// noted in 'spv.mutated_callsites' metadata for later restoration. For ASM we
756// also have to mutate the constraint string as IRTranslator tries to handle
757// multiple outputs and expects an aggregate return type in their presence.
758bool SPIRVPrepareFunctionsImpl::removeAggregateTypesFromCalls(Function *F) {
759 if (F->isDeclaration() || F->isIntrinsic())
760 return false;
761
762 SmallVector<std::pair<CallBase *, FunctionType *>> Calls;
763 for (auto &&I : instructions(F)) {
764 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
765 if (!CB->getCalledOperand() || CB->getCalledFunction())
766 continue;
767 if (CB->getType()->isAggregateType() ||
768 any_of(Range: CB->args(),
769 P: [](auto &&Arg) { return Arg->getType()->isAggregateType(); }))
770 Calls.emplace_back(Args&: CB, Args: nullptr);
771 }
772 }
773
774 if (Calls.empty())
775 return false;
776
777 IRBuilder<> B(F->getContext());
778
779 unsigned MutatedCallIdx = 0;
780 for (auto &&[CB, NewFnTy] : Calls) {
781 SmallVector<std::pair<int, Type *>> ChangedTypes;
782 SmallVector<Type *> NewArgTypes;
783
784 Type *RetTy = CB->getType();
785 if (RetTy->isAggregateType()) {
786 ChangedTypes.emplace_back(Args: -1, Args&: RetTy);
787 RetTy = B.getInt32Ty();
788 }
789
790 for (auto &&Arg : CB->args()) {
791 if (Arg->getType()->isAggregateType()) {
792 NewArgTypes.push_back(Elt: B.getInt32Ty());
793 ChangedTypes.emplace_back(Args: Arg.getOperandNo(), Args: Arg->getType());
794 } else {
795 NewArgTypes.push_back(Elt: Arg->getType());
796 }
797 }
798 NewFnTy = FunctionType::get(Result: RetTy, Params: NewArgTypes,
799 isVarArg: CB->getFunctionType()->isVarArg());
800
801 // Keyed via instruction metadata, not a name.
802 std::string Key =
803 ("spv.mutated_callsite." + F->getName() + "." + Twine(MutatedCallIdx++))
804 .str();
805 CB->setMetadata(
806 Kind: "spv.mutated_callsite",
807 Node: MDNode::get(Context&: F->getContext(), MDs: MDString::get(Context&: F->getContext(), Str: Key)));
808
809 std::string Constraints;
810 if (auto *ASM = dyn_cast<InlineAsm>(Val: CB->getCalledOperand())) {
811 Constraints = ASM->getConstraintString();
812
813 CB->setCalledOperand(InlineAsm::get(
814 Ty: NewFnTy, AsmString: ASM->getAsmString(),
815 Constraints: fixMultiOutputConstraintString(Constraints), hasSideEffects: ASM->hasSideEffects(),
816 isAlignStack: ASM->isAlignStack(), asmDialect: ASM->getDialect(), canThrow: ASM->canThrow()));
817 }
818
819 addFunctionTypeMutation(
820 NMD: F->getParent()->getOrInsertNamedMetadata(Name: "spv.mutated_callsites"),
821 ChangedTys: std::move(ChangedTypes), Name: Key, AsmConstraints: Constraints);
822 }
823
824 for (auto &&[CB, NewFTy] : Calls) {
825 if (NewFTy->getReturnType() != CB->getType())
826 TM.getSubtarget<SPIRVSubtarget>(F: *F).getSPIRVGlobalRegistry()->addMutated(
827 Val: CB, Ty: CB->getType());
828 CB->mutateFunctionType(FTy: NewFTy);
829 }
830
831 return true;
832}
833
834bool SPIRVPrepareFunctionsImpl::runOnModule(Module &M) {
835 // Resolve the SPIR-V environment from module content before any
836 // function-level processing. This must happen before legalization so that
837 // isShader()/isKernel() return correct values.
838 const_cast<SPIRVTargetMachine &>(TM)
839 .getMutableSubtargetImpl()
840 ->resolveEnvFromModule(M);
841
842 bool Changed = false;
843 if (M.getFunctionDefs().empty()) {
844 // If there are no function definitions, insert a service
845 // function so that the global/constant tracking intrinsics
846 // will be created. Without these intrinsics the generated SPIR-V
847 // will be empty. The service function itself is not emitted.
848 Function *SF = getOrCreateBackendServiceFunction(M);
849 BasicBlock *BB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: SF);
850 IRBuilder<> IRB(BB);
851 IRB.CreateRetVoid();
852 Changed = true;
853 }
854
855 Changed |= terminateBlocksAfterTrap(M, IID: Intrinsic::trap);
856 Changed |= terminateBlocksAfterTrap(M, IID: Intrinsic::ubsantrap);
857
858 for (GlobalVariable &GV : M.globals()) {
859 // Strip + tag available_externally globals so AuxData can re-emit the
860 // original linkage as NonSemantic.AuxData::Linkage.
861 if (GV.hasAvailableExternallyLinkage() && !GV.isDeclaration()) {
862 GV.addAttribute(SPIRV_WAS_AVAILABLE_EXTERNALLY_ATTR);
863 GV.setLinkage(GlobalValue::ExternalLinkage);
864 Changed = true;
865 }
866 }
867
868 for (Function &F : M) {
869 // MachineFunctionPass skips available_externally; strip + tag so AuxData
870 // can re-emit the original linkage as NonSemantic.AuxData::Linkage.
871 if (F.hasAvailableExternallyLinkage() && !F.isDeclaration()) {
872 F.addFnAttr(SPIRV_WAS_AVAILABLE_EXTERNALLY_ATTR);
873 F.setLinkage(GlobalValue::ExternalLinkage);
874 Changed = true;
875 }
876 Changed |= substituteAbortKHRCalls(F: &F);
877 Changed |= substituteIntrinsicCalls(F: &F);
878 Changed |= sortBlocks(F);
879 Changed |= removeAggregateTypesFromCalls(F: &F);
880 }
881
882 std::vector<Function *> FuncsWorklist;
883 for (auto &F : M)
884 FuncsWorklist.push_back(x: &F);
885
886 for (auto *F : FuncsWorklist) {
887 Function *NewF = removeAggregateTypesFromSignature(F);
888
889 if (NewF != F) {
890 F->eraseFromParent();
891 Changed = true;
892 }
893 }
894 return Changed;
895}
896
897PreservedAnalyses SPIRVPrepareFunctions::run(Module &M,
898 ModuleAnalysisManager &AM) {
899 return SPIRVPrepareFunctionsImpl(TM).runOnModule(M)
900 ? PreservedAnalyses::none()
901 : PreservedAnalyses::all();
902}
903
904ModulePass *
905llvm::createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM) {
906 return new SPIRVPrepareFunctionsLegacy(TM);
907}
908