1//===-- SPIRVPrepareFunctions.cpp - modify function signatures --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass modifies function signatures containing aggregate arguments
10// and/or return value before IRTranslator. Information about the original
11// signatures is stored in metadata. It is used during call lowering to
12// restore correct SPIR-V types of function arguments and return values.
13// This pass also substitutes some llvm intrinsic calls with calls to newly
14// generated functions (as the Khronos LLVM/SPIR-V Translator does).
15//
16// NOTE: this pass is a module-level one due to the necessity to modify
17// GVs/functions.
18//
19//===----------------------------------------------------------------------===//
20
21#include "SPIRVPrepareFunctions.h"
22#include "SPIRV.h"
23#include "SPIRVBuiltins.h"
24#include "SPIRVSubtarget.h"
25#include "SPIRVTargetMachine.h"
26#include "SPIRVUtils.h"
27#include "llvm/ADT/StringExtras.h"
28#include "llvm/Analysis/TargetTransformInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/CodeGen/IntrinsicLowering.h"
31#include "llvm/IR/DiagnosticInfo.h"
32#include "llvm/IR/IRBuilder.h"
33#include "llvm/IR/InstIterator.h"
34#include "llvm/IR/Instructions.h"
35#include "llvm/IR/IntrinsicInst.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/IntrinsicsSPIRV.h"
38#include "llvm/InitializePasses.h"
39#include "llvm/Transforms/Utils/Cloning.h"
40#include "llvm/Transforms/Utils/Local.h"
41#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
42#include <regex>
43
44using namespace llvm;
45
46namespace {
47
48class SPIRVPrepareFunctionsImpl {
49 const SPIRVTargetMachine &TM;
50 function_ref<const TargetTransformInfo &(Function &)> GetTTI;
51 bool substituteIntrinsicCalls(Function *F);
52 bool substituteAbortKHRCalls(Function *F);
53 bool terminateBlocksAfterTrap(Module &M, Intrinsic::ID IID);
54 Function *removeAggregateTypesFromSignature(Function *F);
55 bool removeAggregateTypesFromCalls(Function *F);
56
57public:
58 SPIRVPrepareFunctionsImpl(
59 const SPIRVTargetMachine &TM,
60 function_ref<const TargetTransformInfo &(Function &)> GetTTI)
61 : TM(TM), GetTTI(GetTTI) {}
62 bool runOnModule(Module &M);
63};
64
65class SPIRVPrepareFunctionsLegacy : public ModulePass {
66 const SPIRVTargetMachine &TM;
67
68public:
69 static char ID;
70 SPIRVPrepareFunctionsLegacy(const SPIRVTargetMachine &TM)
71 : ModulePass(ID), TM(TM) {}
72
73 bool runOnModule(Module &M) override {
74 auto GetTTI = [this](Function &F) -> const TargetTransformInfo & {
75 return getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
76 };
77 return SPIRVPrepareFunctionsImpl(TM, GetTTI).runOnModule(M);
78 }
79
80 void getAnalysisUsage(AnalysisUsage &AU) const override {
81 AU.addRequired<TargetTransformInfoWrapperPass>();
82 }
83
84 StringRef getPassName() const override { return "SPIRV prepare functions"; }
85};
86
87static cl::list<std::string> SPVAllowUnknownIntrinsics(
88 "spv-allow-unknown-intrinsics", cl::CommaSeparated,
89 cl::desc("Emit unknown intrinsics as calls to external functions. A "
90 "comma-separated input list of intrinsic prefixes must be "
91 "provided, and only intrinsics carrying a listed prefix get "
92 "emitted as described."),
93 cl::value_desc("intrinsic_prefix_0,intrinsic_prefix_1"), cl::ValueOptional);
94} // namespace
95
96char SPIRVPrepareFunctionsLegacy::ID = 0;
97
98INITIALIZE_PASS_BEGIN(SPIRVPrepareFunctionsLegacy, "spirv-prepare-functions",
99 "SPIRV prepare functions", false, false)
100INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
101INITIALIZE_PASS_END(SPIRVPrepareFunctionsLegacy, "spirv-prepare-functions",
102 "SPIRV prepare functions", false, false)
103
104static std::string lowerLLVMIntrinsicName(IntrinsicInst *II) {
105 Function *IntrinsicFunc = II->getCalledFunction();
106 assert(IntrinsicFunc && "Missing function");
107 std::string FuncName = IntrinsicFunc->getName().str();
108 llvm::replace(Range&: FuncName, OldValue: '.', NewValue: '_');
109 FuncName = "spirv." + FuncName;
110 return FuncName;
111}
112
113static Function *getOrCreateFunction(Module *M, Type *RetTy,
114 ArrayRef<Type *> ArgTypes,
115 StringRef Name) {
116 FunctionType *FT = FunctionType::get(Result: RetTy, Params: ArgTypes, isVarArg: false);
117 Function *F = M->getFunction(Name);
118 if (F && F->getFunctionType() == FT)
119 return F;
120 Function *NewF = Function::Create(Ty: FT, Linkage: GlobalValue::ExternalLinkage, N: Name, M);
121 if (F)
122 NewF->setDSOLocal(F->isDSOLocal());
123 NewF->setCallingConv(CallingConv::SPIR_FUNC);
124 return NewF;
125}
126
127static bool lowerIntrinsicToFunction(IntrinsicInst *Intrinsic,
128 const TargetTransformInfo &TTI) {
129 // For @llvm.memset.* intrinsic cases with constant value and length arguments
130 // are emulated via "storing" a constant array to the destination. For other
131 // cases we wrap the intrinsic in @spirv.llvm_memset_* function and expand the
132 // intrinsic to a loop via expandMemSetAsLoop().
133 if (auto *MSI = dyn_cast<MemSetInst>(Val: Intrinsic))
134 if (isa<Constant>(Val: MSI->getValue()) && isa<ConstantInt>(Val: MSI->getLength()))
135 return false; // It is handled later using OpCopyMemorySized.
136
137 // An intrinsic with a metadata argument has no SPIR-V lowering and can't be
138 // turned into a function.
139 if (any_of(Range: Intrinsic->args(), P: IsaPred<MetadataAsValue>)) {
140 const Function *F = Intrinsic->getFunction();
141 F->getContext().diagnose(DI: DiagnosticInfoUnsupported(
142 *F,
143 "cannot lower the intrinsic '" +
144 Intrinsic->getCalledFunction()->getName() +
145 "' that takes a metadata argument",
146 Intrinsic->getDebugLoc()));
147 if (!Intrinsic->getType()->isVoidTy())
148 Intrinsic->replaceAllUsesWith(V: PoisonValue::get(T: Intrinsic->getType()));
149 Intrinsic->eraseFromParent();
150 return true;
151 }
152
153 Module *M = Intrinsic->getModule();
154 std::string FuncName = lowerLLVMIntrinsicName(II: Intrinsic);
155 if (Intrinsic->isVolatile())
156 FuncName += ".volatile";
157 // Redirect @llvm.intrinsic.* call to @spirv.llvm_intrinsic_*
158 Function *F = M->getFunction(Name: FuncName);
159 if (F) {
160 Intrinsic->setCalledFunction(F);
161 return true;
162 }
163 FunctionCallee FC =
164 M->getOrInsertFunction(Name: FuncName, T: Intrinsic->getFunctionType());
165 auto IntrinsicID = Intrinsic->getIntrinsicID();
166 Intrinsic->setCalledFunction(FC);
167 F = cast<Function>(Val: FC.getCallee());
168 F->setAttributes(Intrinsic->getAttributes());
169
170 switch (IntrinsicID) {
171 case Intrinsic::memset: {
172 auto *MSI = static_cast<MemSetInst *>(Intrinsic);
173 Argument *Dest = F->getArg(i: 0);
174 Argument *Val = F->getArg(i: 1);
175 Argument *Len = F->getArg(i: 2);
176 Argument *IsVolatile = F->getArg(i: 3);
177 Dest->setName("dest");
178 Val->setName("val");
179 Len->setName("len");
180 IsVolatile->setName("isvolatile");
181 BasicBlock *EntryBB = BasicBlock::Create(Context&: M->getContext(), Name: "entry", Parent: F);
182 IRBuilder<> IRB(EntryBB);
183 auto *MemSet = IRB.CreateMemSet(Ptr: Dest, Val, Size: Len, Align: MSI->getDestAlign(),
184 isVolatile: MSI->isVolatile());
185 IRB.CreateRetVoid();
186 expandMemSetAsLoop(MemSet: cast<MemSetInst>(Val: MemSet), TTI);
187 MemSet->eraseFromParent();
188 break;
189 }
190 case Intrinsic::bswap: {
191 BasicBlock *EntryBB = BasicBlock::Create(Context&: M->getContext(), Name: "entry", Parent: F);
192 IRBuilder<> IRB(EntryBB);
193 CallInst *BSwap = IRB.CreateIntrinsicWithoutFolding(
194 ID: Intrinsic::bswap, OverloadTypes: Intrinsic->getType(), Args: F->getArg(i: 0));
195 IRB.CreateRet(V: BSwap);
196 IntrinsicLowering IL(M->getDataLayout());
197 IL.LowerIntrinsicCall(CI: BSwap);
198 break;
199 }
200 default:
201 break;
202 }
203 return true;
204}
205
206static std::string getAnnotation(Value *AnnoVal, Value *OptAnnoVal) {
207 if (auto *Ref = dyn_cast_or_null<GetElementPtrInst>(Val: AnnoVal))
208 AnnoVal = Ref->getOperand(i_nocapture: 0);
209 if (auto *Ref = dyn_cast_or_null<BitCastInst>(Val: OptAnnoVal))
210 OptAnnoVal = Ref->getOperand(i_nocapture: 0);
211
212 std::string Anno;
213 if (auto *C = dyn_cast_or_null<Constant>(Val: AnnoVal)) {
214 StringRef Str;
215 if (getConstantStringInfo(V: C, Str))
216 Anno = Str;
217 }
218 // handle optional annotation parameter in a way that Khronos Translator do
219 // (collect integers wrapped in a struct)
220 if (auto *C = dyn_cast_or_null<Constant>(Val: OptAnnoVal);
221 C && C->getNumOperands()) {
222 Value *MaybeStruct = C->getOperand(i: 0);
223 if (auto *Struct = dyn_cast<ConstantStruct>(Val: MaybeStruct)) {
224 for (unsigned I = 0, E = Struct->getNumOperands(); I != E; ++I) {
225 if (auto *CInt = dyn_cast<ConstantInt>(Val: Struct->getOperand(i_nocapture: I)))
226 Anno += (I == 0 ? ": " : ", ") +
227 std::to_string(val: CInt->getType()->getIntegerBitWidth() == 1
228 ? CInt->getZExtValue()
229 : CInt->getSExtValue());
230 }
231 } else if (auto *Struct = dyn_cast<ConstantAggregateZero>(Val: MaybeStruct)) {
232 // { i32 i32 ... } zeroinitializer
233 for (unsigned I = 0, E = Struct->getType()->getStructNumElements();
234 I != E; ++I)
235 Anno += I == 0 ? ": 0" : ", 0";
236 }
237 }
238 return Anno;
239}
240
241static SmallVector<Metadata *> parseAnnotation(Value *I,
242 const std::string &Anno,
243 LLVMContext &Ctx,
244 Type *Int32Ty) {
245 // Try to parse the annotation string according to the following rules:
246 // annotation := ({kind} | {kind:value,value,...})+
247 // kind := number
248 // value := number | string
249 static const std::regex R(
250 "\\{(\\d+)(?:[:,](\\d+|\"[^\"]*\")(?:,(\\d+|\"[^\"]*\"))*)?\\}");
251 SmallVector<Metadata *> MDs;
252 int Pos = 0;
253 for (std::sregex_iterator
254 It = std::sregex_iterator(Anno.begin(), Anno.end(), R),
255 ItEnd = std::sregex_iterator();
256 It != ItEnd; ++It) {
257 if (It->position() != Pos)
258 return SmallVector<Metadata *>{};
259 Pos = It->position() + It->length();
260 std::smatch Match = *It;
261 SmallVector<Metadata *> MDsItem;
262 for (std::size_t i = 1; i < Match.size(); ++i) {
263 std::ssub_match SMatch = Match[i];
264 std::string Item = SMatch.str();
265 if (Item.length() == 0)
266 break;
267 if (Item[0] == '"') {
268 Item = Item.substr(pos: 1, n: Item.length() - 2);
269 // Acceptable format of the string snippet is:
270 static const std::regex RStr("^(\\d+)(?:,(\\d+))*$");
271 if (std::smatch MatchStr; std::regex_match(s: Item, m&: MatchStr, re: RStr)) {
272 for (std::size_t SubIdx = 1; SubIdx < MatchStr.size(); ++SubIdx)
273 if (std::string SubStr = MatchStr[SubIdx].str(); SubStr.length())
274 MDsItem.push_back(Elt: ConstantAsMetadata::get(
275 C: ConstantInt::get(Ty: Int32Ty, V: std::stoi(str: SubStr))));
276 } else {
277 MDsItem.push_back(Elt: MDString::get(Context&: Ctx, Str: Item));
278 }
279 } else if (int32_t Num; llvm::to_integer(S: StringRef(Item), Num, Base: 10)) {
280 MDsItem.push_back(
281 Elt: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: Num)));
282 } else {
283 MDsItem.push_back(Elt: MDString::get(Context&: Ctx, Str: Item));
284 }
285 }
286 if (MDsItem.size() == 0)
287 return SmallVector<Metadata *>{};
288 MDs.push_back(Elt: MDNode::get(Context&: Ctx, MDs: MDsItem));
289 }
290 return Pos == static_cast<int>(Anno.length()) ? std::move(MDs)
291 : SmallVector<Metadata *>{};
292}
293
294static void lowerPtrAnnotation(IntrinsicInst *II) {
295 LLVMContext &Ctx = II->getContext();
296 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
297
298 // Retrieve an annotation string from arguments.
299 Value *PtrArg = nullptr;
300 if (auto *BI = dyn_cast<BitCastInst>(Val: II->getArgOperand(i: 0)))
301 PtrArg = BI->getOperand(i_nocapture: 0);
302 else
303 PtrArg = II->getOperand(i_nocapture: 0);
304 std::string Anno =
305 getAnnotation(AnnoVal: II->getArgOperand(i: 1),
306 OptAnnoVal: 4 < II->arg_size() ? II->getArgOperand(i: 4) : nullptr);
307
308 // Parse the annotation.
309 SmallVector<Metadata *> MDs = parseAnnotation(I: II, Anno, Ctx, Int32Ty);
310
311 // If the annotation string is not parsed successfully we don't know the
312 // format used and output it as a general UserSemantic decoration.
313 // Otherwise MDs is a Metadata tuple (a decoration list) in the format
314 // expected by `spirv.Decorations`.
315 if (MDs.size() == 0) {
316 auto UserSemantic = ConstantAsMetadata::get(C: ConstantInt::get(
317 Ty: Int32Ty, V: static_cast<uint32_t>(SPIRV::Decoration::UserSemantic)));
318 MDs.push_back(Elt: MDNode::get(Context&: Ctx, MDs: {UserSemantic, MDString::get(Context&: Ctx, Str: Anno)}));
319 }
320
321 // Build the internal intrinsic function.
322 IRBuilder<> IRB(II->getParent());
323 IRB.SetInsertPoint(II);
324 IRB.CreateIntrinsic(
325 ID: Intrinsic::spv_assign_decoration, OverloadTypes: {PtrArg->getType()},
326 Args: {PtrArg, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs))});
327 II->replaceAllUsesWith(V: II->getOperand(i_nocapture: 0));
328}
329
330static void lowerFunnelShifts(IntrinsicInst *FSHIntrinsic) {
331 // Get a separate function - otherwise, we'd have to rework the CFG of the
332 // current one. Then simply replace the intrinsic uses with a call to the new
333 // function.
334 // Generate LLVM IR for i* @spirv.llvm_fsh?_i* (i* %a, i* %b, i* %c)
335 Module *M = FSHIntrinsic->getModule();
336 FunctionType *FSHFuncTy = FSHIntrinsic->getFunctionType();
337 Type *FSHRetTy = FSHFuncTy->getReturnType();
338 const std::string FuncName = lowerLLVMIntrinsicName(II: FSHIntrinsic);
339 Function *FSHFunc =
340 getOrCreateFunction(M, RetTy: FSHRetTy, ArgTypes: FSHFuncTy->params(), Name: FuncName);
341
342 if (!FSHFunc->empty()) {
343 FSHIntrinsic->setCalledFunction(FSHFunc);
344 return;
345 }
346 BasicBlock *RotateBB = BasicBlock::Create(Context&: M->getContext(), Name: "rotate", Parent: FSHFunc);
347 IRBuilder<> IRB(RotateBB);
348 Type *Ty = FSHFunc->getReturnType();
349 // Build the actual funnel shift rotate logic.
350 // In the comments, "int" is used interchangeably with "vector of int
351 // elements".
352 FixedVectorType *VectorTy = dyn_cast<FixedVectorType>(Val: Ty);
353 Type *IntTy = VectorTy ? VectorTy->getElementType() : Ty;
354 unsigned BitWidth = IntTy->getIntegerBitWidth();
355 ConstantInt *BitWidthConstant = IRB.getInt(AI: {BitWidth, BitWidth});
356 Value *BitWidthForInsts =
357 VectorTy
358 ? IRB.CreateVectorSplat(NumElts: VectorTy->getNumElements(), V: BitWidthConstant)
359 : BitWidthConstant;
360 Value *RotateModVal =
361 IRB.CreateURem(/*Rotate*/ LHS: FSHFunc->getArg(i: 2), RHS: BitWidthForInsts);
362 Value *FirstShift = nullptr, *SecShift = nullptr;
363 if (FSHIntrinsic->getIntrinsicID() == Intrinsic::fshr) {
364 // Shift the less significant number right, the "rotate" number of bits
365 // will be 0-filled on the left as a result of this regular shift.
366 FirstShift = IRB.CreateLShr(LHS: FSHFunc->getArg(i: 1), RHS: RotateModVal);
367 } else {
368 // Shift the more significant number left, the "rotate" number of bits
369 // will be 0-filled on the right as a result of this regular shift.
370 FirstShift = IRB.CreateShl(LHS: FSHFunc->getArg(i: 0), RHS: RotateModVal);
371 }
372 // We want the "rotate" number of the more significant int's LSBs (MSBs) to
373 // occupy the leftmost (rightmost) "0 space" left by the previous operation.
374 // Therefore, subtract the "rotate" number from the integer bitsize...
375 Value *SubRotateVal = IRB.CreateSub(LHS: BitWidthForInsts, RHS: RotateModVal);
376 if (FSHIntrinsic->getIntrinsicID() == Intrinsic::fshr) {
377 // ...and left-shift the more significant int by this number, zero-filling
378 // the LSBs.
379 SecShift = IRB.CreateShl(LHS: FSHFunc->getArg(i: 0), RHS: SubRotateVal);
380 } else {
381 // ...and right-shift the less significant int by this number, zero-filling
382 // the MSBs.
383 SecShift = IRB.CreateLShr(LHS: FSHFunc->getArg(i: 1), RHS: SubRotateVal);
384 }
385 // A simple binary addition of the shifted ints yields the final result.
386 IRB.CreateRet(V: IRB.CreateOr(LHS: FirstShift, RHS: SecShift));
387
388 FSHIntrinsic->setCalledFunction(FSHFunc);
389}
390
391static void lowerConstrainedFPCmpIntrinsic(
392 ConstrainedFPCmpIntrinsic *ConstrainedCmpIntrinsic,
393 SmallVector<Instruction *> &EraseFromParent) {
394 if (!ConstrainedCmpIntrinsic)
395 return;
396 // Extract the floating-point values being compared
397 Value *LHS = ConstrainedCmpIntrinsic->getArgOperand(i: 0);
398 Value *RHS = ConstrainedCmpIntrinsic->getArgOperand(i: 1);
399 FCmpInst::Predicate Pred = ConstrainedCmpIntrinsic->getPredicate();
400 IRBuilder<> Builder(ConstrainedCmpIntrinsic);
401 Value *FCmp = Builder.CreateFCmp(P: Pred, LHS, RHS);
402 ConstrainedCmpIntrinsic->replaceAllUsesWith(V: FCmp);
403 EraseFromParent.push_back(Elt: dyn_cast<Instruction>(Val: ConstrainedCmpIntrinsic));
404}
405
406static void lowerExpectAssume(IntrinsicInst *II) {
407 // If we cannot use the SPV_KHR_expect_assume extension, then we need to
408 // ignore the intrinsic and move on. It should be removed later on by LLVM.
409 // Otherwise we should lower the intrinsic to the corresponding SPIR-V
410 // instruction.
411 // For @llvm.assume we have OpAssumeTrueKHR.
412 // For @llvm.expect we have OpExpectKHR.
413 //
414 // We need to lower this into a builtin and then the builtin into a SPIR-V
415 // instruction.
416 if (II->getIntrinsicID() == Intrinsic::assume) {
417 Function *F = Intrinsic::getOrInsertDeclaration(
418 M: II->getModule(), id: Intrinsic::SPVIntrinsics::spv_assume);
419 II->setCalledFunction(F);
420 } else if (II->getIntrinsicID() == Intrinsic::expect) {
421 Function *F = Intrinsic::getOrInsertDeclaration(
422 M: II->getModule(), id: Intrinsic::SPVIntrinsics::spv_expect,
423 OverloadTys: {II->getOperand(i_nocapture: 0)->getType()});
424 II->setCalledFunction(F);
425 } else {
426 llvm_unreachable("Unknown intrinsic");
427 }
428}
429
430static bool toSpvLifetimeIntrinsic(IntrinsicInst *II, Intrinsic::ID NewID) {
431 auto *LifetimeArg0 = II->getArgOperand(i: 0);
432
433 // If the lifetime argument is a poison value, the intrinsic has no effect.
434 if (isa<PoisonValue>(Val: LifetimeArg0)) {
435 II->eraseFromParent();
436 return true;
437 }
438
439 IRBuilder<> Builder(II);
440 auto *Alloca = cast<AllocaInst>(Val: LifetimeArg0);
441 std::optional<TypeSize> Size =
442 Alloca->getAllocationSize(DL: Alloca->getDataLayout());
443 Value *SizeVal = Builder.getInt64(C: Size ? *Size : -1);
444 Builder.CreateIntrinsic(ID: NewID, OverloadTypes: Alloca->getType(), Args: {SizeVal, LifetimeArg0});
445 II->eraseFromParent();
446 return true;
447}
448
449static void
450lowerConstrainedFmuladd(IntrinsicInst *II,
451 SmallVector<Instruction *> &EraseFromParent) {
452 auto *FPI = cast<ConstrainedFPIntrinsic>(Val: II);
453 Value *A = FPI->getArgOperand(i: 0);
454 Value *Mul = FPI->getArgOperand(i: 1);
455 Value *Add = FPI->getArgOperand(i: 2);
456 IRBuilder<> Builder(II->getParent());
457 Builder.SetInsertPoint(II);
458 std::optional<RoundingMode> Rounding = FPI->getRoundingMode();
459 Value *Product = Builder.CreateFMul(L: A, R: Mul, Name: II->getName() + ".mul");
460 Value *Result = Builder.CreateConstrainedFPBinOp(
461 ID: Intrinsic::experimental_constrained_fadd, L: Product, R: Add, FMFSource: {},
462 Name: II->getName() + ".add", FPMathTag: nullptr, Rounding);
463 II->replaceAllUsesWith(V: Result);
464 EraseFromParent.push_back(Elt: II);
465}
466
467// Substitutes calls to LLVM intrinsics with either calls to SPIR-V intrinsics
468// or calls to proper generated functions. Returns True if F was modified.
469bool SPIRVPrepareFunctionsImpl::substituteIntrinsicCalls(Function *F) {
470 if (F->isDeclaration())
471 return false;
472
473 bool Changed = false;
474 const SPIRVSubtarget &STI = TM.getSubtarget<SPIRVSubtarget>(F: *F);
475 SmallVector<Instruction *> EraseFromParent;
476 const TargetTransformInfo &TTI = GetTTI(*F);
477 for (BasicBlock &BB : *F) {
478 for (Instruction &I : make_early_inc_range(Range&: BB)) {
479 auto Call = dyn_cast<CallInst>(Val: &I);
480 if (!Call)
481 continue;
482 Function *CF = Call->getCalledFunction();
483 if (!CF || !CF->isIntrinsic())
484 continue;
485 auto *II = cast<IntrinsicInst>(Val: Call);
486 if (Intrinsic::isTargetIntrinsic(IID: II->getIntrinsicID()) &&
487 II->getCalledOperand()->getName().starts_with(Prefix: "llvm.spv"))
488 continue;
489 switch (II->getIntrinsicID()) {
490 case Intrinsic::memset:
491 case Intrinsic::bswap:
492 Changed |= lowerIntrinsicToFunction(Intrinsic: II, TTI);
493 break;
494 case Intrinsic::fshl:
495 case Intrinsic::fshr:
496 lowerFunnelShifts(FSHIntrinsic: II);
497 Changed = true;
498 break;
499 case Intrinsic::assume:
500 case Intrinsic::expect:
501 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume))
502 lowerExpectAssume(II);
503 Changed = true;
504 break;
505 case Intrinsic::lifetime_start:
506 if (!STI.isShader()) {
507 Changed |= toSpvLifetimeIntrinsic(
508 II, NewID: Intrinsic::SPVIntrinsics::spv_lifetime_start);
509 } else {
510 II->eraseFromParent();
511 Changed = true;
512 }
513 break;
514 case Intrinsic::lifetime_end:
515 if (!STI.isShader()) {
516 Changed |= toSpvLifetimeIntrinsic(
517 II, NewID: Intrinsic::SPVIntrinsics::spv_lifetime_end);
518 } else {
519 II->eraseFromParent();
520 Changed = true;
521 }
522 break;
523 case Intrinsic::ptr_annotation:
524 lowerPtrAnnotation(II);
525 Changed = true;
526 break;
527 case Intrinsic::experimental_constrained_fmuladd:
528 lowerConstrainedFmuladd(II, EraseFromParent);
529 Changed = true;
530 break;
531 case Intrinsic::experimental_constrained_fcmp:
532 case Intrinsic::experimental_constrained_fcmps:
533 lowerConstrainedFPCmpIntrinsic(ConstrainedCmpIntrinsic: dyn_cast<ConstrainedFPCmpIntrinsic>(Val: II),
534 EraseFromParent);
535 Changed = true;
536 break;
537 default:
538 // Drop assume-like intrinsics that have no SPIR-V representation.
539 if (II->isAssumeLikeIntrinsic()) {
540 if (!II->getType()->isVoidTy())
541 II->replaceAllUsesWith(V: PoisonValue::get(T: II->getType()));
542 II->eraseFromParent();
543 Changed = true;
544 break;
545 }
546 if (TM.getTargetTriple().getVendor() == Triple::AMD ||
547 any_of(Range&: SPVAllowUnknownIntrinsics, P: [II](auto &&Prefix) {
548 if (Prefix.empty())
549 return false;
550 return II->getCalledFunction()->getName().starts_with(Prefix);
551 }))
552 Changed |= lowerIntrinsicToFunction(Intrinsic: II, TTI);
553 break;
554 }
555 }
556 }
557 for (auto *I : EraseFromParent)
558 I->eraseFromParent();
559 return Changed;
560}
561
562static void
563addFunctionTypeMutation(NamedMDNode *NMD,
564 SmallVector<std::pair<int, Type *>> ChangedTys,
565 StringRef Name, StringRef AsmConstraints = "") {
566
567 LLVMContext &Ctx = NMD->getParent()->getContext();
568 Type *I32Ty = IntegerType::getInt32Ty(C&: Ctx);
569
570 SmallVector<Metadata *> MDArgs;
571 MDArgs.push_back(Elt: MDString::get(Context&: Ctx, Str: Name));
572 transform(Range&: ChangedTys, d_first: std::back_inserter(x&: MDArgs), F: [=, &Ctx](auto &&CTy) {
573 return MDNode::get(
574 Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(I32Ty, CTy.first, true)),
575 ValueAsMetadata::get(V: Constant::getNullValue(Ty: CTy.second))});
576 });
577 if (!AsmConstraints.empty())
578 MDArgs.push_back(Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: AsmConstraints)));
579 NMD->addOperand(M: MDNode::get(Context&: Ctx, MDs: MDArgs));
580}
581
582// Returns F if aggregate argument/return types are not present or cloned F
583// function with the types replaced by i32 types. The change in types is
584// noted in 'spv.cloned_funcs' metadata for later restoration.
585Function *
586SPIRVPrepareFunctionsImpl::removeAggregateTypesFromSignature(Function *F) {
587 bool IsRetAggr = F->getReturnType()->isAggregateType();
588 // Allow intrinsics with aggregate return/argument types to reach GlobalISel.
589 // Renaming/mutating the signature of an intrinsic would desync its name from
590 // its argument types and break the IR verifier.
591 if (F->isIntrinsic())
592 return F;
593
594 IRBuilder<> B(F->getContext());
595
596 bool HasAggrArg = llvm::any_of(Range: F->args(), P: [](Argument &Arg) {
597 return Arg.getType()->isAggregateType();
598 });
599 bool DoClone = IsRetAggr || HasAggrArg;
600 if (!DoClone)
601 return F;
602 SmallVector<std::pair<int, Type *>, 4> ChangedTypes;
603 Type *RetType = IsRetAggr ? B.getInt32Ty() : F->getReturnType();
604 if (IsRetAggr)
605 ChangedTypes.push_back(Elt: std::pair<int, Type *>(-1, F->getReturnType()));
606 SmallVector<Type *, 4> ArgTypes;
607 for (const auto &Arg : F->args()) {
608 if (Arg.getType()->isAggregateType()) {
609 ArgTypes.push_back(Elt: B.getInt32Ty());
610 ChangedTypes.push_back(
611 Elt: std::pair<int, Type *>(Arg.getArgNo(), Arg.getType()));
612 } else
613 ArgTypes.push_back(Elt: Arg.getType());
614 }
615 FunctionType *NewFTy =
616 FunctionType::get(Result: RetType, Params: ArgTypes, isVarArg: F->getFunctionType()->isVarArg());
617 Function *NewF =
618 Function::Create(Ty: NewFTy, Linkage: F->getLinkage(), AddrSpace: F->getAddressSpace(),
619 N: F->getName(), M: F->getParent());
620
621 ValueToValueMapTy VMap;
622 auto NewFArgIt = NewF->arg_begin();
623 for (auto &Arg : F->args()) {
624 StringRef ArgName = Arg.getName();
625 NewFArgIt->setName(ArgName);
626 VMap[&Arg] = &(*NewFArgIt++);
627 }
628 SmallVector<ReturnInst *, 8> Returns;
629
630 CloneFunctionInto(NewFunc: NewF, OldFunc: F, VMap, Changes: CloneFunctionChangeType::LocalChangesOnly,
631 Returns);
632 NewF->takeName(V: F);
633
634 addFunctionTypeMutation(
635 NMD: NewF->getParent()->getOrInsertNamedMetadata(Name: "spv.cloned_funcs"),
636 ChangedTys: std::move(ChangedTypes), Name: NewF->getName());
637
638 for (auto *U : make_early_inc_range(Range: F->users())) {
639 if (CallInst *CI;
640 (CI = dyn_cast<CallInst>(Val: U)) && CI->getCalledFunction() == F)
641 CI->mutateFunctionType(FTy: NewF->getFunctionType());
642 if (auto *C = dyn_cast<Constant>(Val: U))
643 C->handleOperandChange(F, NewF);
644 else
645 U->replaceUsesOfWith(From: F, To: NewF);
646 }
647
648 // register the mutation
649 if (RetType != F->getReturnType())
650 TM.getSubtarget<SPIRVSubtarget>(F: *F).getSPIRVGlobalRegistry()->addMutated(
651 Val: NewF, Ty: F->getReturnType());
652 return NewF;
653}
654
655// Returns true iff `F`'s name resolves (after OpenCL/SPIR-V demangling and
656// builtin-name lookup) to the SPIR-V friendly built-in `__spirv_AbortKHR`.
657static bool isAbortKHRBuiltin(const Function &F) {
658 if (F.isIntrinsic())
659 return false;
660 StringRef Name = F.getName();
661 // Quick reject: the mangled or unmangled name must contain the substring.
662 if (!Name.contains(Other: "__spirv_AbortKHR"))
663 return false;
664 std::string Demangled = getOclOrSpirvBuiltinDemangledName(Name);
665 if (Demangled.empty())
666 return false;
667 return SPIRV::lookupBuiltinNameHelper(DemangledCall: Demangled) == "__spirv_AbortKHR";
668}
669
670// Rewrites a single call to `__spirv_AbortKHR` into a call to the
671// `llvm.spv.abort` target intrinsic, then re-terminates the block with
672// `unreachable`. OpAbortKHR is itself a SPIR-V function-termination
673// instruction and must be the last instruction in its block, so any trailing
674// stores/lifetime intrinsics/`ret` emitted by the OpenCL ABI are dropped.
675// `changeToUnreachable` cleans up any successor PHI predecessor entries.
676static void rewriteAbortKHRCall(CallInst *CI) {
677 IRBuilder<> B(CI);
678 Value *Msg = CI->getArgOperand(i: 0);
679 // The OpenCL C ABI may pass aggregate arguments by pointer (byval). In that
680 // case load the underlying value so that OpAbortKHR receives the composite
681 // itself, as required by the SPV_KHR_abort spec ("Message Type must be a
682 // concrete type").
683 if (CI->isByValArgument(ArgNo: 0)) {
684 Type *AggTy = CI->getParamByValType(ArgNo: 0);
685 Msg = B.CreateLoad(Ty: AggTy, Ptr: Msg);
686 }
687 B.CreateIntrinsic(ID: Intrinsic::spv_abort, OverloadTypes: {Msg->getType()}, Args: {Msg});
688 changeToUnreachable(I: CI);
689}
690
691// Replace OpenCL/SPIR-V style calls to `__spirv_AbortKHR(message)` (i.e.
692// calls to `F` when `F` is the `__spirv_AbortKHR` built-in) with calls to the
693// `llvm.spv.abort` target intrinsic.
694bool SPIRVPrepareFunctionsImpl::substituteAbortKHRCalls(Function *F) {
695 if (!isAbortKHRBuiltin(F: *F))
696 return false;
697
698 SmallVector<CallInst *> Calls;
699 for (User *U : F->users()) {
700 auto *CI = dyn_cast<CallInst>(Val: U);
701 if (!CI || CI->getCalledFunction() != F)
702 continue;
703 if (CI->arg_size() != 1)
704 continue;
705 Calls.push_back(Elt: CI);
706 }
707
708 for (CallInst *CI : Calls)
709 rewriteAbortKHRCall(CI);
710
711 return !Calls.empty();
712}
713
714// When the SPV_KHR_abort extension is enabled, `llvm.trap` and
715// `llvm.ubsantrap` are lowered to `OpAbortKHR` during instruction selection.
716// `OpAbortKHR` is itself a SPIR-V block terminator, so any instructions that
717// follow the trap call within the same basic block (e.g. `ret`, lifetime
718// markers) would produce SPIR-V ops after `OpAbortKHR` and break validation.
719// Terminate the block right after each call to the trap intrinsics by replacing
720// the next instruction with `unreachable`.
721bool SPIRVPrepareFunctionsImpl::terminateBlocksAfterTrap(Module &M,
722 Intrinsic::ID IID) {
723 assert((IID == Intrinsic::trap || IID == Intrinsic::ubsantrap) &&
724 "Expected trap intrinsic ID");
725
726 Function *F = Intrinsic::getDeclarationIfExists(M: &M, id: IID);
727 if (!F)
728 return false;
729
730 // If the target doesn't support SPV_KHR_abort, we won't be able to lower
731 // the trap intrinsic to OpAbortKHR, so we can skip the block-terminating
732 // transformation.
733 const auto &ST = TM.getSubtarget<SPIRVSubtarget>(F: *F);
734 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_abort))
735 return false;
736
737 SmallVector<CallInst *> Calls;
738 for (User *U : F->users()) {
739 auto *CI = dyn_cast<CallInst>(Val: U);
740 if (!CI || CI->getCalledFunction() != F)
741 continue;
742 Calls.push_back(Elt: CI);
743 }
744
745 bool Changed = false;
746 for (CallInst *CI : Calls) {
747 Instruction *Next = CI->getNextNode();
748 if (!Next || isa<UnreachableInst>(Val: Next))
749 continue;
750 changeToUnreachable(I: Next);
751 Changed = true;
752 }
753 return Changed;
754}
755
756static std::string fixMultiOutputConstraintString(StringRef Constraints) {
757 // We should only have one =r return for the made up ASM type.
758 SmallVector<StringRef> Tmp;
759 SplitString(Source: Constraints, OutFragments&: Tmp, Delimiters: ",");
760 std::string SafeConstraints("=r,");
761 for (unsigned I = 0u; I != Tmp.size() - 1; ++I) {
762 if (Tmp[I].starts_with(Prefix: '=') && (Tmp[I][1] == '&' || isalnum(Tmp[I][1])))
763 continue;
764 SafeConstraints.append(svt: Tmp[I]).append(l: {','});
765 }
766 SafeConstraints.append(svt: Tmp.back());
767
768 return SafeConstraints;
769}
770
771// Mutates indirect and inline ASM callsites iff aggregate argument/return types
772// are present with the types replaced by i32 types. The change in types is
773// noted in 'spv.mutated_callsites' metadata for later restoration. For ASM we
774// also have to mutate the constraint string as IRTranslator tries to handle
775// multiple outputs and expects an aggregate return type in their presence.
776bool SPIRVPrepareFunctionsImpl::removeAggregateTypesFromCalls(Function *F) {
777 if (F->isDeclaration() || F->isIntrinsic())
778 return false;
779
780 SmallVector<std::pair<CallBase *, FunctionType *>> Calls;
781 for (auto &&I : instructions(F)) {
782 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
783 if (!CB->getCalledOperand() || CB->getCalledFunction())
784 continue;
785 if (CB->getType()->isAggregateType() ||
786 any_of(Range: CB->args(),
787 P: [](auto &&Arg) { return Arg->getType()->isAggregateType(); }))
788 Calls.emplace_back(Args&: CB, Args: nullptr);
789 }
790 }
791
792 if (Calls.empty())
793 return false;
794
795 IRBuilder<> B(F->getContext());
796
797 unsigned MutatedCallIdx = 0;
798 for (auto &&[CB, NewFnTy] : Calls) {
799 SmallVector<std::pair<int, Type *>> ChangedTypes;
800 SmallVector<Type *> NewArgTypes;
801
802 Type *RetTy = CB->getType();
803 if (RetTy->isAggregateType()) {
804 ChangedTypes.emplace_back(Args: -1, Args&: RetTy);
805 RetTy = B.getInt32Ty();
806 }
807
808 for (auto &&Arg : CB->args()) {
809 if (Arg->getType()->isAggregateType()) {
810 NewArgTypes.push_back(Elt: B.getInt32Ty());
811 ChangedTypes.emplace_back(Args: Arg.getOperandNo(), Args: Arg->getType());
812 } else {
813 NewArgTypes.push_back(Elt: Arg->getType());
814 }
815 }
816 NewFnTy = FunctionType::get(Result: RetTy, Params: NewArgTypes,
817 isVarArg: CB->getFunctionType()->isVarArg());
818
819 // Keyed via instruction metadata, not a name.
820 std::string Key =
821 ("spv.mutated_callsite." + F->getName() + "." + Twine(MutatedCallIdx++))
822 .str();
823 CB->setMetadata(
824 Kind: "spv.mutated_callsite",
825 Node: MDNode::get(Context&: F->getContext(), MDs: MDString::get(Context&: F->getContext(), Str: Key)));
826
827 std::string Constraints;
828 if (auto *ASM = dyn_cast<InlineAsm>(Val: CB->getCalledOperand())) {
829 Constraints = ASM->getConstraintString();
830
831 CB->setCalledOperand(InlineAsm::get(
832 Ty: NewFnTy, AsmString: ASM->getAsmString(),
833 Constraints: fixMultiOutputConstraintString(Constraints), hasSideEffects: ASM->hasSideEffects(),
834 isAlignStack: ASM->isAlignStack(), asmDialect: ASM->getDialect(), canThrow: ASM->canThrow()));
835 }
836
837 addFunctionTypeMutation(
838 NMD: F->getParent()->getOrInsertNamedMetadata(Name: "spv.mutated_callsites"),
839 ChangedTys: std::move(ChangedTypes), Name: Key, AsmConstraints: Constraints);
840 }
841
842 for (auto &&[CB, NewFTy] : Calls) {
843 if (NewFTy->getReturnType() != CB->getType())
844 TM.getSubtarget<SPIRVSubtarget>(F: *F).getSPIRVGlobalRegistry()->addMutated(
845 Val: CB, Ty: CB->getType());
846 CB->mutateFunctionType(FTy: NewFTy);
847 }
848
849 return true;
850}
851
852bool SPIRVPrepareFunctionsImpl::runOnModule(Module &M) {
853 // Resolve the SPIR-V environment from module content before any
854 // function-level processing. This must happen before legalization so that
855 // isShader()/isKernel() return correct values.
856 const_cast<SPIRVTargetMachine &>(TM)
857 .getMutableSubtargetImpl()
858 ->resolveEnvFromModule(M);
859
860 bool Changed = false;
861 if (M.getFunctionDefs().empty()) {
862 // If there are no function definitions, insert a service
863 // function so that the global/constant tracking intrinsics
864 // will be created. Without these intrinsics the generated SPIR-V
865 // will be empty. The service function itself is not emitted.
866 Function *SF = getOrCreateBackendServiceFunction(M);
867 BasicBlock *BB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: SF);
868 IRBuilder<> IRB(BB);
869 IRB.CreateRetVoid();
870 Changed = true;
871 }
872
873 Changed |= terminateBlocksAfterTrap(M, IID: Intrinsic::trap);
874 Changed |= terminateBlocksAfterTrap(M, IID: Intrinsic::ubsantrap);
875
876 for (GlobalVariable &GV : M.globals()) {
877 // Strip + tag available_externally globals so AuxData can re-emit the
878 // original linkage as NonSemantic.AuxData::Linkage.
879 if (GV.hasAvailableExternallyLinkage() && !GV.isDeclaration()) {
880 GV.addAttribute(SPIRV_WAS_AVAILABLE_EXTERNALLY_ATTR);
881 GV.setLinkage(GlobalValue::ExternalLinkage);
882 Changed = true;
883 }
884 }
885
886 for (Function &F : M) {
887 // MachineFunctionPass skips available_externally; strip + tag so AuxData
888 // can re-emit the original linkage as NonSemantic.AuxData::Linkage.
889 if (F.hasAvailableExternallyLinkage() && !F.isDeclaration()) {
890 F.addFnAttr(SPIRV_WAS_AVAILABLE_EXTERNALLY_ATTR);
891 F.setLinkage(GlobalValue::ExternalLinkage);
892 Changed = true;
893 }
894 Changed |= substituteAbortKHRCalls(F: &F);
895 Changed |= substituteIntrinsicCalls(F: &F);
896 Changed |= sortBlocks(F);
897 Changed |= removeAggregateTypesFromCalls(F: &F);
898 }
899
900 std::vector<Function *> FuncsWorklist;
901 for (auto &F : M)
902 FuncsWorklist.push_back(x: &F);
903
904 for (auto *F : FuncsWorklist) {
905 Function *NewF = removeAggregateTypesFromSignature(F);
906
907 if (NewF != F) {
908 F->eraseFromParent();
909 Changed = true;
910 }
911 }
912 return Changed;
913}
914
915PreservedAnalyses SPIRVPrepareFunctions::run(Module &M,
916 ModuleAnalysisManager &AM) {
917 FunctionAnalysisManager &FAM =
918 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
919 auto GetTTI = [&FAM](Function &F) -> const TargetTransformInfo & {
920 return FAM.getResult<TargetIRAnalysis>(IR&: F);
921 };
922 return SPIRVPrepareFunctionsImpl(TM, GetTTI).runOnModule(M)
923 ? PreservedAnalyses::none()
924 : PreservedAnalyses::all();
925}
926
927ModulePass *
928llvm::createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM) {
929 return new SPIRVPrepareFunctionsLegacy(TM);
930}
931