1//===-- SPIRVPrepareFunctions.cpp - modify function signatures --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass modifies function signatures containing aggregate arguments
10// and/or return value before IRTranslator. Information about the original
11// signatures is stored in metadata. It is used during call lowering to
12// restore correct SPIR-V types of function arguments and return values.
13// This pass also substitutes some llvm intrinsic calls with calls to newly
14// generated functions (as the Khronos LLVM/SPIR-V Translator does).
15//
16// NOTE: this pass is a module-level one due to the necessity to modify
17// GVs/functions.
18//
19//===----------------------------------------------------------------------===//
20
21#include "SPIRV.h"
22#include "SPIRVBuiltins.h"
23#include "SPIRVSubtarget.h"
24#include "SPIRVTargetMachine.h"
25#include "SPIRVUtils.h"
26#include "llvm/ADT/StringExtras.h"
27#include "llvm/Analysis/TargetTransformInfo.h"
28#include "llvm/Analysis/ValueTracking.h"
29#include "llvm/CodeGen/IntrinsicLowering.h"
30#include "llvm/IR/AttributeMask.h"
31#include "llvm/IR/DiagnosticInfo.h"
32#include "llvm/IR/IRBuilder.h"
33#include "llvm/IR/InstIterator.h"
34#include "llvm/IR/Instructions.h"
35#include "llvm/IR/IntrinsicInst.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/IntrinsicsSPIRV.h"
38#include "llvm/InitializePasses.h"
39#include "llvm/Transforms/Utils/BasicBlockUtils.h"
40#include "llvm/Transforms/Utils/Cloning.h"
41#include "llvm/Transforms/Utils/Local.h"
42#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
43#include <regex>
44
45using namespace llvm;
46
47namespace {
48
49class SPIRVPrepareFunctionsImpl {
50 const SPIRVTargetMachine &TM;
51 function_ref<const TargetTransformInfo &(Function &)> GetTTI;
52 bool substituteIntrinsicCalls(Function *F);
53 bool substituteAbortKHRCalls(Function *F);
54 bool terminateBlocksAfterTrap(Module &M, Intrinsic::ID IID);
55 Function *removeAggregateTypesFromSignature(Function *F);
56 bool removeAggregateTypesFromCalls(Function *F);
57
58public:
59 SPIRVPrepareFunctionsImpl(
60 const SPIRVTargetMachine &TM,
61 function_ref<const TargetTransformInfo &(Function &)> GetTTI)
62 : TM(TM), GetTTI(GetTTI) {}
63 bool runOnModule(Module &M);
64};
65
66class SPIRVPrepareFunctionsLegacy : public ModulePass {
67 const SPIRVTargetMachine &TM;
68
69public:
70 static char ID;
71 SPIRVPrepareFunctionsLegacy(const SPIRVTargetMachine &TM)
72 : ModulePass(ID), TM(TM) {}
73
74 bool runOnModule(Module &M) override {
75 auto GetTTI = [this](Function &F) -> const TargetTransformInfo & {
76 return getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
77 };
78 return SPIRVPrepareFunctionsImpl(TM, GetTTI).runOnModule(M);
79 }
80
81 void getAnalysisUsage(AnalysisUsage &AU) const override {
82 AU.addRequired<TargetTransformInfoWrapperPass>();
83 }
84
85 StringRef getPassName() const override { return "SPIRV prepare functions"; }
86};
87
88static cl::list<std::string> SPVAllowUnknownIntrinsics(
89 "spv-allow-unknown-intrinsics", cl::CommaSeparated,
90 cl::desc("Emit unknown intrinsics as calls to external functions. A "
91 "comma-separated input list of intrinsic prefixes must be "
92 "provided, and only intrinsics carrying a listed prefix get "
93 "emitted as described."),
94 cl::value_desc("intrinsic_prefix_0,intrinsic_prefix_1"), cl::ValueOptional);
95} // namespace
96
97char SPIRVPrepareFunctionsLegacy::ID = 0;
98
99INITIALIZE_PASS_BEGIN(SPIRVPrepareFunctionsLegacy, "spirv-prepare-functions",
100 "SPIRV prepare functions", false, false)
101INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
102INITIALIZE_PASS_END(SPIRVPrepareFunctionsLegacy, "spirv-prepare-functions",
103 "SPIRV prepare functions", false, false)
104
105static std::string lowerLLVMIntrinsicName(IntrinsicInst *II) {
106 Function *IntrinsicFunc = II->getCalledFunction();
107 assert(IntrinsicFunc && "Missing function");
108 std::string FuncName = IntrinsicFunc->getName().str();
109 llvm::replace(Range&: FuncName, OldValue: '.', NewValue: '_');
110 FuncName = "spirv." + FuncName;
111 return FuncName;
112}
113
114static Function *getOrCreateFunction(Module *M, Type *RetTy,
115 ArrayRef<Type *> ArgTypes,
116 StringRef Name) {
117 FunctionType *FT = FunctionType::get(Result: RetTy, Params: ArgTypes, isVarArg: false);
118 Function *F = M->getFunction(Name);
119 if (F && F->getFunctionType() == FT)
120 return F;
121 Function *NewF = Function::Create(Ty: FT, Linkage: GlobalValue::ExternalLinkage, N: Name, M);
122 if (F)
123 NewF->setDSOLocal(F->isDSOLocal());
124 NewF->setCallingConv(CallingConv::SPIR_FUNC);
125 return NewF;
126}
127
128static bool lowerIntrinsicToFunction(IntrinsicInst *Intrinsic,
129 const TargetTransformInfo &TTI) {
130 // For @llvm.memset.* intrinsic cases with constant value and length arguments
131 // are emulated via "storing" a constant array to the destination. For other
132 // cases we wrap the intrinsic in @spirv.llvm_memset_* function and expand the
133 // intrinsic to a loop via expandMemSetAsLoop().
134 if (auto *MSI = dyn_cast<MemSetInst>(Val: Intrinsic))
135 if (isa<Constant>(Val: MSI->getValue()) && isa<ConstantInt>(Val: MSI->getLength()))
136 return false; // It is handled later using OpCopyMemorySized.
137
138 // An intrinsic with a metadata argument has no SPIR-V lowering and can't be
139 // turned into a function.
140 if (any_of(Range: Intrinsic->args(), P: IsaPred<MetadataAsValue>)) {
141 const Function *F = Intrinsic->getFunction();
142 F->getContext().diagnose(DI: DiagnosticInfoUnsupported(
143 *F,
144 "cannot lower the intrinsic '" +
145 Intrinsic->getCalledFunction()->getName() +
146 "' that takes a metadata argument",
147 Intrinsic->getDebugLoc()));
148 if (!Intrinsic->getType()->isVoidTy())
149 Intrinsic->replaceAllUsesWith(V: PoisonValue::get(T: Intrinsic->getType()));
150 Intrinsic->eraseFromParent();
151 return true;
152 }
153
154 Module *M = Intrinsic->getModule();
155 std::string FuncName = lowerLLVMIntrinsicName(II: Intrinsic);
156 if (Intrinsic->isVolatile())
157 FuncName += ".volatile";
158 // Redirect @llvm.intrinsic.* call to @spirv.llvm_intrinsic_*
159 Function *F = M->getFunction(Name: FuncName);
160 if (F) {
161 Intrinsic->setCalledFunction(F);
162 return true;
163 }
164 FunctionCallee FC =
165 M->getOrInsertFunction(Name: FuncName, T: Intrinsic->getFunctionType());
166 auto IntrinsicID = Intrinsic->getIntrinsicID();
167 Intrinsic->setCalledFunction(FC);
168 F = cast<Function>(Val: FC.getCallee());
169 F->setAttributes(Intrinsic->getAttributes());
170
171 switch (IntrinsicID) {
172 case Intrinsic::memset: {
173 auto *MSI = static_cast<MemSetInst *>(Intrinsic);
174 Argument *Dest = F->getArg(i: 0);
175 Argument *Val = F->getArg(i: 1);
176 Argument *Len = F->getArg(i: 2);
177 Argument *IsVolatile = F->getArg(i: 3);
178 Dest->setName("dest");
179 Val->setName("val");
180 Len->setName("len");
181 IsVolatile->setName("isvolatile");
182 BasicBlock *EntryBB = BasicBlock::Create(Context&: M->getContext(), Name: "entry", Parent: F);
183 IRBuilder<> IRB(EntryBB);
184 auto *MemSet = IRB.CreateMemSet(Ptr: Dest, Val, Size: Len, Align: MSI->getDestAlign(),
185 isVolatile: MSI->isVolatile());
186 IRB.CreateRetVoid();
187 expandMemSetAsLoop(MemSet: cast<MemSetInst>(Val: MemSet), TTI);
188 MemSet->eraseFromParent();
189 break;
190 }
191 case Intrinsic::bswap: {
192 BasicBlock *EntryBB = BasicBlock::Create(Context&: M->getContext(), Name: "entry", Parent: F);
193 IRBuilder<> IRB(EntryBB);
194 CallInst *BSwap = IRB.CreateIntrinsicWithoutFolding(
195 ID: Intrinsic::bswap, OverloadTypes: Intrinsic->getType(), Args: F->getArg(i: 0));
196 IRB.CreateRet(V: BSwap);
197 IntrinsicLowering IL(M->getDataLayout());
198 IL.LowerIntrinsicCall(CI: BSwap);
199 break;
200 }
201 default:
202 break;
203 }
204 return true;
205}
206
207static std::string getAnnotation(Value *AnnoVal, Value *OptAnnoVal) {
208 if (auto *Ref = dyn_cast_or_null<GetElementPtrInst>(Val: AnnoVal))
209 AnnoVal = Ref->getOperand(i_nocapture: 0);
210 if (auto *Ref = dyn_cast_or_null<BitCastInst>(Val: OptAnnoVal))
211 OptAnnoVal = Ref->getOperand(i_nocapture: 0);
212
213 std::string Anno;
214 if (auto *C = dyn_cast_or_null<Constant>(Val: AnnoVal)) {
215 StringRef Str;
216 if (getConstantStringInfo(V: C, Str))
217 Anno = Str;
218 }
219 // handle optional annotation parameter in a way that Khronos Translator do
220 // (collect integers wrapped in a struct)
221 if (auto *C = dyn_cast_or_null<Constant>(Val: OptAnnoVal);
222 C && C->getNumOperands()) {
223 Value *MaybeStruct = C->getOperand(i: 0);
224 if (auto *Struct = dyn_cast<ConstantStruct>(Val: MaybeStruct)) {
225 for (unsigned I = 0, E = Struct->getNumOperands(); I != E; ++I) {
226 if (auto *CInt = dyn_cast<ConstantInt>(Val: Struct->getOperand(i_nocapture: I)))
227 Anno += (I == 0 ? ": " : ", ") +
228 std::to_string(val: CInt->getType()->getIntegerBitWidth() == 1
229 ? CInt->getZExtValue()
230 : CInt->getSExtValue());
231 }
232 } else if (auto *Struct = dyn_cast<ConstantAggregateZero>(Val: MaybeStruct)) {
233 // { i32 i32 ... } zeroinitializer
234 for (unsigned I = 0, E = Struct->getType()->getStructNumElements();
235 I != E; ++I)
236 Anno += I == 0 ? ": 0" : ", 0";
237 }
238 }
239 return Anno;
240}
241
242static SmallVector<Metadata *> parseAnnotation(Value *I,
243 const std::string &Anno,
244 LLVMContext &Ctx,
245 Type *Int32Ty) {
246 // Try to parse the annotation string according to the following rules:
247 // annotation := ({kind} | {kind:value,value,...})+
248 // kind := number
249 // value := number | string
250 static const std::regex R(
251 "\\{(\\d+)(?:[:,](\\d+|\"[^\"]*\")(?:,(\\d+|\"[^\"]*\"))*)?\\}");
252 SmallVector<Metadata *> MDs;
253 int Pos = 0;
254 for (std::sregex_iterator
255 It = std::sregex_iterator(Anno.begin(), Anno.end(), R),
256 ItEnd = std::sregex_iterator();
257 It != ItEnd; ++It) {
258 if (It->position() != Pos)
259 return SmallVector<Metadata *>{};
260 Pos = It->position() + It->length();
261 std::smatch Match = *It;
262 SmallVector<Metadata *> MDsItem;
263 for (std::size_t i = 1; i < Match.size(); ++i) {
264 std::ssub_match SMatch = Match[i];
265 std::string Item = SMatch.str();
266 if (Item.length() == 0)
267 break;
268 if (Item[0] == '"') {
269 Item = Item.substr(pos: 1, n: Item.length() - 2);
270 // Acceptable format of the string snippet is:
271 static const std::regex RStr("^(\\d+)(?:,(\\d+))*$");
272 if (std::smatch MatchStr; std::regex_match(s: Item, m&: MatchStr, re: RStr)) {
273 for (std::size_t SubIdx = 1; SubIdx < MatchStr.size(); ++SubIdx)
274 if (std::string SubStr = MatchStr[SubIdx].str(); SubStr.length())
275 MDsItem.push_back(Elt: ConstantAsMetadata::get(
276 C: ConstantInt::get(Ty: Int32Ty, V: std::stoi(str: SubStr))));
277 } else {
278 MDsItem.push_back(Elt: MDString::get(Context&: Ctx, Str: Item));
279 }
280 } else if (int32_t Num; llvm::to_integer(S: StringRef(Item), Num, Base: 10)) {
281 MDsItem.push_back(
282 Elt: ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: Num)));
283 } else {
284 MDsItem.push_back(Elt: MDString::get(Context&: Ctx, Str: Item));
285 }
286 }
287 if (MDsItem.size() == 0)
288 return SmallVector<Metadata *>{};
289 MDs.push_back(Elt: MDNode::get(Context&: Ctx, MDs: MDsItem));
290 }
291 return Pos == static_cast<int>(Anno.length()) ? std::move(MDs)
292 : SmallVector<Metadata *>{};
293}
294
295static void lowerPtrAnnotation(IntrinsicInst *II) {
296 LLVMContext &Ctx = II->getContext();
297 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
298
299 // Retrieve an annotation string from arguments.
300 Value *PtrArg = nullptr;
301 if (auto *BI = dyn_cast<BitCastInst>(Val: II->getArgOperand(i: 0)))
302 PtrArg = BI->getOperand(i_nocapture: 0);
303 else
304 PtrArg = II->getOperand(i_nocapture: 0);
305 std::string Anno =
306 getAnnotation(AnnoVal: II->getArgOperand(i: 1),
307 OptAnnoVal: 4 < II->arg_size() ? II->getArgOperand(i: 4) : nullptr);
308
309 // Parse the annotation.
310 SmallVector<Metadata *> MDs = parseAnnotation(I: II, Anno, Ctx, Int32Ty);
311
312 // If the annotation string is not parsed successfully we don't know the
313 // format used and output it as a general UserSemantic decoration.
314 // Otherwise MDs is a Metadata tuple (a decoration list) in the format
315 // expected by `spirv.Decorations`.
316 if (MDs.size() == 0) {
317 auto UserSemantic = ConstantAsMetadata::get(C: ConstantInt::get(
318 Ty: Int32Ty, V: static_cast<uint32_t>(SPIRV::Decoration::UserSemantic)));
319 MDs.push_back(Elt: MDNode::get(Context&: Ctx, MDs: {UserSemantic, MDString::get(Context&: Ctx, Str: Anno)}));
320 }
321
322 // Build the internal intrinsic function.
323 IRBuilder<> IRB(II->getParent());
324 IRB.SetInsertPoint(II);
325 IRB.CreateIntrinsic(
326 ID: Intrinsic::spv_assign_decoration, OverloadTypes: {PtrArg->getType()},
327 Args: {PtrArg, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs))});
328 II->replaceAllUsesWith(V: II->getOperand(i_nocapture: 0));
329}
330
331static void lowerFunnelShifts(IntrinsicInst *FSHIntrinsic) {
332 // Get a separate function - otherwise, we'd have to rework the CFG of the
333 // current one. Then simply replace the intrinsic uses with a call to the new
334 // function.
335 // Generate LLVM IR for i* @spirv.llvm_fsh?_i* (i* %a, i* %b, i* %c)
336 Module *M = FSHIntrinsic->getModule();
337 FunctionType *FSHFuncTy = FSHIntrinsic->getFunctionType();
338 Type *FSHRetTy = FSHFuncTy->getReturnType();
339 const std::string FuncName = lowerLLVMIntrinsicName(II: FSHIntrinsic);
340 Function *FSHFunc =
341 getOrCreateFunction(M, RetTy: FSHRetTy, ArgTypes: FSHFuncTy->params(), Name: FuncName);
342
343 if (!FSHFunc->empty()) {
344 FSHIntrinsic->setCalledFunction(FSHFunc);
345 return;
346 }
347 BasicBlock *RotateBB = BasicBlock::Create(Context&: M->getContext(), Name: "rotate", Parent: FSHFunc);
348 IRBuilder<> IRB(RotateBB);
349 Type *Ty = FSHFunc->getReturnType();
350 // Build the actual funnel shift rotate logic.
351 // In the comments, "int" is used interchangeably with "vector of int
352 // elements".
353 FixedVectorType *VectorTy = dyn_cast<FixedVectorType>(Val: Ty);
354 Type *IntTy = VectorTy ? VectorTy->getElementType() : Ty;
355 unsigned BitWidth = IntTy->getIntegerBitWidth();
356 ConstantInt *BitWidthConstant = IRB.getInt(AI: {BitWidth, BitWidth});
357 Value *BitWidthForInsts =
358 VectorTy
359 ? IRB.CreateVectorSplat(NumElts: VectorTy->getNumElements(), V: BitWidthConstant)
360 : BitWidthConstant;
361 Value *RotateModVal =
362 IRB.CreateURem(/*Rotate*/ LHS: FSHFunc->getArg(i: 2), RHS: BitWidthForInsts);
363 Value *FirstShift = nullptr, *SecShift = nullptr;
364 if (FSHIntrinsic->getIntrinsicID() == Intrinsic::fshr) {
365 // Shift the less significant number right, the "rotate" number of bits
366 // will be 0-filled on the left as a result of this regular shift.
367 FirstShift = IRB.CreateLShr(LHS: FSHFunc->getArg(i: 1), RHS: RotateModVal);
368 } else {
369 // Shift the more significant number left, the "rotate" number of bits
370 // will be 0-filled on the right as a result of this regular shift.
371 FirstShift = IRB.CreateShl(LHS: FSHFunc->getArg(i: 0), RHS: RotateModVal);
372 }
373 // We want the "rotate" number of the more significant int's LSBs (MSBs) to
374 // occupy the leftmost (rightmost) "0 space" left by the previous operation.
375 // Therefore, subtract the "rotate" number from the integer bitsize...
376 Value *SubRotateVal = IRB.CreateSub(LHS: BitWidthForInsts, RHS: RotateModVal);
377 if (FSHIntrinsic->getIntrinsicID() == Intrinsic::fshr) {
378 // ...and left-shift the more significant int by this number, zero-filling
379 // the LSBs.
380 SecShift = IRB.CreateShl(LHS: FSHFunc->getArg(i: 0), RHS: SubRotateVal);
381 } else {
382 // ...and right-shift the less significant int by this number, zero-filling
383 // the MSBs.
384 SecShift = IRB.CreateLShr(LHS: FSHFunc->getArg(i: 1), RHS: SubRotateVal);
385 }
386 // A simple binary addition of the shifted ints yields the final result.
387 IRB.CreateRet(V: IRB.CreateOr(LHS: FirstShift, RHS: SecShift));
388
389 FSHIntrinsic->setCalledFunction(FSHFunc);
390}
391
392static void lowerConstrainedFPCmpIntrinsic(
393 ConstrainedFPCmpIntrinsic *ConstrainedCmpIntrinsic,
394 SmallVector<Instruction *> &EraseFromParent) {
395 if (!ConstrainedCmpIntrinsic)
396 return;
397 // Extract the floating-point values being compared
398 Value *LHS = ConstrainedCmpIntrinsic->getArgOperand(i: 0);
399 Value *RHS = ConstrainedCmpIntrinsic->getArgOperand(i: 1);
400 FCmpInst::Predicate Pred = ConstrainedCmpIntrinsic->getPredicate();
401 IRBuilder<> Builder(ConstrainedCmpIntrinsic);
402 Value *FCmp = Builder.CreateFCmp(P: Pred, LHS, RHS);
403 ConstrainedCmpIntrinsic->replaceAllUsesWith(V: FCmp);
404 EraseFromParent.push_back(Elt: dyn_cast<Instruction>(Val: ConstrainedCmpIntrinsic));
405}
406
407static void lowerExpectAssume(IntrinsicInst *II) {
408 // If we cannot use the SPV_KHR_expect_assume extension, then we need to
409 // ignore the intrinsic and move on. It should be removed later on by LLVM.
410 // Otherwise we should lower the intrinsic to the corresponding SPIR-V
411 // instruction.
412 // For @llvm.assume we have OpAssumeTrueKHR.
413 // For @llvm.expect and @llvm.expect.with.probability we have OpExpectKHR.
414 //
415 // We need to lower this into a builtin and then the builtin into a SPIR-V
416 // instruction.
417 if (II->getIntrinsicID() == Intrinsic::assume) {
418 Function *F = Intrinsic::getOrInsertDeclaration(
419 M: II->getModule(), id: Intrinsic::SPVIntrinsics::spv_assume);
420 II->setCalledFunction(F);
421 } else if (II->getIntrinsicID() == Intrinsic::expect) {
422 Function *F = Intrinsic::getOrInsertDeclaration(
423 M: II->getModule(), id: Intrinsic::SPVIntrinsics::spv_expect,
424 OverloadTys: {II->getOperand(i_nocapture: 0)->getType()});
425 II->setCalledFunction(F);
426 } else if (II->getIntrinsicID() == Intrinsic::expect_with_probability) {
427 // OpExpectKHR has no probability operand. Keep the hint only if the
428 // expected value is likely, as LowerExpectIntrinsic does for PHIs.
429 Value *Res = II->getArgOperand(i: 0);
430 double Prob =
431 cast<ConstantFP>(Val: II->getArgOperand(i: 2))->getValueAPF().convertToDouble();
432 if (Prob > 0.5)
433 Res = IRBuilder<>(II).CreateIntrinsic(
434 ID: Intrinsic::SPVIntrinsics::spv_expect, OverloadTypes: {II->getType()},
435 Args: {II->getArgOperand(i: 0), II->getArgOperand(i: 1)});
436 BasicBlock::iterator BI = II->getIterator();
437 ReplaceInstWithValue(BI, V: Res);
438 } else {
439 llvm_unreachable("Unknown intrinsic");
440 }
441}
442
443static bool toSpvLifetimeIntrinsic(IntrinsicInst *II, Intrinsic::ID NewID) {
444 auto *LifetimeArg0 = II->getArgOperand(i: 0);
445
446 // If the lifetime argument is a poison value, the intrinsic has no effect.
447 if (isa<PoisonValue>(Val: LifetimeArg0)) {
448 II->eraseFromParent();
449 return true;
450 }
451
452 auto *Alloca = dyn_cast<AllocaInst>(Val: LifetimeArg0);
453 if (!Alloca || !Alloca->isStaticAlloca()) {
454 II->eraseFromParent();
455 return true;
456 }
457
458 IRBuilder<> Builder(II);
459 std::optional<TypeSize> Size =
460 Alloca->getAllocationSize(DL: Alloca->getDataLayout());
461 Value *SizeVal = Builder.getInt64(C: Size ? *Size : -1);
462 Builder.CreateIntrinsic(ID: NewID, OverloadTypes: Alloca->getType(), Args: {SizeVal, LifetimeArg0});
463 II->eraseFromParent();
464 return true;
465}
466
467static void
468lowerConstrainedFmuladd(IntrinsicInst *II,
469 SmallVector<Instruction *> &EraseFromParent) {
470 auto *FPI = cast<ConstrainedFPIntrinsic>(Val: II);
471 Value *A = FPI->getArgOperand(i: 0);
472 Value *Mul = FPI->getArgOperand(i: 1);
473 Value *Add = FPI->getArgOperand(i: 2);
474 IRBuilder<> Builder(II->getParent());
475 Builder.SetInsertPoint(II);
476 std::optional<RoundingMode> Rounding = FPI->getRoundingMode();
477 Value *Product = Builder.CreateFMul(L: A, R: Mul, Name: II->getName() + ".mul");
478 Value *Result = Builder.CreateConstrainedFPBinOp(
479 ID: Intrinsic::experimental_constrained_fadd, L: Product, R: Add, FMFSource: {},
480 Name: II->getName() + ".add", FPMathTag: nullptr, Rounding);
481 II->replaceAllUsesWith(V: Result);
482 EraseFromParent.push_back(Elt: II);
483}
484
485// llvm.minimum/llvm.maximum return NaN if either operand is NaN and order -0.0
486// before +0.0, whereas OpenCL.std fmin/fmax and GLSL.std.450 NMin/NMax return
487// the other operand for a NaN and may return either zero. Expand them like
488// SPIRV-LLVM-Translator does, so that both produce the same OpenCL SPIR-V. For
489// llvm.minimum (llvm.maximum is symmetrical):
490// %m = call @llvm.minnum(%x, %y)
491// ; unless nsz: for non-NaN operands, the result has the sign of %x if that
492// ; is set (maximum: clear), and the sign of %y otherwise. This only changes
493// ; %m if it is a zero of the wrong sign.
494// %xsign = call @llvm.copysign(1.0, %x)
495// %xneg = fcmp olt %xsign, 0.0
496// %s = select %xneg, %x, %y
497// %m.s = call @llvm.copysign(%m, %s)
498// ; unless nnan
499// %uno = fcmp uno %x, %y
500// %res = select %uno, NaN, %m.s
501static Value *expandFMinimumMaximum(IRBuilderBase &Builder, bool IsMax,
502 Value *X, Value *Y, FastMathFlags FMF) {
503 Type *Ty = X->getType();
504 Value *Res = Builder.CreateIntrinsic(
505 ID: IsMax ? Intrinsic::maxnum : Intrinsic::minnum, OverloadTypes: {Ty}, Args: {X, Y}, FMFSource: FMF);
506 if (!FMF.noSignedZeros()) {
507 Value *XSign = Builder.CreateCopySign(LHS: ConstantFP::get(Ty, V: 1.0), RHS: X);
508 Value *XNeg = Builder.CreateFCmpOLT(LHS: XSign, RHS: ConstantFP::getZero(Ty));
509 Value *Sign = IsMax ? Builder.CreateSelect(C: XNeg, True: Y, False: X)
510 : Builder.CreateSelect(C: XNeg, True: X, False: Y);
511 Res = Builder.CreateCopySign(LHS: Res, RHS: Sign);
512 }
513 if (!FMF.noNaNs()) {
514 Value *Uno = Builder.CreateFCmpUNO(LHS: X, RHS: Y);
515 Res = Builder.CreateSelect(C: Uno, True: ConstantFP::getQNaN(Ty), False: Res);
516 }
517 return Res;
518}
519
520static void lowerFMinimumMaximum(IntrinsicInst *II,
521 SmallVector<Instruction *> &EraseFromParent) {
522 IRBuilder<> Builder(II);
523 Value *Res = expandFMinimumMaximum(
524 Builder, IsMax: II->getIntrinsicID() == Intrinsic::maximum, X: II->getArgOperand(i: 0),
525 Y: II->getArgOperand(i: 1), FMF: II->getFastMathFlags());
526 Res->takeName(V: II);
527 II->replaceAllUsesWith(V: Res);
528 EraseFromParent.push_back(Elt: II);
529}
530
531// ExpandReductions only expands llvm.vector.reduce.fminimum/fmaximum on vectors
532// with a power-of-two number of elements. Expand the others into a chain of
533// llvm.minimum/llvm.maximum, expanded as above.
534static void
535lowerVectorReduceFMinimumMaximum(IntrinsicInst *II,
536 SmallVector<Instruction *> &EraseFromParent) {
537 bool IsMax = II->getIntrinsicID() == Intrinsic::vector_reduce_fmaximum;
538 Value *Vec = II->getArgOperand(i: 0);
539 unsigned NumElts = cast<FixedVectorType>(Val: Vec->getType())->getNumElements();
540 IRBuilder<> Builder(II);
541 SmallVector<Value *> Elts;
542 for (unsigned I = 0; I < NumElts; ++I)
543 Elts.push_back(Elt: Builder.CreateExtractElement(Vec, Idx: I));
544 Value *Res = Elts[0];
545 for (Value *Elt : drop_begin(RangeOrContainer&: Elts))
546 Res =
547 expandFMinimumMaximum(Builder, IsMax, X: Res, Y: Elt, FMF: II->getFastMathFlags());
548 Res->takeName(V: II);
549 II->replaceAllUsesWith(V: Res);
550 EraseFromParent.push_back(Elt: II);
551}
552
553// Substitutes calls to LLVM intrinsics with either calls to SPIR-V intrinsics
554// or calls to proper generated functions. Returns True if F was modified.
555bool SPIRVPrepareFunctionsImpl::substituteIntrinsicCalls(Function *F) {
556 if (F->isDeclaration())
557 return false;
558
559 bool Changed = false;
560 const SPIRVSubtarget &STI = TM.getSubtarget<SPIRVSubtarget>(F: *F);
561 SmallVector<Instruction *> EraseFromParent;
562 const TargetTransformInfo &TTI = GetTTI(*F);
563 for (BasicBlock &BB : *F) {
564 for (Instruction &I : make_early_inc_range(Range&: BB)) {
565 auto Call = dyn_cast<CallInst>(Val: &I);
566 if (!Call)
567 continue;
568 Function *CF = Call->getCalledFunction();
569 if (!CF || !CF->isIntrinsic())
570 continue;
571 auto *II = cast<IntrinsicInst>(Val: Call);
572 if (Intrinsic::isTargetIntrinsic(IID: II->getIntrinsicID()) &&
573 II->getCalledOperand()->getName().starts_with(Prefix: "llvm.spv"))
574 continue;
575 switch (II->getIntrinsicID()) {
576 case Intrinsic::memset:
577 case Intrinsic::bswap:
578 Changed |= lowerIntrinsicToFunction(Intrinsic: II, TTI);
579 break;
580 case Intrinsic::fshl:
581 case Intrinsic::fshr:
582 lowerFunnelShifts(FSHIntrinsic: II);
583 Changed = true;
584 break;
585 case Intrinsic::minimum:
586 case Intrinsic::maximum:
587 lowerFMinimumMaximum(II, EraseFromParent);
588 Changed = true;
589 break;
590 case Intrinsic::vector_reduce_fminimum:
591 case Intrinsic::vector_reduce_fmaximum:
592 lowerVectorReduceFMinimumMaximum(II, EraseFromParent);
593 Changed = true;
594 break;
595 case Intrinsic::assume:
596 case Intrinsic::expect:
597 case Intrinsic::expect_with_probability:
598 if (STI.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume))
599 lowerExpectAssume(II);
600 Changed = true;
601 break;
602 case Intrinsic::lifetime_start:
603 if (!STI.isShader()) {
604 Changed |= toSpvLifetimeIntrinsic(
605 II, NewID: Intrinsic::SPVIntrinsics::spv_lifetime_start);
606 } else {
607 II->eraseFromParent();
608 Changed = true;
609 }
610 break;
611 case Intrinsic::lifetime_end:
612 if (!STI.isShader()) {
613 Changed |= toSpvLifetimeIntrinsic(
614 II, NewID: Intrinsic::SPVIntrinsics::spv_lifetime_end);
615 } else {
616 II->eraseFromParent();
617 Changed = true;
618 }
619 break;
620 case Intrinsic::ptr_annotation:
621 lowerPtrAnnotation(II);
622 Changed = true;
623 break;
624 case Intrinsic::experimental_constrained_fmuladd:
625 lowerConstrainedFmuladd(II, EraseFromParent);
626 Changed = true;
627 break;
628 case Intrinsic::experimental_constrained_fcmp:
629 case Intrinsic::experimental_constrained_fcmps:
630 lowerConstrainedFPCmpIntrinsic(ConstrainedCmpIntrinsic: dyn_cast<ConstrainedFPCmpIntrinsic>(Val: II),
631 EraseFromParent);
632 Changed = true;
633 break;
634 default:
635 // Drop assume-like intrinsics that have no SPIR-V representation.
636 if (II->isAssumeLikeIntrinsic()) {
637 if (!II->getType()->isVoidTy())
638 II->replaceAllUsesWith(V: PoisonValue::get(T: II->getType()));
639 II->eraseFromParent();
640 Changed = true;
641 break;
642 }
643 if (STI.getTargetTriple().getVendor() == Triple::AMD ||
644 any_of(Range&: SPVAllowUnknownIntrinsics, P: [II](auto &&Prefix) {
645 if (Prefix.empty())
646 return false;
647 return II->getCalledFunction()->getName().starts_with(Prefix);
648 }))
649 Changed |= lowerIntrinsicToFunction(Intrinsic: II, TTI);
650 break;
651 }
652 }
653 }
654 for (auto *I : EraseFromParent)
655 I->eraseFromParent();
656 return Changed;
657}
658
659static void
660addFunctionTypeMutation(NamedMDNode *NMD,
661 SmallVector<std::pair<int, Type *>> ChangedTys,
662 StringRef Name, StringRef AsmConstraints = "") {
663
664 LLVMContext &Ctx = NMD->getParent()->getContext();
665 Type *I32Ty = IntegerType::getInt32Ty(C&: Ctx);
666
667 SmallVector<Metadata *> MDArgs;
668 MDArgs.push_back(Elt: MDString::get(Context&: Ctx, Str: Name));
669 transform(Range&: ChangedTys, d_first: std::back_inserter(x&: MDArgs), F: [=, &Ctx](auto &&CTy) {
670 return MDNode::get(
671 Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(I32Ty, CTy.first, true)),
672 ValueAsMetadata::get(V: Constant::getNullValue(Ty: CTy.second))});
673 });
674 if (!AsmConstraints.empty())
675 MDArgs.push_back(Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: AsmConstraints)));
676 NMD->addOperand(M: MDNode::get(Context&: Ctx, MDs: MDArgs));
677}
678
679// Drops attributes that are invalid on the i32 placeholders in the type of V.
680template <typename T> static void removeIncompatibleAttrs(T *V) {
681 FunctionType *FTy = V->getFunctionType();
682 AttributeList Attrs = V->getAttributes();
683 V->removeRetAttrs(AttributeFuncs::typeIncompatible(Ty: FTy->getReturnType(),
684 AS: Attrs.getRetAttrs()));
685 for (auto [I, Ty] : enumerate(First: FTy->params()))
686 V->removeParamAttrs(
687 I, AttributeFuncs::typeIncompatible(Ty, AS: Attrs.getParamAttrs(ArgNo: I)));
688}
689
690// Returns F if aggregate argument/return types are not present or cloned F
691// function with the types replaced by i32 types. The change in types is
692// noted in 'spv.cloned_funcs' metadata for later restoration.
693Function *
694SPIRVPrepareFunctionsImpl::removeAggregateTypesFromSignature(Function *F) {
695 bool IsRetAggr = F->getReturnType()->isAggregateType();
696 // Allow intrinsics with aggregate return/argument types to reach GlobalISel.
697 // Renaming/mutating the signature of an intrinsic would desync its name from
698 // its argument types and break the IR verifier.
699 if (F->isIntrinsic())
700 return F;
701
702 IRBuilder<> B(*F->getParent());
703
704 bool HasAggrArg = llvm::any_of(Range: F->args(), P: [](Argument &Arg) {
705 return Arg.getType()->isAggregateType();
706 });
707 bool DoClone = IsRetAggr || HasAggrArg;
708 if (!DoClone)
709 return F;
710 SmallVector<std::pair<int, Type *>, 4> ChangedTypes;
711 Type *RetType = IsRetAggr ? B.getInt32Ty() : F->getReturnType();
712 if (IsRetAggr)
713 ChangedTypes.push_back(Elt: std::pair<int, Type *>(-1, F->getReturnType()));
714 SmallVector<Type *, 4> ArgTypes;
715 for (const auto &Arg : F->args()) {
716 if (Arg.getType()->isAggregateType()) {
717 ArgTypes.push_back(Elt: B.getInt32Ty());
718 ChangedTypes.push_back(
719 Elt: std::pair<int, Type *>(Arg.getArgNo(), Arg.getType()));
720 } else
721 ArgTypes.push_back(Elt: Arg.getType());
722 }
723 FunctionType *NewFTy =
724 FunctionType::get(Result: RetType, Params: ArgTypes, isVarArg: F->getFunctionType()->isVarArg());
725 Function *NewF =
726 Function::Create(Ty: NewFTy, Linkage: F->getLinkage(), AddrSpace: F->getAddressSpace(),
727 N: F->getName(), M: F->getParent());
728
729 ValueToValueMapTy VMap;
730 auto NewFArgIt = NewF->arg_begin();
731 for (auto &Arg : F->args()) {
732 StringRef ArgName = Arg.getName();
733 NewFArgIt->setName(ArgName);
734 VMap[&Arg] = &(*NewFArgIt++);
735 }
736 SmallVector<ReturnInst *, 8> Returns;
737
738 CloneFunctionInto(NewFunc: NewF, OldFunc: F, VMap, Changes: CloneFunctionChangeType::LocalChangesOnly,
739 Returns);
740 NewF->takeName(V: F);
741 NewF->setComdat(F->getComdat());
742 removeIncompatibleAttrs(V: NewF);
743
744 addFunctionTypeMutation(
745 NMD: NewF->getParent()->getOrInsertNamedMetadata(Name: "spv.cloned_funcs"),
746 ChangedTys: std::move(ChangedTypes), Name: NewF->getName());
747
748 for (User *U : F->users()) {
749 if (auto *CB = dyn_cast<CallBase>(Val: U); CB && CB->getCalledFunction() == F) {
750 CB->mutateFunctionType(FTy: NewF->getFunctionType());
751 removeIncompatibleAttrs(V: CB);
752 }
753 }
754 // NewF keeps F's address space, so their pointer types match and
755 // RAUW is safe despite the differing signatures.
756 assert(F->getType() == NewF->getType() &&
757 "RAUW requires F and NewF to share the same pointer type");
758 F->replaceAllUsesWith(V: NewF);
759
760 // register the mutation
761 if (RetType != F->getReturnType())
762 TM.getSubtarget<SPIRVSubtarget>(F: *F).getSPIRVGlobalRegistry()->addMutated(
763 Val: NewF, Ty: F->getReturnType());
764 return NewF;
765}
766
767// Returns true iff `F`'s name resolves (after OpenCL/SPIR-V demangling and
768// builtin-name lookup) to the SPIR-V friendly built-in `__spirv_AbortKHR`.
769static bool isAbortKHRBuiltin(const Function &F) {
770 if (F.isIntrinsic())
771 return false;
772 StringRef Name = F.getName();
773 // Quick reject: the mangled or unmangled name must contain the substring.
774 if (!Name.contains(Other: "__spirv_AbortKHR"))
775 return false;
776 std::string Demangled = getOclOrSpirvBuiltinDemangledName(Name);
777 if (Demangled.empty())
778 return false;
779 return SPIRV::lookupBuiltinNameHelper(DemangledCall: Demangled) == "__spirv_AbortKHR";
780}
781
782// Rewrites a single call to `__spirv_AbortKHR` into a call to the
783// `llvm.spv.abort` target intrinsic, then re-terminates the block with
784// `unreachable`. OpAbortKHR is itself a SPIR-V function-termination
785// instruction and must be the last instruction in its block, so any trailing
786// stores/lifetime intrinsics/`ret` emitted by the OpenCL ABI are dropped.
787// `changeToUnreachable` cleans up any successor PHI predecessor entries.
788static void rewriteAbortKHRCall(CallInst *CI) {
789 IRBuilder<> B(CI);
790 Value *Msg = CI->getArgOperand(i: 0);
791 // The OpenCL C ABI may pass aggregate arguments by pointer (byval). In that
792 // case load the underlying value so that OpAbortKHR receives the composite
793 // itself, as required by the SPV_KHR_abort spec ("Message Type must be a
794 // concrete type").
795 if (CI->isByValArgument(ArgNo: 0)) {
796 Type *AggTy = CI->getParamByValType(ArgNo: 0);
797 Msg = B.CreateLoad(Ty: AggTy, Ptr: Msg);
798 }
799 B.CreateIntrinsic(ID: Intrinsic::spv_abort, OverloadTypes: {Msg->getType()}, Args: {Msg});
800 changeToUnreachable(I: CI);
801}
802
803// Replace OpenCL/SPIR-V style calls to `__spirv_AbortKHR(message)` (i.e.
804// calls to `F` when `F` is the `__spirv_AbortKHR` built-in) with calls to the
805// `llvm.spv.abort` target intrinsic.
806bool SPIRVPrepareFunctionsImpl::substituteAbortKHRCalls(Function *F) {
807 if (!isAbortKHRBuiltin(F: *F))
808 return false;
809
810 bool Changed = false;
811 for (User *U : make_early_inc_range(Range: F->users())) {
812 auto *CI = dyn_cast<CallInst>(Val: U);
813 if (!CI || CI->getCalledFunction() != F)
814 continue;
815 if (CI->arg_size() != 1)
816 continue;
817 rewriteAbortKHRCall(CI);
818 Changed = true;
819 }
820
821 return Changed;
822}
823
824// When the SPV_KHR_abort extension is enabled, `llvm.trap` and
825// `llvm.ubsantrap` are lowered to `OpAbortKHR` during instruction selection.
826// `OpAbortKHR` is itself a SPIR-V block terminator, so any instructions that
827// follow the trap call within the same basic block (e.g. `ret`, lifetime
828// markers) would produce SPIR-V ops after `OpAbortKHR` and break validation.
829// Terminate the block right after each call to the trap intrinsics by replacing
830// the next instruction with `unreachable`.
831bool SPIRVPrepareFunctionsImpl::terminateBlocksAfterTrap(Module &M,
832 Intrinsic::ID IID) {
833 assert((IID == Intrinsic::trap || IID == Intrinsic::ubsantrap) &&
834 "Expected trap intrinsic ID");
835
836 Function *F = Intrinsic::getDeclarationIfExists(M: &M, id: IID);
837 if (!F)
838 return false;
839
840 // If the target doesn't support SPV_KHR_abort, we won't be able to lower
841 // the trap intrinsic to OpAbortKHR, so we can skip the block-terminating
842 // transformation.
843 const auto &ST = TM.getSubtarget<SPIRVSubtarget>(F: *F);
844 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_abort))
845 return false;
846
847 bool Changed = false;
848 for (User *U : make_early_inc_range(Range: F->users())) {
849 auto *CI = dyn_cast<CallInst>(Val: U);
850 if (!CI || CI->getCalledFunction() != F)
851 continue;
852 Instruction *Next = CI->getNextNode();
853 if (!Next || isa<UnreachableInst>(Val: Next))
854 continue;
855 changeToUnreachable(I: Next);
856 Changed = true;
857 }
858 return Changed;
859}
860
861static std::string fixMultiOutputConstraintString(StringRef Constraints) {
862 // We should only have one =r return for the made up ASM type.
863 SmallVector<StringRef> Tmp;
864 SplitString(Source: Constraints, OutFragments&: Tmp, Delimiters: ",");
865 std::string SafeConstraints("=r,");
866 for (unsigned I = 0u; I != Tmp.size() - 1; ++I) {
867 if (Tmp[I].starts_with(Prefix: '=') && (Tmp[I][1] == '&' || isalnum(Tmp[I][1])))
868 continue;
869 SafeConstraints.append(svt: Tmp[I]).append(l: {','});
870 }
871 SafeConstraints.append(svt: Tmp.back());
872
873 return SafeConstraints;
874}
875
876// Mutates indirect and inline ASM callsites iff aggregate argument/return types
877// are present with the types replaced by i32 types. The change in types is
878// noted in 'spv.mutated_callsites' metadata for later restoration. For ASM we
879// also have to mutate the constraint string as IRTranslator tries to handle
880// multiple outputs and expects an aggregate return type in their presence.
881bool SPIRVPrepareFunctionsImpl::removeAggregateTypesFromCalls(Function *F) {
882 if (F->isDeclaration() || F->isIntrinsic())
883 return false;
884
885 SmallVector<std::pair<CallBase *, FunctionType *>> Calls;
886 for (auto &&I : instructions(F)) {
887 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
888 if (!CB->getCalledOperand() || CB->getCalledFunction())
889 continue;
890 if (CB->getType()->isAggregateType() ||
891 any_of(Range: CB->args(),
892 P: [](auto &&Arg) { return Arg->getType()->isAggregateType(); }))
893 Calls.emplace_back(Args&: CB, Args: nullptr);
894 }
895 }
896
897 if (Calls.empty())
898 return false;
899
900 IRBuilder<> B(*F->getParent());
901
902 unsigned MutatedCallIdx = 0;
903 for (auto &&[CB, NewFnTy] : Calls) {
904 SmallVector<std::pair<int, Type *>> ChangedTypes;
905 SmallVector<Type *> NewArgTypes;
906
907 Type *RetTy = CB->getType();
908 if (RetTy->isAggregateType()) {
909 ChangedTypes.emplace_back(Args: -1, Args&: RetTy);
910 RetTy = B.getInt32Ty();
911 }
912
913 for (auto &&Arg : CB->args()) {
914 if (Arg->getType()->isAggregateType()) {
915 NewArgTypes.push_back(Elt: B.getInt32Ty());
916 ChangedTypes.emplace_back(Args: Arg.getOperandNo(), Args: Arg->getType());
917 } else {
918 NewArgTypes.push_back(Elt: Arg->getType());
919 }
920 }
921 NewFnTy = FunctionType::get(Result: RetTy, Params: NewArgTypes,
922 isVarArg: CB->getFunctionType()->isVarArg());
923
924 // Keyed via instruction metadata, not a name.
925 std::string Key =
926 ("spv.mutated_callsite." + F->getName() + "." + Twine(MutatedCallIdx++))
927 .str();
928 CB->setMetadata(
929 Kind: "spv.mutated_callsite",
930 Node: MDNode::get(Context&: F->getContext(), MDs: MDString::get(Context&: F->getContext(), Str: Key)));
931
932 std::string Constraints;
933 if (auto *ASM = dyn_cast<InlineAsm>(Val: CB->getCalledOperand())) {
934 Constraints = ASM->getConstraintString();
935
936 CB->setCalledOperand(InlineAsm::get(
937 Ty: NewFnTy, AsmString: ASM->getAsmString(),
938 Constraints: fixMultiOutputConstraintString(Constraints), hasSideEffects: ASM->hasSideEffects(),
939 isAlignStack: ASM->isAlignStack(), asmDialect: ASM->getDialect(), canThrow: ASM->canThrow()));
940 }
941
942 addFunctionTypeMutation(
943 NMD: F->getParent()->getOrInsertNamedMetadata(Name: "spv.mutated_callsites"),
944 ChangedTys: std::move(ChangedTypes), Name: Key, AsmConstraints: Constraints);
945 }
946
947 for (auto &&[CB, NewFTy] : Calls) {
948 if (NewFTy->getReturnType() != CB->getType())
949 TM.getSubtarget<SPIRVSubtarget>(F: *F).getSPIRVGlobalRegistry()->addMutated(
950 Val: CB, Ty: CB->getType());
951 CB->mutateFunctionType(FTy: NewFTy);
952 removeIncompatibleAttrs(V: CB);
953 }
954
955 return true;
956}
957
958bool SPIRVPrepareFunctionsImpl::runOnModule(Module &M) {
959 // Resolve the SPIR-V environment from module content before any
960 // function-level processing. This must happen before legalization so that
961 // isShader()/isKernel() return correct values.
962 const_cast<SPIRVTargetMachine &>(TM)
963 .getMutableSubtargetImpl()
964 ->resolveEnvFromModule(M);
965
966 bool Changed = false;
967 if (M.getFunctionDefs().empty()) {
968 // If there are no function definitions, insert a service
969 // function so that the global/constant tracking intrinsics
970 // will be created. Without these intrinsics the generated SPIR-V
971 // will be empty. The service function itself is not emitted.
972 Function *SF = getOrCreateBackendServiceFunction(M);
973 BasicBlock *BB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: SF);
974 IRBuilder<> IRB(BB);
975 IRB.CreateRetVoid();
976 Changed = true;
977 }
978
979 Changed |= terminateBlocksAfterTrap(M, IID: Intrinsic::trap);
980 Changed |= terminateBlocksAfterTrap(M, IID: Intrinsic::ubsantrap);
981
982 for (GlobalVariable &GV : M.globals()) {
983 // Strip + tag available_externally globals so AuxData can re-emit the
984 // original linkage as NonSemantic.AuxData::Linkage.
985 if (GV.hasAvailableExternallyLinkage() && !GV.isDeclaration()) {
986 GV.addAttribute(SPIRV_WAS_AVAILABLE_EXTERNALLY_ATTR);
987 GV.setLinkage(GlobalValue::ExternalLinkage);
988 Changed = true;
989 }
990 }
991
992 std::vector<Function *> FuncsWorklist;
993 for (Function &F : M) {
994 // MachineFunctionPass skips available_externally; strip + tag so AuxData
995 // can re-emit the original linkage as NonSemantic.AuxData::Linkage.
996 if (F.hasAvailableExternallyLinkage() && !F.isDeclaration()) {
997 F.addFnAttr(SPIRV_WAS_AVAILABLE_EXTERNALLY_ATTR);
998 F.setLinkage(GlobalValue::ExternalLinkage);
999 Changed = true;
1000 }
1001 Changed |= substituteAbortKHRCalls(F: &F);
1002 Changed |= substituteIntrinsicCalls(F: &F);
1003 Changed |= sortBlocks(F);
1004 Changed |= removeAggregateTypesFromCalls(F: &F);
1005 FuncsWorklist.push_back(x: &F);
1006 }
1007
1008 for (auto *F : FuncsWorklist) {
1009 Function *NewF = removeAggregateTypesFromSignature(F);
1010
1011 if (NewF != F) {
1012 F->eraseFromParent();
1013 Changed = true;
1014 }
1015 }
1016 return Changed;
1017}
1018
1019PreservedAnalyses SPIRVPrepareFunctionsPass::run(Module &M,
1020 ModuleAnalysisManager &AM) {
1021 FunctionAnalysisManager &FAM =
1022 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
1023 auto GetTTI = [&FAM](Function &F) -> const TargetTransformInfo & {
1024 return FAM.getResult<TargetIRAnalysis>(IR&: F);
1025 };
1026 return SPIRVPrepareFunctionsImpl(TM, GetTTI).runOnModule(M)
1027 ? PreservedAnalyses::none()
1028 : PreservedAnalyses::all();
1029}
1030
1031ModulePass *
1032llvm::createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM) {
1033 return new SPIRVPrepareFunctionsLegacy(TM);
1034}
1035