1//===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass emits SPIRV intrinsics keeping essential high-level information for
10// the translation of LLVM IR to SPIR-V.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRV.h"
15#include "SPIRVBuiltins.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVTargetMachine.h"
18#include "SPIRVUtils.h"
19#include "llvm/ADT/DenseSet.h"
20#include "llvm/ADT/StringSet.h"
21#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/InstIterator.h"
23#include "llvm/IR/InstVisitor.h"
24#include "llvm/IR/IntrinsicsSPIRV.h"
25#include "llvm/IR/PatternMatch.h"
26#include "llvm/IR/TypedPointerType.h"
27#include "llvm/Support/CommandLine.h"
28#include "llvm/Transforms/Utils/Local.h"
29
30#include <cassert>
31#include <queue>
32#include <unordered_set>
33
34// This pass performs the following transformation on LLVM IR level required
35// for the following translation to SPIR-V:
36// - replaces direct usages of aggregate constants with target-specific
37// intrinsics;
38// - replaces aggregates-related instructions (extract/insert, ld/st, etc)
39// with a target-specific intrinsics;
40// - emits intrinsics for the global variable initializers since IRTranslator
41// doesn't handle them and it's not very convenient to translate them
42// ourselves;
43// - emits intrinsics to keep track of the string names assigned to the values;
44// - emits intrinsics to keep track of constants (this is necessary to have an
45// LLVM IR constant after the IRTranslation is completed) for their further
46// deduplication;
47// - emits intrinsics to keep track of original LLVM types of the values
48// to be able to emit proper SPIR-V types eventually.
49//
50// TODO: consider removing spv.track.constant in favor of spv.assign.type.
51
52using namespace llvm;
53
54static cl::opt<bool>
55 SpirvEmitOpNames("spirv-emit-op-names",
56 cl::desc("Emit OpName for all instructions"),
57 cl::init(Val: false));
58
59namespace llvm::SPIRV {
60#define GET_BuiltinGroup_DECL
61#include "SPIRVGenTables.inc"
62} // namespace llvm::SPIRV
63
64namespace {
65// This class keeps track of which functions reference which global variables.
66class GlobalVariableUsers {
67 template <typename T1, typename T2>
68 using OneToManyMapTy = DenseMap<T1, SmallPtrSet<T2, 4>>;
69
70 OneToManyMapTy<const GlobalVariable *, const Function *> GlobalIsUsedByFun;
71
72 void collectGlobalUsers(
73 const GlobalVariable *GV,
74 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
75 &GlobalIsUsedByGlobal) {
76 SmallVector<const Value *> Stack = {GV->user_begin(), GV->user_end()};
77 while (!Stack.empty()) {
78 const Value *V = Stack.pop_back_val();
79
80 if (const Instruction *I = dyn_cast<Instruction>(Val: V)) {
81 GlobalIsUsedByFun[GV].insert(Ptr: I->getFunction());
82 continue;
83 }
84
85 if (const GlobalVariable *UserGV = dyn_cast<GlobalVariable>(Val: V)) {
86 GlobalIsUsedByGlobal[GV].insert(Ptr: UserGV);
87 continue;
88 }
89
90 if (const Constant *C = dyn_cast<Constant>(Val: V))
91 Stack.append(in_start: C->user_begin(), in_end: C->user_end());
92 }
93 }
94
95 bool propagateGlobalToGlobalUsers(
96 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
97 &GlobalIsUsedByGlobal) {
98 SmallVector<const GlobalVariable *> OldUsersGlobals;
99 bool Changed = false;
100 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
101 OldUsersGlobals.assign(in_start: UserGlobals.begin(), in_end: UserGlobals.end());
102 for (const GlobalVariable *UserGV : OldUsersGlobals) {
103 auto It = GlobalIsUsedByGlobal.find(Val: UserGV);
104 if (It == GlobalIsUsedByGlobal.end())
105 continue;
106 Changed |= set_union(S1&: UserGlobals, S2: It->second);
107 }
108 }
109 return Changed;
110 }
111
112 void propagateGlobalToFunctionReferences(
113 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
114 &GlobalIsUsedByGlobal) {
115 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
116 auto &UserFunctions = GlobalIsUsedByFun[GV];
117 for (const GlobalVariable *UserGV : UserGlobals) {
118 auto It = GlobalIsUsedByFun.find(Val: UserGV);
119 if (It == GlobalIsUsedByFun.end())
120 continue;
121 set_union(S1&: UserFunctions, S2: It->second);
122 }
123 }
124 }
125
126public:
127 void init(Module &M) {
128 // Collect which global variables are referenced by which global variables
129 // and which functions reference each global variables.
130 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
131 GlobalIsUsedByGlobal;
132 GlobalIsUsedByFun.clear();
133 for (GlobalVariable &GV : M.globals())
134 collectGlobalUsers(GV: &GV, GlobalIsUsedByGlobal);
135
136 // Compute indirect references by iterating until a fixed point is reached.
137 while (propagateGlobalToGlobalUsers(GlobalIsUsedByGlobal))
138 (void)0;
139
140 propagateGlobalToFunctionReferences(GlobalIsUsedByGlobal);
141 }
142
143 using FunctionSetType = typename decltype(GlobalIsUsedByFun)::mapped_type;
144 const FunctionSetType &
145 getTransitiveUserFunctions(const GlobalVariable &GV) const {
146 auto It = GlobalIsUsedByFun.find(Val: &GV);
147 if (It != GlobalIsUsedByFun.end())
148 return It->second;
149
150 static const FunctionSetType Empty{};
151 return Empty;
152 }
153};
154
155static bool isaGEP(const Value *V) {
156 return isa<StructuredGEPInst>(Val: V) || isa<GetElementPtrInst>(Val: V);
157}
158
159class SPIRVEmitIntrinsics
160 : public ModulePass,
161 public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
162 SPIRVTargetMachine *TM = nullptr;
163 SPIRVGlobalRegistry *GR = nullptr;
164 Function *CurrF = nullptr;
165 bool TrackConstants = true;
166 bool HaveFunPtrs = false;
167 DenseMap<Instruction *, Constant *> AggrConsts;
168 DenseMap<Instruction *, Type *> AggrConstTypes;
169 DenseSet<Instruction *> AggrStores;
170 GlobalVariableUsers GVUsers;
171 std::unordered_set<Value *> Named;
172
173 // map of function declarations to <pointer arg index => element type>
174 DenseMap<Function *, SmallVector<std::pair<unsigned, Type *>>> FDeclPtrTys;
175
176 // a register of Instructions that don't have a complete type definition
177 bool CanTodoType = true;
178 unsigned TodoTypeSz = 0;
179 DenseMap<Value *, bool> TodoType;
180 void insertTodoType(Value *Op) {
181 // TODO: add isa<CallInst>(Op) to no-insert
182 if (CanTodoType && !isaGEP(V: Op)) {
183 auto It = TodoType.try_emplace(Key: Op, Args: true);
184 if (It.second)
185 ++TodoTypeSz;
186 }
187 }
188 void eraseTodoType(Value *Op) {
189 auto It = TodoType.find(Val: Op);
190 if (It != TodoType.end() && It->second) {
191 It->second = false;
192 --TodoTypeSz;
193 }
194 }
195 bool isTodoType(Value *Op) {
196 if (isaGEP(V: Op))
197 return false;
198 auto It = TodoType.find(Val: Op);
199 return It != TodoType.end() && It->second;
200 }
201 // a register of Instructions that were visited by deduceOperandElementType()
202 // to validate operand types with an instruction
203 std::unordered_set<Instruction *> TypeValidated;
204
205 // well known result types of builtins
206 enum WellKnownTypes { Event };
207
208 // deduce element type of untyped pointers
209 Type *deduceElementType(Value *I, bool UnknownElemTypeI8);
210 Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8);
211 Type *deduceElementTypeHelper(Value *I, std::unordered_set<Value *> &Visited,
212 bool UnknownElemTypeI8,
213 bool IgnoreKnownType = false);
214 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
215 bool UnknownElemTypeI8);
216 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
217 std::unordered_set<Value *> &Visited,
218 bool UnknownElemTypeI8);
219 Type *deduceElementTypeByUsersDeep(Value *Op,
220 std::unordered_set<Value *> &Visited,
221 bool UnknownElemTypeI8);
222 void maybeAssignPtrType(Type *&Ty, Value *I, Type *RefTy,
223 bool UnknownElemTypeI8);
224
225 // deduce nested types of composites
226 Type *deduceNestedTypeHelper(User *U, bool UnknownElemTypeI8);
227 Type *deduceNestedTypeHelper(User *U, Type *Ty,
228 std::unordered_set<Value *> &Visited,
229 bool UnknownElemTypeI8);
230
231 // deduce Types of operands of the Instruction if possible
232 void deduceOperandElementType(Instruction *I,
233 SmallPtrSet<Instruction *, 4> *IncompleteRets,
234 const SmallPtrSet<Value *, 4> *AskOps = nullptr,
235 bool IsPostprocessing = false);
236
237 void preprocessCompositeConstants(IRBuilder<> &B);
238 void preprocessUndefs(IRBuilder<> &B);
239
240 Type *reconstructType(Value *Op, bool UnknownElemTypeI8,
241 bool IsPostprocessing);
242
243 void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
244 void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
245 bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
246 bool UnknownElemTypeI8);
247 void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B);
248 void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V,
249 IRBuilder<> &B);
250 void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer,
251 Type *ExpectedElementType,
252 unsigned OperandToReplace,
253 IRBuilder<> &B);
254 void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
255 bool shouldTryToAddMemAliasingDecoration(Instruction *Inst);
256 void insertSpirvDecorations(Instruction *I, IRBuilder<> &B);
257 void insertConstantsForFPFastMathDefault(Module &M);
258 Value *buildSpvUndefComposite(Type *AggrTy, IRBuilder<> &B);
259 void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
260 void processParamTypes(Function *F, IRBuilder<> &B);
261 void processParamTypesByFunHeader(Function *F, IRBuilder<> &B);
262 Type *deduceFunParamElementType(Function *F, unsigned OpIdx);
263 Type *deduceFunParamElementType(Function *F, unsigned OpIdx,
264 std::unordered_set<Function *> &FVisited);
265
266 bool deduceOperandElementTypeCalledFunction(
267 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
268 Type *&KnownElemTy, bool &Incomplete);
269 void deduceOperandElementTypeFunctionPointer(
270 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
271 Type *&KnownElemTy, bool IsPostprocessing);
272 bool deduceOperandElementTypeFunctionRet(
273 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
274 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing,
275 Type *&KnownElemTy, Value *Op, Function *F);
276
277 CallInst *buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy);
278 void replaceUsesOfWithSpvPtrcast(Value *Op, Type *ElemTy, Instruction *I,
279 DenseMap<Function *, CallInst *> Ptrcasts);
280 void propagateElemType(Value *Op, Type *ElemTy,
281 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
282 void
283 propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
284 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
285 void propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
286 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
287 std::unordered_set<Value *> &Visited,
288 DenseMap<Function *, CallInst *> Ptrcasts);
289
290 void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
291 void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src,
292 Instruction *Dest, bool DeleteOld = true);
293
294 void applyDemangledPtrArgTypes(IRBuilder<> &B);
295
296 GetElementPtrInst *simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP);
297
298 bool runOnFunction(Function &F);
299 bool postprocessTypes(Module &M);
300 bool processFunctionPointers(Module &M);
301 void parseFunDeclarations(Module &M);
302 void useRoundingMode(ConstrainedFPIntrinsic *FPI, IRBuilder<> &B);
303 bool processMaskedMemIntrinsic(IntrinsicInst &I);
304 bool convertMaskedMemIntrinsics(Module &M);
305
306 void emitUnstructuredLoopControls(Function &F, IRBuilder<> &B);
307
308 // Tries to walk the type accessed by the given GEP instruction.
309 // For each nested type access, one of the 2 callbacks is called:
310 // - OnLiteralIndexing when the index is a known constant value.
311 // Parameters:
312 // PointedType: the pointed type resulting of this indexing.
313 // If the parent type is an array, this is the index in the array.
314 // If the parent type is a struct, this is the field index.
315 // Index: index of the element in the parent type.
316 // - OnDynamnicIndexing when the index is a non-constant value.
317 // This callback is only called when indexing into an array.
318 // Parameters:
319 // ElementType: the type of the elements stored in the parent array.
320 // Offset: the Value* containing the byte offset into the array.
321 // Return true if an error occurred during the walk, false otherwise.
322 bool walkLogicalAccessChain(
323 GetElementPtrInst &GEP,
324 const std::function<void(Type *PointedType, uint64_t Index)>
325 &OnLiteralIndexing,
326 const std::function<void(Type *ElementType, Value *Offset)>
327 &OnDynamicIndexing);
328
329 // Returns the type accessed using the given GEP instruction by relying
330 // on the GEP type.
331 // FIXME: GEP types are not supposed to be used to retrieve the pointed
332 // type. This must be fixed.
333 Type *getGEPType(GetElementPtrInst *GEP);
334
335 // Returns the type accessed using the given GEP instruction by walking
336 // the source type using the GEP indices.
337 // FIXME: without help from the frontend, this method cannot reliably retrieve
338 // the stored type, nor can robustly determine the depth of the type
339 // we are accessing.
340 Type *getGEPTypeLogical(GetElementPtrInst *GEP);
341
342 Instruction *buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP);
343
344public:
345 static char ID;
346 SPIRVEmitIntrinsics(SPIRVTargetMachine *TM = nullptr)
347 : ModulePass(ID), TM(TM) {}
348 Instruction *visitInstruction(Instruction &I) { return &I; }
349 Instruction *visitSwitchInst(SwitchInst &I);
350 Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
351 Instruction *visitIntrinsicInst(IntrinsicInst &I);
352 Instruction *visitBitCastInst(BitCastInst &I);
353 Instruction *visitInsertElementInst(InsertElementInst &I);
354 Instruction *visitExtractElementInst(ExtractElementInst &I);
355 Instruction *visitInsertValueInst(InsertValueInst &I);
356 Instruction *visitExtractValueInst(ExtractValueInst &I);
357 Instruction *visitLoadInst(LoadInst &I);
358 Instruction *visitStoreInst(StoreInst &I);
359 Instruction *visitAllocaInst(AllocaInst &I);
360 Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
361 Instruction *visitUnreachableInst(UnreachableInst &I);
362 Instruction *visitCallInst(CallInst &I);
363
364 StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
365
366 bool runOnModule(Module &M) override;
367
368 void getAnalysisUsage(AnalysisUsage &AU) const override {
369 ModulePass::getAnalysisUsage(AU);
370 }
371};
372
373bool isConvergenceIntrinsic(const Instruction *I) {
374 const auto *II = dyn_cast<IntrinsicInst>(Val: I);
375 if (!II)
376 return false;
377
378 return II->getIntrinsicID() == Intrinsic::experimental_convergence_entry ||
379 II->getIntrinsicID() == Intrinsic::experimental_convergence_loop ||
380 II->getIntrinsicID() == Intrinsic::experimental_convergence_anchor;
381}
382
383bool expectIgnoredInIRTranslation(const Instruction *I) {
384 const auto *II = dyn_cast<IntrinsicInst>(Val: I);
385 if (!II)
386 return false;
387 switch (II->getIntrinsicID()) {
388 case Intrinsic::invariant_start:
389 case Intrinsic::spv_resource_handlefrombinding:
390 case Intrinsic::spv_resource_getpointer:
391 return true;
392 default:
393 return false;
394 }
395}
396
397// Returns the source pointer from `I` ignoring intermediate ptrcast.
398Value *getPointerRoot(Value *I) {
399 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
400 if (II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
401 Value *V = II->getArgOperand(i: 0);
402 return getPointerRoot(I: V);
403 }
404 }
405 return I;
406}
407
408} // namespace
409
410char SPIRVEmitIntrinsics::ID = 0;
411
412INITIALIZE_PASS(SPIRVEmitIntrinsics, "emit-intrinsics", "SPIRV emit intrinsics",
413 false, false)
414
415static inline bool isAssignTypeInstr(const Instruction *I) {
416 return isa<IntrinsicInst>(Val: I) &&
417 cast<IntrinsicInst>(Val: I)->getIntrinsicID() == Intrinsic::spv_assign_type;
418}
419
420static bool isMemInstrToReplace(Instruction *I) {
421 return isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I) || isa<InsertValueInst>(Val: I) ||
422 isa<ExtractValueInst>(Val: I) || isa<AtomicCmpXchgInst>(Val: I);
423}
424
425static bool isAggrConstForceInt32(const Value *V) {
426 bool IsAggrZero =
427 isa<ConstantAggregateZero>(Val: V) && !V->getType()->isVectorTy();
428 bool IsUndefAggregate = isa<UndefValue>(Val: V) && V->getType()->isAggregateType();
429 return isa<ConstantArray>(Val: V) || isa<ConstantStruct>(Val: V) ||
430 isa<ConstantDataArray>(Val: V) || IsAggrZero || IsUndefAggregate;
431}
432
433static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
434 if (isa<PHINode>(Val: I))
435 B.SetInsertPoint(I->getParent()->getFirstNonPHIOrDbgOrAlloca());
436 else
437 B.SetInsertPoint(I);
438}
439
440static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I) {
441 B.SetCurrentDebugLocation(I->getDebugLoc());
442 if (I->getType()->isVoidTy())
443 B.SetInsertPoint(I->getNextNode());
444 else
445 B.SetInsertPoint(*I->getInsertionPointAfterDef());
446}
447
448static bool requireAssignType(Instruction *I) {
449 if (const auto *Intr = dyn_cast<IntrinsicInst>(Val: I)) {
450 switch (Intr->getIntrinsicID()) {
451 case Intrinsic::invariant_start:
452 case Intrinsic::invariant_end:
453 return false;
454 }
455 }
456 return true;
457}
458
459static inline void reportFatalOnTokenType(const Instruction *I) {
460 if (I->getType()->isTokenTy())
461 report_fatal_error(reason: "A token is encountered but SPIR-V without extensions "
462 "does not support token type",
463 gen_crash_diag: false);
464}
465
466static void emitAssignName(Instruction *I, IRBuilder<> &B) {
467 if (!I->hasName() || I->getType()->isAggregateType() ||
468 expectIgnoredInIRTranslation(I))
469 return;
470
471 // We want to be conservative when adding the names because they can interfere
472 // with later optimizations.
473 bool KeepName = SpirvEmitOpNames;
474 if (!KeepName) {
475 if (isa<AllocaInst>(Val: I)) {
476 KeepName = true;
477 } else if (auto *CI = dyn_cast<CallBase>(Val: I)) {
478 Function *F = CI->getCalledFunction();
479 if (F && F->getName().starts_with(Prefix: "llvm.spv.alloca"))
480 KeepName = true;
481 }
482 }
483
484 if (!KeepName)
485 return;
486
487 reportFatalOnTokenType(I);
488 setInsertPointAfterDef(B, I);
489 LLVMContext &Ctx = I->getContext();
490 std::vector<Value *> Args = {
491 I, MetadataAsValue::get(
492 Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: I->getName())))};
493 B.CreateIntrinsic(ID: Intrinsic::spv_assign_name, Types: {I->getType()}, Args);
494}
495
496void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest,
497 bool DeleteOld) {
498 GR->replaceAllUsesWith(Old: Src, New: Dest, DeleteOld);
499 // Update uncomplete type records if any
500 if (isTodoType(Op: Src)) {
501 if (DeleteOld)
502 eraseTodoType(Op: Src);
503 insertTodoType(Op: Dest);
504 }
505}
506
507void SPIRVEmitIntrinsics::replaceAllUsesWithAndErase(IRBuilder<> &B,
508 Instruction *Src,
509 Instruction *Dest,
510 bool DeleteOld) {
511 replaceAllUsesWith(Src, Dest, DeleteOld);
512 std::string Name = Src->hasName() ? Src->getName().str() : "";
513 Src->eraseFromParent();
514 if (!Name.empty()) {
515 Dest->setName(Name);
516 if (Named.insert(x: Dest).second)
517 emitAssignName(I: Dest, B);
518 }
519}
520
521static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
522 return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
523 isPointerTy(T: SI->getValueOperand()->getType()) &&
524 isa<Argument>(Val: SI->getValueOperand());
525}
526
527// Maybe restore original function return type.
528static inline Type *restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I,
529 Type *Ty) {
530 CallInst *CI = dyn_cast<CallInst>(Val: I);
531 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
532 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
533 return Ty;
534 if (Type *OriginalTy = GR->findMutated(Val: CI->getCalledFunction()))
535 return OriginalTy;
536 return Ty;
537}
538
539// Reconstruct type with nested element types according to deduced type info.
540// Return nullptr if no detailed type info is available.
541Type *SPIRVEmitIntrinsics::reconstructType(Value *Op, bool UnknownElemTypeI8,
542 bool IsPostprocessing) {
543 Type *Ty = Op->getType();
544 if (auto *OpI = dyn_cast<Instruction>(Val: Op))
545 Ty = restoreMutatedType(GR, I: OpI, Ty);
546 if (!isUntypedPointerTy(T: Ty))
547 return Ty;
548 // try to find the pointee type
549 if (Type *NestedTy = GR->findDeducedElementType(Val: Op))
550 return getTypedPointerWrapper(ElemTy: NestedTy, AS: getPointerAddressSpace(T: Ty));
551 // not a pointer according to the type info (e.g., Event object)
552 CallInst *CI = GR->findAssignPtrTypeInstr(Val: Op);
553 if (CI) {
554 MetadataAsValue *MD = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
555 return cast<ConstantAsMetadata>(Val: MD->getMetadata())->getType();
556 }
557 if (UnknownElemTypeI8) {
558 if (!IsPostprocessing)
559 insertTodoType(Op);
560 return getTypedPointerWrapper(ElemTy: IntegerType::getInt8Ty(C&: Op->getContext()),
561 AS: getPointerAddressSpace(T: Ty));
562 }
563 return nullptr;
564}
565
566CallInst *SPIRVEmitIntrinsics::buildSpvPtrcast(Function *F, Value *Op,
567 Type *ElemTy) {
568 IRBuilder<> B(Op->getContext());
569 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
570 // spv_ptrcast's argument Op denotes an instruction that generates
571 // a value, and we may use getInsertionPointAfterDef()
572 setInsertPointAfterDef(B, I: OpI);
573 } else if (auto *OpA = dyn_cast<Argument>(Val: Op)) {
574 B.SetInsertPointPastAllocas(OpA->getParent());
575 B.SetCurrentDebugLocation(DebugLoc());
576 } else {
577 B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
578 }
579 Type *OpTy = Op->getType();
580 SmallVector<Type *, 2> Types = {OpTy, OpTy};
581 SmallVector<Value *, 2> Args = {Op, buildMD(Arg: getNormalizedPoisonValue(Ty: ElemTy)),
582 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
583 CallInst *PtrCasted =
584 B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
585 GR->buildAssignPtr(B, ElemTy, Arg: PtrCasted);
586 return PtrCasted;
587}
588
589void SPIRVEmitIntrinsics::replaceUsesOfWithSpvPtrcast(
590 Value *Op, Type *ElemTy, Instruction *I,
591 DenseMap<Function *, CallInst *> Ptrcasts) {
592 Function *F = I->getParent()->getParent();
593 CallInst *PtrCastedI = nullptr;
594 auto It = Ptrcasts.find(Val: F);
595 if (It == Ptrcasts.end()) {
596 PtrCastedI = buildSpvPtrcast(F, Op, ElemTy);
597 Ptrcasts[F] = PtrCastedI;
598 } else {
599 PtrCastedI = It->second;
600 }
601 I->replaceUsesOfWith(From: Op, To: PtrCastedI);
602}
603
604void SPIRVEmitIntrinsics::propagateElemType(
605 Value *Op, Type *ElemTy,
606 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
607 DenseMap<Function *, CallInst *> Ptrcasts;
608 SmallVector<User *> Users(Op->users());
609 for (auto *U : Users) {
610 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
611 continue;
612 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
613 continue;
614 Instruction *UI = dyn_cast<Instruction>(Val: U);
615 // If the instruction was validated already, we need to keep it valid by
616 // keeping current Op type.
617 if (isaGEP(V: UI) || TypeValidated.find(x: UI) != TypeValidated.end())
618 replaceUsesOfWithSpvPtrcast(Op, ElemTy, I: UI, Ptrcasts);
619 }
620}
621
622void SPIRVEmitIntrinsics::propagateElemTypeRec(
623 Value *Op, Type *PtrElemTy, Type *CastElemTy,
624 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
625 std::unordered_set<Value *> Visited;
626 DenseMap<Function *, CallInst *> Ptrcasts;
627 propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited,
628 Ptrcasts: std::move(Ptrcasts));
629}
630
631void SPIRVEmitIntrinsics::propagateElemTypeRec(
632 Value *Op, Type *PtrElemTy, Type *CastElemTy,
633 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
634 std::unordered_set<Value *> &Visited,
635 DenseMap<Function *, CallInst *> Ptrcasts) {
636 if (!Visited.insert(x: Op).second)
637 return;
638 SmallVector<User *> Users(Op->users());
639 for (auto *U : Users) {
640 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
641 continue;
642 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
643 continue;
644 Instruction *UI = dyn_cast<Instruction>(Val: U);
645 // If the instruction was validated already, we need to keep it valid by
646 // keeping current Op type.
647 if (isaGEP(V: UI) || TypeValidated.find(x: UI) != TypeValidated.end())
648 replaceUsesOfWithSpvPtrcast(Op, ElemTy: CastElemTy, I: UI, Ptrcasts);
649 }
650}
651
652// Set element pointer type to the given value of ValueTy and tries to
653// specify this type further (recursively) by Operand value, if needed.
654
655Type *
656SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
657 bool UnknownElemTypeI8) {
658 std::unordered_set<Value *> Visited;
659 return deduceElementTypeByValueDeep(ValueTy, Operand, Visited,
660 UnknownElemTypeI8);
661}
662
663Type *SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(
664 Type *ValueTy, Value *Operand, std::unordered_set<Value *> &Visited,
665 bool UnknownElemTypeI8) {
666 Type *Ty = ValueTy;
667 if (Operand) {
668 if (auto *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
669 if (Type *NestedTy =
670 deduceElementTypeHelper(I: Operand, Visited, UnknownElemTypeI8))
671 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
672 } else {
673 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Operand), Ty, Visited,
674 UnknownElemTypeI8);
675 }
676 }
677 return Ty;
678}
679
680// Traverse User instructions to deduce an element pointer type of the operand.
681Type *SPIRVEmitIntrinsics::deduceElementTypeByUsersDeep(
682 Value *Op, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8) {
683 if (!Op || !isPointerTy(T: Op->getType()) || isa<ConstantPointerNull>(Val: Op) ||
684 isa<UndefValue>(Val: Op))
685 return nullptr;
686
687 if (auto ElemTy = getPointeeType(Ty: Op->getType()))
688 return ElemTy;
689
690 // maybe we already know operand's element type
691 if (Type *KnownTy = GR->findDeducedElementType(Val: Op))
692 return KnownTy;
693
694 for (User *OpU : Op->users()) {
695 if (Instruction *Inst = dyn_cast<Instruction>(Val: OpU)) {
696 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8))
697 return Ty;
698 }
699 }
700 return nullptr;
701}
702
703// Implements what we know in advance about intrinsics and builtin calls
704// TODO: consider feasibility of this particular case to be generalized by
705// encoding knowledge about intrinsics and builtin calls by corresponding
706// specification rules
707static Type *getPointeeTypeByCallInst(StringRef DemangledName,
708 Function *CalledF, unsigned OpIdx) {
709 if ((DemangledName.starts_with(Prefix: "__spirv_ocl_printf(") ||
710 DemangledName.starts_with(Prefix: "printf(")) &&
711 OpIdx == 0)
712 return IntegerType::getInt8Ty(C&: CalledF->getContext());
713 return nullptr;
714}
715
716// Deduce and return a successfully deduced Type of the Instruction,
717// or nullptr otherwise.
718Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(Value *I,
719 bool UnknownElemTypeI8) {
720 std::unordered_set<Value *> Visited;
721 return deduceElementTypeHelper(I, Visited, UnknownElemTypeI8);
722}
723
724void SPIRVEmitIntrinsics::maybeAssignPtrType(Type *&Ty, Value *Op, Type *RefTy,
725 bool UnknownElemTypeI8) {
726 if (isUntypedPointerTy(T: RefTy)) {
727 if (!UnknownElemTypeI8)
728 return;
729 insertTodoType(Op);
730 }
731 Ty = RefTy;
732}
733
734bool SPIRVEmitIntrinsics::walkLogicalAccessChain(
735 GetElementPtrInst &GEP,
736 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
737 const std::function<void(Type *, Value *)> &OnDynamicIndexing) {
738 // We only rewrite i8* GEP. Other should be left as-is.
739 // Valid i8* GEP must always have a single index.
740 assert(GEP.getSourceElementType() ==
741 IntegerType::getInt8Ty(CurrF->getContext()));
742 assert(GEP.getNumIndices() == 1);
743
744 auto &DL = CurrF->getDataLayout();
745 Value *Src = getPointerRoot(I: GEP.getPointerOperand());
746 Type *CurType = deduceElementType(I: Src, UnknownElemTypeI8: true);
747
748 Value *Operand = *GEP.idx_begin();
749 ConstantInt *CI = dyn_cast<ConstantInt>(Val: Operand);
750 if (!CI) {
751 ArrayType *AT = dyn_cast<ArrayType>(Val: CurType);
752 // Operand is not constant. Either we have an array and accept it, or we
753 // give up.
754 if (AT)
755 OnDynamicIndexing(AT->getElementType(), Operand);
756 return AT == nullptr;
757 }
758
759 assert(CI);
760 uint64_t Offset = CI->getZExtValue();
761
762 do {
763 if (ArrayType *AT = dyn_cast<ArrayType>(Val: CurType)) {
764 uint32_t EltTypeSize = DL.getTypeSizeInBits(Ty: AT->getElementType()) / 8;
765 assert(Offset < AT->getNumElements() * EltTypeSize);
766 uint64_t Index = Offset / EltTypeSize;
767 Offset = Offset - (Index * EltTypeSize);
768 CurType = AT->getElementType();
769 OnLiteralIndexing(CurType, Index);
770 } else if (StructType *ST = dyn_cast<StructType>(Val: CurType)) {
771 uint32_t StructSize = DL.getTypeSizeInBits(Ty: ST) / 8;
772 assert(Offset < StructSize);
773 (void)StructSize;
774 const auto &STL = DL.getStructLayout(Ty: ST);
775 unsigned Element = STL->getElementContainingOffset(FixedOffset: Offset);
776 Offset -= STL->getElementOffset(Idx: Element);
777 CurType = ST->getElementType(N: Element);
778 OnLiteralIndexing(CurType, Element);
779 } else if (auto *VT = dyn_cast<FixedVectorType>(Val: CurType)) {
780 Type *EltTy = VT->getElementType();
781 TypeSize EltSizeBits = DL.getTypeSizeInBits(Ty: EltTy);
782 assert(EltSizeBits % 8 == 0 &&
783 "Element type size in bits must be a multiple of 8.");
784 uint32_t EltTypeSize = EltSizeBits / 8;
785 assert(Offset < VT->getNumElements() * EltTypeSize);
786 uint64_t Index = Offset / EltTypeSize;
787 Offset -= Index * EltTypeSize;
788 CurType = EltTy;
789 OnLiteralIndexing(CurType, Index);
790
791 } else {
792 // Unknown composite kind; give up.
793 return true;
794 }
795 } while (Offset > 0);
796
797 return false;
798}
799
800Instruction *
801SPIRVEmitIntrinsics::buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP) {
802 auto &DL = CurrF->getDataLayout();
803 IRBuilder<> B(GEP.getParent());
804 B.SetInsertPoint(&GEP);
805
806 std::vector<Value *> Indices;
807 Indices.push_back(x: ConstantInt::get(
808 Ty: IntegerType::getInt32Ty(C&: CurrF->getContext()), V: 0, /* Signed= */ IsSigned: false));
809 walkLogicalAccessChain(
810 GEP,
811 OnLiteralIndexing: [&Indices, &B](Type *EltType, uint64_t Index) {
812 Indices.push_back(
813 x: ConstantInt::get(Ty: B.getInt64Ty(), V: Index, /* Signed= */ IsSigned: false));
814 },
815 OnDynamicIndexing: [&Indices, &B, &DL](Type *EltType, Value *Offset) {
816 uint32_t EltTypeSize = DL.getTypeSizeInBits(Ty: EltType) / 8;
817 Value *Index = B.CreateUDiv(
818 LHS: Offset, RHS: ConstantInt::get(Ty: Offset->getType(), V: EltTypeSize,
819 /* Signed= */ IsSigned: false));
820 Indices.push_back(x: Index);
821 });
822
823 SmallVector<Type *, 2> Types = {GEP.getType(), GEP.getOperand(i_nocapture: 0)->getType()};
824 SmallVector<Value *, 4> Args;
825 Args.push_back(Elt: B.getInt1(V: GEP.isInBounds()));
826 Args.push_back(Elt: GEP.getOperand(i_nocapture: 0));
827 llvm::append_range(C&: Args, R&: Indices);
828 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types: {Types}, Args: {Args});
829 replaceAllUsesWithAndErase(B, Src: &GEP, Dest: NewI);
830 return NewI;
831}
832
833Type *SPIRVEmitIntrinsics::getGEPTypeLogical(GetElementPtrInst *GEP) {
834
835 Type *CurType = GEP->getResultElementType();
836
837 bool Interrupted = walkLogicalAccessChain(
838 GEP&: *GEP, OnLiteralIndexing: [&CurType](Type *EltType, uint64_t Index) { CurType = EltType; },
839 OnDynamicIndexing: [&CurType](Type *EltType, Value *Index) { CurType = EltType; });
840
841 return Interrupted ? GEP->getResultElementType() : CurType;
842}
843
844Type *SPIRVEmitIntrinsics::getGEPType(GetElementPtrInst *Ref) {
845 if (Ref->getSourceElementType() ==
846 IntegerType::getInt8Ty(C&: CurrF->getContext()) &&
847 TM->getSubtargetImpl()->isLogicalSPIRV()) {
848 return getGEPTypeLogical(GEP: Ref);
849 }
850
851 Type *Ty = nullptr;
852 // TODO: not sure if GetElementPtrInst::getTypeAtIndex() does anything
853 // useful here
854 if (isNestedPointer(Ty: Ref->getSourceElementType())) {
855 Ty = Ref->getSourceElementType();
856 for (Use &U : drop_begin(RangeOrContainer: Ref->indices()))
857 Ty = GetElementPtrInst::getTypeAtIndex(Ty, Idx: U.get());
858 } else {
859 Ty = Ref->getResultElementType();
860 }
861 return Ty;
862}
863
864Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
865 Value *I, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8,
866 bool IgnoreKnownType) {
867 // allow to pass nullptr as an argument
868 if (!I)
869 return nullptr;
870
871 // maybe already known
872 if (!IgnoreKnownType)
873 if (Type *KnownTy = GR->findDeducedElementType(Val: I))
874 return KnownTy;
875
876 // maybe a cycle
877 if (!Visited.insert(x: I).second)
878 return nullptr;
879
880 // fallback value in case when we fail to deduce a type
881 Type *Ty = nullptr;
882 // look for known basic patterns of type inference
883 if (auto *Ref = dyn_cast<AllocaInst>(Val: I)) {
884 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getAllocatedType(), UnknownElemTypeI8);
885 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
886 Ty = getGEPType(Ref);
887 } else if (auto *SGEP = dyn_cast<StructuredGEPInst>(Val: I)) {
888 Ty = SGEP->getResultElementType();
889 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
890 Value *Op = Ref->getPointerOperand();
891 Type *KnownTy = GR->findDeducedElementType(Val: Op);
892 if (!KnownTy)
893 KnownTy = Op->getType();
894 if (Type *ElemTy = getPointeeType(Ty: KnownTy))
895 maybeAssignPtrType(Ty, Op: I, RefTy: ElemTy, UnknownElemTypeI8);
896 } else if (auto *Ref = dyn_cast<GlobalValue>(Val: I)) {
897 if (auto *Fn = dyn_cast<Function>(Val: Ref)) {
898 Ty = SPIRV::getOriginalFunctionType(F: *Fn);
899 GR->addDeducedElementType(Val: I, Ty);
900 } else {
901 Ty = deduceElementTypeByValueDeep(
902 ValueTy: Ref->getValueType(),
903 Operand: Ref->getNumOperands() > 0 ? Ref->getOperand(i: 0) : nullptr, Visited,
904 UnknownElemTypeI8);
905 }
906 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
907 Type *RefTy = deduceElementTypeHelper(I: Ref->getPointerOperand(), Visited,
908 UnknownElemTypeI8);
909 maybeAssignPtrType(Ty, Op: I, RefTy, UnknownElemTypeI8);
910 } else if (auto *Ref = dyn_cast<IntToPtrInst>(Val: I)) {
911 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getDestTy(), UnknownElemTypeI8);
912 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
913 if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
914 isPointerTy(T: Src) && isPointerTy(T: Dest))
915 Ty = deduceElementTypeHelper(I: Ref->getOperand(i_nocapture: 0), Visited,
916 UnknownElemTypeI8);
917 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
918 Value *Op = Ref->getNewValOperand();
919 if (isPointerTy(T: Op->getType()))
920 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
921 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
922 Value *Op = Ref->getValOperand();
923 if (isPointerTy(T: Op->getType()))
924 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
925 } else if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
926 Type *BestTy = nullptr;
927 unsigned MaxN = 1;
928 DenseMap<Type *, unsigned> PhiTys;
929 for (int i = Ref->getNumIncomingValues() - 1; i >= 0; --i) {
930 Ty = deduceElementTypeByUsersDeep(Op: Ref->getIncomingValue(i), Visited,
931 UnknownElemTypeI8);
932 if (!Ty)
933 continue;
934 auto It = PhiTys.try_emplace(Key: Ty, Args: 1);
935 if (!It.second) {
936 ++It.first->second;
937 if (It.first->second > MaxN) {
938 MaxN = It.first->second;
939 BestTy = Ty;
940 }
941 }
942 }
943 if (BestTy)
944 Ty = BestTy;
945 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
946 for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) {
947 Ty = deduceElementTypeByUsersDeep(Op, Visited, UnknownElemTypeI8);
948 if (Ty)
949 break;
950 }
951 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
952 static StringMap<unsigned> ResTypeByArg = {
953 {"to_global", 0},
954 {"to_local", 0},
955 {"to_private", 0},
956 {"__spirv_GenericCastToPtr_ToGlobal", 0},
957 {"__spirv_GenericCastToPtr_ToLocal", 0},
958 {"__spirv_GenericCastToPtr_ToPrivate", 0},
959 {"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
960 {"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
961 {"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
962 // TODO: maybe improve performance by caching demangled names
963
964 auto *II = dyn_cast<IntrinsicInst>(Val: I);
965 if (II && II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
966 auto *HandleType = cast<TargetExtType>(Val: II->getOperand(i_nocapture: 0)->getType());
967 if (HandleType->getTargetExtName() == "spirv.Image" ||
968 HandleType->getTargetExtName() == "spirv.SignedImage") {
969 for (User *U : II->users()) {
970 Ty = cast<Instruction>(Val: U)->getAccessType();
971 if (Ty)
972 break;
973 }
974 } else if (HandleType->getTargetExtName() == "spirv.VulkanBuffer") {
975 // This call is supposed to index into an array
976 Ty = HandleType->getTypeParameter(i: 0);
977 if (Ty->isArrayTy())
978 Ty = Ty->getArrayElementType();
979 else {
980 assert(Ty && Ty->isStructTy());
981 uint32_t Index = cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1))->getZExtValue();
982 Ty = cast<StructType>(Val: Ty)->getElementType(N: Index);
983 }
984 Ty = reconstitutePeeledArrayType(Ty);
985 } else {
986 llvm_unreachable("Unknown handle type for spv_resource_getpointer.");
987 }
988 } else if (II && II->getIntrinsicID() ==
989 Intrinsic::spv_generic_cast_to_ptr_explicit) {
990 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: 0), Visited,
991 UnknownElemTypeI8);
992 } else if (Function *CalledF = CI->getCalledFunction()) {
993 std::string DemangledName =
994 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
995 if (DemangledName.length() > 0)
996 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName);
997 auto AsArgIt = ResTypeByArg.find(Key: DemangledName);
998 if (AsArgIt != ResTypeByArg.end())
999 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: AsArgIt->second),
1000 Visited, UnknownElemTypeI8);
1001 else if (Type *KnownRetTy = GR->findDeducedElementType(Val: CalledF))
1002 Ty = KnownRetTy;
1003 }
1004 }
1005
1006 // remember the found relationship
1007 if (Ty && !IgnoreKnownType) {
1008 // specify nested types if needed, otherwise return unchanged
1009 GR->addDeducedElementType(Val: I, Ty: normalizeType(Ty));
1010 }
1011
1012 return Ty;
1013}
1014
1015// Re-create a type of the value if it has untyped pointer fields, also nested.
1016// Return the original value type if no corrections of untyped pointer
1017// information is found or needed.
1018Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(User *U,
1019 bool UnknownElemTypeI8) {
1020 std::unordered_set<Value *> Visited;
1021 return deduceNestedTypeHelper(U, Ty: U->getType(), Visited, UnknownElemTypeI8);
1022}
1023
1024Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
1025 User *U, Type *OrigTy, std::unordered_set<Value *> &Visited,
1026 bool UnknownElemTypeI8) {
1027 if (!U)
1028 return OrigTy;
1029
1030 // maybe already known
1031 if (Type *KnownTy = GR->findDeducedCompositeType(Val: U))
1032 return KnownTy;
1033
1034 // maybe a cycle
1035 if (!Visited.insert(x: U).second)
1036 return OrigTy;
1037
1038 if (isa<StructType>(Val: OrigTy)) {
1039 SmallVector<Type *> Tys;
1040 bool Change = false;
1041 for (unsigned i = 0; i < U->getNumOperands(); ++i) {
1042 Value *Op = U->getOperand(i);
1043 assert(Op && "Operands should not be null.");
1044 Type *OpTy = Op->getType();
1045 Type *Ty = OpTy;
1046 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1047 if (Type *NestedTy =
1048 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1049 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1050 } else {
1051 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1052 UnknownElemTypeI8);
1053 }
1054 Tys.push_back(Elt: Ty);
1055 Change |= Ty != OpTy;
1056 }
1057 if (Change) {
1058 Type *NewTy = StructType::create(Elements: Tys);
1059 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1060 return NewTy;
1061 }
1062 } else if (auto *ArrTy = dyn_cast<ArrayType>(Val: OrigTy)) {
1063 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1064 Type *OpTy = ArrTy->getElementType();
1065 Type *Ty = OpTy;
1066 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1067 if (Type *NestedTy =
1068 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1069 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1070 } else {
1071 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1072 UnknownElemTypeI8);
1073 }
1074 if (Ty != OpTy) {
1075 Type *NewTy = ArrayType::get(ElementType: Ty, NumElements: ArrTy->getNumElements());
1076 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1077 return NewTy;
1078 }
1079 }
1080 } else if (auto *VecTy = dyn_cast<VectorType>(Val: OrigTy)) {
1081 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1082 Type *OpTy = VecTy->getElementType();
1083 Type *Ty = OpTy;
1084 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1085 if (Type *NestedTy =
1086 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1087 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1088 } else {
1089 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1090 UnknownElemTypeI8);
1091 }
1092 if (Ty != OpTy) {
1093 Type *NewTy = VectorType::get(ElementType: Ty, EC: VecTy->getElementCount());
1094 GR->addDeducedCompositeType(Val: U, Ty: normalizeType(Ty: NewTy));
1095 return NewTy;
1096 }
1097 }
1098 }
1099
1100 return OrigTy;
1101}
1102
1103Type *SPIRVEmitIntrinsics::deduceElementType(Value *I, bool UnknownElemTypeI8) {
1104 if (Type *Ty = deduceElementTypeHelper(I, UnknownElemTypeI8))
1105 return Ty;
1106 if (!UnknownElemTypeI8)
1107 return nullptr;
1108 insertTodoType(Op: I);
1109 return IntegerType::getInt8Ty(C&: I->getContext());
1110}
1111
1112static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I,
1113 Value *PointerOperand) {
1114 Type *PointeeTy = GR->findDeducedElementType(Val: PointerOperand);
1115 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1116 return nullptr;
1117 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType());
1118 if (!PtrTy)
1119 return I->getType();
1120 if (Type *NestedTy = GR->findDeducedElementType(Val: I))
1121 return getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1122 return nullptr;
1123}
1124
1125// Try to deduce element type for a call base. Returns false if this is an
1126// indirect function invocation, and true otherwise.
1127bool SPIRVEmitIntrinsics::deduceOperandElementTypeCalledFunction(
1128 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1129 Type *&KnownElemTy, bool &Incomplete) {
1130 Function *CalledF = CI->getCalledFunction();
1131 if (!CalledF)
1132 return false;
1133 std::string DemangledName =
1134 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1135 if (DemangledName.length() > 0 &&
1136 !StringRef(DemangledName).starts_with(Prefix: "llvm.")) {
1137 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F: *CalledF);
1138 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
1139 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
1140 if (Opcode == SPIRV::OpGroupAsyncCopy) {
1141 for (unsigned i = 0, PtrCnt = 0; i < CI->arg_size() && PtrCnt < 2; ++i) {
1142 Value *Op = CI->getArgOperand(i);
1143 if (!isPointerTy(T: Op->getType()))
1144 continue;
1145 ++PtrCnt;
1146 if (Type *ElemTy = GR->findDeducedElementType(Val: Op))
1147 KnownElemTy = ElemTy; // src will rewrite dest if both are defined
1148 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1149 }
1150 } else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) {
1151 if (CI->arg_size() == 0)
1152 return true;
1153 Value *Op = CI->getArgOperand(i: 0);
1154 if (!isPointerTy(T: Op->getType()))
1155 return true;
1156 switch (Opcode) {
1157 case SPIRV::OpAtomicFAddEXT:
1158 case SPIRV::OpAtomicFMinEXT:
1159 case SPIRV::OpAtomicFMaxEXT:
1160 case SPIRV::OpAtomicLoad:
1161 case SPIRV::OpAtomicCompareExchangeWeak:
1162 case SPIRV::OpAtomicCompareExchange:
1163 case SPIRV::OpAtomicExchange:
1164 case SPIRV::OpAtomicIAdd:
1165 case SPIRV::OpAtomicISub:
1166 case SPIRV::OpAtomicOr:
1167 case SPIRV::OpAtomicXor:
1168 case SPIRV::OpAtomicAnd:
1169 case SPIRV::OpAtomicUMin:
1170 case SPIRV::OpAtomicUMax:
1171 case SPIRV::OpAtomicSMin:
1172 case SPIRV::OpAtomicSMax: {
1173 KnownElemTy = isPointerTy(T: CI->getType()) ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1174 : CI->getType();
1175 if (!KnownElemTy)
1176 return true;
1177 Incomplete = isTodoType(Op);
1178 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1179 } break;
1180 case SPIRV::OpAtomicStore: {
1181 if (CI->arg_size() < 4)
1182 return true;
1183 Value *ValOp = CI->getArgOperand(i: 3);
1184 KnownElemTy = isPointerTy(T: ValOp->getType())
1185 ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1186 : ValOp->getType();
1187 if (!KnownElemTy)
1188 return true;
1189 Incomplete = isTodoType(Op);
1190 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1191 } break;
1192 }
1193 }
1194 }
1195 return true;
1196}
1197
1198// Try to deduce element type for a function pointer.
1199void SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionPointer(
1200 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1201 Type *&KnownElemTy, bool IsPostprocessing) {
1202 Value *Op = CI->getCalledOperand();
1203 if (!Op || !isPointerTy(T: Op->getType()))
1204 return;
1205 Ops.push_back(Elt: std::make_pair(x&: Op, y: std::numeric_limits<unsigned>::max()));
1206 FunctionType *FTy = SPIRV::getOriginalFunctionType(CB: *CI);
1207 bool IsNewFTy = false, IsIncomplete = false;
1208 SmallVector<Type *, 4> ArgTys;
1209 for (auto &&[ParmIdx, Arg] : llvm::enumerate(First: CI->args())) {
1210 Type *ArgTy = Arg->getType();
1211 if (ArgTy->isPointerTy()) {
1212 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg)) {
1213 IsNewFTy = true;
1214 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
1215 if (isTodoType(Op: Arg))
1216 IsIncomplete = true;
1217 } else {
1218 IsIncomplete = true;
1219 }
1220 } else {
1221 ArgTy = FTy->getFunctionParamType(i: ParmIdx);
1222 }
1223 ArgTys.push_back(Elt: ArgTy);
1224 }
1225 Type *RetTy = FTy->getReturnType();
1226 if (CI->getType()->isPointerTy()) {
1227 if (Type *ElemTy = GR->findDeducedElementType(Val: CI)) {
1228 IsNewFTy = true;
1229 RetTy =
1230 getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: CI->getType()));
1231 if (isTodoType(Op: CI))
1232 IsIncomplete = true;
1233 } else {
1234 IsIncomplete = true;
1235 }
1236 }
1237 if (!IsPostprocessing && IsIncomplete)
1238 insertTodoType(Op);
1239 KnownElemTy =
1240 IsNewFTy ? FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: FTy->isVarArg()) : FTy;
1241}
1242
1243bool SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionRet(
1244 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
1245 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing,
1246 Type *&KnownElemTy, Value *Op, Function *F) {
1247 KnownElemTy = GR->findDeducedElementType(Val: F);
1248 if (KnownElemTy)
1249 return false;
1250 if (Type *OpElemTy = GR->findDeducedElementType(Val: Op)) {
1251 OpElemTy = normalizeType(Ty: OpElemTy);
1252 GR->addDeducedElementType(Val: F, Ty: OpElemTy);
1253 GR->addReturnType(
1254 ArgF: F, DerivedTy: TypedPointerType::get(ElementType: OpElemTy,
1255 AddressSpace: getPointerAddressSpace(T: F->getReturnType())));
1256 // non-recursive update of types in function uses
1257 DenseSet<std::pair<Value *, Value *>> VisitedSubst{std::make_pair(x&: I, y&: Op)};
1258 for (User *U : F->users()) {
1259 CallInst *CI = dyn_cast<CallInst>(Val: U);
1260 if (!CI || CI->getCalledFunction() != F)
1261 continue;
1262 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: CI)) {
1263 if (Type *PrevElemTy = GR->findDeducedElementType(Val: CI)) {
1264 GR->updateAssignType(AssignCI, Arg: CI,
1265 OfType: getNormalizedPoisonValue(Ty: OpElemTy));
1266 propagateElemType(Op: CI, ElemTy: PrevElemTy, VisitedSubst);
1267 }
1268 }
1269 }
1270 // Non-recursive update of types in the function uncomplete returns.
1271 // This may happen just once per a function, the latch is a pair of
1272 // findDeducedElementType(F) / addDeducedElementType(F, ...).
1273 // With or without the latch it is a non-recursive call due to
1274 // IncompleteRets set to nullptr in this call.
1275 if (IncompleteRets)
1276 for (Instruction *IncompleteRetI : *IncompleteRets)
1277 deduceOperandElementType(I: IncompleteRetI, IncompleteRets: nullptr, AskOps,
1278 IsPostprocessing);
1279 } else if (IncompleteRets) {
1280 IncompleteRets->insert(Ptr: I);
1281 }
1282 TypeValidated.insert(x: I);
1283 return true;
1284}
1285
1286// If the Instruction has Pointer operands with unresolved types, this function
1287// tries to deduce them. If the Instruction has Pointer operands with known
1288// types which differ from expected, this function tries to insert a bitcast to
1289// resolve the issue.
1290void SPIRVEmitIntrinsics::deduceOperandElementType(
1291 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
1292 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing) {
1293 SmallVector<std::pair<Value *, unsigned>> Ops;
1294 Type *KnownElemTy = nullptr;
1295 bool Incomplete = false;
1296 // look for known basic patterns of type inference
1297 if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1298 if (!isPointerTy(T: I->getType()) ||
1299 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1300 return;
1301 Incomplete = isTodoType(Op: I);
1302 for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
1303 Value *Op = Ref->getIncomingValue(i);
1304 if (isPointerTy(T: Op->getType()))
1305 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1306 }
1307 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1308 KnownElemTy = GR->findDeducedElementType(Val: I);
1309 if (!KnownElemTy)
1310 return;
1311 Incomplete = isTodoType(Op: I);
1312 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(), y: 0));
1313 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1314 if (!isPointerTy(T: I->getType()))
1315 return;
1316 KnownElemTy = GR->findDeducedElementType(Val: I);
1317 if (!KnownElemTy)
1318 return;
1319 Incomplete = isTodoType(Op: I);
1320 Ops.push_back(Elt: std::make_pair(x: Ref->getOperand(i_nocapture: 0), y: 0));
1321 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
1322 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1323 return;
1324 KnownElemTy = Ref->getSourceElementType();
1325 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1326 y: GetElementPtrInst::getPointerOperandIndex()));
1327 } else if (auto *Ref = dyn_cast<StructuredGEPInst>(Val: I)) {
1328 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1329 return;
1330 KnownElemTy = Ref->getBaseType();
1331 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1332 y: StructuredGEPInst::getPointerOperandIndex()));
1333 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
1334 KnownElemTy = I->getType();
1335 if (isUntypedPointerTy(T: KnownElemTy))
1336 return;
1337 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1338 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1339 return;
1340 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1341 y: LoadInst::getPointerOperandIndex()));
1342 } else if (auto *Ref = dyn_cast<StoreInst>(Val: I)) {
1343 if (!(KnownElemTy =
1344 reconstructType(Op: Ref->getValueOperand(), UnknownElemTypeI8: false, IsPostprocessing)))
1345 return;
1346 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1347 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1348 return;
1349 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1350 y: StoreInst::getPointerOperandIndex()));
1351 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1352 KnownElemTy = isPointerTy(T: I->getType())
1353 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1354 : I->getType();
1355 if (!KnownElemTy)
1356 return;
1357 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1358 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1359 y: AtomicCmpXchgInst::getPointerOperandIndex()));
1360 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1361 KnownElemTy = isPointerTy(T: I->getType())
1362 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1363 : I->getType();
1364 if (!KnownElemTy)
1365 return;
1366 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1367 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1368 y: AtomicRMWInst::getPointerOperandIndex()));
1369 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1370 if (!isPointerTy(T: I->getType()) ||
1371 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1372 return;
1373 Incomplete = isTodoType(Op: I);
1374 for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
1375 Value *Op = Ref->getOperand(i_nocapture: i);
1376 if (isPointerTy(T: Op->getType()))
1377 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1378 }
1379 } else if (auto *Ref = dyn_cast<ReturnInst>(Val: I)) {
1380 if (!isPointerTy(T: CurrF->getReturnType()))
1381 return;
1382 Value *Op = Ref->getReturnValue();
1383 if (!Op)
1384 return;
1385 if (deduceOperandElementTypeFunctionRet(I, IncompleteRets, AskOps,
1386 IsPostprocessing, KnownElemTy, Op,
1387 F: CurrF))
1388 return;
1389 Incomplete = isTodoType(Op: CurrF);
1390 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1391 } else if (auto *Ref = dyn_cast<ICmpInst>(Val: I)) {
1392 if (!isPointerTy(T: Ref->getOperand(i_nocapture: 0)->getType()))
1393 return;
1394 Value *Op0 = Ref->getOperand(i_nocapture: 0);
1395 Value *Op1 = Ref->getOperand(i_nocapture: 1);
1396 bool Incomplete0 = isTodoType(Op: Op0);
1397 bool Incomplete1 = isTodoType(Op: Op1);
1398 Type *ElemTy1 = GR->findDeducedElementType(Val: Op1);
1399 Type *ElemTy0 = (Incomplete0 && !Incomplete1 && ElemTy1)
1400 ? nullptr
1401 : GR->findDeducedElementType(Val: Op0);
1402 if (ElemTy0) {
1403 KnownElemTy = ElemTy0;
1404 Incomplete = Incomplete0;
1405 Ops.push_back(Elt: std::make_pair(x&: Op1, y: 1));
1406 } else if (ElemTy1) {
1407 KnownElemTy = ElemTy1;
1408 Incomplete = Incomplete1;
1409 Ops.push_back(Elt: std::make_pair(x&: Op0, y: 0));
1410 }
1411 } else if (CallInst *CI = dyn_cast<CallInst>(Val: I)) {
1412 if (!CI->isIndirectCall())
1413 deduceOperandElementTypeCalledFunction(CI, Ops, KnownElemTy, Incomplete);
1414 else if (HaveFunPtrs)
1415 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy,
1416 IsPostprocessing);
1417 }
1418
1419 // There is no enough info to deduce types or all is valid.
1420 if (!KnownElemTy || Ops.size() == 0)
1421 return;
1422
1423 LLVMContext &Ctx = CurrF->getContext();
1424 IRBuilder<> B(Ctx);
1425 for (auto &OpIt : Ops) {
1426 Value *Op = OpIt.first;
1427 if (AskOps && !AskOps->contains(Ptr: Op))
1428 continue;
1429 Type *AskTy = nullptr;
1430 CallInst *AskCI = nullptr;
1431 if (IsPostprocessing && AskOps) {
1432 AskTy = GR->findDeducedElementType(Val: Op);
1433 AskCI = GR->findAssignPtrTypeInstr(Val: Op);
1434 assert(AskTy && AskCI);
1435 }
1436 Type *Ty = AskTy ? AskTy : GR->findDeducedElementType(Val: Op);
1437 if (Ty == KnownElemTy)
1438 continue;
1439 Value *OpTyVal = getNormalizedPoisonValue(Ty: KnownElemTy);
1440 Type *OpTy = Op->getType();
1441 if (Op->hasUseList() &&
1442 (!Ty || AskTy || isUntypedPointerTy(T: Ty) || isTodoType(Op))) {
1443 Type *PrevElemTy = GR->findDeducedElementType(Val: Op);
1444 GR->addDeducedElementType(Val: Op, Ty: normalizeType(Ty: KnownElemTy));
1445 // check if KnownElemTy is complete
1446 if (!Incomplete)
1447 eraseTodoType(Op);
1448 else if (!IsPostprocessing)
1449 insertTodoType(Op);
1450 // check if there is existing Intrinsic::spv_assign_ptr_type instruction
1451 CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Val: Op);
1452 if (AssignCI == nullptr) {
1453 Instruction *User = dyn_cast<Instruction>(Val: Op->use_begin()->get());
1454 setInsertPointSkippingPhis(B, I: User ? User->getNextNode() : I);
1455 CallInst *CI =
1456 buildIntrWithMD(IntrID: Intrinsic::spv_assign_ptr_type, Types: {OpTy}, Arg: OpTyVal, Arg2: Op,
1457 Imms: {B.getInt32(C: getPointerAddressSpace(T: OpTy))}, B);
1458 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: CI);
1459 } else {
1460 GR->updateAssignType(AssignCI, Arg: Op, OfType: OpTyVal);
1461 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1462 std::make_pair(x&: I, y&: Op)};
1463 propagateElemTypeRec(Op, PtrElemTy: KnownElemTy, CastElemTy: PrevElemTy, VisitedSubst);
1464 }
1465 } else {
1466 eraseTodoType(Op);
1467 CallInst *PtrCastI =
1468 buildSpvPtrcast(F: I->getParent()->getParent(), Op, ElemTy: KnownElemTy);
1469 if (OpIt.second == std::numeric_limits<unsigned>::max())
1470 dyn_cast<CallInst>(Val: I)->setCalledOperand(PtrCastI);
1471 else
1472 I->setOperand(i: OpIt.second, Val: PtrCastI);
1473 }
1474 }
1475 TypeValidated.insert(x: I);
1476}
1477
1478void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
1479 Instruction *New,
1480 IRBuilder<> &B) {
1481 while (!Old->user_empty()) {
1482 auto *U = Old->user_back();
1483 if (isAssignTypeInstr(I: U)) {
1484 B.SetInsertPoint(U);
1485 SmallVector<Value *, 2> Args = {New, U->getOperand(i: 1)};
1486 CallInst *AssignCI =
1487 B.CreateIntrinsic(ID: Intrinsic::spv_assign_type, Types: {New->getType()}, Args);
1488 GR->addAssignPtrTypeInstr(Val: New, AssignPtrTyCI: AssignCI);
1489 U->eraseFromParent();
1490 } else if (isMemInstrToReplace(I: U) || isa<ReturnInst>(Val: U) ||
1491 isa<CallInst>(Val: U)) {
1492 U->replaceUsesOfWith(From: Old, To: New);
1493 } else {
1494 llvm_unreachable("illegal aggregate intrinsic user");
1495 }
1496 }
1497 New->copyMetadata(SrcInst: *Old);
1498 Old->eraseFromParent();
1499}
1500
1501void SPIRVEmitIntrinsics::preprocessUndefs(IRBuilder<> &B) {
1502 std::queue<Instruction *> Worklist;
1503 for (auto &I : instructions(F: CurrF))
1504 Worklist.push(x: &I);
1505
1506 while (!Worklist.empty()) {
1507 Instruction *I = Worklist.front();
1508 bool BPrepared = false;
1509 Worklist.pop();
1510
1511 for (auto &Op : I->operands()) {
1512 auto *AggrUndef = dyn_cast<UndefValue>(Val&: Op);
1513 if (!AggrUndef || !Op->getType()->isAggregateType())
1514 continue;
1515
1516 if (!BPrepared) {
1517 setInsertPointSkippingPhis(B, I);
1518 BPrepared = true;
1519 }
1520 auto *IntrUndef = B.CreateIntrinsic(ID: Intrinsic::spv_undef, Args: {});
1521 Worklist.push(x: IntrUndef);
1522 I->replaceUsesOfWith(From: Op, To: IntrUndef);
1523 AggrConsts[IntrUndef] = AggrUndef;
1524 AggrConstTypes[IntrUndef] = AggrUndef->getType();
1525 }
1526 }
1527}
1528
1529void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) {
1530 std::queue<Instruction *> Worklist;
1531 for (auto &I : instructions(F: CurrF))
1532 Worklist.push(x: &I);
1533
1534 while (!Worklist.empty()) {
1535 auto *I = Worklist.front();
1536 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
1537 assert(I);
1538 bool KeepInst = false;
1539 for (const auto &Op : I->operands()) {
1540 Constant *AggrConst = nullptr;
1541 Type *ResTy = nullptr;
1542 if (auto *COp = dyn_cast<ConstantVector>(Val: Op)) {
1543 AggrConst = COp;
1544 ResTy = COp->getType();
1545 } else if (auto *COp = dyn_cast<ConstantArray>(Val: Op)) {
1546 AggrConst = COp;
1547 ResTy = B.getInt32Ty();
1548 } else if (auto *COp = dyn_cast<ConstantStruct>(Val: Op)) {
1549 AggrConst = COp;
1550 ResTy = B.getInt32Ty();
1551 } else if (auto *COp = dyn_cast<ConstantDataArray>(Val: Op)) {
1552 AggrConst = COp;
1553 ResTy = B.getInt32Ty();
1554 } else if (auto *COp = dyn_cast<ConstantAggregateZero>(Val: Op)) {
1555 AggrConst = COp;
1556 ResTy = Op->getType()->isVectorTy() ? COp->getType() : B.getInt32Ty();
1557 }
1558 if (AggrConst) {
1559 SmallVector<Value *> Args;
1560 if (auto *COp = dyn_cast<ConstantDataSequential>(Val: Op))
1561 for (unsigned i = 0; i < COp->getNumElements(); ++i)
1562 Args.push_back(Elt: COp->getElementAsConstant(i));
1563 else
1564 llvm::append_range(C&: Args, R: AggrConst->operands());
1565 if (!BPrepared) {
1566 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
1567 : B.SetInsertPoint(I);
1568 BPrepared = true;
1569 }
1570 auto *CI =
1571 B.CreateIntrinsic(ID: Intrinsic::spv_const_composite, Types: {ResTy}, Args: {Args});
1572 Worklist.push(x: CI);
1573 I->replaceUsesOfWith(From: Op, To: CI);
1574 KeepInst = true;
1575 AggrConsts[CI] = AggrConst;
1576 AggrConstTypes[CI] = deduceNestedTypeHelper(U: AggrConst, UnknownElemTypeI8: false);
1577 }
1578 }
1579 if (!KeepInst)
1580 Worklist.pop();
1581 }
1582}
1583
1584static void createDecorationIntrinsic(Instruction *I, MDNode *Node,
1585 IRBuilder<> &B) {
1586 LLVMContext &Ctx = I->getContext();
1587 setInsertPointAfterDef(B, I);
1588 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, Types: {I->getType()},
1589 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: {Node}))});
1590}
1591
1592static void createRoundingModeDecoration(Instruction *I,
1593 unsigned RoundingModeDeco,
1594 IRBuilder<> &B) {
1595 LLVMContext &Ctx = I->getContext();
1596 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1597 MDNode *RoundingModeNode = MDNode::get(
1598 Context&: Ctx,
1599 MDs: {ConstantAsMetadata::get(
1600 C: ConstantInt::get(Ty: Int32Ty, V: SPIRV::Decoration::FPRoundingMode)),
1601 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: RoundingModeDeco))});
1602 createDecorationIntrinsic(I, Node: RoundingModeNode, B);
1603}
1604
1605static void createSaturatedConversionDecoration(Instruction *I,
1606 IRBuilder<> &B) {
1607 LLVMContext &Ctx = I->getContext();
1608 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1609 MDNode *SaturatedConversionNode =
1610 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1611 Ty: Int32Ty, V: SPIRV::Decoration::SaturatedConversion))});
1612 createDecorationIntrinsic(I, Node: SaturatedConversionNode, B);
1613}
1614
1615static void addSaturatedDecorationToIntrinsic(Instruction *I, IRBuilder<> &B) {
1616 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
1617 if (Function *Fu = CI->getCalledFunction()) {
1618 if (Fu->isIntrinsic()) {
1619 unsigned const int IntrinsicId = Fu->getIntrinsicID();
1620 switch (IntrinsicId) {
1621 case Intrinsic::fptosi_sat:
1622 case Intrinsic::fptoui_sat:
1623 createSaturatedConversionDecoration(I, B);
1624 break;
1625 default:
1626 break;
1627 }
1628 }
1629 }
1630 }
1631}
1632
1633Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
1634 if (!Call.isInlineAsm())
1635 return &Call;
1636
1637 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
1638 LLVMContext &Ctx = CurrF->getContext();
1639
1640 Constant *TyC = UndefValue::get(T: IA->getFunctionType());
1641 MDString *ConstraintString = MDString::get(Context&: Ctx, Str: IA->getConstraintString());
1642 SmallVector<Value *> Args = {
1643 buildMD(Arg: TyC),
1644 MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: ConstraintString))};
1645 for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++)
1646 Args.push_back(Elt: Call.getArgOperand(i: OpIdx));
1647
1648 IRBuilder<> B(Call.getParent());
1649 B.SetInsertPoint(&Call);
1650 B.CreateIntrinsic(ID: Intrinsic::spv_inline_asm, Args: {Args});
1651 return &Call;
1652}
1653
1654// Use a tip about rounding mode to create a decoration.
1655void SPIRVEmitIntrinsics::useRoundingMode(ConstrainedFPIntrinsic *FPI,
1656 IRBuilder<> &B) {
1657 std::optional<RoundingMode> RM = FPI->getRoundingMode();
1658 if (!RM.has_value())
1659 return;
1660 unsigned RoundingModeDeco = std::numeric_limits<unsigned>::max();
1661 switch (RM.value()) {
1662 default:
1663 // ignore unknown rounding modes
1664 break;
1665 case RoundingMode::NearestTiesToEven:
1666 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTE;
1667 break;
1668 case RoundingMode::TowardNegative:
1669 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTN;
1670 break;
1671 case RoundingMode::TowardPositive:
1672 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTP;
1673 break;
1674 case RoundingMode::TowardZero:
1675 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTZ;
1676 break;
1677 case RoundingMode::Dynamic:
1678 case RoundingMode::NearestTiesToAway:
1679 // TODO: check if supported
1680 break;
1681 }
1682 if (RoundingModeDeco == std::numeric_limits<unsigned>::max())
1683 return;
1684 // Convert the tip about rounding mode into a decoration record.
1685 createRoundingModeDecoration(I: FPI, RoundingModeDeco, B);
1686}
1687
1688Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
1689 BasicBlock *ParentBB = I.getParent();
1690 Function *F = ParentBB->getParent();
1691 IRBuilder<> B(ParentBB);
1692 B.SetInsertPoint(&I);
1693 SmallVector<Value *, 4> Args;
1694 SmallVector<BasicBlock *> BBCases;
1695 Args.push_back(Elt: I.getCondition());
1696 BBCases.push_back(Elt: I.getDefaultDest());
1697 Args.push_back(Elt: BlockAddress::get(F, BB: I.getDefaultDest()));
1698 for (auto &Case : I.cases()) {
1699 Args.push_back(Elt: Case.getCaseValue());
1700 BBCases.push_back(Elt: Case.getCaseSuccessor());
1701 Args.push_back(Elt: BlockAddress::get(F, BB: Case.getCaseSuccessor()));
1702 }
1703 CallInst *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_switch,
1704 Types: {I.getOperand(i_nocapture: 0)->getType()}, Args: {Args});
1705 // remove switch to avoid its unneeded and undesirable unwrap into branches
1706 // and conditions
1707 replaceAllUsesWith(Src: &I, Dest: NewI);
1708 I.eraseFromParent();
1709 // insert artificial and temporary instruction to preserve valid CFG,
1710 // it will be removed after IR translation pass
1711 B.SetInsertPoint(ParentBB);
1712 IndirectBrInst *BrI = B.CreateIndirectBr(
1713 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: ParentBB->getContext())),
1714 NumDests: BBCases.size());
1715 for (BasicBlock *BBCase : BBCases)
1716 BrI->addDestination(Dest: BBCase);
1717 return BrI;
1718}
1719
1720static bool isFirstIndexZero(const GetElementPtrInst *GEP) {
1721 if (GEP->getNumIndices() == 0)
1722 return false;
1723 if (const auto *CI = dyn_cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: 1))) {
1724 return CI->getZExtValue() == 0;
1725 }
1726 return false;
1727}
1728
1729Instruction *SPIRVEmitIntrinsics::visitIntrinsicInst(IntrinsicInst &I) {
1730 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
1731 if (!SGEP)
1732 return &I;
1733
1734 IRBuilder<> B(I.getParent());
1735 B.SetInsertPoint(&I);
1736 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
1737 SmallVector<Value *, 4> Args;
1738 Args.push_back(/* inBounds= */ Elt: B.getInt1(V: true));
1739 Args.push_back(Elt: I.getOperand(i_nocapture: 0));
1740 Args.push_back(/* zero index */ Elt: B.getInt32(C: 0));
1741 for (unsigned J = 0; J < SGEP->getNumIndices(); ++J)
1742 Args.push_back(Elt: SGEP->getIndexOperand(Index: J));
1743
1744 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types, Args);
1745 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1746 return NewI;
1747}
1748
1749Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
1750 IRBuilder<> B(I.getParent());
1751 B.SetInsertPoint(&I);
1752
1753 if (TM->getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: &I)) {
1754 // Logical SPIR-V cannot use the OpPtrAccessChain instruction. If the first
1755 // index of the GEP is not 0, then we need to try to adjust it.
1756 //
1757 // If the GEP is doing byte addressing, try to rebuild the full access chain
1758 // from the type of the pointer.
1759 if (I.getSourceElementType() ==
1760 IntegerType::getInt8Ty(C&: CurrF->getContext())) {
1761 return buildLogicalAccessChainFromGEP(GEP&: I);
1762 }
1763
1764 // Look for the array-to-pointer decay. If this is the pattern
1765 // we can adjust the types, and prepend a 0 to the indices.
1766 Value *PtrOp = I.getPointerOperand();
1767 Type *SrcElemTy = I.getSourceElementType();
1768 Type *DeducedPointeeTy = deduceElementType(I: PtrOp, UnknownElemTypeI8: true);
1769
1770 if (auto *ArrTy = dyn_cast<ArrayType>(Val: DeducedPointeeTy)) {
1771 if (ArrTy->getElementType() == SrcElemTy) {
1772 SmallVector<Value *> NewIndices;
1773 Type *FirstIdxType = I.getOperand(i_nocapture: 1)->getType();
1774 NewIndices.push_back(Elt: ConstantInt::get(Ty: FirstIdxType, V: 0));
1775 for (Value *Idx : I.indices())
1776 NewIndices.push_back(Elt: Idx);
1777
1778 SmallVector<Type *, 2> Types = {I.getType(), I.getPointerOperandType()};
1779 SmallVector<Value *, 4> Args;
1780 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
1781 Args.push_back(Elt: I.getPointerOperand());
1782 Args.append(in_start: NewIndices.begin(), in_end: NewIndices.end());
1783
1784 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types: {Types}, Args: {Args});
1785 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1786 return NewI;
1787 }
1788 }
1789 }
1790
1791 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
1792 SmallVector<Value *, 4> Args;
1793 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
1794 llvm::append_range(C&: Args, R: I.operands());
1795 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types: {Types}, Args: {Args});
1796 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1797 return NewI;
1798}
1799
1800Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
1801 IRBuilder<> B(I.getParent());
1802 B.SetInsertPoint(&I);
1803 Value *Source = I.getOperand(i_nocapture: 0);
1804
1805 // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of
1806 // varying element types. In case of IR coming from older versions of LLVM
1807 // such bitcasts do not provide sufficient information, should be just skipped
1808 // here, and handled in insertPtrCastOrAssignTypeInstr.
1809 if (isPointerTy(T: I.getType())) {
1810 replaceAllUsesWith(Src: &I, Dest: Source);
1811 I.eraseFromParent();
1812 return nullptr;
1813 }
1814
1815 SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
1816 SmallVector<Value *> Args(I.op_begin(), I.op_end());
1817 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_bitcast, Types: {Types}, Args: {Args});
1818 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1819 return NewI;
1820}
1821
1822void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt(
1823 TargetExtType *AssignedType, Value *V, IRBuilder<> &B) {
1824 Type *VTy = V->getType();
1825
1826 // A couple of sanity checks.
1827 assert((isPointerTy(VTy)) && "Expect a pointer type!");
1828 if (Type *ElemTy = getPointeeType(Ty: VTy))
1829 if (ElemTy != AssignedType)
1830 report_fatal_error(reason: "Unexpected pointer element type!");
1831
1832 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: V);
1833 if (!AssignCI) {
1834 GR->buildAssignType(B, Ty: AssignedType, Arg: V);
1835 return;
1836 }
1837
1838 Type *CurrentType =
1839 dyn_cast<ConstantAsMetadata>(
1840 Val: cast<MetadataAsValue>(Val: AssignCI->getOperand(i_nocapture: 1))->getMetadata())
1841 ->getType();
1842 if (CurrentType == AssignedType)
1843 return;
1844
1845 // Builtin types cannot be redeclared or casted.
1846 if (CurrentType->isTargetExtTy())
1847 report_fatal_error(reason: "Type mismatch " + CurrentType->getTargetExtName() +
1848 "/" + AssignedType->getTargetExtName() +
1849 " for value " + V->getName(),
1850 gen_crash_diag: false);
1851
1852 // Our previous guess about the type seems to be wrong, let's update
1853 // inferred type according to a new, more precise type information.
1854 GR->updateAssignType(AssignCI, Arg: V, OfType: getNormalizedPoisonValue(Ty: AssignedType));
1855}
1856
1857void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
1858 Instruction *I, Value *Pointer, Type *ExpectedElementType,
1859 unsigned OperandToReplace, IRBuilder<> &B) {
1860 TypeValidated.insert(x: I);
1861
1862 // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType
1863 Type *PointerElemTy = deduceElementTypeHelper(I: Pointer, UnknownElemTypeI8: false);
1864 if (PointerElemTy == ExpectedElementType ||
1865 isEquivalentTypes(Ty1: PointerElemTy, Ty2: ExpectedElementType))
1866 return;
1867
1868 setInsertPointSkippingPhis(B, I);
1869 Value *ExpectedElementVal = getNormalizedPoisonValue(Ty: ExpectedElementType);
1870 MetadataAsValue *VMD = buildMD(Arg: ExpectedElementVal);
1871 unsigned AddressSpace = getPointerAddressSpace(T: Pointer->getType());
1872 bool FirstPtrCastOrAssignPtrType = true;
1873
1874 // Do not emit new spv_ptrcast if equivalent one already exists or when
1875 // spv_assign_ptr_type already targets this pointer with the same element
1876 // type.
1877 if (Pointer->hasUseList()) {
1878 for (auto User : Pointer->users()) {
1879 auto *II = dyn_cast<IntrinsicInst>(Val: User);
1880 if (!II ||
1881 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
1882 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
1883 II->getOperand(i_nocapture: 0) != Pointer)
1884 continue;
1885
1886 // There is some spv_ptrcast/spv_assign_ptr_type already targeting this
1887 // pointer.
1888 FirstPtrCastOrAssignPtrType = false;
1889 if (II->getOperand(i_nocapture: 1) != VMD ||
1890 dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 2))->getSExtValue() !=
1891 AddressSpace)
1892 continue;
1893
1894 // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the
1895 // same element type and address space.
1896 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
1897 return;
1898
1899 // This must be a spv_ptrcast, do not emit new if this one has the same BB
1900 // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type.
1901 if (II->getParent() != I->getParent())
1902 continue;
1903
1904 I->setOperand(i: OperandToReplace, Val: II);
1905 return;
1906 }
1907 }
1908
1909 if (isa<Instruction>(Val: Pointer) || isa<Argument>(Val: Pointer)) {
1910 if (FirstPtrCastOrAssignPtrType) {
1911 // If this would be the first spv_ptrcast, do not emit spv_ptrcast and
1912 // emit spv_assign_ptr_type instead.
1913 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
1914 return;
1915 } else if (isTodoType(Op: Pointer)) {
1916 eraseTodoType(Op: Pointer);
1917 if (!isa<CallInst>(Val: Pointer) && !isaGEP(V: Pointer) &&
1918 !isa<AllocaInst>(Val: Pointer)) {
1919 // If this wouldn't be the first spv_ptrcast but existing type info is
1920 // uncomplete, update spv_assign_ptr_type arguments.
1921 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Pointer)) {
1922 Type *PrevElemTy = GR->findDeducedElementType(Val: Pointer);
1923 assert(PrevElemTy);
1924 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1925 std::make_pair(x&: I, y&: Pointer)};
1926 GR->updateAssignType(AssignCI, Arg: Pointer, OfType: ExpectedElementVal);
1927 propagateElemType(Op: Pointer, ElemTy: PrevElemTy, VisitedSubst);
1928 } else {
1929 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
1930 }
1931 return;
1932 }
1933 }
1934 }
1935
1936 // Emit spv_ptrcast
1937 SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()};
1938 SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(C: AddressSpace)};
1939 auto *PtrCastI = B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
1940 I->setOperand(i: OperandToReplace, Val: PtrCastI);
1941 // We need to set up a pointee type for the newly created spv_ptrcast.
1942 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: PtrCastI);
1943}
1944
1945void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
1946 IRBuilder<> &B) {
1947 // Handle basic instructions:
1948 StoreInst *SI = dyn_cast<StoreInst>(Val: I);
1949 if (IsKernelArgInt8(F: CurrF, SI)) {
1950 replacePointerOperandWithPtrCast(
1951 I, Pointer: SI->getValueOperand(), ExpectedElementType: IntegerType::getInt8Ty(C&: CurrF->getContext()),
1952 OperandToReplace: 0, B);
1953 }
1954 if (SI) {
1955 Value *Op = SI->getValueOperand();
1956 Value *Pointer = SI->getPointerOperand();
1957 Type *OpTy = Op->getType();
1958 if (auto *OpI = dyn_cast<Instruction>(Val: Op))
1959 OpTy = restoreMutatedType(GR, I: OpI, Ty: OpTy);
1960 if (OpTy == Op->getType())
1961 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: Op, UnknownElemTypeI8: false);
1962 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 1, B);
1963 return;
1964 }
1965 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
1966 Value *Pointer = LI->getPointerOperand();
1967 Type *OpTy = LI->getType();
1968 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1969 if (Type *ElemTy = GR->findDeducedElementType(Val: LI)) {
1970 OpTy = getTypedPointerWrapper(ElemTy, AS: PtrTy->getAddressSpace());
1971 } else {
1972 Type *NewOpTy = OpTy;
1973 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: LI, UnknownElemTypeI8: false);
1974 if (OpTy == NewOpTy)
1975 insertTodoType(Op: Pointer);
1976 }
1977 }
1978 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
1979 return;
1980 }
1981 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(Val: I)) {
1982 Value *Pointer = GEPI->getPointerOperand();
1983 Type *OpTy = nullptr;
1984
1985 // Logical SPIR-V is not allowed to use Op*PtrAccessChain instructions. If
1986 // the first index is 0, then we can trivially lower to OpAccessChain. If
1987 // not we need to try to rewrite the GEP. We avoid adding a pointer cast at
1988 // this time, and will rewrite the GEP when visiting it.
1989 if (TM->getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: GEPI)) {
1990 return;
1991 }
1992
1993 // In all cases, fall back to the GEP type if type scavenging failed.
1994 if (!OpTy)
1995 OpTy = GEPI->getSourceElementType();
1996
1997 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
1998 if (isNestedPointer(Ty: OpTy))
1999 insertTodoType(Op: Pointer);
2000 return;
2001 }
2002
2003 // TODO: review and merge with existing logics:
2004 // Handle calls to builtins (non-intrinsics):
2005 CallInst *CI = dyn_cast<CallInst>(Val: I);
2006 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
2007 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
2008 return;
2009
2010 // collect information about formal parameter types
2011 std::string DemangledName =
2012 getOclOrSpirvBuiltinDemangledName(Name: CI->getCalledFunction()->getName());
2013 Function *CalledF = CI->getCalledFunction();
2014 SmallVector<Type *, 4> CalledArgTys;
2015 bool HaveTypes = false;
2016 for (unsigned OpIdx = 0; OpIdx < CalledF->arg_size(); ++OpIdx) {
2017 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2018 Type *ArgType = CalledArg->getType();
2019 if (!isPointerTy(T: ArgType)) {
2020 CalledArgTys.push_back(Elt: nullptr);
2021 } else if (Type *ArgTypeElem = getPointeeType(Ty: ArgType)) {
2022 CalledArgTys.push_back(Elt: ArgTypeElem);
2023 HaveTypes = true;
2024 } else {
2025 Type *ElemTy = GR->findDeducedElementType(Val: CalledArg);
2026 if (!ElemTy && hasPointeeTypeAttr(Arg: CalledArg))
2027 ElemTy = getPointeeTypeByAttr(Arg: CalledArg);
2028 if (!ElemTy) {
2029 ElemTy = getPointeeTypeByCallInst(DemangledName, CalledF, OpIdx);
2030 if (ElemTy) {
2031 GR->addDeducedElementType(Val: CalledArg, Ty: normalizeType(Ty: ElemTy));
2032 } else {
2033 for (User *U : CalledArg->users()) {
2034 if (Instruction *Inst = dyn_cast<Instruction>(Val: U)) {
2035 if ((ElemTy = deduceElementTypeHelper(I: Inst, UnknownElemTypeI8: false)) != nullptr)
2036 break;
2037 }
2038 }
2039 }
2040 }
2041 HaveTypes |= ElemTy != nullptr;
2042 CalledArgTys.push_back(Elt: ElemTy);
2043 }
2044 }
2045
2046 if (DemangledName.empty() && !HaveTypes)
2047 return;
2048
2049 for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) {
2050 Value *ArgOperand = CI->getArgOperand(i: OpIdx);
2051 if (!isPointerTy(T: ArgOperand->getType()))
2052 continue;
2053
2054 // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs()
2055 if (!isa<Instruction>(Val: ArgOperand) && !isa<Argument>(Val: ArgOperand)) {
2056 // However, we may have assumptions about the formal argument's type and
2057 // may have a need to insert a ptr cast for the actual parameter of this
2058 // call.
2059 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2060 if (!GR->findDeducedElementType(Val: CalledArg))
2061 continue;
2062 }
2063
2064 Type *ExpectedType =
2065 OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr;
2066 if (!ExpectedType && !DemangledName.empty())
2067 ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType(
2068 DemangledCall: DemangledName, ArgIdx: OpIdx, Ctx&: I->getContext());
2069 if (!ExpectedType || ExpectedType->isVoidTy())
2070 continue;
2071
2072 if (ExpectedType->isTargetExtTy() &&
2073 !isTypedPointerWrapper(ExtTy: cast<TargetExtType>(Val: ExpectedType)))
2074 insertAssignPtrTypeTargetExt(AssignedType: cast<TargetExtType>(Val: ExpectedType),
2075 V: ArgOperand, B);
2076 else
2077 replacePointerOperandWithPtrCast(I: CI, Pointer: ArgOperand, ExpectedElementType: ExpectedType, OperandToReplace: OpIdx, B);
2078 }
2079}
2080
2081Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
2082 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2083 // type in LLT and IRTranslator will replace it by the scalar.
2084 if (isVector1(Ty: I.getType()))
2085 return &I;
2086
2087 SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType(),
2088 I.getOperand(i_nocapture: 1)->getType(),
2089 I.getOperand(i_nocapture: 2)->getType()};
2090 IRBuilder<> B(I.getParent());
2091 B.SetInsertPoint(&I);
2092 SmallVector<Value *> Args(I.op_begin(), I.op_end());
2093 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_insertelt, Types: {Types}, Args: {Args});
2094 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2095 return NewI;
2096}
2097
2098Instruction *
2099SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
2100 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2101 // type in LLT and IRTranslator will replace it by the scalar.
2102 if (isVector1(Ty: I.getVectorOperandType()))
2103 return &I;
2104
2105 IRBuilder<> B(I.getParent());
2106 B.SetInsertPoint(&I);
2107 SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
2108 I.getIndexOperand()->getType()};
2109 SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
2110 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_extractelt, Types: {Types}, Args: {Args});
2111 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2112 return NewI;
2113}
2114
2115Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
2116 IRBuilder<> B(I.getParent());
2117 B.SetInsertPoint(&I);
2118 SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
2119 SmallVector<Value *> Args;
2120 Value *AggregateOp = I.getAggregateOperand();
2121 if (isa<UndefValue>(Val: AggregateOp))
2122 Args.push_back(Elt: UndefValue::get(T: B.getInt32Ty()));
2123 else
2124 Args.push_back(Elt: AggregateOp);
2125 Args.push_back(Elt: I.getInsertedValueOperand());
2126 for (auto &Op : I.indices())
2127 Args.push_back(Elt: B.getInt32(C: Op));
2128 Instruction *NewI =
2129 B.CreateIntrinsic(ID: Intrinsic::spv_insertv, Types: {Types}, Args: {Args});
2130 replaceMemInstrUses(Old: &I, New: NewI, B);
2131 return NewI;
2132}
2133
2134Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
2135 if (I.getAggregateOperand()->getType()->isAggregateType())
2136 return &I;
2137 IRBuilder<> B(I.getParent());
2138 B.SetInsertPoint(&I);
2139 SmallVector<Value *> Args(I.operands());
2140 for (auto &Op : I.indices())
2141 Args.push_back(Elt: B.getInt32(C: Op));
2142 auto *NewI =
2143 B.CreateIntrinsic(ID: Intrinsic::spv_extractv, Types: {I.getType()}, Args: {Args});
2144 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2145 return NewI;
2146}
2147
2148Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
2149 if (!I.getType()->isAggregateType())
2150 return &I;
2151 IRBuilder<> B(I.getParent());
2152 B.SetInsertPoint(&I);
2153 TrackConstants = false;
2154 const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
2155 MachineMemOperand::Flags Flags =
2156 TLI->getLoadMemOperandFlags(LI: I, DL: CurrF->getDataLayout());
2157 auto *NewI =
2158 B.CreateIntrinsic(ID: Intrinsic::spv_load, Types: {I.getOperand(i_nocapture: 0)->getType()},
2159 Args: {I.getPointerOperand(), B.getInt16(C: Flags),
2160 B.getInt32(C: I.getAlign().value())});
2161 replaceMemInstrUses(Old: &I, New: NewI, B);
2162 return NewI;
2163}
2164
2165Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
2166 if (!AggrStores.contains(V: &I))
2167 return &I;
2168 IRBuilder<> B(I.getParent());
2169 B.SetInsertPoint(&I);
2170 TrackConstants = false;
2171 const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
2172 MachineMemOperand::Flags Flags =
2173 TLI->getStoreMemOperandFlags(SI: I, DL: CurrF->getDataLayout());
2174 auto *PtrOp = I.getPointerOperand();
2175
2176 if (I.getValueOperand()->getType()->isAggregateType()) {
2177 // It is possible that what used to be an ExtractValueInst has been replaced
2178 // with a call to the spv_extractv intrinsic, and that said call hasn't
2179 // had its return type replaced with i32 during the dedicated pass (because
2180 // it was emitted later); we have to handle this here, because IRTranslator
2181 // cannot deal with multi-register types at the moment.
2182 CallBase *CB = dyn_cast<CallBase>(Val: I.getValueOperand());
2183 assert(CB && CB->getIntrinsicID() == Intrinsic::spv_extractv &&
2184 "Unexpected argument of aggregate type, should be spv_extractv!");
2185 CB->mutateType(Ty: B.getInt32Ty());
2186 }
2187
2188 auto *NewI = B.CreateIntrinsic(
2189 ID: Intrinsic::spv_store, Types: {I.getValueOperand()->getType(), PtrOp->getType()},
2190 Args: {I.getValueOperand(), PtrOp, B.getInt16(C: Flags),
2191 B.getInt32(C: I.getAlign().value())});
2192 NewI->copyMetadata(SrcInst: I);
2193 I.eraseFromParent();
2194 return NewI;
2195}
2196
2197Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
2198 Value *ArraySize = nullptr;
2199 if (I.isArrayAllocation()) {
2200 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*I.getFunction());
2201 if (!STI->canUseExtension(
2202 E: SPIRV::Extension::SPV_INTEL_variable_length_array))
2203 report_fatal_error(
2204 reason: "array allocation: this instruction requires the following "
2205 "SPIR-V extension: SPV_INTEL_variable_length_array",
2206 gen_crash_diag: false);
2207 ArraySize = I.getArraySize();
2208 }
2209 IRBuilder<> B(I.getParent());
2210 B.SetInsertPoint(&I);
2211 TrackConstants = false;
2212 Type *PtrTy = I.getType();
2213 auto *NewI =
2214 ArraySize
2215 ? B.CreateIntrinsic(ID: Intrinsic::spv_alloca_array,
2216 Types: {PtrTy, ArraySize->getType()},
2217 Args: {ArraySize, B.getInt32(C: I.getAlign().value())})
2218 : B.CreateIntrinsic(ID: Intrinsic::spv_alloca, Types: {PtrTy},
2219 Args: {B.getInt32(C: I.getAlign().value())});
2220 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2221 return NewI;
2222}
2223
2224Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2225 assert(I.getType()->isAggregateType() && "Aggregate result is expected");
2226 IRBuilder<> B(I.getParent());
2227 B.SetInsertPoint(&I);
2228 SmallVector<Value *> Args(I.operands());
2229 Args.push_back(Elt: B.getInt32(
2230 C: static_cast<uint32_t>(getMemScope(Ctx&: I.getContext(), Id: I.getSyncScopeID()))));
2231 Args.push_back(Elt: B.getInt32(
2232 C: static_cast<uint32_t>(getMemSemantics(Ord: I.getSuccessOrdering()))));
2233 Args.push_back(Elt: B.getInt32(
2234 C: static_cast<uint32_t>(getMemSemantics(Ord: I.getFailureOrdering()))));
2235 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_cmpxchg,
2236 Types: {I.getPointerOperand()->getType()}, Args: {Args});
2237 replaceMemInstrUses(Old: &I, New: NewI, B);
2238 return NewI;
2239}
2240
2241Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
2242 IRBuilder<> B(I.getParent());
2243 B.SetInsertPoint(&I);
2244 B.CreateIntrinsic(ID: Intrinsic::spv_unreachable, Args: {});
2245 return &I;
2246}
2247
2248static bool
2249shouldEmitIntrinsicsForGlobalValue(const GlobalVariableUsers &GVUsers,
2250 const GlobalVariable &GV,
2251 const Function *F) {
2252 // Skip special artificial variables.
2253 static const StringSet<> ArtificialGlobals{"llvm.global.annotations",
2254 "llvm.compiler.used", "llvm.used"};
2255
2256 if (ArtificialGlobals.contains(key: GV.getName()))
2257 return false;
2258
2259 auto &UserFunctions = GVUsers.getTransitiveUserFunctions(GV);
2260 if (UserFunctions.contains(Ptr: F))
2261 return true;
2262
2263 // Do not emit the intrinsics in this function, it's going to be emitted on
2264 // the functions that reference it.
2265 if (!UserFunctions.empty())
2266 return false;
2267
2268 // Emit definitions for globals that are not referenced by any function on the
2269 // first function definition.
2270 const Module &M = *F->getParent();
2271 const Function &FirstDefinition = *M.getFunctionDefs().begin();
2272 return F == &FirstDefinition;
2273}
2274
2275Value *SPIRVEmitIntrinsics::buildSpvUndefComposite(Type *AggrTy,
2276 IRBuilder<> &B) {
2277 SmallVector<Value *, 4> Elems;
2278 if (auto *ArrTy = dyn_cast<ArrayType>(Val: AggrTy)) {
2279 Type *ElemTy = ArrTy->getElementType();
2280 auto *UI = B.CreateIntrinsic(ID: Intrinsic::spv_undef, Args: {});
2281 AggrConsts[UI] = PoisonValue::get(T: ElemTy);
2282 AggrConstTypes[UI] = ElemTy;
2283 Elems.assign(NumElts: ArrTy->getNumElements(), Elt: UI);
2284 } else {
2285 auto *StructTy = cast<StructType>(Val: AggrTy);
2286 DenseMap<Type *, Instruction *> UndefByType;
2287 for (unsigned I = 0; I < StructTy->getNumElements(); ++I) {
2288 Type *ElemTy = StructTy->getContainedType(i: I);
2289 auto &Entry = UndefByType[ElemTy];
2290 if (!Entry) {
2291 Entry = B.CreateIntrinsic(ID: Intrinsic::spv_undef, Args: {});
2292 AggrConsts[Entry] = PoisonValue::get(T: ElemTy);
2293 AggrConstTypes[Entry] = ElemTy;
2294 }
2295 Elems.push_back(Elt: Entry);
2296 }
2297 }
2298 auto *Composite = B.CreateIntrinsic(ID: Intrinsic::spv_const_composite,
2299 Types: {B.getInt32Ty()}, Args: Elems);
2300 AggrConsts[Composite] = PoisonValue::get(T: AggrTy);
2301 AggrConstTypes[Composite] = AggrTy;
2302 return Composite;
2303}
2304
2305void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV,
2306 IRBuilder<> &B) {
2307
2308 if (!shouldEmitIntrinsicsForGlobalValue(GVUsers, GV, F: CurrF))
2309 return;
2310
2311 Constant *Init = nullptr;
2312 if (hasInitializer(GV: &GV)) {
2313 // Deduce element type and store results in Global Registry.
2314 // Result is ignored, because TypedPointerType is not supported
2315 // by llvm IR general logic.
2316 deduceElementTypeHelper(I: &GV, UnknownElemTypeI8: false);
2317 Init = GV.getInitializer();
2318 Value *InitOp = Init;
2319 if (isa<UndefValue>(Val: Init) && Init->getType()->isAggregateType())
2320 InitOp = buildSpvUndefComposite(AggrTy: Init->getType(), B);
2321 Type *Ty = isAggrConstForceInt32(V: Init) ? B.getInt32Ty() : Init->getType();
2322 Constant *Const = isAggrConstForceInt32(V: Init) ? B.getInt32(C: 1) : Init;
2323 auto *InitInst = B.CreateIntrinsic(ID: Intrinsic::spv_init_global,
2324 Types: {GV.getType(), Ty}, Args: {&GV, Const});
2325 InitInst->setArgOperand(i: 1, v: InitOp);
2326 }
2327 if (!Init && GV.use_empty())
2328 B.CreateIntrinsic(ID: Intrinsic::spv_unref_global, Types: GV.getType(), Args: &GV);
2329}
2330
2331// Return true, if we can't decide what is the pointee type now and will get
2332// back to the question later. Return false is spv_assign_ptr_type is not needed
2333// or can be inserted immediately.
2334bool SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I,
2335 IRBuilder<> &B,
2336 bool UnknownElemTypeI8) {
2337 reportFatalOnTokenType(I);
2338 if (!isPointerTy(T: I->getType()) || !requireAssignType(I))
2339 return false;
2340
2341 setInsertPointAfterDef(B, I);
2342 if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) {
2343 GR->buildAssignPtr(B, ElemTy, Arg: I);
2344 return false;
2345 }
2346 return true;
2347}
2348
2349void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I,
2350 IRBuilder<> &B) {
2351 // TODO: extend the list of functions with known result types
2352 static StringMap<unsigned> ResTypeWellKnown = {
2353 {"async_work_group_copy", WellKnownTypes::Event},
2354 {"async_work_group_strided_copy", WellKnownTypes::Event},
2355 {"__spirv_GroupAsyncCopy", WellKnownTypes::Event}};
2356
2357 reportFatalOnTokenType(I);
2358
2359 bool IsKnown = false;
2360 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
2361 if (!CI->isIndirectCall() && !CI->isInlineAsm() &&
2362 CI->getCalledFunction() && !CI->getCalledFunction()->isIntrinsic()) {
2363 Function *CalledF = CI->getCalledFunction();
2364 std::string DemangledName =
2365 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
2366 FPDecorationId DecorationId = FPDecorationId::NONE;
2367 if (DemangledName.length() > 0)
2368 DemangledName =
2369 SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName, DecorationId: &DecorationId);
2370 auto ResIt = ResTypeWellKnown.find(Key: DemangledName);
2371 if (ResIt != ResTypeWellKnown.end()) {
2372 IsKnown = true;
2373 setInsertPointAfterDef(B, I);
2374 switch (ResIt->second) {
2375 case WellKnownTypes::Event:
2376 GR->buildAssignType(
2377 B, Ty: TargetExtType::get(Context&: I->getContext(), Name: "spirv.Event"), Arg: I);
2378 break;
2379 }
2380 }
2381 // check if a floating rounding mode or saturation info is present
2382 switch (DecorationId) {
2383 default:
2384 break;
2385 case FPDecorationId::SAT:
2386 createSaturatedConversionDecoration(I: CI, B);
2387 break;
2388 case FPDecorationId::RTE:
2389 createRoundingModeDecoration(
2390 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTE, B);
2391 break;
2392 case FPDecorationId::RTZ:
2393 createRoundingModeDecoration(
2394 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTZ, B);
2395 break;
2396 case FPDecorationId::RTP:
2397 createRoundingModeDecoration(
2398 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTP, B);
2399 break;
2400 case FPDecorationId::RTN:
2401 createRoundingModeDecoration(
2402 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTN, B);
2403 break;
2404 }
2405 }
2406 }
2407
2408 Type *Ty = I->getType();
2409 if (!IsKnown && !Ty->isVoidTy() && !isPointerTy(T: Ty) && requireAssignType(I)) {
2410 setInsertPointAfterDef(B, I);
2411 Type *TypeToAssign = Ty;
2412 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2413 if (II->getIntrinsicID() == Intrinsic::spv_const_composite ||
2414 II->getIntrinsicID() == Intrinsic::spv_undef) {
2415 auto It = AggrConstTypes.find(Val: II);
2416 if (It == AggrConstTypes.end())
2417 report_fatal_error(reason: "Unknown composite intrinsic type");
2418 TypeToAssign = It->second;
2419 }
2420 }
2421 TypeToAssign = restoreMutatedType(GR, I, Ty: TypeToAssign);
2422 GR->buildAssignType(B, Ty: TypeToAssign, Arg: I);
2423 }
2424 for (const auto &Op : I->operands()) {
2425 if (isa<ConstantPointerNull>(Val: Op) || isa<UndefValue>(Val: Op) ||
2426 // Check GetElementPtrConstantExpr case.
2427 (isa<ConstantExpr>(Val: Op) &&
2428 (isa<GEPOperator>(Val: Op) ||
2429 (cast<ConstantExpr>(Val: Op)->getOpcode() == CastInst::IntToPtr)))) {
2430 setInsertPointSkippingPhis(B, I);
2431 Type *OpTy = Op->getType();
2432 if (isa<UndefValue>(Val: Op) && OpTy->isAggregateType()) {
2433 CallInst *AssignCI =
2434 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {B.getInt32Ty()}, Arg: Op,
2435 Arg2: UndefValue::get(T: B.getInt32Ty()), Imms: {}, B);
2436 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: AssignCI);
2437 } else if (!isa<Instruction>(Val: Op)) {
2438 Type *OpTy = Op->getType();
2439 Type *OpTyElem = getPointeeType(Ty: OpTy);
2440 if (OpTyElem) {
2441 GR->buildAssignPtr(B, ElemTy: OpTyElem, Arg: Op);
2442 } else if (isPointerTy(T: OpTy)) {
2443 Type *ElemTy = GR->findDeducedElementType(Val: Op);
2444 GR->buildAssignPtr(B, ElemTy: ElemTy ? ElemTy : deduceElementType(I: Op, UnknownElemTypeI8: true),
2445 Arg: Op);
2446 } else {
2447 Value *OpTyVal = Op;
2448 if (OpTy->isTargetExtTy()) {
2449 // We need to do this in order to be consistent with how target ext
2450 // types are handled in `processInstrAfterVisit`
2451 OpTyVal = getNormalizedPoisonValue(Ty: OpTy);
2452 }
2453 CallInst *AssignCI =
2454 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {OpTy},
2455 Arg: getNormalizedPoisonValue(Ty: OpTy), Arg2: OpTyVal, Imms: {}, B);
2456 GR->addAssignPtrTypeInstr(Val: OpTyVal, AssignPtrTyCI: AssignCI);
2457 }
2458 }
2459 }
2460 }
2461}
2462
2463bool SPIRVEmitIntrinsics::shouldTryToAddMemAliasingDecoration(
2464 Instruction *Inst) {
2465 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*Inst->getFunction());
2466 if (!STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing))
2467 return false;
2468 // Add aliasing decorations to internal load and store intrinsics
2469 // and atomic instructions, skipping atomic store as it won't have ID to
2470 // attach the decoration.
2471 CallInst *CI = dyn_cast<CallInst>(Val: Inst);
2472 if (!CI)
2473 return false;
2474 if (Function *Fun = CI->getCalledFunction()) {
2475 if (Fun->isIntrinsic()) {
2476 switch (Fun->getIntrinsicID()) {
2477 case Intrinsic::spv_load:
2478 case Intrinsic::spv_store:
2479 return true;
2480 default:
2481 return false;
2482 }
2483 }
2484 std::string Name = getOclOrSpirvBuiltinDemangledName(Name: Fun->getName());
2485 const std::string Prefix = "__spirv_Atomic";
2486 const bool IsAtomic = Name.find(str: Prefix) == 0;
2487
2488 if (!Fun->getReturnType()->isVoidTy() && IsAtomic)
2489 return true;
2490 }
2491 return false;
2492}
2493
2494void SPIRVEmitIntrinsics::insertSpirvDecorations(Instruction *I,
2495 IRBuilder<> &B) {
2496 if (MDNode *MD = I->getMetadata(Kind: "spirv.Decorations")) {
2497 setInsertPointAfterDef(B, I);
2498 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, Types: {I->getType()},
2499 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
2500 }
2501 // Lower alias.scope/noalias metadata
2502 {
2503 auto processMemAliasingDecoration = [&](unsigned Kind) {
2504 if (MDNode *AliasListMD = I->getMetadata(KindID: Kind)) {
2505 if (shouldTryToAddMemAliasingDecoration(Inst: I)) {
2506 uint32_t Dec = Kind == LLVMContext::MD_alias_scope
2507 ? SPIRV::Decoration::AliasScopeINTEL
2508 : SPIRV::Decoration::NoAliasINTEL;
2509 SmallVector<Value *, 3> Args = {
2510 I, ConstantInt::get(Ty: B.getInt32Ty(), V: Dec),
2511 MetadataAsValue::get(Context&: I->getContext(), MD: AliasListMD)};
2512 setInsertPointAfterDef(B, I);
2513 B.CreateIntrinsic(ID: Intrinsic::spv_assign_aliasing_decoration,
2514 Types: {I->getType()}, Args: {Args});
2515 }
2516 }
2517 };
2518 processMemAliasingDecoration(LLVMContext::MD_alias_scope);
2519 processMemAliasingDecoration(LLVMContext::MD_noalias);
2520 }
2521 // MD_fpmath
2522 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_fpmath)) {
2523 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*I->getFunction());
2524 bool AllowFPMaxError =
2525 STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_fp_max_error);
2526 if (!AllowFPMaxError)
2527 return;
2528
2529 setInsertPointAfterDef(B, I);
2530 B.CreateIntrinsic(ID: Intrinsic::spv_assign_fpmaxerror_decoration,
2531 Types: {I->getType()},
2532 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
2533 }
2534}
2535
2536static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
2537 const Module &M,
2538 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
2539 &FPFastMathDefaultInfoMap,
2540 Function *F) {
2541 auto it = FPFastMathDefaultInfoMap.find(Val: F);
2542 if (it != FPFastMathDefaultInfoMap.end())
2543 return it->second;
2544
2545 // If the map does not contain the entry, create a new one. Initialize it to
2546 // contain all 3 elements sorted by bit width of target type: {half, float,
2547 // double}.
2548 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
2549 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
2550 Args: SPIRV::FPFastMathMode::None);
2551 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
2552 Args: SPIRV::FPFastMathMode::None);
2553 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
2554 Args: SPIRV::FPFastMathMode::None);
2555 return FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
2556}
2557
2558static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
2559 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
2560 const Type *Ty) {
2561 size_t BitWidth = Ty->getScalarSizeInBits();
2562 int Index =
2563 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
2564 BitWidth);
2565 assert(Index >= 0 && Index < 3 &&
2566 "Expected FPFastMathDefaultInfo for half, float, or double");
2567 assert(FPFastMathDefaultInfoVec.size() == 3 &&
2568 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
2569 return FPFastMathDefaultInfoVec[Index];
2570}
2571
2572void SPIRVEmitIntrinsics::insertConstantsForFPFastMathDefault(Module &M) {
2573 const SPIRVSubtarget *ST = TM->getSubtargetImpl();
2574 if (!ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
2575 return;
2576
2577 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
2578 // We need the entry point (function) as the key, and the target
2579 // type and flags as the value.
2580 // We also need to check ContractionOff and SignedZeroInfNanPreserve
2581 // execution modes, as they are now deprecated and must be replaced
2582 // with FPFastMathDefaultInfo.
2583 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
2584 if (!Node) {
2585 if (!M.getNamedMetadata(Name: "opencl.enable.FP_CONTRACT")) {
2586 // This requires emitting ContractionOff. However, because
2587 // ContractionOff is now deprecated, we need to replace it with
2588 // FPFastMathDefaultInfo with FP Fast Math Mode bitmask set to all 0.
2589 // We need to create the constant for that.
2590
2591 // Create constant instruction with the bitmask flags.
2592 Constant *InitValue =
2593 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: 0);
2594 // TODO: Reuse constant if there is one already with the required
2595 // value.
2596 [[maybe_unused]] GlobalVariable *GV =
2597 new GlobalVariable(M, // Module
2598 Type::getInt32Ty(C&: M.getContext()), // Type
2599 true, // isConstant
2600 GlobalValue::InternalLinkage, // Linkage
2601 InitValue // Initializer
2602 );
2603 }
2604 return;
2605 }
2606
2607 // The table maps function pointers to their default FP fast math info. It
2608 // can be assumed that the SmallVector is sorted by the bit width of the
2609 // type. The first element is the smallest bit width, and the last element
2610 // is the largest bit width, therefore, we will have {half, float, double}
2611 // in the order of their bit widths.
2612 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
2613 FPFastMathDefaultInfoMap;
2614
2615 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
2616 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
2617 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
2618 Function *F = cast<Function>(
2619 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
2620 const auto EM =
2621 cast<ConstantInt>(
2622 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
2623 ->getZExtValue();
2624 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
2625 assert(MDN->getNumOperands() == 4 &&
2626 "Expected 4 operands for FPFastMathDefault");
2627 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
2628 unsigned Flags =
2629 cast<ConstantInt>(
2630 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
2631 ->getZExtValue();
2632 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2633 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
2634 SPIRV::FPFastMathDefaultInfo &Info =
2635 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
2636 Info.FastMathFlags = Flags;
2637 Info.FPFastMathDefault = true;
2638 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
2639 assert(MDN->getNumOperands() == 2 &&
2640 "Expected no operands for ContractionOff");
2641
2642 // We need to save this info for every possible FP type, i.e. {half,
2643 // float, double, fp128}.
2644 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2645 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
2646 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
2647 Info.ContractionOff = true;
2648 }
2649 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
2650 assert(MDN->getNumOperands() == 3 &&
2651 "Expected 1 operand for SignedZeroInfNanPreserve");
2652 unsigned TargetWidth =
2653 cast<ConstantInt>(
2654 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
2655 ->getZExtValue();
2656 // We need to save this info only for the FP type with TargetWidth.
2657 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2658 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
2659 int Index = SPIRV::FPFastMathDefaultInfoVector::
2660 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
2661 assert(Index >= 0 && Index < 3 &&
2662 "Expected FPFastMathDefaultInfo for half, float, or double");
2663 assert(FPFastMathDefaultInfoVec.size() == 3 &&
2664 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
2665 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
2666 }
2667 }
2668
2669 std::unordered_map<unsigned, GlobalVariable *> GlobalVars;
2670 for (auto &[Func, FPFastMathDefaultInfoVec] : FPFastMathDefaultInfoMap) {
2671 if (FPFastMathDefaultInfoVec.empty())
2672 continue;
2673
2674 for (const SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
2675 assert(Info.Ty && "Expected target type for FPFastMathDefaultInfo");
2676 // Skip if none of the execution modes was used.
2677 unsigned Flags = Info.FastMathFlags;
2678 if (Flags == SPIRV::FPFastMathMode::None && !Info.ContractionOff &&
2679 !Info.SignedZeroInfNanPreserve && !Info.FPFastMathDefault)
2680 continue;
2681
2682 // Check if flags are compatible.
2683 if (Info.ContractionOff && (Flags & SPIRV::FPFastMathMode::AllowContract))
2684 report_fatal_error(reason: "Conflicting FPFastMathFlags: ContractionOff "
2685 "and AllowContract");
2686
2687 if (Info.SignedZeroInfNanPreserve &&
2688 !(Flags &
2689 (SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
2690 SPIRV::FPFastMathMode::NSZ))) {
2691 if (Info.FPFastMathDefault)
2692 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
2693 "SignedZeroInfNanPreserve but at least one of "
2694 "NotNaN/NotInf/NSZ is enabled.");
2695 }
2696
2697 if ((Flags & SPIRV::FPFastMathMode::AllowTransform) &&
2698 !((Flags & SPIRV::FPFastMathMode::AllowReassoc) &&
2699 (Flags & SPIRV::FPFastMathMode::AllowContract))) {
2700 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
2701 "AllowTransform requires AllowReassoc and "
2702 "AllowContract to be set.");
2703 }
2704
2705 auto it = GlobalVars.find(x: Flags);
2706 GlobalVariable *GV = nullptr;
2707 if (it != GlobalVars.end()) {
2708 // Reuse existing global variable.
2709 GV = it->second;
2710 } else {
2711 // Create constant instruction with the bitmask flags.
2712 Constant *InitValue =
2713 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: Flags);
2714 // TODO: Reuse constant if there is one already with the required
2715 // value.
2716 GV = new GlobalVariable(M, // Module
2717 Type::getInt32Ty(C&: M.getContext()), // Type
2718 true, // isConstant
2719 GlobalValue::InternalLinkage, // Linkage
2720 InitValue // Initializer
2721 );
2722 GlobalVars[Flags] = GV;
2723 }
2724 }
2725 }
2726}
2727
2728void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
2729 IRBuilder<> &B) {
2730 auto *II = dyn_cast<IntrinsicInst>(Val: I);
2731 bool IsConstComposite =
2732 II && II->getIntrinsicID() == Intrinsic::spv_const_composite;
2733 if (IsConstComposite && TrackConstants) {
2734 setInsertPointAfterDef(B, I);
2735 auto t = AggrConsts.find(Val: I);
2736 assert(t != AggrConsts.end());
2737 auto *NewOp =
2738 buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
2739 Types: {II->getType(), II->getType()}, Arg: t->second, Arg2: I, Imms: {}, B);
2740 replaceAllUsesWith(Src: I, Dest: NewOp, DeleteOld: false);
2741 NewOp->setArgOperand(i: 0, v: I);
2742 }
2743 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
2744 for (const auto &Op : I->operands()) {
2745 if (isa<PHINode>(Val: I) || isa<SwitchInst>(Val: I) ||
2746 !(isa<ConstantData>(Val: Op) || isa<ConstantExpr>(Val: Op)))
2747 continue;
2748 unsigned OpNo = Op.getOperandNo();
2749 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
2750 (!II->isBundleOperand(Idx: OpNo) &&
2751 II->paramHasAttr(ArgNo: OpNo, Kind: Attribute::ImmArg))))
2752 continue;
2753
2754 if (!BPrepared) {
2755 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
2756 : B.SetInsertPoint(I);
2757 BPrepared = true;
2758 }
2759 Type *OpTy = Op->getType();
2760 Type *OpElemTy = GR->findDeducedElementType(Val: Op);
2761 Value *NewOp = Op;
2762 if (OpTy->isTargetExtTy()) {
2763 // Since this value is replaced by poison, we need to do the same in
2764 // `insertAssignTypeIntrs`.
2765 Value *OpTyVal = getNormalizedPoisonValue(Ty: OpTy);
2766 NewOp = buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
2767 Types: {OpTy, OpTyVal->getType()}, Arg: Op, Arg2: OpTyVal, Imms: {}, B);
2768 }
2769 if (!IsConstComposite && isPointerTy(T: OpTy) && OpElemTy != nullptr &&
2770 OpElemTy != IntegerType::getInt8Ty(C&: I->getContext())) {
2771 SmallVector<Type *, 2> Types = {OpTy, OpTy};
2772 SmallVector<Value *, 2> Args = {
2773 NewOp, buildMD(Arg: getNormalizedPoisonValue(Ty: OpElemTy)),
2774 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
2775 CallInst *PtrCasted =
2776 B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
2777 GR->buildAssignPtr(B, ElemTy: OpElemTy, Arg: PtrCasted);
2778 NewOp = PtrCasted;
2779 }
2780 if (NewOp != Op)
2781 I->setOperand(i: OpNo, Val: NewOp);
2782 }
2783 if (Named.insert(x: I).second)
2784 emitAssignName(I, B);
2785}
2786
2787Type *SPIRVEmitIntrinsics::deduceFunParamElementType(Function *F,
2788 unsigned OpIdx) {
2789 std::unordered_set<Function *> FVisited;
2790 return deduceFunParamElementType(F, OpIdx, FVisited);
2791}
2792
2793Type *SPIRVEmitIntrinsics::deduceFunParamElementType(
2794 Function *F, unsigned OpIdx, std::unordered_set<Function *> &FVisited) {
2795 // maybe a cycle
2796 if (!FVisited.insert(x: F).second)
2797 return nullptr;
2798
2799 std::unordered_set<Value *> Visited;
2800 SmallVector<std::pair<Function *, unsigned>> Lookup;
2801 // search in function's call sites
2802 for (User *U : F->users()) {
2803 CallInst *CI = dyn_cast<CallInst>(Val: U);
2804 if (!CI || OpIdx >= CI->arg_size())
2805 continue;
2806 Value *OpArg = CI->getArgOperand(i: OpIdx);
2807 if (!isPointerTy(T: OpArg->getType()))
2808 continue;
2809 // maybe we already know operand's element type
2810 if (Type *KnownTy = GR->findDeducedElementType(Val: OpArg))
2811 return KnownTy;
2812 // try to deduce from the operand itself
2813 Visited.clear();
2814 if (Type *Ty = deduceElementTypeHelper(I: OpArg, Visited, UnknownElemTypeI8: false))
2815 return Ty;
2816 // search in actual parameter's users
2817 for (User *OpU : OpArg->users()) {
2818 Instruction *Inst = dyn_cast<Instruction>(Val: OpU);
2819 if (!Inst || Inst == CI)
2820 continue;
2821 Visited.clear();
2822 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8: false))
2823 return Ty;
2824 }
2825 // check if it's a formal parameter of the outer function
2826 if (!CI->getParent() || !CI->getParent()->getParent())
2827 continue;
2828 Function *OuterF = CI->getParent()->getParent();
2829 if (FVisited.find(x: OuterF) != FVisited.end())
2830 continue;
2831 for (unsigned i = 0; i < OuterF->arg_size(); ++i) {
2832 if (OuterF->getArg(i) == OpArg) {
2833 Lookup.push_back(Elt: std::make_pair(x&: OuterF, y&: i));
2834 break;
2835 }
2836 }
2837 }
2838
2839 // search in function parameters
2840 for (auto &Pair : Lookup) {
2841 if (Type *Ty = deduceFunParamElementType(F: Pair.first, OpIdx: Pair.second, FVisited))
2842 return Ty;
2843 }
2844
2845 return nullptr;
2846}
2847
2848void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F,
2849 IRBuilder<> &B) {
2850 B.SetInsertPointPastAllocas(F);
2851 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
2852 Argument *Arg = F->getArg(i: OpIdx);
2853 if (!isUntypedPointerTy(T: Arg->getType()))
2854 continue;
2855 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
2856 if (ElemTy)
2857 continue;
2858 if (hasPointeeTypeAttr(Arg) &&
2859 (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) {
2860 GR->buildAssignPtr(B, ElemTy, Arg);
2861 continue;
2862 }
2863 // search in function's call sites
2864 for (User *U : F->users()) {
2865 CallInst *CI = dyn_cast<CallInst>(Val: U);
2866 if (!CI || OpIdx >= CI->arg_size())
2867 continue;
2868 Value *OpArg = CI->getArgOperand(i: OpIdx);
2869 if (!isPointerTy(T: OpArg->getType()))
2870 continue;
2871 // maybe we already know operand's element type
2872 if ((ElemTy = GR->findDeducedElementType(Val: OpArg)) != nullptr)
2873 break;
2874 }
2875 if (ElemTy) {
2876 GR->buildAssignPtr(B, ElemTy, Arg);
2877 continue;
2878 }
2879 if (HaveFunPtrs) {
2880 for (User *U : Arg->users()) {
2881 CallInst *CI = dyn_cast<CallInst>(Val: U);
2882 if (CI && !isa<IntrinsicInst>(Val: CI) && CI->isIndirectCall() &&
2883 CI->getCalledOperand() == Arg &&
2884 CI->getParent()->getParent() == CurrF) {
2885 SmallVector<std::pair<Value *, unsigned>> Ops;
2886 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy&: ElemTy, IsPostprocessing: false);
2887 if (ElemTy) {
2888 GR->buildAssignPtr(B, ElemTy, Arg);
2889 break;
2890 }
2891 }
2892 }
2893 }
2894 }
2895}
2896
2897void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) {
2898 B.SetInsertPointPastAllocas(F);
2899 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
2900 Argument *Arg = F->getArg(i: OpIdx);
2901 if (!isUntypedPointerTy(T: Arg->getType()))
2902 continue;
2903 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
2904 if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) {
2905 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Arg)) {
2906 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
2907 GR->updateAssignType(AssignCI, Arg, OfType: getNormalizedPoisonValue(Ty: ElemTy));
2908 propagateElemType(Op: Arg, ElemTy: IntegerType::getInt8Ty(C&: F->getContext()),
2909 VisitedSubst);
2910 } else {
2911 GR->buildAssignPtr(B, ElemTy, Arg);
2912 }
2913 }
2914 }
2915}
2916
2917static FunctionType *getFunctionPointerElemType(Function *F,
2918 SPIRVGlobalRegistry *GR) {
2919 FunctionType *FTy = F->getFunctionType();
2920 bool IsNewFTy = false;
2921 SmallVector<Type *, 4> ArgTys;
2922 for (Argument &Arg : F->args()) {
2923 Type *ArgTy = Arg.getType();
2924 if (ArgTy->isPointerTy())
2925 if (Type *ElemTy = GR->findDeducedElementType(Val: &Arg)) {
2926 IsNewFTy = true;
2927 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
2928 }
2929 ArgTys.push_back(Elt: ArgTy);
2930 }
2931 return IsNewFTy
2932 ? FunctionType::get(Result: FTy->getReturnType(), Params: ArgTys, isVarArg: FTy->isVarArg())
2933 : FTy;
2934}
2935
2936bool SPIRVEmitIntrinsics::processFunctionPointers(Module &M) {
2937 SmallVector<Function *> Worklist;
2938 for (auto &F : M) {
2939 if (F.isIntrinsic())
2940 continue;
2941 if (F.isDeclaration()) {
2942 for (User *U : F.users()) {
2943 CallInst *CI = dyn_cast<CallInst>(Val: U);
2944 if (!CI || CI->getCalledFunction() != &F) {
2945 Worklist.push_back(Elt: &F);
2946 break;
2947 }
2948 }
2949 } else {
2950 if (F.user_empty())
2951 continue;
2952 Type *FPElemTy = GR->findDeducedElementType(Val: &F);
2953 if (!FPElemTy)
2954 FPElemTy = getFunctionPointerElemType(F: &F, GR);
2955 for (User *U : F.users()) {
2956 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
2957 if (!II || II->arg_size() != 3 || II->getOperand(i_nocapture: 0) != &F)
2958 continue;
2959 if (II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type ||
2960 II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
2961 GR->updateAssignType(AssignCI: II, Arg: &F, OfType: getNormalizedPoisonValue(Ty: FPElemTy));
2962 break;
2963 }
2964 }
2965 }
2966 }
2967 if (Worklist.empty())
2968 return false;
2969
2970 LLVMContext &Ctx = M.getContext();
2971 Function *SF = getOrCreateBackendServiceFunction(M);
2972 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: SF);
2973 IRBuilder<> IRB(BB);
2974
2975 for (Function *F : Worklist) {
2976 SmallVector<Value *> Args;
2977 for (const auto &Arg : F->args())
2978 Args.push_back(Elt: getNormalizedPoisonValue(Ty: Arg.getType()));
2979 IRB.CreateCall(Callee: F, Args);
2980 }
2981 IRB.CreateRetVoid();
2982
2983 return true;
2984}
2985
2986// Apply types parsed from demangled function declarations.
2987void SPIRVEmitIntrinsics::applyDemangledPtrArgTypes(IRBuilder<> &B) {
2988 DenseMap<Function *, CallInst *> Ptrcasts;
2989 for (auto It : FDeclPtrTys) {
2990 Function *F = It.first;
2991 for (auto *U : F->users()) {
2992 CallInst *CI = dyn_cast<CallInst>(Val: U);
2993 if (!CI || CI->getCalledFunction() != F)
2994 continue;
2995 unsigned Sz = CI->arg_size();
2996 for (auto [Idx, ElemTy] : It.second) {
2997 if (Idx >= Sz)
2998 continue;
2999 Value *Param = CI->getArgOperand(i: Idx);
3000 if (GR->findDeducedElementType(Val: Param) || isa<GlobalValue>(Val: Param))
3001 continue;
3002 if (Argument *Arg = dyn_cast<Argument>(Val: Param)) {
3003 if (!hasPointeeTypeAttr(Arg)) {
3004 B.SetInsertPointPastAllocas(Arg->getParent());
3005 B.SetCurrentDebugLocation(DebugLoc());
3006 GR->buildAssignPtr(B, ElemTy, Arg);
3007 }
3008 } else if (isaGEP(V: Param)) {
3009 replaceUsesOfWithSpvPtrcast(Op: Param, ElemTy: normalizeType(Ty: ElemTy), I: CI,
3010 Ptrcasts);
3011 } else if (isa<Instruction>(Val: Param)) {
3012 GR->addDeducedElementType(Val: Param, Ty: normalizeType(Ty: ElemTy));
3013 // insertAssignTypeIntrs() will complete buildAssignPtr()
3014 } else {
3015 B.SetInsertPoint(CI->getParent()
3016 ->getParent()
3017 ->getEntryBlock()
3018 .getFirstNonPHIOrDbgOrAlloca());
3019 GR->buildAssignPtr(B, ElemTy, Arg: Param);
3020 }
3021 CallInst *Ref = dyn_cast<CallInst>(Val: Param);
3022 if (!Ref)
3023 continue;
3024 Function *RefF = Ref->getCalledFunction();
3025 if (!RefF || !isPointerTy(T: RefF->getReturnType()) ||
3026 GR->findDeducedElementType(Val: RefF))
3027 continue;
3028 ElemTy = normalizeType(Ty: ElemTy);
3029 GR->addDeducedElementType(Val: RefF, Ty: ElemTy);
3030 GR->addReturnType(
3031 ArgF: RefF, DerivedTy: TypedPointerType::get(
3032 ElementType: ElemTy, AddressSpace: getPointerAddressSpace(T: RefF->getReturnType())));
3033 }
3034 }
3035 }
3036}
3037
3038GetElementPtrInst *
3039SPIRVEmitIntrinsics::simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP) {
3040 // getelementptr [0 x T], P, 0 (zero), I -> getelementptr T, P, I.
3041 // If type is 0-length array and first index is 0 (zero), drop both the
3042 // 0-length array type and the first index. This is a common pattern in
3043 // the IR, e.g. when using a zero-length array as a placeholder for a
3044 // flexible array such as unbound arrays.
3045 assert(GEP && "GEP is null");
3046 Type *SrcTy = GEP->getSourceElementType();
3047 SmallVector<Value *, 8> Indices(GEP->indices());
3048 ArrayType *ArrTy = dyn_cast<ArrayType>(Val: SrcTy);
3049 if (ArrTy && ArrTy->getNumElements() == 0 &&
3050 PatternMatch::match(V: Indices[0], P: PatternMatch::m_Zero())) {
3051 Indices.erase(CI: Indices.begin());
3052 SrcTy = ArrTy->getElementType();
3053 return GetElementPtrInst::Create(PointeeType: SrcTy, Ptr: GEP->getPointerOperand(), IdxList: Indices,
3054 NW: GEP->getNoWrapFlags(), NameStr: "",
3055 InsertBefore: GEP->getIterator());
3056 }
3057 return nullptr;
3058}
3059
3060void SPIRVEmitIntrinsics::emitUnstructuredLoopControls(Function &F,
3061 IRBuilder<> &B) {
3062 const SPIRVSubtarget *ST = TM->getSubtargetImpl(F);
3063 // Shaders use SPIRVStructurizer which emits OpLoopMerge via spv_loop_merge.
3064 if (ST->isShader())
3065 return;
3066 if (!ST->canUseExtension(
3067 E: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls))
3068 return;
3069
3070 for (BasicBlock &BB : F) {
3071 Instruction *Term = BB.getTerminator();
3072 MDNode *LoopMD = Term->getMetadata(KindID: LLVMContext::MD_loop);
3073 if (!LoopMD)
3074 continue;
3075
3076 SmallVector<unsigned, 1> Ops =
3077 getSpirvLoopControlOperandsFromLoopMetadata(LoopMD);
3078 unsigned LC = Ops[0];
3079 if (LC == SPIRV::LoopControl::None)
3080 continue;
3081
3082 // Emit intrinsic: loop control mask + optional parameters.
3083 B.SetInsertPoint(Term);
3084 SmallVector<Value *, 4> IntrArgs;
3085 IntrArgs.push_back(Elt: B.getInt32(C: LC));
3086 for (unsigned I = 1; I < Ops.size(); ++I)
3087 IntrArgs.push_back(Elt: B.getInt32(C: Ops[I]));
3088 B.CreateIntrinsic(ID: Intrinsic::spv_loop_control_intel, Args: IntrArgs);
3089 }
3090}
3091
3092bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
3093 if (Func.isDeclaration())
3094 return false;
3095
3096 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F: Func);
3097 GR = ST.getSPIRVGlobalRegistry();
3098
3099 if (!CurrF)
3100 HaveFunPtrs =
3101 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers);
3102
3103 CurrF = &Func;
3104 IRBuilder<> B(Func.getContext());
3105 AggrConsts.clear();
3106 AggrConstTypes.clear();
3107 AggrStores.clear();
3108
3109 // Fix GEP result types ahead of inference, and simplify if possible.
3110 // Data structure for dead instructions that were simplified and replaced.
3111 SmallPtrSet<Instruction *, 4> DeadInsts;
3112 for (auto &I : instructions(F&: Func)) {
3113 auto *GEP = dyn_cast<GetElementPtrInst>(Val: &I);
3114 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
3115
3116 if ((!GEP && !SGEP) || GR->findDeducedElementType(Val: &I))
3117 continue;
3118
3119 if (SGEP) {
3120 GR->addDeducedElementType(Val: SGEP,
3121 Ty: normalizeType(Ty: SGEP->getResultElementType()));
3122 continue;
3123 }
3124
3125 GetElementPtrInst *NewGEP = simplifyZeroLengthArrayGepInst(GEP);
3126 if (NewGEP) {
3127 GEP->replaceAllUsesWith(V: NewGEP);
3128 DeadInsts.insert(Ptr: GEP);
3129 GEP = NewGEP;
3130 }
3131 if (Type *GepTy = getGEPType(Ref: GEP))
3132 GR->addDeducedElementType(Val: GEP, Ty: normalizeType(Ty: GepTy));
3133 }
3134 // Remove dead instructions that were simplified and replaced.
3135 for (auto *I : DeadInsts) {
3136 assert(I->use_empty() && "Dead instruction should not have any uses left");
3137 I->eraseFromParent();
3138 }
3139
3140 processParamTypesByFunHeader(F: CurrF, B);
3141
3142 // StoreInst's operand type can be changed during the next
3143 // transformations, so we need to store it in the set. Also store already
3144 // transformed types.
3145 for (auto &I : instructions(F&: Func)) {
3146 StoreInst *SI = dyn_cast<StoreInst>(Val: &I);
3147 if (!SI)
3148 continue;
3149 Type *ElTy = SI->getValueOperand()->getType();
3150 if (ElTy->isAggregateType() || ElTy->isVectorTy())
3151 AggrStores.insert(V: &I);
3152 }
3153
3154 B.SetInsertPoint(TheBB: &Func.getEntryBlock(), IP: Func.getEntryBlock().begin());
3155 for (auto &GV : Func.getParent()->globals())
3156 processGlobalValue(GV, B);
3157
3158 preprocessUndefs(B);
3159 preprocessCompositeConstants(B);
3160 SmallVector<Instruction *> Worklist(
3161 llvm::make_pointer_range(Range: instructions(F&: Func)));
3162
3163 applyDemangledPtrArgTypes(B);
3164
3165 // Pass forward: use operand to deduce instructions result.
3166 for (auto &I : Worklist) {
3167 // Don't emit intrinsincs for convergence intrinsics.
3168 if (isConvergenceIntrinsic(I))
3169 continue;
3170
3171 bool Postpone = insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: false);
3172 // if Postpone is true, we can't decide on pointee type yet
3173 insertAssignTypeIntrs(I, B);
3174 insertPtrCastOrAssignTypeInstr(I, B);
3175 insertSpirvDecorations(I, B);
3176 // if instruction requires a pointee type set, let's check if we know it
3177 // already, and force it to be i8 if not
3178 if (Postpone && !GR->findAssignPtrTypeInstr(Val: I))
3179 insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: true);
3180
3181 if (auto *FPI = dyn_cast<ConstrainedFPIntrinsic>(Val: I))
3182 useRoundingMode(FPI, B);
3183 }
3184
3185 // Pass backward: use instructions results to specify/update/cast operands
3186 // where needed.
3187 SmallPtrSet<Instruction *, 4> IncompleteRets;
3188 for (auto &I : llvm::reverse(C: instructions(F&: Func)))
3189 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets);
3190
3191 // Pass forward for PHIs only, their operands are not preceed the
3192 // instruction in meaning of `instructions(Func)`.
3193 for (BasicBlock &BB : Func)
3194 for (PHINode &Phi : BB.phis())
3195 if (isPointerTy(T: Phi.getType()))
3196 deduceOperandElementType(I: &Phi, IncompleteRets: nullptr);
3197
3198 for (auto *I : Worklist) {
3199 TrackConstants = true;
3200 if (!I->getType()->isVoidTy() || isa<StoreInst>(Val: I))
3201 setInsertPointAfterDef(B, I);
3202 // Visitors return either the original/newly created instruction for
3203 // further processing, nullptr otherwise.
3204 I = visit(I&: *I);
3205 if (!I)
3206 continue;
3207
3208 // Don't emit intrinsics for convergence operations.
3209 if (isConvergenceIntrinsic(I))
3210 continue;
3211
3212 addSaturatedDecorationToIntrinsic(I, B);
3213 processInstrAfterVisit(I, B);
3214 }
3215
3216 emitUnstructuredLoopControls(F&: Func, B);
3217
3218 return true;
3219}
3220
3221// Try to deduce a better type for pointers to untyped ptr.
3222bool SPIRVEmitIntrinsics::postprocessTypes(Module &M) {
3223 if (!GR || TodoTypeSz == 0)
3224 return false;
3225
3226 unsigned SzTodo = TodoTypeSz;
3227 DenseMap<Value *, SmallPtrSet<Value *, 4>> ToProcess;
3228 for (auto [Op, Enabled] : TodoType) {
3229 // TODO: add isa<CallInst>(Op) to continue
3230 if (!Enabled || isaGEP(V: Op))
3231 continue;
3232 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Op);
3233 Type *KnownTy = GR->findDeducedElementType(Val: Op);
3234 if (!KnownTy || !AssignCI)
3235 continue;
3236 assert(Op == AssignCI->getArgOperand(0));
3237 // Try to improve the type deduced after all Functions are processed.
3238 if (auto *CI = dyn_cast<Instruction>(Val: Op)) {
3239 CurrF = CI->getParent()->getParent();
3240 std::unordered_set<Value *> Visited;
3241 if (Type *ElemTy = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8: false, IgnoreKnownType: true)) {
3242 if (ElemTy != KnownTy) {
3243 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3244 propagateElemType(Op: CI, ElemTy, VisitedSubst);
3245 eraseTodoType(Op);
3246 continue;
3247 }
3248 }
3249 }
3250
3251 if (Op->hasUseList()) {
3252 for (User *U : Op->users()) {
3253 Instruction *Inst = dyn_cast<Instruction>(Val: U);
3254 if (Inst && !isa<IntrinsicInst>(Val: Inst))
3255 ToProcess[Inst].insert(Ptr: Op);
3256 }
3257 }
3258 }
3259 if (TodoTypeSz == 0)
3260 return true;
3261
3262 for (auto &F : M) {
3263 CurrF = &F;
3264 SmallPtrSet<Instruction *, 4> IncompleteRets;
3265 for (auto &I : llvm::reverse(C: instructions(F))) {
3266 auto It = ToProcess.find(Val: &I);
3267 if (It == ToProcess.end())
3268 continue;
3269 It->second.remove_if(P: [this](Value *V) { return !isTodoType(Op: V); });
3270 if (It->second.size() == 0)
3271 continue;
3272 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets, AskOps: &It->second, IsPostprocessing: true);
3273 if (TodoTypeSz == 0)
3274 return true;
3275 }
3276 }
3277
3278 return SzTodo > TodoTypeSz;
3279}
3280
3281// Parse and store argument types of function declarations where needed.
3282void SPIRVEmitIntrinsics::parseFunDeclarations(Module &M) {
3283 for (auto &F : M) {
3284 if (!F.isDeclaration() || F.isIntrinsic())
3285 continue;
3286 // get the demangled name
3287 std::string DemangledName = getOclOrSpirvBuiltinDemangledName(Name: F.getName());
3288 if (DemangledName.empty())
3289 continue;
3290 // allow only OpGroupAsyncCopy use case at the moment
3291 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F);
3292 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
3293 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
3294 if (Opcode != SPIRV::OpGroupAsyncCopy)
3295 continue;
3296 // find pointer arguments
3297 SmallVector<unsigned> Idxs;
3298 for (unsigned OpIdx = 0; OpIdx < F.arg_size(); ++OpIdx) {
3299 Argument *Arg = F.getArg(i: OpIdx);
3300 if (isPointerTy(T: Arg->getType()) && !hasPointeeTypeAttr(Arg))
3301 Idxs.push_back(Elt: OpIdx);
3302 }
3303 if (!Idxs.size())
3304 continue;
3305 // parse function arguments
3306 LLVMContext &Ctx = F.getContext();
3307 SmallVector<StringRef, 10> TypeStrs;
3308 SPIRV::parseBuiltinTypeStr(BuiltinArgsTypeStrs&: TypeStrs, DemangledCall: DemangledName, Ctx);
3309 if (!TypeStrs.size())
3310 continue;
3311 // find type info for pointer arguments
3312 for (unsigned Idx : Idxs) {
3313 if (Idx >= TypeStrs.size())
3314 continue;
3315 if (Type *ElemTy =
3316 SPIRV::parseBuiltinCallArgumentType(TypeStr: TypeStrs[Idx].trim(), Ctx))
3317 if (TypedPointerType::isValidElementType(ElemTy) &&
3318 !ElemTy->isTargetExtTy())
3319 FDeclPtrTys[&F].push_back(Elt: std::make_pair(x&: Idx, y&: ElemTy));
3320 }
3321 }
3322}
3323
3324bool SPIRVEmitIntrinsics::processMaskedMemIntrinsic(IntrinsicInst &I) {
3325 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
3326
3327 if (I.getIntrinsicID() == Intrinsic::masked_gather) {
3328 if (!ST.canUseExtension(
3329 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
3330 I.getContext().emitError(
3331 I: &I, ErrorStr: "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter "
3332 "extension");
3333 // Replace with poison to allow compilation to continue and report error.
3334 I.replaceAllUsesWith(V: PoisonValue::get(T: I.getType()));
3335 I.eraseFromParent();
3336 return true;
3337 }
3338
3339 IRBuilder<> B(&I);
3340
3341 Value *Ptrs = I.getArgOperand(i: 0);
3342 Value *Mask = I.getArgOperand(i: 1);
3343 Value *Passthru = I.getArgOperand(i: 2);
3344
3345 // Alignment is stored as a parameter attribute, not as a regular parameter.
3346 uint32_t Alignment = I.getParamAlign(ArgNo: 0).valueOrOne().value();
3347
3348 SmallVector<Value *, 4> Args = {Ptrs, B.getInt32(C: Alignment), Mask,
3349 Passthru};
3350 SmallVector<Type *, 4> Types = {I.getType(), Ptrs->getType(),
3351 Mask->getType(), Passthru->getType()};
3352
3353 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_masked_gather, Types, Args);
3354 I.replaceAllUsesWith(V: NewI);
3355 I.eraseFromParent();
3356 return true;
3357 }
3358
3359 if (I.getIntrinsicID() == Intrinsic::masked_scatter) {
3360 if (!ST.canUseExtension(
3361 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
3362 I.getContext().emitError(
3363 I: &I, ErrorStr: "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter "
3364 "extension");
3365 // Erase the intrinsic to allow compilation to continue and report error.
3366 I.eraseFromParent();
3367 return true;
3368 }
3369
3370 IRBuilder<> B(&I);
3371
3372 Value *Values = I.getArgOperand(i: 0);
3373 Value *Ptrs = I.getArgOperand(i: 1);
3374 Value *Mask = I.getArgOperand(i: 2);
3375
3376 // Alignment is stored as a parameter attribute on the ptrs parameter (arg
3377 // 1).
3378 uint32_t Alignment = I.getParamAlign(ArgNo: 1).valueOrOne().value();
3379
3380 SmallVector<Value *, 4> Args = {Values, Ptrs, B.getInt32(C: Alignment), Mask};
3381 SmallVector<Type *, 3> Types = {Values->getType(), Ptrs->getType(),
3382 Mask->getType()};
3383
3384 B.CreateIntrinsic(ID: Intrinsic::spv_masked_scatter, Types, Args);
3385 I.eraseFromParent();
3386 return true;
3387 }
3388
3389 return false;
3390}
3391
3392bool SPIRVEmitIntrinsics::convertMaskedMemIntrinsics(Module &M) {
3393 bool Changed = false;
3394
3395 for (Function &F : make_early_inc_range(Range&: M)) {
3396 if (!F.isIntrinsic())
3397 continue;
3398 Intrinsic::ID IID = F.getIntrinsicID();
3399 if (IID != Intrinsic::masked_gather && IID != Intrinsic::masked_scatter)
3400 continue;
3401
3402 for (User *U : make_early_inc_range(Range: F.users())) {
3403 if (auto *II = dyn_cast<IntrinsicInst>(Val: U))
3404 Changed |= processMaskedMemIntrinsic(I&: *II);
3405 }
3406
3407 if (F.use_empty())
3408 F.eraseFromParent();
3409 }
3410
3411 return Changed;
3412}
3413
3414bool SPIRVEmitIntrinsics::runOnModule(Module &M) {
3415 bool Changed = false;
3416
3417 Changed |= convertMaskedMemIntrinsics(M);
3418
3419 parseFunDeclarations(M);
3420 insertConstantsForFPFastMathDefault(M);
3421 GVUsers.init(M);
3422
3423 TodoType.clear();
3424 for (auto &F : M)
3425 Changed |= runOnFunction(Func&: F);
3426
3427 // Specify function parameters after all functions were processed.
3428 for (auto &F : M) {
3429 // check if function parameter types are set
3430 CurrF = &F;
3431 if (!F.isDeclaration() && !F.isIntrinsic()) {
3432 IRBuilder<> B(F.getContext());
3433 processParamTypes(F: &F, B);
3434 }
3435 }
3436
3437 CanTodoType = false;
3438 Changed |= postprocessTypes(M);
3439
3440 if (HaveFunPtrs)
3441 Changed |= processFunctionPointers(M);
3442
3443 return Changed;
3444}
3445
3446ModulePass *llvm::createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM) {
3447 return new SPIRVEmitIntrinsics(TM);
3448}
3449