1//===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass emits SPIRV intrinsics keeping essential high-level information for
10// the translation of LLVM IR to SPIR-V.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRV.h"
15#include "SPIRVBuiltins.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVTargetMachine.h"
18#include "SPIRVUtils.h"
19#include "llvm/ADT/DenseSet.h"
20#include "llvm/IR/IRBuilder.h"
21#include "llvm/IR/InstIterator.h"
22#include "llvm/IR/InstVisitor.h"
23#include "llvm/IR/IntrinsicsSPIRV.h"
24#include "llvm/IR/TypedPointerType.h"
25
26#include <queue>
27#include <unordered_set>
28
29// This pass performs the following transformation on LLVM IR level required
30// for the following translation to SPIR-V:
31// - replaces direct usages of aggregate constants with target-specific
32// intrinsics;
33// - replaces aggregates-related instructions (extract/insert, ld/st, etc)
34// with a target-specific intrinsics;
35// - emits intrinsics for the global variable initializers since IRTranslator
36// doesn't handle them and it's not very convenient to translate them
37// ourselves;
38// - emits intrinsics to keep track of the string names assigned to the values;
39// - emits intrinsics to keep track of constants (this is necessary to have an
40// LLVM IR constant after the IRTranslation is completed) for their further
41// deduplication;
42// - emits intrinsics to keep track of original LLVM types of the values
43// to be able to emit proper SPIR-V types eventually.
44//
45// TODO: consider removing spv.track.constant in favor of spv.assign.type.
46
47using namespace llvm;
48
49namespace llvm::SPIRV {
50#define GET_BuiltinGroup_DECL
51#include "SPIRVGenTables.inc"
52} // namespace llvm::SPIRV
53
54namespace {
55
56class SPIRVEmitIntrinsics
57 : public ModulePass,
58 public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
59 SPIRVTargetMachine *TM = nullptr;
60 SPIRVGlobalRegistry *GR = nullptr;
61 Function *CurrF = nullptr;
62 bool TrackConstants = true;
63 bool HaveFunPtrs = false;
64 DenseMap<Instruction *, Constant *> AggrConsts;
65 DenseMap<Instruction *, Type *> AggrConstTypes;
66 DenseSet<Instruction *> AggrStores;
67 std::unordered_set<Value *> Named;
68
69 // map of function declarations to <pointer arg index => element type>
70 DenseMap<Function *, SmallVector<std::pair<unsigned, Type *>>> FDeclPtrTys;
71
72 // a register of Instructions that don't have a complete type definition
73 bool CanTodoType = true;
74 unsigned TodoTypeSz = 0;
75 DenseMap<Value *, bool> TodoType;
76 void insertTodoType(Value *Op) {
77 // TODO: add isa<CallInst>(Op) to no-insert
78 if (CanTodoType && !isa<GetElementPtrInst>(Val: Op)) {
79 auto It = TodoType.try_emplace(Key: Op, Args: true);
80 if (It.second)
81 ++TodoTypeSz;
82 }
83 }
84 void eraseTodoType(Value *Op) {
85 auto It = TodoType.find(Val: Op);
86 if (It != TodoType.end() && It->second) {
87 It->second = false;
88 --TodoTypeSz;
89 }
90 }
91 bool isTodoType(Value *Op) {
92 if (isa<GetElementPtrInst>(Val: Op))
93 return false;
94 auto It = TodoType.find(Val: Op);
95 return It != TodoType.end() && It->second;
96 }
97 // a register of Instructions that were visited by deduceOperandElementType()
98 // to validate operand types with an instruction
99 std::unordered_set<Instruction *> TypeValidated;
100
101 // well known result types of builtins
102 enum WellKnownTypes { Event };
103
104 // deduce element type of untyped pointers
105 Type *deduceElementType(Value *I, bool UnknownElemTypeI8);
106 Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8);
107 Type *deduceElementTypeHelper(Value *I, std::unordered_set<Value *> &Visited,
108 bool UnknownElemTypeI8,
109 bool IgnoreKnownType = false);
110 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
111 bool UnknownElemTypeI8);
112 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
113 std::unordered_set<Value *> &Visited,
114 bool UnknownElemTypeI8);
115 Type *deduceElementTypeByUsersDeep(Value *Op,
116 std::unordered_set<Value *> &Visited,
117 bool UnknownElemTypeI8);
118 void maybeAssignPtrType(Type *&Ty, Value *I, Type *RefTy,
119 bool UnknownElemTypeI8);
120
121 // deduce nested types of composites
122 Type *deduceNestedTypeHelper(User *U, bool UnknownElemTypeI8);
123 Type *deduceNestedTypeHelper(User *U, Type *Ty,
124 std::unordered_set<Value *> &Visited,
125 bool UnknownElemTypeI8);
126
127 // deduce Types of operands of the Instruction if possible
128 void deduceOperandElementType(Instruction *I,
129 SmallPtrSet<Instruction *, 4> *IncompleteRets,
130 const SmallPtrSet<Value *, 4> *AskOps = nullptr,
131 bool IsPostprocessing = false);
132
133 void preprocessCompositeConstants(IRBuilder<> &B);
134 void preprocessUndefs(IRBuilder<> &B);
135
136 Type *reconstructType(Value *Op, bool UnknownElemTypeI8,
137 bool IsPostprocessing);
138
139 void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
140 void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
141 bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
142 bool UnknownElemTypeI8);
143 void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B);
144 void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V,
145 IRBuilder<> &B);
146 void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer,
147 Type *ExpectedElementType,
148 unsigned OperandToReplace,
149 IRBuilder<> &B);
150 void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
151 bool shouldTryToAddMemAliasingDecoration(Instruction *Inst);
152 void insertSpirvDecorations(Instruction *I, IRBuilder<> &B);
153 void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
154 void processParamTypes(Function *F, IRBuilder<> &B);
155 void processParamTypesByFunHeader(Function *F, IRBuilder<> &B);
156 Type *deduceFunParamElementType(Function *F, unsigned OpIdx);
157 Type *deduceFunParamElementType(Function *F, unsigned OpIdx,
158 std::unordered_set<Function *> &FVisited);
159
160 bool deduceOperandElementTypeCalledFunction(
161 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
162 Type *&KnownElemTy, bool &Incomplete);
163 void deduceOperandElementTypeFunctionPointer(
164 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
165 Type *&KnownElemTy, bool IsPostprocessing);
166 bool deduceOperandElementTypeFunctionRet(
167 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
168 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing,
169 Type *&KnownElemTy, Value *Op, Function *F);
170
171 CallInst *buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy);
172 void replaceUsesOfWithSpvPtrcast(Value *Op, Type *ElemTy, Instruction *I,
173 DenseMap<Function *, CallInst *> Ptrcasts);
174 void propagateElemType(Value *Op, Type *ElemTy,
175 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
176 void
177 propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
178 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
179 void propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
180 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
181 std::unordered_set<Value *> &Visited,
182 DenseMap<Function *, CallInst *> Ptrcasts);
183
184 void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
185 void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src,
186 Instruction *Dest, bool DeleteOld = true);
187
188 void applyDemangledPtrArgTypes(IRBuilder<> &B);
189
190 bool runOnFunction(Function &F);
191 bool postprocessTypes(Module &M);
192 bool processFunctionPointers(Module &M);
193 void parseFunDeclarations(Module &M);
194
195 void useRoundingMode(ConstrainedFPIntrinsic *FPI, IRBuilder<> &B);
196
197public:
198 static char ID;
199 SPIRVEmitIntrinsics(SPIRVTargetMachine *TM = nullptr)
200 : ModulePass(ID), TM(TM) {}
201 Instruction *visitInstruction(Instruction &I) { return &I; }
202 Instruction *visitSwitchInst(SwitchInst &I);
203 Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
204 Instruction *visitBitCastInst(BitCastInst &I);
205 Instruction *visitInsertElementInst(InsertElementInst &I);
206 Instruction *visitExtractElementInst(ExtractElementInst &I);
207 Instruction *visitInsertValueInst(InsertValueInst &I);
208 Instruction *visitExtractValueInst(ExtractValueInst &I);
209 Instruction *visitLoadInst(LoadInst &I);
210 Instruction *visitStoreInst(StoreInst &I);
211 Instruction *visitAllocaInst(AllocaInst &I);
212 Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
213 Instruction *visitUnreachableInst(UnreachableInst &I);
214 Instruction *visitCallInst(CallInst &I);
215
216 StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
217
218 bool runOnModule(Module &M) override;
219
220 void getAnalysisUsage(AnalysisUsage &AU) const override {
221 ModulePass::getAnalysisUsage(AU);
222 }
223};
224
225bool isConvergenceIntrinsic(const Instruction *I) {
226 const auto *II = dyn_cast<IntrinsicInst>(Val: I);
227 if (!II)
228 return false;
229
230 return II->getIntrinsicID() == Intrinsic::experimental_convergence_entry ||
231 II->getIntrinsicID() == Intrinsic::experimental_convergence_loop ||
232 II->getIntrinsicID() == Intrinsic::experimental_convergence_anchor;
233}
234
235bool expectIgnoredInIRTranslation(const Instruction *I) {
236 const auto *II = dyn_cast<IntrinsicInst>(Val: I);
237 if (!II)
238 return false;
239 switch (II->getIntrinsicID()) {
240 case Intrinsic::invariant_start:
241 case Intrinsic::spv_resource_handlefrombinding:
242 case Intrinsic::spv_resource_getpointer:
243 return true;
244 default:
245 return false;
246 }
247}
248
249} // namespace
250
251char SPIRVEmitIntrinsics::ID = 0;
252
253INITIALIZE_PASS(SPIRVEmitIntrinsics, "emit-intrinsics", "SPIRV emit intrinsics",
254 false, false)
255
256static inline bool isAssignTypeInstr(const Instruction *I) {
257 return isa<IntrinsicInst>(Val: I) &&
258 cast<IntrinsicInst>(Val: I)->getIntrinsicID() == Intrinsic::spv_assign_type;
259}
260
261static bool isMemInstrToReplace(Instruction *I) {
262 return isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I) || isa<InsertValueInst>(Val: I) ||
263 isa<ExtractValueInst>(Val: I) || isa<AtomicCmpXchgInst>(Val: I);
264}
265
266static bool isAggrConstForceInt32(const Value *V) {
267 return isa<ConstantArray>(Val: V) || isa<ConstantStruct>(Val: V) ||
268 isa<ConstantDataArray>(Val: V) ||
269 (isa<ConstantAggregateZero>(Val: V) && !V->getType()->isVectorTy());
270}
271
272static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
273 if (isa<PHINode>(Val: I))
274 B.SetInsertPoint(I->getParent()->getFirstNonPHIOrDbgOrAlloca());
275 else
276 B.SetInsertPoint(I);
277}
278
279static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I) {
280 B.SetCurrentDebugLocation(I->getDebugLoc());
281 if (I->getType()->isVoidTy())
282 B.SetInsertPoint(I->getNextNode());
283 else
284 B.SetInsertPoint(*I->getInsertionPointAfterDef());
285}
286
287static bool requireAssignType(Instruction *I) {
288 if (const auto *Intr = dyn_cast<IntrinsicInst>(Val: I)) {
289 switch (Intr->getIntrinsicID()) {
290 case Intrinsic::invariant_start:
291 case Intrinsic::invariant_end:
292 return false;
293 }
294 }
295 return true;
296}
297
298static inline void reportFatalOnTokenType(const Instruction *I) {
299 if (I->getType()->isTokenTy())
300 report_fatal_error(reason: "A token is encountered but SPIR-V without extensions "
301 "does not support token type",
302 gen_crash_diag: false);
303}
304
305static void emitAssignName(Instruction *I, IRBuilder<> &B) {
306 if (!I->hasName() || I->getType()->isAggregateType() ||
307 expectIgnoredInIRTranslation(I))
308 return;
309 reportFatalOnTokenType(I);
310 setInsertPointAfterDef(B, I);
311 LLVMContext &Ctx = I->getContext();
312 std::vector<Value *> Args = {
313 I, MetadataAsValue::get(
314 Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: I->getName())))};
315 B.CreateIntrinsic(ID: Intrinsic::spv_assign_name, Types: {I->getType()}, Args);
316}
317
318void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest,
319 bool DeleteOld) {
320 GR->replaceAllUsesWith(Old: Src, New: Dest, DeleteOld);
321 // Update uncomplete type records if any
322 if (isTodoType(Op: Src)) {
323 if (DeleteOld)
324 eraseTodoType(Op: Src);
325 insertTodoType(Op: Dest);
326 }
327}
328
329void SPIRVEmitIntrinsics::replaceAllUsesWithAndErase(IRBuilder<> &B,
330 Instruction *Src,
331 Instruction *Dest,
332 bool DeleteOld) {
333 replaceAllUsesWith(Src, Dest, DeleteOld);
334 std::string Name = Src->hasName() ? Src->getName().str() : "";
335 Src->eraseFromParent();
336 if (!Name.empty()) {
337 Dest->setName(Name);
338 if (Named.insert(x: Dest).second)
339 emitAssignName(I: Dest, B);
340 }
341}
342
343static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
344 return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
345 isPointerTy(T: SI->getValueOperand()->getType()) &&
346 isa<Argument>(Val: SI->getValueOperand());
347}
348
349// Maybe restore original function return type.
350static inline Type *restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I,
351 Type *Ty) {
352 CallInst *CI = dyn_cast<CallInst>(Val: I);
353 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
354 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
355 return Ty;
356 if (Type *OriginalTy = GR->findMutated(Val: CI->getCalledFunction()))
357 return OriginalTy;
358 return Ty;
359}
360
361// Reconstruct type with nested element types according to deduced type info.
362// Return nullptr if no detailed type info is available.
363Type *SPIRVEmitIntrinsics::reconstructType(Value *Op, bool UnknownElemTypeI8,
364 bool IsPostprocessing) {
365 Type *Ty = Op->getType();
366 if (auto *OpI = dyn_cast<Instruction>(Val: Op))
367 Ty = restoreMutatedType(GR, I: OpI, Ty);
368 if (!isUntypedPointerTy(T: Ty))
369 return Ty;
370 // try to find the pointee type
371 if (Type *NestedTy = GR->findDeducedElementType(Val: Op))
372 return getTypedPointerWrapper(ElemTy: NestedTy, AS: getPointerAddressSpace(T: Ty));
373 // not a pointer according to the type info (e.g., Event object)
374 CallInst *CI = GR->findAssignPtrTypeInstr(Val: Op);
375 if (CI) {
376 MetadataAsValue *MD = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
377 return cast<ConstantAsMetadata>(Val: MD->getMetadata())->getType();
378 }
379 if (UnknownElemTypeI8) {
380 if (!IsPostprocessing)
381 insertTodoType(Op);
382 return getTypedPointerWrapper(ElemTy: IntegerType::getInt8Ty(C&: Op->getContext()),
383 AS: getPointerAddressSpace(T: Ty));
384 }
385 return nullptr;
386}
387
388CallInst *SPIRVEmitIntrinsics::buildSpvPtrcast(Function *F, Value *Op,
389 Type *ElemTy) {
390 IRBuilder<> B(Op->getContext());
391 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
392 // spv_ptrcast's argument Op denotes an instruction that generates
393 // a value, and we may use getInsertionPointAfterDef()
394 setInsertPointAfterDef(B, I: OpI);
395 } else if (auto *OpA = dyn_cast<Argument>(Val: Op)) {
396 B.SetInsertPointPastAllocas(OpA->getParent());
397 B.SetCurrentDebugLocation(DebugLoc());
398 } else {
399 B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
400 }
401 Type *OpTy = Op->getType();
402 SmallVector<Type *, 2> Types = {OpTy, OpTy};
403 SmallVector<Value *, 2> Args = {Op, buildMD(Arg: getNormalizedPoisonValue(Ty: ElemTy)),
404 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
405 CallInst *PtrCasted =
406 B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
407 GR->buildAssignPtr(B, ElemTy, Arg: PtrCasted);
408 return PtrCasted;
409}
410
411void SPIRVEmitIntrinsics::replaceUsesOfWithSpvPtrcast(
412 Value *Op, Type *ElemTy, Instruction *I,
413 DenseMap<Function *, CallInst *> Ptrcasts) {
414 Function *F = I->getParent()->getParent();
415 CallInst *PtrCastedI = nullptr;
416 auto It = Ptrcasts.find(Val: F);
417 if (It == Ptrcasts.end()) {
418 PtrCastedI = buildSpvPtrcast(F, Op, ElemTy);
419 Ptrcasts[F] = PtrCastedI;
420 } else {
421 PtrCastedI = It->second;
422 }
423 I->replaceUsesOfWith(From: Op, To: PtrCastedI);
424}
425
426void SPIRVEmitIntrinsics::propagateElemType(
427 Value *Op, Type *ElemTy,
428 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
429 DenseMap<Function *, CallInst *> Ptrcasts;
430 SmallVector<User *> Users(Op->users());
431 for (auto *U : Users) {
432 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
433 continue;
434 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
435 continue;
436 Instruction *UI = dyn_cast<Instruction>(Val: U);
437 // If the instruction was validated already, we need to keep it valid by
438 // keeping current Op type.
439 if (isa<GetElementPtrInst>(Val: UI) ||
440 TypeValidated.find(x: UI) != TypeValidated.end())
441 replaceUsesOfWithSpvPtrcast(Op, ElemTy, I: UI, Ptrcasts);
442 }
443}
444
445void SPIRVEmitIntrinsics::propagateElemTypeRec(
446 Value *Op, Type *PtrElemTy, Type *CastElemTy,
447 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
448 std::unordered_set<Value *> Visited;
449 DenseMap<Function *, CallInst *> Ptrcasts;
450 propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited,
451 Ptrcasts);
452}
453
454void SPIRVEmitIntrinsics::propagateElemTypeRec(
455 Value *Op, Type *PtrElemTy, Type *CastElemTy,
456 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
457 std::unordered_set<Value *> &Visited,
458 DenseMap<Function *, CallInst *> Ptrcasts) {
459 if (!Visited.insert(x: Op).second)
460 return;
461 SmallVector<User *> Users(Op->users());
462 for (auto *U : Users) {
463 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
464 continue;
465 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
466 continue;
467 Instruction *UI = dyn_cast<Instruction>(Val: U);
468 // If the instruction was validated already, we need to keep it valid by
469 // keeping current Op type.
470 if (isa<GetElementPtrInst>(Val: UI) ||
471 TypeValidated.find(x: UI) != TypeValidated.end())
472 replaceUsesOfWithSpvPtrcast(Op, ElemTy: CastElemTy, I: UI, Ptrcasts);
473 }
474}
475
476// Set element pointer type to the given value of ValueTy and tries to
477// specify this type further (recursively) by Operand value, if needed.
478
479Type *
480SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
481 bool UnknownElemTypeI8) {
482 std::unordered_set<Value *> Visited;
483 return deduceElementTypeByValueDeep(ValueTy, Operand, Visited,
484 UnknownElemTypeI8);
485}
486
487Type *SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(
488 Type *ValueTy, Value *Operand, std::unordered_set<Value *> &Visited,
489 bool UnknownElemTypeI8) {
490 Type *Ty = ValueTy;
491 if (Operand) {
492 if (auto *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
493 if (Type *NestedTy =
494 deduceElementTypeHelper(I: Operand, Visited, UnknownElemTypeI8))
495 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
496 } else {
497 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Operand), Ty, Visited,
498 UnknownElemTypeI8);
499 }
500 }
501 return Ty;
502}
503
504// Traverse User instructions to deduce an element pointer type of the operand.
505Type *SPIRVEmitIntrinsics::deduceElementTypeByUsersDeep(
506 Value *Op, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8) {
507 if (!Op || !isPointerTy(T: Op->getType()) || isa<ConstantPointerNull>(Val: Op) ||
508 isa<UndefValue>(Val: Op))
509 return nullptr;
510
511 if (auto ElemTy = getPointeeType(Ty: Op->getType()))
512 return ElemTy;
513
514 // maybe we already know operand's element type
515 if (Type *KnownTy = GR->findDeducedElementType(Val: Op))
516 return KnownTy;
517
518 for (User *OpU : Op->users()) {
519 if (Instruction *Inst = dyn_cast<Instruction>(Val: OpU)) {
520 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8))
521 return Ty;
522 }
523 }
524 return nullptr;
525}
526
527// Implements what we know in advance about intrinsics and builtin calls
528// TODO: consider feasibility of this particular case to be generalized by
529// encoding knowledge about intrinsics and builtin calls by corresponding
530// specification rules
531static Type *getPointeeTypeByCallInst(StringRef DemangledName,
532 Function *CalledF, unsigned OpIdx) {
533 if ((DemangledName.starts_with(Prefix: "__spirv_ocl_printf(") ||
534 DemangledName.starts_with(Prefix: "printf(")) &&
535 OpIdx == 0)
536 return IntegerType::getInt8Ty(C&: CalledF->getContext());
537 return nullptr;
538}
539
540// Deduce and return a successfully deduced Type of the Instruction,
541// or nullptr otherwise.
542Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(Value *I,
543 bool UnknownElemTypeI8) {
544 std::unordered_set<Value *> Visited;
545 return deduceElementTypeHelper(I, Visited, UnknownElemTypeI8);
546}
547
548void SPIRVEmitIntrinsics::maybeAssignPtrType(Type *&Ty, Value *Op, Type *RefTy,
549 bool UnknownElemTypeI8) {
550 if (isUntypedPointerTy(T: RefTy)) {
551 if (!UnknownElemTypeI8)
552 return;
553 insertTodoType(Op);
554 }
555 Ty = RefTy;
556}
557
558Type *getGEPType(GetElementPtrInst *Ref) {
559 Type *Ty = nullptr;
560 // TODO: not sure if GetElementPtrInst::getTypeAtIndex() does anything
561 // useful here
562 if (isNestedPointer(Ty: Ref->getSourceElementType())) {
563 Ty = Ref->getSourceElementType();
564 for (Use &U : drop_begin(RangeOrContainer: Ref->indices()))
565 Ty = GetElementPtrInst::getTypeAtIndex(Ty, Idx: U.get());
566 } else {
567 Ty = Ref->getResultElementType();
568 }
569 return Ty;
570}
571
572Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
573 Value *I, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8,
574 bool IgnoreKnownType) {
575 // allow to pass nullptr as an argument
576 if (!I)
577 return nullptr;
578
579 // maybe already known
580 if (!IgnoreKnownType)
581 if (Type *KnownTy = GR->findDeducedElementType(Val: I))
582 return KnownTy;
583
584 // maybe a cycle
585 if (!Visited.insert(x: I).second)
586 return nullptr;
587
588 // fallback value in case when we fail to deduce a type
589 Type *Ty = nullptr;
590 // look for known basic patterns of type inference
591 if (auto *Ref = dyn_cast<AllocaInst>(Val: I)) {
592 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getAllocatedType(), UnknownElemTypeI8);
593 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
594 Ty = getGEPType(Ref);
595 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
596 Value *Op = Ref->getPointerOperand();
597 Type *KnownTy = GR->findDeducedElementType(Val: Op);
598 if (!KnownTy)
599 KnownTy = Op->getType();
600 if (Type *ElemTy = getPointeeType(Ty: KnownTy))
601 maybeAssignPtrType(Ty, Op: I, RefTy: ElemTy, UnknownElemTypeI8);
602 } else if (auto *Ref = dyn_cast<GlobalValue>(Val: I)) {
603 Ty = deduceElementTypeByValueDeep(
604 ValueTy: Ref->getValueType(),
605 Operand: Ref->getNumOperands() > 0 ? Ref->getOperand(i: 0) : nullptr, Visited,
606 UnknownElemTypeI8);
607 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
608 Type *RefTy = deduceElementTypeHelper(I: Ref->getPointerOperand(), Visited,
609 UnknownElemTypeI8);
610 maybeAssignPtrType(Ty, Op: I, RefTy, UnknownElemTypeI8);
611 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
612 if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
613 isPointerTy(T: Src) && isPointerTy(T: Dest))
614 Ty = deduceElementTypeHelper(I: Ref->getOperand(i_nocapture: 0), Visited,
615 UnknownElemTypeI8);
616 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
617 Value *Op = Ref->getNewValOperand();
618 if (isPointerTy(T: Op->getType()))
619 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
620 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
621 Value *Op = Ref->getValOperand();
622 if (isPointerTy(T: Op->getType()))
623 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
624 } else if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
625 Type *BestTy = nullptr;
626 unsigned MaxN = 1;
627 DenseMap<Type *, unsigned> PhiTys;
628 for (int i = Ref->getNumIncomingValues() - 1; i >= 0; --i) {
629 Ty = deduceElementTypeByUsersDeep(Op: Ref->getIncomingValue(i), Visited,
630 UnknownElemTypeI8);
631 if (!Ty)
632 continue;
633 auto It = PhiTys.try_emplace(Key: Ty, Args: 1);
634 if (!It.second) {
635 ++It.first->second;
636 if (It.first->second > MaxN) {
637 MaxN = It.first->second;
638 BestTy = Ty;
639 }
640 }
641 }
642 if (BestTy)
643 Ty = BestTy;
644 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
645 for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) {
646 Ty = deduceElementTypeByUsersDeep(Op, Visited, UnknownElemTypeI8);
647 if (Ty)
648 break;
649 }
650 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
651 static StringMap<unsigned> ResTypeByArg = {
652 {"to_global", 0},
653 {"to_local", 0},
654 {"to_private", 0},
655 {"__spirv_GenericCastToPtr_ToGlobal", 0},
656 {"__spirv_GenericCastToPtr_ToLocal", 0},
657 {"__spirv_GenericCastToPtr_ToPrivate", 0},
658 {"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
659 {"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
660 {"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
661 // TODO: maybe improve performance by caching demangled names
662
663 auto *II = dyn_cast<IntrinsicInst>(Val: I);
664 if (II && II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
665 auto *HandleType = cast<TargetExtType>(Val: II->getOperand(i_nocapture: 0)->getType());
666 if (HandleType->getTargetExtName() == "spirv.Image" ||
667 HandleType->getTargetExtName() == "spirv.SignedImage") {
668 if (II->hasOneUse()) {
669 auto *U = *II->users().begin();
670 Ty = cast<Instruction>(Val: U)->getAccessType();
671 assert(Ty && "Unable to get type for resource pointer.");
672 }
673 } else if (HandleType->getTargetExtName() == "spirv.VulkanBuffer") {
674 // This call is supposed to index into an array
675 Ty = HandleType->getTypeParameter(i: 0);
676 if (Ty->isArrayTy())
677 Ty = Ty->getArrayElementType();
678 else {
679 TargetExtType *BufferTy = cast<TargetExtType>(Val: Ty);
680 assert(BufferTy->getTargetExtName() == "spirv.Layout");
681 Ty = BufferTy->getTypeParameter(i: 0);
682 assert(Ty && Ty->isStructTy());
683 uint32_t Index = cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1))->getZExtValue();
684 Ty = cast<StructType>(Val: Ty)->getElementType(N: Index);
685 }
686 } else {
687 llvm_unreachable("Unknown handle type for spv_resource_getpointer.");
688 }
689 } else if (II && II->getIntrinsicID() ==
690 Intrinsic::spv_generic_cast_to_ptr_explicit) {
691 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: 0), Visited,
692 UnknownElemTypeI8);
693 } else if (Function *CalledF = CI->getCalledFunction()) {
694 std::string DemangledName =
695 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
696 if (DemangledName.length() > 0)
697 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName);
698 auto AsArgIt = ResTypeByArg.find(Key: DemangledName);
699 if (AsArgIt != ResTypeByArg.end())
700 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: AsArgIt->second),
701 Visited, UnknownElemTypeI8);
702 else if (Type *KnownRetTy = GR->findDeducedElementType(Val: CalledF))
703 Ty = KnownRetTy;
704 }
705 }
706
707 // remember the found relationship
708 if (Ty && !IgnoreKnownType) {
709 // specify nested types if needed, otherwise return unchanged
710 GR->addDeducedElementType(Val: I, Ty: normalizeType(Ty));
711 }
712
713 return Ty;
714}
715
716// Re-create a type of the value if it has untyped pointer fields, also nested.
717// Return the original value type if no corrections of untyped pointer
718// information is found or needed.
719Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(User *U,
720 bool UnknownElemTypeI8) {
721 std::unordered_set<Value *> Visited;
722 return deduceNestedTypeHelper(U, Ty: U->getType(), Visited, UnknownElemTypeI8);
723}
724
725Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
726 User *U, Type *OrigTy, std::unordered_set<Value *> &Visited,
727 bool UnknownElemTypeI8) {
728 if (!U)
729 return OrigTy;
730
731 // maybe already known
732 if (Type *KnownTy = GR->findDeducedCompositeType(Val: U))
733 return KnownTy;
734
735 // maybe a cycle
736 if (!Visited.insert(x: U).second)
737 return OrigTy;
738
739 if (isa<StructType>(Val: OrigTy)) {
740 SmallVector<Type *> Tys;
741 bool Change = false;
742 for (unsigned i = 0; i < U->getNumOperands(); ++i) {
743 Value *Op = U->getOperand(i);
744 Type *OpTy = Op->getType();
745 Type *Ty = OpTy;
746 if (Op) {
747 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
748 if (Type *NestedTy =
749 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
750 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
751 } else {
752 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
753 UnknownElemTypeI8);
754 }
755 }
756 Tys.push_back(Elt: Ty);
757 Change |= Ty != OpTy;
758 }
759 if (Change) {
760 Type *NewTy = StructType::create(Elements: Tys);
761 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
762 return NewTy;
763 }
764 } else if (auto *ArrTy = dyn_cast<ArrayType>(Val: OrigTy)) {
765 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
766 Type *OpTy = ArrTy->getElementType();
767 Type *Ty = OpTy;
768 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
769 if (Type *NestedTy =
770 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
771 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
772 } else {
773 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
774 UnknownElemTypeI8);
775 }
776 if (Ty != OpTy) {
777 Type *NewTy = ArrayType::get(ElementType: Ty, NumElements: ArrTy->getNumElements());
778 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
779 return NewTy;
780 }
781 }
782 } else if (auto *VecTy = dyn_cast<VectorType>(Val: OrigTy)) {
783 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
784 Type *OpTy = VecTy->getElementType();
785 Type *Ty = OpTy;
786 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
787 if (Type *NestedTy =
788 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
789 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
790 } else {
791 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
792 UnknownElemTypeI8);
793 }
794 if (Ty != OpTy) {
795 Type *NewTy = VectorType::get(ElementType: Ty, EC: VecTy->getElementCount());
796 GR->addDeducedCompositeType(Val: U, Ty: normalizeType(Ty: NewTy));
797 return NewTy;
798 }
799 }
800 }
801
802 return OrigTy;
803}
804
805Type *SPIRVEmitIntrinsics::deduceElementType(Value *I, bool UnknownElemTypeI8) {
806 if (Type *Ty = deduceElementTypeHelper(I, UnknownElemTypeI8))
807 return Ty;
808 if (!UnknownElemTypeI8)
809 return nullptr;
810 insertTodoType(Op: I);
811 return IntegerType::getInt8Ty(C&: I->getContext());
812}
813
814static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I,
815 Value *PointerOperand) {
816 Type *PointeeTy = GR->findDeducedElementType(Val: PointerOperand);
817 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
818 return nullptr;
819 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType());
820 if (!PtrTy)
821 return I->getType();
822 if (Type *NestedTy = GR->findDeducedElementType(Val: I))
823 return getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
824 return nullptr;
825}
826
827// Try to deduce element type for a call base. Returns false if this is an
828// indirect function invocation, and true otherwise.
829bool SPIRVEmitIntrinsics::deduceOperandElementTypeCalledFunction(
830 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
831 Type *&KnownElemTy, bool &Incomplete) {
832 Function *CalledF = CI->getCalledFunction();
833 if (!CalledF)
834 return false;
835 std::string DemangledName =
836 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
837 if (DemangledName.length() > 0 &&
838 !StringRef(DemangledName).starts_with(Prefix: "llvm.")) {
839 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F: *CalledF);
840 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
841 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
842 if (Opcode == SPIRV::OpGroupAsyncCopy) {
843 for (unsigned i = 0, PtrCnt = 0; i < CI->arg_size() && PtrCnt < 2; ++i) {
844 Value *Op = CI->getArgOperand(i);
845 if (!isPointerTy(T: Op->getType()))
846 continue;
847 ++PtrCnt;
848 if (Type *ElemTy = GR->findDeducedElementType(Val: Op))
849 KnownElemTy = ElemTy; // src will rewrite dest if both are defined
850 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
851 }
852 } else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) {
853 if (CI->arg_size() == 0)
854 return true;
855 Value *Op = CI->getArgOperand(i: 0);
856 if (!isPointerTy(T: Op->getType()))
857 return true;
858 switch (Opcode) {
859 case SPIRV::OpAtomicFAddEXT:
860 case SPIRV::OpAtomicFMinEXT:
861 case SPIRV::OpAtomicFMaxEXT:
862 case SPIRV::OpAtomicLoad:
863 case SPIRV::OpAtomicCompareExchangeWeak:
864 case SPIRV::OpAtomicCompareExchange:
865 case SPIRV::OpAtomicExchange:
866 case SPIRV::OpAtomicIAdd:
867 case SPIRV::OpAtomicISub:
868 case SPIRV::OpAtomicOr:
869 case SPIRV::OpAtomicXor:
870 case SPIRV::OpAtomicAnd:
871 case SPIRV::OpAtomicUMin:
872 case SPIRV::OpAtomicUMax:
873 case SPIRV::OpAtomicSMin:
874 case SPIRV::OpAtomicSMax: {
875 KnownElemTy = isPointerTy(T: CI->getType()) ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
876 : CI->getType();
877 if (!KnownElemTy)
878 return true;
879 Incomplete = isTodoType(Op);
880 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
881 } break;
882 case SPIRV::OpAtomicStore: {
883 if (CI->arg_size() < 4)
884 return true;
885 Value *ValOp = CI->getArgOperand(i: 3);
886 KnownElemTy = isPointerTy(T: ValOp->getType())
887 ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
888 : ValOp->getType();
889 if (!KnownElemTy)
890 return true;
891 Incomplete = isTodoType(Op);
892 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
893 } break;
894 }
895 }
896 }
897 return true;
898}
899
900// Try to deduce element type for a function pointer.
901void SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionPointer(
902 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
903 Type *&KnownElemTy, bool IsPostprocessing) {
904 Value *Op = CI->getCalledOperand();
905 if (!Op || !isPointerTy(T: Op->getType()))
906 return;
907 Ops.push_back(Elt: std::make_pair(x&: Op, y: std::numeric_limits<unsigned>::max()));
908 FunctionType *FTy = CI->getFunctionType();
909 bool IsNewFTy = false, IsIncomplete = false;
910 SmallVector<Type *, 4> ArgTys;
911 for (Value *Arg : CI->args()) {
912 Type *ArgTy = Arg->getType();
913 if (ArgTy->isPointerTy()) {
914 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg)) {
915 IsNewFTy = true;
916 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
917 if (isTodoType(Op: Arg))
918 IsIncomplete = true;
919 } else {
920 IsIncomplete = true;
921 }
922 }
923 ArgTys.push_back(Elt: ArgTy);
924 }
925 Type *RetTy = FTy->getReturnType();
926 if (CI->getType()->isPointerTy()) {
927 if (Type *ElemTy = GR->findDeducedElementType(Val: CI)) {
928 IsNewFTy = true;
929 RetTy =
930 getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: CI->getType()));
931 if (isTodoType(Op: CI))
932 IsIncomplete = true;
933 } else {
934 IsIncomplete = true;
935 }
936 }
937 if (!IsPostprocessing && IsIncomplete)
938 insertTodoType(Op);
939 KnownElemTy =
940 IsNewFTy ? FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: FTy->isVarArg()) : FTy;
941}
942
943bool SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionRet(
944 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
945 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing,
946 Type *&KnownElemTy, Value *Op, Function *F) {
947 KnownElemTy = GR->findDeducedElementType(Val: F);
948 if (KnownElemTy)
949 return false;
950 if (Type *OpElemTy = GR->findDeducedElementType(Val: Op)) {
951 OpElemTy = normalizeType(Ty: OpElemTy);
952 GR->addDeducedElementType(Val: F, Ty: OpElemTy);
953 GR->addReturnType(
954 ArgF: F, DerivedTy: TypedPointerType::get(ElementType: OpElemTy,
955 AddressSpace: getPointerAddressSpace(T: F->getReturnType())));
956 // non-recursive update of types in function uses
957 DenseSet<std::pair<Value *, Value *>> VisitedSubst{std::make_pair(x&: I, y&: Op)};
958 for (User *U : F->users()) {
959 CallInst *CI = dyn_cast<CallInst>(Val: U);
960 if (!CI || CI->getCalledFunction() != F)
961 continue;
962 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: CI)) {
963 if (Type *PrevElemTy = GR->findDeducedElementType(Val: CI)) {
964 GR->updateAssignType(AssignCI, Arg: CI,
965 OfType: getNormalizedPoisonValue(Ty: OpElemTy));
966 propagateElemType(Op: CI, ElemTy: PrevElemTy, VisitedSubst);
967 }
968 }
969 }
970 // Non-recursive update of types in the function uncomplete returns.
971 // This may happen just once per a function, the latch is a pair of
972 // findDeducedElementType(F) / addDeducedElementType(F, ...).
973 // With or without the latch it is a non-recursive call due to
974 // IncompleteRets set to nullptr in this call.
975 if (IncompleteRets)
976 for (Instruction *IncompleteRetI : *IncompleteRets)
977 deduceOperandElementType(I: IncompleteRetI, IncompleteRets: nullptr, AskOps,
978 IsPostprocessing);
979 } else if (IncompleteRets) {
980 IncompleteRets->insert(Ptr: I);
981 }
982 TypeValidated.insert(x: I);
983 return true;
984}
985
986// If the Instruction has Pointer operands with unresolved types, this function
987// tries to deduce them. If the Instruction has Pointer operands with known
988// types which differ from expected, this function tries to insert a bitcast to
989// resolve the issue.
990void SPIRVEmitIntrinsics::deduceOperandElementType(
991 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
992 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing) {
993 SmallVector<std::pair<Value *, unsigned>> Ops;
994 Type *KnownElemTy = nullptr;
995 bool Incomplete = false;
996 // look for known basic patterns of type inference
997 if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
998 if (!isPointerTy(T: I->getType()) ||
999 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1000 return;
1001 Incomplete = isTodoType(Op: I);
1002 for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
1003 Value *Op = Ref->getIncomingValue(i);
1004 if (isPointerTy(T: Op->getType()))
1005 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1006 }
1007 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1008 KnownElemTy = GR->findDeducedElementType(Val: I);
1009 if (!KnownElemTy)
1010 return;
1011 Incomplete = isTodoType(Op: I);
1012 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(), y: 0));
1013 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1014 if (!isPointerTy(T: I->getType()))
1015 return;
1016 KnownElemTy = GR->findDeducedElementType(Val: I);
1017 if (!KnownElemTy)
1018 return;
1019 Incomplete = isTodoType(Op: I);
1020 Ops.push_back(Elt: std::make_pair(x: Ref->getOperand(i_nocapture: 0), y: 0));
1021 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
1022 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1023 return;
1024 KnownElemTy = Ref->getSourceElementType();
1025 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1026 y: GetElementPtrInst::getPointerOperandIndex()));
1027 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
1028 KnownElemTy = I->getType();
1029 if (isUntypedPointerTy(T: KnownElemTy))
1030 return;
1031 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1032 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1033 return;
1034 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1035 y: LoadInst::getPointerOperandIndex()));
1036 } else if (auto *Ref = dyn_cast<StoreInst>(Val: I)) {
1037 if (!(KnownElemTy =
1038 reconstructType(Op: Ref->getValueOperand(), UnknownElemTypeI8: false, IsPostprocessing)))
1039 return;
1040 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1041 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1042 return;
1043 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1044 y: StoreInst::getPointerOperandIndex()));
1045 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1046 KnownElemTy = isPointerTy(T: I->getType())
1047 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1048 : I->getType();
1049 if (!KnownElemTy)
1050 return;
1051 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1052 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1053 y: AtomicCmpXchgInst::getPointerOperandIndex()));
1054 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1055 KnownElemTy = isPointerTy(T: I->getType())
1056 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1057 : I->getType();
1058 if (!KnownElemTy)
1059 return;
1060 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1061 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1062 y: AtomicRMWInst::getPointerOperandIndex()));
1063 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1064 if (!isPointerTy(T: I->getType()) ||
1065 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1066 return;
1067 Incomplete = isTodoType(Op: I);
1068 for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
1069 Value *Op = Ref->getOperand(i_nocapture: i);
1070 if (isPointerTy(T: Op->getType()))
1071 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1072 }
1073 } else if (auto *Ref = dyn_cast<ReturnInst>(Val: I)) {
1074 if (!isPointerTy(T: CurrF->getReturnType()))
1075 return;
1076 Value *Op = Ref->getReturnValue();
1077 if (!Op)
1078 return;
1079 if (deduceOperandElementTypeFunctionRet(I, IncompleteRets, AskOps,
1080 IsPostprocessing, KnownElemTy, Op,
1081 F: CurrF))
1082 return;
1083 Incomplete = isTodoType(Op: CurrF);
1084 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1085 } else if (auto *Ref = dyn_cast<ICmpInst>(Val: I)) {
1086 if (!isPointerTy(T: Ref->getOperand(i_nocapture: 0)->getType()))
1087 return;
1088 Value *Op0 = Ref->getOperand(i_nocapture: 0);
1089 Value *Op1 = Ref->getOperand(i_nocapture: 1);
1090 bool Incomplete0 = isTodoType(Op: Op0);
1091 bool Incomplete1 = isTodoType(Op: Op1);
1092 Type *ElemTy1 = GR->findDeducedElementType(Val: Op1);
1093 Type *ElemTy0 = (Incomplete0 && !Incomplete1 && ElemTy1)
1094 ? nullptr
1095 : GR->findDeducedElementType(Val: Op0);
1096 if (ElemTy0) {
1097 KnownElemTy = ElemTy0;
1098 Incomplete = Incomplete0;
1099 Ops.push_back(Elt: std::make_pair(x&: Op1, y: 1));
1100 } else if (ElemTy1) {
1101 KnownElemTy = ElemTy1;
1102 Incomplete = Incomplete1;
1103 Ops.push_back(Elt: std::make_pair(x&: Op0, y: 0));
1104 }
1105 } else if (CallInst *CI = dyn_cast<CallInst>(Val: I)) {
1106 if (!CI->isIndirectCall())
1107 deduceOperandElementTypeCalledFunction(CI, Ops, KnownElemTy, Incomplete);
1108 else if (HaveFunPtrs)
1109 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy,
1110 IsPostprocessing);
1111 }
1112
1113 // There is no enough info to deduce types or all is valid.
1114 if (!KnownElemTy || Ops.size() == 0)
1115 return;
1116
1117 LLVMContext &Ctx = CurrF->getContext();
1118 IRBuilder<> B(Ctx);
1119 for (auto &OpIt : Ops) {
1120 Value *Op = OpIt.first;
1121 if (AskOps && !AskOps->contains(Ptr: Op))
1122 continue;
1123 Type *AskTy = nullptr;
1124 CallInst *AskCI = nullptr;
1125 if (IsPostprocessing && AskOps) {
1126 AskTy = GR->findDeducedElementType(Val: Op);
1127 AskCI = GR->findAssignPtrTypeInstr(Val: Op);
1128 assert(AskTy && AskCI);
1129 }
1130 Type *Ty = AskTy ? AskTy : GR->findDeducedElementType(Val: Op);
1131 if (Ty == KnownElemTy)
1132 continue;
1133 Value *OpTyVal = getNormalizedPoisonValue(Ty: KnownElemTy);
1134 Type *OpTy = Op->getType();
1135 if (Op->hasUseList() &&
1136 (!Ty || AskTy || isUntypedPointerTy(T: Ty) || isTodoType(Op))) {
1137 Type *PrevElemTy = GR->findDeducedElementType(Val: Op);
1138 GR->addDeducedElementType(Val: Op, Ty: normalizeType(Ty: KnownElemTy));
1139 // check if KnownElemTy is complete
1140 if (!Incomplete)
1141 eraseTodoType(Op);
1142 else if (!IsPostprocessing)
1143 insertTodoType(Op);
1144 // check if there is existing Intrinsic::spv_assign_ptr_type instruction
1145 CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Val: Op);
1146 if (AssignCI == nullptr) {
1147 Instruction *User = dyn_cast<Instruction>(Val: Op->use_begin()->get());
1148 setInsertPointSkippingPhis(B, I: User ? User->getNextNode() : I);
1149 CallInst *CI =
1150 buildIntrWithMD(IntrID: Intrinsic::spv_assign_ptr_type, Types: {OpTy}, Arg: OpTyVal, Arg2: Op,
1151 Imms: {B.getInt32(C: getPointerAddressSpace(T: OpTy))}, B);
1152 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: CI);
1153 } else {
1154 GR->updateAssignType(AssignCI, Arg: Op, OfType: OpTyVal);
1155 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1156 std::make_pair(x&: I, y&: Op)};
1157 propagateElemTypeRec(Op, PtrElemTy: KnownElemTy, CastElemTy: PrevElemTy, VisitedSubst);
1158 }
1159 } else {
1160 eraseTodoType(Op);
1161 CallInst *PtrCastI =
1162 buildSpvPtrcast(F: I->getParent()->getParent(), Op, ElemTy: KnownElemTy);
1163 if (OpIt.second == std::numeric_limits<unsigned>::max())
1164 dyn_cast<CallInst>(Val: I)->setCalledOperand(PtrCastI);
1165 else
1166 I->setOperand(i: OpIt.second, Val: PtrCastI);
1167 }
1168 }
1169 TypeValidated.insert(x: I);
1170}
1171
1172void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
1173 Instruction *New,
1174 IRBuilder<> &B) {
1175 while (!Old->user_empty()) {
1176 auto *U = Old->user_back();
1177 if (isAssignTypeInstr(I: U)) {
1178 B.SetInsertPoint(U);
1179 SmallVector<Value *, 2> Args = {New, U->getOperand(i: 1)};
1180 CallInst *AssignCI =
1181 B.CreateIntrinsic(ID: Intrinsic::spv_assign_type, Types: {New->getType()}, Args);
1182 GR->addAssignPtrTypeInstr(Val: New, AssignPtrTyCI: AssignCI);
1183 U->eraseFromParent();
1184 } else if (isMemInstrToReplace(I: U) || isa<ReturnInst>(Val: U) ||
1185 isa<CallInst>(Val: U)) {
1186 U->replaceUsesOfWith(From: Old, To: New);
1187 } else {
1188 llvm_unreachable("illegal aggregate intrinsic user");
1189 }
1190 }
1191 New->copyMetadata(SrcInst: *Old);
1192 Old->eraseFromParent();
1193}
1194
1195void SPIRVEmitIntrinsics::preprocessUndefs(IRBuilder<> &B) {
1196 std::queue<Instruction *> Worklist;
1197 for (auto &I : instructions(F: CurrF))
1198 Worklist.push(x: &I);
1199
1200 while (!Worklist.empty()) {
1201 Instruction *I = Worklist.front();
1202 bool BPrepared = false;
1203 Worklist.pop();
1204
1205 for (auto &Op : I->operands()) {
1206 auto *AggrUndef = dyn_cast<UndefValue>(Val&: Op);
1207 if (!AggrUndef || !Op->getType()->isAggregateType())
1208 continue;
1209
1210 if (!BPrepared) {
1211 setInsertPointSkippingPhis(B, I);
1212 BPrepared = true;
1213 }
1214 auto *IntrUndef = B.CreateIntrinsic(ID: Intrinsic::spv_undef, Args: {});
1215 Worklist.push(x: IntrUndef);
1216 I->replaceUsesOfWith(From: Op, To: IntrUndef);
1217 AggrConsts[IntrUndef] = AggrUndef;
1218 AggrConstTypes[IntrUndef] = AggrUndef->getType();
1219 }
1220 }
1221}
1222
1223void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) {
1224 std::queue<Instruction *> Worklist;
1225 for (auto &I : instructions(F: CurrF))
1226 Worklist.push(x: &I);
1227
1228 while (!Worklist.empty()) {
1229 auto *I = Worklist.front();
1230 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
1231 assert(I);
1232 bool KeepInst = false;
1233 for (const auto &Op : I->operands()) {
1234 Constant *AggrConst = nullptr;
1235 Type *ResTy = nullptr;
1236 if (auto *COp = dyn_cast<ConstantVector>(Val: Op)) {
1237 AggrConst = cast<Constant>(Val: COp);
1238 ResTy = COp->getType();
1239 } else if (auto *COp = dyn_cast<ConstantArray>(Val: Op)) {
1240 AggrConst = cast<Constant>(Val: COp);
1241 ResTy = B.getInt32Ty();
1242 } else if (auto *COp = dyn_cast<ConstantStruct>(Val: Op)) {
1243 AggrConst = cast<Constant>(Val: COp);
1244 ResTy = B.getInt32Ty();
1245 } else if (auto *COp = dyn_cast<ConstantDataArray>(Val: Op)) {
1246 AggrConst = cast<Constant>(Val: COp);
1247 ResTy = B.getInt32Ty();
1248 } else if (auto *COp = dyn_cast<ConstantAggregateZero>(Val: Op)) {
1249 AggrConst = cast<Constant>(Val: COp);
1250 ResTy = Op->getType()->isVectorTy() ? COp->getType() : B.getInt32Ty();
1251 }
1252 if (AggrConst) {
1253 SmallVector<Value *> Args;
1254 if (auto *COp = dyn_cast<ConstantDataSequential>(Val: Op))
1255 for (unsigned i = 0; i < COp->getNumElements(); ++i)
1256 Args.push_back(Elt: COp->getElementAsConstant(i));
1257 else
1258 llvm::append_range(C&: Args, R: AggrConst->operands());
1259 if (!BPrepared) {
1260 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
1261 : B.SetInsertPoint(I);
1262 BPrepared = true;
1263 }
1264 auto *CI =
1265 B.CreateIntrinsic(ID: Intrinsic::spv_const_composite, Types: {ResTy}, Args: {Args});
1266 Worklist.push(x: CI);
1267 I->replaceUsesOfWith(From: Op, To: CI);
1268 KeepInst = true;
1269 AggrConsts[CI] = AggrConst;
1270 AggrConstTypes[CI] = deduceNestedTypeHelper(U: AggrConst, UnknownElemTypeI8: false);
1271 }
1272 }
1273 if (!KeepInst)
1274 Worklist.pop();
1275 }
1276}
1277
1278static void createDecorationIntrinsic(Instruction *I, MDNode *Node,
1279 IRBuilder<> &B) {
1280 LLVMContext &Ctx = I->getContext();
1281 setInsertPointAfterDef(B, I);
1282 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, Types: {I->getType()},
1283 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: {Node}))});
1284}
1285
1286static void createRoundingModeDecoration(Instruction *I,
1287 unsigned RoundingModeDeco,
1288 IRBuilder<> &B) {
1289 LLVMContext &Ctx = I->getContext();
1290 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1291 MDNode *RoundingModeNode = MDNode::get(
1292 Context&: Ctx,
1293 MDs: {ConstantAsMetadata::get(
1294 C: ConstantInt::get(Ty: Int32Ty, V: SPIRV::Decoration::FPRoundingMode)),
1295 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: RoundingModeDeco))});
1296 createDecorationIntrinsic(I, Node: RoundingModeNode, B);
1297}
1298
1299static void createSaturatedConversionDecoration(Instruction *I,
1300 IRBuilder<> &B) {
1301 LLVMContext &Ctx = I->getContext();
1302 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1303 MDNode *SaturatedConversionNode =
1304 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1305 Ty: Int32Ty, V: SPIRV::Decoration::SaturatedConversion))});
1306 createDecorationIntrinsic(I, Node: SaturatedConversionNode, B);
1307}
1308
1309Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
1310 if (!Call.isInlineAsm())
1311 return &Call;
1312
1313 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
1314 LLVMContext &Ctx = CurrF->getContext();
1315
1316 Constant *TyC = UndefValue::get(T: IA->getFunctionType());
1317 MDString *ConstraintString = MDString::get(Context&: Ctx, Str: IA->getConstraintString());
1318 SmallVector<Value *> Args = {
1319 buildMD(Arg: TyC),
1320 MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: ConstraintString))};
1321 for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++)
1322 Args.push_back(Elt: Call.getArgOperand(i: OpIdx));
1323
1324 IRBuilder<> B(Call.getParent());
1325 B.SetInsertPoint(&Call);
1326 B.CreateIntrinsic(ID: Intrinsic::spv_inline_asm, Args: {Args});
1327 return &Call;
1328}
1329
1330// Use a tip about rounding mode to create a decoration.
1331void SPIRVEmitIntrinsics::useRoundingMode(ConstrainedFPIntrinsic *FPI,
1332 IRBuilder<> &B) {
1333 std::optional<RoundingMode> RM = FPI->getRoundingMode();
1334 if (!RM.has_value())
1335 return;
1336 unsigned RoundingModeDeco = std::numeric_limits<unsigned>::max();
1337 switch (RM.value()) {
1338 default:
1339 // ignore unknown rounding modes
1340 break;
1341 case RoundingMode::NearestTiesToEven:
1342 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTE;
1343 break;
1344 case RoundingMode::TowardNegative:
1345 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTN;
1346 break;
1347 case RoundingMode::TowardPositive:
1348 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTP;
1349 break;
1350 case RoundingMode::TowardZero:
1351 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTZ;
1352 break;
1353 case RoundingMode::Dynamic:
1354 case RoundingMode::NearestTiesToAway:
1355 // TODO: check if supported
1356 break;
1357 }
1358 if (RoundingModeDeco == std::numeric_limits<unsigned>::max())
1359 return;
1360 // Convert the tip about rounding mode into a decoration record.
1361 createRoundingModeDecoration(I: FPI, RoundingModeDeco, B);
1362}
1363
1364Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
1365 BasicBlock *ParentBB = I.getParent();
1366 IRBuilder<> B(ParentBB);
1367 B.SetInsertPoint(&I);
1368 SmallVector<Value *, 4> Args;
1369 SmallVector<BasicBlock *> BBCases;
1370 for (auto &Op : I.operands()) {
1371 if (Op.get()->getType()->isSized()) {
1372 Args.push_back(Elt: Op);
1373 } else if (BasicBlock *BB = dyn_cast<BasicBlock>(Val: Op.get())) {
1374 BBCases.push_back(Elt: BB);
1375 Args.push_back(Elt: BlockAddress::get(F: BB->getParent(), BB));
1376 } else {
1377 report_fatal_error(reason: "Unexpected switch operand");
1378 }
1379 }
1380 CallInst *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_switch,
1381 Types: {I.getOperand(i_nocapture: 0)->getType()}, Args: {Args});
1382 // remove switch to avoid its unneeded and undesirable unwrap into branches
1383 // and conditions
1384 replaceAllUsesWith(Src: &I, Dest: NewI);
1385 I.eraseFromParent();
1386 // insert artificial and temporary instruction to preserve valid CFG,
1387 // it will be removed after IR translation pass
1388 B.SetInsertPoint(ParentBB);
1389 IndirectBrInst *BrI = B.CreateIndirectBr(
1390 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: ParentBB->getContext())),
1391 NumDests: BBCases.size());
1392 for (BasicBlock *BBCase : BBCases)
1393 BrI->addDestination(Dest: BBCase);
1394 return BrI;
1395}
1396
1397Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
1398 IRBuilder<> B(I.getParent());
1399 B.SetInsertPoint(&I);
1400 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
1401 SmallVector<Value *, 4> Args;
1402 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
1403 llvm::append_range(C&: Args, R: I.operands());
1404 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types: {Types}, Args: {Args});
1405 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1406 return NewI;
1407}
1408
1409Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
1410 IRBuilder<> B(I.getParent());
1411 B.SetInsertPoint(&I);
1412 Value *Source = I.getOperand(i_nocapture: 0);
1413
1414 // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of
1415 // varying element types. In case of IR coming from older versions of LLVM
1416 // such bitcasts do not provide sufficient information, should be just skipped
1417 // here, and handled in insertPtrCastOrAssignTypeInstr.
1418 if (isPointerTy(T: I.getType())) {
1419 replaceAllUsesWith(Src: &I, Dest: Source);
1420 I.eraseFromParent();
1421 return nullptr;
1422 }
1423
1424 SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
1425 SmallVector<Value *> Args(I.op_begin(), I.op_end());
1426 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_bitcast, Types: {Types}, Args: {Args});
1427 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1428 return NewI;
1429}
1430
1431void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt(
1432 TargetExtType *AssignedType, Value *V, IRBuilder<> &B) {
1433 Type *VTy = V->getType();
1434
1435 // A couple of sanity checks.
1436 assert((isPointerTy(VTy)) && "Expect a pointer type!");
1437 if (Type *ElemTy = getPointeeType(Ty: VTy))
1438 if (ElemTy != AssignedType)
1439 report_fatal_error(reason: "Unexpected pointer element type!");
1440
1441 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: V);
1442 if (!AssignCI) {
1443 GR->buildAssignType(B, Ty: AssignedType, Arg: V);
1444 return;
1445 }
1446
1447 Type *CurrentType =
1448 dyn_cast<ConstantAsMetadata>(
1449 Val: cast<MetadataAsValue>(Val: AssignCI->getOperand(i_nocapture: 1))->getMetadata())
1450 ->getType();
1451 if (CurrentType == AssignedType)
1452 return;
1453
1454 // Builtin types cannot be redeclared or casted.
1455 if (CurrentType->isTargetExtTy())
1456 report_fatal_error(reason: "Type mismatch " + CurrentType->getTargetExtName() +
1457 "/" + AssignedType->getTargetExtName() +
1458 " for value " + V->getName(),
1459 gen_crash_diag: false);
1460
1461 // Our previous guess about the type seems to be wrong, let's update
1462 // inferred type according to a new, more precise type information.
1463 GR->updateAssignType(AssignCI, Arg: V, OfType: getNormalizedPoisonValue(Ty: AssignedType));
1464}
1465
1466void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
1467 Instruction *I, Value *Pointer, Type *ExpectedElementType,
1468 unsigned OperandToReplace, IRBuilder<> &B) {
1469 TypeValidated.insert(x: I);
1470
1471 // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType
1472 Type *PointerElemTy = deduceElementTypeHelper(I: Pointer, UnknownElemTypeI8: false);
1473 if (PointerElemTy == ExpectedElementType ||
1474 isEquivalentTypes(Ty1: PointerElemTy, Ty2: ExpectedElementType))
1475 return;
1476
1477 setInsertPointSkippingPhis(B, I);
1478 Value *ExpectedElementVal = getNormalizedPoisonValue(Ty: ExpectedElementType);
1479 MetadataAsValue *VMD = buildMD(Arg: ExpectedElementVal);
1480 unsigned AddressSpace = getPointerAddressSpace(T: Pointer->getType());
1481 bool FirstPtrCastOrAssignPtrType = true;
1482
1483 // Do not emit new spv_ptrcast if equivalent one already exists or when
1484 // spv_assign_ptr_type already targets this pointer with the same element
1485 // type.
1486 if (Pointer->hasUseList()) {
1487 for (auto User : Pointer->users()) {
1488 auto *II = dyn_cast<IntrinsicInst>(Val: User);
1489 if (!II ||
1490 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
1491 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
1492 II->getOperand(i_nocapture: 0) != Pointer)
1493 continue;
1494
1495 // There is some spv_ptrcast/spv_assign_ptr_type already targeting this
1496 // pointer.
1497 FirstPtrCastOrAssignPtrType = false;
1498 if (II->getOperand(i_nocapture: 1) != VMD ||
1499 dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 2))->getSExtValue() !=
1500 AddressSpace)
1501 continue;
1502
1503 // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the
1504 // same element type and address space.
1505 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
1506 return;
1507
1508 // This must be a spv_ptrcast, do not emit new if this one has the same BB
1509 // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type.
1510 if (II->getParent() != I->getParent())
1511 continue;
1512
1513 I->setOperand(i: OperandToReplace, Val: II);
1514 return;
1515 }
1516 }
1517
1518 if (isa<Instruction>(Val: Pointer) || isa<Argument>(Val: Pointer)) {
1519 if (FirstPtrCastOrAssignPtrType) {
1520 // If this would be the first spv_ptrcast, do not emit spv_ptrcast and
1521 // emit spv_assign_ptr_type instead.
1522 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
1523 return;
1524 } else if (isTodoType(Op: Pointer)) {
1525 eraseTodoType(Op: Pointer);
1526 if (!isa<CallInst>(Val: Pointer) && !isa<GetElementPtrInst>(Val: Pointer)) {
1527 // If this wouldn't be the first spv_ptrcast but existing type info is
1528 // uncomplete, update spv_assign_ptr_type arguments.
1529 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Pointer)) {
1530 Type *PrevElemTy = GR->findDeducedElementType(Val: Pointer);
1531 assert(PrevElemTy);
1532 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1533 std::make_pair(x&: I, y&: Pointer)};
1534 GR->updateAssignType(AssignCI, Arg: Pointer, OfType: ExpectedElementVal);
1535 propagateElemType(Op: Pointer, ElemTy: PrevElemTy, VisitedSubst);
1536 } else {
1537 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
1538 }
1539 return;
1540 }
1541 }
1542 }
1543
1544 // Emit spv_ptrcast
1545 SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()};
1546 SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(C: AddressSpace)};
1547 auto *PtrCastI = B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
1548 I->setOperand(i: OperandToReplace, Val: PtrCastI);
1549 // We need to set up a pointee type for the newly created spv_ptrcast.
1550 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: PtrCastI);
1551}
1552
1553void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
1554 IRBuilder<> &B) {
1555 // Handle basic instructions:
1556 StoreInst *SI = dyn_cast<StoreInst>(Val: I);
1557 if (IsKernelArgInt8(F: CurrF, SI)) {
1558 replacePointerOperandWithPtrCast(
1559 I, Pointer: SI->getValueOperand(), ExpectedElementType: IntegerType::getInt8Ty(C&: CurrF->getContext()),
1560 OperandToReplace: 0, B);
1561 }
1562 if (SI) {
1563 Value *Op = SI->getValueOperand();
1564 Value *Pointer = SI->getPointerOperand();
1565 Type *OpTy = Op->getType();
1566 if (auto *OpI = dyn_cast<Instruction>(Val: Op))
1567 OpTy = restoreMutatedType(GR, I: OpI, Ty: OpTy);
1568 if (OpTy == Op->getType())
1569 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: Op, UnknownElemTypeI8: false);
1570 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 1, B);
1571 return;
1572 }
1573 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
1574 Value *Pointer = LI->getPointerOperand();
1575 Type *OpTy = LI->getType();
1576 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1577 if (Type *ElemTy = GR->findDeducedElementType(Val: LI)) {
1578 OpTy = getTypedPointerWrapper(ElemTy, AS: PtrTy->getAddressSpace());
1579 } else {
1580 Type *NewOpTy = OpTy;
1581 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: LI, UnknownElemTypeI8: false);
1582 if (OpTy == NewOpTy)
1583 insertTodoType(Op: Pointer);
1584 }
1585 }
1586 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
1587 return;
1588 }
1589 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(Val: I)) {
1590 Value *Pointer = GEPI->getPointerOperand();
1591 Type *OpTy = GEPI->getSourceElementType();
1592 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
1593 if (isNestedPointer(Ty: OpTy))
1594 insertTodoType(Op: Pointer);
1595 return;
1596 }
1597
1598 // TODO: review and merge with existing logics:
1599 // Handle calls to builtins (non-intrinsics):
1600 CallInst *CI = dyn_cast<CallInst>(Val: I);
1601 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
1602 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
1603 return;
1604
1605 // collect information about formal parameter types
1606 std::string DemangledName =
1607 getOclOrSpirvBuiltinDemangledName(Name: CI->getCalledFunction()->getName());
1608 Function *CalledF = CI->getCalledFunction();
1609 SmallVector<Type *, 4> CalledArgTys;
1610 bool HaveTypes = false;
1611 for (unsigned OpIdx = 0; OpIdx < CalledF->arg_size(); ++OpIdx) {
1612 Argument *CalledArg = CalledF->getArg(i: OpIdx);
1613 Type *ArgType = CalledArg->getType();
1614 if (!isPointerTy(T: ArgType)) {
1615 CalledArgTys.push_back(Elt: nullptr);
1616 } else if (Type *ArgTypeElem = getPointeeType(Ty: ArgType)) {
1617 CalledArgTys.push_back(Elt: ArgTypeElem);
1618 HaveTypes = true;
1619 } else {
1620 Type *ElemTy = GR->findDeducedElementType(Val: CalledArg);
1621 if (!ElemTy && hasPointeeTypeAttr(Arg: CalledArg))
1622 ElemTy = getPointeeTypeByAttr(Arg: CalledArg);
1623 if (!ElemTy) {
1624 ElemTy = getPointeeTypeByCallInst(DemangledName, CalledF, OpIdx);
1625 if (ElemTy) {
1626 GR->addDeducedElementType(Val: CalledArg, Ty: normalizeType(Ty: ElemTy));
1627 } else {
1628 for (User *U : CalledArg->users()) {
1629 if (Instruction *Inst = dyn_cast<Instruction>(Val: U)) {
1630 if ((ElemTy = deduceElementTypeHelper(I: Inst, UnknownElemTypeI8: false)) != nullptr)
1631 break;
1632 }
1633 }
1634 }
1635 }
1636 HaveTypes |= ElemTy != nullptr;
1637 CalledArgTys.push_back(Elt: ElemTy);
1638 }
1639 }
1640
1641 if (DemangledName.empty() && !HaveTypes)
1642 return;
1643
1644 for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) {
1645 Value *ArgOperand = CI->getArgOperand(i: OpIdx);
1646 if (!isPointerTy(T: ArgOperand->getType()))
1647 continue;
1648
1649 // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs()
1650 if (!isa<Instruction>(Val: ArgOperand) && !isa<Argument>(Val: ArgOperand)) {
1651 // However, we may have assumptions about the formal argument's type and
1652 // may have a need to insert a ptr cast for the actual parameter of this
1653 // call.
1654 Argument *CalledArg = CalledF->getArg(i: OpIdx);
1655 if (!GR->findDeducedElementType(Val: CalledArg))
1656 continue;
1657 }
1658
1659 Type *ExpectedType =
1660 OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr;
1661 if (!ExpectedType && !DemangledName.empty())
1662 ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType(
1663 DemangledCall: DemangledName, ArgIdx: OpIdx, Ctx&: I->getContext());
1664 if (!ExpectedType || ExpectedType->isVoidTy())
1665 continue;
1666
1667 if (ExpectedType->isTargetExtTy() &&
1668 !isTypedPointerWrapper(ExtTy: cast<TargetExtType>(Val: ExpectedType)))
1669 insertAssignPtrTypeTargetExt(AssignedType: cast<TargetExtType>(Val: ExpectedType),
1670 V: ArgOperand, B);
1671 else
1672 replacePointerOperandWithPtrCast(I: CI, Pointer: ArgOperand, ExpectedElementType: ExpectedType, OperandToReplace: OpIdx, B);
1673 }
1674}
1675
1676Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
1677 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
1678 // type in LLT and IRTranslator will replace it by the scalar.
1679 if (isVector1(Ty: I.getType()))
1680 return &I;
1681
1682 SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType(),
1683 I.getOperand(i_nocapture: 1)->getType(),
1684 I.getOperand(i_nocapture: 2)->getType()};
1685 IRBuilder<> B(I.getParent());
1686 B.SetInsertPoint(&I);
1687 SmallVector<Value *> Args(I.op_begin(), I.op_end());
1688 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_insertelt, Types: {Types}, Args: {Args});
1689 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1690 return NewI;
1691}
1692
1693Instruction *
1694SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
1695 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
1696 // type in LLT and IRTranslator will replace it by the scalar.
1697 if (isVector1(Ty: I.getVectorOperandType()))
1698 return &I;
1699
1700 IRBuilder<> B(I.getParent());
1701 B.SetInsertPoint(&I);
1702 SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
1703 I.getIndexOperand()->getType()};
1704 SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
1705 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_extractelt, Types: {Types}, Args: {Args});
1706 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1707 return NewI;
1708}
1709
1710Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
1711 IRBuilder<> B(I.getParent());
1712 B.SetInsertPoint(&I);
1713 SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
1714 SmallVector<Value *> Args;
1715 for (auto &Op : I.operands())
1716 if (isa<UndefValue>(Val: Op))
1717 Args.push_back(Elt: UndefValue::get(T: B.getInt32Ty()));
1718 else
1719 Args.push_back(Elt: Op);
1720 for (auto &Op : I.indices())
1721 Args.push_back(Elt: B.getInt32(C: Op));
1722 Instruction *NewI =
1723 B.CreateIntrinsic(ID: Intrinsic::spv_insertv, Types: {Types}, Args: {Args});
1724 replaceMemInstrUses(Old: &I, New: NewI, B);
1725 return NewI;
1726}
1727
1728Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
1729 if (I.getAggregateOperand()->getType()->isAggregateType())
1730 return &I;
1731 IRBuilder<> B(I.getParent());
1732 B.SetInsertPoint(&I);
1733 SmallVector<Value *> Args(I.operands());
1734 for (auto &Op : I.indices())
1735 Args.push_back(Elt: B.getInt32(C: Op));
1736 auto *NewI =
1737 B.CreateIntrinsic(ID: Intrinsic::spv_extractv, Types: {I.getType()}, Args: {Args});
1738 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1739 return NewI;
1740}
1741
1742Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
1743 if (!I.getType()->isAggregateType())
1744 return &I;
1745 IRBuilder<> B(I.getParent());
1746 B.SetInsertPoint(&I);
1747 TrackConstants = false;
1748 const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
1749 MachineMemOperand::Flags Flags =
1750 TLI->getLoadMemOperandFlags(LI: I, DL: CurrF->getDataLayout());
1751 auto *NewI =
1752 B.CreateIntrinsic(ID: Intrinsic::spv_load, Types: {I.getOperand(i_nocapture: 0)->getType()},
1753 Args: {I.getPointerOperand(), B.getInt16(C: Flags),
1754 B.getInt8(C: I.getAlign().value())});
1755 replaceMemInstrUses(Old: &I, New: NewI, B);
1756 return NewI;
1757}
1758
1759Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
1760 if (!AggrStores.contains(V: &I))
1761 return &I;
1762 IRBuilder<> B(I.getParent());
1763 B.SetInsertPoint(&I);
1764 TrackConstants = false;
1765 const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
1766 MachineMemOperand::Flags Flags =
1767 TLI->getStoreMemOperandFlags(SI: I, DL: CurrF->getDataLayout());
1768 auto *PtrOp = I.getPointerOperand();
1769 auto *NewI = B.CreateIntrinsic(
1770 ID: Intrinsic::spv_store, Types: {I.getValueOperand()->getType(), PtrOp->getType()},
1771 Args: {I.getValueOperand(), PtrOp, B.getInt16(C: Flags),
1772 B.getInt8(C: I.getAlign().value())});
1773 NewI->copyMetadata(SrcInst: I);
1774 I.eraseFromParent();
1775 return NewI;
1776}
1777
1778Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
1779 Value *ArraySize = nullptr;
1780 if (I.isArrayAllocation()) {
1781 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*I.getFunction());
1782 if (!STI->canUseExtension(
1783 E: SPIRV::Extension::SPV_INTEL_variable_length_array))
1784 report_fatal_error(
1785 reason: "array allocation: this instruction requires the following "
1786 "SPIR-V extension: SPV_INTEL_variable_length_array",
1787 gen_crash_diag: false);
1788 ArraySize = I.getArraySize();
1789 }
1790 IRBuilder<> B(I.getParent());
1791 B.SetInsertPoint(&I);
1792 TrackConstants = false;
1793 Type *PtrTy = I.getType();
1794 auto *NewI =
1795 ArraySize
1796 ? B.CreateIntrinsic(ID: Intrinsic::spv_alloca_array,
1797 Types: {PtrTy, ArraySize->getType()},
1798 Args: {ArraySize, B.getInt8(C: I.getAlign().value())})
1799 : B.CreateIntrinsic(ID: Intrinsic::spv_alloca, Types: {PtrTy},
1800 Args: {B.getInt8(C: I.getAlign().value())});
1801 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1802 return NewI;
1803}
1804
1805Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
1806 assert(I.getType()->isAggregateType() && "Aggregate result is expected");
1807 IRBuilder<> B(I.getParent());
1808 B.SetInsertPoint(&I);
1809 SmallVector<Value *> Args(I.operands());
1810 Args.push_back(Elt: B.getInt32(
1811 C: static_cast<uint32_t>(getMemScope(Ctx&: I.getContext(), Id: I.getSyncScopeID()))));
1812 Args.push_back(Elt: B.getInt32(
1813 C: static_cast<uint32_t>(getMemSemantics(Ord: I.getSuccessOrdering()))));
1814 Args.push_back(Elt: B.getInt32(
1815 C: static_cast<uint32_t>(getMemSemantics(Ord: I.getFailureOrdering()))));
1816 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_cmpxchg,
1817 Types: {I.getPointerOperand()->getType()}, Args: {Args});
1818 replaceMemInstrUses(Old: &I, New: NewI, B);
1819 return NewI;
1820}
1821
1822Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
1823 IRBuilder<> B(I.getParent());
1824 B.SetInsertPoint(&I);
1825 B.CreateIntrinsic(ID: Intrinsic::spv_unreachable, Args: {});
1826 return &I;
1827}
1828
1829void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV,
1830 IRBuilder<> &B) {
1831 // Skip special artifical variable llvm.global.annotations.
1832 if (GV.getName() == "llvm.global.annotations")
1833 return;
1834 Constant *Init = nullptr;
1835 if (hasInitializer(GV: &GV)) {
1836 // Deduce element type and store results in Global Registry.
1837 // Result is ignored, because TypedPointerType is not supported
1838 // by llvm IR general logic.
1839 deduceElementTypeHelper(I: &GV, UnknownElemTypeI8: false);
1840 Init = GV.getInitializer();
1841 Type *Ty = isAggrConstForceInt32(V: Init) ? B.getInt32Ty() : Init->getType();
1842 Constant *Const = isAggrConstForceInt32(V: Init) ? B.getInt32(C: 1) : Init;
1843 auto *InitInst = B.CreateIntrinsic(ID: Intrinsic::spv_init_global,
1844 Types: {GV.getType(), Ty}, Args: {&GV, Const});
1845 InitInst->setArgOperand(i: 1, v: Init);
1846 }
1847 if (!Init && GV.use_empty())
1848 B.CreateIntrinsic(ID: Intrinsic::spv_unref_global, Types: GV.getType(), Args: &GV);
1849}
1850
1851// Return true, if we can't decide what is the pointee type now and will get
1852// back to the question later. Return false is spv_assign_ptr_type is not needed
1853// or can be inserted immediately.
1854bool SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I,
1855 IRBuilder<> &B,
1856 bool UnknownElemTypeI8) {
1857 reportFatalOnTokenType(I);
1858 if (!isPointerTy(T: I->getType()) || !requireAssignType(I))
1859 return false;
1860
1861 setInsertPointAfterDef(B, I);
1862 if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) {
1863 GR->buildAssignPtr(B, ElemTy, Arg: I);
1864 return false;
1865 }
1866 return true;
1867}
1868
1869void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I,
1870 IRBuilder<> &B) {
1871 // TODO: extend the list of functions with known result types
1872 static StringMap<unsigned> ResTypeWellKnown = {
1873 {"async_work_group_copy", WellKnownTypes::Event},
1874 {"async_work_group_strided_copy", WellKnownTypes::Event},
1875 {"__spirv_GroupAsyncCopy", WellKnownTypes::Event}};
1876
1877 reportFatalOnTokenType(I);
1878
1879 bool IsKnown = false;
1880 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
1881 if (!CI->isIndirectCall() && !CI->isInlineAsm() &&
1882 CI->getCalledFunction() && !CI->getCalledFunction()->isIntrinsic()) {
1883 Function *CalledF = CI->getCalledFunction();
1884 std::string DemangledName =
1885 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1886 FPDecorationId DecorationId = FPDecorationId::NONE;
1887 if (DemangledName.length() > 0)
1888 DemangledName =
1889 SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName, DecorationId: &DecorationId);
1890 auto ResIt = ResTypeWellKnown.find(Key: DemangledName);
1891 if (ResIt != ResTypeWellKnown.end()) {
1892 IsKnown = true;
1893 setInsertPointAfterDef(B, I);
1894 switch (ResIt->second) {
1895 case WellKnownTypes::Event:
1896 GR->buildAssignType(
1897 B, Ty: TargetExtType::get(Context&: I->getContext(), Name: "spirv.Event"), Arg: I);
1898 break;
1899 }
1900 }
1901 // check if a floating rounding mode or saturation info is present
1902 switch (DecorationId) {
1903 default:
1904 break;
1905 case FPDecorationId::SAT:
1906 createSaturatedConversionDecoration(I: CI, B);
1907 break;
1908 case FPDecorationId::RTE:
1909 createRoundingModeDecoration(
1910 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTE, B);
1911 break;
1912 case FPDecorationId::RTZ:
1913 createRoundingModeDecoration(
1914 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTZ, B);
1915 break;
1916 case FPDecorationId::RTP:
1917 createRoundingModeDecoration(
1918 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTP, B);
1919 break;
1920 case FPDecorationId::RTN:
1921 createRoundingModeDecoration(
1922 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTN, B);
1923 break;
1924 }
1925 }
1926 }
1927
1928 Type *Ty = I->getType();
1929 if (!IsKnown && !Ty->isVoidTy() && !isPointerTy(T: Ty) && requireAssignType(I)) {
1930 setInsertPointAfterDef(B, I);
1931 Type *TypeToAssign = Ty;
1932 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
1933 if (II->getIntrinsicID() == Intrinsic::spv_const_composite ||
1934 II->getIntrinsicID() == Intrinsic::spv_undef) {
1935 auto It = AggrConstTypes.find(Val: II);
1936 if (It == AggrConstTypes.end())
1937 report_fatal_error(reason: "Unknown composite intrinsic type");
1938 TypeToAssign = It->second;
1939 }
1940 }
1941 TypeToAssign = restoreMutatedType(GR, I, Ty: TypeToAssign);
1942 GR->buildAssignType(B, Ty: TypeToAssign, Arg: I);
1943 }
1944 for (const auto &Op : I->operands()) {
1945 if (isa<ConstantPointerNull>(Val: Op) || isa<UndefValue>(Val: Op) ||
1946 // Check GetElementPtrConstantExpr case.
1947 (isa<ConstantExpr>(Val: Op) && isa<GEPOperator>(Val: Op))) {
1948 setInsertPointSkippingPhis(B, I);
1949 Type *OpTy = Op->getType();
1950 if (isa<UndefValue>(Val: Op) && OpTy->isAggregateType()) {
1951 CallInst *AssignCI =
1952 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {B.getInt32Ty()}, Arg: Op,
1953 Arg2: UndefValue::get(T: B.getInt32Ty()), Imms: {}, B);
1954 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: AssignCI);
1955 } else if (!isa<Instruction>(Val: Op)) {
1956 Type *OpTy = Op->getType();
1957 Type *OpTyElem = getPointeeType(Ty: OpTy);
1958 if (OpTyElem) {
1959 GR->buildAssignPtr(B, ElemTy: OpTyElem, Arg: Op);
1960 } else if (isPointerTy(T: OpTy)) {
1961 Type *ElemTy = GR->findDeducedElementType(Val: Op);
1962 GR->buildAssignPtr(B, ElemTy: ElemTy ? ElemTy : deduceElementType(I: Op, UnknownElemTypeI8: true),
1963 Arg: Op);
1964 } else {
1965 Value *OpTyVal = Op;
1966 if (OpTy->isTargetExtTy()) {
1967 // We need to do this in order to be consistent with how target ext
1968 // types are handled in `processInstrAfterVisit`
1969 OpTyVal = getNormalizedPoisonValue(Ty: OpTy);
1970 }
1971 CallInst *AssignCI =
1972 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {OpTy},
1973 Arg: getNormalizedPoisonValue(Ty: OpTy), Arg2: OpTyVal, Imms: {}, B);
1974 GR->addAssignPtrTypeInstr(Val: OpTyVal, AssignPtrTyCI: AssignCI);
1975 }
1976 }
1977 }
1978 }
1979}
1980
1981bool SPIRVEmitIntrinsics::shouldTryToAddMemAliasingDecoration(
1982 Instruction *Inst) {
1983 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*Inst->getFunction());
1984 if (!STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing))
1985 return false;
1986 // Add aliasing decorations to internal load and store intrinsics
1987 // and atomic instructions, skipping atomic store as it won't have ID to
1988 // attach the decoration.
1989 CallInst *CI = dyn_cast<CallInst>(Val: Inst);
1990 if (!CI)
1991 return false;
1992 if (Function *Fun = CI->getCalledFunction()) {
1993 if (Fun->isIntrinsic()) {
1994 switch (Fun->getIntrinsicID()) {
1995 case Intrinsic::spv_load:
1996 case Intrinsic::spv_store:
1997 return true;
1998 default:
1999 return false;
2000 }
2001 }
2002 std::string Name = getOclOrSpirvBuiltinDemangledName(Name: Fun->getName());
2003 const std::string Prefix = "__spirv_Atomic";
2004 const bool IsAtomic = Name.find(str: Prefix) == 0;
2005
2006 if (!Fun->getReturnType()->isVoidTy() && IsAtomic)
2007 return true;
2008 }
2009 return false;
2010}
2011
2012void SPIRVEmitIntrinsics::insertSpirvDecorations(Instruction *I,
2013 IRBuilder<> &B) {
2014 if (MDNode *MD = I->getMetadata(Kind: "spirv.Decorations")) {
2015 setInsertPointAfterDef(B, I);
2016 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, Types: {I->getType()},
2017 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
2018 }
2019 // Lower alias.scope/noalias metadata
2020 {
2021 auto processMemAliasingDecoration = [&](unsigned Kind) {
2022 if (MDNode *AliasListMD = I->getMetadata(KindID: Kind)) {
2023 if (shouldTryToAddMemAliasingDecoration(Inst: I)) {
2024 uint32_t Dec = Kind == LLVMContext::MD_alias_scope
2025 ? SPIRV::Decoration::AliasScopeINTEL
2026 : SPIRV::Decoration::NoAliasINTEL;
2027 SmallVector<Value *, 3> Args = {
2028 I, ConstantInt::get(Ty: B.getInt32Ty(), V: Dec),
2029 MetadataAsValue::get(Context&: I->getContext(), MD: AliasListMD)};
2030 setInsertPointAfterDef(B, I);
2031 B.CreateIntrinsic(ID: Intrinsic::spv_assign_aliasing_decoration,
2032 Types: {I->getType()}, Args: {Args});
2033 }
2034 }
2035 };
2036 processMemAliasingDecoration(LLVMContext::MD_alias_scope);
2037 processMemAliasingDecoration(LLVMContext::MD_noalias);
2038 }
2039 // MD_fpmath
2040 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_fpmath)) {
2041 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*I->getFunction());
2042 bool AllowFPMaxError =
2043 STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_fp_max_error);
2044 if (!AllowFPMaxError)
2045 return;
2046
2047 setInsertPointAfterDef(B, I);
2048 B.CreateIntrinsic(ID: Intrinsic::spv_assign_fpmaxerror_decoration,
2049 Types: {I->getType()},
2050 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
2051 }
2052}
2053
2054void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
2055 IRBuilder<> &B) {
2056 auto *II = dyn_cast<IntrinsicInst>(Val: I);
2057 bool IsConstComposite =
2058 II && II->getIntrinsicID() == Intrinsic::spv_const_composite;
2059 if (IsConstComposite && TrackConstants) {
2060 setInsertPointAfterDef(B, I);
2061 auto t = AggrConsts.find(Val: I);
2062 assert(t != AggrConsts.end());
2063 auto *NewOp =
2064 buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
2065 Types: {II->getType(), II->getType()}, Arg: t->second, Arg2: I, Imms: {}, B);
2066 replaceAllUsesWith(Src: I, Dest: NewOp, DeleteOld: false);
2067 NewOp->setArgOperand(i: 0, v: I);
2068 }
2069 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
2070 for (const auto &Op : I->operands()) {
2071 if (isa<PHINode>(Val: I) || isa<SwitchInst>(Val: I) ||
2072 !(isa<ConstantData>(Val: Op) || isa<ConstantExpr>(Val: Op)))
2073 continue;
2074 unsigned OpNo = Op.getOperandNo();
2075 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
2076 (II->paramHasAttr(ArgNo: OpNo, Kind: Attribute::ImmArg))))
2077 continue;
2078
2079 if (!BPrepared) {
2080 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
2081 : B.SetInsertPoint(I);
2082 BPrepared = true;
2083 }
2084 Type *OpTy = Op->getType();
2085 Type *OpElemTy = GR->findDeducedElementType(Val: Op);
2086 Value *NewOp = Op;
2087 if (OpTy->isTargetExtTy()) {
2088 // Since this value is replaced by poison, we need to do the same in
2089 // `insertAssignTypeIntrs`.
2090 Value *OpTyVal = getNormalizedPoisonValue(Ty: OpTy);
2091 NewOp = buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
2092 Types: {OpTy, OpTyVal->getType()}, Arg: Op, Arg2: OpTyVal, Imms: {}, B);
2093 }
2094 if (!IsConstComposite && isPointerTy(T: OpTy) && OpElemTy != nullptr &&
2095 OpElemTy != IntegerType::getInt8Ty(C&: I->getContext())) {
2096 SmallVector<Type *, 2> Types = {OpTy, OpTy};
2097 SmallVector<Value *, 2> Args = {
2098 NewOp, buildMD(Arg: getNormalizedPoisonValue(Ty: OpElemTy)),
2099 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
2100 CallInst *PtrCasted =
2101 B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
2102 GR->buildAssignPtr(B, ElemTy: OpElemTy, Arg: PtrCasted);
2103 NewOp = PtrCasted;
2104 }
2105 if (NewOp != Op)
2106 I->setOperand(i: OpNo, Val: NewOp);
2107 }
2108 if (Named.insert(x: I).second)
2109 emitAssignName(I, B);
2110}
2111
2112Type *SPIRVEmitIntrinsics::deduceFunParamElementType(Function *F,
2113 unsigned OpIdx) {
2114 std::unordered_set<Function *> FVisited;
2115 return deduceFunParamElementType(F, OpIdx, FVisited);
2116}
2117
2118Type *SPIRVEmitIntrinsics::deduceFunParamElementType(
2119 Function *F, unsigned OpIdx, std::unordered_set<Function *> &FVisited) {
2120 // maybe a cycle
2121 if (!FVisited.insert(x: F).second)
2122 return nullptr;
2123
2124 std::unordered_set<Value *> Visited;
2125 SmallVector<std::pair<Function *, unsigned>> Lookup;
2126 // search in function's call sites
2127 for (User *U : F->users()) {
2128 CallInst *CI = dyn_cast<CallInst>(Val: U);
2129 if (!CI || OpIdx >= CI->arg_size())
2130 continue;
2131 Value *OpArg = CI->getArgOperand(i: OpIdx);
2132 if (!isPointerTy(T: OpArg->getType()))
2133 continue;
2134 // maybe we already know operand's element type
2135 if (Type *KnownTy = GR->findDeducedElementType(Val: OpArg))
2136 return KnownTy;
2137 // try to deduce from the operand itself
2138 Visited.clear();
2139 if (Type *Ty = deduceElementTypeHelper(I: OpArg, Visited, UnknownElemTypeI8: false))
2140 return Ty;
2141 // search in actual parameter's users
2142 for (User *OpU : OpArg->users()) {
2143 Instruction *Inst = dyn_cast<Instruction>(Val: OpU);
2144 if (!Inst || Inst == CI)
2145 continue;
2146 Visited.clear();
2147 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8: false))
2148 return Ty;
2149 }
2150 // check if it's a formal parameter of the outer function
2151 if (!CI->getParent() || !CI->getParent()->getParent())
2152 continue;
2153 Function *OuterF = CI->getParent()->getParent();
2154 if (FVisited.find(x: OuterF) != FVisited.end())
2155 continue;
2156 for (unsigned i = 0; i < OuterF->arg_size(); ++i) {
2157 if (OuterF->getArg(i) == OpArg) {
2158 Lookup.push_back(Elt: std::make_pair(x&: OuterF, y&: i));
2159 break;
2160 }
2161 }
2162 }
2163
2164 // search in function parameters
2165 for (auto &Pair : Lookup) {
2166 if (Type *Ty = deduceFunParamElementType(F: Pair.first, OpIdx: Pair.second, FVisited))
2167 return Ty;
2168 }
2169
2170 return nullptr;
2171}
2172
2173void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F,
2174 IRBuilder<> &B) {
2175 B.SetInsertPointPastAllocas(F);
2176 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
2177 Argument *Arg = F->getArg(i: OpIdx);
2178 if (!isUntypedPointerTy(T: Arg->getType()))
2179 continue;
2180 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
2181 if (ElemTy)
2182 continue;
2183 if (hasPointeeTypeAttr(Arg) &&
2184 (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) {
2185 GR->buildAssignPtr(B, ElemTy, Arg);
2186 continue;
2187 }
2188 // search in function's call sites
2189 for (User *U : F->users()) {
2190 CallInst *CI = dyn_cast<CallInst>(Val: U);
2191 if (!CI || OpIdx >= CI->arg_size())
2192 continue;
2193 Value *OpArg = CI->getArgOperand(i: OpIdx);
2194 if (!isPointerTy(T: OpArg->getType()))
2195 continue;
2196 // maybe we already know operand's element type
2197 if ((ElemTy = GR->findDeducedElementType(Val: OpArg)) != nullptr)
2198 break;
2199 }
2200 if (ElemTy) {
2201 GR->buildAssignPtr(B, ElemTy, Arg);
2202 continue;
2203 }
2204 if (HaveFunPtrs) {
2205 for (User *U : Arg->users()) {
2206 CallInst *CI = dyn_cast<CallInst>(Val: U);
2207 if (CI && !isa<IntrinsicInst>(Val: CI) && CI->isIndirectCall() &&
2208 CI->getCalledOperand() == Arg &&
2209 CI->getParent()->getParent() == CurrF) {
2210 SmallVector<std::pair<Value *, unsigned>> Ops;
2211 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy&: ElemTy, IsPostprocessing: false);
2212 if (ElemTy) {
2213 GR->buildAssignPtr(B, ElemTy, Arg);
2214 break;
2215 }
2216 }
2217 }
2218 }
2219 }
2220}
2221
2222void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) {
2223 B.SetInsertPointPastAllocas(F);
2224 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
2225 Argument *Arg = F->getArg(i: OpIdx);
2226 if (!isUntypedPointerTy(T: Arg->getType()))
2227 continue;
2228 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
2229 if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) {
2230 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Arg)) {
2231 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
2232 GR->updateAssignType(AssignCI, Arg, OfType: getNormalizedPoisonValue(Ty: ElemTy));
2233 propagateElemType(Op: Arg, ElemTy: IntegerType::getInt8Ty(C&: F->getContext()),
2234 VisitedSubst);
2235 } else {
2236 GR->buildAssignPtr(B, ElemTy, Arg);
2237 }
2238 }
2239 }
2240}
2241
2242static FunctionType *getFunctionPointerElemType(Function *F,
2243 SPIRVGlobalRegistry *GR) {
2244 FunctionType *FTy = F->getFunctionType();
2245 bool IsNewFTy = false;
2246 SmallVector<Type *, 4> ArgTys;
2247 for (Argument &Arg : F->args()) {
2248 Type *ArgTy = Arg.getType();
2249 if (ArgTy->isPointerTy())
2250 if (Type *ElemTy = GR->findDeducedElementType(Val: &Arg)) {
2251 IsNewFTy = true;
2252 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
2253 }
2254 ArgTys.push_back(Elt: ArgTy);
2255 }
2256 return IsNewFTy
2257 ? FunctionType::get(Result: FTy->getReturnType(), Params: ArgTys, isVarArg: FTy->isVarArg())
2258 : FTy;
2259}
2260
2261bool SPIRVEmitIntrinsics::processFunctionPointers(Module &M) {
2262 SmallVector<Function *> Worklist;
2263 for (auto &F : M) {
2264 if (F.isIntrinsic())
2265 continue;
2266 if (F.isDeclaration()) {
2267 for (User *U : F.users()) {
2268 CallInst *CI = dyn_cast<CallInst>(Val: U);
2269 if (!CI || CI->getCalledFunction() != &F) {
2270 Worklist.push_back(Elt: &F);
2271 break;
2272 }
2273 }
2274 } else {
2275 if (F.user_empty())
2276 continue;
2277 Type *FPElemTy = GR->findDeducedElementType(Val: &F);
2278 if (!FPElemTy)
2279 FPElemTy = getFunctionPointerElemType(F: &F, GR);
2280 for (User *U : F.users()) {
2281 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
2282 if (!II || II->arg_size() != 3 || II->getOperand(i_nocapture: 0) != &F)
2283 continue;
2284 if (II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type ||
2285 II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
2286 GR->updateAssignType(AssignCI: II, Arg: &F, OfType: getNormalizedPoisonValue(Ty: FPElemTy));
2287 break;
2288 }
2289 }
2290 }
2291 }
2292 if (Worklist.empty())
2293 return false;
2294
2295 std::string ServiceFunName = SPIRV_BACKEND_SERVICE_FUN_NAME;
2296 if (!getVacantFunctionName(M, Name&: ServiceFunName))
2297 report_fatal_error(
2298 reason: "cannot allocate a name for the internal service function");
2299 LLVMContext &Ctx = M.getContext();
2300 Function *SF =
2301 Function::Create(Ty: FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: {}, isVarArg: false),
2302 Linkage: GlobalValue::PrivateLinkage, N: ServiceFunName, M);
2303 SF->addFnAttr(SPIRV_BACKEND_SERVICE_FUN_NAME, Val: "");
2304 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: SF);
2305 IRBuilder<> IRB(BB);
2306
2307 for (Function *F : Worklist) {
2308 SmallVector<Value *> Args;
2309 for (const auto &Arg : F->args())
2310 Args.push_back(Elt: getNormalizedPoisonValue(Ty: Arg.getType()));
2311 IRB.CreateCall(Callee: F, Args);
2312 }
2313 IRB.CreateRetVoid();
2314
2315 return true;
2316}
2317
2318// Apply types parsed from demangled function declarations.
2319void SPIRVEmitIntrinsics::applyDemangledPtrArgTypes(IRBuilder<> &B) {
2320 DenseMap<Function *, CallInst *> Ptrcasts;
2321 for (auto It : FDeclPtrTys) {
2322 Function *F = It.first;
2323 for (auto *U : F->users()) {
2324 CallInst *CI = dyn_cast<CallInst>(Val: U);
2325 if (!CI || CI->getCalledFunction() != F)
2326 continue;
2327 unsigned Sz = CI->arg_size();
2328 for (auto [Idx, ElemTy] : It.second) {
2329 if (Idx >= Sz)
2330 continue;
2331 Value *Param = CI->getArgOperand(i: Idx);
2332 if (GR->findDeducedElementType(Val: Param) || isa<GlobalValue>(Val: Param))
2333 continue;
2334 if (Argument *Arg = dyn_cast<Argument>(Val: Param)) {
2335 if (!hasPointeeTypeAttr(Arg)) {
2336 B.SetInsertPointPastAllocas(Arg->getParent());
2337 B.SetCurrentDebugLocation(DebugLoc());
2338 GR->buildAssignPtr(B, ElemTy, Arg);
2339 }
2340 } else if (isa<GetElementPtrInst>(Val: Param)) {
2341 replaceUsesOfWithSpvPtrcast(Op: Param, ElemTy: normalizeType(Ty: ElemTy), I: CI,
2342 Ptrcasts);
2343 } else if (isa<Instruction>(Val: Param)) {
2344 GR->addDeducedElementType(Val: Param, Ty: normalizeType(Ty: ElemTy));
2345 // insertAssignTypeIntrs() will complete buildAssignPtr()
2346 } else {
2347 B.SetInsertPoint(CI->getParent()
2348 ->getParent()
2349 ->getEntryBlock()
2350 .getFirstNonPHIOrDbgOrAlloca());
2351 GR->buildAssignPtr(B, ElemTy, Arg: Param);
2352 }
2353 CallInst *Ref = dyn_cast<CallInst>(Val: Param);
2354 if (!Ref)
2355 continue;
2356 Function *RefF = Ref->getCalledFunction();
2357 if (!RefF || !isPointerTy(T: RefF->getReturnType()) ||
2358 GR->findDeducedElementType(Val: RefF))
2359 continue;
2360 ElemTy = normalizeType(Ty: ElemTy);
2361 GR->addDeducedElementType(Val: RefF, Ty: ElemTy);
2362 GR->addReturnType(
2363 ArgF: RefF, DerivedTy: TypedPointerType::get(
2364 ElementType: ElemTy, AddressSpace: getPointerAddressSpace(T: RefF->getReturnType())));
2365 }
2366 }
2367 }
2368}
2369
2370bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
2371 if (Func.isDeclaration())
2372 return false;
2373
2374 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F: Func);
2375 GR = ST.getSPIRVGlobalRegistry();
2376
2377 if (!CurrF)
2378 HaveFunPtrs =
2379 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers);
2380
2381 CurrF = &Func;
2382 IRBuilder<> B(Func.getContext());
2383 AggrConsts.clear();
2384 AggrConstTypes.clear();
2385 AggrStores.clear();
2386
2387 // fix GEP result types ahead of inference
2388 for (auto &I : instructions(F&: Func)) {
2389 auto *Ref = dyn_cast<GetElementPtrInst>(Val: &I);
2390 if (!Ref || GR->findDeducedElementType(Val: Ref))
2391 continue;
2392 if (Type *GepTy = getGEPType(Ref))
2393 GR->addDeducedElementType(Val: Ref, Ty: normalizeType(Ty: GepTy));
2394 }
2395
2396 processParamTypesByFunHeader(F: CurrF, B);
2397
2398 // StoreInst's operand type can be changed during the next transformations,
2399 // so we need to store it in the set. Also store already transformed types.
2400 for (auto &I : instructions(F&: Func)) {
2401 StoreInst *SI = dyn_cast<StoreInst>(Val: &I);
2402 if (!SI)
2403 continue;
2404 Type *ElTy = SI->getValueOperand()->getType();
2405 if (ElTy->isAggregateType() || ElTy->isVectorTy())
2406 AggrStores.insert(V: &I);
2407 }
2408
2409 B.SetInsertPoint(TheBB: &Func.getEntryBlock(), IP: Func.getEntryBlock().begin());
2410 for (auto &GV : Func.getParent()->globals())
2411 processGlobalValue(GV, B);
2412
2413 preprocessUndefs(B);
2414 preprocessCompositeConstants(B);
2415 SmallVector<Instruction *> Worklist(
2416 llvm::make_pointer_range(Range: instructions(F&: Func)));
2417
2418 applyDemangledPtrArgTypes(B);
2419
2420 // Pass forward: use operand to deduce instructions result.
2421 for (auto &I : Worklist) {
2422 // Don't emit intrinsincs for convergence intrinsics.
2423 if (isConvergenceIntrinsic(I))
2424 continue;
2425
2426 bool Postpone = insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: false);
2427 // if Postpone is true, we can't decide on pointee type yet
2428 insertAssignTypeIntrs(I, B);
2429 insertPtrCastOrAssignTypeInstr(I, B);
2430 insertSpirvDecorations(I, B);
2431 // if instruction requires a pointee type set, let's check if we know it
2432 // already, and force it to be i8 if not
2433 if (Postpone && !GR->findAssignPtrTypeInstr(Val: I))
2434 insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: true);
2435
2436 if (auto *FPI = dyn_cast<ConstrainedFPIntrinsic>(Val: I))
2437 useRoundingMode(FPI, B);
2438 }
2439
2440 // Pass backward: use instructions results to specify/update/cast operands
2441 // where needed.
2442 SmallPtrSet<Instruction *, 4> IncompleteRets;
2443 for (auto &I : llvm::reverse(C: instructions(F&: Func)))
2444 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets);
2445
2446 // Pass forward for PHIs only, their operands are not preceed the instruction
2447 // in meaning of `instructions(Func)`.
2448 for (BasicBlock &BB : Func)
2449 for (PHINode &Phi : BB.phis())
2450 if (isPointerTy(T: Phi.getType()))
2451 deduceOperandElementType(I: &Phi, IncompleteRets: nullptr);
2452
2453 for (auto *I : Worklist) {
2454 TrackConstants = true;
2455 if (!I->getType()->isVoidTy() || isa<StoreInst>(Val: I))
2456 setInsertPointAfterDef(B, I);
2457 // Visitors return either the original/newly created instruction for further
2458 // processing, nullptr otherwise.
2459 I = visit(I&: *I);
2460 if (!I)
2461 continue;
2462
2463 // Don't emit intrinsics for convergence operations.
2464 if (isConvergenceIntrinsic(I))
2465 continue;
2466
2467 processInstrAfterVisit(I, B);
2468 }
2469
2470 return true;
2471}
2472
2473// Try to deduce a better type for pointers to untyped ptr.
2474bool SPIRVEmitIntrinsics::postprocessTypes(Module &M) {
2475 if (!GR || TodoTypeSz == 0)
2476 return false;
2477
2478 unsigned SzTodo = TodoTypeSz;
2479 DenseMap<Value *, SmallPtrSet<Value *, 4>> ToProcess;
2480 for (auto [Op, Enabled] : TodoType) {
2481 // TODO: add isa<CallInst>(Op) to continue
2482 if (!Enabled || isa<GetElementPtrInst>(Val: Op))
2483 continue;
2484 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Op);
2485 Type *KnownTy = GR->findDeducedElementType(Val: Op);
2486 if (!KnownTy || !AssignCI)
2487 continue;
2488 assert(Op == AssignCI->getArgOperand(0));
2489 // Try to improve the type deduced after all Functions are processed.
2490 if (auto *CI = dyn_cast<Instruction>(Val: Op)) {
2491 CurrF = CI->getParent()->getParent();
2492 std::unordered_set<Value *> Visited;
2493 if (Type *ElemTy = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8: false, IgnoreKnownType: true)) {
2494 if (ElemTy != KnownTy) {
2495 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
2496 propagateElemType(Op: CI, ElemTy, VisitedSubst);
2497 eraseTodoType(Op);
2498 continue;
2499 }
2500 }
2501 }
2502
2503 if (Op->hasUseList()) {
2504 for (User *U : Op->users()) {
2505 Instruction *Inst = dyn_cast<Instruction>(Val: U);
2506 if (Inst && !isa<IntrinsicInst>(Val: Inst))
2507 ToProcess[Inst].insert(Ptr: Op);
2508 }
2509 }
2510 }
2511 if (TodoTypeSz == 0)
2512 return true;
2513
2514 for (auto &F : M) {
2515 CurrF = &F;
2516 SmallPtrSet<Instruction *, 4> IncompleteRets;
2517 for (auto &I : llvm::reverse(C: instructions(F))) {
2518 auto It = ToProcess.find(Val: &I);
2519 if (It == ToProcess.end())
2520 continue;
2521 It->second.remove_if(P: [this](Value *V) { return !isTodoType(Op: V); });
2522 if (It->second.size() == 0)
2523 continue;
2524 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets, AskOps: &It->second, IsPostprocessing: true);
2525 if (TodoTypeSz == 0)
2526 return true;
2527 }
2528 }
2529
2530 return SzTodo > TodoTypeSz;
2531}
2532
2533// Parse and store argument types of function declarations where needed.
2534void SPIRVEmitIntrinsics::parseFunDeclarations(Module &M) {
2535 for (auto &F : M) {
2536 if (!F.isDeclaration() || F.isIntrinsic())
2537 continue;
2538 // get the demangled name
2539 std::string DemangledName = getOclOrSpirvBuiltinDemangledName(Name: F.getName());
2540 if (DemangledName.empty())
2541 continue;
2542 // allow only OpGroupAsyncCopy use case at the moment
2543 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F);
2544 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
2545 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
2546 if (Opcode != SPIRV::OpGroupAsyncCopy)
2547 continue;
2548 // find pointer arguments
2549 SmallVector<unsigned> Idxs;
2550 for (unsigned OpIdx = 0; OpIdx < F.arg_size(); ++OpIdx) {
2551 Argument *Arg = F.getArg(i: OpIdx);
2552 if (isPointerTy(T: Arg->getType()) && !hasPointeeTypeAttr(Arg))
2553 Idxs.push_back(Elt: OpIdx);
2554 }
2555 if (!Idxs.size())
2556 continue;
2557 // parse function arguments
2558 LLVMContext &Ctx = F.getContext();
2559 SmallVector<StringRef, 10> TypeStrs;
2560 SPIRV::parseBuiltinTypeStr(BuiltinArgsTypeStrs&: TypeStrs, DemangledCall: DemangledName, Ctx);
2561 if (!TypeStrs.size())
2562 continue;
2563 // find type info for pointer arguments
2564 for (unsigned Idx : Idxs) {
2565 if (Idx >= TypeStrs.size())
2566 continue;
2567 if (Type *ElemTy =
2568 SPIRV::parseBuiltinCallArgumentType(TypeStr: TypeStrs[Idx].trim(), Ctx))
2569 if (TypedPointerType::isValidElementType(ElemTy) &&
2570 !ElemTy->isTargetExtTy())
2571 FDeclPtrTys[&F].push_back(Elt: std::make_pair(x&: Idx, y&: ElemTy));
2572 }
2573 }
2574}
2575
2576bool SPIRVEmitIntrinsics::runOnModule(Module &M) {
2577 bool Changed = false;
2578
2579 parseFunDeclarations(M);
2580
2581 TodoType.clear();
2582 for (auto &F : M)
2583 Changed |= runOnFunction(Func&: F);
2584
2585 // Specify function parameters after all functions were processed.
2586 for (auto &F : M) {
2587 // check if function parameter types are set
2588 CurrF = &F;
2589 if (!F.isDeclaration() && !F.isIntrinsic()) {
2590 IRBuilder<> B(F.getContext());
2591 processParamTypes(F: &F, B);
2592 }
2593 }
2594
2595 CanTodoType = false;
2596 Changed |= postprocessTypes(M);
2597
2598 if (HaveFunPtrs)
2599 Changed |= processFunctionPointers(M);
2600
2601 return Changed;
2602}
2603
2604ModulePass *llvm::createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM) {
2605 return new SPIRVEmitIntrinsics(TM);
2606}
2607