1//===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass emits SPIRV intrinsics keeping essential high-level information for
10// the translation of LLVM IR to SPIR-V.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRV.h"
15#include "SPIRVBuiltins.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVTargetMachine.h"
18#include "SPIRVUtils.h"
19#include "llvm/ADT/DenseSet.h"
20#include "llvm/ADT/StringSet.h"
21#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/InstIterator.h"
23#include "llvm/IR/InstVisitor.h"
24#include "llvm/IR/IntrinsicsSPIRV.h"
25#include "llvm/IR/PatternMatch.h"
26#include "llvm/IR/TypedPointerType.h"
27#include "llvm/Support/CommandLine.h"
28#include "llvm/Transforms/Utils/Local.h"
29
30#include <cassert>
31#include <queue>
32#include <unordered_set>
33
34// This pass performs the following transformation on LLVM IR level required
35// for the following translation to SPIR-V:
36// - replaces direct usages of aggregate constants with target-specific
37// intrinsics;
38// - replaces aggregates-related instructions (extract/insert, ld/st, etc)
39// with a target-specific intrinsics;
40// - emits intrinsics for the global variable initializers since IRTranslator
41// doesn't handle them and it's not very convenient to translate them
42// ourselves;
43// - emits intrinsics to keep track of the string names assigned to the values;
44// - emits intrinsics to keep track of constants (this is necessary to have an
45// LLVM IR constant after the IRTranslation is completed) for their further
46// deduplication;
47// - emits intrinsics to keep track of original LLVM types of the values
48// to be able to emit proper SPIR-V types eventually.
49//
50// TODO: consider removing spv.track.constant in favor of spv.assign.type.
51
52using namespace llvm;
53
54static cl::opt<bool>
55 SpirvEmitOpNames("spirv-emit-op-names",
56 cl::desc("Emit OpName for all instructions"),
57 cl::init(Val: false));
58
59namespace llvm::SPIRV {
60#define GET_BuiltinGroup_DECL
61#include "SPIRVGenTables.inc"
62} // namespace llvm::SPIRV
63
64namespace {
65
66static bool isaGEP(const Value *V) {
67 return isa<StructuredGEPInst>(Val: V) || isa<GetElementPtrInst>(Val: V);
68}
69
70class SPIRVEmitIntrinsics
71 : public ModulePass,
72 public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
73 SPIRVTargetMachine *TM = nullptr;
74 SPIRVGlobalRegistry *GR = nullptr;
75 Function *CurrF = nullptr;
76 bool TrackConstants = true;
77 bool HaveFunPtrs = false;
78 DenseMap<Instruction *, Constant *> AggrConsts;
79 DenseMap<Instruction *, Type *> AggrConstTypes;
80 DenseSet<Instruction *> AggrStores;
81 std::unordered_set<Value *> Named;
82
83 // map of function declarations to <pointer arg index => element type>
84 DenseMap<Function *, SmallVector<std::pair<unsigned, Type *>>> FDeclPtrTys;
85
86 // a register of Instructions that don't have a complete type definition
87 bool CanTodoType = true;
88 unsigned TodoTypeSz = 0;
89 DenseMap<Value *, bool> TodoType;
90 void insertTodoType(Value *Op) {
91 // TODO: add isa<CallInst>(Op) to no-insert
92 if (CanTodoType && !isaGEP(V: Op)) {
93 auto It = TodoType.try_emplace(Key: Op, Args: true);
94 if (It.second)
95 ++TodoTypeSz;
96 }
97 }
98 void eraseTodoType(Value *Op) {
99 auto It = TodoType.find(Val: Op);
100 if (It != TodoType.end() && It->second) {
101 It->second = false;
102 --TodoTypeSz;
103 }
104 }
105 bool isTodoType(Value *Op) {
106 if (isaGEP(V: Op))
107 return false;
108 auto It = TodoType.find(Val: Op);
109 return It != TodoType.end() && It->second;
110 }
111 // a register of Instructions that were visited by deduceOperandElementType()
112 // to validate operand types with an instruction
113 std::unordered_set<Instruction *> TypeValidated;
114
115 // well known result types of builtins
116 enum WellKnownTypes { Event };
117
118 // deduce element type of untyped pointers
119 Type *deduceElementType(Value *I, bool UnknownElemTypeI8);
120 Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8);
121 Type *deduceElementTypeHelper(Value *I, std::unordered_set<Value *> &Visited,
122 bool UnknownElemTypeI8,
123 bool IgnoreKnownType = false);
124 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
125 bool UnknownElemTypeI8);
126 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
127 std::unordered_set<Value *> &Visited,
128 bool UnknownElemTypeI8);
129 Type *deduceElementTypeByUsersDeep(Value *Op,
130 std::unordered_set<Value *> &Visited,
131 bool UnknownElemTypeI8);
132 void maybeAssignPtrType(Type *&Ty, Value *I, Type *RefTy,
133 bool UnknownElemTypeI8);
134
135 // deduce nested types of composites
136 Type *deduceNestedTypeHelper(User *U, bool UnknownElemTypeI8);
137 Type *deduceNestedTypeHelper(User *U, Type *Ty,
138 std::unordered_set<Value *> &Visited,
139 bool UnknownElemTypeI8);
140
141 // deduce Types of operands of the Instruction if possible
142 void deduceOperandElementType(Instruction *I,
143 SmallPtrSet<Instruction *, 4> *IncompleteRets,
144 const SmallPtrSet<Value *, 4> *AskOps = nullptr,
145 bool IsPostprocessing = false);
146
147 void preprocessCompositeConstants(IRBuilder<> &B);
148 void preprocessUndefs(IRBuilder<> &B);
149
150 Type *reconstructType(Value *Op, bool UnknownElemTypeI8,
151 bool IsPostprocessing);
152
153 void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
154 void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
155 bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
156 bool UnknownElemTypeI8);
157 void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B);
158 void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V,
159 IRBuilder<> &B);
160 void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer,
161 Type *ExpectedElementType,
162 unsigned OperandToReplace,
163 IRBuilder<> &B);
164 void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
165 bool shouldTryToAddMemAliasingDecoration(Instruction *Inst);
166 void insertSpirvDecorations(Instruction *I, IRBuilder<> &B);
167 void insertConstantsForFPFastMathDefault(Module &M);
168 void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
169 void processParamTypes(Function *F, IRBuilder<> &B);
170 void processParamTypesByFunHeader(Function *F, IRBuilder<> &B);
171 Type *deduceFunParamElementType(Function *F, unsigned OpIdx);
172 Type *deduceFunParamElementType(Function *F, unsigned OpIdx,
173 std::unordered_set<Function *> &FVisited);
174
175 bool deduceOperandElementTypeCalledFunction(
176 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
177 Type *&KnownElemTy, bool &Incomplete);
178 void deduceOperandElementTypeFunctionPointer(
179 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
180 Type *&KnownElemTy, bool IsPostprocessing);
181 bool deduceOperandElementTypeFunctionRet(
182 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
183 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing,
184 Type *&KnownElemTy, Value *Op, Function *F);
185
186 CallInst *buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy);
187 void replaceUsesOfWithSpvPtrcast(Value *Op, Type *ElemTy, Instruction *I,
188 DenseMap<Function *, CallInst *> Ptrcasts);
189 void propagateElemType(Value *Op, Type *ElemTy,
190 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
191 void
192 propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
193 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
194 void propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
195 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
196 std::unordered_set<Value *> &Visited,
197 DenseMap<Function *, CallInst *> Ptrcasts);
198
199 void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
200 void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src,
201 Instruction *Dest, bool DeleteOld = true);
202
203 void applyDemangledPtrArgTypes(IRBuilder<> &B);
204
205 GetElementPtrInst *simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP);
206
207 bool runOnFunction(Function &F);
208 bool postprocessTypes(Module &M);
209 bool processFunctionPointers(Module &M);
210 void parseFunDeclarations(Module &M);
211
212 void useRoundingMode(ConstrainedFPIntrinsic *FPI, IRBuilder<> &B);
213
214 void emitUnstructuredLoopControls(Function &F, IRBuilder<> &B);
215
216 // Tries to walk the type accessed by the given GEP instruction.
217 // For each nested type access, one of the 2 callbacks is called:
218 // - OnLiteralIndexing when the index is a known constant value.
219 // Parameters:
220 // PointedType: the pointed type resulting of this indexing.
221 // If the parent type is an array, this is the index in the array.
222 // If the parent type is a struct, this is the field index.
223 // Index: index of the element in the parent type.
224 // - OnDynamnicIndexing when the index is a non-constant value.
225 // This callback is only called when indexing into an array.
226 // Parameters:
227 // ElementType: the type of the elements stored in the parent array.
228 // Offset: the Value* containing the byte offset into the array.
229 // Return true if an error occurred during the walk, false otherwise.
230 bool walkLogicalAccessChain(
231 GetElementPtrInst &GEP,
232 const std::function<void(Type *PointedType, uint64_t Index)>
233 &OnLiteralIndexing,
234 const std::function<void(Type *ElementType, Value *Offset)>
235 &OnDynamicIndexing);
236
237 // Returns the type accessed using the given GEP instruction by relying
238 // on the GEP type.
239 // FIXME: GEP types are not supposed to be used to retrieve the pointed
240 // type. This must be fixed.
241 Type *getGEPType(GetElementPtrInst *GEP);
242
243 // Returns the type accessed using the given GEP instruction by walking
244 // the source type using the GEP indices.
245 // FIXME: without help from the frontend, this method cannot reliably retrieve
246 // the stored type, nor can robustly determine the depth of the type
247 // we are accessing.
248 Type *getGEPTypeLogical(GetElementPtrInst *GEP);
249
250 Instruction *buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP);
251
252public:
253 static char ID;
254 SPIRVEmitIntrinsics(SPIRVTargetMachine *TM = nullptr)
255 : ModulePass(ID), TM(TM) {}
256 Instruction *visitInstruction(Instruction &I) { return &I; }
257 Instruction *visitSwitchInst(SwitchInst &I);
258 Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
259 Instruction *visitIntrinsicInst(IntrinsicInst &I);
260 Instruction *visitBitCastInst(BitCastInst &I);
261 Instruction *visitInsertElementInst(InsertElementInst &I);
262 Instruction *visitExtractElementInst(ExtractElementInst &I);
263 Instruction *visitInsertValueInst(InsertValueInst &I);
264 Instruction *visitExtractValueInst(ExtractValueInst &I);
265 Instruction *visitLoadInst(LoadInst &I);
266 Instruction *visitStoreInst(StoreInst &I);
267 Instruction *visitAllocaInst(AllocaInst &I);
268 Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
269 Instruction *visitUnreachableInst(UnreachableInst &I);
270 Instruction *visitCallInst(CallInst &I);
271
272 StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
273
274 bool runOnModule(Module &M) override;
275
276 void getAnalysisUsage(AnalysisUsage &AU) const override {
277 ModulePass::getAnalysisUsage(AU);
278 }
279};
280
281bool isConvergenceIntrinsic(const Instruction *I) {
282 const auto *II = dyn_cast<IntrinsicInst>(Val: I);
283 if (!II)
284 return false;
285
286 return II->getIntrinsicID() == Intrinsic::experimental_convergence_entry ||
287 II->getIntrinsicID() == Intrinsic::experimental_convergence_loop ||
288 II->getIntrinsicID() == Intrinsic::experimental_convergence_anchor;
289}
290
291bool expectIgnoredInIRTranslation(const Instruction *I) {
292 const auto *II = dyn_cast<IntrinsicInst>(Val: I);
293 if (!II)
294 return false;
295 switch (II->getIntrinsicID()) {
296 case Intrinsic::invariant_start:
297 case Intrinsic::spv_resource_handlefrombinding:
298 case Intrinsic::spv_resource_getpointer:
299 return true;
300 default:
301 return false;
302 }
303}
304
305// Returns the source pointer from `I` ignoring intermediate ptrcast.
306Value *getPointerRoot(Value *I) {
307 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
308 if (II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
309 Value *V = II->getArgOperand(i: 0);
310 return getPointerRoot(I: V);
311 }
312 }
313 return I;
314}
315
316} // namespace
317
318char SPIRVEmitIntrinsics::ID = 0;
319
320INITIALIZE_PASS(SPIRVEmitIntrinsics, "emit-intrinsics", "SPIRV emit intrinsics",
321 false, false)
322
323static inline bool isAssignTypeInstr(const Instruction *I) {
324 return isa<IntrinsicInst>(Val: I) &&
325 cast<IntrinsicInst>(Val: I)->getIntrinsicID() == Intrinsic::spv_assign_type;
326}
327
328static bool isMemInstrToReplace(Instruction *I) {
329 return isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I) || isa<InsertValueInst>(Val: I) ||
330 isa<ExtractValueInst>(Val: I) || isa<AtomicCmpXchgInst>(Val: I);
331}
332
333static bool isAggrConstForceInt32(const Value *V) {
334 return isa<ConstantArray>(Val: V) || isa<ConstantStruct>(Val: V) ||
335 isa<ConstantDataArray>(Val: V) ||
336 (isa<ConstantAggregateZero>(Val: V) && !V->getType()->isVectorTy());
337}
338
339static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
340 if (isa<PHINode>(Val: I))
341 B.SetInsertPoint(I->getParent()->getFirstNonPHIOrDbgOrAlloca());
342 else
343 B.SetInsertPoint(I);
344}
345
346static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I) {
347 B.SetCurrentDebugLocation(I->getDebugLoc());
348 if (I->getType()->isVoidTy())
349 B.SetInsertPoint(I->getNextNode());
350 else
351 B.SetInsertPoint(*I->getInsertionPointAfterDef());
352}
353
354static bool requireAssignType(Instruction *I) {
355 if (const auto *Intr = dyn_cast<IntrinsicInst>(Val: I)) {
356 switch (Intr->getIntrinsicID()) {
357 case Intrinsic::invariant_start:
358 case Intrinsic::invariant_end:
359 return false;
360 }
361 }
362 return true;
363}
364
365static inline void reportFatalOnTokenType(const Instruction *I) {
366 if (I->getType()->isTokenTy())
367 report_fatal_error(reason: "A token is encountered but SPIR-V without extensions "
368 "does not support token type",
369 gen_crash_diag: false);
370}
371
372static void emitAssignName(Instruction *I, IRBuilder<> &B) {
373 if (!I->hasName() || I->getType()->isAggregateType() ||
374 expectIgnoredInIRTranslation(I))
375 return;
376
377 // We want to be conservative when adding the names because they can interfere
378 // with later optimizations.
379 bool KeepName = SpirvEmitOpNames;
380 if (!KeepName) {
381 if (isa<AllocaInst>(Val: I)) {
382 KeepName = true;
383 } else if (auto *CI = dyn_cast<CallBase>(Val: I)) {
384 Function *F = CI->getCalledFunction();
385 if (F && F->getName().starts_with(Prefix: "llvm.spv.alloca"))
386 KeepName = true;
387 }
388 }
389
390 if (!KeepName)
391 return;
392
393 reportFatalOnTokenType(I);
394 setInsertPointAfterDef(B, I);
395 LLVMContext &Ctx = I->getContext();
396 std::vector<Value *> Args = {
397 I, MetadataAsValue::get(
398 Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: I->getName())))};
399 B.CreateIntrinsic(ID: Intrinsic::spv_assign_name, Types: {I->getType()}, Args);
400}
401
402void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest,
403 bool DeleteOld) {
404 GR->replaceAllUsesWith(Old: Src, New: Dest, DeleteOld);
405 // Update uncomplete type records if any
406 if (isTodoType(Op: Src)) {
407 if (DeleteOld)
408 eraseTodoType(Op: Src);
409 insertTodoType(Op: Dest);
410 }
411}
412
413void SPIRVEmitIntrinsics::replaceAllUsesWithAndErase(IRBuilder<> &B,
414 Instruction *Src,
415 Instruction *Dest,
416 bool DeleteOld) {
417 replaceAllUsesWith(Src, Dest, DeleteOld);
418 std::string Name = Src->hasName() ? Src->getName().str() : "";
419 Src->eraseFromParent();
420 if (!Name.empty()) {
421 Dest->setName(Name);
422 if (Named.insert(x: Dest).second)
423 emitAssignName(I: Dest, B);
424 }
425}
426
427static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
428 return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
429 isPointerTy(T: SI->getValueOperand()->getType()) &&
430 isa<Argument>(Val: SI->getValueOperand());
431}
432
433// Maybe restore original function return type.
434static inline Type *restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I,
435 Type *Ty) {
436 CallInst *CI = dyn_cast<CallInst>(Val: I);
437 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
438 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
439 return Ty;
440 if (Type *OriginalTy = GR->findMutated(Val: CI->getCalledFunction()))
441 return OriginalTy;
442 return Ty;
443}
444
445// Reconstruct type with nested element types according to deduced type info.
446// Return nullptr if no detailed type info is available.
447Type *SPIRVEmitIntrinsics::reconstructType(Value *Op, bool UnknownElemTypeI8,
448 bool IsPostprocessing) {
449 Type *Ty = Op->getType();
450 if (auto *OpI = dyn_cast<Instruction>(Val: Op))
451 Ty = restoreMutatedType(GR, I: OpI, Ty);
452 if (!isUntypedPointerTy(T: Ty))
453 return Ty;
454 // try to find the pointee type
455 if (Type *NestedTy = GR->findDeducedElementType(Val: Op))
456 return getTypedPointerWrapper(ElemTy: NestedTy, AS: getPointerAddressSpace(T: Ty));
457 // not a pointer according to the type info (e.g., Event object)
458 CallInst *CI = GR->findAssignPtrTypeInstr(Val: Op);
459 if (CI) {
460 MetadataAsValue *MD = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
461 return cast<ConstantAsMetadata>(Val: MD->getMetadata())->getType();
462 }
463 if (UnknownElemTypeI8) {
464 if (!IsPostprocessing)
465 insertTodoType(Op);
466 return getTypedPointerWrapper(ElemTy: IntegerType::getInt8Ty(C&: Op->getContext()),
467 AS: getPointerAddressSpace(T: Ty));
468 }
469 return nullptr;
470}
471
472CallInst *SPIRVEmitIntrinsics::buildSpvPtrcast(Function *F, Value *Op,
473 Type *ElemTy) {
474 IRBuilder<> B(Op->getContext());
475 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
476 // spv_ptrcast's argument Op denotes an instruction that generates
477 // a value, and we may use getInsertionPointAfterDef()
478 setInsertPointAfterDef(B, I: OpI);
479 } else if (auto *OpA = dyn_cast<Argument>(Val: Op)) {
480 B.SetInsertPointPastAllocas(OpA->getParent());
481 B.SetCurrentDebugLocation(DebugLoc());
482 } else {
483 B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
484 }
485 Type *OpTy = Op->getType();
486 SmallVector<Type *, 2> Types = {OpTy, OpTy};
487 SmallVector<Value *, 2> Args = {Op, buildMD(Arg: getNormalizedPoisonValue(Ty: ElemTy)),
488 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
489 CallInst *PtrCasted =
490 B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
491 GR->buildAssignPtr(B, ElemTy, Arg: PtrCasted);
492 return PtrCasted;
493}
494
495void SPIRVEmitIntrinsics::replaceUsesOfWithSpvPtrcast(
496 Value *Op, Type *ElemTy, Instruction *I,
497 DenseMap<Function *, CallInst *> Ptrcasts) {
498 Function *F = I->getParent()->getParent();
499 CallInst *PtrCastedI = nullptr;
500 auto It = Ptrcasts.find(Val: F);
501 if (It == Ptrcasts.end()) {
502 PtrCastedI = buildSpvPtrcast(F, Op, ElemTy);
503 Ptrcasts[F] = PtrCastedI;
504 } else {
505 PtrCastedI = It->second;
506 }
507 I->replaceUsesOfWith(From: Op, To: PtrCastedI);
508}
509
510void SPIRVEmitIntrinsics::propagateElemType(
511 Value *Op, Type *ElemTy,
512 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
513 DenseMap<Function *, CallInst *> Ptrcasts;
514 SmallVector<User *> Users(Op->users());
515 for (auto *U : Users) {
516 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
517 continue;
518 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
519 continue;
520 Instruction *UI = dyn_cast<Instruction>(Val: U);
521 // If the instruction was validated already, we need to keep it valid by
522 // keeping current Op type.
523 if (isaGEP(V: UI) || TypeValidated.find(x: UI) != TypeValidated.end())
524 replaceUsesOfWithSpvPtrcast(Op, ElemTy, I: UI, Ptrcasts);
525 }
526}
527
528void SPIRVEmitIntrinsics::propagateElemTypeRec(
529 Value *Op, Type *PtrElemTy, Type *CastElemTy,
530 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
531 std::unordered_set<Value *> Visited;
532 DenseMap<Function *, CallInst *> Ptrcasts;
533 propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited,
534 Ptrcasts: std::move(Ptrcasts));
535}
536
537void SPIRVEmitIntrinsics::propagateElemTypeRec(
538 Value *Op, Type *PtrElemTy, Type *CastElemTy,
539 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
540 std::unordered_set<Value *> &Visited,
541 DenseMap<Function *, CallInst *> Ptrcasts) {
542 if (!Visited.insert(x: Op).second)
543 return;
544 SmallVector<User *> Users(Op->users());
545 for (auto *U : Users) {
546 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
547 continue;
548 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
549 continue;
550 Instruction *UI = dyn_cast<Instruction>(Val: U);
551 // If the instruction was validated already, we need to keep it valid by
552 // keeping current Op type.
553 if (isaGEP(V: UI) || TypeValidated.find(x: UI) != TypeValidated.end())
554 replaceUsesOfWithSpvPtrcast(Op, ElemTy: CastElemTy, I: UI, Ptrcasts);
555 }
556}
557
558// Set element pointer type to the given value of ValueTy and tries to
559// specify this type further (recursively) by Operand value, if needed.
560
561Type *
562SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
563 bool UnknownElemTypeI8) {
564 std::unordered_set<Value *> Visited;
565 return deduceElementTypeByValueDeep(ValueTy, Operand, Visited,
566 UnknownElemTypeI8);
567}
568
569Type *SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(
570 Type *ValueTy, Value *Operand, std::unordered_set<Value *> &Visited,
571 bool UnknownElemTypeI8) {
572 Type *Ty = ValueTy;
573 if (Operand) {
574 if (auto *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
575 if (Type *NestedTy =
576 deduceElementTypeHelper(I: Operand, Visited, UnknownElemTypeI8))
577 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
578 } else {
579 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Operand), Ty, Visited,
580 UnknownElemTypeI8);
581 }
582 }
583 return Ty;
584}
585
586// Traverse User instructions to deduce an element pointer type of the operand.
587Type *SPIRVEmitIntrinsics::deduceElementTypeByUsersDeep(
588 Value *Op, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8) {
589 if (!Op || !isPointerTy(T: Op->getType()) || isa<ConstantPointerNull>(Val: Op) ||
590 isa<UndefValue>(Val: Op))
591 return nullptr;
592
593 if (auto ElemTy = getPointeeType(Ty: Op->getType()))
594 return ElemTy;
595
596 // maybe we already know operand's element type
597 if (Type *KnownTy = GR->findDeducedElementType(Val: Op))
598 return KnownTy;
599
600 for (User *OpU : Op->users()) {
601 if (Instruction *Inst = dyn_cast<Instruction>(Val: OpU)) {
602 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8))
603 return Ty;
604 }
605 }
606 return nullptr;
607}
608
609// Implements what we know in advance about intrinsics and builtin calls
610// TODO: consider feasibility of this particular case to be generalized by
611// encoding knowledge about intrinsics and builtin calls by corresponding
612// specification rules
613static Type *getPointeeTypeByCallInst(StringRef DemangledName,
614 Function *CalledF, unsigned OpIdx) {
615 if ((DemangledName.starts_with(Prefix: "__spirv_ocl_printf(") ||
616 DemangledName.starts_with(Prefix: "printf(")) &&
617 OpIdx == 0)
618 return IntegerType::getInt8Ty(C&: CalledF->getContext());
619 return nullptr;
620}
621
622// Deduce and return a successfully deduced Type of the Instruction,
623// or nullptr otherwise.
624Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(Value *I,
625 bool UnknownElemTypeI8) {
626 std::unordered_set<Value *> Visited;
627 return deduceElementTypeHelper(I, Visited, UnknownElemTypeI8);
628}
629
630void SPIRVEmitIntrinsics::maybeAssignPtrType(Type *&Ty, Value *Op, Type *RefTy,
631 bool UnknownElemTypeI8) {
632 if (isUntypedPointerTy(T: RefTy)) {
633 if (!UnknownElemTypeI8)
634 return;
635 insertTodoType(Op);
636 }
637 Ty = RefTy;
638}
639
640bool SPIRVEmitIntrinsics::walkLogicalAccessChain(
641 GetElementPtrInst &GEP,
642 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
643 const std::function<void(Type *, Value *)> &OnDynamicIndexing) {
644 // We only rewrite i8* GEP. Other should be left as-is.
645 // Valid i8* GEP must always have a single index.
646 assert(GEP.getSourceElementType() ==
647 IntegerType::getInt8Ty(CurrF->getContext()));
648 assert(GEP.getNumIndices() == 1);
649
650 auto &DL = CurrF->getDataLayout();
651 Value *Src = getPointerRoot(I: GEP.getPointerOperand());
652 Type *CurType = deduceElementType(I: Src, UnknownElemTypeI8: true);
653
654 Value *Operand = *GEP.idx_begin();
655 ConstantInt *CI = dyn_cast<ConstantInt>(Val: Operand);
656 if (!CI) {
657 ArrayType *AT = dyn_cast<ArrayType>(Val: CurType);
658 // Operand is not constant. Either we have an array and accept it, or we
659 // give up.
660 if (AT)
661 OnDynamicIndexing(AT->getElementType(), Operand);
662 return AT == nullptr;
663 }
664
665 assert(CI);
666 uint64_t Offset = CI->getZExtValue();
667
668 do {
669 if (ArrayType *AT = dyn_cast<ArrayType>(Val: CurType)) {
670 uint32_t EltTypeSize = DL.getTypeSizeInBits(Ty: AT->getElementType()) / 8;
671 assert(Offset < AT->getNumElements() * EltTypeSize);
672 uint64_t Index = Offset / EltTypeSize;
673 Offset = Offset - (Index * EltTypeSize);
674 CurType = AT->getElementType();
675 OnLiteralIndexing(CurType, Index);
676 } else if (StructType *ST = dyn_cast<StructType>(Val: CurType)) {
677 uint32_t StructSize = DL.getTypeSizeInBits(Ty: ST) / 8;
678 assert(Offset < StructSize);
679 (void)StructSize;
680 const auto &STL = DL.getStructLayout(Ty: ST);
681 unsigned Element = STL->getElementContainingOffset(FixedOffset: Offset);
682 Offset -= STL->getElementOffset(Idx: Element);
683 CurType = ST->getElementType(N: Element);
684 OnLiteralIndexing(CurType, Element);
685 } else if (auto *VT = dyn_cast<FixedVectorType>(Val: CurType)) {
686 Type *EltTy = VT->getElementType();
687 TypeSize EltSizeBits = DL.getTypeSizeInBits(Ty: EltTy);
688 assert(EltSizeBits % 8 == 0 &&
689 "Element type size in bits must be a multiple of 8.");
690 uint32_t EltTypeSize = EltSizeBits / 8;
691 assert(Offset < VT->getNumElements() * EltTypeSize);
692 uint64_t Index = Offset / EltTypeSize;
693 Offset -= Index * EltTypeSize;
694 CurType = EltTy;
695 OnLiteralIndexing(CurType, Index);
696
697 } else {
698 // Unknown composite kind; give up.
699 return true;
700 }
701 } while (Offset > 0);
702
703 return false;
704}
705
706Instruction *
707SPIRVEmitIntrinsics::buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP) {
708 auto &DL = CurrF->getDataLayout();
709 IRBuilder<> B(GEP.getParent());
710 B.SetInsertPoint(&GEP);
711
712 std::vector<Value *> Indices;
713 Indices.push_back(x: ConstantInt::get(
714 Ty: IntegerType::getInt32Ty(C&: CurrF->getContext()), V: 0, /* Signed= */ IsSigned: false));
715 walkLogicalAccessChain(
716 GEP,
717 OnLiteralIndexing: [&Indices, &B](Type *EltType, uint64_t Index) {
718 Indices.push_back(
719 x: ConstantInt::get(Ty: B.getInt64Ty(), V: Index, /* Signed= */ IsSigned: false));
720 },
721 OnDynamicIndexing: [&Indices, &B, &DL](Type *EltType, Value *Offset) {
722 uint32_t EltTypeSize = DL.getTypeSizeInBits(Ty: EltType) / 8;
723 Value *Index = B.CreateUDiv(
724 LHS: Offset, RHS: ConstantInt::get(Ty: Offset->getType(), V: EltTypeSize,
725 /* Signed= */ IsSigned: false));
726 Indices.push_back(x: Index);
727 });
728
729 SmallVector<Type *, 2> Types = {GEP.getType(), GEP.getOperand(i_nocapture: 0)->getType()};
730 SmallVector<Value *, 4> Args;
731 Args.push_back(Elt: B.getInt1(V: GEP.isInBounds()));
732 Args.push_back(Elt: GEP.getOperand(i_nocapture: 0));
733 llvm::append_range(C&: Args, R&: Indices);
734 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types: {Types}, Args: {Args});
735 replaceAllUsesWithAndErase(B, Src: &GEP, Dest: NewI);
736 return NewI;
737}
738
739Type *SPIRVEmitIntrinsics::getGEPTypeLogical(GetElementPtrInst *GEP) {
740
741 Type *CurType = GEP->getResultElementType();
742
743 bool Interrupted = walkLogicalAccessChain(
744 GEP&: *GEP, OnLiteralIndexing: [&CurType](Type *EltType, uint64_t Index) { CurType = EltType; },
745 OnDynamicIndexing: [&CurType](Type *EltType, Value *Index) { CurType = EltType; });
746
747 return Interrupted ? GEP->getResultElementType() : CurType;
748}
749
750Type *SPIRVEmitIntrinsics::getGEPType(GetElementPtrInst *Ref) {
751 if (Ref->getSourceElementType() ==
752 IntegerType::getInt8Ty(C&: CurrF->getContext()) &&
753 TM->getSubtargetImpl()->isLogicalSPIRV()) {
754 return getGEPTypeLogical(GEP: Ref);
755 }
756
757 Type *Ty = nullptr;
758 // TODO: not sure if GetElementPtrInst::getTypeAtIndex() does anything
759 // useful here
760 if (isNestedPointer(Ty: Ref->getSourceElementType())) {
761 Ty = Ref->getSourceElementType();
762 for (Use &U : drop_begin(RangeOrContainer: Ref->indices()))
763 Ty = GetElementPtrInst::getTypeAtIndex(Ty, Idx: U.get());
764 } else {
765 Ty = Ref->getResultElementType();
766 }
767 return Ty;
768}
769
770Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
771 Value *I, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8,
772 bool IgnoreKnownType) {
773 // allow to pass nullptr as an argument
774 if (!I)
775 return nullptr;
776
777 // maybe already known
778 if (!IgnoreKnownType)
779 if (Type *KnownTy = GR->findDeducedElementType(Val: I))
780 return KnownTy;
781
782 // maybe a cycle
783 if (!Visited.insert(x: I).second)
784 return nullptr;
785
786 // fallback value in case when we fail to deduce a type
787 Type *Ty = nullptr;
788 // look for known basic patterns of type inference
789 if (auto *Ref = dyn_cast<AllocaInst>(Val: I)) {
790 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getAllocatedType(), UnknownElemTypeI8);
791 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
792 Ty = getGEPType(Ref);
793 } else if (auto *SGEP = dyn_cast<StructuredGEPInst>(Val: I)) {
794 Ty = SGEP->getResultElementType();
795 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
796 Value *Op = Ref->getPointerOperand();
797 Type *KnownTy = GR->findDeducedElementType(Val: Op);
798 if (!KnownTy)
799 KnownTy = Op->getType();
800 if (Type *ElemTy = getPointeeType(Ty: KnownTy))
801 maybeAssignPtrType(Ty, Op: I, RefTy: ElemTy, UnknownElemTypeI8);
802 } else if (auto *Ref = dyn_cast<GlobalValue>(Val: I)) {
803 if (auto *Fn = dyn_cast<Function>(Val: Ref)) {
804 Ty = SPIRV::getOriginalFunctionType(F: *Fn);
805 GR->addDeducedElementType(Val: I, Ty);
806 } else {
807 Ty = deduceElementTypeByValueDeep(
808 ValueTy: Ref->getValueType(),
809 Operand: Ref->getNumOperands() > 0 ? Ref->getOperand(i: 0) : nullptr, Visited,
810 UnknownElemTypeI8);
811 }
812 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
813 Type *RefTy = deduceElementTypeHelper(I: Ref->getPointerOperand(), Visited,
814 UnknownElemTypeI8);
815 maybeAssignPtrType(Ty, Op: I, RefTy, UnknownElemTypeI8);
816 } else if (auto *Ref = dyn_cast<IntToPtrInst>(Val: I)) {
817 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getDestTy(), UnknownElemTypeI8);
818 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
819 if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
820 isPointerTy(T: Src) && isPointerTy(T: Dest))
821 Ty = deduceElementTypeHelper(I: Ref->getOperand(i_nocapture: 0), Visited,
822 UnknownElemTypeI8);
823 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
824 Value *Op = Ref->getNewValOperand();
825 if (isPointerTy(T: Op->getType()))
826 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
827 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
828 Value *Op = Ref->getValOperand();
829 if (isPointerTy(T: Op->getType()))
830 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
831 } else if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
832 Type *BestTy = nullptr;
833 unsigned MaxN = 1;
834 DenseMap<Type *, unsigned> PhiTys;
835 for (int i = Ref->getNumIncomingValues() - 1; i >= 0; --i) {
836 Ty = deduceElementTypeByUsersDeep(Op: Ref->getIncomingValue(i), Visited,
837 UnknownElemTypeI8);
838 if (!Ty)
839 continue;
840 auto It = PhiTys.try_emplace(Key: Ty, Args: 1);
841 if (!It.second) {
842 ++It.first->second;
843 if (It.first->second > MaxN) {
844 MaxN = It.first->second;
845 BestTy = Ty;
846 }
847 }
848 }
849 if (BestTy)
850 Ty = BestTy;
851 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
852 for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) {
853 Ty = deduceElementTypeByUsersDeep(Op, Visited, UnknownElemTypeI8);
854 if (Ty)
855 break;
856 }
857 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
858 static StringMap<unsigned> ResTypeByArg = {
859 {"to_global", 0},
860 {"to_local", 0},
861 {"to_private", 0},
862 {"__spirv_GenericCastToPtr_ToGlobal", 0},
863 {"__spirv_GenericCastToPtr_ToLocal", 0},
864 {"__spirv_GenericCastToPtr_ToPrivate", 0},
865 {"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
866 {"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
867 {"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
868 // TODO: maybe improve performance by caching demangled names
869
870 auto *II = dyn_cast<IntrinsicInst>(Val: I);
871 if (II && II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
872 auto *HandleType = cast<TargetExtType>(Val: II->getOperand(i_nocapture: 0)->getType());
873 if (HandleType->getTargetExtName() == "spirv.Image" ||
874 HandleType->getTargetExtName() == "spirv.SignedImage") {
875 for (User *U : II->users()) {
876 Ty = cast<Instruction>(Val: U)->getAccessType();
877 if (Ty)
878 break;
879 }
880 } else if (HandleType->getTargetExtName() == "spirv.VulkanBuffer") {
881 // This call is supposed to index into an array
882 Ty = HandleType->getTypeParameter(i: 0);
883 if (Ty->isArrayTy())
884 Ty = Ty->getArrayElementType();
885 else {
886 assert(Ty && Ty->isStructTy());
887 uint32_t Index = cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1))->getZExtValue();
888 Ty = cast<StructType>(Val: Ty)->getElementType(N: Index);
889 }
890 Ty = reconstitutePeeledArrayType(Ty);
891 } else {
892 llvm_unreachable("Unknown handle type for spv_resource_getpointer.");
893 }
894 } else if (II && II->getIntrinsicID() ==
895 Intrinsic::spv_generic_cast_to_ptr_explicit) {
896 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: 0), Visited,
897 UnknownElemTypeI8);
898 } else if (Function *CalledF = CI->getCalledFunction()) {
899 std::string DemangledName =
900 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
901 if (DemangledName.length() > 0)
902 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName);
903 auto AsArgIt = ResTypeByArg.find(Key: DemangledName);
904 if (AsArgIt != ResTypeByArg.end())
905 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: AsArgIt->second),
906 Visited, UnknownElemTypeI8);
907 else if (Type *KnownRetTy = GR->findDeducedElementType(Val: CalledF))
908 Ty = KnownRetTy;
909 }
910 }
911
912 // remember the found relationship
913 if (Ty && !IgnoreKnownType) {
914 // specify nested types if needed, otherwise return unchanged
915 GR->addDeducedElementType(Val: I, Ty: normalizeType(Ty));
916 }
917
918 return Ty;
919}
920
921// Re-create a type of the value if it has untyped pointer fields, also nested.
922// Return the original value type if no corrections of untyped pointer
923// information is found or needed.
924Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(User *U,
925 bool UnknownElemTypeI8) {
926 std::unordered_set<Value *> Visited;
927 return deduceNestedTypeHelper(U, Ty: U->getType(), Visited, UnknownElemTypeI8);
928}
929
930Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
931 User *U, Type *OrigTy, std::unordered_set<Value *> &Visited,
932 bool UnknownElemTypeI8) {
933 if (!U)
934 return OrigTy;
935
936 // maybe already known
937 if (Type *KnownTy = GR->findDeducedCompositeType(Val: U))
938 return KnownTy;
939
940 // maybe a cycle
941 if (!Visited.insert(x: U).second)
942 return OrigTy;
943
944 if (isa<StructType>(Val: OrigTy)) {
945 SmallVector<Type *> Tys;
946 bool Change = false;
947 for (unsigned i = 0; i < U->getNumOperands(); ++i) {
948 Value *Op = U->getOperand(i);
949 assert(Op && "Operands should not be null.");
950 Type *OpTy = Op->getType();
951 Type *Ty = OpTy;
952 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
953 if (Type *NestedTy =
954 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
955 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
956 } else {
957 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
958 UnknownElemTypeI8);
959 }
960 Tys.push_back(Elt: Ty);
961 Change |= Ty != OpTy;
962 }
963 if (Change) {
964 Type *NewTy = StructType::create(Elements: Tys);
965 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
966 return NewTy;
967 }
968 } else if (auto *ArrTy = dyn_cast<ArrayType>(Val: OrigTy)) {
969 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
970 Type *OpTy = ArrTy->getElementType();
971 Type *Ty = OpTy;
972 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
973 if (Type *NestedTy =
974 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
975 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
976 } else {
977 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
978 UnknownElemTypeI8);
979 }
980 if (Ty != OpTy) {
981 Type *NewTy = ArrayType::get(ElementType: Ty, NumElements: ArrTy->getNumElements());
982 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
983 return NewTy;
984 }
985 }
986 } else if (auto *VecTy = dyn_cast<VectorType>(Val: OrigTy)) {
987 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
988 Type *OpTy = VecTy->getElementType();
989 Type *Ty = OpTy;
990 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
991 if (Type *NestedTy =
992 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
993 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
994 } else {
995 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
996 UnknownElemTypeI8);
997 }
998 if (Ty != OpTy) {
999 Type *NewTy = VectorType::get(ElementType: Ty, EC: VecTy->getElementCount());
1000 GR->addDeducedCompositeType(Val: U, Ty: normalizeType(Ty: NewTy));
1001 return NewTy;
1002 }
1003 }
1004 }
1005
1006 return OrigTy;
1007}
1008
1009Type *SPIRVEmitIntrinsics::deduceElementType(Value *I, bool UnknownElemTypeI8) {
1010 if (Type *Ty = deduceElementTypeHelper(I, UnknownElemTypeI8))
1011 return Ty;
1012 if (!UnknownElemTypeI8)
1013 return nullptr;
1014 insertTodoType(Op: I);
1015 return IntegerType::getInt8Ty(C&: I->getContext());
1016}
1017
1018static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I,
1019 Value *PointerOperand) {
1020 Type *PointeeTy = GR->findDeducedElementType(Val: PointerOperand);
1021 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1022 return nullptr;
1023 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType());
1024 if (!PtrTy)
1025 return I->getType();
1026 if (Type *NestedTy = GR->findDeducedElementType(Val: I))
1027 return getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1028 return nullptr;
1029}
1030
1031// Try to deduce element type for a call base. Returns false if this is an
1032// indirect function invocation, and true otherwise.
1033bool SPIRVEmitIntrinsics::deduceOperandElementTypeCalledFunction(
1034 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1035 Type *&KnownElemTy, bool &Incomplete) {
1036 Function *CalledF = CI->getCalledFunction();
1037 if (!CalledF)
1038 return false;
1039 std::string DemangledName =
1040 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1041 if (DemangledName.length() > 0 &&
1042 !StringRef(DemangledName).starts_with(Prefix: "llvm.")) {
1043 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F: *CalledF);
1044 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
1045 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
1046 if (Opcode == SPIRV::OpGroupAsyncCopy) {
1047 for (unsigned i = 0, PtrCnt = 0; i < CI->arg_size() && PtrCnt < 2; ++i) {
1048 Value *Op = CI->getArgOperand(i);
1049 if (!isPointerTy(T: Op->getType()))
1050 continue;
1051 ++PtrCnt;
1052 if (Type *ElemTy = GR->findDeducedElementType(Val: Op))
1053 KnownElemTy = ElemTy; // src will rewrite dest if both are defined
1054 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1055 }
1056 } else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) {
1057 if (CI->arg_size() == 0)
1058 return true;
1059 Value *Op = CI->getArgOperand(i: 0);
1060 if (!isPointerTy(T: Op->getType()))
1061 return true;
1062 switch (Opcode) {
1063 case SPIRV::OpAtomicFAddEXT:
1064 case SPIRV::OpAtomicFMinEXT:
1065 case SPIRV::OpAtomicFMaxEXT:
1066 case SPIRV::OpAtomicLoad:
1067 case SPIRV::OpAtomicCompareExchangeWeak:
1068 case SPIRV::OpAtomicCompareExchange:
1069 case SPIRV::OpAtomicExchange:
1070 case SPIRV::OpAtomicIAdd:
1071 case SPIRV::OpAtomicISub:
1072 case SPIRV::OpAtomicOr:
1073 case SPIRV::OpAtomicXor:
1074 case SPIRV::OpAtomicAnd:
1075 case SPIRV::OpAtomicUMin:
1076 case SPIRV::OpAtomicUMax:
1077 case SPIRV::OpAtomicSMin:
1078 case SPIRV::OpAtomicSMax: {
1079 KnownElemTy = isPointerTy(T: CI->getType()) ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1080 : CI->getType();
1081 if (!KnownElemTy)
1082 return true;
1083 Incomplete = isTodoType(Op);
1084 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1085 } break;
1086 case SPIRV::OpAtomicStore: {
1087 if (CI->arg_size() < 4)
1088 return true;
1089 Value *ValOp = CI->getArgOperand(i: 3);
1090 KnownElemTy = isPointerTy(T: ValOp->getType())
1091 ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1092 : ValOp->getType();
1093 if (!KnownElemTy)
1094 return true;
1095 Incomplete = isTodoType(Op);
1096 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1097 } break;
1098 }
1099 }
1100 }
1101 return true;
1102}
1103
1104// Try to deduce element type for a function pointer.
1105void SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionPointer(
1106 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1107 Type *&KnownElemTy, bool IsPostprocessing) {
1108 Value *Op = CI->getCalledOperand();
1109 if (!Op || !isPointerTy(T: Op->getType()))
1110 return;
1111 Ops.push_back(Elt: std::make_pair(x&: Op, y: std::numeric_limits<unsigned>::max()));
1112 FunctionType *FTy = SPIRV::getOriginalFunctionType(CB: *CI);
1113 bool IsNewFTy = false, IsIncomplete = false;
1114 SmallVector<Type *, 4> ArgTys;
1115 for (auto &&[ParmIdx, Arg] : llvm::enumerate(First: CI->args())) {
1116 Type *ArgTy = Arg->getType();
1117 if (ArgTy->isPointerTy()) {
1118 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg)) {
1119 IsNewFTy = true;
1120 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
1121 if (isTodoType(Op: Arg))
1122 IsIncomplete = true;
1123 } else {
1124 IsIncomplete = true;
1125 }
1126 } else {
1127 ArgTy = FTy->getFunctionParamType(i: ParmIdx);
1128 }
1129 ArgTys.push_back(Elt: ArgTy);
1130 }
1131 Type *RetTy = FTy->getReturnType();
1132 if (CI->getType()->isPointerTy()) {
1133 if (Type *ElemTy = GR->findDeducedElementType(Val: CI)) {
1134 IsNewFTy = true;
1135 RetTy =
1136 getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: CI->getType()));
1137 if (isTodoType(Op: CI))
1138 IsIncomplete = true;
1139 } else {
1140 IsIncomplete = true;
1141 }
1142 }
1143 if (!IsPostprocessing && IsIncomplete)
1144 insertTodoType(Op);
1145 KnownElemTy =
1146 IsNewFTy ? FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: FTy->isVarArg()) : FTy;
1147}
1148
1149bool SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionRet(
1150 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
1151 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing,
1152 Type *&KnownElemTy, Value *Op, Function *F) {
1153 KnownElemTy = GR->findDeducedElementType(Val: F);
1154 if (KnownElemTy)
1155 return false;
1156 if (Type *OpElemTy = GR->findDeducedElementType(Val: Op)) {
1157 OpElemTy = normalizeType(Ty: OpElemTy);
1158 GR->addDeducedElementType(Val: F, Ty: OpElemTy);
1159 GR->addReturnType(
1160 ArgF: F, DerivedTy: TypedPointerType::get(ElementType: OpElemTy,
1161 AddressSpace: getPointerAddressSpace(T: F->getReturnType())));
1162 // non-recursive update of types in function uses
1163 DenseSet<std::pair<Value *, Value *>> VisitedSubst{std::make_pair(x&: I, y&: Op)};
1164 for (User *U : F->users()) {
1165 CallInst *CI = dyn_cast<CallInst>(Val: U);
1166 if (!CI || CI->getCalledFunction() != F)
1167 continue;
1168 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: CI)) {
1169 if (Type *PrevElemTy = GR->findDeducedElementType(Val: CI)) {
1170 GR->updateAssignType(AssignCI, Arg: CI,
1171 OfType: getNormalizedPoisonValue(Ty: OpElemTy));
1172 propagateElemType(Op: CI, ElemTy: PrevElemTy, VisitedSubst);
1173 }
1174 }
1175 }
1176 // Non-recursive update of types in the function uncomplete returns.
1177 // This may happen just once per a function, the latch is a pair of
1178 // findDeducedElementType(F) / addDeducedElementType(F, ...).
1179 // With or without the latch it is a non-recursive call due to
1180 // IncompleteRets set to nullptr in this call.
1181 if (IncompleteRets)
1182 for (Instruction *IncompleteRetI : *IncompleteRets)
1183 deduceOperandElementType(I: IncompleteRetI, IncompleteRets: nullptr, AskOps,
1184 IsPostprocessing);
1185 } else if (IncompleteRets) {
1186 IncompleteRets->insert(Ptr: I);
1187 }
1188 TypeValidated.insert(x: I);
1189 return true;
1190}
1191
1192// If the Instruction has Pointer operands with unresolved types, this function
1193// tries to deduce them. If the Instruction has Pointer operands with known
1194// types which differ from expected, this function tries to insert a bitcast to
1195// resolve the issue.
1196void SPIRVEmitIntrinsics::deduceOperandElementType(
1197 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
1198 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing) {
1199 SmallVector<std::pair<Value *, unsigned>> Ops;
1200 Type *KnownElemTy = nullptr;
1201 bool Incomplete = false;
1202 // look for known basic patterns of type inference
1203 if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1204 if (!isPointerTy(T: I->getType()) ||
1205 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1206 return;
1207 Incomplete = isTodoType(Op: I);
1208 for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
1209 Value *Op = Ref->getIncomingValue(i);
1210 if (isPointerTy(T: Op->getType()))
1211 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1212 }
1213 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1214 KnownElemTy = GR->findDeducedElementType(Val: I);
1215 if (!KnownElemTy)
1216 return;
1217 Incomplete = isTodoType(Op: I);
1218 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(), y: 0));
1219 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1220 if (!isPointerTy(T: I->getType()))
1221 return;
1222 KnownElemTy = GR->findDeducedElementType(Val: I);
1223 if (!KnownElemTy)
1224 return;
1225 Incomplete = isTodoType(Op: I);
1226 Ops.push_back(Elt: std::make_pair(x: Ref->getOperand(i_nocapture: 0), y: 0));
1227 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
1228 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1229 return;
1230 KnownElemTy = Ref->getSourceElementType();
1231 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1232 y: GetElementPtrInst::getPointerOperandIndex()));
1233 } else if (auto *Ref = dyn_cast<StructuredGEPInst>(Val: I)) {
1234 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1235 return;
1236 KnownElemTy = Ref->getBaseType();
1237 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1238 y: StructuredGEPInst::getPointerOperandIndex()));
1239 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
1240 KnownElemTy = I->getType();
1241 if (isUntypedPointerTy(T: KnownElemTy))
1242 return;
1243 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1244 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1245 return;
1246 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1247 y: LoadInst::getPointerOperandIndex()));
1248 } else if (auto *Ref = dyn_cast<StoreInst>(Val: I)) {
1249 if (!(KnownElemTy =
1250 reconstructType(Op: Ref->getValueOperand(), UnknownElemTypeI8: false, IsPostprocessing)))
1251 return;
1252 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1253 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1254 return;
1255 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1256 y: StoreInst::getPointerOperandIndex()));
1257 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1258 KnownElemTy = isPointerTy(T: I->getType())
1259 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1260 : I->getType();
1261 if (!KnownElemTy)
1262 return;
1263 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1264 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1265 y: AtomicCmpXchgInst::getPointerOperandIndex()));
1266 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1267 KnownElemTy = isPointerTy(T: I->getType())
1268 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1269 : I->getType();
1270 if (!KnownElemTy)
1271 return;
1272 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1273 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1274 y: AtomicRMWInst::getPointerOperandIndex()));
1275 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1276 if (!isPointerTy(T: I->getType()) ||
1277 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1278 return;
1279 Incomplete = isTodoType(Op: I);
1280 for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
1281 Value *Op = Ref->getOperand(i_nocapture: i);
1282 if (isPointerTy(T: Op->getType()))
1283 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1284 }
1285 } else if (auto *Ref = dyn_cast<ReturnInst>(Val: I)) {
1286 if (!isPointerTy(T: CurrF->getReturnType()))
1287 return;
1288 Value *Op = Ref->getReturnValue();
1289 if (!Op)
1290 return;
1291 if (deduceOperandElementTypeFunctionRet(I, IncompleteRets, AskOps,
1292 IsPostprocessing, KnownElemTy, Op,
1293 F: CurrF))
1294 return;
1295 Incomplete = isTodoType(Op: CurrF);
1296 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1297 } else if (auto *Ref = dyn_cast<ICmpInst>(Val: I)) {
1298 if (!isPointerTy(T: Ref->getOperand(i_nocapture: 0)->getType()))
1299 return;
1300 Value *Op0 = Ref->getOperand(i_nocapture: 0);
1301 Value *Op1 = Ref->getOperand(i_nocapture: 1);
1302 bool Incomplete0 = isTodoType(Op: Op0);
1303 bool Incomplete1 = isTodoType(Op: Op1);
1304 Type *ElemTy1 = GR->findDeducedElementType(Val: Op1);
1305 Type *ElemTy0 = (Incomplete0 && !Incomplete1 && ElemTy1)
1306 ? nullptr
1307 : GR->findDeducedElementType(Val: Op0);
1308 if (ElemTy0) {
1309 KnownElemTy = ElemTy0;
1310 Incomplete = Incomplete0;
1311 Ops.push_back(Elt: std::make_pair(x&: Op1, y: 1));
1312 } else if (ElemTy1) {
1313 KnownElemTy = ElemTy1;
1314 Incomplete = Incomplete1;
1315 Ops.push_back(Elt: std::make_pair(x&: Op0, y: 0));
1316 }
1317 } else if (CallInst *CI = dyn_cast<CallInst>(Val: I)) {
1318 if (!CI->isIndirectCall())
1319 deduceOperandElementTypeCalledFunction(CI, Ops, KnownElemTy, Incomplete);
1320 else if (HaveFunPtrs)
1321 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy,
1322 IsPostprocessing);
1323 }
1324
1325 // There is no enough info to deduce types or all is valid.
1326 if (!KnownElemTy || Ops.size() == 0)
1327 return;
1328
1329 LLVMContext &Ctx = CurrF->getContext();
1330 IRBuilder<> B(Ctx);
1331 for (auto &OpIt : Ops) {
1332 Value *Op = OpIt.first;
1333 if (AskOps && !AskOps->contains(Ptr: Op))
1334 continue;
1335 Type *AskTy = nullptr;
1336 CallInst *AskCI = nullptr;
1337 if (IsPostprocessing && AskOps) {
1338 AskTy = GR->findDeducedElementType(Val: Op);
1339 AskCI = GR->findAssignPtrTypeInstr(Val: Op);
1340 assert(AskTy && AskCI);
1341 }
1342 Type *Ty = AskTy ? AskTy : GR->findDeducedElementType(Val: Op);
1343 if (Ty == KnownElemTy)
1344 continue;
1345 Value *OpTyVal = getNormalizedPoisonValue(Ty: KnownElemTy);
1346 Type *OpTy = Op->getType();
1347 if (Op->hasUseList() &&
1348 (!Ty || AskTy || isUntypedPointerTy(T: Ty) || isTodoType(Op))) {
1349 Type *PrevElemTy = GR->findDeducedElementType(Val: Op);
1350 GR->addDeducedElementType(Val: Op, Ty: normalizeType(Ty: KnownElemTy));
1351 // check if KnownElemTy is complete
1352 if (!Incomplete)
1353 eraseTodoType(Op);
1354 else if (!IsPostprocessing)
1355 insertTodoType(Op);
1356 // check if there is existing Intrinsic::spv_assign_ptr_type instruction
1357 CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Val: Op);
1358 if (AssignCI == nullptr) {
1359 Instruction *User = dyn_cast<Instruction>(Val: Op->use_begin()->get());
1360 setInsertPointSkippingPhis(B, I: User ? User->getNextNode() : I);
1361 CallInst *CI =
1362 buildIntrWithMD(IntrID: Intrinsic::spv_assign_ptr_type, Types: {OpTy}, Arg: OpTyVal, Arg2: Op,
1363 Imms: {B.getInt32(C: getPointerAddressSpace(T: OpTy))}, B);
1364 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: CI);
1365 } else {
1366 GR->updateAssignType(AssignCI, Arg: Op, OfType: OpTyVal);
1367 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1368 std::make_pair(x&: I, y&: Op)};
1369 propagateElemTypeRec(Op, PtrElemTy: KnownElemTy, CastElemTy: PrevElemTy, VisitedSubst);
1370 }
1371 } else {
1372 eraseTodoType(Op);
1373 CallInst *PtrCastI =
1374 buildSpvPtrcast(F: I->getParent()->getParent(), Op, ElemTy: KnownElemTy);
1375 if (OpIt.second == std::numeric_limits<unsigned>::max())
1376 dyn_cast<CallInst>(Val: I)->setCalledOperand(PtrCastI);
1377 else
1378 I->setOperand(i: OpIt.second, Val: PtrCastI);
1379 }
1380 }
1381 TypeValidated.insert(x: I);
1382}
1383
1384void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
1385 Instruction *New,
1386 IRBuilder<> &B) {
1387 while (!Old->user_empty()) {
1388 auto *U = Old->user_back();
1389 if (isAssignTypeInstr(I: U)) {
1390 B.SetInsertPoint(U);
1391 SmallVector<Value *, 2> Args = {New, U->getOperand(i: 1)};
1392 CallInst *AssignCI =
1393 B.CreateIntrinsic(ID: Intrinsic::spv_assign_type, Types: {New->getType()}, Args);
1394 GR->addAssignPtrTypeInstr(Val: New, AssignPtrTyCI: AssignCI);
1395 U->eraseFromParent();
1396 } else if (isMemInstrToReplace(I: U) || isa<ReturnInst>(Val: U) ||
1397 isa<CallInst>(Val: U)) {
1398 U->replaceUsesOfWith(From: Old, To: New);
1399 } else {
1400 llvm_unreachable("illegal aggregate intrinsic user");
1401 }
1402 }
1403 New->copyMetadata(SrcInst: *Old);
1404 Old->eraseFromParent();
1405}
1406
1407void SPIRVEmitIntrinsics::preprocessUndefs(IRBuilder<> &B) {
1408 std::queue<Instruction *> Worklist;
1409 for (auto &I : instructions(F: CurrF))
1410 Worklist.push(x: &I);
1411
1412 while (!Worklist.empty()) {
1413 Instruction *I = Worklist.front();
1414 bool BPrepared = false;
1415 Worklist.pop();
1416
1417 for (auto &Op : I->operands()) {
1418 auto *AggrUndef = dyn_cast<UndefValue>(Val&: Op);
1419 if (!AggrUndef || !Op->getType()->isAggregateType())
1420 continue;
1421
1422 if (!BPrepared) {
1423 setInsertPointSkippingPhis(B, I);
1424 BPrepared = true;
1425 }
1426 auto *IntrUndef = B.CreateIntrinsic(ID: Intrinsic::spv_undef, Args: {});
1427 Worklist.push(x: IntrUndef);
1428 I->replaceUsesOfWith(From: Op, To: IntrUndef);
1429 AggrConsts[IntrUndef] = AggrUndef;
1430 AggrConstTypes[IntrUndef] = AggrUndef->getType();
1431 }
1432 }
1433}
1434
1435void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) {
1436 std::queue<Instruction *> Worklist;
1437 for (auto &I : instructions(F: CurrF))
1438 Worklist.push(x: &I);
1439
1440 while (!Worklist.empty()) {
1441 auto *I = Worklist.front();
1442 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
1443 assert(I);
1444 bool KeepInst = false;
1445 for (const auto &Op : I->operands()) {
1446 Constant *AggrConst = nullptr;
1447 Type *ResTy = nullptr;
1448 if (auto *COp = dyn_cast<ConstantVector>(Val: Op)) {
1449 AggrConst = COp;
1450 ResTy = COp->getType();
1451 } else if (auto *COp = dyn_cast<ConstantArray>(Val: Op)) {
1452 AggrConst = COp;
1453 ResTy = B.getInt32Ty();
1454 } else if (auto *COp = dyn_cast<ConstantStruct>(Val: Op)) {
1455 AggrConst = COp;
1456 ResTy = B.getInt32Ty();
1457 } else if (auto *COp = dyn_cast<ConstantDataArray>(Val: Op)) {
1458 AggrConst = COp;
1459 ResTy = B.getInt32Ty();
1460 } else if (auto *COp = dyn_cast<ConstantAggregateZero>(Val: Op)) {
1461 AggrConst = COp;
1462 ResTy = Op->getType()->isVectorTy() ? COp->getType() : B.getInt32Ty();
1463 }
1464 if (AggrConst) {
1465 SmallVector<Value *> Args;
1466 if (auto *COp = dyn_cast<ConstantDataSequential>(Val: Op))
1467 for (unsigned i = 0; i < COp->getNumElements(); ++i)
1468 Args.push_back(Elt: COp->getElementAsConstant(i));
1469 else
1470 llvm::append_range(C&: Args, R: AggrConst->operands());
1471 if (!BPrepared) {
1472 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
1473 : B.SetInsertPoint(I);
1474 BPrepared = true;
1475 }
1476 auto *CI =
1477 B.CreateIntrinsic(ID: Intrinsic::spv_const_composite, Types: {ResTy}, Args: {Args});
1478 Worklist.push(x: CI);
1479 I->replaceUsesOfWith(From: Op, To: CI);
1480 KeepInst = true;
1481 AggrConsts[CI] = AggrConst;
1482 AggrConstTypes[CI] = deduceNestedTypeHelper(U: AggrConst, UnknownElemTypeI8: false);
1483 }
1484 }
1485 if (!KeepInst)
1486 Worklist.pop();
1487 }
1488}
1489
1490static void createDecorationIntrinsic(Instruction *I, MDNode *Node,
1491 IRBuilder<> &B) {
1492 LLVMContext &Ctx = I->getContext();
1493 setInsertPointAfterDef(B, I);
1494 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, Types: {I->getType()},
1495 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: {Node}))});
1496}
1497
1498static void createRoundingModeDecoration(Instruction *I,
1499 unsigned RoundingModeDeco,
1500 IRBuilder<> &B) {
1501 LLVMContext &Ctx = I->getContext();
1502 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1503 MDNode *RoundingModeNode = MDNode::get(
1504 Context&: Ctx,
1505 MDs: {ConstantAsMetadata::get(
1506 C: ConstantInt::get(Ty: Int32Ty, V: SPIRV::Decoration::FPRoundingMode)),
1507 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: RoundingModeDeco))});
1508 createDecorationIntrinsic(I, Node: RoundingModeNode, B);
1509}
1510
1511static void createSaturatedConversionDecoration(Instruction *I,
1512 IRBuilder<> &B) {
1513 LLVMContext &Ctx = I->getContext();
1514 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1515 MDNode *SaturatedConversionNode =
1516 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1517 Ty: Int32Ty, V: SPIRV::Decoration::SaturatedConversion))});
1518 createDecorationIntrinsic(I, Node: SaturatedConversionNode, B);
1519}
1520
1521static void addSaturatedDecorationToIntrinsic(Instruction *I, IRBuilder<> &B) {
1522 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
1523 if (Function *Fu = CI->getCalledFunction()) {
1524 if (Fu->isIntrinsic()) {
1525 unsigned const int IntrinsicId = Fu->getIntrinsicID();
1526 switch (IntrinsicId) {
1527 case Intrinsic::fptosi_sat:
1528 case Intrinsic::fptoui_sat:
1529 createSaturatedConversionDecoration(I, B);
1530 break;
1531 default:
1532 break;
1533 }
1534 }
1535 }
1536 }
1537}
1538
1539Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
1540 if (!Call.isInlineAsm())
1541 return &Call;
1542
1543 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
1544 LLVMContext &Ctx = CurrF->getContext();
1545
1546 Constant *TyC = UndefValue::get(T: IA->getFunctionType());
1547 MDString *ConstraintString = MDString::get(Context&: Ctx, Str: IA->getConstraintString());
1548 SmallVector<Value *> Args = {
1549 buildMD(Arg: TyC),
1550 MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: ConstraintString))};
1551 for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++)
1552 Args.push_back(Elt: Call.getArgOperand(i: OpIdx));
1553
1554 IRBuilder<> B(Call.getParent());
1555 B.SetInsertPoint(&Call);
1556 B.CreateIntrinsic(ID: Intrinsic::spv_inline_asm, Args: {Args});
1557 return &Call;
1558}
1559
1560// Use a tip about rounding mode to create a decoration.
1561void SPIRVEmitIntrinsics::useRoundingMode(ConstrainedFPIntrinsic *FPI,
1562 IRBuilder<> &B) {
1563 std::optional<RoundingMode> RM = FPI->getRoundingMode();
1564 if (!RM.has_value())
1565 return;
1566 unsigned RoundingModeDeco = std::numeric_limits<unsigned>::max();
1567 switch (RM.value()) {
1568 default:
1569 // ignore unknown rounding modes
1570 break;
1571 case RoundingMode::NearestTiesToEven:
1572 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTE;
1573 break;
1574 case RoundingMode::TowardNegative:
1575 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTN;
1576 break;
1577 case RoundingMode::TowardPositive:
1578 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTP;
1579 break;
1580 case RoundingMode::TowardZero:
1581 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTZ;
1582 break;
1583 case RoundingMode::Dynamic:
1584 case RoundingMode::NearestTiesToAway:
1585 // TODO: check if supported
1586 break;
1587 }
1588 if (RoundingModeDeco == std::numeric_limits<unsigned>::max())
1589 return;
1590 // Convert the tip about rounding mode into a decoration record.
1591 createRoundingModeDecoration(I: FPI, RoundingModeDeco, B);
1592}
1593
1594Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
1595 BasicBlock *ParentBB = I.getParent();
1596 Function *F = ParentBB->getParent();
1597 IRBuilder<> B(ParentBB);
1598 B.SetInsertPoint(&I);
1599 SmallVector<Value *, 4> Args;
1600 SmallVector<BasicBlock *> BBCases;
1601 Args.push_back(Elt: I.getCondition());
1602 BBCases.push_back(Elt: I.getDefaultDest());
1603 Args.push_back(Elt: BlockAddress::get(F, BB: I.getDefaultDest()));
1604 for (auto &Case : I.cases()) {
1605 Args.push_back(Elt: Case.getCaseValue());
1606 BBCases.push_back(Elt: Case.getCaseSuccessor());
1607 Args.push_back(Elt: BlockAddress::get(F, BB: Case.getCaseSuccessor()));
1608 }
1609 CallInst *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_switch,
1610 Types: {I.getOperand(i_nocapture: 0)->getType()}, Args: {Args});
1611 // remove switch to avoid its unneeded and undesirable unwrap into branches
1612 // and conditions
1613 replaceAllUsesWith(Src: &I, Dest: NewI);
1614 I.eraseFromParent();
1615 // insert artificial and temporary instruction to preserve valid CFG,
1616 // it will be removed after IR translation pass
1617 B.SetInsertPoint(ParentBB);
1618 IndirectBrInst *BrI = B.CreateIndirectBr(
1619 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: ParentBB->getContext())),
1620 NumDests: BBCases.size());
1621 for (BasicBlock *BBCase : BBCases)
1622 BrI->addDestination(Dest: BBCase);
1623 return BrI;
1624}
1625
1626static bool isFirstIndexZero(const GetElementPtrInst *GEP) {
1627 if (GEP->getNumIndices() == 0)
1628 return false;
1629 if (const auto *CI = dyn_cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: 1))) {
1630 return CI->getZExtValue() == 0;
1631 }
1632 return false;
1633}
1634
1635Instruction *SPIRVEmitIntrinsics::visitIntrinsicInst(IntrinsicInst &I) {
1636 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
1637 if (!SGEP)
1638 return &I;
1639
1640 IRBuilder<> B(I.getParent());
1641 B.SetInsertPoint(&I);
1642 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
1643 SmallVector<Value *, 4> Args;
1644 Args.push_back(/* inBounds= */ Elt: B.getInt1(V: true));
1645 Args.push_back(Elt: I.getOperand(i_nocapture: 0));
1646 Args.push_back(/* zero index */ Elt: B.getInt32(C: 0));
1647 for (unsigned J = 0; J < SGEP->getNumIndices(); ++J)
1648 Args.push_back(Elt: SGEP->getIndexOperand(Index: J));
1649
1650 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types, Args);
1651 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1652 return NewI;
1653}
1654
1655Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
1656 IRBuilder<> B(I.getParent());
1657 B.SetInsertPoint(&I);
1658
1659 if (TM->getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: &I)) {
1660 // Logical SPIR-V cannot use the OpPtrAccessChain instruction. If the first
1661 // index of the GEP is not 0, then we need to try to adjust it.
1662 //
1663 // If the GEP is doing byte addressing, try to rebuild the full access chain
1664 // from the type of the pointer.
1665 if (I.getSourceElementType() ==
1666 IntegerType::getInt8Ty(C&: CurrF->getContext())) {
1667 return buildLogicalAccessChainFromGEP(GEP&: I);
1668 }
1669
1670 // Look for the array-to-pointer decay. If this is the pattern
1671 // we can adjust the types, and prepend a 0 to the indices.
1672 Value *PtrOp = I.getPointerOperand();
1673 Type *SrcElemTy = I.getSourceElementType();
1674 Type *DeducedPointeeTy = deduceElementType(I: PtrOp, UnknownElemTypeI8: true);
1675
1676 if (auto *ArrTy = dyn_cast<ArrayType>(Val: DeducedPointeeTy)) {
1677 if (ArrTy->getElementType() == SrcElemTy) {
1678 SmallVector<Value *> NewIndices;
1679 Type *FirstIdxType = I.getOperand(i_nocapture: 1)->getType();
1680 NewIndices.push_back(Elt: ConstantInt::get(Ty: FirstIdxType, V: 0));
1681 for (Value *Idx : I.indices())
1682 NewIndices.push_back(Elt: Idx);
1683
1684 SmallVector<Type *, 2> Types = {I.getType(), I.getPointerOperandType()};
1685 SmallVector<Value *, 4> Args;
1686 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
1687 Args.push_back(Elt: I.getPointerOperand());
1688 Args.append(in_start: NewIndices.begin(), in_end: NewIndices.end());
1689
1690 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types: {Types}, Args: {Args});
1691 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1692 return NewI;
1693 }
1694 }
1695 }
1696
1697 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
1698 SmallVector<Value *, 4> Args;
1699 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
1700 llvm::append_range(C&: Args, R: I.operands());
1701 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types: {Types}, Args: {Args});
1702 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1703 return NewI;
1704}
1705
1706Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
1707 IRBuilder<> B(I.getParent());
1708 B.SetInsertPoint(&I);
1709 Value *Source = I.getOperand(i_nocapture: 0);
1710
1711 // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of
1712 // varying element types. In case of IR coming from older versions of LLVM
1713 // such bitcasts do not provide sufficient information, should be just skipped
1714 // here, and handled in insertPtrCastOrAssignTypeInstr.
1715 if (isPointerTy(T: I.getType())) {
1716 replaceAllUsesWith(Src: &I, Dest: Source);
1717 I.eraseFromParent();
1718 return nullptr;
1719 }
1720
1721 SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
1722 SmallVector<Value *> Args(I.op_begin(), I.op_end());
1723 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_bitcast, Types: {Types}, Args: {Args});
1724 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1725 return NewI;
1726}
1727
1728void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt(
1729 TargetExtType *AssignedType, Value *V, IRBuilder<> &B) {
1730 Type *VTy = V->getType();
1731
1732 // A couple of sanity checks.
1733 assert((isPointerTy(VTy)) && "Expect a pointer type!");
1734 if (Type *ElemTy = getPointeeType(Ty: VTy))
1735 if (ElemTy != AssignedType)
1736 report_fatal_error(reason: "Unexpected pointer element type!");
1737
1738 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: V);
1739 if (!AssignCI) {
1740 GR->buildAssignType(B, Ty: AssignedType, Arg: V);
1741 return;
1742 }
1743
1744 Type *CurrentType =
1745 dyn_cast<ConstantAsMetadata>(
1746 Val: cast<MetadataAsValue>(Val: AssignCI->getOperand(i_nocapture: 1))->getMetadata())
1747 ->getType();
1748 if (CurrentType == AssignedType)
1749 return;
1750
1751 // Builtin types cannot be redeclared or casted.
1752 if (CurrentType->isTargetExtTy())
1753 report_fatal_error(reason: "Type mismatch " + CurrentType->getTargetExtName() +
1754 "/" + AssignedType->getTargetExtName() +
1755 " for value " + V->getName(),
1756 gen_crash_diag: false);
1757
1758 // Our previous guess about the type seems to be wrong, let's update
1759 // inferred type according to a new, more precise type information.
1760 GR->updateAssignType(AssignCI, Arg: V, OfType: getNormalizedPoisonValue(Ty: AssignedType));
1761}
1762
1763void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
1764 Instruction *I, Value *Pointer, Type *ExpectedElementType,
1765 unsigned OperandToReplace, IRBuilder<> &B) {
1766 TypeValidated.insert(x: I);
1767
1768 // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType
1769 Type *PointerElemTy = deduceElementTypeHelper(I: Pointer, UnknownElemTypeI8: false);
1770 if (PointerElemTy == ExpectedElementType ||
1771 isEquivalentTypes(Ty1: PointerElemTy, Ty2: ExpectedElementType))
1772 return;
1773
1774 setInsertPointSkippingPhis(B, I);
1775 Value *ExpectedElementVal = getNormalizedPoisonValue(Ty: ExpectedElementType);
1776 MetadataAsValue *VMD = buildMD(Arg: ExpectedElementVal);
1777 unsigned AddressSpace = getPointerAddressSpace(T: Pointer->getType());
1778 bool FirstPtrCastOrAssignPtrType = true;
1779
1780 // Do not emit new spv_ptrcast if equivalent one already exists or when
1781 // spv_assign_ptr_type already targets this pointer with the same element
1782 // type.
1783 if (Pointer->hasUseList()) {
1784 for (auto User : Pointer->users()) {
1785 auto *II = dyn_cast<IntrinsicInst>(Val: User);
1786 if (!II ||
1787 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
1788 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
1789 II->getOperand(i_nocapture: 0) != Pointer)
1790 continue;
1791
1792 // There is some spv_ptrcast/spv_assign_ptr_type already targeting this
1793 // pointer.
1794 FirstPtrCastOrAssignPtrType = false;
1795 if (II->getOperand(i_nocapture: 1) != VMD ||
1796 dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 2))->getSExtValue() !=
1797 AddressSpace)
1798 continue;
1799
1800 // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the
1801 // same element type and address space.
1802 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
1803 return;
1804
1805 // This must be a spv_ptrcast, do not emit new if this one has the same BB
1806 // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type.
1807 if (II->getParent() != I->getParent())
1808 continue;
1809
1810 I->setOperand(i: OperandToReplace, Val: II);
1811 return;
1812 }
1813 }
1814
1815 if (isa<Instruction>(Val: Pointer) || isa<Argument>(Val: Pointer)) {
1816 if (FirstPtrCastOrAssignPtrType) {
1817 // If this would be the first spv_ptrcast, do not emit spv_ptrcast and
1818 // emit spv_assign_ptr_type instead.
1819 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
1820 return;
1821 } else if (isTodoType(Op: Pointer)) {
1822 eraseTodoType(Op: Pointer);
1823 if (!isa<CallInst>(Val: Pointer) && !isaGEP(V: Pointer)) {
1824 // If this wouldn't be the first spv_ptrcast but existing type info is
1825 // uncomplete, update spv_assign_ptr_type arguments.
1826 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Pointer)) {
1827 Type *PrevElemTy = GR->findDeducedElementType(Val: Pointer);
1828 assert(PrevElemTy);
1829 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1830 std::make_pair(x&: I, y&: Pointer)};
1831 GR->updateAssignType(AssignCI, Arg: Pointer, OfType: ExpectedElementVal);
1832 propagateElemType(Op: Pointer, ElemTy: PrevElemTy, VisitedSubst);
1833 } else {
1834 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
1835 }
1836 return;
1837 }
1838 }
1839 }
1840
1841 // Emit spv_ptrcast
1842 SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()};
1843 SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(C: AddressSpace)};
1844 auto *PtrCastI = B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
1845 I->setOperand(i: OperandToReplace, Val: PtrCastI);
1846 // We need to set up a pointee type for the newly created spv_ptrcast.
1847 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: PtrCastI);
1848}
1849
1850void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
1851 IRBuilder<> &B) {
1852 // Handle basic instructions:
1853 StoreInst *SI = dyn_cast<StoreInst>(Val: I);
1854 if (IsKernelArgInt8(F: CurrF, SI)) {
1855 replacePointerOperandWithPtrCast(
1856 I, Pointer: SI->getValueOperand(), ExpectedElementType: IntegerType::getInt8Ty(C&: CurrF->getContext()),
1857 OperandToReplace: 0, B);
1858 }
1859 if (SI) {
1860 Value *Op = SI->getValueOperand();
1861 Value *Pointer = SI->getPointerOperand();
1862 Type *OpTy = Op->getType();
1863 if (auto *OpI = dyn_cast<Instruction>(Val: Op))
1864 OpTy = restoreMutatedType(GR, I: OpI, Ty: OpTy);
1865 if (OpTy == Op->getType())
1866 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: Op, UnknownElemTypeI8: false);
1867 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 1, B);
1868 return;
1869 }
1870 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
1871 Value *Pointer = LI->getPointerOperand();
1872 Type *OpTy = LI->getType();
1873 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1874 if (Type *ElemTy = GR->findDeducedElementType(Val: LI)) {
1875 OpTy = getTypedPointerWrapper(ElemTy, AS: PtrTy->getAddressSpace());
1876 } else {
1877 Type *NewOpTy = OpTy;
1878 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: LI, UnknownElemTypeI8: false);
1879 if (OpTy == NewOpTy)
1880 insertTodoType(Op: Pointer);
1881 }
1882 }
1883 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
1884 return;
1885 }
1886 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(Val: I)) {
1887 Value *Pointer = GEPI->getPointerOperand();
1888 Type *OpTy = nullptr;
1889
1890 // Logical SPIR-V is not allowed to use Op*PtrAccessChain instructions. If
1891 // the first index is 0, then we can trivially lower to OpAccessChain. If
1892 // not we need to try to rewrite the GEP. We avoid adding a pointer cast at
1893 // this time, and will rewrite the GEP when visiting it.
1894 if (TM->getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: GEPI)) {
1895 return;
1896 }
1897
1898 // In all cases, fall back to the GEP type if type scavenging failed.
1899 if (!OpTy)
1900 OpTy = GEPI->getSourceElementType();
1901
1902 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
1903 if (isNestedPointer(Ty: OpTy))
1904 insertTodoType(Op: Pointer);
1905 return;
1906 }
1907
1908 // TODO: review and merge with existing logics:
1909 // Handle calls to builtins (non-intrinsics):
1910 CallInst *CI = dyn_cast<CallInst>(Val: I);
1911 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
1912 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
1913 return;
1914
1915 // collect information about formal parameter types
1916 std::string DemangledName =
1917 getOclOrSpirvBuiltinDemangledName(Name: CI->getCalledFunction()->getName());
1918 Function *CalledF = CI->getCalledFunction();
1919 SmallVector<Type *, 4> CalledArgTys;
1920 bool HaveTypes = false;
1921 for (unsigned OpIdx = 0; OpIdx < CalledF->arg_size(); ++OpIdx) {
1922 Argument *CalledArg = CalledF->getArg(i: OpIdx);
1923 Type *ArgType = CalledArg->getType();
1924 if (!isPointerTy(T: ArgType)) {
1925 CalledArgTys.push_back(Elt: nullptr);
1926 } else if (Type *ArgTypeElem = getPointeeType(Ty: ArgType)) {
1927 CalledArgTys.push_back(Elt: ArgTypeElem);
1928 HaveTypes = true;
1929 } else {
1930 Type *ElemTy = GR->findDeducedElementType(Val: CalledArg);
1931 if (!ElemTy && hasPointeeTypeAttr(Arg: CalledArg))
1932 ElemTy = getPointeeTypeByAttr(Arg: CalledArg);
1933 if (!ElemTy) {
1934 ElemTy = getPointeeTypeByCallInst(DemangledName, CalledF, OpIdx);
1935 if (ElemTy) {
1936 GR->addDeducedElementType(Val: CalledArg, Ty: normalizeType(Ty: ElemTy));
1937 } else {
1938 for (User *U : CalledArg->users()) {
1939 if (Instruction *Inst = dyn_cast<Instruction>(Val: U)) {
1940 if ((ElemTy = deduceElementTypeHelper(I: Inst, UnknownElemTypeI8: false)) != nullptr)
1941 break;
1942 }
1943 }
1944 }
1945 }
1946 HaveTypes |= ElemTy != nullptr;
1947 CalledArgTys.push_back(Elt: ElemTy);
1948 }
1949 }
1950
1951 if (DemangledName.empty() && !HaveTypes)
1952 return;
1953
1954 for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) {
1955 Value *ArgOperand = CI->getArgOperand(i: OpIdx);
1956 if (!isPointerTy(T: ArgOperand->getType()))
1957 continue;
1958
1959 // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs()
1960 if (!isa<Instruction>(Val: ArgOperand) && !isa<Argument>(Val: ArgOperand)) {
1961 // However, we may have assumptions about the formal argument's type and
1962 // may have a need to insert a ptr cast for the actual parameter of this
1963 // call.
1964 Argument *CalledArg = CalledF->getArg(i: OpIdx);
1965 if (!GR->findDeducedElementType(Val: CalledArg))
1966 continue;
1967 }
1968
1969 Type *ExpectedType =
1970 OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr;
1971 if (!ExpectedType && !DemangledName.empty())
1972 ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType(
1973 DemangledCall: DemangledName, ArgIdx: OpIdx, Ctx&: I->getContext());
1974 if (!ExpectedType || ExpectedType->isVoidTy())
1975 continue;
1976
1977 if (ExpectedType->isTargetExtTy() &&
1978 !isTypedPointerWrapper(ExtTy: cast<TargetExtType>(Val: ExpectedType)))
1979 insertAssignPtrTypeTargetExt(AssignedType: cast<TargetExtType>(Val: ExpectedType),
1980 V: ArgOperand, B);
1981 else
1982 replacePointerOperandWithPtrCast(I: CI, Pointer: ArgOperand, ExpectedElementType: ExpectedType, OperandToReplace: OpIdx, B);
1983 }
1984}
1985
1986Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
1987 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
1988 // type in LLT and IRTranslator will replace it by the scalar.
1989 if (isVector1(Ty: I.getType()))
1990 return &I;
1991
1992 SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType(),
1993 I.getOperand(i_nocapture: 1)->getType(),
1994 I.getOperand(i_nocapture: 2)->getType()};
1995 IRBuilder<> B(I.getParent());
1996 B.SetInsertPoint(&I);
1997 SmallVector<Value *> Args(I.op_begin(), I.op_end());
1998 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_insertelt, Types: {Types}, Args: {Args});
1999 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2000 return NewI;
2001}
2002
2003Instruction *
2004SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
2005 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2006 // type in LLT and IRTranslator will replace it by the scalar.
2007 if (isVector1(Ty: I.getVectorOperandType()))
2008 return &I;
2009
2010 IRBuilder<> B(I.getParent());
2011 B.SetInsertPoint(&I);
2012 SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
2013 I.getIndexOperand()->getType()};
2014 SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
2015 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_extractelt, Types: {Types}, Args: {Args});
2016 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2017 return NewI;
2018}
2019
2020Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
2021 IRBuilder<> B(I.getParent());
2022 B.SetInsertPoint(&I);
2023 SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
2024 SmallVector<Value *> Args;
2025 Value *AggregateOp = I.getAggregateOperand();
2026 if (isa<UndefValue>(Val: AggregateOp))
2027 Args.push_back(Elt: UndefValue::get(T: B.getInt32Ty()));
2028 else
2029 Args.push_back(Elt: AggregateOp);
2030 Args.push_back(Elt: I.getInsertedValueOperand());
2031 for (auto &Op : I.indices())
2032 Args.push_back(Elt: B.getInt32(C: Op));
2033 Instruction *NewI =
2034 B.CreateIntrinsic(ID: Intrinsic::spv_insertv, Types: {Types}, Args: {Args});
2035 replaceMemInstrUses(Old: &I, New: NewI, B);
2036 return NewI;
2037}
2038
2039Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
2040 if (I.getAggregateOperand()->getType()->isAggregateType())
2041 return &I;
2042 IRBuilder<> B(I.getParent());
2043 B.SetInsertPoint(&I);
2044 SmallVector<Value *> Args(I.operands());
2045 for (auto &Op : I.indices())
2046 Args.push_back(Elt: B.getInt32(C: Op));
2047 auto *NewI =
2048 B.CreateIntrinsic(ID: Intrinsic::spv_extractv, Types: {I.getType()}, Args: {Args});
2049 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2050 return NewI;
2051}
2052
2053Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
2054 if (!I.getType()->isAggregateType())
2055 return &I;
2056 IRBuilder<> B(I.getParent());
2057 B.SetInsertPoint(&I);
2058 TrackConstants = false;
2059 const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
2060 MachineMemOperand::Flags Flags =
2061 TLI->getLoadMemOperandFlags(LI: I, DL: CurrF->getDataLayout());
2062 auto *NewI =
2063 B.CreateIntrinsic(ID: Intrinsic::spv_load, Types: {I.getOperand(i_nocapture: 0)->getType()},
2064 Args: {I.getPointerOperand(), B.getInt16(C: Flags),
2065 B.getInt32(C: I.getAlign().value())});
2066 replaceMemInstrUses(Old: &I, New: NewI, B);
2067 return NewI;
2068}
2069
2070Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
2071 if (!AggrStores.contains(V: &I))
2072 return &I;
2073 IRBuilder<> B(I.getParent());
2074 B.SetInsertPoint(&I);
2075 TrackConstants = false;
2076 const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
2077 MachineMemOperand::Flags Flags =
2078 TLI->getStoreMemOperandFlags(SI: I, DL: CurrF->getDataLayout());
2079 auto *PtrOp = I.getPointerOperand();
2080
2081 if (I.getValueOperand()->getType()->isAggregateType()) {
2082 // It is possible that what used to be an ExtractValueInst has been replaced
2083 // with a call to the spv_extractv intrinsic, and that said call hasn't
2084 // had its return type replaced with i32 during the dedicated pass (because
2085 // it was emitted later); we have to handle this here, because IRTranslator
2086 // cannot deal with multi-register types at the moment.
2087 CallBase *CB = dyn_cast<CallBase>(Val: I.getValueOperand());
2088 assert(CB && CB->getIntrinsicID() == Intrinsic::spv_extractv &&
2089 "Unexpected argument of aggregate type, should be spv_extractv!");
2090 CB->mutateType(Ty: B.getInt32Ty());
2091 }
2092
2093 auto *NewI = B.CreateIntrinsic(
2094 ID: Intrinsic::spv_store, Types: {I.getValueOperand()->getType(), PtrOp->getType()},
2095 Args: {I.getValueOperand(), PtrOp, B.getInt16(C: Flags),
2096 B.getInt32(C: I.getAlign().value())});
2097 NewI->copyMetadata(SrcInst: I);
2098 I.eraseFromParent();
2099 return NewI;
2100}
2101
2102Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
2103 Value *ArraySize = nullptr;
2104 if (I.isArrayAllocation()) {
2105 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*I.getFunction());
2106 if (!STI->canUseExtension(
2107 E: SPIRV::Extension::SPV_INTEL_variable_length_array))
2108 report_fatal_error(
2109 reason: "array allocation: this instruction requires the following "
2110 "SPIR-V extension: SPV_INTEL_variable_length_array",
2111 gen_crash_diag: false);
2112 ArraySize = I.getArraySize();
2113 }
2114 IRBuilder<> B(I.getParent());
2115 B.SetInsertPoint(&I);
2116 TrackConstants = false;
2117 Type *PtrTy = I.getType();
2118 auto *NewI =
2119 ArraySize
2120 ? B.CreateIntrinsic(ID: Intrinsic::spv_alloca_array,
2121 Types: {PtrTy, ArraySize->getType()},
2122 Args: {ArraySize, B.getInt32(C: I.getAlign().value())})
2123 : B.CreateIntrinsic(ID: Intrinsic::spv_alloca, Types: {PtrTy},
2124 Args: {B.getInt32(C: I.getAlign().value())});
2125 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2126 return NewI;
2127}
2128
2129Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2130 assert(I.getType()->isAggregateType() && "Aggregate result is expected");
2131 IRBuilder<> B(I.getParent());
2132 B.SetInsertPoint(&I);
2133 SmallVector<Value *> Args(I.operands());
2134 Args.push_back(Elt: B.getInt32(
2135 C: static_cast<uint32_t>(getMemScope(Ctx&: I.getContext(), Id: I.getSyncScopeID()))));
2136 Args.push_back(Elt: B.getInt32(
2137 C: static_cast<uint32_t>(getMemSemantics(Ord: I.getSuccessOrdering()))));
2138 Args.push_back(Elt: B.getInt32(
2139 C: static_cast<uint32_t>(getMemSemantics(Ord: I.getFailureOrdering()))));
2140 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_cmpxchg,
2141 Types: {I.getPointerOperand()->getType()}, Args: {Args});
2142 replaceMemInstrUses(Old: &I, New: NewI, B);
2143 return NewI;
2144}
2145
2146Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
2147 IRBuilder<> B(I.getParent());
2148 B.SetInsertPoint(&I);
2149 B.CreateIntrinsic(ID: Intrinsic::spv_unreachable, Args: {});
2150 return &I;
2151}
2152
2153void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV,
2154 IRBuilder<> &B) {
2155 // Skip special artificial variables.
2156 static const StringSet<> ArtificialGlobals{"llvm.global.annotations",
2157 "llvm.compiler.used", "llvm.used"};
2158
2159 if (ArtificialGlobals.contains(key: GV.getName()))
2160 return;
2161
2162 Constant *Init = nullptr;
2163 if (hasInitializer(GV: &GV)) {
2164 // Deduce element type and store results in Global Registry.
2165 // Result is ignored, because TypedPointerType is not supported
2166 // by llvm IR general logic.
2167 deduceElementTypeHelper(I: &GV, UnknownElemTypeI8: false);
2168 Init = GV.getInitializer();
2169 Type *Ty = isAggrConstForceInt32(V: Init) ? B.getInt32Ty() : Init->getType();
2170 Constant *Const = isAggrConstForceInt32(V: Init) ? B.getInt32(C: 1) : Init;
2171 auto *InitInst = B.CreateIntrinsic(ID: Intrinsic::spv_init_global,
2172 Types: {GV.getType(), Ty}, Args: {&GV, Const});
2173 InitInst->setArgOperand(i: 1, v: Init);
2174 }
2175 if (!Init && GV.use_empty())
2176 B.CreateIntrinsic(ID: Intrinsic::spv_unref_global, Types: GV.getType(), Args: &GV);
2177}
2178
2179// Return true, if we can't decide what is the pointee type now and will get
2180// back to the question later. Return false is spv_assign_ptr_type is not needed
2181// or can be inserted immediately.
2182bool SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I,
2183 IRBuilder<> &B,
2184 bool UnknownElemTypeI8) {
2185 reportFatalOnTokenType(I);
2186 if (!isPointerTy(T: I->getType()) || !requireAssignType(I))
2187 return false;
2188
2189 setInsertPointAfterDef(B, I);
2190 if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) {
2191 GR->buildAssignPtr(B, ElemTy, Arg: I);
2192 return false;
2193 }
2194 return true;
2195}
2196
2197void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I,
2198 IRBuilder<> &B) {
2199 // TODO: extend the list of functions with known result types
2200 static StringMap<unsigned> ResTypeWellKnown = {
2201 {"async_work_group_copy", WellKnownTypes::Event},
2202 {"async_work_group_strided_copy", WellKnownTypes::Event},
2203 {"__spirv_GroupAsyncCopy", WellKnownTypes::Event}};
2204
2205 reportFatalOnTokenType(I);
2206
2207 bool IsKnown = false;
2208 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
2209 if (!CI->isIndirectCall() && !CI->isInlineAsm() &&
2210 CI->getCalledFunction() && !CI->getCalledFunction()->isIntrinsic()) {
2211 Function *CalledF = CI->getCalledFunction();
2212 std::string DemangledName =
2213 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
2214 FPDecorationId DecorationId = FPDecorationId::NONE;
2215 if (DemangledName.length() > 0)
2216 DemangledName =
2217 SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName, DecorationId: &DecorationId);
2218 auto ResIt = ResTypeWellKnown.find(Key: DemangledName);
2219 if (ResIt != ResTypeWellKnown.end()) {
2220 IsKnown = true;
2221 setInsertPointAfterDef(B, I);
2222 switch (ResIt->second) {
2223 case WellKnownTypes::Event:
2224 GR->buildAssignType(
2225 B, Ty: TargetExtType::get(Context&: I->getContext(), Name: "spirv.Event"), Arg: I);
2226 break;
2227 }
2228 }
2229 // check if a floating rounding mode or saturation info is present
2230 switch (DecorationId) {
2231 default:
2232 break;
2233 case FPDecorationId::SAT:
2234 createSaturatedConversionDecoration(I: CI, B);
2235 break;
2236 case FPDecorationId::RTE:
2237 createRoundingModeDecoration(
2238 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTE, B);
2239 break;
2240 case FPDecorationId::RTZ:
2241 createRoundingModeDecoration(
2242 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTZ, B);
2243 break;
2244 case FPDecorationId::RTP:
2245 createRoundingModeDecoration(
2246 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTP, B);
2247 break;
2248 case FPDecorationId::RTN:
2249 createRoundingModeDecoration(
2250 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTN, B);
2251 break;
2252 }
2253 }
2254 }
2255
2256 Type *Ty = I->getType();
2257 if (!IsKnown && !Ty->isVoidTy() && !isPointerTy(T: Ty) && requireAssignType(I)) {
2258 setInsertPointAfterDef(B, I);
2259 Type *TypeToAssign = Ty;
2260 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2261 if (II->getIntrinsicID() == Intrinsic::spv_const_composite ||
2262 II->getIntrinsicID() == Intrinsic::spv_undef) {
2263 auto It = AggrConstTypes.find(Val: II);
2264 if (It == AggrConstTypes.end())
2265 report_fatal_error(reason: "Unknown composite intrinsic type");
2266 TypeToAssign = It->second;
2267 }
2268 }
2269 TypeToAssign = restoreMutatedType(GR, I, Ty: TypeToAssign);
2270 GR->buildAssignType(B, Ty: TypeToAssign, Arg: I);
2271 }
2272 for (const auto &Op : I->operands()) {
2273 if (isa<ConstantPointerNull>(Val: Op) || isa<UndefValue>(Val: Op) ||
2274 // Check GetElementPtrConstantExpr case.
2275 (isa<ConstantExpr>(Val: Op) &&
2276 (isa<GEPOperator>(Val: Op) ||
2277 (cast<ConstantExpr>(Val: Op)->getOpcode() == CastInst::IntToPtr)))) {
2278 setInsertPointSkippingPhis(B, I);
2279 Type *OpTy = Op->getType();
2280 if (isa<UndefValue>(Val: Op) && OpTy->isAggregateType()) {
2281 CallInst *AssignCI =
2282 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {B.getInt32Ty()}, Arg: Op,
2283 Arg2: UndefValue::get(T: B.getInt32Ty()), Imms: {}, B);
2284 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: AssignCI);
2285 } else if (!isa<Instruction>(Val: Op)) {
2286 Type *OpTy = Op->getType();
2287 Type *OpTyElem = getPointeeType(Ty: OpTy);
2288 if (OpTyElem) {
2289 GR->buildAssignPtr(B, ElemTy: OpTyElem, Arg: Op);
2290 } else if (isPointerTy(T: OpTy)) {
2291 Type *ElemTy = GR->findDeducedElementType(Val: Op);
2292 GR->buildAssignPtr(B, ElemTy: ElemTy ? ElemTy : deduceElementType(I: Op, UnknownElemTypeI8: true),
2293 Arg: Op);
2294 } else {
2295 Value *OpTyVal = Op;
2296 if (OpTy->isTargetExtTy()) {
2297 // We need to do this in order to be consistent with how target ext
2298 // types are handled in `processInstrAfterVisit`
2299 OpTyVal = getNormalizedPoisonValue(Ty: OpTy);
2300 }
2301 CallInst *AssignCI =
2302 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {OpTy},
2303 Arg: getNormalizedPoisonValue(Ty: OpTy), Arg2: OpTyVal, Imms: {}, B);
2304 GR->addAssignPtrTypeInstr(Val: OpTyVal, AssignPtrTyCI: AssignCI);
2305 }
2306 }
2307 }
2308 }
2309}
2310
2311bool SPIRVEmitIntrinsics::shouldTryToAddMemAliasingDecoration(
2312 Instruction *Inst) {
2313 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*Inst->getFunction());
2314 if (!STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing))
2315 return false;
2316 // Add aliasing decorations to internal load and store intrinsics
2317 // and atomic instructions, skipping atomic store as it won't have ID to
2318 // attach the decoration.
2319 CallInst *CI = dyn_cast<CallInst>(Val: Inst);
2320 if (!CI)
2321 return false;
2322 if (Function *Fun = CI->getCalledFunction()) {
2323 if (Fun->isIntrinsic()) {
2324 switch (Fun->getIntrinsicID()) {
2325 case Intrinsic::spv_load:
2326 case Intrinsic::spv_store:
2327 return true;
2328 default:
2329 return false;
2330 }
2331 }
2332 std::string Name = getOclOrSpirvBuiltinDemangledName(Name: Fun->getName());
2333 const std::string Prefix = "__spirv_Atomic";
2334 const bool IsAtomic = Name.find(str: Prefix) == 0;
2335
2336 if (!Fun->getReturnType()->isVoidTy() && IsAtomic)
2337 return true;
2338 }
2339 return false;
2340}
2341
2342void SPIRVEmitIntrinsics::insertSpirvDecorations(Instruction *I,
2343 IRBuilder<> &B) {
2344 if (MDNode *MD = I->getMetadata(Kind: "spirv.Decorations")) {
2345 setInsertPointAfterDef(B, I);
2346 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, Types: {I->getType()},
2347 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
2348 }
2349 // Lower alias.scope/noalias metadata
2350 {
2351 auto processMemAliasingDecoration = [&](unsigned Kind) {
2352 if (MDNode *AliasListMD = I->getMetadata(KindID: Kind)) {
2353 if (shouldTryToAddMemAliasingDecoration(Inst: I)) {
2354 uint32_t Dec = Kind == LLVMContext::MD_alias_scope
2355 ? SPIRV::Decoration::AliasScopeINTEL
2356 : SPIRV::Decoration::NoAliasINTEL;
2357 SmallVector<Value *, 3> Args = {
2358 I, ConstantInt::get(Ty: B.getInt32Ty(), V: Dec),
2359 MetadataAsValue::get(Context&: I->getContext(), MD: AliasListMD)};
2360 setInsertPointAfterDef(B, I);
2361 B.CreateIntrinsic(ID: Intrinsic::spv_assign_aliasing_decoration,
2362 Types: {I->getType()}, Args: {Args});
2363 }
2364 }
2365 };
2366 processMemAliasingDecoration(LLVMContext::MD_alias_scope);
2367 processMemAliasingDecoration(LLVMContext::MD_noalias);
2368 }
2369 // MD_fpmath
2370 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_fpmath)) {
2371 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*I->getFunction());
2372 bool AllowFPMaxError =
2373 STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_fp_max_error);
2374 if (!AllowFPMaxError)
2375 return;
2376
2377 setInsertPointAfterDef(B, I);
2378 B.CreateIntrinsic(ID: Intrinsic::spv_assign_fpmaxerror_decoration,
2379 Types: {I->getType()},
2380 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
2381 }
2382}
2383
2384static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
2385 const Module &M,
2386 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
2387 &FPFastMathDefaultInfoMap,
2388 Function *F) {
2389 auto it = FPFastMathDefaultInfoMap.find(Val: F);
2390 if (it != FPFastMathDefaultInfoMap.end())
2391 return it->second;
2392
2393 // If the map does not contain the entry, create a new one. Initialize it to
2394 // contain all 3 elements sorted by bit width of target type: {half, float,
2395 // double}.
2396 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
2397 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
2398 Args: SPIRV::FPFastMathMode::None);
2399 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
2400 Args: SPIRV::FPFastMathMode::None);
2401 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
2402 Args: SPIRV::FPFastMathMode::None);
2403 return FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
2404}
2405
2406static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
2407 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
2408 const Type *Ty) {
2409 size_t BitWidth = Ty->getScalarSizeInBits();
2410 int Index =
2411 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
2412 BitWidth);
2413 assert(Index >= 0 && Index < 3 &&
2414 "Expected FPFastMathDefaultInfo for half, float, or double");
2415 assert(FPFastMathDefaultInfoVec.size() == 3 &&
2416 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
2417 return FPFastMathDefaultInfoVec[Index];
2418}
2419
2420void SPIRVEmitIntrinsics::insertConstantsForFPFastMathDefault(Module &M) {
2421 const SPIRVSubtarget *ST = TM->getSubtargetImpl();
2422 if (!ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
2423 return;
2424
2425 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
2426 // We need the entry point (function) as the key, and the target
2427 // type and flags as the value.
2428 // We also need to check ContractionOff and SignedZeroInfNanPreserve
2429 // execution modes, as they are now deprecated and must be replaced
2430 // with FPFastMathDefaultInfo.
2431 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
2432 if (!Node) {
2433 if (!M.getNamedMetadata(Name: "opencl.enable.FP_CONTRACT")) {
2434 // This requires emitting ContractionOff. However, because
2435 // ContractionOff is now deprecated, we need to replace it with
2436 // FPFastMathDefaultInfo with FP Fast Math Mode bitmask set to all 0.
2437 // We need to create the constant for that.
2438
2439 // Create constant instruction with the bitmask flags.
2440 Constant *InitValue =
2441 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: 0);
2442 // TODO: Reuse constant if there is one already with the required
2443 // value.
2444 [[maybe_unused]] GlobalVariable *GV =
2445 new GlobalVariable(M, // Module
2446 Type::getInt32Ty(C&: M.getContext()), // Type
2447 true, // isConstant
2448 GlobalValue::InternalLinkage, // Linkage
2449 InitValue // Initializer
2450 );
2451 }
2452 return;
2453 }
2454
2455 // The table maps function pointers to their default FP fast math info. It
2456 // can be assumed that the SmallVector is sorted by the bit width of the
2457 // type. The first element is the smallest bit width, and the last element
2458 // is the largest bit width, therefore, we will have {half, float, double}
2459 // in the order of their bit widths.
2460 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
2461 FPFastMathDefaultInfoMap;
2462
2463 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
2464 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
2465 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
2466 Function *F = cast<Function>(
2467 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
2468 const auto EM =
2469 cast<ConstantInt>(
2470 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
2471 ->getZExtValue();
2472 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
2473 assert(MDN->getNumOperands() == 4 &&
2474 "Expected 4 operands for FPFastMathDefault");
2475 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
2476 unsigned Flags =
2477 cast<ConstantInt>(
2478 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
2479 ->getZExtValue();
2480 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2481 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
2482 SPIRV::FPFastMathDefaultInfo &Info =
2483 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
2484 Info.FastMathFlags = Flags;
2485 Info.FPFastMathDefault = true;
2486 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
2487 assert(MDN->getNumOperands() == 2 &&
2488 "Expected no operands for ContractionOff");
2489
2490 // We need to save this info for every possible FP type, i.e. {half,
2491 // float, double, fp128}.
2492 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2493 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
2494 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
2495 Info.ContractionOff = true;
2496 }
2497 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
2498 assert(MDN->getNumOperands() == 3 &&
2499 "Expected 1 operand for SignedZeroInfNanPreserve");
2500 unsigned TargetWidth =
2501 cast<ConstantInt>(
2502 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
2503 ->getZExtValue();
2504 // We need to save this info only for the FP type with TargetWidth.
2505 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2506 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
2507 int Index = SPIRV::FPFastMathDefaultInfoVector::
2508 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
2509 assert(Index >= 0 && Index < 3 &&
2510 "Expected FPFastMathDefaultInfo for half, float, or double");
2511 assert(FPFastMathDefaultInfoVec.size() == 3 &&
2512 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
2513 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
2514 }
2515 }
2516
2517 std::unordered_map<unsigned, GlobalVariable *> GlobalVars;
2518 for (auto &[Func, FPFastMathDefaultInfoVec] : FPFastMathDefaultInfoMap) {
2519 if (FPFastMathDefaultInfoVec.empty())
2520 continue;
2521
2522 for (const SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
2523 assert(Info.Ty && "Expected target type for FPFastMathDefaultInfo");
2524 // Skip if none of the execution modes was used.
2525 unsigned Flags = Info.FastMathFlags;
2526 if (Flags == SPIRV::FPFastMathMode::None && !Info.ContractionOff &&
2527 !Info.SignedZeroInfNanPreserve && !Info.FPFastMathDefault)
2528 continue;
2529
2530 // Check if flags are compatible.
2531 if (Info.ContractionOff && (Flags & SPIRV::FPFastMathMode::AllowContract))
2532 report_fatal_error(reason: "Conflicting FPFastMathFlags: ContractionOff "
2533 "and AllowContract");
2534
2535 if (Info.SignedZeroInfNanPreserve &&
2536 !(Flags &
2537 (SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
2538 SPIRV::FPFastMathMode::NSZ))) {
2539 if (Info.FPFastMathDefault)
2540 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
2541 "SignedZeroInfNanPreserve but at least one of "
2542 "NotNaN/NotInf/NSZ is enabled.");
2543 }
2544
2545 if ((Flags & SPIRV::FPFastMathMode::AllowTransform) &&
2546 !((Flags & SPIRV::FPFastMathMode::AllowReassoc) &&
2547 (Flags & SPIRV::FPFastMathMode::AllowContract))) {
2548 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
2549 "AllowTransform requires AllowReassoc and "
2550 "AllowContract to be set.");
2551 }
2552
2553 auto it = GlobalVars.find(x: Flags);
2554 GlobalVariable *GV = nullptr;
2555 if (it != GlobalVars.end()) {
2556 // Reuse existing global variable.
2557 GV = it->second;
2558 } else {
2559 // Create constant instruction with the bitmask flags.
2560 Constant *InitValue =
2561 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: Flags);
2562 // TODO: Reuse constant if there is one already with the required
2563 // value.
2564 GV = new GlobalVariable(M, // Module
2565 Type::getInt32Ty(C&: M.getContext()), // Type
2566 true, // isConstant
2567 GlobalValue::InternalLinkage, // Linkage
2568 InitValue // Initializer
2569 );
2570 GlobalVars[Flags] = GV;
2571 }
2572 }
2573 }
2574}
2575
2576void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
2577 IRBuilder<> &B) {
2578 auto *II = dyn_cast<IntrinsicInst>(Val: I);
2579 bool IsConstComposite =
2580 II && II->getIntrinsicID() == Intrinsic::spv_const_composite;
2581 if (IsConstComposite && TrackConstants) {
2582 setInsertPointAfterDef(B, I);
2583 auto t = AggrConsts.find(Val: I);
2584 assert(t != AggrConsts.end());
2585 auto *NewOp =
2586 buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
2587 Types: {II->getType(), II->getType()}, Arg: t->second, Arg2: I, Imms: {}, B);
2588 replaceAllUsesWith(Src: I, Dest: NewOp, DeleteOld: false);
2589 NewOp->setArgOperand(i: 0, v: I);
2590 }
2591 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
2592 for (const auto &Op : I->operands()) {
2593 if (isa<PHINode>(Val: I) || isa<SwitchInst>(Val: I) ||
2594 !(isa<ConstantData>(Val: Op) || isa<ConstantExpr>(Val: Op)))
2595 continue;
2596 unsigned OpNo = Op.getOperandNo();
2597 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
2598 (II->paramHasAttr(ArgNo: OpNo, Kind: Attribute::ImmArg))))
2599 continue;
2600
2601 if (!BPrepared) {
2602 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
2603 : B.SetInsertPoint(I);
2604 BPrepared = true;
2605 }
2606 Type *OpTy = Op->getType();
2607 Type *OpElemTy = GR->findDeducedElementType(Val: Op);
2608 Value *NewOp = Op;
2609 if (OpTy->isTargetExtTy()) {
2610 // Since this value is replaced by poison, we need to do the same in
2611 // `insertAssignTypeIntrs`.
2612 Value *OpTyVal = getNormalizedPoisonValue(Ty: OpTy);
2613 NewOp = buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
2614 Types: {OpTy, OpTyVal->getType()}, Arg: Op, Arg2: OpTyVal, Imms: {}, B);
2615 }
2616 if (!IsConstComposite && isPointerTy(T: OpTy) && OpElemTy != nullptr &&
2617 OpElemTy != IntegerType::getInt8Ty(C&: I->getContext())) {
2618 SmallVector<Type *, 2> Types = {OpTy, OpTy};
2619 SmallVector<Value *, 2> Args = {
2620 NewOp, buildMD(Arg: getNormalizedPoisonValue(Ty: OpElemTy)),
2621 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
2622 CallInst *PtrCasted =
2623 B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
2624 GR->buildAssignPtr(B, ElemTy: OpElemTy, Arg: PtrCasted);
2625 NewOp = PtrCasted;
2626 }
2627 if (NewOp != Op)
2628 I->setOperand(i: OpNo, Val: NewOp);
2629 }
2630 if (Named.insert(x: I).second)
2631 emitAssignName(I, B);
2632}
2633
2634Type *SPIRVEmitIntrinsics::deduceFunParamElementType(Function *F,
2635 unsigned OpIdx) {
2636 std::unordered_set<Function *> FVisited;
2637 return deduceFunParamElementType(F, OpIdx, FVisited);
2638}
2639
2640Type *SPIRVEmitIntrinsics::deduceFunParamElementType(
2641 Function *F, unsigned OpIdx, std::unordered_set<Function *> &FVisited) {
2642 // maybe a cycle
2643 if (!FVisited.insert(x: F).second)
2644 return nullptr;
2645
2646 std::unordered_set<Value *> Visited;
2647 SmallVector<std::pair<Function *, unsigned>> Lookup;
2648 // search in function's call sites
2649 for (User *U : F->users()) {
2650 CallInst *CI = dyn_cast<CallInst>(Val: U);
2651 if (!CI || OpIdx >= CI->arg_size())
2652 continue;
2653 Value *OpArg = CI->getArgOperand(i: OpIdx);
2654 if (!isPointerTy(T: OpArg->getType()))
2655 continue;
2656 // maybe we already know operand's element type
2657 if (Type *KnownTy = GR->findDeducedElementType(Val: OpArg))
2658 return KnownTy;
2659 // try to deduce from the operand itself
2660 Visited.clear();
2661 if (Type *Ty = deduceElementTypeHelper(I: OpArg, Visited, UnknownElemTypeI8: false))
2662 return Ty;
2663 // search in actual parameter's users
2664 for (User *OpU : OpArg->users()) {
2665 Instruction *Inst = dyn_cast<Instruction>(Val: OpU);
2666 if (!Inst || Inst == CI)
2667 continue;
2668 Visited.clear();
2669 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8: false))
2670 return Ty;
2671 }
2672 // check if it's a formal parameter of the outer function
2673 if (!CI->getParent() || !CI->getParent()->getParent())
2674 continue;
2675 Function *OuterF = CI->getParent()->getParent();
2676 if (FVisited.find(x: OuterF) != FVisited.end())
2677 continue;
2678 for (unsigned i = 0; i < OuterF->arg_size(); ++i) {
2679 if (OuterF->getArg(i) == OpArg) {
2680 Lookup.push_back(Elt: std::make_pair(x&: OuterF, y&: i));
2681 break;
2682 }
2683 }
2684 }
2685
2686 // search in function parameters
2687 for (auto &Pair : Lookup) {
2688 if (Type *Ty = deduceFunParamElementType(F: Pair.first, OpIdx: Pair.second, FVisited))
2689 return Ty;
2690 }
2691
2692 return nullptr;
2693}
2694
2695void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F,
2696 IRBuilder<> &B) {
2697 B.SetInsertPointPastAllocas(F);
2698 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
2699 Argument *Arg = F->getArg(i: OpIdx);
2700 if (!isUntypedPointerTy(T: Arg->getType()))
2701 continue;
2702 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
2703 if (ElemTy)
2704 continue;
2705 if (hasPointeeTypeAttr(Arg) &&
2706 (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) {
2707 GR->buildAssignPtr(B, ElemTy, Arg);
2708 continue;
2709 }
2710 // search in function's call sites
2711 for (User *U : F->users()) {
2712 CallInst *CI = dyn_cast<CallInst>(Val: U);
2713 if (!CI || OpIdx >= CI->arg_size())
2714 continue;
2715 Value *OpArg = CI->getArgOperand(i: OpIdx);
2716 if (!isPointerTy(T: OpArg->getType()))
2717 continue;
2718 // maybe we already know operand's element type
2719 if ((ElemTy = GR->findDeducedElementType(Val: OpArg)) != nullptr)
2720 break;
2721 }
2722 if (ElemTy) {
2723 GR->buildAssignPtr(B, ElemTy, Arg);
2724 continue;
2725 }
2726 if (HaveFunPtrs) {
2727 for (User *U : Arg->users()) {
2728 CallInst *CI = dyn_cast<CallInst>(Val: U);
2729 if (CI && !isa<IntrinsicInst>(Val: CI) && CI->isIndirectCall() &&
2730 CI->getCalledOperand() == Arg &&
2731 CI->getParent()->getParent() == CurrF) {
2732 SmallVector<std::pair<Value *, unsigned>> Ops;
2733 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy&: ElemTy, IsPostprocessing: false);
2734 if (ElemTy) {
2735 GR->buildAssignPtr(B, ElemTy, Arg);
2736 break;
2737 }
2738 }
2739 }
2740 }
2741 }
2742}
2743
2744void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) {
2745 B.SetInsertPointPastAllocas(F);
2746 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
2747 Argument *Arg = F->getArg(i: OpIdx);
2748 if (!isUntypedPointerTy(T: Arg->getType()))
2749 continue;
2750 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
2751 if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) {
2752 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Arg)) {
2753 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
2754 GR->updateAssignType(AssignCI, Arg, OfType: getNormalizedPoisonValue(Ty: ElemTy));
2755 propagateElemType(Op: Arg, ElemTy: IntegerType::getInt8Ty(C&: F->getContext()),
2756 VisitedSubst);
2757 } else {
2758 GR->buildAssignPtr(B, ElemTy, Arg);
2759 }
2760 }
2761 }
2762}
2763
2764static FunctionType *getFunctionPointerElemType(Function *F,
2765 SPIRVGlobalRegistry *GR) {
2766 FunctionType *FTy = F->getFunctionType();
2767 bool IsNewFTy = false;
2768 SmallVector<Type *, 4> ArgTys;
2769 for (Argument &Arg : F->args()) {
2770 Type *ArgTy = Arg.getType();
2771 if (ArgTy->isPointerTy())
2772 if (Type *ElemTy = GR->findDeducedElementType(Val: &Arg)) {
2773 IsNewFTy = true;
2774 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
2775 }
2776 ArgTys.push_back(Elt: ArgTy);
2777 }
2778 return IsNewFTy
2779 ? FunctionType::get(Result: FTy->getReturnType(), Params: ArgTys, isVarArg: FTy->isVarArg())
2780 : FTy;
2781}
2782
2783bool SPIRVEmitIntrinsics::processFunctionPointers(Module &M) {
2784 SmallVector<Function *> Worklist;
2785 for (auto &F : M) {
2786 if (F.isIntrinsic())
2787 continue;
2788 if (F.isDeclaration()) {
2789 for (User *U : F.users()) {
2790 CallInst *CI = dyn_cast<CallInst>(Val: U);
2791 if (!CI || CI->getCalledFunction() != &F) {
2792 Worklist.push_back(Elt: &F);
2793 break;
2794 }
2795 }
2796 } else {
2797 if (F.user_empty())
2798 continue;
2799 Type *FPElemTy = GR->findDeducedElementType(Val: &F);
2800 if (!FPElemTy)
2801 FPElemTy = getFunctionPointerElemType(F: &F, GR);
2802 for (User *U : F.users()) {
2803 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
2804 if (!II || II->arg_size() != 3 || II->getOperand(i_nocapture: 0) != &F)
2805 continue;
2806 if (II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type ||
2807 II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
2808 GR->updateAssignType(AssignCI: II, Arg: &F, OfType: getNormalizedPoisonValue(Ty: FPElemTy));
2809 break;
2810 }
2811 }
2812 }
2813 }
2814 if (Worklist.empty())
2815 return false;
2816
2817 std::string ServiceFunName = SPIRV_BACKEND_SERVICE_FUN_NAME;
2818 if (!getVacantFunctionName(M, Name&: ServiceFunName))
2819 report_fatal_error(
2820 reason: "cannot allocate a name for the internal service function");
2821 LLVMContext &Ctx = M.getContext();
2822 Function *SF =
2823 Function::Create(Ty: FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: {}, isVarArg: false),
2824 Linkage: GlobalValue::PrivateLinkage, N: ServiceFunName, M);
2825 SF->addFnAttr(SPIRV_BACKEND_SERVICE_FUN_NAME, Val: "");
2826 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: SF);
2827 IRBuilder<> IRB(BB);
2828
2829 for (Function *F : Worklist) {
2830 SmallVector<Value *> Args;
2831 for (const auto &Arg : F->args())
2832 Args.push_back(Elt: getNormalizedPoisonValue(Ty: Arg.getType()));
2833 IRB.CreateCall(Callee: F, Args);
2834 }
2835 IRB.CreateRetVoid();
2836
2837 return true;
2838}
2839
2840// Apply types parsed from demangled function declarations.
2841void SPIRVEmitIntrinsics::applyDemangledPtrArgTypes(IRBuilder<> &B) {
2842 DenseMap<Function *, CallInst *> Ptrcasts;
2843 for (auto It : FDeclPtrTys) {
2844 Function *F = It.first;
2845 for (auto *U : F->users()) {
2846 CallInst *CI = dyn_cast<CallInst>(Val: U);
2847 if (!CI || CI->getCalledFunction() != F)
2848 continue;
2849 unsigned Sz = CI->arg_size();
2850 for (auto [Idx, ElemTy] : It.second) {
2851 if (Idx >= Sz)
2852 continue;
2853 Value *Param = CI->getArgOperand(i: Idx);
2854 if (GR->findDeducedElementType(Val: Param) || isa<GlobalValue>(Val: Param))
2855 continue;
2856 if (Argument *Arg = dyn_cast<Argument>(Val: Param)) {
2857 if (!hasPointeeTypeAttr(Arg)) {
2858 B.SetInsertPointPastAllocas(Arg->getParent());
2859 B.SetCurrentDebugLocation(DebugLoc());
2860 GR->buildAssignPtr(B, ElemTy, Arg);
2861 }
2862 } else if (isaGEP(V: Param)) {
2863 replaceUsesOfWithSpvPtrcast(Op: Param, ElemTy: normalizeType(Ty: ElemTy), I: CI,
2864 Ptrcasts);
2865 } else if (isa<Instruction>(Val: Param)) {
2866 GR->addDeducedElementType(Val: Param, Ty: normalizeType(Ty: ElemTy));
2867 // insertAssignTypeIntrs() will complete buildAssignPtr()
2868 } else {
2869 B.SetInsertPoint(CI->getParent()
2870 ->getParent()
2871 ->getEntryBlock()
2872 .getFirstNonPHIOrDbgOrAlloca());
2873 GR->buildAssignPtr(B, ElemTy, Arg: Param);
2874 }
2875 CallInst *Ref = dyn_cast<CallInst>(Val: Param);
2876 if (!Ref)
2877 continue;
2878 Function *RefF = Ref->getCalledFunction();
2879 if (!RefF || !isPointerTy(T: RefF->getReturnType()) ||
2880 GR->findDeducedElementType(Val: RefF))
2881 continue;
2882 ElemTy = normalizeType(Ty: ElemTy);
2883 GR->addDeducedElementType(Val: RefF, Ty: ElemTy);
2884 GR->addReturnType(
2885 ArgF: RefF, DerivedTy: TypedPointerType::get(
2886 ElementType: ElemTy, AddressSpace: getPointerAddressSpace(T: RefF->getReturnType())));
2887 }
2888 }
2889 }
2890}
2891
2892GetElementPtrInst *
2893SPIRVEmitIntrinsics::simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP) {
2894 // getelementptr [0 x T], P, 0 (zero), I -> getelementptr T, P, I.
2895 // If type is 0-length array and first index is 0 (zero), drop both the
2896 // 0-length array type and the first index. This is a common pattern in
2897 // the IR, e.g. when using a zero-length array as a placeholder for a
2898 // flexible array such as unbound arrays.
2899 assert(GEP && "GEP is null");
2900 Type *SrcTy = GEP->getSourceElementType();
2901 SmallVector<Value *, 8> Indices(GEP->indices());
2902 ArrayType *ArrTy = dyn_cast<ArrayType>(Val: SrcTy);
2903 if (ArrTy && ArrTy->getNumElements() == 0 &&
2904 PatternMatch::match(V: Indices[0], P: PatternMatch::m_Zero())) {
2905 Indices.erase(CI: Indices.begin());
2906 SrcTy = ArrTy->getElementType();
2907 return GetElementPtrInst::Create(PointeeType: SrcTy, Ptr: GEP->getPointerOperand(), IdxList: Indices,
2908 NW: GEP->getNoWrapFlags(), NameStr: "",
2909 InsertBefore: GEP->getIterator());
2910 }
2911 return nullptr;
2912}
2913
2914void SPIRVEmitIntrinsics::emitUnstructuredLoopControls(Function &F,
2915 IRBuilder<> &B) {
2916 const SPIRVSubtarget *ST = TM->getSubtargetImpl(F);
2917 // Shaders use SPIRVStructurizer which emits OpLoopMerge via spv_loop_merge.
2918 if (ST->isShader())
2919 return;
2920 if (!ST->canUseExtension(
2921 E: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls))
2922 return;
2923
2924 for (BasicBlock &BB : F) {
2925 Instruction *Term = BB.getTerminator();
2926 MDNode *LoopMD = Term->getMetadata(KindID: LLVMContext::MD_loop);
2927 if (!LoopMD)
2928 continue;
2929
2930 SmallVector<unsigned, 1> Ops =
2931 getSpirvLoopControlOperandsFromLoopMetadata(LoopMD);
2932 unsigned LC = Ops[0];
2933 if (LC == SPIRV::LoopControl::None)
2934 continue;
2935
2936 // Emit intrinsic: loop control mask + optional parameters.
2937 B.SetInsertPoint(Term);
2938 SmallVector<Value *, 4> IntrArgs;
2939 IntrArgs.push_back(Elt: B.getInt32(C: LC));
2940 for (unsigned I = 1; I < Ops.size(); ++I)
2941 IntrArgs.push_back(Elt: B.getInt32(C: Ops[I]));
2942 B.CreateIntrinsic(ID: Intrinsic::spv_loop_control_intel, Args: IntrArgs);
2943 }
2944}
2945
2946bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
2947 if (Func.isDeclaration())
2948 return false;
2949
2950 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F: Func);
2951 GR = ST.getSPIRVGlobalRegistry();
2952
2953 if (!CurrF)
2954 HaveFunPtrs =
2955 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers);
2956
2957 CurrF = &Func;
2958 IRBuilder<> B(Func.getContext());
2959 AggrConsts.clear();
2960 AggrConstTypes.clear();
2961 AggrStores.clear();
2962
2963 // Fix GEP result types ahead of inference, and simplify if possible.
2964 // Data structure for dead instructions that were simplified and replaced.
2965 SmallPtrSet<Instruction *, 4> DeadInsts;
2966 for (auto &I : instructions(F&: Func)) {
2967 auto *GEP = dyn_cast<GetElementPtrInst>(Val: &I);
2968 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
2969
2970 if ((!GEP && !SGEP) || GR->findDeducedElementType(Val: &I))
2971 continue;
2972
2973 if (SGEP) {
2974 GR->addDeducedElementType(Val: SGEP,
2975 Ty: normalizeType(Ty: SGEP->getResultElementType()));
2976 continue;
2977 }
2978
2979 GetElementPtrInst *NewGEP = simplifyZeroLengthArrayGepInst(GEP);
2980 if (NewGEP) {
2981 GEP->replaceAllUsesWith(V: NewGEP);
2982 DeadInsts.insert(Ptr: GEP);
2983 GEP = NewGEP;
2984 }
2985 if (Type *GepTy = getGEPType(Ref: GEP))
2986 GR->addDeducedElementType(Val: GEP, Ty: normalizeType(Ty: GepTy));
2987 }
2988 // Remove dead instructions that were simplified and replaced.
2989 for (auto *I : DeadInsts) {
2990 assert(I->use_empty() && "Dead instruction should not have any uses left");
2991 I->eraseFromParent();
2992 }
2993
2994 processParamTypesByFunHeader(F: CurrF, B);
2995
2996 // StoreInst's operand type can be changed during the next
2997 // transformations, so we need to store it in the set. Also store already
2998 // transformed types.
2999 for (auto &I : instructions(F&: Func)) {
3000 StoreInst *SI = dyn_cast<StoreInst>(Val: &I);
3001 if (!SI)
3002 continue;
3003 Type *ElTy = SI->getValueOperand()->getType();
3004 if (ElTy->isAggregateType() || ElTy->isVectorTy())
3005 AggrStores.insert(V: &I);
3006 }
3007
3008 B.SetInsertPoint(TheBB: &Func.getEntryBlock(), IP: Func.getEntryBlock().begin());
3009 for (auto &GV : Func.getParent()->globals())
3010 processGlobalValue(GV, B);
3011
3012 preprocessUndefs(B);
3013 preprocessCompositeConstants(B);
3014 SmallVector<Instruction *> Worklist(
3015 llvm::make_pointer_range(Range: instructions(F&: Func)));
3016
3017 applyDemangledPtrArgTypes(B);
3018
3019 // Pass forward: use operand to deduce instructions result.
3020 for (auto &I : Worklist) {
3021 // Don't emit intrinsincs for convergence intrinsics.
3022 if (isConvergenceIntrinsic(I))
3023 continue;
3024
3025 bool Postpone = insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: false);
3026 // if Postpone is true, we can't decide on pointee type yet
3027 insertAssignTypeIntrs(I, B);
3028 insertPtrCastOrAssignTypeInstr(I, B);
3029 insertSpirvDecorations(I, B);
3030 // if instruction requires a pointee type set, let's check if we know it
3031 // already, and force it to be i8 if not
3032 if (Postpone && !GR->findAssignPtrTypeInstr(Val: I))
3033 insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: true);
3034
3035 if (auto *FPI = dyn_cast<ConstrainedFPIntrinsic>(Val: I))
3036 useRoundingMode(FPI, B);
3037 }
3038
3039 // Pass backward: use instructions results to specify/update/cast operands
3040 // where needed.
3041 SmallPtrSet<Instruction *, 4> IncompleteRets;
3042 for (auto &I : llvm::reverse(C: instructions(F&: Func)))
3043 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets);
3044
3045 // Pass forward for PHIs only, their operands are not preceed the
3046 // instruction in meaning of `instructions(Func)`.
3047 for (BasicBlock &BB : Func)
3048 for (PHINode &Phi : BB.phis())
3049 if (isPointerTy(T: Phi.getType()))
3050 deduceOperandElementType(I: &Phi, IncompleteRets: nullptr);
3051
3052 for (auto *I : Worklist) {
3053 TrackConstants = true;
3054 if (!I->getType()->isVoidTy() || isa<StoreInst>(Val: I))
3055 setInsertPointAfterDef(B, I);
3056 // Visitors return either the original/newly created instruction for
3057 // further processing, nullptr otherwise.
3058 I = visit(I&: *I);
3059 if (!I)
3060 continue;
3061
3062 // Don't emit intrinsics for convergence operations.
3063 if (isConvergenceIntrinsic(I))
3064 continue;
3065
3066 addSaturatedDecorationToIntrinsic(I, B);
3067 processInstrAfterVisit(I, B);
3068 }
3069
3070 emitUnstructuredLoopControls(F&: Func, B);
3071
3072 return true;
3073}
3074
3075// Try to deduce a better type for pointers to untyped ptr.
3076bool SPIRVEmitIntrinsics::postprocessTypes(Module &M) {
3077 if (!GR || TodoTypeSz == 0)
3078 return false;
3079
3080 unsigned SzTodo = TodoTypeSz;
3081 DenseMap<Value *, SmallPtrSet<Value *, 4>> ToProcess;
3082 for (auto [Op, Enabled] : TodoType) {
3083 // TODO: add isa<CallInst>(Op) to continue
3084 if (!Enabled || isaGEP(V: Op))
3085 continue;
3086 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Op);
3087 Type *KnownTy = GR->findDeducedElementType(Val: Op);
3088 if (!KnownTy || !AssignCI)
3089 continue;
3090 assert(Op == AssignCI->getArgOperand(0));
3091 // Try to improve the type deduced after all Functions are processed.
3092 if (auto *CI = dyn_cast<Instruction>(Val: Op)) {
3093 CurrF = CI->getParent()->getParent();
3094 std::unordered_set<Value *> Visited;
3095 if (Type *ElemTy = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8: false, IgnoreKnownType: true)) {
3096 if (ElemTy != KnownTy) {
3097 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3098 propagateElemType(Op: CI, ElemTy, VisitedSubst);
3099 eraseTodoType(Op);
3100 continue;
3101 }
3102 }
3103 }
3104
3105 if (Op->hasUseList()) {
3106 for (User *U : Op->users()) {
3107 Instruction *Inst = dyn_cast<Instruction>(Val: U);
3108 if (Inst && !isa<IntrinsicInst>(Val: Inst))
3109 ToProcess[Inst].insert(Ptr: Op);
3110 }
3111 }
3112 }
3113 if (TodoTypeSz == 0)
3114 return true;
3115
3116 for (auto &F : M) {
3117 CurrF = &F;
3118 SmallPtrSet<Instruction *, 4> IncompleteRets;
3119 for (auto &I : llvm::reverse(C: instructions(F))) {
3120 auto It = ToProcess.find(Val: &I);
3121 if (It == ToProcess.end())
3122 continue;
3123 It->second.remove_if(P: [this](Value *V) { return !isTodoType(Op: V); });
3124 if (It->second.size() == 0)
3125 continue;
3126 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets, AskOps: &It->second, IsPostprocessing: true);
3127 if (TodoTypeSz == 0)
3128 return true;
3129 }
3130 }
3131
3132 return SzTodo > TodoTypeSz;
3133}
3134
3135// Parse and store argument types of function declarations where needed.
3136void SPIRVEmitIntrinsics::parseFunDeclarations(Module &M) {
3137 for (auto &F : M) {
3138 if (!F.isDeclaration() || F.isIntrinsic())
3139 continue;
3140 // get the demangled name
3141 std::string DemangledName = getOclOrSpirvBuiltinDemangledName(Name: F.getName());
3142 if (DemangledName.empty())
3143 continue;
3144 // allow only OpGroupAsyncCopy use case at the moment
3145 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F);
3146 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
3147 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
3148 if (Opcode != SPIRV::OpGroupAsyncCopy)
3149 continue;
3150 // find pointer arguments
3151 SmallVector<unsigned> Idxs;
3152 for (unsigned OpIdx = 0; OpIdx < F.arg_size(); ++OpIdx) {
3153 Argument *Arg = F.getArg(i: OpIdx);
3154 if (isPointerTy(T: Arg->getType()) && !hasPointeeTypeAttr(Arg))
3155 Idxs.push_back(Elt: OpIdx);
3156 }
3157 if (!Idxs.size())
3158 continue;
3159 // parse function arguments
3160 LLVMContext &Ctx = F.getContext();
3161 SmallVector<StringRef, 10> TypeStrs;
3162 SPIRV::parseBuiltinTypeStr(BuiltinArgsTypeStrs&: TypeStrs, DemangledCall: DemangledName, Ctx);
3163 if (!TypeStrs.size())
3164 continue;
3165 // find type info for pointer arguments
3166 for (unsigned Idx : Idxs) {
3167 if (Idx >= TypeStrs.size())
3168 continue;
3169 if (Type *ElemTy =
3170 SPIRV::parseBuiltinCallArgumentType(TypeStr: TypeStrs[Idx].trim(), Ctx))
3171 if (TypedPointerType::isValidElementType(ElemTy) &&
3172 !ElemTy->isTargetExtTy())
3173 FDeclPtrTys[&F].push_back(Elt: std::make_pair(x&: Idx, y&: ElemTy));
3174 }
3175 }
3176}
3177
3178bool SPIRVEmitIntrinsics::runOnModule(Module &M) {
3179 bool Changed = false;
3180
3181 parseFunDeclarations(M);
3182 insertConstantsForFPFastMathDefault(M);
3183
3184 TodoType.clear();
3185 for (auto &F : M)
3186 Changed |= runOnFunction(Func&: F);
3187
3188 // Specify function parameters after all functions were processed.
3189 for (auto &F : M) {
3190 // check if function parameter types are set
3191 CurrF = &F;
3192 if (!F.isDeclaration() && !F.isIntrinsic()) {
3193 IRBuilder<> B(F.getContext());
3194 processParamTypes(F: &F, B);
3195 }
3196 }
3197
3198 CanTodoType = false;
3199 Changed |= postprocessTypes(M);
3200
3201 if (HaveFunPtrs)
3202 Changed |= processFunctionPointers(M);
3203
3204 return Changed;
3205}
3206
3207ModulePass *llvm::createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM) {
3208 return new SPIRVEmitIntrinsics(TM);
3209}
3210