1//===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass emits SPIRV intrinsics keeping essential high-level information for
10// the translation of LLVM IR to SPIR-V.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRV.h"
15#include "SPIRVBuiltins.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVTargetMachine.h"
18#include "SPIRVUtils.h"
19#include "llvm/ADT/DenseSet.h"
20#include "llvm/ADT/StringSet.h"
21#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/InstIterator.h"
23#include "llvm/IR/InstVisitor.h"
24#include "llvm/IR/IntrinsicsSPIRV.h"
25#include "llvm/IR/PatternMatch.h"
26#include "llvm/IR/TypedPointerType.h"
27#include "llvm/Support/CommandLine.h"
28#include "llvm/Transforms/Utils/Local.h"
29
30#include <cassert>
31#include <queue>
32#include <unordered_set>
33
34// This pass performs the following transformation on LLVM IR level required
35// for the following translation to SPIR-V:
36// - replaces direct usages of aggregate constants with target-specific
37// intrinsics;
38// - replaces aggregates-related instructions (extract/insert, ld/st, etc)
39// with a target-specific intrinsics;
40// - emits intrinsics for the global variable initializers since IRTranslator
41// doesn't handle them and it's not very convenient to translate them
42// ourselves;
43// - emits intrinsics to keep track of the string names assigned to the values;
44// - emits intrinsics to keep track of constants (this is necessary to have an
45// LLVM IR constant after the IRTranslation is completed) for their further
46// deduplication;
47// - emits intrinsics to keep track of original LLVM types of the values
48// to be able to emit proper SPIR-V types eventually.
49//
50// TODO: consider removing spv.track.constant in favor of spv.assign.type.
51
52using namespace llvm;
53
54static cl::opt<bool>
55 SpirvEmitOpNames("spirv-emit-op-names",
56 cl::desc("Emit OpName for all instructions"),
57 cl::init(Val: false));
58
59namespace llvm::SPIRV {
60#define GET_BuiltinGroup_DECL
61#include "SPIRVGenTables.inc"
62} // namespace llvm::SPIRV
63
64namespace {
65
66class SPIRVEmitIntrinsics
67 : public ModulePass,
68 public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
69 SPIRVTargetMachine *TM = nullptr;
70 SPIRVGlobalRegistry *GR = nullptr;
71 Function *CurrF = nullptr;
72 bool TrackConstants = true;
73 bool HaveFunPtrs = false;
74 DenseMap<Instruction *, Constant *> AggrConsts;
75 DenseMap<Instruction *, Type *> AggrConstTypes;
76 DenseSet<Instruction *> AggrStores;
77 std::unordered_set<Value *> Named;
78
79 // map of function declarations to <pointer arg index => element type>
80 DenseMap<Function *, SmallVector<std::pair<unsigned, Type *>>> FDeclPtrTys;
81
82 // a register of Instructions that don't have a complete type definition
83 bool CanTodoType = true;
84 unsigned TodoTypeSz = 0;
85 DenseMap<Value *, bool> TodoType;
86 void insertTodoType(Value *Op) {
87 // TODO: add isa<CallInst>(Op) to no-insert
88 if (CanTodoType && !isa<GetElementPtrInst>(Val: Op)) {
89 auto It = TodoType.try_emplace(Key: Op, Args: true);
90 if (It.second)
91 ++TodoTypeSz;
92 }
93 }
94 void eraseTodoType(Value *Op) {
95 auto It = TodoType.find(Val: Op);
96 if (It != TodoType.end() && It->second) {
97 It->second = false;
98 --TodoTypeSz;
99 }
100 }
101 bool isTodoType(Value *Op) {
102 if (isa<GetElementPtrInst>(Val: Op))
103 return false;
104 auto It = TodoType.find(Val: Op);
105 return It != TodoType.end() && It->second;
106 }
107 // a register of Instructions that were visited by deduceOperandElementType()
108 // to validate operand types with an instruction
109 std::unordered_set<Instruction *> TypeValidated;
110
111 // well known result types of builtins
112 enum WellKnownTypes { Event };
113
114 // deduce element type of untyped pointers
115 Type *deduceElementType(Value *I, bool UnknownElemTypeI8);
116 Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8);
117 Type *deduceElementTypeHelper(Value *I, std::unordered_set<Value *> &Visited,
118 bool UnknownElemTypeI8,
119 bool IgnoreKnownType = false);
120 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
121 bool UnknownElemTypeI8);
122 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
123 std::unordered_set<Value *> &Visited,
124 bool UnknownElemTypeI8);
125 Type *deduceElementTypeByUsersDeep(Value *Op,
126 std::unordered_set<Value *> &Visited,
127 bool UnknownElemTypeI8);
128 void maybeAssignPtrType(Type *&Ty, Value *I, Type *RefTy,
129 bool UnknownElemTypeI8);
130
131 // deduce nested types of composites
132 Type *deduceNestedTypeHelper(User *U, bool UnknownElemTypeI8);
133 Type *deduceNestedTypeHelper(User *U, Type *Ty,
134 std::unordered_set<Value *> &Visited,
135 bool UnknownElemTypeI8);
136
137 // deduce Types of operands of the Instruction if possible
138 void deduceOperandElementType(Instruction *I,
139 SmallPtrSet<Instruction *, 4> *IncompleteRets,
140 const SmallPtrSet<Value *, 4> *AskOps = nullptr,
141 bool IsPostprocessing = false);
142
143 void preprocessCompositeConstants(IRBuilder<> &B);
144 void preprocessUndefs(IRBuilder<> &B);
145
146 Type *reconstructType(Value *Op, bool UnknownElemTypeI8,
147 bool IsPostprocessing);
148
149 void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
150 void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
151 bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
152 bool UnknownElemTypeI8);
153 void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B);
154 void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V,
155 IRBuilder<> &B);
156 void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer,
157 Type *ExpectedElementType,
158 unsigned OperandToReplace,
159 IRBuilder<> &B);
160 void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
161 bool shouldTryToAddMemAliasingDecoration(Instruction *Inst);
162 void insertSpirvDecorations(Instruction *I, IRBuilder<> &B);
163 void insertConstantsForFPFastMathDefault(Module &M);
164 void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
165 void processParamTypes(Function *F, IRBuilder<> &B);
166 void processParamTypesByFunHeader(Function *F, IRBuilder<> &B);
167 Type *deduceFunParamElementType(Function *F, unsigned OpIdx);
168 Type *deduceFunParamElementType(Function *F, unsigned OpIdx,
169 std::unordered_set<Function *> &FVisited);
170
171 bool deduceOperandElementTypeCalledFunction(
172 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
173 Type *&KnownElemTy, bool &Incomplete);
174 void deduceOperandElementTypeFunctionPointer(
175 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
176 Type *&KnownElemTy, bool IsPostprocessing);
177 bool deduceOperandElementTypeFunctionRet(
178 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
179 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing,
180 Type *&KnownElemTy, Value *Op, Function *F);
181
182 CallInst *buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy);
183 void replaceUsesOfWithSpvPtrcast(Value *Op, Type *ElemTy, Instruction *I,
184 DenseMap<Function *, CallInst *> Ptrcasts);
185 void propagateElemType(Value *Op, Type *ElemTy,
186 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
187 void
188 propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
189 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
190 void propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
191 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
192 std::unordered_set<Value *> &Visited,
193 DenseMap<Function *, CallInst *> Ptrcasts);
194
195 void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
196 void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src,
197 Instruction *Dest, bool DeleteOld = true);
198
199 void applyDemangledPtrArgTypes(IRBuilder<> &B);
200
201 GetElementPtrInst *simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP);
202
203 bool runOnFunction(Function &F);
204 bool postprocessTypes(Module &M);
205 bool processFunctionPointers(Module &M);
206 void parseFunDeclarations(Module &M);
207
208 void useRoundingMode(ConstrainedFPIntrinsic *FPI, IRBuilder<> &B);
209
210 void emitUnstructuredLoopControls(Function &F, IRBuilder<> &B);
211
212 // Tries to walk the type accessed by the given GEP instruction.
213 // For each nested type access, one of the 2 callbacks is called:
214 // - OnLiteralIndexing when the index is a known constant value.
215 // Parameters:
216 // PointedType: the pointed type resulting of this indexing.
217 // If the parent type is an array, this is the index in the array.
218 // If the parent type is a struct, this is the field index.
219 // Index: index of the element in the parent type.
220 // - OnDynamnicIndexing when the index is a non-constant value.
221 // This callback is only called when indexing into an array.
222 // Parameters:
223 // ElementType: the type of the elements stored in the parent array.
224 // Offset: the Value* containing the byte offset into the array.
225 // Return true if an error occurred during the walk, false otherwise.
226 bool walkLogicalAccessChain(
227 GetElementPtrInst &GEP,
228 const std::function<void(Type *PointedType, uint64_t Index)>
229 &OnLiteralIndexing,
230 const std::function<void(Type *ElementType, Value *Offset)>
231 &OnDynamicIndexing);
232
233 // Returns the type accessed using the given GEP instruction by relying
234 // on the GEP type.
235 // FIXME: GEP types are not supposed to be used to retrieve the pointed
236 // type. This must be fixed.
237 Type *getGEPType(GetElementPtrInst *GEP);
238
239 // Returns the type accessed using the given GEP instruction by walking
240 // the source type using the GEP indices.
241 // FIXME: without help from the frontend, this method cannot reliably retrieve
242 // the stored type, nor can robustly determine the depth of the type
243 // we are accessing.
244 Type *getGEPTypeLogical(GetElementPtrInst *GEP);
245
246 Instruction *buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP);
247
248public:
249 static char ID;
250 SPIRVEmitIntrinsics(SPIRVTargetMachine *TM = nullptr)
251 : ModulePass(ID), TM(TM) {}
252 Instruction *visitInstruction(Instruction &I) { return &I; }
253 Instruction *visitSwitchInst(SwitchInst &I);
254 Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
255 Instruction *visitBitCastInst(BitCastInst &I);
256 Instruction *visitInsertElementInst(InsertElementInst &I);
257 Instruction *visitExtractElementInst(ExtractElementInst &I);
258 Instruction *visitInsertValueInst(InsertValueInst &I);
259 Instruction *visitExtractValueInst(ExtractValueInst &I);
260 Instruction *visitLoadInst(LoadInst &I);
261 Instruction *visitStoreInst(StoreInst &I);
262 Instruction *visitAllocaInst(AllocaInst &I);
263 Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
264 Instruction *visitUnreachableInst(UnreachableInst &I);
265 Instruction *visitCallInst(CallInst &I);
266
267 StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
268
269 bool runOnModule(Module &M) override;
270
271 void getAnalysisUsage(AnalysisUsage &AU) const override {
272 ModulePass::getAnalysisUsage(AU);
273 }
274};
275
276bool isConvergenceIntrinsic(const Instruction *I) {
277 const auto *II = dyn_cast<IntrinsicInst>(Val: I);
278 if (!II)
279 return false;
280
281 return II->getIntrinsicID() == Intrinsic::experimental_convergence_entry ||
282 II->getIntrinsicID() == Intrinsic::experimental_convergence_loop ||
283 II->getIntrinsicID() == Intrinsic::experimental_convergence_anchor;
284}
285
286bool expectIgnoredInIRTranslation(const Instruction *I) {
287 const auto *II = dyn_cast<IntrinsicInst>(Val: I);
288 if (!II)
289 return false;
290 switch (II->getIntrinsicID()) {
291 case Intrinsic::invariant_start:
292 case Intrinsic::spv_resource_handlefrombinding:
293 case Intrinsic::spv_resource_getpointer:
294 return true;
295 default:
296 return false;
297 }
298}
299
300// Returns the source pointer from `I` ignoring intermediate ptrcast.
301Value *getPointerRoot(Value *I) {
302 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
303 if (II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
304 Value *V = II->getArgOperand(i: 0);
305 return getPointerRoot(I: V);
306 }
307 }
308 return I;
309}
310
311} // namespace
312
313char SPIRVEmitIntrinsics::ID = 0;
314
315INITIALIZE_PASS(SPIRVEmitIntrinsics, "emit-intrinsics", "SPIRV emit intrinsics",
316 false, false)
317
318static inline bool isAssignTypeInstr(const Instruction *I) {
319 return isa<IntrinsicInst>(Val: I) &&
320 cast<IntrinsicInst>(Val: I)->getIntrinsicID() == Intrinsic::spv_assign_type;
321}
322
323static bool isMemInstrToReplace(Instruction *I) {
324 return isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I) || isa<InsertValueInst>(Val: I) ||
325 isa<ExtractValueInst>(Val: I) || isa<AtomicCmpXchgInst>(Val: I);
326}
327
328static bool isAggrConstForceInt32(const Value *V) {
329 return isa<ConstantArray>(Val: V) || isa<ConstantStruct>(Val: V) ||
330 isa<ConstantDataArray>(Val: V) ||
331 (isa<ConstantAggregateZero>(Val: V) && !V->getType()->isVectorTy());
332}
333
334static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
335 if (isa<PHINode>(Val: I))
336 B.SetInsertPoint(I->getParent()->getFirstNonPHIOrDbgOrAlloca());
337 else
338 B.SetInsertPoint(I);
339}
340
341static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I) {
342 B.SetCurrentDebugLocation(I->getDebugLoc());
343 if (I->getType()->isVoidTy())
344 B.SetInsertPoint(I->getNextNode());
345 else
346 B.SetInsertPoint(*I->getInsertionPointAfterDef());
347}
348
349static bool requireAssignType(Instruction *I) {
350 if (const auto *Intr = dyn_cast<IntrinsicInst>(Val: I)) {
351 switch (Intr->getIntrinsicID()) {
352 case Intrinsic::invariant_start:
353 case Intrinsic::invariant_end:
354 return false;
355 }
356 }
357 return true;
358}
359
360static inline void reportFatalOnTokenType(const Instruction *I) {
361 if (I->getType()->isTokenTy())
362 report_fatal_error(reason: "A token is encountered but SPIR-V without extensions "
363 "does not support token type",
364 gen_crash_diag: false);
365}
366
367static void emitAssignName(Instruction *I, IRBuilder<> &B) {
368 if (!I->hasName() || I->getType()->isAggregateType() ||
369 expectIgnoredInIRTranslation(I))
370 return;
371
372 // We want to be conservative when adding the names because they can interfere
373 // with later optimizations.
374 bool KeepName = SpirvEmitOpNames;
375 if (!KeepName) {
376 if (isa<AllocaInst>(Val: I)) {
377 KeepName = true;
378 } else if (auto *CI = dyn_cast<CallBase>(Val: I)) {
379 Function *F = CI->getCalledFunction();
380 if (F && F->getName().starts_with(Prefix: "llvm.spv.alloca"))
381 KeepName = true;
382 }
383 }
384
385 if (!KeepName)
386 return;
387
388 reportFatalOnTokenType(I);
389 setInsertPointAfterDef(B, I);
390 LLVMContext &Ctx = I->getContext();
391 std::vector<Value *> Args = {
392 I, MetadataAsValue::get(
393 Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: I->getName())))};
394 B.CreateIntrinsic(ID: Intrinsic::spv_assign_name, Types: {I->getType()}, Args);
395}
396
397void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest,
398 bool DeleteOld) {
399 GR->replaceAllUsesWith(Old: Src, New: Dest, DeleteOld);
400 // Update uncomplete type records if any
401 if (isTodoType(Op: Src)) {
402 if (DeleteOld)
403 eraseTodoType(Op: Src);
404 insertTodoType(Op: Dest);
405 }
406}
407
408void SPIRVEmitIntrinsics::replaceAllUsesWithAndErase(IRBuilder<> &B,
409 Instruction *Src,
410 Instruction *Dest,
411 bool DeleteOld) {
412 replaceAllUsesWith(Src, Dest, DeleteOld);
413 std::string Name = Src->hasName() ? Src->getName().str() : "";
414 Src->eraseFromParent();
415 if (!Name.empty()) {
416 Dest->setName(Name);
417 if (Named.insert(x: Dest).second)
418 emitAssignName(I: Dest, B);
419 }
420}
421
422static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
423 return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
424 isPointerTy(T: SI->getValueOperand()->getType()) &&
425 isa<Argument>(Val: SI->getValueOperand());
426}
427
428// Maybe restore original function return type.
429static inline Type *restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I,
430 Type *Ty) {
431 CallInst *CI = dyn_cast<CallInst>(Val: I);
432 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
433 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
434 return Ty;
435 if (Type *OriginalTy = GR->findMutated(Val: CI->getCalledFunction()))
436 return OriginalTy;
437 return Ty;
438}
439
440// Reconstruct type with nested element types according to deduced type info.
441// Return nullptr if no detailed type info is available.
442Type *SPIRVEmitIntrinsics::reconstructType(Value *Op, bool UnknownElemTypeI8,
443 bool IsPostprocessing) {
444 Type *Ty = Op->getType();
445 if (auto *OpI = dyn_cast<Instruction>(Val: Op))
446 Ty = restoreMutatedType(GR, I: OpI, Ty);
447 if (!isUntypedPointerTy(T: Ty))
448 return Ty;
449 // try to find the pointee type
450 if (Type *NestedTy = GR->findDeducedElementType(Val: Op))
451 return getTypedPointerWrapper(ElemTy: NestedTy, AS: getPointerAddressSpace(T: Ty));
452 // not a pointer according to the type info (e.g., Event object)
453 CallInst *CI = GR->findAssignPtrTypeInstr(Val: Op);
454 if (CI) {
455 MetadataAsValue *MD = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
456 return cast<ConstantAsMetadata>(Val: MD->getMetadata())->getType();
457 }
458 if (UnknownElemTypeI8) {
459 if (!IsPostprocessing)
460 insertTodoType(Op);
461 return getTypedPointerWrapper(ElemTy: IntegerType::getInt8Ty(C&: Op->getContext()),
462 AS: getPointerAddressSpace(T: Ty));
463 }
464 return nullptr;
465}
466
467CallInst *SPIRVEmitIntrinsics::buildSpvPtrcast(Function *F, Value *Op,
468 Type *ElemTy) {
469 IRBuilder<> B(Op->getContext());
470 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
471 // spv_ptrcast's argument Op denotes an instruction that generates
472 // a value, and we may use getInsertionPointAfterDef()
473 setInsertPointAfterDef(B, I: OpI);
474 } else if (auto *OpA = dyn_cast<Argument>(Val: Op)) {
475 B.SetInsertPointPastAllocas(OpA->getParent());
476 B.SetCurrentDebugLocation(DebugLoc());
477 } else {
478 B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
479 }
480 Type *OpTy = Op->getType();
481 SmallVector<Type *, 2> Types = {OpTy, OpTy};
482 SmallVector<Value *, 2> Args = {Op, buildMD(Arg: getNormalizedPoisonValue(Ty: ElemTy)),
483 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
484 CallInst *PtrCasted =
485 B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
486 GR->buildAssignPtr(B, ElemTy, Arg: PtrCasted);
487 return PtrCasted;
488}
489
490void SPIRVEmitIntrinsics::replaceUsesOfWithSpvPtrcast(
491 Value *Op, Type *ElemTy, Instruction *I,
492 DenseMap<Function *, CallInst *> Ptrcasts) {
493 Function *F = I->getParent()->getParent();
494 CallInst *PtrCastedI = nullptr;
495 auto It = Ptrcasts.find(Val: F);
496 if (It == Ptrcasts.end()) {
497 PtrCastedI = buildSpvPtrcast(F, Op, ElemTy);
498 Ptrcasts[F] = PtrCastedI;
499 } else {
500 PtrCastedI = It->second;
501 }
502 I->replaceUsesOfWith(From: Op, To: PtrCastedI);
503}
504
505void SPIRVEmitIntrinsics::propagateElemType(
506 Value *Op, Type *ElemTy,
507 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
508 DenseMap<Function *, CallInst *> Ptrcasts;
509 SmallVector<User *> Users(Op->users());
510 for (auto *U : Users) {
511 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
512 continue;
513 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
514 continue;
515 Instruction *UI = dyn_cast<Instruction>(Val: U);
516 // If the instruction was validated already, we need to keep it valid by
517 // keeping current Op type.
518 if (isa<GetElementPtrInst>(Val: UI) ||
519 TypeValidated.find(x: UI) != TypeValidated.end())
520 replaceUsesOfWithSpvPtrcast(Op, ElemTy, I: UI, Ptrcasts);
521 }
522}
523
524void SPIRVEmitIntrinsics::propagateElemTypeRec(
525 Value *Op, Type *PtrElemTy, Type *CastElemTy,
526 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
527 std::unordered_set<Value *> Visited;
528 DenseMap<Function *, CallInst *> Ptrcasts;
529 propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited,
530 Ptrcasts: std::move(Ptrcasts));
531}
532
533void SPIRVEmitIntrinsics::propagateElemTypeRec(
534 Value *Op, Type *PtrElemTy, Type *CastElemTy,
535 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
536 std::unordered_set<Value *> &Visited,
537 DenseMap<Function *, CallInst *> Ptrcasts) {
538 if (!Visited.insert(x: Op).second)
539 return;
540 SmallVector<User *> Users(Op->users());
541 for (auto *U : Users) {
542 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
543 continue;
544 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
545 continue;
546 Instruction *UI = dyn_cast<Instruction>(Val: U);
547 // If the instruction was validated already, we need to keep it valid by
548 // keeping current Op type.
549 if (isa<GetElementPtrInst>(Val: UI) ||
550 TypeValidated.find(x: UI) != TypeValidated.end())
551 replaceUsesOfWithSpvPtrcast(Op, ElemTy: CastElemTy, I: UI, Ptrcasts);
552 }
553}
554
555// Set element pointer type to the given value of ValueTy and tries to
556// specify this type further (recursively) by Operand value, if needed.
557
558Type *
559SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
560 bool UnknownElemTypeI8) {
561 std::unordered_set<Value *> Visited;
562 return deduceElementTypeByValueDeep(ValueTy, Operand, Visited,
563 UnknownElemTypeI8);
564}
565
566Type *SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(
567 Type *ValueTy, Value *Operand, std::unordered_set<Value *> &Visited,
568 bool UnknownElemTypeI8) {
569 Type *Ty = ValueTy;
570 if (Operand) {
571 if (auto *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
572 if (Type *NestedTy =
573 deduceElementTypeHelper(I: Operand, Visited, UnknownElemTypeI8))
574 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
575 } else {
576 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Operand), Ty, Visited,
577 UnknownElemTypeI8);
578 }
579 }
580 return Ty;
581}
582
583// Traverse User instructions to deduce an element pointer type of the operand.
584Type *SPIRVEmitIntrinsics::deduceElementTypeByUsersDeep(
585 Value *Op, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8) {
586 if (!Op || !isPointerTy(T: Op->getType()) || isa<ConstantPointerNull>(Val: Op) ||
587 isa<UndefValue>(Val: Op))
588 return nullptr;
589
590 if (auto ElemTy = getPointeeType(Ty: Op->getType()))
591 return ElemTy;
592
593 // maybe we already know operand's element type
594 if (Type *KnownTy = GR->findDeducedElementType(Val: Op))
595 return KnownTy;
596
597 for (User *OpU : Op->users()) {
598 if (Instruction *Inst = dyn_cast<Instruction>(Val: OpU)) {
599 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8))
600 return Ty;
601 }
602 }
603 return nullptr;
604}
605
606// Implements what we know in advance about intrinsics and builtin calls
607// TODO: consider feasibility of this particular case to be generalized by
608// encoding knowledge about intrinsics and builtin calls by corresponding
609// specification rules
610static Type *getPointeeTypeByCallInst(StringRef DemangledName,
611 Function *CalledF, unsigned OpIdx) {
612 if ((DemangledName.starts_with(Prefix: "__spirv_ocl_printf(") ||
613 DemangledName.starts_with(Prefix: "printf(")) &&
614 OpIdx == 0)
615 return IntegerType::getInt8Ty(C&: CalledF->getContext());
616 return nullptr;
617}
618
619// Deduce and return a successfully deduced Type of the Instruction,
620// or nullptr otherwise.
621Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(Value *I,
622 bool UnknownElemTypeI8) {
623 std::unordered_set<Value *> Visited;
624 return deduceElementTypeHelper(I, Visited, UnknownElemTypeI8);
625}
626
627void SPIRVEmitIntrinsics::maybeAssignPtrType(Type *&Ty, Value *Op, Type *RefTy,
628 bool UnknownElemTypeI8) {
629 if (isUntypedPointerTy(T: RefTy)) {
630 if (!UnknownElemTypeI8)
631 return;
632 insertTodoType(Op);
633 }
634 Ty = RefTy;
635}
636
637bool SPIRVEmitIntrinsics::walkLogicalAccessChain(
638 GetElementPtrInst &GEP,
639 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
640 const std::function<void(Type *, Value *)> &OnDynamicIndexing) {
641 // We only rewrite i8* GEP. Other should be left as-is.
642 // Valid i8* GEP must always have a single index.
643 assert(GEP.getSourceElementType() ==
644 IntegerType::getInt8Ty(CurrF->getContext()));
645 assert(GEP.getNumIndices() == 1);
646
647 auto &DL = CurrF->getDataLayout();
648 Value *Src = getPointerRoot(I: GEP.getPointerOperand());
649 Type *CurType = deduceElementType(I: Src, UnknownElemTypeI8: true);
650
651 Value *Operand = *GEP.idx_begin();
652 ConstantInt *CI = dyn_cast<ConstantInt>(Val: Operand);
653 if (!CI) {
654 ArrayType *AT = dyn_cast<ArrayType>(Val: CurType);
655 // Operand is not constant. Either we have an array and accept it, or we
656 // give up.
657 if (AT)
658 OnDynamicIndexing(AT->getElementType(), Operand);
659 return AT == nullptr;
660 }
661
662 assert(CI);
663 uint64_t Offset = CI->getZExtValue();
664
665 do {
666 if (ArrayType *AT = dyn_cast<ArrayType>(Val: CurType)) {
667 uint32_t EltTypeSize = DL.getTypeSizeInBits(Ty: AT->getElementType()) / 8;
668 assert(Offset < AT->getNumElements() * EltTypeSize);
669 uint64_t Index = Offset / EltTypeSize;
670 Offset = Offset - (Index * EltTypeSize);
671 CurType = AT->getElementType();
672 OnLiteralIndexing(CurType, Index);
673 } else if (StructType *ST = dyn_cast<StructType>(Val: CurType)) {
674 uint32_t StructSize = DL.getTypeSizeInBits(Ty: ST) / 8;
675 assert(Offset < StructSize);
676 (void)StructSize;
677 const auto &STL = DL.getStructLayout(Ty: ST);
678 unsigned Element = STL->getElementContainingOffset(FixedOffset: Offset);
679 Offset -= STL->getElementOffset(Idx: Element);
680 CurType = ST->getElementType(N: Element);
681 OnLiteralIndexing(CurType, Element);
682 } else if (auto *VT = dyn_cast<FixedVectorType>(Val: CurType)) {
683 Type *EltTy = VT->getElementType();
684 TypeSize EltSizeBits = DL.getTypeSizeInBits(Ty: EltTy);
685 assert(EltSizeBits % 8 == 0 &&
686 "Element type size in bits must be a multiple of 8.");
687 uint32_t EltTypeSize = EltSizeBits / 8;
688 assert(Offset < VT->getNumElements() * EltTypeSize);
689 uint64_t Index = Offset / EltTypeSize;
690 Offset -= Index * EltTypeSize;
691 CurType = EltTy;
692 OnLiteralIndexing(CurType, Index);
693
694 } else {
695 // Unknown composite kind; give up.
696 return true;
697 }
698 } while (Offset > 0);
699
700 return false;
701}
702
703Instruction *
704SPIRVEmitIntrinsics::buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP) {
705 auto &DL = CurrF->getDataLayout();
706 IRBuilder<> B(GEP.getParent());
707 B.SetInsertPoint(&GEP);
708
709 std::vector<Value *> Indices;
710 Indices.push_back(x: ConstantInt::get(
711 Ty: IntegerType::getInt32Ty(C&: CurrF->getContext()), V: 0, /* Signed= */ IsSigned: false));
712 walkLogicalAccessChain(
713 GEP,
714 OnLiteralIndexing: [&Indices, &B](Type *EltType, uint64_t Index) {
715 Indices.push_back(
716 x: ConstantInt::get(Ty: B.getInt64Ty(), V: Index, /* Signed= */ IsSigned: false));
717 },
718 OnDynamicIndexing: [&Indices, &B, &DL](Type *EltType, Value *Offset) {
719 uint32_t EltTypeSize = DL.getTypeSizeInBits(Ty: EltType) / 8;
720 Value *Index = B.CreateUDiv(
721 LHS: Offset, RHS: ConstantInt::get(Ty: Offset->getType(), V: EltTypeSize,
722 /* Signed= */ IsSigned: false));
723 Indices.push_back(x: Index);
724 });
725
726 SmallVector<Type *, 2> Types = {GEP.getType(), GEP.getOperand(i_nocapture: 0)->getType()};
727 SmallVector<Value *, 4> Args;
728 Args.push_back(Elt: B.getInt1(V: GEP.isInBounds()));
729 Args.push_back(Elt: GEP.getOperand(i_nocapture: 0));
730 llvm::append_range(C&: Args, R&: Indices);
731 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types: {Types}, Args: {Args});
732 replaceAllUsesWithAndErase(B, Src: &GEP, Dest: NewI);
733 return NewI;
734}
735
736Type *SPIRVEmitIntrinsics::getGEPTypeLogical(GetElementPtrInst *GEP) {
737
738 Type *CurType = GEP->getResultElementType();
739
740 bool Interrupted = walkLogicalAccessChain(
741 GEP&: *GEP, OnLiteralIndexing: [&CurType](Type *EltType, uint64_t Index) { CurType = EltType; },
742 OnDynamicIndexing: [&CurType](Type *EltType, Value *Index) { CurType = EltType; });
743
744 return Interrupted ? GEP->getResultElementType() : CurType;
745}
746
747Type *SPIRVEmitIntrinsics::getGEPType(GetElementPtrInst *Ref) {
748 if (Ref->getSourceElementType() ==
749 IntegerType::getInt8Ty(C&: CurrF->getContext()) &&
750 TM->getSubtargetImpl()->isLogicalSPIRV()) {
751 return getGEPTypeLogical(GEP: Ref);
752 }
753
754 Type *Ty = nullptr;
755 // TODO: not sure if GetElementPtrInst::getTypeAtIndex() does anything
756 // useful here
757 if (isNestedPointer(Ty: Ref->getSourceElementType())) {
758 Ty = Ref->getSourceElementType();
759 for (Use &U : drop_begin(RangeOrContainer: Ref->indices()))
760 Ty = GetElementPtrInst::getTypeAtIndex(Ty, Idx: U.get());
761 } else {
762 Ty = Ref->getResultElementType();
763 }
764 return Ty;
765}
766
767Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
768 Value *I, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8,
769 bool IgnoreKnownType) {
770 // allow to pass nullptr as an argument
771 if (!I)
772 return nullptr;
773
774 // maybe already known
775 if (!IgnoreKnownType)
776 if (Type *KnownTy = GR->findDeducedElementType(Val: I))
777 return KnownTy;
778
779 // maybe a cycle
780 if (!Visited.insert(x: I).second)
781 return nullptr;
782
783 // fallback value in case when we fail to deduce a type
784 Type *Ty = nullptr;
785 // look for known basic patterns of type inference
786 if (auto *Ref = dyn_cast<AllocaInst>(Val: I)) {
787 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getAllocatedType(), UnknownElemTypeI8);
788 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
789 Ty = getGEPType(Ref);
790 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
791 Value *Op = Ref->getPointerOperand();
792 Type *KnownTy = GR->findDeducedElementType(Val: Op);
793 if (!KnownTy)
794 KnownTy = Op->getType();
795 if (Type *ElemTy = getPointeeType(Ty: KnownTy))
796 maybeAssignPtrType(Ty, Op: I, RefTy: ElemTy, UnknownElemTypeI8);
797 } else if (auto *Ref = dyn_cast<GlobalValue>(Val: I)) {
798 if (auto *Fn = dyn_cast<Function>(Val: Ref)) {
799 Ty = SPIRV::getOriginalFunctionType(F: *Fn);
800 GR->addDeducedElementType(Val: I, Ty);
801 } else {
802 Ty = deduceElementTypeByValueDeep(
803 ValueTy: Ref->getValueType(),
804 Operand: Ref->getNumOperands() > 0 ? Ref->getOperand(i: 0) : nullptr, Visited,
805 UnknownElemTypeI8);
806 }
807 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
808 Type *RefTy = deduceElementTypeHelper(I: Ref->getPointerOperand(), Visited,
809 UnknownElemTypeI8);
810 maybeAssignPtrType(Ty, Op: I, RefTy, UnknownElemTypeI8);
811 } else if (auto *Ref = dyn_cast<IntToPtrInst>(Val: I)) {
812 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getDestTy(), UnknownElemTypeI8);
813 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
814 if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
815 isPointerTy(T: Src) && isPointerTy(T: Dest))
816 Ty = deduceElementTypeHelper(I: Ref->getOperand(i_nocapture: 0), Visited,
817 UnknownElemTypeI8);
818 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
819 Value *Op = Ref->getNewValOperand();
820 if (isPointerTy(T: Op->getType()))
821 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
822 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
823 Value *Op = Ref->getValOperand();
824 if (isPointerTy(T: Op->getType()))
825 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
826 } else if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
827 Type *BestTy = nullptr;
828 unsigned MaxN = 1;
829 DenseMap<Type *, unsigned> PhiTys;
830 for (int i = Ref->getNumIncomingValues() - 1; i >= 0; --i) {
831 Ty = deduceElementTypeByUsersDeep(Op: Ref->getIncomingValue(i), Visited,
832 UnknownElemTypeI8);
833 if (!Ty)
834 continue;
835 auto It = PhiTys.try_emplace(Key: Ty, Args: 1);
836 if (!It.second) {
837 ++It.first->second;
838 if (It.first->second > MaxN) {
839 MaxN = It.first->second;
840 BestTy = Ty;
841 }
842 }
843 }
844 if (BestTy)
845 Ty = BestTy;
846 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
847 for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) {
848 Ty = deduceElementTypeByUsersDeep(Op, Visited, UnknownElemTypeI8);
849 if (Ty)
850 break;
851 }
852 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
853 static StringMap<unsigned> ResTypeByArg = {
854 {"to_global", 0},
855 {"to_local", 0},
856 {"to_private", 0},
857 {"__spirv_GenericCastToPtr_ToGlobal", 0},
858 {"__spirv_GenericCastToPtr_ToLocal", 0},
859 {"__spirv_GenericCastToPtr_ToPrivate", 0},
860 {"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
861 {"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
862 {"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
863 // TODO: maybe improve performance by caching demangled names
864
865 auto *II = dyn_cast<IntrinsicInst>(Val: I);
866 if (II && II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
867 auto *HandleType = cast<TargetExtType>(Val: II->getOperand(i_nocapture: 0)->getType());
868 if (HandleType->getTargetExtName() == "spirv.Image" ||
869 HandleType->getTargetExtName() == "spirv.SignedImage") {
870 for (User *U : II->users()) {
871 Ty = cast<Instruction>(Val: U)->getAccessType();
872 if (Ty)
873 break;
874 }
875 } else if (HandleType->getTargetExtName() == "spirv.VulkanBuffer") {
876 // This call is supposed to index into an array
877 Ty = HandleType->getTypeParameter(i: 0);
878 if (Ty->isArrayTy())
879 Ty = Ty->getArrayElementType();
880 else {
881 assert(Ty && Ty->isStructTy());
882 uint32_t Index = cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1))->getZExtValue();
883 Ty = cast<StructType>(Val: Ty)->getElementType(N: Index);
884 }
885 Ty = reconstitutePeeledArrayType(Ty);
886 } else {
887 llvm_unreachable("Unknown handle type for spv_resource_getpointer.");
888 }
889 } else if (II && II->getIntrinsicID() ==
890 Intrinsic::spv_generic_cast_to_ptr_explicit) {
891 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: 0), Visited,
892 UnknownElemTypeI8);
893 } else if (Function *CalledF = CI->getCalledFunction()) {
894 std::string DemangledName =
895 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
896 if (DemangledName.length() > 0)
897 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName);
898 auto AsArgIt = ResTypeByArg.find(Key: DemangledName);
899 if (AsArgIt != ResTypeByArg.end())
900 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: AsArgIt->second),
901 Visited, UnknownElemTypeI8);
902 else if (Type *KnownRetTy = GR->findDeducedElementType(Val: CalledF))
903 Ty = KnownRetTy;
904 }
905 }
906
907 // remember the found relationship
908 if (Ty && !IgnoreKnownType) {
909 // specify nested types if needed, otherwise return unchanged
910 GR->addDeducedElementType(Val: I, Ty: normalizeType(Ty));
911 }
912
913 return Ty;
914}
915
916// Re-create a type of the value if it has untyped pointer fields, also nested.
917// Return the original value type if no corrections of untyped pointer
918// information is found or needed.
919Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(User *U,
920 bool UnknownElemTypeI8) {
921 std::unordered_set<Value *> Visited;
922 return deduceNestedTypeHelper(U, Ty: U->getType(), Visited, UnknownElemTypeI8);
923}
924
925Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
926 User *U, Type *OrigTy, std::unordered_set<Value *> &Visited,
927 bool UnknownElemTypeI8) {
928 if (!U)
929 return OrigTy;
930
931 // maybe already known
932 if (Type *KnownTy = GR->findDeducedCompositeType(Val: U))
933 return KnownTy;
934
935 // maybe a cycle
936 if (!Visited.insert(x: U).second)
937 return OrigTy;
938
939 if (isa<StructType>(Val: OrigTy)) {
940 SmallVector<Type *> Tys;
941 bool Change = false;
942 for (unsigned i = 0; i < U->getNumOperands(); ++i) {
943 Value *Op = U->getOperand(i);
944 assert(Op && "Operands should not be null.");
945 Type *OpTy = Op->getType();
946 Type *Ty = OpTy;
947 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
948 if (Type *NestedTy =
949 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
950 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
951 } else {
952 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
953 UnknownElemTypeI8);
954 }
955 Tys.push_back(Elt: Ty);
956 Change |= Ty != OpTy;
957 }
958 if (Change) {
959 Type *NewTy = StructType::create(Elements: Tys);
960 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
961 return NewTy;
962 }
963 } else if (auto *ArrTy = dyn_cast<ArrayType>(Val: OrigTy)) {
964 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
965 Type *OpTy = ArrTy->getElementType();
966 Type *Ty = OpTy;
967 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
968 if (Type *NestedTy =
969 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
970 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
971 } else {
972 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
973 UnknownElemTypeI8);
974 }
975 if (Ty != OpTy) {
976 Type *NewTy = ArrayType::get(ElementType: Ty, NumElements: ArrTy->getNumElements());
977 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
978 return NewTy;
979 }
980 }
981 } else if (auto *VecTy = dyn_cast<VectorType>(Val: OrigTy)) {
982 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
983 Type *OpTy = VecTy->getElementType();
984 Type *Ty = OpTy;
985 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
986 if (Type *NestedTy =
987 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
988 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
989 } else {
990 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
991 UnknownElemTypeI8);
992 }
993 if (Ty != OpTy) {
994 Type *NewTy = VectorType::get(ElementType: Ty, EC: VecTy->getElementCount());
995 GR->addDeducedCompositeType(Val: U, Ty: normalizeType(Ty: NewTy));
996 return NewTy;
997 }
998 }
999 }
1000
1001 return OrigTy;
1002}
1003
1004Type *SPIRVEmitIntrinsics::deduceElementType(Value *I, bool UnknownElemTypeI8) {
1005 if (Type *Ty = deduceElementTypeHelper(I, UnknownElemTypeI8))
1006 return Ty;
1007 if (!UnknownElemTypeI8)
1008 return nullptr;
1009 insertTodoType(Op: I);
1010 return IntegerType::getInt8Ty(C&: I->getContext());
1011}
1012
1013static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I,
1014 Value *PointerOperand) {
1015 Type *PointeeTy = GR->findDeducedElementType(Val: PointerOperand);
1016 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1017 return nullptr;
1018 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType());
1019 if (!PtrTy)
1020 return I->getType();
1021 if (Type *NestedTy = GR->findDeducedElementType(Val: I))
1022 return getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1023 return nullptr;
1024}
1025
1026// Try to deduce element type for a call base. Returns false if this is an
1027// indirect function invocation, and true otherwise.
1028bool SPIRVEmitIntrinsics::deduceOperandElementTypeCalledFunction(
1029 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1030 Type *&KnownElemTy, bool &Incomplete) {
1031 Function *CalledF = CI->getCalledFunction();
1032 if (!CalledF)
1033 return false;
1034 std::string DemangledName =
1035 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1036 if (DemangledName.length() > 0 &&
1037 !StringRef(DemangledName).starts_with(Prefix: "llvm.")) {
1038 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F: *CalledF);
1039 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
1040 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
1041 if (Opcode == SPIRV::OpGroupAsyncCopy) {
1042 for (unsigned i = 0, PtrCnt = 0; i < CI->arg_size() && PtrCnt < 2; ++i) {
1043 Value *Op = CI->getArgOperand(i);
1044 if (!isPointerTy(T: Op->getType()))
1045 continue;
1046 ++PtrCnt;
1047 if (Type *ElemTy = GR->findDeducedElementType(Val: Op))
1048 KnownElemTy = ElemTy; // src will rewrite dest if both are defined
1049 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1050 }
1051 } else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) {
1052 if (CI->arg_size() == 0)
1053 return true;
1054 Value *Op = CI->getArgOperand(i: 0);
1055 if (!isPointerTy(T: Op->getType()))
1056 return true;
1057 switch (Opcode) {
1058 case SPIRV::OpAtomicFAddEXT:
1059 case SPIRV::OpAtomicFMinEXT:
1060 case SPIRV::OpAtomicFMaxEXT:
1061 case SPIRV::OpAtomicLoad:
1062 case SPIRV::OpAtomicCompareExchangeWeak:
1063 case SPIRV::OpAtomicCompareExchange:
1064 case SPIRV::OpAtomicExchange:
1065 case SPIRV::OpAtomicIAdd:
1066 case SPIRV::OpAtomicISub:
1067 case SPIRV::OpAtomicOr:
1068 case SPIRV::OpAtomicXor:
1069 case SPIRV::OpAtomicAnd:
1070 case SPIRV::OpAtomicUMin:
1071 case SPIRV::OpAtomicUMax:
1072 case SPIRV::OpAtomicSMin:
1073 case SPIRV::OpAtomicSMax: {
1074 KnownElemTy = isPointerTy(T: CI->getType()) ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1075 : CI->getType();
1076 if (!KnownElemTy)
1077 return true;
1078 Incomplete = isTodoType(Op);
1079 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1080 } break;
1081 case SPIRV::OpAtomicStore: {
1082 if (CI->arg_size() < 4)
1083 return true;
1084 Value *ValOp = CI->getArgOperand(i: 3);
1085 KnownElemTy = isPointerTy(T: ValOp->getType())
1086 ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1087 : ValOp->getType();
1088 if (!KnownElemTy)
1089 return true;
1090 Incomplete = isTodoType(Op);
1091 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1092 } break;
1093 }
1094 }
1095 }
1096 return true;
1097}
1098
1099// Try to deduce element type for a function pointer.
1100void SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionPointer(
1101 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1102 Type *&KnownElemTy, bool IsPostprocessing) {
1103 Value *Op = CI->getCalledOperand();
1104 if (!Op || !isPointerTy(T: Op->getType()))
1105 return;
1106 Ops.push_back(Elt: std::make_pair(x&: Op, y: std::numeric_limits<unsigned>::max()));
1107 FunctionType *FTy = SPIRV::getOriginalFunctionType(CB: *CI);
1108 bool IsNewFTy = false, IsIncomplete = false;
1109 SmallVector<Type *, 4> ArgTys;
1110 for (auto &&[ParmIdx, Arg] : llvm::enumerate(First: CI->args())) {
1111 Type *ArgTy = Arg->getType();
1112 if (ArgTy->isPointerTy()) {
1113 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg)) {
1114 IsNewFTy = true;
1115 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
1116 if (isTodoType(Op: Arg))
1117 IsIncomplete = true;
1118 } else {
1119 IsIncomplete = true;
1120 }
1121 } else {
1122 ArgTy = FTy->getFunctionParamType(i: ParmIdx);
1123 }
1124 ArgTys.push_back(Elt: ArgTy);
1125 }
1126 Type *RetTy = FTy->getReturnType();
1127 if (CI->getType()->isPointerTy()) {
1128 if (Type *ElemTy = GR->findDeducedElementType(Val: CI)) {
1129 IsNewFTy = true;
1130 RetTy =
1131 getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: CI->getType()));
1132 if (isTodoType(Op: CI))
1133 IsIncomplete = true;
1134 } else {
1135 IsIncomplete = true;
1136 }
1137 }
1138 if (!IsPostprocessing && IsIncomplete)
1139 insertTodoType(Op);
1140 KnownElemTy =
1141 IsNewFTy ? FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: FTy->isVarArg()) : FTy;
1142}
1143
1144bool SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionRet(
1145 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
1146 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing,
1147 Type *&KnownElemTy, Value *Op, Function *F) {
1148 KnownElemTy = GR->findDeducedElementType(Val: F);
1149 if (KnownElemTy)
1150 return false;
1151 if (Type *OpElemTy = GR->findDeducedElementType(Val: Op)) {
1152 OpElemTy = normalizeType(Ty: OpElemTy);
1153 GR->addDeducedElementType(Val: F, Ty: OpElemTy);
1154 GR->addReturnType(
1155 ArgF: F, DerivedTy: TypedPointerType::get(ElementType: OpElemTy,
1156 AddressSpace: getPointerAddressSpace(T: F->getReturnType())));
1157 // non-recursive update of types in function uses
1158 DenseSet<std::pair<Value *, Value *>> VisitedSubst{std::make_pair(x&: I, y&: Op)};
1159 for (User *U : F->users()) {
1160 CallInst *CI = dyn_cast<CallInst>(Val: U);
1161 if (!CI || CI->getCalledFunction() != F)
1162 continue;
1163 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: CI)) {
1164 if (Type *PrevElemTy = GR->findDeducedElementType(Val: CI)) {
1165 GR->updateAssignType(AssignCI, Arg: CI,
1166 OfType: getNormalizedPoisonValue(Ty: OpElemTy));
1167 propagateElemType(Op: CI, ElemTy: PrevElemTy, VisitedSubst);
1168 }
1169 }
1170 }
1171 // Non-recursive update of types in the function uncomplete returns.
1172 // This may happen just once per a function, the latch is a pair of
1173 // findDeducedElementType(F) / addDeducedElementType(F, ...).
1174 // With or without the latch it is a non-recursive call due to
1175 // IncompleteRets set to nullptr in this call.
1176 if (IncompleteRets)
1177 for (Instruction *IncompleteRetI : *IncompleteRets)
1178 deduceOperandElementType(I: IncompleteRetI, IncompleteRets: nullptr, AskOps,
1179 IsPostprocessing);
1180 } else if (IncompleteRets) {
1181 IncompleteRets->insert(Ptr: I);
1182 }
1183 TypeValidated.insert(x: I);
1184 return true;
1185}
1186
1187// If the Instruction has Pointer operands with unresolved types, this function
1188// tries to deduce them. If the Instruction has Pointer operands with known
1189// types which differ from expected, this function tries to insert a bitcast to
1190// resolve the issue.
1191void SPIRVEmitIntrinsics::deduceOperandElementType(
1192 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
1193 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing) {
1194 SmallVector<std::pair<Value *, unsigned>> Ops;
1195 Type *KnownElemTy = nullptr;
1196 bool Incomplete = false;
1197 // look for known basic patterns of type inference
1198 if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1199 if (!isPointerTy(T: I->getType()) ||
1200 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1201 return;
1202 Incomplete = isTodoType(Op: I);
1203 for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
1204 Value *Op = Ref->getIncomingValue(i);
1205 if (isPointerTy(T: Op->getType()))
1206 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1207 }
1208 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1209 KnownElemTy = GR->findDeducedElementType(Val: I);
1210 if (!KnownElemTy)
1211 return;
1212 Incomplete = isTodoType(Op: I);
1213 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(), y: 0));
1214 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1215 if (!isPointerTy(T: I->getType()))
1216 return;
1217 KnownElemTy = GR->findDeducedElementType(Val: I);
1218 if (!KnownElemTy)
1219 return;
1220 Incomplete = isTodoType(Op: I);
1221 Ops.push_back(Elt: std::make_pair(x: Ref->getOperand(i_nocapture: 0), y: 0));
1222 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
1223 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1224 return;
1225 KnownElemTy = Ref->getSourceElementType();
1226 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1227 y: GetElementPtrInst::getPointerOperandIndex()));
1228 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
1229 KnownElemTy = I->getType();
1230 if (isUntypedPointerTy(T: KnownElemTy))
1231 return;
1232 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1233 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1234 return;
1235 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1236 y: LoadInst::getPointerOperandIndex()));
1237 } else if (auto *Ref = dyn_cast<StoreInst>(Val: I)) {
1238 if (!(KnownElemTy =
1239 reconstructType(Op: Ref->getValueOperand(), UnknownElemTypeI8: false, IsPostprocessing)))
1240 return;
1241 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1242 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1243 return;
1244 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1245 y: StoreInst::getPointerOperandIndex()));
1246 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1247 KnownElemTy = isPointerTy(T: I->getType())
1248 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1249 : I->getType();
1250 if (!KnownElemTy)
1251 return;
1252 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1253 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1254 y: AtomicCmpXchgInst::getPointerOperandIndex()));
1255 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1256 KnownElemTy = isPointerTy(T: I->getType())
1257 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1258 : I->getType();
1259 if (!KnownElemTy)
1260 return;
1261 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1262 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1263 y: AtomicRMWInst::getPointerOperandIndex()));
1264 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1265 if (!isPointerTy(T: I->getType()) ||
1266 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1267 return;
1268 Incomplete = isTodoType(Op: I);
1269 for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
1270 Value *Op = Ref->getOperand(i_nocapture: i);
1271 if (isPointerTy(T: Op->getType()))
1272 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1273 }
1274 } else if (auto *Ref = dyn_cast<ReturnInst>(Val: I)) {
1275 if (!isPointerTy(T: CurrF->getReturnType()))
1276 return;
1277 Value *Op = Ref->getReturnValue();
1278 if (!Op)
1279 return;
1280 if (deduceOperandElementTypeFunctionRet(I, IncompleteRets, AskOps,
1281 IsPostprocessing, KnownElemTy, Op,
1282 F: CurrF))
1283 return;
1284 Incomplete = isTodoType(Op: CurrF);
1285 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1286 } else if (auto *Ref = dyn_cast<ICmpInst>(Val: I)) {
1287 if (!isPointerTy(T: Ref->getOperand(i_nocapture: 0)->getType()))
1288 return;
1289 Value *Op0 = Ref->getOperand(i_nocapture: 0);
1290 Value *Op1 = Ref->getOperand(i_nocapture: 1);
1291 bool Incomplete0 = isTodoType(Op: Op0);
1292 bool Incomplete1 = isTodoType(Op: Op1);
1293 Type *ElemTy1 = GR->findDeducedElementType(Val: Op1);
1294 Type *ElemTy0 = (Incomplete0 && !Incomplete1 && ElemTy1)
1295 ? nullptr
1296 : GR->findDeducedElementType(Val: Op0);
1297 if (ElemTy0) {
1298 KnownElemTy = ElemTy0;
1299 Incomplete = Incomplete0;
1300 Ops.push_back(Elt: std::make_pair(x&: Op1, y: 1));
1301 } else if (ElemTy1) {
1302 KnownElemTy = ElemTy1;
1303 Incomplete = Incomplete1;
1304 Ops.push_back(Elt: std::make_pair(x&: Op0, y: 0));
1305 }
1306 } else if (CallInst *CI = dyn_cast<CallInst>(Val: I)) {
1307 if (!CI->isIndirectCall())
1308 deduceOperandElementTypeCalledFunction(CI, Ops, KnownElemTy, Incomplete);
1309 else if (HaveFunPtrs)
1310 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy,
1311 IsPostprocessing);
1312 }
1313
1314 // There is no enough info to deduce types or all is valid.
1315 if (!KnownElemTy || Ops.size() == 0)
1316 return;
1317
1318 LLVMContext &Ctx = CurrF->getContext();
1319 IRBuilder<> B(Ctx);
1320 for (auto &OpIt : Ops) {
1321 Value *Op = OpIt.first;
1322 if (AskOps && !AskOps->contains(Ptr: Op))
1323 continue;
1324 Type *AskTy = nullptr;
1325 CallInst *AskCI = nullptr;
1326 if (IsPostprocessing && AskOps) {
1327 AskTy = GR->findDeducedElementType(Val: Op);
1328 AskCI = GR->findAssignPtrTypeInstr(Val: Op);
1329 assert(AskTy && AskCI);
1330 }
1331 Type *Ty = AskTy ? AskTy : GR->findDeducedElementType(Val: Op);
1332 if (Ty == KnownElemTy)
1333 continue;
1334 Value *OpTyVal = getNormalizedPoisonValue(Ty: KnownElemTy);
1335 Type *OpTy = Op->getType();
1336 if (Op->hasUseList() &&
1337 (!Ty || AskTy || isUntypedPointerTy(T: Ty) || isTodoType(Op))) {
1338 Type *PrevElemTy = GR->findDeducedElementType(Val: Op);
1339 GR->addDeducedElementType(Val: Op, Ty: normalizeType(Ty: KnownElemTy));
1340 // check if KnownElemTy is complete
1341 if (!Incomplete)
1342 eraseTodoType(Op);
1343 else if (!IsPostprocessing)
1344 insertTodoType(Op);
1345 // check if there is existing Intrinsic::spv_assign_ptr_type instruction
1346 CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Val: Op);
1347 if (AssignCI == nullptr) {
1348 Instruction *User = dyn_cast<Instruction>(Val: Op->use_begin()->get());
1349 setInsertPointSkippingPhis(B, I: User ? User->getNextNode() : I);
1350 CallInst *CI =
1351 buildIntrWithMD(IntrID: Intrinsic::spv_assign_ptr_type, Types: {OpTy}, Arg: OpTyVal, Arg2: Op,
1352 Imms: {B.getInt32(C: getPointerAddressSpace(T: OpTy))}, B);
1353 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: CI);
1354 } else {
1355 GR->updateAssignType(AssignCI, Arg: Op, OfType: OpTyVal);
1356 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1357 std::make_pair(x&: I, y&: Op)};
1358 propagateElemTypeRec(Op, PtrElemTy: KnownElemTy, CastElemTy: PrevElemTy, VisitedSubst);
1359 }
1360 } else {
1361 eraseTodoType(Op);
1362 CallInst *PtrCastI =
1363 buildSpvPtrcast(F: I->getParent()->getParent(), Op, ElemTy: KnownElemTy);
1364 if (OpIt.second == std::numeric_limits<unsigned>::max())
1365 dyn_cast<CallInst>(Val: I)->setCalledOperand(PtrCastI);
1366 else
1367 I->setOperand(i: OpIt.second, Val: PtrCastI);
1368 }
1369 }
1370 TypeValidated.insert(x: I);
1371}
1372
1373void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
1374 Instruction *New,
1375 IRBuilder<> &B) {
1376 while (!Old->user_empty()) {
1377 auto *U = Old->user_back();
1378 if (isAssignTypeInstr(I: U)) {
1379 B.SetInsertPoint(U);
1380 SmallVector<Value *, 2> Args = {New, U->getOperand(i: 1)};
1381 CallInst *AssignCI =
1382 B.CreateIntrinsic(ID: Intrinsic::spv_assign_type, Types: {New->getType()}, Args);
1383 GR->addAssignPtrTypeInstr(Val: New, AssignPtrTyCI: AssignCI);
1384 U->eraseFromParent();
1385 } else if (isMemInstrToReplace(I: U) || isa<ReturnInst>(Val: U) ||
1386 isa<CallInst>(Val: U)) {
1387 U->replaceUsesOfWith(From: Old, To: New);
1388 } else {
1389 llvm_unreachable("illegal aggregate intrinsic user");
1390 }
1391 }
1392 New->copyMetadata(SrcInst: *Old);
1393 Old->eraseFromParent();
1394}
1395
1396void SPIRVEmitIntrinsics::preprocessUndefs(IRBuilder<> &B) {
1397 std::queue<Instruction *> Worklist;
1398 for (auto &I : instructions(F: CurrF))
1399 Worklist.push(x: &I);
1400
1401 while (!Worklist.empty()) {
1402 Instruction *I = Worklist.front();
1403 bool BPrepared = false;
1404 Worklist.pop();
1405
1406 for (auto &Op : I->operands()) {
1407 auto *AggrUndef = dyn_cast<UndefValue>(Val&: Op);
1408 if (!AggrUndef || !Op->getType()->isAggregateType())
1409 continue;
1410
1411 if (!BPrepared) {
1412 setInsertPointSkippingPhis(B, I);
1413 BPrepared = true;
1414 }
1415 auto *IntrUndef = B.CreateIntrinsic(ID: Intrinsic::spv_undef, Args: {});
1416 Worklist.push(x: IntrUndef);
1417 I->replaceUsesOfWith(From: Op, To: IntrUndef);
1418 AggrConsts[IntrUndef] = AggrUndef;
1419 AggrConstTypes[IntrUndef] = AggrUndef->getType();
1420 }
1421 }
1422}
1423
1424void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) {
1425 std::queue<Instruction *> Worklist;
1426 for (auto &I : instructions(F: CurrF))
1427 Worklist.push(x: &I);
1428
1429 while (!Worklist.empty()) {
1430 auto *I = Worklist.front();
1431 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
1432 assert(I);
1433 bool KeepInst = false;
1434 for (const auto &Op : I->operands()) {
1435 Constant *AggrConst = nullptr;
1436 Type *ResTy = nullptr;
1437 if (auto *COp = dyn_cast<ConstantVector>(Val: Op)) {
1438 AggrConst = COp;
1439 ResTy = COp->getType();
1440 } else if (auto *COp = dyn_cast<ConstantArray>(Val: Op)) {
1441 AggrConst = COp;
1442 ResTy = B.getInt32Ty();
1443 } else if (auto *COp = dyn_cast<ConstantStruct>(Val: Op)) {
1444 AggrConst = COp;
1445 ResTy = B.getInt32Ty();
1446 } else if (auto *COp = dyn_cast<ConstantDataArray>(Val: Op)) {
1447 AggrConst = COp;
1448 ResTy = B.getInt32Ty();
1449 } else if (auto *COp = dyn_cast<ConstantAggregateZero>(Val: Op)) {
1450 AggrConst = COp;
1451 ResTy = Op->getType()->isVectorTy() ? COp->getType() : B.getInt32Ty();
1452 }
1453 if (AggrConst) {
1454 SmallVector<Value *> Args;
1455 if (auto *COp = dyn_cast<ConstantDataSequential>(Val: Op))
1456 for (unsigned i = 0; i < COp->getNumElements(); ++i)
1457 Args.push_back(Elt: COp->getElementAsConstant(i));
1458 else
1459 llvm::append_range(C&: Args, R: AggrConst->operands());
1460 if (!BPrepared) {
1461 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
1462 : B.SetInsertPoint(I);
1463 BPrepared = true;
1464 }
1465 auto *CI =
1466 B.CreateIntrinsic(ID: Intrinsic::spv_const_composite, Types: {ResTy}, Args: {Args});
1467 Worklist.push(x: CI);
1468 I->replaceUsesOfWith(From: Op, To: CI);
1469 KeepInst = true;
1470 AggrConsts[CI] = AggrConst;
1471 AggrConstTypes[CI] = deduceNestedTypeHelper(U: AggrConst, UnknownElemTypeI8: false);
1472 }
1473 }
1474 if (!KeepInst)
1475 Worklist.pop();
1476 }
1477}
1478
1479static void createDecorationIntrinsic(Instruction *I, MDNode *Node,
1480 IRBuilder<> &B) {
1481 LLVMContext &Ctx = I->getContext();
1482 setInsertPointAfterDef(B, I);
1483 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, Types: {I->getType()},
1484 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: {Node}))});
1485}
1486
1487static void createRoundingModeDecoration(Instruction *I,
1488 unsigned RoundingModeDeco,
1489 IRBuilder<> &B) {
1490 LLVMContext &Ctx = I->getContext();
1491 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1492 MDNode *RoundingModeNode = MDNode::get(
1493 Context&: Ctx,
1494 MDs: {ConstantAsMetadata::get(
1495 C: ConstantInt::get(Ty: Int32Ty, V: SPIRV::Decoration::FPRoundingMode)),
1496 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: RoundingModeDeco))});
1497 createDecorationIntrinsic(I, Node: RoundingModeNode, B);
1498}
1499
1500static void createSaturatedConversionDecoration(Instruction *I,
1501 IRBuilder<> &B) {
1502 LLVMContext &Ctx = I->getContext();
1503 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1504 MDNode *SaturatedConversionNode =
1505 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1506 Ty: Int32Ty, V: SPIRV::Decoration::SaturatedConversion))});
1507 createDecorationIntrinsic(I, Node: SaturatedConversionNode, B);
1508}
1509
1510static void addSaturatedDecorationToIntrinsic(Instruction *I, IRBuilder<> &B) {
1511 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
1512 if (Function *Fu = CI->getCalledFunction()) {
1513 if (Fu->isIntrinsic()) {
1514 unsigned const int IntrinsicId = Fu->getIntrinsicID();
1515 switch (IntrinsicId) {
1516 case Intrinsic::fptosi_sat:
1517 case Intrinsic::fptoui_sat:
1518 createSaturatedConversionDecoration(I, B);
1519 break;
1520 default:
1521 break;
1522 }
1523 }
1524 }
1525 }
1526}
1527
1528Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
1529 if (!Call.isInlineAsm())
1530 return &Call;
1531
1532 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
1533 LLVMContext &Ctx = CurrF->getContext();
1534
1535 Constant *TyC = UndefValue::get(T: IA->getFunctionType());
1536 MDString *ConstraintString = MDString::get(Context&: Ctx, Str: IA->getConstraintString());
1537 SmallVector<Value *> Args = {
1538 buildMD(Arg: TyC),
1539 MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: ConstraintString))};
1540 for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++)
1541 Args.push_back(Elt: Call.getArgOperand(i: OpIdx));
1542
1543 IRBuilder<> B(Call.getParent());
1544 B.SetInsertPoint(&Call);
1545 B.CreateIntrinsic(ID: Intrinsic::spv_inline_asm, Args: {Args});
1546 return &Call;
1547}
1548
1549// Use a tip about rounding mode to create a decoration.
1550void SPIRVEmitIntrinsics::useRoundingMode(ConstrainedFPIntrinsic *FPI,
1551 IRBuilder<> &B) {
1552 std::optional<RoundingMode> RM = FPI->getRoundingMode();
1553 if (!RM.has_value())
1554 return;
1555 unsigned RoundingModeDeco = std::numeric_limits<unsigned>::max();
1556 switch (RM.value()) {
1557 default:
1558 // ignore unknown rounding modes
1559 break;
1560 case RoundingMode::NearestTiesToEven:
1561 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTE;
1562 break;
1563 case RoundingMode::TowardNegative:
1564 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTN;
1565 break;
1566 case RoundingMode::TowardPositive:
1567 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTP;
1568 break;
1569 case RoundingMode::TowardZero:
1570 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTZ;
1571 break;
1572 case RoundingMode::Dynamic:
1573 case RoundingMode::NearestTiesToAway:
1574 // TODO: check if supported
1575 break;
1576 }
1577 if (RoundingModeDeco == std::numeric_limits<unsigned>::max())
1578 return;
1579 // Convert the tip about rounding mode into a decoration record.
1580 createRoundingModeDecoration(I: FPI, RoundingModeDeco, B);
1581}
1582
1583Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
1584 BasicBlock *ParentBB = I.getParent();
1585 Function *F = ParentBB->getParent();
1586 IRBuilder<> B(ParentBB);
1587 B.SetInsertPoint(&I);
1588 SmallVector<Value *, 4> Args;
1589 SmallVector<BasicBlock *> BBCases;
1590 Args.push_back(Elt: I.getCondition());
1591 BBCases.push_back(Elt: I.getDefaultDest());
1592 Args.push_back(Elt: BlockAddress::get(F, BB: I.getDefaultDest()));
1593 for (auto &Case : I.cases()) {
1594 Args.push_back(Elt: Case.getCaseValue());
1595 BBCases.push_back(Elt: Case.getCaseSuccessor());
1596 Args.push_back(Elt: BlockAddress::get(F, BB: Case.getCaseSuccessor()));
1597 }
1598 CallInst *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_switch,
1599 Types: {I.getOperand(i_nocapture: 0)->getType()}, Args: {Args});
1600 // remove switch to avoid its unneeded and undesirable unwrap into branches
1601 // and conditions
1602 replaceAllUsesWith(Src: &I, Dest: NewI);
1603 I.eraseFromParent();
1604 // insert artificial and temporary instruction to preserve valid CFG,
1605 // it will be removed after IR translation pass
1606 B.SetInsertPoint(ParentBB);
1607 IndirectBrInst *BrI = B.CreateIndirectBr(
1608 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: ParentBB->getContext())),
1609 NumDests: BBCases.size());
1610 for (BasicBlock *BBCase : BBCases)
1611 BrI->addDestination(Dest: BBCase);
1612 return BrI;
1613}
1614
1615static bool isFirstIndexZero(const GetElementPtrInst *GEP) {
1616 if (GEP->getNumIndices() == 0)
1617 return false;
1618 if (const auto *CI = dyn_cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: 1))) {
1619 return CI->getZExtValue() == 0;
1620 }
1621 return false;
1622}
1623
1624Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
1625 IRBuilder<> B(I.getParent());
1626 B.SetInsertPoint(&I);
1627
1628 if (TM->getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: &I)) {
1629 // Logical SPIR-V cannot use the OpPtrAccessChain instruction. If the first
1630 // index of the GEP is not 0, then we need to try to adjust it.
1631 //
1632 // If the GEP is doing byte addressing, try to rebuild the full access chain
1633 // from the type of the pointer.
1634 if (I.getSourceElementType() ==
1635 IntegerType::getInt8Ty(C&: CurrF->getContext())) {
1636 return buildLogicalAccessChainFromGEP(GEP&: I);
1637 }
1638
1639 // Look for the array-to-pointer decay. If this is the pattern
1640 // we can adjust the types, and prepend a 0 to the indices.
1641 Value *PtrOp = I.getPointerOperand();
1642 Type *SrcElemTy = I.getSourceElementType();
1643 Type *DeducedPointeeTy = deduceElementType(I: PtrOp, UnknownElemTypeI8: true);
1644
1645 if (auto *ArrTy = dyn_cast<ArrayType>(Val: DeducedPointeeTy)) {
1646 if (ArrTy->getElementType() == SrcElemTy) {
1647 SmallVector<Value *> NewIndices;
1648 Type *FirstIdxType = I.getOperand(i_nocapture: 1)->getType();
1649 NewIndices.push_back(Elt: ConstantInt::get(Ty: FirstIdxType, V: 0));
1650 for (Value *Idx : I.indices())
1651 NewIndices.push_back(Elt: Idx);
1652
1653 SmallVector<Type *, 2> Types = {I.getType(), I.getPointerOperandType()};
1654 SmallVector<Value *, 4> Args;
1655 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
1656 Args.push_back(Elt: I.getPointerOperand());
1657 Args.append(in_start: NewIndices.begin(), in_end: NewIndices.end());
1658
1659 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types: {Types}, Args: {Args});
1660 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1661 return NewI;
1662 }
1663 }
1664 }
1665
1666 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
1667 SmallVector<Value *, 4> Args;
1668 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
1669 llvm::append_range(C&: Args, R: I.operands());
1670 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types: {Types}, Args: {Args});
1671 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1672 return NewI;
1673}
1674
1675Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
1676 IRBuilder<> B(I.getParent());
1677 B.SetInsertPoint(&I);
1678 Value *Source = I.getOperand(i_nocapture: 0);
1679
1680 // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of
1681 // varying element types. In case of IR coming from older versions of LLVM
1682 // such bitcasts do not provide sufficient information, should be just skipped
1683 // here, and handled in insertPtrCastOrAssignTypeInstr.
1684 if (isPointerTy(T: I.getType())) {
1685 replaceAllUsesWith(Src: &I, Dest: Source);
1686 I.eraseFromParent();
1687 return nullptr;
1688 }
1689
1690 SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
1691 SmallVector<Value *> Args(I.op_begin(), I.op_end());
1692 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_bitcast, Types: {Types}, Args: {Args});
1693 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1694 return NewI;
1695}
1696
1697void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt(
1698 TargetExtType *AssignedType, Value *V, IRBuilder<> &B) {
1699 Type *VTy = V->getType();
1700
1701 // A couple of sanity checks.
1702 assert((isPointerTy(VTy)) && "Expect a pointer type!");
1703 if (Type *ElemTy = getPointeeType(Ty: VTy))
1704 if (ElemTy != AssignedType)
1705 report_fatal_error(reason: "Unexpected pointer element type!");
1706
1707 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: V);
1708 if (!AssignCI) {
1709 GR->buildAssignType(B, Ty: AssignedType, Arg: V);
1710 return;
1711 }
1712
1713 Type *CurrentType =
1714 dyn_cast<ConstantAsMetadata>(
1715 Val: cast<MetadataAsValue>(Val: AssignCI->getOperand(i_nocapture: 1))->getMetadata())
1716 ->getType();
1717 if (CurrentType == AssignedType)
1718 return;
1719
1720 // Builtin types cannot be redeclared or casted.
1721 if (CurrentType->isTargetExtTy())
1722 report_fatal_error(reason: "Type mismatch " + CurrentType->getTargetExtName() +
1723 "/" + AssignedType->getTargetExtName() +
1724 " for value " + V->getName(),
1725 gen_crash_diag: false);
1726
1727 // Our previous guess about the type seems to be wrong, let's update
1728 // inferred type according to a new, more precise type information.
1729 GR->updateAssignType(AssignCI, Arg: V, OfType: getNormalizedPoisonValue(Ty: AssignedType));
1730}
1731
1732void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
1733 Instruction *I, Value *Pointer, Type *ExpectedElementType,
1734 unsigned OperandToReplace, IRBuilder<> &B) {
1735 TypeValidated.insert(x: I);
1736
1737 // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType
1738 Type *PointerElemTy = deduceElementTypeHelper(I: Pointer, UnknownElemTypeI8: false);
1739 if (PointerElemTy == ExpectedElementType ||
1740 isEquivalentTypes(Ty1: PointerElemTy, Ty2: ExpectedElementType))
1741 return;
1742
1743 setInsertPointSkippingPhis(B, I);
1744 Value *ExpectedElementVal = getNormalizedPoisonValue(Ty: ExpectedElementType);
1745 MetadataAsValue *VMD = buildMD(Arg: ExpectedElementVal);
1746 unsigned AddressSpace = getPointerAddressSpace(T: Pointer->getType());
1747 bool FirstPtrCastOrAssignPtrType = true;
1748
1749 // Do not emit new spv_ptrcast if equivalent one already exists or when
1750 // spv_assign_ptr_type already targets this pointer with the same element
1751 // type.
1752 if (Pointer->hasUseList()) {
1753 for (auto User : Pointer->users()) {
1754 auto *II = dyn_cast<IntrinsicInst>(Val: User);
1755 if (!II ||
1756 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
1757 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
1758 II->getOperand(i_nocapture: 0) != Pointer)
1759 continue;
1760
1761 // There is some spv_ptrcast/spv_assign_ptr_type already targeting this
1762 // pointer.
1763 FirstPtrCastOrAssignPtrType = false;
1764 if (II->getOperand(i_nocapture: 1) != VMD ||
1765 dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 2))->getSExtValue() !=
1766 AddressSpace)
1767 continue;
1768
1769 // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the
1770 // same element type and address space.
1771 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
1772 return;
1773
1774 // This must be a spv_ptrcast, do not emit new if this one has the same BB
1775 // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type.
1776 if (II->getParent() != I->getParent())
1777 continue;
1778
1779 I->setOperand(i: OperandToReplace, Val: II);
1780 return;
1781 }
1782 }
1783
1784 if (isa<Instruction>(Val: Pointer) || isa<Argument>(Val: Pointer)) {
1785 if (FirstPtrCastOrAssignPtrType) {
1786 // If this would be the first spv_ptrcast, do not emit spv_ptrcast and
1787 // emit spv_assign_ptr_type instead.
1788 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
1789 return;
1790 } else if (isTodoType(Op: Pointer)) {
1791 eraseTodoType(Op: Pointer);
1792 if (!isa<CallInst>(Val: Pointer) && !isa<GetElementPtrInst>(Val: Pointer)) {
1793 // If this wouldn't be the first spv_ptrcast but existing type info is
1794 // uncomplete, update spv_assign_ptr_type arguments.
1795 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Pointer)) {
1796 Type *PrevElemTy = GR->findDeducedElementType(Val: Pointer);
1797 assert(PrevElemTy);
1798 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1799 std::make_pair(x&: I, y&: Pointer)};
1800 GR->updateAssignType(AssignCI, Arg: Pointer, OfType: ExpectedElementVal);
1801 propagateElemType(Op: Pointer, ElemTy: PrevElemTy, VisitedSubst);
1802 } else {
1803 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
1804 }
1805 return;
1806 }
1807 }
1808 }
1809
1810 // Emit spv_ptrcast
1811 SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()};
1812 SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(C: AddressSpace)};
1813 auto *PtrCastI = B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
1814 I->setOperand(i: OperandToReplace, Val: PtrCastI);
1815 // We need to set up a pointee type for the newly created spv_ptrcast.
1816 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: PtrCastI);
1817}
1818
1819void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
1820 IRBuilder<> &B) {
1821 // Handle basic instructions:
1822 StoreInst *SI = dyn_cast<StoreInst>(Val: I);
1823 if (IsKernelArgInt8(F: CurrF, SI)) {
1824 replacePointerOperandWithPtrCast(
1825 I, Pointer: SI->getValueOperand(), ExpectedElementType: IntegerType::getInt8Ty(C&: CurrF->getContext()),
1826 OperandToReplace: 0, B);
1827 }
1828 if (SI) {
1829 Value *Op = SI->getValueOperand();
1830 Value *Pointer = SI->getPointerOperand();
1831 Type *OpTy = Op->getType();
1832 if (auto *OpI = dyn_cast<Instruction>(Val: Op))
1833 OpTy = restoreMutatedType(GR, I: OpI, Ty: OpTy);
1834 if (OpTy == Op->getType())
1835 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: Op, UnknownElemTypeI8: false);
1836 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 1, B);
1837 return;
1838 }
1839 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
1840 Value *Pointer = LI->getPointerOperand();
1841 Type *OpTy = LI->getType();
1842 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1843 if (Type *ElemTy = GR->findDeducedElementType(Val: LI)) {
1844 OpTy = getTypedPointerWrapper(ElemTy, AS: PtrTy->getAddressSpace());
1845 } else {
1846 Type *NewOpTy = OpTy;
1847 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: LI, UnknownElemTypeI8: false);
1848 if (OpTy == NewOpTy)
1849 insertTodoType(Op: Pointer);
1850 }
1851 }
1852 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
1853 return;
1854 }
1855 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(Val: I)) {
1856 Value *Pointer = GEPI->getPointerOperand();
1857 Type *OpTy = nullptr;
1858
1859 // Logical SPIR-V is not allowed to use Op*PtrAccessChain instructions. If
1860 // the first index is 0, then we can trivially lower to OpAccessChain. If
1861 // not we need to try to rewrite the GEP. We avoid adding a pointer cast at
1862 // this time, and will rewrite the GEP when visiting it.
1863 if (TM->getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: GEPI)) {
1864 return;
1865 }
1866
1867 // In all cases, fall back to the GEP type if type scavenging failed.
1868 if (!OpTy)
1869 OpTy = GEPI->getSourceElementType();
1870
1871 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
1872 if (isNestedPointer(Ty: OpTy))
1873 insertTodoType(Op: Pointer);
1874 return;
1875 }
1876
1877 // TODO: review and merge with existing logics:
1878 // Handle calls to builtins (non-intrinsics):
1879 CallInst *CI = dyn_cast<CallInst>(Val: I);
1880 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
1881 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
1882 return;
1883
1884 // collect information about formal parameter types
1885 std::string DemangledName =
1886 getOclOrSpirvBuiltinDemangledName(Name: CI->getCalledFunction()->getName());
1887 Function *CalledF = CI->getCalledFunction();
1888 SmallVector<Type *, 4> CalledArgTys;
1889 bool HaveTypes = false;
1890 for (unsigned OpIdx = 0; OpIdx < CalledF->arg_size(); ++OpIdx) {
1891 Argument *CalledArg = CalledF->getArg(i: OpIdx);
1892 Type *ArgType = CalledArg->getType();
1893 if (!isPointerTy(T: ArgType)) {
1894 CalledArgTys.push_back(Elt: nullptr);
1895 } else if (Type *ArgTypeElem = getPointeeType(Ty: ArgType)) {
1896 CalledArgTys.push_back(Elt: ArgTypeElem);
1897 HaveTypes = true;
1898 } else {
1899 Type *ElemTy = GR->findDeducedElementType(Val: CalledArg);
1900 if (!ElemTy && hasPointeeTypeAttr(Arg: CalledArg))
1901 ElemTy = getPointeeTypeByAttr(Arg: CalledArg);
1902 if (!ElemTy) {
1903 ElemTy = getPointeeTypeByCallInst(DemangledName, CalledF, OpIdx);
1904 if (ElemTy) {
1905 GR->addDeducedElementType(Val: CalledArg, Ty: normalizeType(Ty: ElemTy));
1906 } else {
1907 for (User *U : CalledArg->users()) {
1908 if (Instruction *Inst = dyn_cast<Instruction>(Val: U)) {
1909 if ((ElemTy = deduceElementTypeHelper(I: Inst, UnknownElemTypeI8: false)) != nullptr)
1910 break;
1911 }
1912 }
1913 }
1914 }
1915 HaveTypes |= ElemTy != nullptr;
1916 CalledArgTys.push_back(Elt: ElemTy);
1917 }
1918 }
1919
1920 if (DemangledName.empty() && !HaveTypes)
1921 return;
1922
1923 for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) {
1924 Value *ArgOperand = CI->getArgOperand(i: OpIdx);
1925 if (!isPointerTy(T: ArgOperand->getType()))
1926 continue;
1927
1928 // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs()
1929 if (!isa<Instruction>(Val: ArgOperand) && !isa<Argument>(Val: ArgOperand)) {
1930 // However, we may have assumptions about the formal argument's type and
1931 // may have a need to insert a ptr cast for the actual parameter of this
1932 // call.
1933 Argument *CalledArg = CalledF->getArg(i: OpIdx);
1934 if (!GR->findDeducedElementType(Val: CalledArg))
1935 continue;
1936 }
1937
1938 Type *ExpectedType =
1939 OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr;
1940 if (!ExpectedType && !DemangledName.empty())
1941 ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType(
1942 DemangledCall: DemangledName, ArgIdx: OpIdx, Ctx&: I->getContext());
1943 if (!ExpectedType || ExpectedType->isVoidTy())
1944 continue;
1945
1946 if (ExpectedType->isTargetExtTy() &&
1947 !isTypedPointerWrapper(ExtTy: cast<TargetExtType>(Val: ExpectedType)))
1948 insertAssignPtrTypeTargetExt(AssignedType: cast<TargetExtType>(Val: ExpectedType),
1949 V: ArgOperand, B);
1950 else
1951 replacePointerOperandWithPtrCast(I: CI, Pointer: ArgOperand, ExpectedElementType: ExpectedType, OperandToReplace: OpIdx, B);
1952 }
1953}
1954
1955Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
1956 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
1957 // type in LLT and IRTranslator will replace it by the scalar.
1958 if (isVector1(Ty: I.getType()))
1959 return &I;
1960
1961 SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType(),
1962 I.getOperand(i_nocapture: 1)->getType(),
1963 I.getOperand(i_nocapture: 2)->getType()};
1964 IRBuilder<> B(I.getParent());
1965 B.SetInsertPoint(&I);
1966 SmallVector<Value *> Args(I.op_begin(), I.op_end());
1967 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_insertelt, Types: {Types}, Args: {Args});
1968 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1969 return NewI;
1970}
1971
1972Instruction *
1973SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
1974 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
1975 // type in LLT and IRTranslator will replace it by the scalar.
1976 if (isVector1(Ty: I.getVectorOperandType()))
1977 return &I;
1978
1979 IRBuilder<> B(I.getParent());
1980 B.SetInsertPoint(&I);
1981 SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
1982 I.getIndexOperand()->getType()};
1983 SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
1984 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_extractelt, Types: {Types}, Args: {Args});
1985 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1986 return NewI;
1987}
1988
1989Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
1990 IRBuilder<> B(I.getParent());
1991 B.SetInsertPoint(&I);
1992 SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
1993 SmallVector<Value *> Args;
1994 Value *AggregateOp = I.getAggregateOperand();
1995 if (isa<UndefValue>(Val: AggregateOp))
1996 Args.push_back(Elt: UndefValue::get(T: B.getInt32Ty()));
1997 else
1998 Args.push_back(Elt: AggregateOp);
1999 Args.push_back(Elt: I.getInsertedValueOperand());
2000 for (auto &Op : I.indices())
2001 Args.push_back(Elt: B.getInt32(C: Op));
2002 Instruction *NewI =
2003 B.CreateIntrinsic(ID: Intrinsic::spv_insertv, Types: {Types}, Args: {Args});
2004 replaceMemInstrUses(Old: &I, New: NewI, B);
2005 return NewI;
2006}
2007
2008Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
2009 if (I.getAggregateOperand()->getType()->isAggregateType())
2010 return &I;
2011 IRBuilder<> B(I.getParent());
2012 B.SetInsertPoint(&I);
2013 SmallVector<Value *> Args(I.operands());
2014 for (auto &Op : I.indices())
2015 Args.push_back(Elt: B.getInt32(C: Op));
2016 auto *NewI =
2017 B.CreateIntrinsic(ID: Intrinsic::spv_extractv, Types: {I.getType()}, Args: {Args});
2018 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2019 return NewI;
2020}
2021
2022Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
2023 if (!I.getType()->isAggregateType())
2024 return &I;
2025 IRBuilder<> B(I.getParent());
2026 B.SetInsertPoint(&I);
2027 TrackConstants = false;
2028 const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
2029 MachineMemOperand::Flags Flags =
2030 TLI->getLoadMemOperandFlags(LI: I, DL: CurrF->getDataLayout());
2031 auto *NewI =
2032 B.CreateIntrinsic(ID: Intrinsic::spv_load, Types: {I.getOperand(i_nocapture: 0)->getType()},
2033 Args: {I.getPointerOperand(), B.getInt16(C: Flags),
2034 B.getInt8(C: I.getAlign().value())});
2035 replaceMemInstrUses(Old: &I, New: NewI, B);
2036 return NewI;
2037}
2038
2039Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
2040 if (!AggrStores.contains(V: &I))
2041 return &I;
2042 IRBuilder<> B(I.getParent());
2043 B.SetInsertPoint(&I);
2044 TrackConstants = false;
2045 const auto *TLI = TM->getSubtargetImpl()->getTargetLowering();
2046 MachineMemOperand::Flags Flags =
2047 TLI->getStoreMemOperandFlags(SI: I, DL: CurrF->getDataLayout());
2048 auto *PtrOp = I.getPointerOperand();
2049
2050 if (I.getValueOperand()->getType()->isAggregateType()) {
2051 // It is possible that what used to be an ExtractValueInst has been replaced
2052 // with a call to the spv_extractv intrinsic, and that said call hasn't
2053 // had its return type replaced with i32 during the dedicated pass (because
2054 // it was emitted later); we have to handle this here, because IRTranslator
2055 // cannot deal with multi-register types at the moment.
2056 CallBase *CB = dyn_cast<CallBase>(Val: I.getValueOperand());
2057 assert(CB && CB->getIntrinsicID() == Intrinsic::spv_extractv &&
2058 "Unexpected argument of aggregate type, should be spv_extractv!");
2059 CB->mutateType(Ty: B.getInt32Ty());
2060 }
2061
2062 auto *NewI = B.CreateIntrinsic(
2063 ID: Intrinsic::spv_store, Types: {I.getValueOperand()->getType(), PtrOp->getType()},
2064 Args: {I.getValueOperand(), PtrOp, B.getInt16(C: Flags),
2065 B.getInt8(C: I.getAlign().value())});
2066 NewI->copyMetadata(SrcInst: I);
2067 I.eraseFromParent();
2068 return NewI;
2069}
2070
2071Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
2072 Value *ArraySize = nullptr;
2073 if (I.isArrayAllocation()) {
2074 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*I.getFunction());
2075 if (!STI->canUseExtension(
2076 E: SPIRV::Extension::SPV_INTEL_variable_length_array))
2077 report_fatal_error(
2078 reason: "array allocation: this instruction requires the following "
2079 "SPIR-V extension: SPV_INTEL_variable_length_array",
2080 gen_crash_diag: false);
2081 ArraySize = I.getArraySize();
2082 }
2083 IRBuilder<> B(I.getParent());
2084 B.SetInsertPoint(&I);
2085 TrackConstants = false;
2086 Type *PtrTy = I.getType();
2087 auto *NewI =
2088 ArraySize
2089 ? B.CreateIntrinsic(ID: Intrinsic::spv_alloca_array,
2090 Types: {PtrTy, ArraySize->getType()},
2091 Args: {ArraySize, B.getInt8(C: I.getAlign().value())})
2092 : B.CreateIntrinsic(ID: Intrinsic::spv_alloca, Types: {PtrTy},
2093 Args: {B.getInt8(C: I.getAlign().value())});
2094 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2095 return NewI;
2096}
2097
2098Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2099 assert(I.getType()->isAggregateType() && "Aggregate result is expected");
2100 IRBuilder<> B(I.getParent());
2101 B.SetInsertPoint(&I);
2102 SmallVector<Value *> Args(I.operands());
2103 Args.push_back(Elt: B.getInt32(
2104 C: static_cast<uint32_t>(getMemScope(Ctx&: I.getContext(), Id: I.getSyncScopeID()))));
2105 Args.push_back(Elt: B.getInt32(
2106 C: static_cast<uint32_t>(getMemSemantics(Ord: I.getSuccessOrdering()))));
2107 Args.push_back(Elt: B.getInt32(
2108 C: static_cast<uint32_t>(getMemSemantics(Ord: I.getFailureOrdering()))));
2109 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_cmpxchg,
2110 Types: {I.getPointerOperand()->getType()}, Args: {Args});
2111 replaceMemInstrUses(Old: &I, New: NewI, B);
2112 return NewI;
2113}
2114
2115Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
2116 IRBuilder<> B(I.getParent());
2117 B.SetInsertPoint(&I);
2118 B.CreateIntrinsic(ID: Intrinsic::spv_unreachable, Args: {});
2119 return &I;
2120}
2121
2122void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV,
2123 IRBuilder<> &B) {
2124 // Skip special artificial variables.
2125 static const StringSet<> ArtificialGlobals{"llvm.global.annotations",
2126 "llvm.compiler.used"};
2127
2128 if (ArtificialGlobals.contains(key: GV.getName()))
2129 return;
2130
2131 Constant *Init = nullptr;
2132 if (hasInitializer(GV: &GV)) {
2133 // Deduce element type and store results in Global Registry.
2134 // Result is ignored, because TypedPointerType is not supported
2135 // by llvm IR general logic.
2136 deduceElementTypeHelper(I: &GV, UnknownElemTypeI8: false);
2137 Init = GV.getInitializer();
2138 Type *Ty = isAggrConstForceInt32(V: Init) ? B.getInt32Ty() : Init->getType();
2139 Constant *Const = isAggrConstForceInt32(V: Init) ? B.getInt32(C: 1) : Init;
2140 auto *InitInst = B.CreateIntrinsic(ID: Intrinsic::spv_init_global,
2141 Types: {GV.getType(), Ty}, Args: {&GV, Const});
2142 InitInst->setArgOperand(i: 1, v: Init);
2143 }
2144 if (!Init && GV.use_empty())
2145 B.CreateIntrinsic(ID: Intrinsic::spv_unref_global, Types: GV.getType(), Args: &GV);
2146}
2147
2148// Return true, if we can't decide what is the pointee type now and will get
2149// back to the question later. Return false is spv_assign_ptr_type is not needed
2150// or can be inserted immediately.
2151bool SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I,
2152 IRBuilder<> &B,
2153 bool UnknownElemTypeI8) {
2154 reportFatalOnTokenType(I);
2155 if (!isPointerTy(T: I->getType()) || !requireAssignType(I))
2156 return false;
2157
2158 setInsertPointAfterDef(B, I);
2159 if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) {
2160 GR->buildAssignPtr(B, ElemTy, Arg: I);
2161 return false;
2162 }
2163 return true;
2164}
2165
2166void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I,
2167 IRBuilder<> &B) {
2168 // TODO: extend the list of functions with known result types
2169 static StringMap<unsigned> ResTypeWellKnown = {
2170 {"async_work_group_copy", WellKnownTypes::Event},
2171 {"async_work_group_strided_copy", WellKnownTypes::Event},
2172 {"__spirv_GroupAsyncCopy", WellKnownTypes::Event}};
2173
2174 reportFatalOnTokenType(I);
2175
2176 bool IsKnown = false;
2177 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
2178 if (!CI->isIndirectCall() && !CI->isInlineAsm() &&
2179 CI->getCalledFunction() && !CI->getCalledFunction()->isIntrinsic()) {
2180 Function *CalledF = CI->getCalledFunction();
2181 std::string DemangledName =
2182 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
2183 FPDecorationId DecorationId = FPDecorationId::NONE;
2184 if (DemangledName.length() > 0)
2185 DemangledName =
2186 SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName, DecorationId: &DecorationId);
2187 auto ResIt = ResTypeWellKnown.find(Key: DemangledName);
2188 if (ResIt != ResTypeWellKnown.end()) {
2189 IsKnown = true;
2190 setInsertPointAfterDef(B, I);
2191 switch (ResIt->second) {
2192 case WellKnownTypes::Event:
2193 GR->buildAssignType(
2194 B, Ty: TargetExtType::get(Context&: I->getContext(), Name: "spirv.Event"), Arg: I);
2195 break;
2196 }
2197 }
2198 // check if a floating rounding mode or saturation info is present
2199 switch (DecorationId) {
2200 default:
2201 break;
2202 case FPDecorationId::SAT:
2203 createSaturatedConversionDecoration(I: CI, B);
2204 break;
2205 case FPDecorationId::RTE:
2206 createRoundingModeDecoration(
2207 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTE, B);
2208 break;
2209 case FPDecorationId::RTZ:
2210 createRoundingModeDecoration(
2211 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTZ, B);
2212 break;
2213 case FPDecorationId::RTP:
2214 createRoundingModeDecoration(
2215 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTP, B);
2216 break;
2217 case FPDecorationId::RTN:
2218 createRoundingModeDecoration(
2219 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTN, B);
2220 break;
2221 }
2222 }
2223 }
2224
2225 Type *Ty = I->getType();
2226 if (!IsKnown && !Ty->isVoidTy() && !isPointerTy(T: Ty) && requireAssignType(I)) {
2227 setInsertPointAfterDef(B, I);
2228 Type *TypeToAssign = Ty;
2229 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2230 if (II->getIntrinsicID() == Intrinsic::spv_const_composite ||
2231 II->getIntrinsicID() == Intrinsic::spv_undef) {
2232 auto It = AggrConstTypes.find(Val: II);
2233 if (It == AggrConstTypes.end())
2234 report_fatal_error(reason: "Unknown composite intrinsic type");
2235 TypeToAssign = It->second;
2236 }
2237 }
2238 TypeToAssign = restoreMutatedType(GR, I, Ty: TypeToAssign);
2239 GR->buildAssignType(B, Ty: TypeToAssign, Arg: I);
2240 }
2241 for (const auto &Op : I->operands()) {
2242 if (isa<ConstantPointerNull>(Val: Op) || isa<UndefValue>(Val: Op) ||
2243 // Check GetElementPtrConstantExpr case.
2244 (isa<ConstantExpr>(Val: Op) &&
2245 (isa<GEPOperator>(Val: Op) ||
2246 (cast<ConstantExpr>(Val: Op)->getOpcode() == CastInst::IntToPtr)))) {
2247 setInsertPointSkippingPhis(B, I);
2248 Type *OpTy = Op->getType();
2249 if (isa<UndefValue>(Val: Op) && OpTy->isAggregateType()) {
2250 CallInst *AssignCI =
2251 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {B.getInt32Ty()}, Arg: Op,
2252 Arg2: UndefValue::get(T: B.getInt32Ty()), Imms: {}, B);
2253 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: AssignCI);
2254 } else if (!isa<Instruction>(Val: Op)) {
2255 Type *OpTy = Op->getType();
2256 Type *OpTyElem = getPointeeType(Ty: OpTy);
2257 if (OpTyElem) {
2258 GR->buildAssignPtr(B, ElemTy: OpTyElem, Arg: Op);
2259 } else if (isPointerTy(T: OpTy)) {
2260 Type *ElemTy = GR->findDeducedElementType(Val: Op);
2261 GR->buildAssignPtr(B, ElemTy: ElemTy ? ElemTy : deduceElementType(I: Op, UnknownElemTypeI8: true),
2262 Arg: Op);
2263 } else {
2264 Value *OpTyVal = Op;
2265 if (OpTy->isTargetExtTy()) {
2266 // We need to do this in order to be consistent with how target ext
2267 // types are handled in `processInstrAfterVisit`
2268 OpTyVal = getNormalizedPoisonValue(Ty: OpTy);
2269 }
2270 CallInst *AssignCI =
2271 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {OpTy},
2272 Arg: getNormalizedPoisonValue(Ty: OpTy), Arg2: OpTyVal, Imms: {}, B);
2273 GR->addAssignPtrTypeInstr(Val: OpTyVal, AssignPtrTyCI: AssignCI);
2274 }
2275 }
2276 }
2277 }
2278}
2279
2280bool SPIRVEmitIntrinsics::shouldTryToAddMemAliasingDecoration(
2281 Instruction *Inst) {
2282 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*Inst->getFunction());
2283 if (!STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing))
2284 return false;
2285 // Add aliasing decorations to internal load and store intrinsics
2286 // and atomic instructions, skipping atomic store as it won't have ID to
2287 // attach the decoration.
2288 CallInst *CI = dyn_cast<CallInst>(Val: Inst);
2289 if (!CI)
2290 return false;
2291 if (Function *Fun = CI->getCalledFunction()) {
2292 if (Fun->isIntrinsic()) {
2293 switch (Fun->getIntrinsicID()) {
2294 case Intrinsic::spv_load:
2295 case Intrinsic::spv_store:
2296 return true;
2297 default:
2298 return false;
2299 }
2300 }
2301 std::string Name = getOclOrSpirvBuiltinDemangledName(Name: Fun->getName());
2302 const std::string Prefix = "__spirv_Atomic";
2303 const bool IsAtomic = Name.find(str: Prefix) == 0;
2304
2305 if (!Fun->getReturnType()->isVoidTy() && IsAtomic)
2306 return true;
2307 }
2308 return false;
2309}
2310
2311void SPIRVEmitIntrinsics::insertSpirvDecorations(Instruction *I,
2312 IRBuilder<> &B) {
2313 if (MDNode *MD = I->getMetadata(Kind: "spirv.Decorations")) {
2314 setInsertPointAfterDef(B, I);
2315 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, Types: {I->getType()},
2316 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
2317 }
2318 // Lower alias.scope/noalias metadata
2319 {
2320 auto processMemAliasingDecoration = [&](unsigned Kind) {
2321 if (MDNode *AliasListMD = I->getMetadata(KindID: Kind)) {
2322 if (shouldTryToAddMemAliasingDecoration(Inst: I)) {
2323 uint32_t Dec = Kind == LLVMContext::MD_alias_scope
2324 ? SPIRV::Decoration::AliasScopeINTEL
2325 : SPIRV::Decoration::NoAliasINTEL;
2326 SmallVector<Value *, 3> Args = {
2327 I, ConstantInt::get(Ty: B.getInt32Ty(), V: Dec),
2328 MetadataAsValue::get(Context&: I->getContext(), MD: AliasListMD)};
2329 setInsertPointAfterDef(B, I);
2330 B.CreateIntrinsic(ID: Intrinsic::spv_assign_aliasing_decoration,
2331 Types: {I->getType()}, Args: {Args});
2332 }
2333 }
2334 };
2335 processMemAliasingDecoration(LLVMContext::MD_alias_scope);
2336 processMemAliasingDecoration(LLVMContext::MD_noalias);
2337 }
2338 // MD_fpmath
2339 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_fpmath)) {
2340 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*I->getFunction());
2341 bool AllowFPMaxError =
2342 STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_fp_max_error);
2343 if (!AllowFPMaxError)
2344 return;
2345
2346 setInsertPointAfterDef(B, I);
2347 B.CreateIntrinsic(ID: Intrinsic::spv_assign_fpmaxerror_decoration,
2348 Types: {I->getType()},
2349 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
2350 }
2351}
2352
2353static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
2354 const Module &M,
2355 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
2356 &FPFastMathDefaultInfoMap,
2357 Function *F) {
2358 auto it = FPFastMathDefaultInfoMap.find(Val: F);
2359 if (it != FPFastMathDefaultInfoMap.end())
2360 return it->second;
2361
2362 // If the map does not contain the entry, create a new one. Initialize it to
2363 // contain all 3 elements sorted by bit width of target type: {half, float,
2364 // double}.
2365 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
2366 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
2367 Args: SPIRV::FPFastMathMode::None);
2368 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
2369 Args: SPIRV::FPFastMathMode::None);
2370 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
2371 Args: SPIRV::FPFastMathMode::None);
2372 return FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
2373}
2374
2375static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
2376 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
2377 const Type *Ty) {
2378 size_t BitWidth = Ty->getScalarSizeInBits();
2379 int Index =
2380 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
2381 BitWidth);
2382 assert(Index >= 0 && Index < 3 &&
2383 "Expected FPFastMathDefaultInfo for half, float, or double");
2384 assert(FPFastMathDefaultInfoVec.size() == 3 &&
2385 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
2386 return FPFastMathDefaultInfoVec[Index];
2387}
2388
2389void SPIRVEmitIntrinsics::insertConstantsForFPFastMathDefault(Module &M) {
2390 const SPIRVSubtarget *ST = TM->getSubtargetImpl();
2391 if (!ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
2392 return;
2393
2394 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
2395 // We need the entry point (function) as the key, and the target
2396 // type and flags as the value.
2397 // We also need to check ContractionOff and SignedZeroInfNanPreserve
2398 // execution modes, as they are now deprecated and must be replaced
2399 // with FPFastMathDefaultInfo.
2400 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
2401 if (!Node) {
2402 if (!M.getNamedMetadata(Name: "opencl.enable.FP_CONTRACT")) {
2403 // This requires emitting ContractionOff. However, because
2404 // ContractionOff is now deprecated, we need to replace it with
2405 // FPFastMathDefaultInfo with FP Fast Math Mode bitmask set to all 0.
2406 // We need to create the constant for that.
2407
2408 // Create constant instruction with the bitmask flags.
2409 Constant *InitValue =
2410 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: 0);
2411 // TODO: Reuse constant if there is one already with the required
2412 // value.
2413 [[maybe_unused]] GlobalVariable *GV =
2414 new GlobalVariable(M, // Module
2415 Type::getInt32Ty(C&: M.getContext()), // Type
2416 true, // isConstant
2417 GlobalValue::InternalLinkage, // Linkage
2418 InitValue // Initializer
2419 );
2420 }
2421 return;
2422 }
2423
2424 // The table maps function pointers to their default FP fast math info. It
2425 // can be assumed that the SmallVector is sorted by the bit width of the
2426 // type. The first element is the smallest bit width, and the last element
2427 // is the largest bit width, therefore, we will have {half, float, double}
2428 // in the order of their bit widths.
2429 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
2430 FPFastMathDefaultInfoMap;
2431
2432 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
2433 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
2434 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
2435 Function *F = cast<Function>(
2436 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
2437 const auto EM =
2438 cast<ConstantInt>(
2439 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
2440 ->getZExtValue();
2441 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
2442 assert(MDN->getNumOperands() == 4 &&
2443 "Expected 4 operands for FPFastMathDefault");
2444 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
2445 unsigned Flags =
2446 cast<ConstantInt>(
2447 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
2448 ->getZExtValue();
2449 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2450 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
2451 SPIRV::FPFastMathDefaultInfo &Info =
2452 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
2453 Info.FastMathFlags = Flags;
2454 Info.FPFastMathDefault = true;
2455 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
2456 assert(MDN->getNumOperands() == 2 &&
2457 "Expected no operands for ContractionOff");
2458
2459 // We need to save this info for every possible FP type, i.e. {half,
2460 // float, double, fp128}.
2461 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2462 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
2463 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
2464 Info.ContractionOff = true;
2465 }
2466 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
2467 assert(MDN->getNumOperands() == 3 &&
2468 "Expected 1 operand for SignedZeroInfNanPreserve");
2469 unsigned TargetWidth =
2470 cast<ConstantInt>(
2471 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
2472 ->getZExtValue();
2473 // We need to save this info only for the FP type with TargetWidth.
2474 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2475 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
2476 int Index = SPIRV::FPFastMathDefaultInfoVector::
2477 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
2478 assert(Index >= 0 && Index < 3 &&
2479 "Expected FPFastMathDefaultInfo for half, float, or double");
2480 assert(FPFastMathDefaultInfoVec.size() == 3 &&
2481 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
2482 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
2483 }
2484 }
2485
2486 std::unordered_map<unsigned, GlobalVariable *> GlobalVars;
2487 for (auto &[Func, FPFastMathDefaultInfoVec] : FPFastMathDefaultInfoMap) {
2488 if (FPFastMathDefaultInfoVec.empty())
2489 continue;
2490
2491 for (const SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
2492 assert(Info.Ty && "Expected target type for FPFastMathDefaultInfo");
2493 // Skip if none of the execution modes was used.
2494 unsigned Flags = Info.FastMathFlags;
2495 if (Flags == SPIRV::FPFastMathMode::None && !Info.ContractionOff &&
2496 !Info.SignedZeroInfNanPreserve && !Info.FPFastMathDefault)
2497 continue;
2498
2499 // Check if flags are compatible.
2500 if (Info.ContractionOff && (Flags & SPIRV::FPFastMathMode::AllowContract))
2501 report_fatal_error(reason: "Conflicting FPFastMathFlags: ContractionOff "
2502 "and AllowContract");
2503
2504 if (Info.SignedZeroInfNanPreserve &&
2505 !(Flags &
2506 (SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
2507 SPIRV::FPFastMathMode::NSZ))) {
2508 if (Info.FPFastMathDefault)
2509 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
2510 "SignedZeroInfNanPreserve but at least one of "
2511 "NotNaN/NotInf/NSZ is enabled.");
2512 }
2513
2514 if ((Flags & SPIRV::FPFastMathMode::AllowTransform) &&
2515 !((Flags & SPIRV::FPFastMathMode::AllowReassoc) &&
2516 (Flags & SPIRV::FPFastMathMode::AllowContract))) {
2517 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
2518 "AllowTransform requires AllowReassoc and "
2519 "AllowContract to be set.");
2520 }
2521
2522 auto it = GlobalVars.find(x: Flags);
2523 GlobalVariable *GV = nullptr;
2524 if (it != GlobalVars.end()) {
2525 // Reuse existing global variable.
2526 GV = it->second;
2527 } else {
2528 // Create constant instruction with the bitmask flags.
2529 Constant *InitValue =
2530 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: Flags);
2531 // TODO: Reuse constant if there is one already with the required
2532 // value.
2533 GV = new GlobalVariable(M, // Module
2534 Type::getInt32Ty(C&: M.getContext()), // Type
2535 true, // isConstant
2536 GlobalValue::InternalLinkage, // Linkage
2537 InitValue // Initializer
2538 );
2539 GlobalVars[Flags] = GV;
2540 }
2541 }
2542 }
2543}
2544
2545void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
2546 IRBuilder<> &B) {
2547 auto *II = dyn_cast<IntrinsicInst>(Val: I);
2548 bool IsConstComposite =
2549 II && II->getIntrinsicID() == Intrinsic::spv_const_composite;
2550 if (IsConstComposite && TrackConstants) {
2551 setInsertPointAfterDef(B, I);
2552 auto t = AggrConsts.find(Val: I);
2553 assert(t != AggrConsts.end());
2554 auto *NewOp =
2555 buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
2556 Types: {II->getType(), II->getType()}, Arg: t->second, Arg2: I, Imms: {}, B);
2557 replaceAllUsesWith(Src: I, Dest: NewOp, DeleteOld: false);
2558 NewOp->setArgOperand(i: 0, v: I);
2559 }
2560 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
2561 for (const auto &Op : I->operands()) {
2562 if (isa<PHINode>(Val: I) || isa<SwitchInst>(Val: I) ||
2563 !(isa<ConstantData>(Val: Op) || isa<ConstantExpr>(Val: Op)))
2564 continue;
2565 unsigned OpNo = Op.getOperandNo();
2566 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
2567 (II->paramHasAttr(ArgNo: OpNo, Kind: Attribute::ImmArg))))
2568 continue;
2569
2570 if (!BPrepared) {
2571 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
2572 : B.SetInsertPoint(I);
2573 BPrepared = true;
2574 }
2575 Type *OpTy = Op->getType();
2576 Type *OpElemTy = GR->findDeducedElementType(Val: Op);
2577 Value *NewOp = Op;
2578 if (OpTy->isTargetExtTy()) {
2579 // Since this value is replaced by poison, we need to do the same in
2580 // `insertAssignTypeIntrs`.
2581 Value *OpTyVal = getNormalizedPoisonValue(Ty: OpTy);
2582 NewOp = buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
2583 Types: {OpTy, OpTyVal->getType()}, Arg: Op, Arg2: OpTyVal, Imms: {}, B);
2584 }
2585 if (!IsConstComposite && isPointerTy(T: OpTy) && OpElemTy != nullptr &&
2586 OpElemTy != IntegerType::getInt8Ty(C&: I->getContext())) {
2587 SmallVector<Type *, 2> Types = {OpTy, OpTy};
2588 SmallVector<Value *, 2> Args = {
2589 NewOp, buildMD(Arg: getNormalizedPoisonValue(Ty: OpElemTy)),
2590 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
2591 CallInst *PtrCasted =
2592 B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
2593 GR->buildAssignPtr(B, ElemTy: OpElemTy, Arg: PtrCasted);
2594 NewOp = PtrCasted;
2595 }
2596 if (NewOp != Op)
2597 I->setOperand(i: OpNo, Val: NewOp);
2598 }
2599 if (Named.insert(x: I).second)
2600 emitAssignName(I, B);
2601}
2602
2603Type *SPIRVEmitIntrinsics::deduceFunParamElementType(Function *F,
2604 unsigned OpIdx) {
2605 std::unordered_set<Function *> FVisited;
2606 return deduceFunParamElementType(F, OpIdx, FVisited);
2607}
2608
2609Type *SPIRVEmitIntrinsics::deduceFunParamElementType(
2610 Function *F, unsigned OpIdx, std::unordered_set<Function *> &FVisited) {
2611 // maybe a cycle
2612 if (!FVisited.insert(x: F).second)
2613 return nullptr;
2614
2615 std::unordered_set<Value *> Visited;
2616 SmallVector<std::pair<Function *, unsigned>> Lookup;
2617 // search in function's call sites
2618 for (User *U : F->users()) {
2619 CallInst *CI = dyn_cast<CallInst>(Val: U);
2620 if (!CI || OpIdx >= CI->arg_size())
2621 continue;
2622 Value *OpArg = CI->getArgOperand(i: OpIdx);
2623 if (!isPointerTy(T: OpArg->getType()))
2624 continue;
2625 // maybe we already know operand's element type
2626 if (Type *KnownTy = GR->findDeducedElementType(Val: OpArg))
2627 return KnownTy;
2628 // try to deduce from the operand itself
2629 Visited.clear();
2630 if (Type *Ty = deduceElementTypeHelper(I: OpArg, Visited, UnknownElemTypeI8: false))
2631 return Ty;
2632 // search in actual parameter's users
2633 for (User *OpU : OpArg->users()) {
2634 Instruction *Inst = dyn_cast<Instruction>(Val: OpU);
2635 if (!Inst || Inst == CI)
2636 continue;
2637 Visited.clear();
2638 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8: false))
2639 return Ty;
2640 }
2641 // check if it's a formal parameter of the outer function
2642 if (!CI->getParent() || !CI->getParent()->getParent())
2643 continue;
2644 Function *OuterF = CI->getParent()->getParent();
2645 if (FVisited.find(x: OuterF) != FVisited.end())
2646 continue;
2647 for (unsigned i = 0; i < OuterF->arg_size(); ++i) {
2648 if (OuterF->getArg(i) == OpArg) {
2649 Lookup.push_back(Elt: std::make_pair(x&: OuterF, y&: i));
2650 break;
2651 }
2652 }
2653 }
2654
2655 // search in function parameters
2656 for (auto &Pair : Lookup) {
2657 if (Type *Ty = deduceFunParamElementType(F: Pair.first, OpIdx: Pair.second, FVisited))
2658 return Ty;
2659 }
2660
2661 return nullptr;
2662}
2663
2664void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F,
2665 IRBuilder<> &B) {
2666 B.SetInsertPointPastAllocas(F);
2667 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
2668 Argument *Arg = F->getArg(i: OpIdx);
2669 if (!isUntypedPointerTy(T: Arg->getType()))
2670 continue;
2671 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
2672 if (ElemTy)
2673 continue;
2674 if (hasPointeeTypeAttr(Arg) &&
2675 (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) {
2676 GR->buildAssignPtr(B, ElemTy, Arg);
2677 continue;
2678 }
2679 // search in function's call sites
2680 for (User *U : F->users()) {
2681 CallInst *CI = dyn_cast<CallInst>(Val: U);
2682 if (!CI || OpIdx >= CI->arg_size())
2683 continue;
2684 Value *OpArg = CI->getArgOperand(i: OpIdx);
2685 if (!isPointerTy(T: OpArg->getType()))
2686 continue;
2687 // maybe we already know operand's element type
2688 if ((ElemTy = GR->findDeducedElementType(Val: OpArg)) != nullptr)
2689 break;
2690 }
2691 if (ElemTy) {
2692 GR->buildAssignPtr(B, ElemTy, Arg);
2693 continue;
2694 }
2695 if (HaveFunPtrs) {
2696 for (User *U : Arg->users()) {
2697 CallInst *CI = dyn_cast<CallInst>(Val: U);
2698 if (CI && !isa<IntrinsicInst>(Val: CI) && CI->isIndirectCall() &&
2699 CI->getCalledOperand() == Arg &&
2700 CI->getParent()->getParent() == CurrF) {
2701 SmallVector<std::pair<Value *, unsigned>> Ops;
2702 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy&: ElemTy, IsPostprocessing: false);
2703 if (ElemTy) {
2704 GR->buildAssignPtr(B, ElemTy, Arg);
2705 break;
2706 }
2707 }
2708 }
2709 }
2710 }
2711}
2712
2713void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) {
2714 B.SetInsertPointPastAllocas(F);
2715 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
2716 Argument *Arg = F->getArg(i: OpIdx);
2717 if (!isUntypedPointerTy(T: Arg->getType()))
2718 continue;
2719 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
2720 if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) {
2721 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Arg)) {
2722 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
2723 GR->updateAssignType(AssignCI, Arg, OfType: getNormalizedPoisonValue(Ty: ElemTy));
2724 propagateElemType(Op: Arg, ElemTy: IntegerType::getInt8Ty(C&: F->getContext()),
2725 VisitedSubst);
2726 } else {
2727 GR->buildAssignPtr(B, ElemTy, Arg);
2728 }
2729 }
2730 }
2731}
2732
2733static FunctionType *getFunctionPointerElemType(Function *F,
2734 SPIRVGlobalRegistry *GR) {
2735 FunctionType *FTy = F->getFunctionType();
2736 bool IsNewFTy = false;
2737 SmallVector<Type *, 4> ArgTys;
2738 for (Argument &Arg : F->args()) {
2739 Type *ArgTy = Arg.getType();
2740 if (ArgTy->isPointerTy())
2741 if (Type *ElemTy = GR->findDeducedElementType(Val: &Arg)) {
2742 IsNewFTy = true;
2743 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
2744 }
2745 ArgTys.push_back(Elt: ArgTy);
2746 }
2747 return IsNewFTy
2748 ? FunctionType::get(Result: FTy->getReturnType(), Params: ArgTys, isVarArg: FTy->isVarArg())
2749 : FTy;
2750}
2751
2752bool SPIRVEmitIntrinsics::processFunctionPointers(Module &M) {
2753 SmallVector<Function *> Worklist;
2754 for (auto &F : M) {
2755 if (F.isIntrinsic())
2756 continue;
2757 if (F.isDeclaration()) {
2758 for (User *U : F.users()) {
2759 CallInst *CI = dyn_cast<CallInst>(Val: U);
2760 if (!CI || CI->getCalledFunction() != &F) {
2761 Worklist.push_back(Elt: &F);
2762 break;
2763 }
2764 }
2765 } else {
2766 if (F.user_empty())
2767 continue;
2768 Type *FPElemTy = GR->findDeducedElementType(Val: &F);
2769 if (!FPElemTy)
2770 FPElemTy = getFunctionPointerElemType(F: &F, GR);
2771 for (User *U : F.users()) {
2772 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
2773 if (!II || II->arg_size() != 3 || II->getOperand(i_nocapture: 0) != &F)
2774 continue;
2775 if (II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type ||
2776 II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
2777 GR->updateAssignType(AssignCI: II, Arg: &F, OfType: getNormalizedPoisonValue(Ty: FPElemTy));
2778 break;
2779 }
2780 }
2781 }
2782 }
2783 if (Worklist.empty())
2784 return false;
2785
2786 std::string ServiceFunName = SPIRV_BACKEND_SERVICE_FUN_NAME;
2787 if (!getVacantFunctionName(M, Name&: ServiceFunName))
2788 report_fatal_error(
2789 reason: "cannot allocate a name for the internal service function");
2790 LLVMContext &Ctx = M.getContext();
2791 Function *SF =
2792 Function::Create(Ty: FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: {}, isVarArg: false),
2793 Linkage: GlobalValue::PrivateLinkage, N: ServiceFunName, M);
2794 SF->addFnAttr(SPIRV_BACKEND_SERVICE_FUN_NAME, Val: "");
2795 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: SF);
2796 IRBuilder<> IRB(BB);
2797
2798 for (Function *F : Worklist) {
2799 SmallVector<Value *> Args;
2800 for (const auto &Arg : F->args())
2801 Args.push_back(Elt: getNormalizedPoisonValue(Ty: Arg.getType()));
2802 IRB.CreateCall(Callee: F, Args);
2803 }
2804 IRB.CreateRetVoid();
2805
2806 return true;
2807}
2808
2809// Apply types parsed from demangled function declarations.
2810void SPIRVEmitIntrinsics::applyDemangledPtrArgTypes(IRBuilder<> &B) {
2811 DenseMap<Function *, CallInst *> Ptrcasts;
2812 for (auto It : FDeclPtrTys) {
2813 Function *F = It.first;
2814 for (auto *U : F->users()) {
2815 CallInst *CI = dyn_cast<CallInst>(Val: U);
2816 if (!CI || CI->getCalledFunction() != F)
2817 continue;
2818 unsigned Sz = CI->arg_size();
2819 for (auto [Idx, ElemTy] : It.second) {
2820 if (Idx >= Sz)
2821 continue;
2822 Value *Param = CI->getArgOperand(i: Idx);
2823 if (GR->findDeducedElementType(Val: Param) || isa<GlobalValue>(Val: Param))
2824 continue;
2825 if (Argument *Arg = dyn_cast<Argument>(Val: Param)) {
2826 if (!hasPointeeTypeAttr(Arg)) {
2827 B.SetInsertPointPastAllocas(Arg->getParent());
2828 B.SetCurrentDebugLocation(DebugLoc());
2829 GR->buildAssignPtr(B, ElemTy, Arg);
2830 }
2831 } else if (isa<GetElementPtrInst>(Val: Param)) {
2832 replaceUsesOfWithSpvPtrcast(Op: Param, ElemTy: normalizeType(Ty: ElemTy), I: CI,
2833 Ptrcasts);
2834 } else if (isa<Instruction>(Val: Param)) {
2835 GR->addDeducedElementType(Val: Param, Ty: normalizeType(Ty: ElemTy));
2836 // insertAssignTypeIntrs() will complete buildAssignPtr()
2837 } else {
2838 B.SetInsertPoint(CI->getParent()
2839 ->getParent()
2840 ->getEntryBlock()
2841 .getFirstNonPHIOrDbgOrAlloca());
2842 GR->buildAssignPtr(B, ElemTy, Arg: Param);
2843 }
2844 CallInst *Ref = dyn_cast<CallInst>(Val: Param);
2845 if (!Ref)
2846 continue;
2847 Function *RefF = Ref->getCalledFunction();
2848 if (!RefF || !isPointerTy(T: RefF->getReturnType()) ||
2849 GR->findDeducedElementType(Val: RefF))
2850 continue;
2851 ElemTy = normalizeType(Ty: ElemTy);
2852 GR->addDeducedElementType(Val: RefF, Ty: ElemTy);
2853 GR->addReturnType(
2854 ArgF: RefF, DerivedTy: TypedPointerType::get(
2855 ElementType: ElemTy, AddressSpace: getPointerAddressSpace(T: RefF->getReturnType())));
2856 }
2857 }
2858 }
2859}
2860
2861GetElementPtrInst *
2862SPIRVEmitIntrinsics::simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP) {
2863 // getelementptr [0 x T], P, 0 (zero), I -> getelementptr T, P, I.
2864 // If type is 0-length array and first index is 0 (zero), drop both the
2865 // 0-length array type and the first index. This is a common pattern in
2866 // the IR, e.g. when using a zero-length array as a placeholder for a
2867 // flexible array such as unbound arrays.
2868 assert(GEP && "GEP is null");
2869 Type *SrcTy = GEP->getSourceElementType();
2870 SmallVector<Value *, 8> Indices(GEP->indices());
2871 ArrayType *ArrTy = dyn_cast<ArrayType>(Val: SrcTy);
2872 if (ArrTy && ArrTy->getNumElements() == 0 &&
2873 PatternMatch::match(V: Indices[0], P: PatternMatch::m_Zero())) {
2874 Indices.erase(CI: Indices.begin());
2875 SrcTy = ArrTy->getElementType();
2876 return GetElementPtrInst::Create(PointeeType: SrcTy, Ptr: GEP->getPointerOperand(), IdxList: Indices,
2877 NW: GEP->getNoWrapFlags(), NameStr: "",
2878 InsertBefore: GEP->getIterator());
2879 }
2880 return nullptr;
2881}
2882
2883void SPIRVEmitIntrinsics::emitUnstructuredLoopControls(Function &F,
2884 IRBuilder<> &B) {
2885 const SPIRVSubtarget *ST = TM->getSubtargetImpl(F);
2886 // Shaders use SPIRVStructurizer which emits OpLoopMerge via spv_loop_merge.
2887 if (ST->isShader())
2888 return;
2889 if (!ST->canUseExtension(
2890 E: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls))
2891 return;
2892
2893 for (BasicBlock &BB : F) {
2894 Instruction *Term = BB.getTerminator();
2895 MDNode *LoopMD = Term->getMetadata(KindID: LLVMContext::MD_loop);
2896 if (!LoopMD)
2897 continue;
2898
2899 SmallVector<unsigned, 1> Ops =
2900 getSpirvLoopControlOperandsFromLoopMetadata(LoopMD);
2901 unsigned LC = Ops[0];
2902 if (LC == SPIRV::LoopControl::None)
2903 continue;
2904
2905 // Emit intrinsic: loop control mask + optional parameters.
2906 B.SetInsertPoint(Term);
2907 SmallVector<Value *, 4> IntrArgs;
2908 IntrArgs.push_back(Elt: B.getInt32(C: LC));
2909 for (unsigned I = 1; I < Ops.size(); ++I)
2910 IntrArgs.push_back(Elt: B.getInt32(C: Ops[I]));
2911 B.CreateIntrinsic(ID: Intrinsic::spv_loop_control_intel, Args: IntrArgs);
2912 }
2913}
2914
2915bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
2916 if (Func.isDeclaration())
2917 return false;
2918
2919 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F: Func);
2920 GR = ST.getSPIRVGlobalRegistry();
2921
2922 if (!CurrF)
2923 HaveFunPtrs =
2924 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers);
2925
2926 CurrF = &Func;
2927 IRBuilder<> B(Func.getContext());
2928 AggrConsts.clear();
2929 AggrConstTypes.clear();
2930 AggrStores.clear();
2931
2932 // Fix GEP result types ahead of inference, and simplify if possible.
2933 // Data structure for dead instructions that were simplified and replaced.
2934 SmallPtrSet<Instruction *, 4> DeadInsts;
2935 for (auto &I : instructions(F&: Func)) {
2936 auto *Ref = dyn_cast<GetElementPtrInst>(Val: &I);
2937 if (!Ref || GR->findDeducedElementType(Val: Ref))
2938 continue;
2939
2940 GetElementPtrInst *NewGEP = simplifyZeroLengthArrayGepInst(GEP: Ref);
2941 if (NewGEP) {
2942 Ref->replaceAllUsesWith(V: NewGEP);
2943 DeadInsts.insert(Ptr: Ref);
2944 Ref = NewGEP;
2945 }
2946 if (Type *GepTy = getGEPType(Ref))
2947 GR->addDeducedElementType(Val: Ref, Ty: normalizeType(Ty: GepTy));
2948 }
2949 // Remove dead instructions that were simplified and replaced.
2950 for (auto *I : DeadInsts) {
2951 assert(I->use_empty() && "Dead instruction should not have any uses left");
2952 I->eraseFromParent();
2953 }
2954
2955 processParamTypesByFunHeader(F: CurrF, B);
2956
2957 // StoreInst's operand type can be changed during the next
2958 // transformations, so we need to store it in the set. Also store already
2959 // transformed types.
2960 for (auto &I : instructions(F&: Func)) {
2961 StoreInst *SI = dyn_cast<StoreInst>(Val: &I);
2962 if (!SI)
2963 continue;
2964 Type *ElTy = SI->getValueOperand()->getType();
2965 if (ElTy->isAggregateType() || ElTy->isVectorTy())
2966 AggrStores.insert(V: &I);
2967 }
2968
2969 B.SetInsertPoint(TheBB: &Func.getEntryBlock(), IP: Func.getEntryBlock().begin());
2970 for (auto &GV : Func.getParent()->globals())
2971 processGlobalValue(GV, B);
2972
2973 preprocessUndefs(B);
2974 preprocessCompositeConstants(B);
2975 SmallVector<Instruction *> Worklist(
2976 llvm::make_pointer_range(Range: instructions(F&: Func)));
2977
2978 applyDemangledPtrArgTypes(B);
2979
2980 // Pass forward: use operand to deduce instructions result.
2981 for (auto &I : Worklist) {
2982 // Don't emit intrinsincs for convergence intrinsics.
2983 if (isConvergenceIntrinsic(I))
2984 continue;
2985
2986 bool Postpone = insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: false);
2987 // if Postpone is true, we can't decide on pointee type yet
2988 insertAssignTypeIntrs(I, B);
2989 insertPtrCastOrAssignTypeInstr(I, B);
2990 insertSpirvDecorations(I, B);
2991 // if instruction requires a pointee type set, let's check if we know it
2992 // already, and force it to be i8 if not
2993 if (Postpone && !GR->findAssignPtrTypeInstr(Val: I))
2994 insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: true);
2995
2996 if (auto *FPI = dyn_cast<ConstrainedFPIntrinsic>(Val: I))
2997 useRoundingMode(FPI, B);
2998 }
2999
3000 // Pass backward: use instructions results to specify/update/cast operands
3001 // where needed.
3002 SmallPtrSet<Instruction *, 4> IncompleteRets;
3003 for (auto &I : llvm::reverse(C: instructions(F&: Func)))
3004 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets);
3005
3006 // Pass forward for PHIs only, their operands are not preceed the
3007 // instruction in meaning of `instructions(Func)`.
3008 for (BasicBlock &BB : Func)
3009 for (PHINode &Phi : BB.phis())
3010 if (isPointerTy(T: Phi.getType()))
3011 deduceOperandElementType(I: &Phi, IncompleteRets: nullptr);
3012
3013 for (auto *I : Worklist) {
3014 TrackConstants = true;
3015 if (!I->getType()->isVoidTy() || isa<StoreInst>(Val: I))
3016 setInsertPointAfterDef(B, I);
3017 // Visitors return either the original/newly created instruction for
3018 // further processing, nullptr otherwise.
3019 I = visit(I&: *I);
3020 if (!I)
3021 continue;
3022
3023 // Don't emit intrinsics for convergence operations.
3024 if (isConvergenceIntrinsic(I))
3025 continue;
3026
3027 addSaturatedDecorationToIntrinsic(I, B);
3028 processInstrAfterVisit(I, B);
3029 }
3030
3031 emitUnstructuredLoopControls(F&: Func, B);
3032
3033 return true;
3034}
3035
3036// Try to deduce a better type for pointers to untyped ptr.
3037bool SPIRVEmitIntrinsics::postprocessTypes(Module &M) {
3038 if (!GR || TodoTypeSz == 0)
3039 return false;
3040
3041 unsigned SzTodo = TodoTypeSz;
3042 DenseMap<Value *, SmallPtrSet<Value *, 4>> ToProcess;
3043 for (auto [Op, Enabled] : TodoType) {
3044 // TODO: add isa<CallInst>(Op) to continue
3045 if (!Enabled || isa<GetElementPtrInst>(Val: Op))
3046 continue;
3047 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Op);
3048 Type *KnownTy = GR->findDeducedElementType(Val: Op);
3049 if (!KnownTy || !AssignCI)
3050 continue;
3051 assert(Op == AssignCI->getArgOperand(0));
3052 // Try to improve the type deduced after all Functions are processed.
3053 if (auto *CI = dyn_cast<Instruction>(Val: Op)) {
3054 CurrF = CI->getParent()->getParent();
3055 std::unordered_set<Value *> Visited;
3056 if (Type *ElemTy = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8: false, IgnoreKnownType: true)) {
3057 if (ElemTy != KnownTy) {
3058 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3059 propagateElemType(Op: CI, ElemTy, VisitedSubst);
3060 eraseTodoType(Op);
3061 continue;
3062 }
3063 }
3064 }
3065
3066 if (Op->hasUseList()) {
3067 for (User *U : Op->users()) {
3068 Instruction *Inst = dyn_cast<Instruction>(Val: U);
3069 if (Inst && !isa<IntrinsicInst>(Val: Inst))
3070 ToProcess[Inst].insert(Ptr: Op);
3071 }
3072 }
3073 }
3074 if (TodoTypeSz == 0)
3075 return true;
3076
3077 for (auto &F : M) {
3078 CurrF = &F;
3079 SmallPtrSet<Instruction *, 4> IncompleteRets;
3080 for (auto &I : llvm::reverse(C: instructions(F))) {
3081 auto It = ToProcess.find(Val: &I);
3082 if (It == ToProcess.end())
3083 continue;
3084 It->second.remove_if(P: [this](Value *V) { return !isTodoType(Op: V); });
3085 if (It->second.size() == 0)
3086 continue;
3087 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets, AskOps: &It->second, IsPostprocessing: true);
3088 if (TodoTypeSz == 0)
3089 return true;
3090 }
3091 }
3092
3093 return SzTodo > TodoTypeSz;
3094}
3095
3096// Parse and store argument types of function declarations where needed.
3097void SPIRVEmitIntrinsics::parseFunDeclarations(Module &M) {
3098 for (auto &F : M) {
3099 if (!F.isDeclaration() || F.isIntrinsic())
3100 continue;
3101 // get the demangled name
3102 std::string DemangledName = getOclOrSpirvBuiltinDemangledName(Name: F.getName());
3103 if (DemangledName.empty())
3104 continue;
3105 // allow only OpGroupAsyncCopy use case at the moment
3106 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F);
3107 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
3108 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
3109 if (Opcode != SPIRV::OpGroupAsyncCopy)
3110 continue;
3111 // find pointer arguments
3112 SmallVector<unsigned> Idxs;
3113 for (unsigned OpIdx = 0; OpIdx < F.arg_size(); ++OpIdx) {
3114 Argument *Arg = F.getArg(i: OpIdx);
3115 if (isPointerTy(T: Arg->getType()) && !hasPointeeTypeAttr(Arg))
3116 Idxs.push_back(Elt: OpIdx);
3117 }
3118 if (!Idxs.size())
3119 continue;
3120 // parse function arguments
3121 LLVMContext &Ctx = F.getContext();
3122 SmallVector<StringRef, 10> TypeStrs;
3123 SPIRV::parseBuiltinTypeStr(BuiltinArgsTypeStrs&: TypeStrs, DemangledCall: DemangledName, Ctx);
3124 if (!TypeStrs.size())
3125 continue;
3126 // find type info for pointer arguments
3127 for (unsigned Idx : Idxs) {
3128 if (Idx >= TypeStrs.size())
3129 continue;
3130 if (Type *ElemTy =
3131 SPIRV::parseBuiltinCallArgumentType(TypeStr: TypeStrs[Idx].trim(), Ctx))
3132 if (TypedPointerType::isValidElementType(ElemTy) &&
3133 !ElemTy->isTargetExtTy())
3134 FDeclPtrTys[&F].push_back(Elt: std::make_pair(x&: Idx, y&: ElemTy));
3135 }
3136 }
3137}
3138
3139bool SPIRVEmitIntrinsics::runOnModule(Module &M) {
3140 bool Changed = false;
3141
3142 parseFunDeclarations(M);
3143 insertConstantsForFPFastMathDefault(M);
3144
3145 TodoType.clear();
3146 for (auto &F : M)
3147 Changed |= runOnFunction(Func&: F);
3148
3149 // Specify function parameters after all functions were processed.
3150 for (auto &F : M) {
3151 // check if function parameter types are set
3152 CurrF = &F;
3153 if (!F.isDeclaration() && !F.isIntrinsic()) {
3154 IRBuilder<> B(F.getContext());
3155 processParamTypes(F: &F, B);
3156 }
3157 }
3158
3159 CanTodoType = false;
3160 Changed |= postprocessTypes(M);
3161
3162 if (HaveFunPtrs)
3163 Changed |= processFunctionPointers(M);
3164
3165 return Changed;
3166}
3167
3168ModulePass *llvm::createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM) {
3169 return new SPIRVEmitIntrinsics(TM);
3170}
3171