1//===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass emits SPIRV intrinsics keeping essential high-level information for
10// the translation of LLVM IR to SPIR-V.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRVEmitIntrinsics.h"
15#include "SPIRV.h"
16#include "SPIRVBuiltins.h"
17#include "SPIRVSubtarget.h"
18#include "SPIRVTargetMachine.h"
19#include "SPIRVUtils.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/StringSet.h"
22#include "llvm/Analysis/LoopInfo.h"
23#include "llvm/IR/Dominators.h"
24#include "llvm/IR/IRBuilder.h"
25#include "llvm/IR/InstIterator.h"
26#include "llvm/IR/InstVisitor.h"
27#include "llvm/IR/IntrinsicsSPIRV.h"
28#include "llvm/IR/PatternMatch.h"
29#include "llvm/IR/TypedPointerType.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Transforms/Utils/Local.h"
32
33#include <cassert>
34#include <queue>
35#include <unordered_set>
36
37// This pass performs the following transformation on LLVM IR level required
38// for the following translation to SPIR-V:
39// - replaces direct usages of aggregate constants with target-specific
40// intrinsics;
41// - replaces aggregates-related instructions (extract/insert, ld/st, etc)
42// with a target-specific intrinsics;
43// - emits intrinsics for the global variable initializers since IRTranslator
44// doesn't handle them and it's not very convenient to translate them
45// ourselves;
46// - emits intrinsics to keep track of the string names assigned to the values;
47// - emits intrinsics to keep track of constants (this is necessary to have an
48// LLVM IR constant after the IRTranslation is completed) for their further
49// deduplication;
50// - emits intrinsics to keep track of original LLVM types of the values
51// to be able to emit proper SPIR-V types eventually.
52//
53// TODO: consider removing spv.track.constant in favor of spv.assign.type.
54
55using namespace llvm;
56
57static cl::opt<bool>
58 SpirvEmitOpNames("spirv-emit-op-names",
59 cl::desc("Emit OpName for all instructions"),
60 cl::init(Val: false));
61
62namespace llvm::SPIRV {
63#define GET_BuiltinGroup_DECL
64#include "SPIRVGenTables.inc"
65} // namespace llvm::SPIRV
66
67namespace {
68// This class keeps track of which functions reference which global variables.
69class GlobalVariableUsers {
70 template <typename T1, typename T2>
71 using OneToManyMapTy = DenseMap<T1, SmallPtrSet<T2, 4>>;
72
73 OneToManyMapTy<const GlobalVariable *, const Function *> GlobalIsUsedByFun;
74
75 void collectGlobalUsers(
76 const GlobalVariable *GV,
77 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
78 &GlobalIsUsedByGlobal) {
79 SmallVector<const Value *> Stack = {GV->user_begin(), GV->user_end()};
80 while (!Stack.empty()) {
81 const Value *V = Stack.pop_back_val();
82
83 if (const Instruction *I = dyn_cast<Instruction>(Val: V)) {
84 GlobalIsUsedByFun[GV].insert(Ptr: I->getFunction());
85 continue;
86 }
87
88 if (const GlobalVariable *UserGV = dyn_cast<GlobalVariable>(Val: V)) {
89 GlobalIsUsedByGlobal[GV].insert(Ptr: UserGV);
90 continue;
91 }
92
93 if (const Constant *C = dyn_cast<Constant>(Val: V))
94 Stack.append(in_start: C->user_begin(), in_end: C->user_end());
95 }
96 }
97
98 bool propagateGlobalToGlobalUsers(
99 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
100 &GlobalIsUsedByGlobal) {
101 SmallVector<const GlobalVariable *> OldUsersGlobals;
102 bool Changed = false;
103 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
104 OldUsersGlobals.assign(in_start: UserGlobals.begin(), in_end: UserGlobals.end());
105 for (const GlobalVariable *UserGV : OldUsersGlobals) {
106 auto It = GlobalIsUsedByGlobal.find(Val: UserGV);
107 if (It == GlobalIsUsedByGlobal.end())
108 continue;
109 Changed |= set_union(S1&: UserGlobals, S2: It->second);
110 }
111 }
112 return Changed;
113 }
114
115 void propagateGlobalToFunctionReferences(
116 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
117 &GlobalIsUsedByGlobal) {
118 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
119 auto &UserFunctions = GlobalIsUsedByFun[GV];
120 for (const GlobalVariable *UserGV : UserGlobals) {
121 auto It = GlobalIsUsedByFun.find(Val: UserGV);
122 if (It == GlobalIsUsedByFun.end())
123 continue;
124 set_union(S1&: UserFunctions, S2: It->second);
125 }
126 }
127 }
128
129public:
130 void init(Module &M) {
131 // Collect which global variables are referenced by which global variables
132 // and which functions reference each global variables.
133 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
134 GlobalIsUsedByGlobal;
135 GlobalIsUsedByFun.clear();
136 for (GlobalVariable &GV : M.globals())
137 collectGlobalUsers(GV: &GV, GlobalIsUsedByGlobal);
138
139 // Compute indirect references by iterating until a fixed point is reached.
140 while (propagateGlobalToGlobalUsers(GlobalIsUsedByGlobal))
141 (void)0;
142
143 propagateGlobalToFunctionReferences(GlobalIsUsedByGlobal);
144 }
145
146 using FunctionSetType = typename decltype(GlobalIsUsedByFun)::mapped_type;
147 const FunctionSetType &
148 getTransitiveUserFunctions(const GlobalVariable &GV) const {
149 auto It = GlobalIsUsedByFun.find(Val: &GV);
150 if (It != GlobalIsUsedByFun.end())
151 return It->second;
152
153 static const FunctionSetType Empty{};
154 return Empty;
155 }
156};
157
158static bool isaGEP(const Value *V) {
159 return isa<StructuredGEPInst>(Val: V) || isa<GetElementPtrInst>(Val: V);
160}
161
162class SPIRVEmitIntrinsics
163 : public ModulePass,
164 public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
165 const SPIRVTargetMachine &TM;
166 SPIRVGlobalRegistry *GR = nullptr;
167 Function *CurrF = nullptr;
168 bool TrackConstants = true;
169 bool HaveFunPtrs = false;
170 DenseMap<Instruction *, Constant *> AggrConsts;
171 DenseMap<Instruction *, Type *> AggrConstTypes;
172 DenseSet<Instruction *> AggrStores;
173 SmallPtrSet<Instruction *, 8> DeletedInstrs;
174 GlobalVariableUsers GVUsers;
175 std::unordered_set<Value *> Named;
176
177 // map of function declarations to <pointer arg index => element type>
178 DenseMap<Function *, SmallVector<std::pair<unsigned, Type *>>> FDeclPtrTys;
179
180 // a register of Instructions that don't have a complete type definition
181 bool CanTodoType = true;
182 unsigned TodoTypeSz = 0;
183 DenseMap<Value *, bool> TodoType;
184 void insertTodoType(Value *Op) {
185 // TODO: add isa<CallInst>(Op) to no-insert
186 if (CanTodoType && !isaGEP(V: Op)) {
187 auto It = TodoType.try_emplace(Key: Op, Args: true);
188 if (It.second)
189 ++TodoTypeSz;
190 }
191 }
192 void eraseTodoType(Value *Op) {
193 auto It = TodoType.find(Val: Op);
194 if (It != TodoType.end() && It->second) {
195 It->second = false;
196 --TodoTypeSz;
197 }
198 }
199 bool isTodoType(Value *Op) {
200 if (isaGEP(V: Op))
201 return false;
202 auto It = TodoType.find(Val: Op);
203 return It != TodoType.end() && It->second;
204 }
205 // a register of Instructions that were visited by deduceOperandElementType()
206 // to validate operand types with an instruction
207 std::unordered_set<Instruction *> TypeValidated;
208
209 // well known result types of builtins
210 enum WellKnownTypes { Event };
211
212 // deduce element type of untyped pointers
213 Type *deduceElementType(Value *I, bool UnknownElemTypeI8);
214 Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8);
215 Type *deduceElementTypeHelper(Value *I, std::unordered_set<Value *> &Visited,
216 bool UnknownElemTypeI8,
217 bool IgnoreKnownType = false);
218 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
219 bool UnknownElemTypeI8);
220 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
221 std::unordered_set<Value *> &Visited,
222 bool UnknownElemTypeI8);
223 Type *deduceElementTypeByUsersDeep(Value *Op,
224 std::unordered_set<Value *> &Visited,
225 bool UnknownElemTypeI8);
226 void maybeAssignPtrType(Type *&Ty, Value *I, Type *RefTy,
227 bool UnknownElemTypeI8);
228
229 // deduce nested types of composites
230 Type *deduceNestedTypeHelper(User *U, bool UnknownElemTypeI8);
231 Type *deduceNestedTypeHelper(User *U, Type *Ty,
232 std::unordered_set<Value *> &Visited,
233 bool UnknownElemTypeI8);
234
235 // deduce Types of operands of the Instruction if possible
236 void deduceOperandElementType(Instruction *I,
237 SmallPtrSet<Instruction *, 4> *IncompleteRets,
238 const SmallPtrSet<Value *, 4> *AskOps = nullptr,
239 bool IsPostprocessing = false);
240
241 void preprocessCompositeConstants(IRBuilder<> &B);
242 void preprocessUndefs(IRBuilder<> &B);
243
244 Type *reconstructType(Value *Op, bool UnknownElemTypeI8,
245 bool IsPostprocessing);
246
247 void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
248 void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
249 bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
250 bool UnknownElemTypeI8);
251 void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B);
252 void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V,
253 IRBuilder<> &B);
254 void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer,
255 Type *ExpectedElementType,
256 unsigned OperandToReplace,
257 IRBuilder<> &B);
258 void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
259 bool shouldTryToAddMemAliasingDecoration(Instruction *Inst);
260 void insertSpirvDecorations(Instruction *I, IRBuilder<> &B);
261 void insertConstantsForFPFastMathDefault(Module &M);
262 Value *buildSpvUndefComposite(Type *AggrTy, IRBuilder<> &B);
263 void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
264 void processParamTypes(Function *F, IRBuilder<> &B);
265 void processParamTypesByFunHeader(Function *F, IRBuilder<> &B);
266 Type *deduceFunParamElementType(Function *F, unsigned OpIdx);
267 Type *deduceFunParamElementType(Function *F, unsigned OpIdx,
268 std::unordered_set<Function *> &FVisited);
269
270 bool deduceOperandElementTypeCalledFunction(
271 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
272 Type *&KnownElemTy, bool &Incomplete);
273 void deduceOperandElementTypeFunctionPointer(
274 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
275 Type *&KnownElemTy, bool IsPostprocessing);
276 bool deduceOperandElementTypeFunctionRet(
277 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
278 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing,
279 Type *&KnownElemTy, Value *Op, Function *F);
280
281 CallInst *buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy);
282 void replaceUsesOfWithSpvPtrcast(Value *Op, Type *ElemTy, Instruction *I,
283 DenseMap<Function *, CallInst *> Ptrcasts);
284 void propagateElemType(Value *Op, Type *ElemTy,
285 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
286 void
287 propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
288 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
289 void propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
290 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
291 std::unordered_set<Value *> &Visited,
292 DenseMap<Function *, CallInst *> Ptrcasts);
293
294 void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
295 void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src,
296 Instruction *Dest, bool DeleteOld = true);
297
298 void applyDemangledPtrArgTypes(IRBuilder<> &B);
299
300 GetElementPtrInst *simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP);
301
302 bool runOnFunction(Function &F);
303 bool postprocessTypes(Module &M);
304 bool processFunctionPointers(Module &M);
305 void parseFunDeclarations(Module &M);
306 void useRoundingMode(ConstrainedFPIntrinsic *FPI, IRBuilder<> &B);
307 bool processMaskedMemIntrinsic(IntrinsicInst &I);
308 bool convertMaskedMemIntrinsics(Module &M);
309 void preprocessBoolVectorBitcasts(Function &F);
310
311 void emitUnstructuredLoopControls(Function &F, IRBuilder<> &B);
312
313 // Tries to walk the type accessed by the given GEP instruction.
314 // For each nested type access, one of the 2 callbacks is called:
315 // - OnLiteralIndexing when the index is a known constant value.
316 // Parameters:
317 // PointedType: the pointed type resulting of this indexing.
318 // If the parent type is an array, this is the index in the array.
319 // If the parent type is a struct, this is the field index.
320 // Index: index of the element in the parent type.
321 // - OnDynamnicIndexing when the index is a non-constant value.
322 // This callback is only called when indexing into an array.
323 // Parameters:
324 // ElementType: the type of the elements stored in the parent array.
325 // Offset: the Value* containing the byte offset into the array.
326 // Return true if an error occurred during the walk, false otherwise.
327 bool walkLogicalAccessChain(
328 GetElementPtrInst &GEP,
329 const std::function<void(Type *PointedType, uint64_t Index)>
330 &OnLiteralIndexing,
331 const std::function<void(Type *ElementType, Value *Offset)>
332 &OnDynamicIndexing);
333
334 // Returns the type accessed using the given GEP instruction by relying
335 // on the GEP type.
336 // FIXME: GEP types are not supposed to be used to retrieve the pointed
337 // type. This must be fixed.
338 Type *getGEPType(GetElementPtrInst *GEP);
339
340 // Returns the type accessed using the given GEP instruction by walking
341 // the source type using the GEP indices.
342 // FIXME: without help from the frontend, this method cannot reliably retrieve
343 // the stored type, nor can robustly determine the depth of the type
344 // we are accessing.
345 Type *getGEPTypeLogical(GetElementPtrInst *GEP);
346
347 Instruction *buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP);
348
349public:
350 static char ID;
351 SPIRVEmitIntrinsics(const SPIRVTargetMachine &TM) : ModulePass(ID), TM(TM) {}
352 Instruction *visitInstruction(Instruction &I) { return &I; }
353 Instruction *visitSwitchInst(SwitchInst &I);
354 Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
355 Instruction *visitIntrinsicInst(IntrinsicInst &I);
356 Instruction *visitBitCastInst(BitCastInst &I);
357 Instruction *visitInsertElementInst(InsertElementInst &I);
358 Instruction *visitExtractElementInst(ExtractElementInst &I);
359 Instruction *visitInsertValueInst(InsertValueInst &I);
360 Instruction *visitExtractValueInst(ExtractValueInst &I);
361 Instruction *visitLoadInst(LoadInst &I);
362 Instruction *visitStoreInst(StoreInst &I);
363 Instruction *visitAllocaInst(AllocaInst &I);
364 Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
365 Instruction *visitUnreachableInst(UnreachableInst &I);
366 Instruction *visitCallInst(CallInst &I);
367
368 StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
369
370 bool runOnModule(Module &M) override;
371
372 void getAnalysisUsage(AnalysisUsage &AU) const override {
373 ModulePass::getAnalysisUsage(AU);
374 }
375};
376
377bool isConvergenceIntrinsic(const Instruction *I) {
378 const auto *II = dyn_cast<IntrinsicInst>(Val: I);
379 if (!II)
380 return false;
381
382 return II->getIntrinsicID() == Intrinsic::experimental_convergence_entry ||
383 II->getIntrinsicID() == Intrinsic::experimental_convergence_loop ||
384 II->getIntrinsicID() == Intrinsic::experimental_convergence_anchor;
385}
386
387bool expectIgnoredInIRTranslation(const Instruction *I) {
388 const auto *II = dyn_cast<IntrinsicInst>(Val: I);
389 if (!II)
390 return false;
391 switch (II->getIntrinsicID()) {
392 case Intrinsic::invariant_start:
393 case Intrinsic::spv_resource_handlefrombinding:
394 case Intrinsic::spv_resource_getpointer:
395 return true;
396 default:
397 return false;
398 }
399}
400
401// Returns the source pointer from `I` ignoring intermediate ptrcast.
402Value *getPointerRoot(Value *I) {
403 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
404 if (II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
405 Value *V = II->getArgOperand(i: 0);
406 return getPointerRoot(I: V);
407 }
408 }
409 return I;
410}
411
412} // namespace
413
414char SPIRVEmitIntrinsics::ID = 0;
415
416INITIALIZE_PASS(SPIRVEmitIntrinsics, "spirv-emit-intrinsics",
417 "SPIRV emit intrinsics", false, false)
418
419static inline bool isAssignTypeInstr(const Instruction *I) {
420 return isa<IntrinsicInst>(Val: I) &&
421 cast<IntrinsicInst>(Val: I)->getIntrinsicID() == Intrinsic::spv_assign_type;
422}
423
424static bool isMemInstrToReplace(Instruction *I) {
425 return isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I) || isa<InsertValueInst>(Val: I) ||
426 isa<ExtractValueInst>(Val: I) || isa<AtomicCmpXchgInst>(Val: I);
427}
428
429static bool isAggrConstForceInt32(const Value *V) {
430 bool IsAggrZero =
431 isa<ConstantAggregateZero>(Val: V) && !V->getType()->isVectorTy();
432 bool IsUndefAggregate = isa<UndefValue>(Val: V) && V->getType()->isAggregateType();
433 return isa<ConstantArray>(Val: V) || isa<ConstantStruct>(Val: V) ||
434 isa<ConstantDataArray>(Val: V) || IsAggrZero || IsUndefAggregate;
435}
436
437static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
438 if (isa<PHINode>(Val: I))
439 B.SetInsertPoint(I->getParent()->getFirstNonPHIOrDbgOrAlloca());
440 else
441 B.SetInsertPoint(I);
442}
443
444static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I) {
445 B.SetCurrentDebugLocation(I->getDebugLoc());
446 if (I->getType()->isVoidTy())
447 B.SetInsertPoint(I->getNextNode());
448 else
449 B.SetInsertPoint(*I->getInsertionPointAfterDef());
450}
451
452static bool requireAssignType(Instruction *I) {
453 if (const auto *Intr = dyn_cast<IntrinsicInst>(Val: I)) {
454 switch (Intr->getIntrinsicID()) {
455 case Intrinsic::invariant_start:
456 case Intrinsic::invariant_end:
457 return false;
458 }
459 }
460 return true;
461}
462
463static inline void reportFatalOnTokenType(const Instruction *I) {
464 if (I->getType()->isTokenTy())
465 report_fatal_error(reason: "A token is encountered but SPIR-V without extensions "
466 "does not support token type",
467 gen_crash_diag: false);
468}
469
470static void emitAssignName(Instruction *I, IRBuilder<> &B) {
471 if (!I->hasName() || I->getType()->isAggregateType() ||
472 expectIgnoredInIRTranslation(I))
473 return;
474
475 // We want to be conservative when adding the names because they can interfere
476 // with later optimizations.
477 bool KeepName = SpirvEmitOpNames;
478 if (!KeepName) {
479 if (isa<AllocaInst>(Val: I)) {
480 KeepName = true;
481 } else if (auto *CI = dyn_cast<CallBase>(Val: I)) {
482 Function *F = CI->getCalledFunction();
483 if (F && F->getName().starts_with(Prefix: "llvm.spv.alloca"))
484 KeepName = true;
485 }
486 }
487
488 if (!KeepName)
489 return;
490
491 reportFatalOnTokenType(I);
492 setInsertPointAfterDef(B, I);
493 LLVMContext &Ctx = I->getContext();
494 std::vector<Value *> Args = {
495 I, MetadataAsValue::get(
496 Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: I->getName())))};
497 B.CreateIntrinsic(ID: Intrinsic::spv_assign_name, Types: {I->getType()}, Args);
498}
499
500void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest,
501 bool DeleteOld) {
502 GR->replaceAllUsesWith(Old: Src, New: Dest, DeleteOld);
503 // Update uncomplete type records if any
504 if (isTodoType(Op: Src)) {
505 if (DeleteOld)
506 eraseTodoType(Op: Src);
507 insertTodoType(Op: Dest);
508 }
509}
510
511void SPIRVEmitIntrinsics::replaceAllUsesWithAndErase(IRBuilder<> &B,
512 Instruction *Src,
513 Instruction *Dest,
514 bool DeleteOld) {
515 replaceAllUsesWith(Src, Dest, DeleteOld);
516 std::string Name = Src->hasName() ? Src->getName().str() : "";
517 Src->eraseFromParent();
518 if (!Name.empty()) {
519 Dest->setName(Name);
520 if (Named.insert(x: Dest).second)
521 emitAssignName(I: Dest, B);
522 }
523}
524
525static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
526 return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
527 isPointerTy(T: SI->getValueOperand()->getType()) &&
528 isa<Argument>(Val: SI->getValueOperand());
529}
530
531// Maybe restore original function return type.
532static inline Type *restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I,
533 Type *Ty) {
534 CallInst *CI = dyn_cast<CallInst>(Val: I);
535 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
536 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
537 return Ty;
538 if (Type *OriginalTy = GR->findMutated(Val: CI->getCalledFunction()))
539 return OriginalTy;
540 return Ty;
541}
542
543// Reconstruct type with nested element types according to deduced type info.
544// Return nullptr if no detailed type info is available.
545Type *SPIRVEmitIntrinsics::reconstructType(Value *Op, bool UnknownElemTypeI8,
546 bool IsPostprocessing) {
547 Type *Ty = Op->getType();
548 if (auto *OpI = dyn_cast<Instruction>(Val: Op))
549 Ty = restoreMutatedType(GR, I: OpI, Ty);
550 if (!isUntypedPointerTy(T: Ty))
551 return Ty;
552 // try to find the pointee type
553 if (Type *NestedTy = GR->findDeducedElementType(Val: Op))
554 return getTypedPointerWrapper(ElemTy: NestedTy, AS: getPointerAddressSpace(T: Ty));
555 // not a pointer according to the type info (e.g., Event object)
556 CallInst *CI = GR->findAssignPtrTypeInstr(Val: Op);
557 if (CI) {
558 MetadataAsValue *MD = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
559 return cast<ConstantAsMetadata>(Val: MD->getMetadata())->getType();
560 }
561 if (UnknownElemTypeI8) {
562 if (!IsPostprocessing)
563 insertTodoType(Op);
564 return getTypedPointerWrapper(ElemTy: IntegerType::getInt8Ty(C&: Op->getContext()),
565 AS: getPointerAddressSpace(T: Ty));
566 }
567 return nullptr;
568}
569
570CallInst *SPIRVEmitIntrinsics::buildSpvPtrcast(Function *F, Value *Op,
571 Type *ElemTy) {
572 IRBuilder<> B(Op->getContext());
573 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
574 // spv_ptrcast's argument Op denotes an instruction that generates
575 // a value, and we may use getInsertionPointAfterDef()
576 setInsertPointAfterDef(B, I: OpI);
577 } else if (auto *OpA = dyn_cast<Argument>(Val: Op)) {
578 B.SetInsertPointPastAllocas(OpA->getParent());
579 B.SetCurrentDebugLocation(DebugLoc());
580 } else {
581 B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
582 }
583 Type *OpTy = Op->getType();
584 SmallVector<Type *, 2> Types = {OpTy, OpTy};
585 SmallVector<Value *, 2> Args = {Op, buildMD(Arg: getNormalizedPoisonValue(Ty: ElemTy)),
586 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
587 CallInst *PtrCasted =
588 B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
589 GR->buildAssignPtr(B, ElemTy, Arg: PtrCasted);
590 return PtrCasted;
591}
592
593void SPIRVEmitIntrinsics::replaceUsesOfWithSpvPtrcast(
594 Value *Op, Type *ElemTy, Instruction *I,
595 DenseMap<Function *, CallInst *> Ptrcasts) {
596 Function *F = I->getParent()->getParent();
597 CallInst *PtrCastedI = nullptr;
598 auto It = Ptrcasts.find(Val: F);
599 if (It == Ptrcasts.end()) {
600 PtrCastedI = buildSpvPtrcast(F, Op, ElemTy);
601 Ptrcasts[F] = PtrCastedI;
602 } else {
603 PtrCastedI = It->second;
604 }
605 I->replaceUsesOfWith(From: Op, To: PtrCastedI);
606}
607
608void SPIRVEmitIntrinsics::propagateElemType(
609 Value *Op, Type *ElemTy,
610 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
611 DenseMap<Function *, CallInst *> Ptrcasts;
612 SmallVector<User *> Users(Op->users());
613 for (auto *U : Users) {
614 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
615 continue;
616 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
617 continue;
618 Instruction *UI = dyn_cast<Instruction>(Val: U);
619 // If the instruction was validated already, we need to keep it valid by
620 // keeping current Op type.
621 if (isaGEP(V: UI) || TypeValidated.find(x: UI) != TypeValidated.end())
622 replaceUsesOfWithSpvPtrcast(Op, ElemTy, I: UI, Ptrcasts);
623 }
624}
625
626void SPIRVEmitIntrinsics::propagateElemTypeRec(
627 Value *Op, Type *PtrElemTy, Type *CastElemTy,
628 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
629 std::unordered_set<Value *> Visited;
630 DenseMap<Function *, CallInst *> Ptrcasts;
631 propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited,
632 Ptrcasts: std::move(Ptrcasts));
633}
634
635void SPIRVEmitIntrinsics::propagateElemTypeRec(
636 Value *Op, Type *PtrElemTy, Type *CastElemTy,
637 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
638 std::unordered_set<Value *> &Visited,
639 DenseMap<Function *, CallInst *> Ptrcasts) {
640 if (!Visited.insert(x: Op).second)
641 return;
642 SmallVector<User *> Users(Op->users());
643 for (auto *U : Users) {
644 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
645 continue;
646 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
647 continue;
648 Instruction *UI = dyn_cast<Instruction>(Val: U);
649 // If the instruction was validated already, we need to keep it valid by
650 // keeping current Op type.
651 if (isaGEP(V: UI) || TypeValidated.find(x: UI) != TypeValidated.end())
652 replaceUsesOfWithSpvPtrcast(Op, ElemTy: CastElemTy, I: UI, Ptrcasts);
653 }
654}
655
656// Set element pointer type to the given value of ValueTy and tries to
657// specify this type further (recursively) by Operand value, if needed.
658
659Type *
660SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
661 bool UnknownElemTypeI8) {
662 std::unordered_set<Value *> Visited;
663 return deduceElementTypeByValueDeep(ValueTy, Operand, Visited,
664 UnknownElemTypeI8);
665}
666
667Type *SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(
668 Type *ValueTy, Value *Operand, std::unordered_set<Value *> &Visited,
669 bool UnknownElemTypeI8) {
670 Type *Ty = ValueTy;
671 if (Operand) {
672 if (auto *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
673 if (Type *NestedTy =
674 deduceElementTypeHelper(I: Operand, Visited, UnknownElemTypeI8))
675 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
676 } else {
677 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Operand), Ty, Visited,
678 UnknownElemTypeI8);
679 }
680 }
681 return Ty;
682}
683
684// Traverse User instructions to deduce an element pointer type of the operand.
685Type *SPIRVEmitIntrinsics::deduceElementTypeByUsersDeep(
686 Value *Op, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8) {
687 if (!Op || !isPointerTy(T: Op->getType()) || isa<ConstantPointerNull>(Val: Op) ||
688 isa<UndefValue>(Val: Op))
689 return nullptr;
690
691 if (auto ElemTy = getPointeeType(Ty: Op->getType()))
692 return ElemTy;
693
694 // maybe we already know operand's element type
695 if (Type *KnownTy = GR->findDeducedElementType(Val: Op))
696 return KnownTy;
697
698 for (User *OpU : Op->users()) {
699 if (Instruction *Inst = dyn_cast<Instruction>(Val: OpU)) {
700 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8))
701 return Ty;
702 }
703 }
704 return nullptr;
705}
706
707// Implements what we know in advance about intrinsics and builtin calls
708// TODO: consider feasibility of this particular case to be generalized by
709// encoding knowledge about intrinsics and builtin calls by corresponding
710// specification rules
711static Type *getPointeeTypeByCallInst(StringRef DemangledName,
712 Function *CalledF, unsigned OpIdx) {
713 if ((DemangledName.starts_with(Prefix: "__spirv_ocl_printf(") ||
714 DemangledName.starts_with(Prefix: "printf(")) &&
715 OpIdx == 0)
716 return IntegerType::getInt8Ty(C&: CalledF->getContext());
717 return nullptr;
718}
719
720// Deduce and return a successfully deduced Type of the Instruction,
721// or nullptr otherwise.
722Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(Value *I,
723 bool UnknownElemTypeI8) {
724 std::unordered_set<Value *> Visited;
725 return deduceElementTypeHelper(I, Visited, UnknownElemTypeI8);
726}
727
728void SPIRVEmitIntrinsics::maybeAssignPtrType(Type *&Ty, Value *Op, Type *RefTy,
729 bool UnknownElemTypeI8) {
730 if (isUntypedPointerTy(T: RefTy)) {
731 if (!UnknownElemTypeI8)
732 return;
733 insertTodoType(Op);
734 }
735 Ty = RefTy;
736}
737
738bool SPIRVEmitIntrinsics::walkLogicalAccessChain(
739 GetElementPtrInst &GEP,
740 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
741 const std::function<void(Type *, Value *)> &OnDynamicIndexing) {
742 // We only rewrite i8* GEP. Other should be left as-is.
743 // Valid i8* GEP must always have a single index.
744 assert(GEP.getSourceElementType() ==
745 IntegerType::getInt8Ty(CurrF->getContext()));
746 assert(GEP.getNumIndices() == 1);
747
748 auto &DL = CurrF->getDataLayout();
749 Value *Src = getPointerRoot(I: GEP.getPointerOperand());
750 Type *CurType = deduceElementType(I: Src, UnknownElemTypeI8: true);
751
752 Value *Operand = *GEP.idx_begin();
753 ConstantInt *CI = dyn_cast<ConstantInt>(Val: Operand);
754 if (!CI) {
755 ArrayType *AT = dyn_cast<ArrayType>(Val: CurType);
756 // Operand is not constant. Either we have an array and accept it, or we
757 // give up.
758 if (AT)
759 OnDynamicIndexing(AT->getElementType(), Operand);
760 return AT == nullptr;
761 }
762
763 assert(CI);
764 uint64_t Offset = CI->getZExtValue();
765
766 do {
767 if (ArrayType *AT = dyn_cast<ArrayType>(Val: CurType)) {
768 uint32_t EltTypeSize = DL.getTypeSizeInBits(Ty: AT->getElementType()) / 8;
769 assert(Offset < AT->getNumElements() * EltTypeSize);
770 uint64_t Index = Offset / EltTypeSize;
771 Offset = Offset - (Index * EltTypeSize);
772 CurType = AT->getElementType();
773 OnLiteralIndexing(CurType, Index);
774 } else if (StructType *ST = dyn_cast<StructType>(Val: CurType)) {
775 uint32_t StructSize = DL.getTypeSizeInBits(Ty: ST) / 8;
776 assert(Offset < StructSize);
777 (void)StructSize;
778 const auto &STL = DL.getStructLayout(Ty: ST);
779 unsigned Element = STL->getElementContainingOffset(FixedOffset: Offset);
780 Offset -= STL->getElementOffset(Idx: Element);
781 CurType = ST->getElementType(N: Element);
782 OnLiteralIndexing(CurType, Element);
783 } else if (auto *VT = dyn_cast<FixedVectorType>(Val: CurType)) {
784 Type *EltTy = VT->getElementType();
785 TypeSize EltSizeBits = DL.getTypeSizeInBits(Ty: EltTy);
786 assert(EltSizeBits % 8 == 0 &&
787 "Element type size in bits must be a multiple of 8.");
788 uint32_t EltTypeSize = EltSizeBits / 8;
789 assert(Offset < VT->getNumElements() * EltTypeSize);
790 uint64_t Index = Offset / EltTypeSize;
791 Offset -= Index * EltTypeSize;
792 CurType = EltTy;
793 OnLiteralIndexing(CurType, Index);
794
795 } else {
796 // Unknown composite kind; give up.
797 return true;
798 }
799 } while (Offset > 0);
800
801 return false;
802}
803
804Instruction *
805SPIRVEmitIntrinsics::buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP) {
806 auto &DL = CurrF->getDataLayout();
807 IRBuilder<> B(GEP.getParent());
808 B.SetInsertPoint(&GEP);
809
810 std::vector<Value *> Indices;
811 Indices.push_back(x: ConstantInt::get(
812 Ty: IntegerType::getInt32Ty(C&: CurrF->getContext()), V: 0, /* Signed= */ IsSigned: false));
813 walkLogicalAccessChain(
814 GEP,
815 OnLiteralIndexing: [&Indices, &B](Type *EltType, uint64_t Index) {
816 Indices.push_back(
817 x: ConstantInt::get(Ty: B.getInt64Ty(), V: Index, /* Signed= */ IsSigned: false));
818 },
819 OnDynamicIndexing: [&Indices, &B, &DL](Type *EltType, Value *Offset) {
820 uint32_t EltTypeSize = DL.getTypeSizeInBits(Ty: EltType) / 8;
821 Value *Index = B.CreateUDiv(
822 LHS: Offset, RHS: ConstantInt::get(Ty: Offset->getType(), V: EltTypeSize,
823 /* Signed= */ IsSigned: false));
824 Indices.push_back(x: Index);
825 });
826
827 SmallVector<Type *, 2> Types = {GEP.getType(), GEP.getOperand(i_nocapture: 0)->getType()};
828 SmallVector<Value *, 4> Args;
829 Args.push_back(Elt: B.getInt1(V: GEP.isInBounds()));
830 Args.push_back(Elt: GEP.getOperand(i_nocapture: 0));
831 llvm::append_range(C&: Args, R&: Indices);
832 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types: {Types}, Args: {Args});
833 replaceAllUsesWithAndErase(B, Src: &GEP, Dest: NewI);
834 return NewI;
835}
836
837Type *SPIRVEmitIntrinsics::getGEPTypeLogical(GetElementPtrInst *GEP) {
838
839 Type *CurType = GEP->getResultElementType();
840
841 bool Interrupted = walkLogicalAccessChain(
842 GEP&: *GEP, OnLiteralIndexing: [&CurType](Type *EltType, uint64_t Index) { CurType = EltType; },
843 OnDynamicIndexing: [&CurType](Type *EltType, Value *Index) { CurType = EltType; });
844
845 return Interrupted ? GEP->getResultElementType() : CurType;
846}
847
848Type *SPIRVEmitIntrinsics::getGEPType(GetElementPtrInst *Ref) {
849 if (Ref->getSourceElementType() ==
850 IntegerType::getInt8Ty(C&: CurrF->getContext()) &&
851 TM.getSubtargetImpl()->isLogicalSPIRV()) {
852 return getGEPTypeLogical(GEP: Ref);
853 }
854
855 Type *Ty = nullptr;
856 // TODO: not sure if GetElementPtrInst::getTypeAtIndex() does anything
857 // useful here
858 if (isNestedPointer(Ty: Ref->getSourceElementType())) {
859 Ty = Ref->getSourceElementType();
860 for (Use &U : drop_begin(RangeOrContainer: Ref->indices()))
861 Ty = GetElementPtrInst::getTypeAtIndex(Ty, Idx: U.get());
862 } else {
863 Ty = Ref->getResultElementType();
864 }
865 return Ty;
866}
867
868Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
869 Value *I, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8,
870 bool IgnoreKnownType) {
871 // allow to pass nullptr as an argument
872 if (!I)
873 return nullptr;
874
875 // maybe already known
876 if (!IgnoreKnownType)
877 if (Type *KnownTy = GR->findDeducedElementType(Val: I))
878 return KnownTy;
879
880 // maybe a cycle
881 if (!Visited.insert(x: I).second)
882 return nullptr;
883
884 // fallback value in case when we fail to deduce a type
885 Type *Ty = nullptr;
886 // look for known basic patterns of type inference
887 if (auto *Ref = dyn_cast<AllocaInst>(Val: I)) {
888 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getAllocatedType(), UnknownElemTypeI8);
889 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
890 Ty = getGEPType(Ref);
891 } else if (auto *SGEP = dyn_cast<StructuredGEPInst>(Val: I)) {
892 Ty = SGEP->getResultElementType();
893 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
894 Value *Op = Ref->getPointerOperand();
895 Type *KnownTy = GR->findDeducedElementType(Val: Op);
896 if (!KnownTy)
897 KnownTy = Op->getType();
898 if (Type *ElemTy = getPointeeType(Ty: KnownTy))
899 maybeAssignPtrType(Ty, Op: I, RefTy: ElemTy, UnknownElemTypeI8);
900 } else if (auto *Ref = dyn_cast<GlobalValue>(Val: I)) {
901 if (auto *Fn = dyn_cast<Function>(Val: Ref)) {
902 Ty = SPIRV::getOriginalFunctionType(F: *Fn);
903 GR->addDeducedElementType(Val: I, Ty);
904 } else {
905 Ty = deduceElementTypeByValueDeep(
906 ValueTy: Ref->getValueType(),
907 Operand: Ref->getNumOperands() > 0 ? Ref->getOperand(i: 0) : nullptr, Visited,
908 UnknownElemTypeI8);
909 }
910 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
911 Type *RefTy = deduceElementTypeHelper(I: Ref->getPointerOperand(), Visited,
912 UnknownElemTypeI8);
913 maybeAssignPtrType(Ty, Op: I, RefTy, UnknownElemTypeI8);
914 } else if (auto *Ref = dyn_cast<IntToPtrInst>(Val: I)) {
915 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getDestTy(), UnknownElemTypeI8);
916 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
917 if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
918 isPointerTy(T: Src) && isPointerTy(T: Dest))
919 Ty = deduceElementTypeHelper(I: Ref->getOperand(i_nocapture: 0), Visited,
920 UnknownElemTypeI8);
921 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
922 Value *Op = Ref->getNewValOperand();
923 if (isPointerTy(T: Op->getType()))
924 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
925 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
926 Value *Op = Ref->getValOperand();
927 if (isPointerTy(T: Op->getType()))
928 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
929 } else if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
930 Type *BestTy = nullptr;
931 unsigned MaxN = 1;
932 DenseMap<Type *, unsigned> PhiTys;
933 for (int i = Ref->getNumIncomingValues() - 1; i >= 0; --i) {
934 Ty = deduceElementTypeByUsersDeep(Op: Ref->getIncomingValue(i), Visited,
935 UnknownElemTypeI8);
936 if (!Ty)
937 continue;
938 auto It = PhiTys.try_emplace(Key: Ty, Args: 1);
939 if (!It.second) {
940 ++It.first->second;
941 if (It.first->second > MaxN) {
942 MaxN = It.first->second;
943 BestTy = Ty;
944 }
945 }
946 }
947 if (BestTy)
948 Ty = BestTy;
949 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
950 for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) {
951 Ty = deduceElementTypeByUsersDeep(Op, Visited, UnknownElemTypeI8);
952 if (Ty)
953 break;
954 }
955 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
956 static StringMap<unsigned> ResTypeByArg = {
957 {"to_global", 0},
958 {"to_local", 0},
959 {"to_private", 0},
960 {"__spirv_GenericCastToPtr_ToGlobal", 0},
961 {"__spirv_GenericCastToPtr_ToLocal", 0},
962 {"__spirv_GenericCastToPtr_ToPrivate", 0},
963 {"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
964 {"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
965 {"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
966 // TODO: maybe improve performance by caching demangled names
967
968 auto *II = dyn_cast<IntrinsicInst>(Val: I);
969 if (II && II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
970 auto *HandleType = cast<TargetExtType>(Val: II->getOperand(i_nocapture: 0)->getType());
971 if (HandleType->getTargetExtName() == "spirv.Image" ||
972 HandleType->getTargetExtName() == "spirv.SignedImage") {
973 for (User *U : II->users()) {
974 Ty = cast<Instruction>(Val: U)->getAccessType();
975 if (Ty)
976 break;
977 }
978 } else if (HandleType->getTargetExtName() == "spirv.VulkanBuffer") {
979 // This call is supposed to index into an array
980 Ty = HandleType->getTypeParameter(i: 0);
981 if (Ty->isArrayTy())
982 Ty = Ty->getArrayElementType();
983 else {
984 assert(Ty && Ty->isStructTy());
985 uint32_t Index = cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1))->getZExtValue();
986 Ty = cast<StructType>(Val: Ty)->getElementType(N: Index);
987 }
988 Ty = reconstitutePeeledArrayType(Ty);
989 } else {
990 llvm_unreachable("Unknown handle type for spv_resource_getpointer.");
991 }
992 } else if (II && II->getIntrinsicID() ==
993 Intrinsic::spv_generic_cast_to_ptr_explicit) {
994 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: 0), Visited,
995 UnknownElemTypeI8);
996 } else if (Function *CalledF = CI->getCalledFunction()) {
997 std::string DemangledName =
998 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
999 if (DemangledName.length() > 0)
1000 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName);
1001 auto AsArgIt = ResTypeByArg.find(Key: DemangledName);
1002 if (AsArgIt != ResTypeByArg.end())
1003 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: AsArgIt->second),
1004 Visited, UnknownElemTypeI8);
1005 else if (Type *KnownRetTy = GR->findDeducedElementType(Val: CalledF))
1006 Ty = KnownRetTy;
1007 }
1008 }
1009
1010 // remember the found relationship
1011 if (Ty && !IgnoreKnownType) {
1012 // specify nested types if needed, otherwise return unchanged
1013 GR->addDeducedElementType(Val: I, Ty: normalizeType(Ty));
1014 }
1015
1016 return Ty;
1017}
1018
1019// Re-create a type of the value if it has untyped pointer fields, also nested.
1020// Return the original value type if no corrections of untyped pointer
1021// information is found or needed.
1022Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(User *U,
1023 bool UnknownElemTypeI8) {
1024 std::unordered_set<Value *> Visited;
1025 return deduceNestedTypeHelper(U, Ty: U->getType(), Visited, UnknownElemTypeI8);
1026}
1027
1028Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
1029 User *U, Type *OrigTy, std::unordered_set<Value *> &Visited,
1030 bool UnknownElemTypeI8) {
1031 if (!U)
1032 return OrigTy;
1033
1034 // maybe already known
1035 if (Type *KnownTy = GR->findDeducedCompositeType(Val: U))
1036 return KnownTy;
1037
1038 // maybe a cycle
1039 if (!Visited.insert(x: U).second)
1040 return OrigTy;
1041
1042 if (isa<StructType>(Val: OrigTy)) {
1043 SmallVector<Type *> Tys;
1044 bool Change = false;
1045 for (unsigned i = 0; i < U->getNumOperands(); ++i) {
1046 Value *Op = U->getOperand(i);
1047 assert(Op && "Operands should not be null.");
1048 Type *OpTy = Op->getType();
1049 Type *Ty = OpTy;
1050 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1051 if (Type *NestedTy =
1052 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1053 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1054 } else {
1055 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1056 UnknownElemTypeI8);
1057 }
1058 Tys.push_back(Elt: Ty);
1059 Change |= Ty != OpTy;
1060 }
1061 if (Change) {
1062 Type *NewTy = StructType::create(Elements: Tys);
1063 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1064 return NewTy;
1065 }
1066 } else if (auto *ArrTy = dyn_cast<ArrayType>(Val: OrigTy)) {
1067 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1068 Type *OpTy = ArrTy->getElementType();
1069 Type *Ty = OpTy;
1070 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1071 if (Type *NestedTy =
1072 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1073 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1074 } else {
1075 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1076 UnknownElemTypeI8);
1077 }
1078 if (Ty != OpTy) {
1079 Type *NewTy = ArrayType::get(ElementType: Ty, NumElements: ArrTy->getNumElements());
1080 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1081 return NewTy;
1082 }
1083 }
1084 } else if (auto *VecTy = dyn_cast<VectorType>(Val: OrigTy)) {
1085 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1086 Type *OpTy = VecTy->getElementType();
1087 Type *Ty = OpTy;
1088 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1089 if (Type *NestedTy =
1090 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1091 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1092 } else {
1093 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1094 UnknownElemTypeI8);
1095 }
1096 if (Ty != OpTy) {
1097 Type *NewTy = VectorType::get(ElementType: Ty, EC: VecTy->getElementCount());
1098 GR->addDeducedCompositeType(Val: U, Ty: normalizeType(Ty: NewTy));
1099 return NewTy;
1100 }
1101 }
1102 }
1103
1104 return OrigTy;
1105}
1106
1107Type *SPIRVEmitIntrinsics::deduceElementType(Value *I, bool UnknownElemTypeI8) {
1108 if (Type *Ty = deduceElementTypeHelper(I, UnknownElemTypeI8))
1109 return Ty;
1110 if (!UnknownElemTypeI8)
1111 return nullptr;
1112 insertTodoType(Op: I);
1113 return IntegerType::getInt8Ty(C&: I->getContext());
1114}
1115
1116static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I,
1117 Value *PointerOperand) {
1118 Type *PointeeTy = GR->findDeducedElementType(Val: PointerOperand);
1119 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1120 return nullptr;
1121 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType());
1122 if (!PtrTy)
1123 return I->getType();
1124 if (Type *NestedTy = GR->findDeducedElementType(Val: I))
1125 return getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1126 return nullptr;
1127}
1128
1129// Try to deduce element type for a call base. Returns false if this is an
1130// indirect function invocation, and true otherwise.
1131bool SPIRVEmitIntrinsics::deduceOperandElementTypeCalledFunction(
1132 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1133 Type *&KnownElemTy, bool &Incomplete) {
1134 Function *CalledF = CI->getCalledFunction();
1135 if (!CalledF)
1136 return false;
1137 std::string DemangledName =
1138 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1139 if (DemangledName.length() > 0 &&
1140 !StringRef(DemangledName).starts_with(Prefix: "llvm.")) {
1141 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *CalledF);
1142 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
1143 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
1144 if (Opcode == SPIRV::OpGroupAsyncCopy) {
1145 for (unsigned i = 0, PtrCnt = 0; i < CI->arg_size() && PtrCnt < 2; ++i) {
1146 Value *Op = CI->getArgOperand(i);
1147 if (!isPointerTy(T: Op->getType()))
1148 continue;
1149 ++PtrCnt;
1150 if (Type *ElemTy = GR->findDeducedElementType(Val: Op))
1151 KnownElemTy = ElemTy; // src will rewrite dest if both are defined
1152 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1153 }
1154 } else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) {
1155 if (CI->arg_size() == 0)
1156 return true;
1157 Value *Op = CI->getArgOperand(i: 0);
1158 if (!isPointerTy(T: Op->getType()))
1159 return true;
1160 switch (Opcode) {
1161 case SPIRV::OpAtomicFAddEXT:
1162 case SPIRV::OpAtomicFMinEXT:
1163 case SPIRV::OpAtomicFMaxEXT:
1164 case SPIRV::OpAtomicLoad:
1165 case SPIRV::OpAtomicCompareExchangeWeak:
1166 case SPIRV::OpAtomicCompareExchange:
1167 case SPIRV::OpAtomicExchange:
1168 case SPIRV::OpAtomicIAdd:
1169 case SPIRV::OpAtomicISub:
1170 case SPIRV::OpAtomicOr:
1171 case SPIRV::OpAtomicXor:
1172 case SPIRV::OpAtomicAnd:
1173 case SPIRV::OpAtomicUMin:
1174 case SPIRV::OpAtomicUMax:
1175 case SPIRV::OpAtomicSMin:
1176 case SPIRV::OpAtomicSMax: {
1177 KnownElemTy = isPointerTy(T: CI->getType()) ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1178 : CI->getType();
1179 if (!KnownElemTy)
1180 return true;
1181 Incomplete = isTodoType(Op);
1182 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1183 } break;
1184 case SPIRV::OpAtomicStore: {
1185 if (CI->arg_size() < 4)
1186 return true;
1187 Value *ValOp = CI->getArgOperand(i: 3);
1188 KnownElemTy = isPointerTy(T: ValOp->getType())
1189 ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1190 : ValOp->getType();
1191 if (!KnownElemTy)
1192 return true;
1193 Incomplete = isTodoType(Op);
1194 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1195 } break;
1196 }
1197 }
1198 }
1199 return true;
1200}
1201
1202// Try to deduce element type for a function pointer.
1203void SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionPointer(
1204 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1205 Type *&KnownElemTy, bool IsPostprocessing) {
1206 Value *Op = CI->getCalledOperand();
1207 if (!Op || !isPointerTy(T: Op->getType()))
1208 return;
1209 Ops.push_back(Elt: std::make_pair(x&: Op, y: std::numeric_limits<unsigned>::max()));
1210 FunctionType *FTy = SPIRV::getOriginalFunctionType(CB: *CI);
1211 bool IsNewFTy = false, IsIncomplete = false;
1212 SmallVector<Type *, 4> ArgTys;
1213 for (auto &&[ParmIdx, Arg] : llvm::enumerate(First: CI->args())) {
1214 Type *ArgTy = Arg->getType();
1215 if (ArgTy->isPointerTy()) {
1216 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg)) {
1217 IsNewFTy = true;
1218 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
1219 if (isTodoType(Op: Arg))
1220 IsIncomplete = true;
1221 } else {
1222 IsIncomplete = true;
1223 }
1224 } else {
1225 ArgTy = FTy->getFunctionParamType(i: ParmIdx);
1226 }
1227 ArgTys.push_back(Elt: ArgTy);
1228 }
1229 Type *RetTy = FTy->getReturnType();
1230 if (CI->getType()->isPointerTy()) {
1231 if (Type *ElemTy = GR->findDeducedElementType(Val: CI)) {
1232 IsNewFTy = true;
1233 RetTy =
1234 getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: CI->getType()));
1235 if (isTodoType(Op: CI))
1236 IsIncomplete = true;
1237 } else {
1238 IsIncomplete = true;
1239 }
1240 }
1241 if (!IsPostprocessing && IsIncomplete)
1242 insertTodoType(Op);
1243 KnownElemTy =
1244 IsNewFTy ? FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: FTy->isVarArg()) : FTy;
1245}
1246
1247bool SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionRet(
1248 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
1249 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing,
1250 Type *&KnownElemTy, Value *Op, Function *F) {
1251 KnownElemTy = GR->findDeducedElementType(Val: F);
1252 if (KnownElemTy)
1253 return false;
1254 if (Type *OpElemTy = GR->findDeducedElementType(Val: Op)) {
1255 OpElemTy = normalizeType(Ty: OpElemTy);
1256 GR->addDeducedElementType(Val: F, Ty: OpElemTy);
1257 GR->addReturnType(
1258 ArgF: F, DerivedTy: TypedPointerType::get(ElementType: OpElemTy,
1259 AddressSpace: getPointerAddressSpace(T: F->getReturnType())));
1260 // non-recursive update of types in function uses
1261 DenseSet<std::pair<Value *, Value *>> VisitedSubst{std::make_pair(x&: I, y&: Op)};
1262 for (User *U : F->users()) {
1263 CallInst *CI = dyn_cast<CallInst>(Val: U);
1264 if (!CI || CI->getCalledFunction() != F)
1265 continue;
1266 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: CI)) {
1267 if (Type *PrevElemTy = GR->findDeducedElementType(Val: CI)) {
1268 GR->updateAssignType(AssignCI, Arg: CI,
1269 OfType: getNormalizedPoisonValue(Ty: OpElemTy));
1270 propagateElemType(Op: CI, ElemTy: PrevElemTy, VisitedSubst);
1271 }
1272 }
1273 }
1274 // Non-recursive update of types in the function uncomplete returns.
1275 // This may happen just once per a function, the latch is a pair of
1276 // findDeducedElementType(F) / addDeducedElementType(F, ...).
1277 // With or without the latch it is a non-recursive call due to
1278 // IncompleteRets set to nullptr in this call.
1279 if (IncompleteRets)
1280 for (Instruction *IncompleteRetI : *IncompleteRets)
1281 deduceOperandElementType(I: IncompleteRetI, IncompleteRets: nullptr, AskOps,
1282 IsPostprocessing);
1283 } else if (IncompleteRets) {
1284 IncompleteRets->insert(Ptr: I);
1285 }
1286 TypeValidated.insert(x: I);
1287 return true;
1288}
1289
1290// If the Instruction has Pointer operands with unresolved types, this function
1291// tries to deduce them. If the Instruction has Pointer operands with known
1292// types which differ from expected, this function tries to insert a bitcast to
1293// resolve the issue.
1294void SPIRVEmitIntrinsics::deduceOperandElementType(
1295 Instruction *I, SmallPtrSet<Instruction *, 4> *IncompleteRets,
1296 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing) {
1297 SmallVector<std::pair<Value *, unsigned>> Ops;
1298 Type *KnownElemTy = nullptr;
1299 bool Incomplete = false;
1300 // look for known basic patterns of type inference
1301 if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1302 if (!isPointerTy(T: I->getType()) ||
1303 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1304 return;
1305 Incomplete = isTodoType(Op: I);
1306 for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
1307 Value *Op = Ref->getIncomingValue(i);
1308 if (isPointerTy(T: Op->getType()))
1309 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1310 }
1311 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1312 KnownElemTy = GR->findDeducedElementType(Val: I);
1313 if (!KnownElemTy)
1314 return;
1315 Incomplete = isTodoType(Op: I);
1316 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(), y: 0));
1317 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1318 if (!isPointerTy(T: I->getType()))
1319 return;
1320 KnownElemTy = GR->findDeducedElementType(Val: I);
1321 if (!KnownElemTy)
1322 return;
1323 Incomplete = isTodoType(Op: I);
1324 Ops.push_back(Elt: std::make_pair(x: Ref->getOperand(i_nocapture: 0), y: 0));
1325 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
1326 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1327 return;
1328 KnownElemTy = Ref->getSourceElementType();
1329 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1330 y: GetElementPtrInst::getPointerOperandIndex()));
1331 } else if (auto *Ref = dyn_cast<StructuredGEPInst>(Val: I)) {
1332 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1333 return;
1334 KnownElemTy = Ref->getBaseType();
1335 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1336 y: StructuredGEPInst::getPointerOperandIndex()));
1337 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
1338 KnownElemTy = I->getType();
1339 if (isUntypedPointerTy(T: KnownElemTy))
1340 return;
1341 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1342 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1343 return;
1344 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1345 y: LoadInst::getPointerOperandIndex()));
1346 } else if (auto *Ref = dyn_cast<StoreInst>(Val: I)) {
1347 if (!(KnownElemTy =
1348 reconstructType(Op: Ref->getValueOperand(), UnknownElemTypeI8: false, IsPostprocessing)))
1349 return;
1350 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1351 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1352 return;
1353 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1354 y: StoreInst::getPointerOperandIndex()));
1355 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1356 KnownElemTy = isPointerTy(T: I->getType())
1357 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1358 : I->getType();
1359 if (!KnownElemTy)
1360 return;
1361 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1362 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1363 y: AtomicCmpXchgInst::getPointerOperandIndex()));
1364 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1365 KnownElemTy = isPointerTy(T: I->getType())
1366 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1367 : I->getType();
1368 if (!KnownElemTy)
1369 return;
1370 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1371 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1372 y: AtomicRMWInst::getPointerOperandIndex()));
1373 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1374 if (!isPointerTy(T: I->getType()) ||
1375 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1376 return;
1377 Incomplete = isTodoType(Op: I);
1378 for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
1379 Value *Op = Ref->getOperand(i_nocapture: i);
1380 if (isPointerTy(T: Op->getType()))
1381 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1382 }
1383 } else if (auto *Ref = dyn_cast<ReturnInst>(Val: I)) {
1384 if (!isPointerTy(T: CurrF->getReturnType()))
1385 return;
1386 Value *Op = Ref->getReturnValue();
1387 if (!Op)
1388 return;
1389 if (deduceOperandElementTypeFunctionRet(I, IncompleteRets, AskOps,
1390 IsPostprocessing, KnownElemTy, Op,
1391 F: CurrF))
1392 return;
1393 Incomplete = isTodoType(Op: CurrF);
1394 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1395 } else if (auto *Ref = dyn_cast<ICmpInst>(Val: I)) {
1396 if (!isPointerTy(T: Ref->getOperand(i_nocapture: 0)->getType()))
1397 return;
1398 Value *Op0 = Ref->getOperand(i_nocapture: 0);
1399 Value *Op1 = Ref->getOperand(i_nocapture: 1);
1400 bool Incomplete0 = isTodoType(Op: Op0);
1401 bool Incomplete1 = isTodoType(Op: Op1);
1402 Type *ElemTy1 = GR->findDeducedElementType(Val: Op1);
1403 Type *ElemTy0 = (Incomplete0 && !Incomplete1 && ElemTy1)
1404 ? nullptr
1405 : GR->findDeducedElementType(Val: Op0);
1406 if (ElemTy0) {
1407 KnownElemTy = ElemTy0;
1408 Incomplete = Incomplete0;
1409 Ops.push_back(Elt: std::make_pair(x&: Op1, y: 1));
1410 } else if (ElemTy1) {
1411 KnownElemTy = ElemTy1;
1412 Incomplete = Incomplete1;
1413 Ops.push_back(Elt: std::make_pair(x&: Op0, y: 0));
1414 }
1415 } else if (CallInst *CI = dyn_cast<CallInst>(Val: I)) {
1416 if (!CI->isIndirectCall())
1417 deduceOperandElementTypeCalledFunction(CI, Ops, KnownElemTy, Incomplete);
1418 else if (HaveFunPtrs)
1419 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy,
1420 IsPostprocessing);
1421 }
1422
1423 // There is no enough info to deduce types or all is valid.
1424 if (!KnownElemTy || Ops.size() == 0)
1425 return;
1426
1427 LLVMContext &Ctx = CurrF->getContext();
1428 IRBuilder<> B(Ctx);
1429 for (auto &OpIt : Ops) {
1430 Value *Op = OpIt.first;
1431 if (AskOps && !AskOps->contains(Ptr: Op))
1432 continue;
1433 Type *AskTy = nullptr;
1434 CallInst *AskCI = nullptr;
1435 if (IsPostprocessing && AskOps) {
1436 AskTy = GR->findDeducedElementType(Val: Op);
1437 AskCI = GR->findAssignPtrTypeInstr(Val: Op);
1438 assert(AskTy && AskCI);
1439 }
1440 Type *Ty = AskTy ? AskTy : GR->findDeducedElementType(Val: Op);
1441 if (Ty == KnownElemTy)
1442 continue;
1443 Value *OpTyVal = getNormalizedPoisonValue(Ty: KnownElemTy);
1444 Type *OpTy = Op->getType();
1445 if (Op->hasUseList() &&
1446 (!Ty || AskTy || isUntypedPointerTy(T: Ty) || isTodoType(Op))) {
1447 Type *PrevElemTy = GR->findDeducedElementType(Val: Op);
1448 GR->addDeducedElementType(Val: Op, Ty: normalizeType(Ty: KnownElemTy));
1449 // check if KnownElemTy is complete
1450 if (!Incomplete)
1451 eraseTodoType(Op);
1452 else if (!IsPostprocessing)
1453 insertTodoType(Op);
1454 // check if there is existing Intrinsic::spv_assign_ptr_type instruction
1455 CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Val: Op);
1456 if (AssignCI == nullptr) {
1457 Instruction *User = dyn_cast<Instruction>(Val: Op->use_begin()->get());
1458 setInsertPointSkippingPhis(B, I: User ? User->getNextNode() : I);
1459 CallInst *CI =
1460 buildIntrWithMD(IntrID: Intrinsic::spv_assign_ptr_type, Types: {OpTy}, Arg: OpTyVal, Arg2: Op,
1461 Imms: {B.getInt32(C: getPointerAddressSpace(T: OpTy))}, B);
1462 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: CI);
1463 } else {
1464 GR->updateAssignType(AssignCI, Arg: Op, OfType: OpTyVal);
1465 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1466 std::make_pair(x&: I, y&: Op)};
1467 propagateElemTypeRec(Op, PtrElemTy: KnownElemTy, CastElemTy: PrevElemTy, VisitedSubst);
1468 }
1469 } else {
1470 eraseTodoType(Op);
1471 CallInst *PtrCastI =
1472 buildSpvPtrcast(F: I->getParent()->getParent(), Op, ElemTy: KnownElemTy);
1473 if (OpIt.second == std::numeric_limits<unsigned>::max())
1474 dyn_cast<CallInst>(Val: I)->setCalledOperand(PtrCastI);
1475 else
1476 I->setOperand(i: OpIt.second, Val: PtrCastI);
1477 }
1478 }
1479 TypeValidated.insert(x: I);
1480}
1481
1482void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
1483 Instruction *New,
1484 IRBuilder<> &B) {
1485 while (!Old->user_empty()) {
1486 auto *U = Old->user_back();
1487 if (isAssignTypeInstr(I: U)) {
1488 B.SetInsertPoint(U);
1489 SmallVector<Value *, 2> Args = {New, U->getOperand(i: 1)};
1490 CallInst *AssignCI =
1491 B.CreateIntrinsic(ID: Intrinsic::spv_assign_type, Types: {New->getType()}, Args);
1492 GR->addAssignPtrTypeInstr(Val: New, AssignPtrTyCI: AssignCI);
1493 U->eraseFromParent();
1494 } else if (isMemInstrToReplace(I: U) || isa<ReturnInst>(Val: U) ||
1495 isa<CallInst>(Val: U)) {
1496 U->replaceUsesOfWith(From: Old, To: New);
1497 } else if (auto *Phi = dyn_cast<PHINode>(Val: U)) {
1498 if (Phi->getType() != New->getType()) {
1499 Phi->mutateType(Ty: New->getType());
1500 Phi->replaceUsesOfWith(From: Old, To: New);
1501 // Convert extractvalue users of the mutated PHI to spv_extractv
1502 SmallVector<ExtractValueInst *, 4> EVUsers;
1503 for (User *PhiUser : Phi->users())
1504 if (auto *EV = dyn_cast<ExtractValueInst>(Val: PhiUser))
1505 EVUsers.push_back(Elt: EV);
1506 for (ExtractValueInst *EV : EVUsers) {
1507 B.SetInsertPoint(EV);
1508 SmallVector<Value *> Args(EV->operand_values());
1509 for (unsigned Idx : EV->indices())
1510 Args.push_back(Elt: B.getInt32(C: Idx));
1511 auto *NewEV =
1512 B.CreateIntrinsic(ID: Intrinsic::spv_extractv, Types: {EV->getType()}, Args);
1513 EV->replaceAllUsesWith(V: NewEV);
1514 DeletedInstrs.insert(Ptr: EV);
1515 EV->eraseFromParent();
1516 }
1517 } else {
1518 Phi->replaceUsesOfWith(From: Old, To: New);
1519 }
1520 } else {
1521 llvm_unreachable("illegal aggregate intrinsic user");
1522 }
1523 }
1524 New->copyMetadata(SrcInst: *Old);
1525 Old->eraseFromParent();
1526}
1527
1528void SPIRVEmitIntrinsics::preprocessUndefs(IRBuilder<> &B) {
1529 std::queue<Instruction *> Worklist;
1530 for (auto &I : instructions(F: CurrF))
1531 Worklist.push(x: &I);
1532
1533 while (!Worklist.empty()) {
1534 Instruction *I = Worklist.front();
1535 bool BPrepared = false;
1536 Worklist.pop();
1537
1538 for (auto &Op : I->operands()) {
1539 auto *AggrUndef = dyn_cast<UndefValue>(Val&: Op);
1540 if (!AggrUndef || !Op->getType()->isAggregateType())
1541 continue;
1542
1543 if (!BPrepared) {
1544 setInsertPointSkippingPhis(B, I);
1545 BPrepared = true;
1546 }
1547 auto *IntrUndef = B.CreateIntrinsic(ID: Intrinsic::spv_undef, Args: {});
1548 Worklist.push(x: IntrUndef);
1549 I->replaceUsesOfWith(From: Op, To: IntrUndef);
1550 AggrConsts[IntrUndef] = AggrUndef;
1551 AggrConstTypes[IntrUndef] = AggrUndef->getType();
1552 }
1553 }
1554}
1555
1556void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) {
1557 std::queue<Instruction *> Worklist;
1558 for (auto &I : instructions(F: CurrF))
1559 Worklist.push(x: &I);
1560
1561 while (!Worklist.empty()) {
1562 auto *I = Worklist.front();
1563 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
1564 assert(I);
1565 bool KeepInst = false;
1566 for (const auto &Op : I->operands()) {
1567 Constant *AggrConst = nullptr;
1568 Type *ResTy = nullptr;
1569 if (auto *COp = dyn_cast<ConstantVector>(Val: Op)) {
1570 AggrConst = COp;
1571 ResTy = COp->getType();
1572 } else if (auto *COp = dyn_cast<ConstantArray>(Val: Op)) {
1573 AggrConst = COp;
1574 ResTy = B.getInt32Ty();
1575 } else if (auto *COp = dyn_cast<ConstantStruct>(Val: Op)) {
1576 AggrConst = COp;
1577 ResTy = B.getInt32Ty();
1578 } else if (auto *COp = dyn_cast<ConstantDataArray>(Val: Op)) {
1579 AggrConst = COp;
1580 ResTy = B.getInt32Ty();
1581 } else if (auto *COp = dyn_cast<ConstantAggregateZero>(Val: Op)) {
1582 AggrConst = COp;
1583 ResTy = Op->getType()->isVectorTy() ? COp->getType() : B.getInt32Ty();
1584 }
1585 if (AggrConst) {
1586 SmallVector<Value *> Args;
1587 if (auto *COp = dyn_cast<ConstantDataSequential>(Val: Op))
1588 for (unsigned i = 0; i < COp->getNumElements(); ++i)
1589 Args.push_back(Elt: COp->getElementAsConstant(i));
1590 else
1591 llvm::append_range(C&: Args, R: AggrConst->operands());
1592 if (!BPrepared) {
1593 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
1594 : B.SetInsertPoint(I);
1595 BPrepared = true;
1596 }
1597 auto *CI =
1598 B.CreateIntrinsic(ID: Intrinsic::spv_const_composite, Types: {ResTy}, Args: {Args});
1599 Worklist.push(x: CI);
1600 I->replaceUsesOfWith(From: Op, To: CI);
1601 KeepInst = true;
1602 AggrConsts[CI] = AggrConst;
1603 AggrConstTypes[CI] = deduceNestedTypeHelper(U: AggrConst, UnknownElemTypeI8: false);
1604 }
1605 }
1606 if (!KeepInst)
1607 Worklist.pop();
1608 }
1609}
1610
1611static void createDecorationIntrinsic(Instruction *I, MDNode *Node,
1612 IRBuilder<> &B) {
1613 LLVMContext &Ctx = I->getContext();
1614 setInsertPointAfterDef(B, I);
1615 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, Types: {I->getType()},
1616 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: {Node}))});
1617}
1618
1619static void createRoundingModeDecoration(Instruction *I,
1620 unsigned RoundingModeDeco,
1621 IRBuilder<> &B) {
1622 LLVMContext &Ctx = I->getContext();
1623 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1624 MDNode *RoundingModeNode = MDNode::get(
1625 Context&: Ctx,
1626 MDs: {ConstantAsMetadata::get(
1627 C: ConstantInt::get(Ty: Int32Ty, V: SPIRV::Decoration::FPRoundingMode)),
1628 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: RoundingModeDeco))});
1629 createDecorationIntrinsic(I, Node: RoundingModeNode, B);
1630}
1631
1632static void createSaturatedConversionDecoration(Instruction *I,
1633 IRBuilder<> &B) {
1634 LLVMContext &Ctx = I->getContext();
1635 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1636 MDNode *SaturatedConversionNode =
1637 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1638 Ty: Int32Ty, V: SPIRV::Decoration::SaturatedConversion))});
1639 createDecorationIntrinsic(I, Node: SaturatedConversionNode, B);
1640}
1641
1642static void addSaturatedDecorationToIntrinsic(Instruction *I, IRBuilder<> &B) {
1643 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
1644 if (Function *Fu = CI->getCalledFunction()) {
1645 if (Fu->isIntrinsic()) {
1646 unsigned const int IntrinsicId = Fu->getIntrinsicID();
1647 switch (IntrinsicId) {
1648 case Intrinsic::fptosi_sat:
1649 case Intrinsic::fptoui_sat:
1650 createSaturatedConversionDecoration(I, B);
1651 break;
1652 default:
1653 break;
1654 }
1655 }
1656 }
1657 }
1658}
1659
1660Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) {
1661 if (!Call.isInlineAsm())
1662 return &Call;
1663
1664 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
1665 LLVMContext &Ctx = CurrF->getContext();
1666
1667 Constant *TyC = UndefValue::get(T: IA->getFunctionType());
1668 MDString *ConstraintString = MDString::get(Context&: Ctx, Str: IA->getConstraintString());
1669 SmallVector<Value *> Args = {
1670 buildMD(Arg: TyC),
1671 MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: ConstraintString))};
1672 for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++)
1673 Args.push_back(Elt: Call.getArgOperand(i: OpIdx));
1674
1675 IRBuilder<> B(Call.getParent());
1676 B.SetInsertPoint(&Call);
1677 B.CreateIntrinsic(ID: Intrinsic::spv_inline_asm, Args: {Args});
1678 return &Call;
1679}
1680
1681// Use a tip about rounding mode to create a decoration.
1682void SPIRVEmitIntrinsics::useRoundingMode(ConstrainedFPIntrinsic *FPI,
1683 IRBuilder<> &B) {
1684 std::optional<RoundingMode> RM = FPI->getRoundingMode();
1685 if (!RM.has_value())
1686 return;
1687 unsigned RoundingModeDeco = std::numeric_limits<unsigned>::max();
1688 switch (RM.value()) {
1689 default:
1690 // ignore unknown rounding modes
1691 break;
1692 case RoundingMode::NearestTiesToEven:
1693 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTE;
1694 break;
1695 case RoundingMode::TowardNegative:
1696 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTN;
1697 break;
1698 case RoundingMode::TowardPositive:
1699 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTP;
1700 break;
1701 case RoundingMode::TowardZero:
1702 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTZ;
1703 break;
1704 case RoundingMode::Dynamic:
1705 case RoundingMode::NearestTiesToAway:
1706 // TODO: check if supported
1707 break;
1708 }
1709 if (RoundingModeDeco == std::numeric_limits<unsigned>::max())
1710 return;
1711 // Convert the tip about rounding mode into a decoration record.
1712 createRoundingModeDecoration(I: FPI, RoundingModeDeco, B);
1713}
1714
1715Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) {
1716 BasicBlock *ParentBB = I.getParent();
1717 Function *F = ParentBB->getParent();
1718 IRBuilder<> B(ParentBB);
1719 B.SetInsertPoint(&I);
1720 SmallVector<Value *, 4> Args;
1721 SmallVector<BasicBlock *> BBCases;
1722 Args.push_back(Elt: I.getCondition());
1723 BBCases.push_back(Elt: I.getDefaultDest());
1724 Args.push_back(Elt: BlockAddress::get(F, BB: I.getDefaultDest()));
1725 for (auto &Case : I.cases()) {
1726 Args.push_back(Elt: Case.getCaseValue());
1727 BBCases.push_back(Elt: Case.getCaseSuccessor());
1728 Args.push_back(Elt: BlockAddress::get(F, BB: Case.getCaseSuccessor()));
1729 }
1730 CallInst *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_switch,
1731 Types: {I.getOperand(i_nocapture: 0)->getType()}, Args: {Args});
1732 // remove switch to avoid its unneeded and undesirable unwrap into branches
1733 // and conditions
1734 replaceAllUsesWith(Src: &I, Dest: NewI);
1735 I.eraseFromParent();
1736 // insert artificial and temporary instruction to preserve valid CFG,
1737 // it will be removed after IR translation pass
1738 B.SetInsertPoint(ParentBB);
1739 IndirectBrInst *BrI = B.CreateIndirectBr(
1740 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: ParentBB->getContext())),
1741 NumDests: BBCases.size());
1742 for (BasicBlock *BBCase : BBCases)
1743 BrI->addDestination(Dest: BBCase);
1744 return BrI;
1745}
1746
1747static bool isFirstIndexZero(const GetElementPtrInst *GEP) {
1748 if (GEP->getNumIndices() == 0)
1749 return false;
1750 if (const auto *CI = dyn_cast<ConstantInt>(Val: GEP->getOperand(i_nocapture: 1))) {
1751 return CI->getZExtValue() == 0;
1752 }
1753 return false;
1754}
1755
1756Instruction *SPIRVEmitIntrinsics::visitIntrinsicInst(IntrinsicInst &I) {
1757 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
1758 if (!SGEP)
1759 return &I;
1760
1761 IRBuilder<> B(I.getParent());
1762 B.SetInsertPoint(&I);
1763 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
1764 SmallVector<Value *, 4> Args;
1765 Args.push_back(/* inBounds= */ Elt: B.getInt1(V: true));
1766 Args.push_back(Elt: I.getOperand(i_nocapture: 0));
1767 Args.push_back(/* zero index */ Elt: B.getInt32(C: 0));
1768 for (unsigned J = 0; J < SGEP->getNumIndices(); ++J)
1769 Args.push_back(Elt: SGEP->getIndexOperand(Index: J));
1770
1771 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types, Args);
1772 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1773 return NewI;
1774}
1775
1776Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) {
1777 IRBuilder<> B(I.getParent());
1778 B.SetInsertPoint(&I);
1779
1780 if (TM.getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: &I)) {
1781 // Logical SPIR-V cannot use the OpPtrAccessChain instruction. If the first
1782 // index of the GEP is not 0, then we need to try to adjust it.
1783 //
1784 // If the GEP is doing byte addressing, try to rebuild the full access chain
1785 // from the type of the pointer.
1786 if (I.getSourceElementType() ==
1787 IntegerType::getInt8Ty(C&: CurrF->getContext())) {
1788 return buildLogicalAccessChainFromGEP(GEP&: I);
1789 }
1790
1791 // Look for the array-to-pointer decay. If this is the pattern
1792 // we can adjust the types, and prepend a 0 to the indices.
1793 Value *PtrOp = I.getPointerOperand();
1794 Type *SrcElemTy = I.getSourceElementType();
1795 Type *DeducedPointeeTy = deduceElementType(I: PtrOp, UnknownElemTypeI8: true);
1796
1797 if (auto *ArrTy = dyn_cast<ArrayType>(Val: DeducedPointeeTy)) {
1798 if (ArrTy->getElementType() == SrcElemTy) {
1799 SmallVector<Value *> NewIndices;
1800 Type *FirstIdxType = I.getOperand(i_nocapture: 1)->getType();
1801 NewIndices.push_back(Elt: ConstantInt::get(Ty: FirstIdxType, V: 0));
1802 for (Value *Idx : I.indices())
1803 NewIndices.push_back(Elt: Idx);
1804
1805 SmallVector<Type *, 2> Types = {I.getType(), I.getPointerOperandType()};
1806 SmallVector<Value *, 4> Args;
1807 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
1808 Args.push_back(Elt: I.getPointerOperand());
1809 Args.append(in_start: NewIndices.begin(), in_end: NewIndices.end());
1810
1811 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types: {Types}, Args: {Args});
1812 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1813 return NewI;
1814 }
1815 }
1816 }
1817
1818 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
1819 SmallVector<Value *, 4> Args;
1820 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
1821 llvm::append_range(C&: Args, R: I.operands());
1822 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_gep, Types: {Types}, Args: {Args});
1823 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1824 return NewI;
1825}
1826
1827Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) {
1828 IRBuilder<> B(I.getParent());
1829 B.SetInsertPoint(&I);
1830 Value *Source = I.getOperand(i_nocapture: 0);
1831
1832 // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of
1833 // varying element types. In case of IR coming from older versions of LLVM
1834 // such bitcasts do not provide sufficient information, should be just skipped
1835 // here, and handled in insertPtrCastOrAssignTypeInstr.
1836 if (isPointerTy(T: I.getType())) {
1837 replaceAllUsesWith(Src: &I, Dest: Source);
1838 I.eraseFromParent();
1839 return nullptr;
1840 }
1841
1842 SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
1843 SmallVector<Value *> Args(I.op_begin(), I.op_end());
1844 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_bitcast, Types: {Types}, Args: {Args});
1845 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1846 return NewI;
1847}
1848
1849void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt(
1850 TargetExtType *AssignedType, Value *V, IRBuilder<> &B) {
1851 Type *VTy = V->getType();
1852
1853 // A couple of sanity checks.
1854 assert((isPointerTy(VTy)) && "Expect a pointer type!");
1855 if (Type *ElemTy = getPointeeType(Ty: VTy))
1856 if (ElemTy != AssignedType)
1857 report_fatal_error(reason: "Unexpected pointer element type!");
1858
1859 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: V);
1860 if (!AssignCI) {
1861 GR->buildAssignType(B, Ty: AssignedType, Arg: V);
1862 return;
1863 }
1864
1865 Type *CurrentType =
1866 dyn_cast<ConstantAsMetadata>(
1867 Val: cast<MetadataAsValue>(Val: AssignCI->getOperand(i_nocapture: 1))->getMetadata())
1868 ->getType();
1869 if (CurrentType == AssignedType)
1870 return;
1871
1872 // Builtin types cannot be redeclared or casted.
1873 if (CurrentType->isTargetExtTy())
1874 report_fatal_error(reason: "Type mismatch " + CurrentType->getTargetExtName() +
1875 "/" + AssignedType->getTargetExtName() +
1876 " for value " + V->getName(),
1877 gen_crash_diag: false);
1878
1879 // Our previous guess about the type seems to be wrong, let's update
1880 // inferred type according to a new, more precise type information.
1881 GR->updateAssignType(AssignCI, Arg: V, OfType: getNormalizedPoisonValue(Ty: AssignedType));
1882}
1883
1884void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
1885 Instruction *I, Value *Pointer, Type *ExpectedElementType,
1886 unsigned OperandToReplace, IRBuilder<> &B) {
1887 TypeValidated.insert(x: I);
1888
1889 // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType
1890 Type *PointerElemTy = deduceElementTypeHelper(I: Pointer, UnknownElemTypeI8: false);
1891 if (PointerElemTy == ExpectedElementType ||
1892 isEquivalentTypes(Ty1: PointerElemTy, Ty2: ExpectedElementType))
1893 return;
1894
1895 setInsertPointSkippingPhis(B, I);
1896 Value *ExpectedElementVal = getNormalizedPoisonValue(Ty: ExpectedElementType);
1897 MetadataAsValue *VMD = buildMD(Arg: ExpectedElementVal);
1898 unsigned AddressSpace = getPointerAddressSpace(T: Pointer->getType());
1899 bool FirstPtrCastOrAssignPtrType = true;
1900
1901 // Do not emit new spv_ptrcast if equivalent one already exists or when
1902 // spv_assign_ptr_type already targets this pointer with the same element
1903 // type.
1904 if (Pointer->hasUseList()) {
1905 for (auto User : Pointer->users()) {
1906 auto *II = dyn_cast<IntrinsicInst>(Val: User);
1907 if (!II ||
1908 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
1909 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
1910 II->getOperand(i_nocapture: 0) != Pointer)
1911 continue;
1912
1913 // There is some spv_ptrcast/spv_assign_ptr_type already targeting this
1914 // pointer.
1915 FirstPtrCastOrAssignPtrType = false;
1916 if (II->getOperand(i_nocapture: 1) != VMD ||
1917 dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 2))->getSExtValue() !=
1918 AddressSpace)
1919 continue;
1920
1921 // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the
1922 // same element type and address space.
1923 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
1924 return;
1925
1926 // This must be a spv_ptrcast, do not emit new if this one has the same BB
1927 // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type.
1928 if (II->getParent() != I->getParent())
1929 continue;
1930
1931 I->setOperand(i: OperandToReplace, Val: II);
1932 return;
1933 }
1934 }
1935
1936 if (isa<Instruction>(Val: Pointer) || isa<Argument>(Val: Pointer)) {
1937 if (FirstPtrCastOrAssignPtrType) {
1938 // If this would be the first spv_ptrcast, do not emit spv_ptrcast and
1939 // emit spv_assign_ptr_type instead.
1940 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
1941 return;
1942 } else if (isTodoType(Op: Pointer)) {
1943 eraseTodoType(Op: Pointer);
1944 if (!isa<CallInst>(Val: Pointer) && !isaGEP(V: Pointer) &&
1945 !isa<AllocaInst>(Val: Pointer)) {
1946 // If this wouldn't be the first spv_ptrcast but existing type info is
1947 // uncomplete, update spv_assign_ptr_type arguments.
1948 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Pointer)) {
1949 Type *PrevElemTy = GR->findDeducedElementType(Val: Pointer);
1950 assert(PrevElemTy);
1951 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1952 std::make_pair(x&: I, y&: Pointer)};
1953 GR->updateAssignType(AssignCI, Arg: Pointer, OfType: ExpectedElementVal);
1954 propagateElemType(Op: Pointer, ElemTy: PrevElemTy, VisitedSubst);
1955 } else {
1956 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
1957 }
1958 return;
1959 }
1960 }
1961 }
1962
1963 // Emit spv_ptrcast
1964 SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()};
1965 SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(C: AddressSpace)};
1966 auto *PtrCastI = B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
1967 I->setOperand(i: OperandToReplace, Val: PtrCastI);
1968 // We need to set up a pointee type for the newly created spv_ptrcast.
1969 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: PtrCastI);
1970}
1971
1972void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I,
1973 IRBuilder<> &B) {
1974 // Handle basic instructions:
1975 StoreInst *SI = dyn_cast<StoreInst>(Val: I);
1976 if (IsKernelArgInt8(F: CurrF, SI)) {
1977 replacePointerOperandWithPtrCast(
1978 I, Pointer: SI->getValueOperand(), ExpectedElementType: IntegerType::getInt8Ty(C&: CurrF->getContext()),
1979 OperandToReplace: 0, B);
1980 }
1981 if (SI) {
1982 Value *Op = SI->getValueOperand();
1983 Value *Pointer = SI->getPointerOperand();
1984 Type *OpTy = Op->getType();
1985 if (auto *OpI = dyn_cast<Instruction>(Val: Op))
1986 OpTy = restoreMutatedType(GR, I: OpI, Ty: OpTy);
1987 if (OpTy == Op->getType())
1988 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: Op, UnknownElemTypeI8: false);
1989 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 1, B);
1990 return;
1991 }
1992 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
1993 Value *Pointer = LI->getPointerOperand();
1994 Type *OpTy = LI->getType();
1995 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1996 if (Type *ElemTy = GR->findDeducedElementType(Val: LI)) {
1997 OpTy = getTypedPointerWrapper(ElemTy, AS: PtrTy->getAddressSpace());
1998 } else {
1999 Type *NewOpTy = OpTy;
2000 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: LI, UnknownElemTypeI8: false);
2001 if (OpTy == NewOpTy)
2002 insertTodoType(Op: Pointer);
2003 }
2004 }
2005 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
2006 return;
2007 }
2008 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(Val: I)) {
2009 Value *Pointer = GEPI->getPointerOperand();
2010 Type *OpTy = nullptr;
2011
2012 // Logical SPIR-V is not allowed to use Op*PtrAccessChain instructions. If
2013 // the first index is 0, then we can trivially lower to OpAccessChain. If
2014 // not we need to try to rewrite the GEP. We avoid adding a pointer cast at
2015 // this time, and will rewrite the GEP when visiting it.
2016 if (TM.getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: GEPI)) {
2017 return;
2018 }
2019
2020 // In all cases, fall back to the GEP type if type scavenging failed.
2021 if (!OpTy)
2022 OpTy = GEPI->getSourceElementType();
2023
2024 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
2025 if (isNestedPointer(Ty: OpTy))
2026 insertTodoType(Op: Pointer);
2027 return;
2028 }
2029
2030 // TODO: review and merge with existing logics:
2031 // Handle calls to builtins (non-intrinsics):
2032 CallInst *CI = dyn_cast<CallInst>(Val: I);
2033 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
2034 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
2035 return;
2036
2037 // collect information about formal parameter types
2038 std::string DemangledName =
2039 getOclOrSpirvBuiltinDemangledName(Name: CI->getCalledFunction()->getName());
2040 Function *CalledF = CI->getCalledFunction();
2041 SmallVector<Type *, 4> CalledArgTys;
2042 bool HaveTypes = false;
2043 for (unsigned OpIdx = 0; OpIdx < CalledF->arg_size(); ++OpIdx) {
2044 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2045 Type *ArgType = CalledArg->getType();
2046 if (!isPointerTy(T: ArgType)) {
2047 CalledArgTys.push_back(Elt: nullptr);
2048 } else if (Type *ArgTypeElem = getPointeeType(Ty: ArgType)) {
2049 CalledArgTys.push_back(Elt: ArgTypeElem);
2050 HaveTypes = true;
2051 } else {
2052 Type *ElemTy = GR->findDeducedElementType(Val: CalledArg);
2053 if (!ElemTy && hasPointeeTypeAttr(Arg: CalledArg))
2054 ElemTy = getPointeeTypeByAttr(Arg: CalledArg);
2055 if (!ElemTy) {
2056 ElemTy = getPointeeTypeByCallInst(DemangledName, CalledF, OpIdx);
2057 if (ElemTy) {
2058 GR->addDeducedElementType(Val: CalledArg, Ty: normalizeType(Ty: ElemTy));
2059 } else {
2060 for (User *U : CalledArg->users()) {
2061 if (Instruction *Inst = dyn_cast<Instruction>(Val: U)) {
2062 if ((ElemTy = deduceElementTypeHelper(I: Inst, UnknownElemTypeI8: false)) != nullptr)
2063 break;
2064 }
2065 }
2066 }
2067 }
2068 HaveTypes |= ElemTy != nullptr;
2069 CalledArgTys.push_back(Elt: ElemTy);
2070 }
2071 }
2072
2073 if (DemangledName.empty() && !HaveTypes)
2074 return;
2075
2076 for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) {
2077 Value *ArgOperand = CI->getArgOperand(i: OpIdx);
2078 if (!isPointerTy(T: ArgOperand->getType()))
2079 continue;
2080
2081 // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs()
2082 if (!isa<Instruction>(Val: ArgOperand) && !isa<Argument>(Val: ArgOperand)) {
2083 // However, we may have assumptions about the formal argument's type and
2084 // may have a need to insert a ptr cast for the actual parameter of this
2085 // call.
2086 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2087 if (!GR->findDeducedElementType(Val: CalledArg))
2088 continue;
2089 }
2090
2091 Type *ExpectedType =
2092 OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr;
2093 if (!ExpectedType && !DemangledName.empty())
2094 ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType(
2095 DemangledCall: DemangledName, ArgIdx: OpIdx, Ctx&: I->getContext());
2096 if (!ExpectedType || ExpectedType->isVoidTy())
2097 continue;
2098
2099 if (ExpectedType->isTargetExtTy() &&
2100 !isTypedPointerWrapper(ExtTy: cast<TargetExtType>(Val: ExpectedType)))
2101 insertAssignPtrTypeTargetExt(AssignedType: cast<TargetExtType>(Val: ExpectedType),
2102 V: ArgOperand, B);
2103 else
2104 replacePointerOperandWithPtrCast(I: CI, Pointer: ArgOperand, ExpectedElementType: ExpectedType, OperandToReplace: OpIdx, B);
2105 }
2106}
2107
2108Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) {
2109 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2110 // type in LLT and IRTranslator will replace it by the scalar.
2111 if (isVector1(Ty: I.getType()))
2112 return &I;
2113
2114 SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType(),
2115 I.getOperand(i_nocapture: 1)->getType(),
2116 I.getOperand(i_nocapture: 2)->getType()};
2117 IRBuilder<> B(I.getParent());
2118 B.SetInsertPoint(&I);
2119 SmallVector<Value *> Args(I.op_begin(), I.op_end());
2120 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_insertelt, Types: {Types}, Args: {Args});
2121 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2122 return NewI;
2123}
2124
2125Instruction *
2126SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) {
2127 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2128 // type in LLT and IRTranslator will replace it by the scalar.
2129 if (isVector1(Ty: I.getVectorOperandType()))
2130 return &I;
2131
2132 IRBuilder<> B(I.getParent());
2133 B.SetInsertPoint(&I);
2134 SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
2135 I.getIndexOperand()->getType()};
2136 SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
2137 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_extractelt, Types: {Types}, Args: {Args});
2138 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2139 return NewI;
2140}
2141
2142Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
2143 IRBuilder<> B(I.getParent());
2144 B.SetInsertPoint(&I);
2145 SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
2146 SmallVector<Value *> Args;
2147 Value *AggregateOp = I.getAggregateOperand();
2148 if (isa<UndefValue>(Val: AggregateOp))
2149 Args.push_back(Elt: UndefValue::get(T: B.getInt32Ty()));
2150 else
2151 Args.push_back(Elt: AggregateOp);
2152 Args.push_back(Elt: I.getInsertedValueOperand());
2153 for (auto &Op : I.indices())
2154 Args.push_back(Elt: B.getInt32(C: Op));
2155 Instruction *NewI =
2156 B.CreateIntrinsic(ID: Intrinsic::spv_insertv, Types: {Types}, Args: {Args});
2157 replaceMemInstrUses(Old: &I, New: NewI, B);
2158 return NewI;
2159}
2160
2161Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
2162 if (I.getAggregateOperand()->getType()->isAggregateType())
2163 return &I;
2164 IRBuilder<> B(I.getParent());
2165 B.SetInsertPoint(&I);
2166 SmallVector<Value *> Args(I.operands());
2167 for (auto &Op : I.indices())
2168 Args.push_back(Elt: B.getInt32(C: Op));
2169 auto *NewI =
2170 B.CreateIntrinsic(ID: Intrinsic::spv_extractv, Types: {I.getType()}, Args: {Args});
2171 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2172 return NewI;
2173}
2174
2175Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) {
2176 if (!I.getType()->isAggregateType())
2177 return &I;
2178 IRBuilder<> B(I.getParent());
2179 B.SetInsertPoint(&I);
2180 TrackConstants = false;
2181 const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
2182 MachineMemOperand::Flags Flags =
2183 TLI->getLoadMemOperandFlags(LI: I, DL: CurrF->getDataLayout());
2184 auto *NewI =
2185 B.CreateIntrinsic(ID: Intrinsic::spv_load, Types: {I.getOperand(i_nocapture: 0)->getType()},
2186 Args: {I.getPointerOperand(), B.getInt16(C: Flags),
2187 B.getInt32(C: I.getAlign().value())});
2188 replaceMemInstrUses(Old: &I, New: NewI, B);
2189 return NewI;
2190}
2191
2192Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) {
2193 if (!AggrStores.contains(V: &I))
2194 return &I;
2195 IRBuilder<> B(I.getParent());
2196 B.SetInsertPoint(&I);
2197 TrackConstants = false;
2198 const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
2199 MachineMemOperand::Flags Flags =
2200 TLI->getStoreMemOperandFlags(SI: I, DL: CurrF->getDataLayout());
2201 auto *PtrOp = I.getPointerOperand();
2202
2203 if (I.getValueOperand()->getType()->isAggregateType()) {
2204 // It is possible that what used to be an ExtractValueInst has been replaced
2205 // with a call to the spv_extractv intrinsic, and that said call hasn't
2206 // had its return type replaced with i32 during the dedicated pass (because
2207 // it was emitted later); we have to handle this here, because IRTranslator
2208 // cannot deal with multi-register types at the moment.
2209 CallBase *CB = dyn_cast<CallBase>(Val: I.getValueOperand());
2210 assert(CB && CB->getIntrinsicID() == Intrinsic::spv_extractv &&
2211 "Unexpected argument of aggregate type, should be spv_extractv!");
2212 CB->mutateType(Ty: B.getInt32Ty());
2213 }
2214
2215 auto *NewI = B.CreateIntrinsic(
2216 ID: Intrinsic::spv_store, Types: {I.getValueOperand()->getType(), PtrOp->getType()},
2217 Args: {I.getValueOperand(), PtrOp, B.getInt16(C: Flags),
2218 B.getInt32(C: I.getAlign().value())});
2219 NewI->copyMetadata(SrcInst: I);
2220 I.eraseFromParent();
2221 return NewI;
2222}
2223
2224Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
2225 Value *ArraySize = nullptr;
2226 if (I.isArrayAllocation()) {
2227 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I.getFunction());
2228 if (!STI->canUseExtension(
2229 E: SPIRV::Extension::SPV_INTEL_variable_length_array))
2230 report_fatal_error(
2231 reason: "array allocation: this instruction requires the following "
2232 "SPIR-V extension: SPV_INTEL_variable_length_array",
2233 gen_crash_diag: false);
2234 ArraySize = I.getArraySize();
2235 }
2236 IRBuilder<> B(I.getParent());
2237 B.SetInsertPoint(&I);
2238 TrackConstants = false;
2239 Type *PtrTy = I.getType();
2240 auto *NewI =
2241 ArraySize
2242 ? B.CreateIntrinsic(ID: Intrinsic::spv_alloca_array,
2243 Types: {PtrTy, ArraySize->getType()},
2244 Args: {ArraySize, B.getInt32(C: I.getAlign().value())})
2245 : B.CreateIntrinsic(ID: Intrinsic::spv_alloca, Types: {PtrTy},
2246 Args: {B.getInt32(C: I.getAlign().value())});
2247 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2248 return NewI;
2249}
2250
2251Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2252 assert(I.getType()->isAggregateType() && "Aggregate result is expected");
2253 IRBuilder<> B(I.getParent());
2254 B.SetInsertPoint(&I);
2255 SmallVector<Value *> Args(I.operands());
2256 Args.push_back(Elt: B.getInt32(
2257 C: static_cast<uint32_t>(getMemScope(Ctx&: I.getContext(), Id: I.getSyncScopeID()))));
2258 Args.push_back(Elt: B.getInt32(
2259 C: static_cast<uint32_t>(getMemSemantics(Ord: I.getSuccessOrdering()))));
2260 Args.push_back(Elt: B.getInt32(
2261 C: static_cast<uint32_t>(getMemSemantics(Ord: I.getFailureOrdering()))));
2262 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_cmpxchg,
2263 Types: {I.getPointerOperand()->getType()}, Args: {Args});
2264 replaceMemInstrUses(Old: &I, New: NewI, B);
2265 return NewI;
2266}
2267
2268Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
2269 IRBuilder<> B(I.getParent());
2270 B.SetInsertPoint(&I);
2271 B.CreateIntrinsic(ID: Intrinsic::spv_unreachable, Args: {});
2272 return &I;
2273}
2274
2275static bool
2276shouldEmitIntrinsicsForGlobalValue(const GlobalVariableUsers &GVUsers,
2277 const GlobalVariable &GV,
2278 const Function *F) {
2279 // Skip special artificial variables.
2280 static const StringSet<> ArtificialGlobals{"llvm.global.annotations",
2281 "llvm.compiler.used", "llvm.used"};
2282
2283 if (ArtificialGlobals.contains(key: GV.getName()))
2284 return false;
2285
2286 auto &UserFunctions = GVUsers.getTransitiveUserFunctions(GV);
2287 if (UserFunctions.contains(Ptr: F))
2288 return true;
2289
2290 // Do not emit the intrinsics in this function, it's going to be emitted on
2291 // the functions that reference it.
2292 if (!UserFunctions.empty())
2293 return false;
2294
2295 // Emit definitions for globals that are not referenced by any function on the
2296 // first function definition.
2297 const Module &M = *F->getParent();
2298 const Function &FirstDefinition = *M.getFunctionDefs().begin();
2299 return F == &FirstDefinition;
2300}
2301
2302Value *SPIRVEmitIntrinsics::buildSpvUndefComposite(Type *AggrTy,
2303 IRBuilder<> &B) {
2304 SmallVector<Value *, 4> Elems;
2305 if (auto *ArrTy = dyn_cast<ArrayType>(Val: AggrTy)) {
2306 Type *ElemTy = ArrTy->getElementType();
2307 auto *UI = B.CreateIntrinsic(ID: Intrinsic::spv_undef, Args: {});
2308 AggrConsts[UI] = PoisonValue::get(T: ElemTy);
2309 AggrConstTypes[UI] = ElemTy;
2310 Elems.assign(NumElts: ArrTy->getNumElements(), Elt: UI);
2311 } else {
2312 auto *StructTy = cast<StructType>(Val: AggrTy);
2313 DenseMap<Type *, Instruction *> UndefByType;
2314 for (unsigned I = 0; I < StructTy->getNumElements(); ++I) {
2315 Type *ElemTy = StructTy->getContainedType(i: I);
2316 auto &Entry = UndefByType[ElemTy];
2317 if (!Entry) {
2318 Entry = B.CreateIntrinsic(ID: Intrinsic::spv_undef, Args: {});
2319 AggrConsts[Entry] = PoisonValue::get(T: ElemTy);
2320 AggrConstTypes[Entry] = ElemTy;
2321 }
2322 Elems.push_back(Elt: Entry);
2323 }
2324 }
2325 auto *Composite = B.CreateIntrinsic(ID: Intrinsic::spv_const_composite,
2326 Types: {B.getInt32Ty()}, Args: Elems);
2327 AggrConsts[Composite] = PoisonValue::get(T: AggrTy);
2328 AggrConstTypes[Composite] = AggrTy;
2329 return Composite;
2330}
2331
2332void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV,
2333 IRBuilder<> &B) {
2334
2335 if (!shouldEmitIntrinsicsForGlobalValue(GVUsers, GV, F: CurrF))
2336 return;
2337
2338 Constant *Init = nullptr;
2339 if (hasInitializer(GV: &GV)) {
2340 // Deduce element type and store results in Global Registry.
2341 // Result is ignored, because TypedPointerType is not supported
2342 // by llvm IR general logic.
2343 deduceElementTypeHelper(I: &GV, UnknownElemTypeI8: false);
2344 Init = GV.getInitializer();
2345 Value *InitOp = Init;
2346 if (isa<UndefValue>(Val: Init) && Init->getType()->isAggregateType())
2347 InitOp = buildSpvUndefComposite(AggrTy: Init->getType(), B);
2348 Type *Ty = isAggrConstForceInt32(V: Init) ? B.getInt32Ty() : Init->getType();
2349 Constant *Const = isAggrConstForceInt32(V: Init) ? B.getInt32(C: 1) : Init;
2350 auto *InitInst = B.CreateIntrinsic(ID: Intrinsic::spv_init_global,
2351 Types: {GV.getType(), Ty}, Args: {&GV, Const});
2352 InitInst->setArgOperand(i: 1, v: InitOp);
2353 }
2354 if (!Init && GV.use_empty())
2355 B.CreateIntrinsic(ID: Intrinsic::spv_unref_global, Types: GV.getType(), Args: &GV);
2356}
2357
2358// Return true, if we can't decide what is the pointee type now and will get
2359// back to the question later. Return false is spv_assign_ptr_type is not needed
2360// or can be inserted immediately.
2361bool SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I,
2362 IRBuilder<> &B,
2363 bool UnknownElemTypeI8) {
2364 reportFatalOnTokenType(I);
2365 if (!isPointerTy(T: I->getType()) || !requireAssignType(I))
2366 return false;
2367
2368 setInsertPointAfterDef(B, I);
2369 if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) {
2370 GR->buildAssignPtr(B, ElemTy, Arg: I);
2371 return false;
2372 }
2373 return true;
2374}
2375
2376void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I,
2377 IRBuilder<> &B) {
2378 // TODO: extend the list of functions with known result types
2379 static StringMap<unsigned> ResTypeWellKnown = {
2380 {"async_work_group_copy", WellKnownTypes::Event},
2381 {"async_work_group_strided_copy", WellKnownTypes::Event},
2382 {"__spirv_GroupAsyncCopy", WellKnownTypes::Event}};
2383
2384 reportFatalOnTokenType(I);
2385
2386 bool IsKnown = false;
2387 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
2388 if (!CI->isIndirectCall() && !CI->isInlineAsm() &&
2389 CI->getCalledFunction() && !CI->getCalledFunction()->isIntrinsic()) {
2390 Function *CalledF = CI->getCalledFunction();
2391 std::string DemangledName =
2392 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
2393 FPDecorationId DecorationId = FPDecorationId::NONE;
2394 if (DemangledName.length() > 0)
2395 DemangledName =
2396 SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName, DecorationId: &DecorationId);
2397 auto ResIt = ResTypeWellKnown.find(Key: DemangledName);
2398 if (ResIt != ResTypeWellKnown.end()) {
2399 IsKnown = true;
2400 setInsertPointAfterDef(B, I);
2401 switch (ResIt->second) {
2402 case WellKnownTypes::Event:
2403 GR->buildAssignType(
2404 B, Ty: TargetExtType::get(Context&: I->getContext(), Name: "spirv.Event"), Arg: I);
2405 break;
2406 }
2407 }
2408 // check if a floating rounding mode or saturation info is present
2409 switch (DecorationId) {
2410 default:
2411 break;
2412 case FPDecorationId::SAT:
2413 createSaturatedConversionDecoration(I: CI, B);
2414 break;
2415 case FPDecorationId::RTE:
2416 createRoundingModeDecoration(
2417 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTE, B);
2418 break;
2419 case FPDecorationId::RTZ:
2420 createRoundingModeDecoration(
2421 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTZ, B);
2422 break;
2423 case FPDecorationId::RTP:
2424 createRoundingModeDecoration(
2425 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTP, B);
2426 break;
2427 case FPDecorationId::RTN:
2428 createRoundingModeDecoration(
2429 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTN, B);
2430 break;
2431 }
2432 }
2433 }
2434
2435 Type *Ty = I->getType();
2436 if (!IsKnown && !Ty->isVoidTy() && !isPointerTy(T: Ty) && requireAssignType(I)) {
2437 setInsertPointAfterDef(B, I);
2438 Type *TypeToAssign = Ty;
2439 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2440 if (II->getIntrinsicID() == Intrinsic::spv_const_composite ||
2441 II->getIntrinsicID() == Intrinsic::spv_undef) {
2442 auto It = AggrConstTypes.find(Val: II);
2443 if (It == AggrConstTypes.end())
2444 report_fatal_error(reason: "Unknown composite intrinsic type");
2445 TypeToAssign = It->second;
2446 }
2447 } else if (auto It = AggrConstTypes.find(Val: I); It != AggrConstTypes.end())
2448 TypeToAssign = It->second;
2449 TypeToAssign = restoreMutatedType(GR, I, Ty: TypeToAssign);
2450 GR->buildAssignType(B, Ty: TypeToAssign, Arg: I);
2451 }
2452 for (const auto &Op : I->operands()) {
2453 if (isa<ConstantPointerNull>(Val: Op) || isa<UndefValue>(Val: Op) ||
2454 // Check GetElementPtrConstantExpr case.
2455 (isa<ConstantExpr>(Val: Op) &&
2456 (isa<GEPOperator>(Val: Op) ||
2457 (cast<ConstantExpr>(Val: Op)->getOpcode() == CastInst::IntToPtr)))) {
2458 setInsertPointSkippingPhis(B, I);
2459 Type *OpTy = Op->getType();
2460 if (isa<UndefValue>(Val: Op) && OpTy->isAggregateType()) {
2461 CallInst *AssignCI =
2462 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {B.getInt32Ty()}, Arg: Op,
2463 Arg2: UndefValue::get(T: B.getInt32Ty()), Imms: {}, B);
2464 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: AssignCI);
2465 } else if (!isa<Instruction>(Val: Op)) {
2466 Type *OpTy = Op->getType();
2467 Type *OpTyElem = getPointeeType(Ty: OpTy);
2468 if (OpTyElem) {
2469 GR->buildAssignPtr(B, ElemTy: OpTyElem, Arg: Op);
2470 } else if (isPointerTy(T: OpTy)) {
2471 Type *ElemTy = GR->findDeducedElementType(Val: Op);
2472 GR->buildAssignPtr(B, ElemTy: ElemTy ? ElemTy : deduceElementType(I: Op, UnknownElemTypeI8: true),
2473 Arg: Op);
2474 } else {
2475 Value *OpTyVal = Op;
2476 if (OpTy->isTargetExtTy()) {
2477 // We need to do this in order to be consistent with how target ext
2478 // types are handled in `processInstrAfterVisit`
2479 OpTyVal = getNormalizedPoisonValue(Ty: OpTy);
2480 }
2481 CallInst *AssignCI =
2482 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {OpTy},
2483 Arg: getNormalizedPoisonValue(Ty: OpTy), Arg2: OpTyVal, Imms: {}, B);
2484 GR->addAssignPtrTypeInstr(Val: OpTyVal, AssignPtrTyCI: AssignCI);
2485 }
2486 }
2487 }
2488 }
2489}
2490
2491bool SPIRVEmitIntrinsics::shouldTryToAddMemAliasingDecoration(
2492 Instruction *Inst) {
2493 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*Inst->getFunction());
2494 if (!STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing))
2495 return false;
2496 // Add aliasing decorations to internal load and store intrinsics
2497 // and atomic instructions, skipping atomic store as it won't have ID to
2498 // attach the decoration.
2499 CallInst *CI = dyn_cast<CallInst>(Val: Inst);
2500 if (!CI)
2501 return false;
2502 if (Function *Fun = CI->getCalledFunction()) {
2503 if (Fun->isIntrinsic()) {
2504 switch (Fun->getIntrinsicID()) {
2505 case Intrinsic::spv_load:
2506 case Intrinsic::spv_store:
2507 return true;
2508 default:
2509 return false;
2510 }
2511 }
2512 std::string Name = getOclOrSpirvBuiltinDemangledName(Name: Fun->getName());
2513 const std::string Prefix = "__spirv_Atomic";
2514 const bool IsAtomic = Name.find(str: Prefix) == 0;
2515
2516 if (!Fun->getReturnType()->isVoidTy() && IsAtomic)
2517 return true;
2518 }
2519 return false;
2520}
2521
2522void SPIRVEmitIntrinsics::insertSpirvDecorations(Instruction *I,
2523 IRBuilder<> &B) {
2524 if (MDNode *MD = I->getMetadata(Kind: "spirv.Decorations")) {
2525 setInsertPointAfterDef(B, I);
2526 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, Types: {I->getType()},
2527 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
2528 }
2529 // Lower alias.scope/noalias metadata
2530 {
2531 auto processMemAliasingDecoration = [&](unsigned Kind) {
2532 if (MDNode *AliasListMD = I->getMetadata(KindID: Kind)) {
2533 if (shouldTryToAddMemAliasingDecoration(Inst: I)) {
2534 uint32_t Dec = Kind == LLVMContext::MD_alias_scope
2535 ? SPIRV::Decoration::AliasScopeINTEL
2536 : SPIRV::Decoration::NoAliasINTEL;
2537 SmallVector<Value *, 3> Args = {
2538 I, ConstantInt::get(Ty: B.getInt32Ty(), V: Dec),
2539 MetadataAsValue::get(Context&: I->getContext(), MD: AliasListMD)};
2540 setInsertPointAfterDef(B, I);
2541 B.CreateIntrinsic(ID: Intrinsic::spv_assign_aliasing_decoration,
2542 Types: {I->getType()}, Args: {Args});
2543 }
2544 }
2545 };
2546 processMemAliasingDecoration(LLVMContext::MD_alias_scope);
2547 processMemAliasingDecoration(LLVMContext::MD_noalias);
2548 }
2549 // MD_fpmath
2550 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_fpmath)) {
2551 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I->getFunction());
2552 bool AllowFPMaxError =
2553 STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_fp_max_error);
2554 if (!AllowFPMaxError)
2555 return;
2556
2557 setInsertPointAfterDef(B, I);
2558 B.CreateIntrinsic(ID: Intrinsic::spv_assign_fpmaxerror_decoration,
2559 Types: {I->getType()},
2560 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
2561 }
2562}
2563
2564static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
2565 const Module &M,
2566 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
2567 &FPFastMathDefaultInfoMap,
2568 Function *F) {
2569 auto it = FPFastMathDefaultInfoMap.find(Val: F);
2570 if (it != FPFastMathDefaultInfoMap.end())
2571 return it->second;
2572
2573 // If the map does not contain the entry, create a new one. Initialize it to
2574 // contain all 3 elements sorted by bit width of target type: {half, float,
2575 // double}.
2576 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
2577 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
2578 Args: SPIRV::FPFastMathMode::None);
2579 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
2580 Args: SPIRV::FPFastMathMode::None);
2581 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
2582 Args: SPIRV::FPFastMathMode::None);
2583 return FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
2584}
2585
2586static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
2587 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
2588 const Type *Ty) {
2589 size_t BitWidth = Ty->getScalarSizeInBits();
2590 int Index =
2591 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
2592 BitWidth);
2593 assert(Index >= 0 && Index < 3 &&
2594 "Expected FPFastMathDefaultInfo for half, float, or double");
2595 assert(FPFastMathDefaultInfoVec.size() == 3 &&
2596 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
2597 return FPFastMathDefaultInfoVec[Index];
2598}
2599
2600void SPIRVEmitIntrinsics::insertConstantsForFPFastMathDefault(Module &M) {
2601 const SPIRVSubtarget *ST = TM.getSubtargetImpl();
2602 if (!ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
2603 return;
2604
2605 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
2606 // We need the entry point (function) as the key, and the target
2607 // type and flags as the value.
2608 // We also need to check ContractionOff and SignedZeroInfNanPreserve
2609 // execution modes, as they are now deprecated and must be replaced
2610 // with FPFastMathDefaultInfo.
2611 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
2612 if (!Node) {
2613 if (!M.getNamedMetadata(Name: "opencl.enable.FP_CONTRACT")) {
2614 // This requires emitting ContractionOff. However, because
2615 // ContractionOff is now deprecated, we need to replace it with
2616 // FPFastMathDefaultInfo with FP Fast Math Mode bitmask set to all 0.
2617 // We need to create the constant for that.
2618
2619 // Create constant instruction with the bitmask flags.
2620 Constant *InitValue =
2621 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: 0);
2622 // TODO: Reuse constant if there is one already with the required
2623 // value.
2624 [[maybe_unused]] GlobalVariable *GV =
2625 new GlobalVariable(M, // Module
2626 Type::getInt32Ty(C&: M.getContext()), // Type
2627 true, // isConstant
2628 GlobalValue::InternalLinkage, // Linkage
2629 InitValue // Initializer
2630 );
2631 }
2632 return;
2633 }
2634
2635 // The table maps function pointers to their default FP fast math info. It
2636 // can be assumed that the SmallVector is sorted by the bit width of the
2637 // type. The first element is the smallest bit width, and the last element
2638 // is the largest bit width, therefore, we will have {half, float, double}
2639 // in the order of their bit widths.
2640 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
2641 FPFastMathDefaultInfoMap;
2642
2643 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
2644 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
2645 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
2646 Function *F = cast<Function>(
2647 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
2648 const auto EM =
2649 cast<ConstantInt>(
2650 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
2651 ->getZExtValue();
2652 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
2653 assert(MDN->getNumOperands() == 4 &&
2654 "Expected 4 operands for FPFastMathDefault");
2655 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
2656 unsigned Flags =
2657 cast<ConstantInt>(
2658 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
2659 ->getZExtValue();
2660 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2661 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
2662 SPIRV::FPFastMathDefaultInfo &Info =
2663 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
2664 Info.FastMathFlags = Flags;
2665 Info.FPFastMathDefault = true;
2666 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
2667 assert(MDN->getNumOperands() == 2 &&
2668 "Expected no operands for ContractionOff");
2669
2670 // We need to save this info for every possible FP type, i.e. {half,
2671 // float, double, fp128}.
2672 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2673 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
2674 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
2675 Info.ContractionOff = true;
2676 }
2677 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
2678 assert(MDN->getNumOperands() == 3 &&
2679 "Expected 1 operand for SignedZeroInfNanPreserve");
2680 unsigned TargetWidth =
2681 cast<ConstantInt>(
2682 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
2683 ->getZExtValue();
2684 // We need to save this info only for the FP type with TargetWidth.
2685 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2686 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
2687 int Index = SPIRV::FPFastMathDefaultInfoVector::
2688 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
2689 assert(Index >= 0 && Index < 3 &&
2690 "Expected FPFastMathDefaultInfo for half, float, or double");
2691 assert(FPFastMathDefaultInfoVec.size() == 3 &&
2692 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
2693 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
2694 }
2695 }
2696
2697 std::unordered_map<unsigned, GlobalVariable *> GlobalVars;
2698 for (auto &[Func, FPFastMathDefaultInfoVec] : FPFastMathDefaultInfoMap) {
2699 if (FPFastMathDefaultInfoVec.empty())
2700 continue;
2701
2702 for (const SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
2703 assert(Info.Ty && "Expected target type for FPFastMathDefaultInfo");
2704 // Skip if none of the execution modes was used.
2705 unsigned Flags = Info.FastMathFlags;
2706 if (Flags == SPIRV::FPFastMathMode::None && !Info.ContractionOff &&
2707 !Info.SignedZeroInfNanPreserve && !Info.FPFastMathDefault)
2708 continue;
2709
2710 // Check if flags are compatible.
2711 if (Info.ContractionOff && (Flags & SPIRV::FPFastMathMode::AllowContract))
2712 report_fatal_error(reason: "Conflicting FPFastMathFlags: ContractionOff "
2713 "and AllowContract");
2714
2715 if (Info.SignedZeroInfNanPreserve &&
2716 !(Flags &
2717 (SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
2718 SPIRV::FPFastMathMode::NSZ))) {
2719 if (Info.FPFastMathDefault)
2720 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
2721 "SignedZeroInfNanPreserve but at least one of "
2722 "NotNaN/NotInf/NSZ is enabled.");
2723 }
2724
2725 if ((Flags & SPIRV::FPFastMathMode::AllowTransform) &&
2726 !((Flags & SPIRV::FPFastMathMode::AllowReassoc) &&
2727 (Flags & SPIRV::FPFastMathMode::AllowContract))) {
2728 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
2729 "AllowTransform requires AllowReassoc and "
2730 "AllowContract to be set.");
2731 }
2732
2733 auto it = GlobalVars.find(x: Flags);
2734 GlobalVariable *GV = nullptr;
2735 if (it != GlobalVars.end()) {
2736 // Reuse existing global variable.
2737 GV = it->second;
2738 } else {
2739 // Create constant instruction with the bitmask flags.
2740 Constant *InitValue =
2741 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: Flags);
2742 // TODO: Reuse constant if there is one already with the required
2743 // value.
2744 GV = new GlobalVariable(M, // Module
2745 Type::getInt32Ty(C&: M.getContext()), // Type
2746 true, // isConstant
2747 GlobalValue::InternalLinkage, // Linkage
2748 InitValue // Initializer
2749 );
2750 GlobalVars[Flags] = GV;
2751 }
2752 }
2753 }
2754}
2755
2756void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
2757 IRBuilder<> &B) {
2758 auto *II = dyn_cast<IntrinsicInst>(Val: I);
2759 bool IsConstComposite =
2760 II && II->getIntrinsicID() == Intrinsic::spv_const_composite;
2761 if (IsConstComposite && TrackConstants) {
2762 setInsertPointAfterDef(B, I);
2763 auto t = AggrConsts.find(Val: I);
2764 assert(t != AggrConsts.end());
2765 auto *NewOp =
2766 buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
2767 Types: {II->getType(), II->getType()}, Arg: t->second, Arg2: I, Imms: {}, B);
2768 replaceAllUsesWith(Src: I, Dest: NewOp, DeleteOld: false);
2769 NewOp->setArgOperand(i: 0, v: I);
2770 }
2771 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
2772 for (const auto &Op : I->operands()) {
2773 if (isa<PHINode>(Val: I) || isa<SwitchInst>(Val: I) ||
2774 !(isa<ConstantData>(Val: Op) || isa<ConstantExpr>(Val: Op)))
2775 continue;
2776 unsigned OpNo = Op.getOperandNo();
2777 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
2778 (!II->isBundleOperand(Idx: OpNo) &&
2779 II->paramHasAttr(ArgNo: OpNo, Kind: Attribute::ImmArg))))
2780 continue;
2781
2782 if (!BPrepared) {
2783 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
2784 : B.SetInsertPoint(I);
2785 BPrepared = true;
2786 }
2787 Type *OpTy = Op->getType();
2788 Type *OpElemTy = GR->findDeducedElementType(Val: Op);
2789 Value *NewOp = Op;
2790 if (OpTy->isTargetExtTy()) {
2791 // Since this value is replaced by poison, we need to do the same in
2792 // `insertAssignTypeIntrs`.
2793 Value *OpTyVal = getNormalizedPoisonValue(Ty: OpTy);
2794 NewOp = buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
2795 Types: {OpTy, OpTyVal->getType()}, Arg: Op, Arg2: OpTyVal, Imms: {}, B);
2796 }
2797 if (!IsConstComposite && isPointerTy(T: OpTy) && OpElemTy != nullptr &&
2798 OpElemTy != IntegerType::getInt8Ty(C&: I->getContext())) {
2799 SmallVector<Type *, 2> Types = {OpTy, OpTy};
2800 SmallVector<Value *, 2> Args = {
2801 NewOp, buildMD(Arg: getNormalizedPoisonValue(Ty: OpElemTy)),
2802 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
2803 CallInst *PtrCasted =
2804 B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, Types: {Types}, Args);
2805 GR->buildAssignPtr(B, ElemTy: OpElemTy, Arg: PtrCasted);
2806 NewOp = PtrCasted;
2807 }
2808 if (NewOp != Op)
2809 I->setOperand(i: OpNo, Val: NewOp);
2810 }
2811 if (Named.insert(x: I).second)
2812 emitAssignName(I, B);
2813}
2814
2815Type *SPIRVEmitIntrinsics::deduceFunParamElementType(Function *F,
2816 unsigned OpIdx) {
2817 std::unordered_set<Function *> FVisited;
2818 return deduceFunParamElementType(F, OpIdx, FVisited);
2819}
2820
2821Type *SPIRVEmitIntrinsics::deduceFunParamElementType(
2822 Function *F, unsigned OpIdx, std::unordered_set<Function *> &FVisited) {
2823 // maybe a cycle
2824 if (!FVisited.insert(x: F).second)
2825 return nullptr;
2826
2827 std::unordered_set<Value *> Visited;
2828 SmallVector<std::pair<Function *, unsigned>> Lookup;
2829 // search in function's call sites
2830 for (User *U : F->users()) {
2831 CallInst *CI = dyn_cast<CallInst>(Val: U);
2832 if (!CI || OpIdx >= CI->arg_size())
2833 continue;
2834 Value *OpArg = CI->getArgOperand(i: OpIdx);
2835 if (!isPointerTy(T: OpArg->getType()))
2836 continue;
2837 // maybe we already know operand's element type
2838 if (Type *KnownTy = GR->findDeducedElementType(Val: OpArg))
2839 return KnownTy;
2840 // try to deduce from the operand itself
2841 Visited.clear();
2842 if (Type *Ty = deduceElementTypeHelper(I: OpArg, Visited, UnknownElemTypeI8: false))
2843 return Ty;
2844 // search in actual parameter's users
2845 for (User *OpU : OpArg->users()) {
2846 Instruction *Inst = dyn_cast<Instruction>(Val: OpU);
2847 if (!Inst || Inst == CI)
2848 continue;
2849 Visited.clear();
2850 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8: false))
2851 return Ty;
2852 }
2853 // check if it's a formal parameter of the outer function
2854 if (!CI->getParent() || !CI->getParent()->getParent())
2855 continue;
2856 Function *OuterF = CI->getParent()->getParent();
2857 if (FVisited.find(x: OuterF) != FVisited.end())
2858 continue;
2859 for (unsigned i = 0; i < OuterF->arg_size(); ++i) {
2860 if (OuterF->getArg(i) == OpArg) {
2861 Lookup.push_back(Elt: std::make_pair(x&: OuterF, y&: i));
2862 break;
2863 }
2864 }
2865 }
2866
2867 // search in function parameters
2868 for (auto &Pair : Lookup) {
2869 if (Type *Ty = deduceFunParamElementType(F: Pair.first, OpIdx: Pair.second, FVisited))
2870 return Ty;
2871 }
2872
2873 return nullptr;
2874}
2875
2876void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F,
2877 IRBuilder<> &B) {
2878 B.SetInsertPointPastAllocas(F);
2879 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
2880 Argument *Arg = F->getArg(i: OpIdx);
2881 if (!isUntypedPointerTy(T: Arg->getType()))
2882 continue;
2883 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
2884 if (ElemTy)
2885 continue;
2886 if (hasPointeeTypeAttr(Arg) &&
2887 (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) {
2888 GR->buildAssignPtr(B, ElemTy, Arg);
2889 continue;
2890 }
2891 // search in function's call sites
2892 for (User *U : F->users()) {
2893 CallInst *CI = dyn_cast<CallInst>(Val: U);
2894 if (!CI || OpIdx >= CI->arg_size())
2895 continue;
2896 Value *OpArg = CI->getArgOperand(i: OpIdx);
2897 if (!isPointerTy(T: OpArg->getType()))
2898 continue;
2899 // maybe we already know operand's element type
2900 if ((ElemTy = GR->findDeducedElementType(Val: OpArg)) != nullptr)
2901 break;
2902 }
2903 if (ElemTy) {
2904 GR->buildAssignPtr(B, ElemTy, Arg);
2905 continue;
2906 }
2907 if (HaveFunPtrs) {
2908 for (User *U : Arg->users()) {
2909 CallInst *CI = dyn_cast<CallInst>(Val: U);
2910 if (CI && !isa<IntrinsicInst>(Val: CI) && CI->isIndirectCall() &&
2911 CI->getCalledOperand() == Arg &&
2912 CI->getParent()->getParent() == CurrF) {
2913 SmallVector<std::pair<Value *, unsigned>> Ops;
2914 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy&: ElemTy, IsPostprocessing: false);
2915 if (ElemTy) {
2916 GR->buildAssignPtr(B, ElemTy, Arg);
2917 break;
2918 }
2919 }
2920 }
2921 }
2922 }
2923}
2924
2925void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) {
2926 B.SetInsertPointPastAllocas(F);
2927 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
2928 Argument *Arg = F->getArg(i: OpIdx);
2929 if (!isUntypedPointerTy(T: Arg->getType()))
2930 continue;
2931 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
2932 if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) {
2933 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Arg)) {
2934 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
2935 GR->updateAssignType(AssignCI, Arg, OfType: getNormalizedPoisonValue(Ty: ElemTy));
2936 propagateElemType(Op: Arg, ElemTy: IntegerType::getInt8Ty(C&: F->getContext()),
2937 VisitedSubst);
2938 } else {
2939 GR->buildAssignPtr(B, ElemTy, Arg);
2940 }
2941 }
2942 }
2943}
2944
2945static FunctionType *getFunctionPointerElemType(Function *F,
2946 SPIRVGlobalRegistry *GR) {
2947 FunctionType *FTy = F->getFunctionType();
2948 bool IsNewFTy = false;
2949 SmallVector<Type *, 4> ArgTys;
2950 for (Argument &Arg : F->args()) {
2951 Type *ArgTy = Arg.getType();
2952 if (ArgTy->isPointerTy())
2953 if (Type *ElemTy = GR->findDeducedElementType(Val: &Arg)) {
2954 IsNewFTy = true;
2955 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
2956 }
2957 ArgTys.push_back(Elt: ArgTy);
2958 }
2959 return IsNewFTy
2960 ? FunctionType::get(Result: FTy->getReturnType(), Params: ArgTys, isVarArg: FTy->isVarArg())
2961 : FTy;
2962}
2963
2964bool SPIRVEmitIntrinsics::processFunctionPointers(Module &M) {
2965 SmallVector<Function *> Worklist;
2966 for (auto &F : M) {
2967 if (F.isIntrinsic())
2968 continue;
2969 if (F.isDeclaration()) {
2970 for (User *U : F.users()) {
2971 CallInst *CI = dyn_cast<CallInst>(Val: U);
2972 if (!CI || CI->getCalledFunction() != &F) {
2973 Worklist.push_back(Elt: &F);
2974 break;
2975 }
2976 }
2977 } else {
2978 if (F.user_empty())
2979 continue;
2980 Type *FPElemTy = GR->findDeducedElementType(Val: &F);
2981 if (!FPElemTy)
2982 FPElemTy = getFunctionPointerElemType(F: &F, GR);
2983 for (User *U : F.users()) {
2984 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
2985 if (!II || II->arg_size() != 3 || II->getOperand(i_nocapture: 0) != &F)
2986 continue;
2987 if (II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type ||
2988 II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
2989 GR->updateAssignType(AssignCI: II, Arg: &F, OfType: getNormalizedPoisonValue(Ty: FPElemTy));
2990 break;
2991 }
2992 }
2993 }
2994 }
2995 if (Worklist.empty())
2996 return false;
2997
2998 LLVMContext &Ctx = M.getContext();
2999 Function *SF = getOrCreateBackendServiceFunction(M);
3000 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: SF);
3001 IRBuilder<> IRB(BB);
3002
3003 for (Function *F : Worklist) {
3004 SmallVector<Value *> Args;
3005 for (const auto &Arg : F->args())
3006 Args.push_back(Elt: getNormalizedPoisonValue(Ty: Arg.getType()));
3007 IRB.CreateCall(Callee: F, Args);
3008 }
3009 IRB.CreateRetVoid();
3010
3011 return true;
3012}
3013
3014// Apply types parsed from demangled function declarations.
3015void SPIRVEmitIntrinsics::applyDemangledPtrArgTypes(IRBuilder<> &B) {
3016 DenseMap<Function *, CallInst *> Ptrcasts;
3017 for (auto It : FDeclPtrTys) {
3018 Function *F = It.first;
3019 for (auto *U : F->users()) {
3020 CallInst *CI = dyn_cast<CallInst>(Val: U);
3021 if (!CI || CI->getCalledFunction() != F)
3022 continue;
3023 unsigned Sz = CI->arg_size();
3024 for (auto [Idx, ElemTy] : It.second) {
3025 if (Idx >= Sz)
3026 continue;
3027 Value *Param = CI->getArgOperand(i: Idx);
3028 if (GR->findDeducedElementType(Val: Param) || isa<GlobalValue>(Val: Param))
3029 continue;
3030 if (Argument *Arg = dyn_cast<Argument>(Val: Param)) {
3031 if (!hasPointeeTypeAttr(Arg)) {
3032 B.SetInsertPointPastAllocas(Arg->getParent());
3033 B.SetCurrentDebugLocation(DebugLoc());
3034 GR->buildAssignPtr(B, ElemTy, Arg);
3035 }
3036 } else if (isaGEP(V: Param)) {
3037 replaceUsesOfWithSpvPtrcast(Op: Param, ElemTy: normalizeType(Ty: ElemTy), I: CI,
3038 Ptrcasts);
3039 } else if (isa<Instruction>(Val: Param)) {
3040 GR->addDeducedElementType(Val: Param, Ty: normalizeType(Ty: ElemTy));
3041 // insertAssignTypeIntrs() will complete buildAssignPtr()
3042 } else {
3043 B.SetInsertPoint(CI->getParent()
3044 ->getParent()
3045 ->getEntryBlock()
3046 .getFirstNonPHIOrDbgOrAlloca());
3047 GR->buildAssignPtr(B, ElemTy, Arg: Param);
3048 }
3049 CallInst *Ref = dyn_cast<CallInst>(Val: Param);
3050 if (!Ref)
3051 continue;
3052 Function *RefF = Ref->getCalledFunction();
3053 if (!RefF || !isPointerTy(T: RefF->getReturnType()) ||
3054 GR->findDeducedElementType(Val: RefF))
3055 continue;
3056 ElemTy = normalizeType(Ty: ElemTy);
3057 GR->addDeducedElementType(Val: RefF, Ty: ElemTy);
3058 GR->addReturnType(
3059 ArgF: RefF, DerivedTy: TypedPointerType::get(
3060 ElementType: ElemTy, AddressSpace: getPointerAddressSpace(T: RefF->getReturnType())));
3061 }
3062 }
3063 }
3064}
3065
3066GetElementPtrInst *
3067SPIRVEmitIntrinsics::simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP) {
3068 // getelementptr [0 x T], P, 0 (zero), I -> getelementptr T, P, I.
3069 // If type is 0-length array and first index is 0 (zero), drop both the
3070 // 0-length array type and the first index. This is a common pattern in
3071 // the IR, e.g. when using a zero-length array as a placeholder for a
3072 // flexible array such as unbound arrays.
3073 assert(GEP && "GEP is null");
3074 Type *SrcTy = GEP->getSourceElementType();
3075 SmallVector<Value *, 8> Indices(GEP->indices());
3076 ArrayType *ArrTy = dyn_cast<ArrayType>(Val: SrcTy);
3077 if (ArrTy && ArrTy->getNumElements() == 0 &&
3078 PatternMatch::match(V: Indices[0], P: PatternMatch::m_Zero())) {
3079 Indices.erase(CI: Indices.begin());
3080 SrcTy = ArrTy->getElementType();
3081 return GetElementPtrInst::Create(PointeeType: SrcTy, Ptr: GEP->getPointerOperand(), IdxList: Indices,
3082 NW: GEP->getNoWrapFlags(), NameStr: "",
3083 InsertBefore: GEP->getIterator());
3084 }
3085 return nullptr;
3086}
3087
3088void SPIRVEmitIntrinsics::emitUnstructuredLoopControls(Function &F,
3089 IRBuilder<> &B) {
3090 const SPIRVSubtarget *ST = TM.getSubtargetImpl(F);
3091 // Shaders use SPIRVStructurizer which emits OpLoopMerge via spv_loop_merge.
3092 if (ST->isShader())
3093 return;
3094
3095 if (ST->canUseExtension(
3096 E: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls)) {
3097 for (BasicBlock &BB : F) {
3098 Instruction *Term = BB.getTerminator();
3099 MDNode *LoopMD = Term->getMetadata(KindID: LLVMContext::MD_loop);
3100 if (!LoopMD)
3101 continue;
3102
3103 SmallVector<unsigned, 1> Ops =
3104 getSpirvLoopControlOperandsFromLoopMetadata(LoopMD);
3105 unsigned LC = Ops[0];
3106 if (LC == SPIRV::LoopControl::None)
3107 continue;
3108
3109 // Emit intrinsic: loop control mask + optional parameters.
3110 B.SetInsertPoint(Term);
3111 SmallVector<Value *, 4> IntrArgs;
3112 for (unsigned Op : Ops)
3113 IntrArgs.push_back(Elt: B.getInt32(C: Op));
3114 B.CreateIntrinsic(ID: Intrinsic::spv_loop_control_intel, Args: IntrArgs);
3115 }
3116 return;
3117 }
3118
3119 // For non-shader targets without the Intel extension, emit OpLoopMerge
3120 // using spv_loop_merge intrinsics, mirroring the structurizer approach.
3121 DominatorTree DT(F);
3122 LoopInfo LI(DT);
3123 if (LI.empty())
3124 return;
3125
3126 for (Loop *L : LI.getLoopsInPreorder()) {
3127 BasicBlock *Latch = L->getLoopLatch();
3128 if (!Latch)
3129 continue;
3130 BasicBlock *MergeBlock = L->getUniqueExitBlock();
3131 if (!MergeBlock)
3132 continue;
3133
3134 // Check for loop unroll metadata on the latch terminator.
3135 SmallVector<unsigned, 1> LoopControlOps =
3136 getSpirvLoopControlOperandsFromLoopMetadata(L);
3137 if (LoopControlOps[0] == SPIRV::LoopControl::None)
3138 continue;
3139
3140 BasicBlock *Header = L->getHeader();
3141 B.SetInsertPoint(Header->getTerminator());
3142 auto *MergeAddress = BlockAddress::get(F: &F, BB: MergeBlock);
3143 auto *ContinueAddress = BlockAddress::get(F: &F, BB: Latch);
3144 SmallVector<Value *, 4> Args = {MergeAddress, ContinueAddress};
3145 for (unsigned Imm : LoopControlOps)
3146 Args.emplace_back(Args: B.getInt32(C: Imm));
3147 B.CreateIntrinsic(ID: Intrinsic::spv_loop_merge, Args: {Args});
3148 }
3149}
3150
3151bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
3152 if (Func.isDeclaration())
3153 return false;
3154
3155 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: Func);
3156 GR = ST.getSPIRVGlobalRegistry();
3157
3158 if (!CurrF)
3159 HaveFunPtrs =
3160 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers);
3161
3162 CurrF = &Func;
3163 IRBuilder<> B(Func.getContext());
3164 AggrConsts.clear();
3165 AggrConstTypes.clear();
3166 AggrStores.clear();
3167 DeletedInstrs.clear();
3168
3169 // Fix GEP result types ahead of inference, and simplify if possible.
3170 // Data structure for dead instructions that were simplified and replaced.
3171 SmallPtrSet<Instruction *, 4> DeadInsts;
3172 for (auto &I : instructions(F&: Func)) {
3173 auto *GEP = dyn_cast<GetElementPtrInst>(Val: &I);
3174 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
3175
3176 if ((!GEP && !SGEP) || GR->findDeducedElementType(Val: &I))
3177 continue;
3178
3179 if (SGEP) {
3180 GR->addDeducedElementType(Val: SGEP,
3181 Ty: normalizeType(Ty: SGEP->getResultElementType()));
3182 continue;
3183 }
3184
3185 GetElementPtrInst *NewGEP = simplifyZeroLengthArrayGepInst(GEP);
3186 if (NewGEP) {
3187 GEP->replaceAllUsesWith(V: NewGEP);
3188 DeadInsts.insert(Ptr: GEP);
3189 GEP = NewGEP;
3190 }
3191 if (Type *GepTy = getGEPType(Ref: GEP))
3192 GR->addDeducedElementType(Val: GEP, Ty: normalizeType(Ty: GepTy));
3193 }
3194 // Remove dead instructions that were simplified and replaced.
3195 for (auto *I : DeadInsts) {
3196 assert(I->use_empty() && "Dead instruction should not have any uses left");
3197 I->eraseFromParent();
3198 }
3199
3200 processParamTypesByFunHeader(F: CurrF, B);
3201
3202 // StoreInst's operand type can be changed during the next
3203 // transformations, so we need to store it in the set. Also store already
3204 // transformed types.
3205 for (auto &I : instructions(F&: Func)) {
3206 StoreInst *SI = dyn_cast<StoreInst>(Val: &I);
3207 if (!SI)
3208 continue;
3209 Type *ElTy = SI->getValueOperand()->getType();
3210 if (ElTy->isAggregateType() || ElTy->isVectorTy())
3211 AggrStores.insert(V: &I);
3212 }
3213
3214 B.SetInsertPoint(TheBB: &Func.getEntryBlock(), IP: Func.getEntryBlock().begin());
3215 for (auto &GV : Func.getParent()->globals())
3216 processGlobalValue(GV, B);
3217
3218 preprocessUndefs(B);
3219 preprocessCompositeConstants(B);
3220
3221 for (BasicBlock &BB : Func)
3222 for (PHINode &Phi : BB.phis())
3223 if (Phi.getType()->isAggregateType()) {
3224 AggrConstTypes[&Phi] = Phi.getType();
3225 Phi.mutateType(Ty: B.getInt32Ty());
3226 }
3227
3228 preprocessBoolVectorBitcasts(F&: Func);
3229 SmallVector<Instruction *> Worklist(
3230 llvm::make_pointer_range(Range: instructions(F&: Func)));
3231
3232 applyDemangledPtrArgTypes(B);
3233
3234 // Pass forward: use operand to deduce instructions result.
3235 for (auto &I : Worklist) {
3236 // Don't emit intrinsincs for convergence intrinsics.
3237 if (isConvergenceIntrinsic(I))
3238 continue;
3239
3240 bool Postpone = insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: false);
3241 // if Postpone is true, we can't decide on pointee type yet
3242 insertAssignTypeIntrs(I, B);
3243 insertPtrCastOrAssignTypeInstr(I, B);
3244 insertSpirvDecorations(I, B);
3245 // if instruction requires a pointee type set, let's check if we know it
3246 // already, and force it to be i8 if not
3247 if (Postpone && !GR->findAssignPtrTypeInstr(Val: I))
3248 insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: true);
3249
3250 if (auto *FPI = dyn_cast<ConstrainedFPIntrinsic>(Val: I))
3251 useRoundingMode(FPI, B);
3252 }
3253
3254 // Pass backward: use instructions results to specify/update/cast operands
3255 // where needed.
3256 SmallPtrSet<Instruction *, 4> IncompleteRets;
3257 for (auto &I : llvm::reverse(C: instructions(F&: Func)))
3258 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets);
3259
3260 // Pass forward for PHIs only, their operands are not preceed the
3261 // instruction in meaning of `instructions(Func)`.
3262 for (BasicBlock &BB : Func)
3263 for (PHINode &Phi : BB.phis())
3264 if (isPointerTy(T: Phi.getType()))
3265 deduceOperandElementType(I: &Phi, IncompleteRets: nullptr);
3266
3267 for (auto *I : Worklist) {
3268 if (DeletedInstrs.count(Ptr: I))
3269 continue;
3270 TrackConstants = true;
3271 if (!I->getType()->isVoidTy() || isa<StoreInst>(Val: I))
3272 setInsertPointAfterDef(B, I);
3273 // Visitors return either the original/newly created instruction for
3274 // further processing, nullptr otherwise.
3275 I = visit(I&: *I);
3276 if (!I)
3277 continue;
3278
3279 // Don't emit intrinsics for convergence operations.
3280 if (isConvergenceIntrinsic(I))
3281 continue;
3282
3283 addSaturatedDecorationToIntrinsic(I, B);
3284 processInstrAfterVisit(I, B);
3285 }
3286
3287 emitUnstructuredLoopControls(F&: Func, B);
3288
3289 return true;
3290}
3291
3292// Try to deduce a better type for pointers to untyped ptr.
3293bool SPIRVEmitIntrinsics::postprocessTypes(Module &M) {
3294 if (!GR || TodoTypeSz == 0)
3295 return false;
3296
3297 unsigned SzTodo = TodoTypeSz;
3298 DenseMap<Value *, SmallPtrSet<Value *, 4>> ToProcess;
3299 for (auto [Op, Enabled] : TodoType) {
3300 // TODO: add isa<CallInst>(Op) to continue
3301 if (!Enabled || isaGEP(V: Op))
3302 continue;
3303 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Op);
3304 Type *KnownTy = GR->findDeducedElementType(Val: Op);
3305 if (!KnownTy || !AssignCI)
3306 continue;
3307 assert(Op == AssignCI->getArgOperand(0));
3308 // Try to improve the type deduced after all Functions are processed.
3309 if (auto *CI = dyn_cast<Instruction>(Val: Op)) {
3310 CurrF = CI->getParent()->getParent();
3311 std::unordered_set<Value *> Visited;
3312 if (Type *ElemTy = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8: false, IgnoreKnownType: true)) {
3313 if (ElemTy != KnownTy) {
3314 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3315 propagateElemType(Op: CI, ElemTy, VisitedSubst);
3316 eraseTodoType(Op);
3317 continue;
3318 }
3319 }
3320 }
3321
3322 if (Op->hasUseList()) {
3323 for (User *U : Op->users()) {
3324 Instruction *Inst = dyn_cast<Instruction>(Val: U);
3325 if (Inst && !isa<IntrinsicInst>(Val: Inst))
3326 ToProcess[Inst].insert(Ptr: Op);
3327 }
3328 }
3329 }
3330 if (TodoTypeSz == 0)
3331 return true;
3332
3333 for (auto &F : M) {
3334 CurrF = &F;
3335 SmallPtrSet<Instruction *, 4> IncompleteRets;
3336 for (auto &I : llvm::reverse(C: instructions(F))) {
3337 auto It = ToProcess.find(Val: &I);
3338 if (It == ToProcess.end())
3339 continue;
3340 It->second.remove_if(P: [this](Value *V) { return !isTodoType(Op: V); });
3341 if (It->second.size() == 0)
3342 continue;
3343 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets, AskOps: &It->second, IsPostprocessing: true);
3344 if (TodoTypeSz == 0)
3345 return true;
3346 }
3347 }
3348
3349 return SzTodo > TodoTypeSz;
3350}
3351
3352// Parse and store argument types of function declarations where needed.
3353void SPIRVEmitIntrinsics::parseFunDeclarations(Module &M) {
3354 for (auto &F : M) {
3355 if (!F.isDeclaration() || F.isIntrinsic())
3356 continue;
3357 // get the demangled name
3358 std::string DemangledName = getOclOrSpirvBuiltinDemangledName(Name: F.getName());
3359 if (DemangledName.empty())
3360 continue;
3361 // allow only OpGroupAsyncCopy use case at the moment
3362 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F);
3363 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
3364 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
3365 if (Opcode != SPIRV::OpGroupAsyncCopy)
3366 continue;
3367 // find pointer arguments
3368 SmallVector<unsigned> Idxs;
3369 for (unsigned OpIdx = 0; OpIdx < F.arg_size(); ++OpIdx) {
3370 Argument *Arg = F.getArg(i: OpIdx);
3371 if (isPointerTy(T: Arg->getType()) && !hasPointeeTypeAttr(Arg))
3372 Idxs.push_back(Elt: OpIdx);
3373 }
3374 if (!Idxs.size())
3375 continue;
3376 // parse function arguments
3377 LLVMContext &Ctx = F.getContext();
3378 SmallVector<StringRef, 10> TypeStrs;
3379 SPIRV::parseBuiltinTypeStr(BuiltinArgsTypeStrs&: TypeStrs, DemangledCall: DemangledName, Ctx);
3380 if (!TypeStrs.size())
3381 continue;
3382 // find type info for pointer arguments
3383 for (unsigned Idx : Idxs) {
3384 if (Idx >= TypeStrs.size())
3385 continue;
3386 if (Type *ElemTy =
3387 SPIRV::parseBuiltinCallArgumentType(TypeStr: TypeStrs[Idx].trim(), Ctx))
3388 if (TypedPointerType::isValidElementType(ElemTy) &&
3389 !ElemTy->isTargetExtTy())
3390 FDeclPtrTys[&F].push_back(Elt: std::make_pair(x&: Idx, y&: ElemTy));
3391 }
3392 }
3393}
3394
3395bool SPIRVEmitIntrinsics::processMaskedMemIntrinsic(IntrinsicInst &I) {
3396 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
3397
3398 if (I.getIntrinsicID() == Intrinsic::masked_gather) {
3399 if (!ST.canUseExtension(
3400 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
3401 I.getContext().emitError(
3402 I: &I, ErrorStr: "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter "
3403 "extension");
3404 // Replace with poison to allow compilation to continue and report error.
3405 I.replaceAllUsesWith(V: PoisonValue::get(T: I.getType()));
3406 I.eraseFromParent();
3407 return true;
3408 }
3409
3410 IRBuilder<> B(&I);
3411
3412 Value *Ptrs = I.getArgOperand(i: 0);
3413 Value *Mask = I.getArgOperand(i: 1);
3414 Value *Passthru = I.getArgOperand(i: 2);
3415
3416 // Alignment is stored as a parameter attribute, not as a regular parameter.
3417 uint32_t Alignment = I.getParamAlign(ArgNo: 0).valueOrOne().value();
3418
3419 SmallVector<Value *, 4> Args = {Ptrs, B.getInt32(C: Alignment), Mask,
3420 Passthru};
3421 SmallVector<Type *, 4> Types = {I.getType(), Ptrs->getType(),
3422 Mask->getType(), Passthru->getType()};
3423
3424 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_masked_gather, Types, Args);
3425 I.replaceAllUsesWith(V: NewI);
3426 I.eraseFromParent();
3427 return true;
3428 }
3429
3430 if (I.getIntrinsicID() == Intrinsic::masked_scatter) {
3431 if (!ST.canUseExtension(
3432 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
3433 I.getContext().emitError(
3434 I: &I, ErrorStr: "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter "
3435 "extension");
3436 // Erase the intrinsic to allow compilation to continue and report error.
3437 I.eraseFromParent();
3438 return true;
3439 }
3440
3441 IRBuilder<> B(&I);
3442
3443 Value *Values = I.getArgOperand(i: 0);
3444 Value *Ptrs = I.getArgOperand(i: 1);
3445 Value *Mask = I.getArgOperand(i: 2);
3446
3447 // Alignment is stored as a parameter attribute on the ptrs parameter (arg
3448 // 1).
3449 uint32_t Alignment = I.getParamAlign(ArgNo: 1).valueOrOne().value();
3450
3451 SmallVector<Value *, 4> Args = {Values, Ptrs, B.getInt32(C: Alignment), Mask};
3452 SmallVector<Type *, 3> Types = {Values->getType(), Ptrs->getType(),
3453 Mask->getType()};
3454
3455 B.CreateIntrinsic(ID: Intrinsic::spv_masked_scatter, Types, Args);
3456 I.eraseFromParent();
3457 return true;
3458 }
3459
3460 return false;
3461}
3462
3463// SPIR-V doesn't support bitcasts involving vector boolean type. Decompose such
3464// bitcasts into element-wise operations before building instructions
3465// worklist, so new instructions are properly visited and converted to
3466// SPIR-V intrinsics.
3467void SPIRVEmitIntrinsics::preprocessBoolVectorBitcasts(Function &F) {
3468 struct BoolVecBitcast {
3469 BitCastInst *BC;
3470 FixedVectorType *BoolVecTy;
3471 bool SrcIsBoolVec;
3472 };
3473
3474 auto getAsBoolVec = [](Type *Ty) -> FixedVectorType * {
3475 auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
3476 return (VTy && VTy->getElementType()->isIntegerTy(Bitwidth: 1)) ? VTy : nullptr;
3477 };
3478
3479 SmallVector<BoolVecBitcast, 4> ToReplace;
3480 for (auto &I : instructions(F)) {
3481 auto *BC = dyn_cast<BitCastInst>(Val: &I);
3482 if (!BC)
3483 continue;
3484 if (auto *BVTy = getAsBoolVec(BC->getSrcTy()))
3485 ToReplace.push_back(Elt: {.BC: BC, .BoolVecTy: BVTy, .SrcIsBoolVec: true});
3486 else if (auto *BVTy = getAsBoolVec(BC->getDestTy()))
3487 ToReplace.push_back(Elt: {.BC: BC, .BoolVecTy: BVTy, .SrcIsBoolVec: false});
3488 }
3489
3490 for (auto &[BC, BoolVecTy, SrcIsBoolVec] : ToReplace) {
3491 IRBuilder<> B(BC);
3492 Value *Src = BC->getOperand(i_nocapture: 0);
3493 unsigned BoolVecN = BoolVecTy->getNumElements();
3494 // Use iN as the scalar intermediate type for the bool vector side.
3495 Type *IntTy = B.getIntNTy(N: BoolVecN);
3496
3497 // Convert source to scalar integer.
3498 Value *IntVal;
3499 if (SrcIsBoolVec) {
3500 // Extract each bool, zext, shift, and OR.
3501 IntVal = ConstantInt::get(Ty: IntTy, V: 0);
3502 for (unsigned I = 0; I < BoolVecN; ++I) {
3503 Value *Elem = B.CreateExtractElement(Vec: Src, Idx: B.getInt32(C: I));
3504 Value *Ext = B.CreateZExt(V: Elem, DestTy: IntTy);
3505 if (I > 0)
3506 Ext = B.CreateShl(LHS: Ext, RHS: ConstantInt::get(Ty: IntTy, V: I));
3507 IntVal = B.CreateOr(LHS: IntVal, RHS: Ext);
3508 }
3509 } else {
3510 // Source is a non-bool type. If it's already a scalar integer, use it
3511 // directly, otherwise bitcast to iN first.
3512 IntVal = Src;
3513 if (!Src->getType()->isIntegerTy())
3514 IntVal = B.CreateBitCast(V: Src, DestTy: IntTy);
3515 }
3516
3517 // Convert scalar integer to destination type.
3518 Value *Result;
3519 if (!SrcIsBoolVec) {
3520 // Test each bit with AND + icmp.
3521 Result = PoisonValue::get(T: BoolVecTy);
3522 for (unsigned I = 0; I < BoolVecN; ++I) {
3523 Value *Mask = ConstantInt::get(Ty: IntTy, V: APInt::getOneBitSet(numBits: BoolVecN, BitNo: I));
3524 Value *And = B.CreateAnd(LHS: IntVal, RHS: Mask);
3525 Value *Cmp = B.CreateICmpNE(LHS: And, RHS: ConstantInt::get(Ty: IntTy, V: 0));
3526 Result = B.CreateInsertElement(Vec: Result, NewElt: Cmp, Idx: B.getInt32(C: I));
3527 }
3528 } else {
3529 // Destination is a non-bool type. If it's a scalar integer, use IntVal
3530 // directly, otherwise bitcast from iN.
3531 Result = IntVal;
3532 if (!BC->getDestTy()->isIntegerTy())
3533 Result = B.CreateBitCast(V: IntVal, DestTy: BC->getDestTy());
3534 }
3535
3536 BC->replaceAllUsesWith(V: Result);
3537 BC->eraseFromParent();
3538 }
3539}
3540
3541bool SPIRVEmitIntrinsics::convertMaskedMemIntrinsics(Module &M) {
3542 bool Changed = false;
3543
3544 for (Function &F : make_early_inc_range(Range&: M)) {
3545 if (!F.isIntrinsic())
3546 continue;
3547 Intrinsic::ID IID = F.getIntrinsicID();
3548 if (IID != Intrinsic::masked_gather && IID != Intrinsic::masked_scatter)
3549 continue;
3550
3551 for (User *U : make_early_inc_range(Range: F.users())) {
3552 if (auto *II = dyn_cast<IntrinsicInst>(Val: U))
3553 Changed |= processMaskedMemIntrinsic(I&: *II);
3554 }
3555
3556 if (F.use_empty())
3557 F.eraseFromParent();
3558 }
3559
3560 return Changed;
3561}
3562
3563bool SPIRVEmitIntrinsics::runOnModule(Module &M) {
3564 bool Changed = false;
3565
3566 Changed |= convertMaskedMemIntrinsics(M);
3567
3568 parseFunDeclarations(M);
3569 insertConstantsForFPFastMathDefault(M);
3570 GVUsers.init(M);
3571
3572 TodoType.clear();
3573 for (auto &F : M)
3574 Changed |= runOnFunction(Func&: F);
3575
3576 // Specify function parameters after all functions were processed.
3577 for (auto &F : M) {
3578 // check if function parameter types are set
3579 CurrF = &F;
3580 if (!F.isDeclaration() && !F.isIntrinsic()) {
3581 IRBuilder<> B(F.getContext());
3582 processParamTypes(F: &F, B);
3583 }
3584 }
3585
3586 CanTodoType = false;
3587 Changed |= postprocessTypes(M);
3588
3589 if (HaveFunPtrs)
3590 Changed |= processFunctionPointers(M);
3591
3592 return Changed;
3593}
3594
3595PreservedAnalyses
3596llvm::SPIRVEmitIntrinsicsPass::run(Module &M, ModuleAnalysisManager &AM) {
3597 SPIRVEmitIntrinsics Legacy(TM);
3598 if (Legacy.runOnModule(M))
3599 return PreservedAnalyses::none();
3600 return PreservedAnalyses::all();
3601}
3602
3603ModulePass *llvm::createSPIRVEmitIntrinsicsPass(const SPIRVTargetMachine &TM) {
3604 return new SPIRVEmitIntrinsics(TM);
3605}
3606