1//===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass emits SPIRV intrinsics keeping essential high-level information for
10// the translation of LLVM IR to SPIR-V.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRV.h"
15#include "SPIRVBuiltins.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVTargetMachine.h"
18#include "SPIRVUtils.h"
19#include "llvm/ADT/DenseMap.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/StringSet.h"
23#include "llvm/Analysis/LoopInfo.h"
24#include "llvm/IR/IRBuilder.h"
25#include "llvm/IR/InstIterator.h"
26#include "llvm/IR/InstVisitor.h"
27#include "llvm/IR/IntrinsicsSPIRV.h"
28#include "llvm/IR/PatternMatch.h"
29#include "llvm/IR/TypedPointerType.h"
30#include "llvm/IR/Value.h"
31#include "llvm/Support/CommandLine.h"
32#include "llvm/Support/Debug.h"
33#include "llvm/Transforms/Utils/Local.h"
34
35#include <cassert>
36#include <optional>
37#include <queue>
38
39// This pass performs the following transformation on LLVM IR level required
40// for the following translation to SPIR-V:
41// - replaces direct usages of aggregate constants with target-specific
42// intrinsics;
43// - replaces aggregates-related instructions (extract/insert, ld/st, etc)
44// with a target-specific intrinsics;
45// - emits intrinsics for the global variable initializers since IRTranslator
46// doesn't handle them and it's not very convenient to translate them
47// ourselves;
48// - emits intrinsics to keep track of the string names assigned to the values;
49// - emits intrinsics to keep track of constants (this is necessary to have an
50// LLVM IR constant after the IRTranslation is completed) for their further
51// deduplication;
52// - emits intrinsics to keep track of original LLVM types of the values
53// to be able to emit proper SPIR-V types eventually.
54//
55// TODO: consider removing spv.track.constant in favor of spv.assign.type.
56
57using namespace llvm;
58using namespace llvm::PatternMatch;
59
60#define DEBUG_TYPE "spirv-emit-intrinsics"
61
62static cl::opt<bool>
63 SpirvEmitOpNames("spirv-emit-op-names",
64 cl::desc("Emit OpName for all instructions"),
65 cl::init(Val: false));
66
67namespace llvm::SPIRV {
68#define GET_BuiltinGroup_DECL
69#include "SPIRVGenTables.inc"
70} // namespace llvm::SPIRV
71
72namespace {
73// This class keeps track of which functions reference which global variables.
74class GlobalVariableUsers {
75 template <typename T1, typename T2>
76 using OneToManyMapTy = DenseMap<T1, SmallPtrSet<T2, 4>>;
77
78 OneToManyMapTy<const GlobalVariable *, const Function *> GlobalIsUsedByFun;
79
80 void collectGlobalUsers(
81 const GlobalVariable *GV,
82 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
83 &GlobalIsUsedByGlobal) {
84 SmallVector<const Value *> Stack = {GV->user_begin(), GV->user_end()};
85 while (!Stack.empty()) {
86 const Value *V = Stack.pop_back_val();
87
88 if (const Instruction *I = dyn_cast<Instruction>(Val: V)) {
89 GlobalIsUsedByFun[GV].insert(Ptr: I->getFunction());
90 continue;
91 }
92
93 if (const GlobalVariable *UserGV = dyn_cast<GlobalVariable>(Val: V)) {
94 GlobalIsUsedByGlobal[GV].insert(Ptr: UserGV);
95 continue;
96 }
97
98 if (const Constant *C = dyn_cast<Constant>(Val: V))
99 Stack.append(in_start: C->user_begin(), in_end: C->user_end());
100 }
101 }
102
103 bool propagateGlobalToGlobalUsers(
104 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
105 &GlobalIsUsedByGlobal) {
106 SmallVector<const GlobalVariable *> OldUsersGlobals;
107 bool Changed = false;
108 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
109 OldUsersGlobals.assign(in_start: UserGlobals.begin(), in_end: UserGlobals.end());
110 for (const GlobalVariable *UserGV : OldUsersGlobals) {
111 auto It = GlobalIsUsedByGlobal.find(Val: UserGV);
112 if (It == GlobalIsUsedByGlobal.end())
113 continue;
114 Changed |= set_union(S1&: UserGlobals, S2: It->second);
115 }
116 }
117 return Changed;
118 }
119
120 void propagateGlobalToFunctionReferences(
121 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
122 &GlobalIsUsedByGlobal) {
123 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
124 auto &UserFunctions = GlobalIsUsedByFun[GV];
125 for (const GlobalVariable *UserGV : UserGlobals) {
126 auto It = GlobalIsUsedByFun.find(Val: UserGV);
127 if (It == GlobalIsUsedByFun.end())
128 continue;
129 set_union(S1&: UserFunctions, S2: It->second);
130 }
131 }
132 }
133
134public:
135 void init(Module &M) {
136 // Collect which global variables are referenced by which global variables
137 // and which functions reference each global variables.
138 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
139 GlobalIsUsedByGlobal;
140 GlobalIsUsedByFun.clear();
141 for (GlobalVariable &GV : M.globals())
142 collectGlobalUsers(GV: &GV, GlobalIsUsedByGlobal);
143
144 // Compute indirect references by iterating until a fixed point is reached.
145 while (propagateGlobalToGlobalUsers(GlobalIsUsedByGlobal))
146 (void)0;
147
148 propagateGlobalToFunctionReferences(GlobalIsUsedByGlobal);
149 }
150
151 using FunctionSetType = typename decltype(GlobalIsUsedByFun)::mapped_type;
152 const FunctionSetType &
153 getTransitiveUserFunctions(const GlobalVariable &GV) const {
154 auto It = GlobalIsUsedByFun.find(Val: &GV);
155 if (It != GlobalIsUsedByFun.end())
156 return It->second;
157
158 static const FunctionSetType Empty{};
159 return Empty;
160 }
161};
162
163static bool isaGEP(const Value *V) {
164 return isa<StructuredGEPInst>(Val: V) || isa<GetElementPtrInst>(Val: V);
165}
166
167// If Ty is a byte-addressing type, return the multiplier for the offset.
168// Otherwise return std::nullopt.
169static std::optional<uint64_t> getByteAddressingMultiplier(Type *Ty) {
170 if (Ty == IntegerType::getInt8Ty(C&: Ty->getContext())) {
171 return 1;
172 }
173 if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
174 if (AT->getElementType() == IntegerType::getInt8Ty(C&: Ty->getContext())) {
175 return AT->getNumElements();
176 }
177 }
178 return std::nullopt;
179}
180
181class SPIRVEmitIntrinsicsImpl
182 : public InstVisitor<SPIRVEmitIntrinsicsImpl, Instruction *> {
183 const SPIRVTargetMachine &TM;
184 SPIRVGlobalRegistry *GR = nullptr;
185 Function *CurrF = nullptr;
186 bool TrackConstants = true;
187 bool HaveFunPtrs = false;
188 bool CanUseAnyVectorRank = false;
189 DenseMap<Instruction *, Constant *> AggrConsts;
190 DenseMap<Instruction *, Type *> AggrConstTypes;
191 SmallPtrSet<Instruction *, 0> AggrStores;
192 GlobalVariableUsers GVUsers;
193 SmallPtrSet<Value *, 0> Named;
194
195 // map of function declarations to <pointer arg index => element type>
196 DenseMap<Function *, SmallVector<std::pair<unsigned, Type *>>> FDeclPtrTys;
197
198 // a register of Instructions that don't have a complete type definition
199 bool CanTodoType = true;
200 unsigned TodoTypeSz = 0;
201 DenseMap<Value *, bool> TodoType;
202 void insertTodoType(Value *Op) {
203 // TODO: add isa<CallInst>(Op) to no-insert
204 if (CanTodoType && !isaGEP(V: Op)) {
205 auto It = TodoType.try_emplace(Key: Op, Args: true);
206 if (It.second)
207 ++TodoTypeSz;
208 }
209 }
210 void eraseTodoType(Value *Op) {
211 auto It = TodoType.find(Val: Op);
212 if (It != TodoType.end() && It->second) {
213 It->second = false;
214 --TodoTypeSz;
215 }
216 }
217 bool isTodoType(Value *Op) {
218 if (isaGEP(V: Op))
219 return false;
220 auto It = TodoType.find(Val: Op);
221 return It != TodoType.end() && It->second;
222 }
223 // a register of Instructions that were visited by deduceOperandElementType()
224 // to validate operand types with an instruction
225 SmallPtrSet<Instruction *, 0> TypeValidated;
226
227 // well known result types of builtins
228 enum WellKnownTypes { Event };
229
230 // deduce element type of untyped pointers
231 Type *deduceElementType(Value *I, bool UnknownElemTypeI8);
232 Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8);
233 Type *deduceElementTypeHelper(Value *I, SmallPtrSetImpl<Value *> &Visited,
234 bool UnknownElemTypeI8,
235 bool IgnoreKnownType = false);
236 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
237 bool UnknownElemTypeI8);
238 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
239 SmallPtrSetImpl<Value *> &Visited,
240 bool UnknownElemTypeI8);
241 Type *deduceElementTypeByUsersDeep(Value *Op,
242 SmallPtrSetImpl<Value *> &Visited,
243 bool UnknownElemTypeI8);
244 void maybeAssignPtrType(Type *&Ty, Value *I, Type *RefTy,
245 bool UnknownElemTypeI8);
246
247 // deduce nested types of composites
248 Type *deduceNestedTypeHelper(User *U, bool UnknownElemTypeI8);
249 Type *deduceNestedTypeHelper(User *U, Type *Ty,
250 SmallPtrSetImpl<Value *> &Visited,
251 bool UnknownElemTypeI8);
252
253 // deduce Types of operands of the Instruction if possible
254 void
255 deduceOperandElementType(Instruction *I,
256 SmallPtrSetImpl<Instruction *> *IncompleteRets,
257 const SmallPtrSetImpl<Value *> *AskOps = nullptr,
258 bool IsPostprocessing = false);
259
260 void preprocessCompositeConstants(IRBuilder<> &B);
261 Value *lowerUndefOrPoison(Value *Op, IRBuilder<> &B, bool HasPoisonExt);
262 void preprocessUndefsAndPoisons(IRBuilder<> &B);
263 void insertCompositeAggregateArms(Instruction *I, IRBuilder<> &B);
264 void simplifyNullAddrSpaceCasts();
265
266 Type *reconstructType(Value *Op, bool UnknownElemTypeI8,
267 bool IsPostprocessing);
268
269 void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
270 void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
271 bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
272 bool UnknownElemTypeI8);
273 void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B);
274 void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V,
275 IRBuilder<> &B);
276 void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer,
277 Type *ExpectedElementType,
278 unsigned OperandToReplace,
279 IRBuilder<> &B);
280 void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
281 bool shouldTryToAddMemAliasingDecoration(Instruction *Inst);
282 void insertSpirvDecorations(Instruction *I, IRBuilder<> &B);
283 void insertConstantsForFPFastMathDefault(Module &M);
284 Value *buildSpvUndefComposite(Type *AggrTy, IRBuilder<> &B);
285 void reconstructAggregateReturns(Function &Func, IRBuilder<> &B);
286 void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
287 void processParamTypes(Function *F, IRBuilder<> &B);
288 void processParamTypesByFunHeader(Function *F, IRBuilder<> &B);
289 Type *deduceFunParamElementType(Function *F, unsigned OpIdx);
290 Type *deduceFunParamElementType(Function *F, unsigned OpIdx,
291 SmallPtrSetImpl<Function *> &FVisited);
292
293 bool deduceOperandElementTypeCalledFunction(
294 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
295 Type *&KnownElemTy, bool &Incomplete);
296 void deduceOperandElementTypeFunctionPointer(
297 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
298 Type *&KnownElemTy, bool IsPostprocessing);
299 bool deduceOperandElementTypeFunctionRet(
300 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
301 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing,
302 Type *&KnownElemTy, Value *Op, Function *F);
303
304 CallInst *buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy);
305 void replaceUsesOfWithSpvPtrcast(Value *Op, Type *ElemTy, Instruction *I,
306 DenseMap<Function *, CallInst *> Ptrcasts);
307 void propagateElemType(Value *Op, Type *ElemTy,
308 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
309 void
310 propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
311 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
312 void propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
313 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
314 SmallPtrSetImpl<Value *> &Visited,
315 DenseMap<Function *, CallInst *> Ptrcasts);
316
317 void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
318 void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src,
319 Instruction *Dest, bool DeleteOld = true);
320
321 void applyDemangledPtrArgTypes(IRBuilder<> &B);
322
323 GetElementPtrInst *simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP);
324
325 bool runOnFunction(Function &F);
326 bool postprocessTypes(Module &M);
327 bool processFunctionPointers(Module &M);
328 void parseFunDeclarations(Module &M);
329 void useRoundingMode(ConstrainedFPIntrinsic *FPI, IRBuilder<> &B);
330 bool processMaskedMemIntrinsic(IntrinsicInst &I);
331 bool convertMaskedMemIntrinsics(Module &M);
332 void preprocessBoolVectorBitcasts(Function &F);
333
334 void emitUnstructuredLoopControls(Function &F, IRBuilder<> &B);
335
336 // Tries to walk the type accessed by the given GEP instruction.
337 // For each nested type access, one of the 2 callbacks is called:
338 // - OnLiteralIndexing when the index is a known constant value.
339 // Parameters:
340 // PointedType: the pointed type resulting of this indexing.
341 // If the parent type is an array, this is the index in the array.
342 // If the parent type is a struct, this is the field index.
343 // Index: index of the element in the parent type.
344 // - OnDynamnicIndexing when the index is a non-constant value.
345 // This callback is only called when indexing into an array.
346 // Parameters:
347 // ElementType: the type of the elements stored in the parent array.
348 // Offset: the Value* containing the byte offset into the array.
349 // Multiplier: a scaling factor for the offset.
350 // Return true if an error occurred during the walk, false otherwise.
351 bool walkLogicalAccessChain(
352 GetElementPtrInst &GEP,
353 const std::function<void(Type *PointedType, uint64_t Index)>
354 &OnLiteralIndexing,
355 const std::function<void(Type *ElementType, Value *Offset,
356 uint64_t Multiplier)> &OnDynamicIndexing);
357
358 bool walkLogicalAccessChainDynamic(
359 Type *CurType, Value *Operand, uint64_t Multiplier,
360 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
361 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing);
362
363 bool walkLogicalAccessChainConstant(
364 Type *CurType, uint64_t Offset,
365 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing);
366
367 // Returns the type accessed using the given GEP instruction by relying
368 // on the GEP type.
369 // FIXME: GEP types are not supposed to be used to retrieve the pointed
370 // type. This must be fixed.
371 Type *getGEPType(GetElementPtrInst *GEP);
372
373 // Returns the type accessed using the given GEP instruction by walking
374 // the source type using the GEP indices.
375 // FIXME: without help from the frontend, this method cannot reliably retrieve
376 // the stored type, nor can robustly determine the depth of the type
377 // we are accessing.
378 Type *getGEPTypeLogical(GetElementPtrInst *GEP);
379
380 Instruction *buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP);
381
382public:
383 SPIRVEmitIntrinsicsImpl(const SPIRVTargetMachine &TM) : TM(TM) {}
384 Instruction *visitInstruction(Instruction &I) { return &I; }
385 Instruction *visitSwitchInst(SwitchInst &I);
386 Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
387 Instruction *visitIntrinsicInst(IntrinsicInst &I);
388 Instruction *visitBitCastInst(BitCastInst &I);
389 Instruction *visitInsertElementInst(InsertElementInst &I);
390 Instruction *visitExtractElementInst(ExtractElementInst &I);
391 Instruction *visitInsertValueInst(InsertValueInst &I);
392 Instruction *visitExtractValueInst(ExtractValueInst &I);
393 Instruction *visitLoadInst(LoadInst &I);
394 Instruction *visitStoreInst(StoreInst &I);
395 Instruction *visitAllocaInst(AllocaInst &I);
396 Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
397 Instruction *visitUnreachableInst(UnreachableInst &I);
398 Instruction *visitCallInst(CallInst &I);
399
400 bool runOnModule(Module &M);
401};
402
403class SPIRVEmitIntrinsicsLegacy : public ModulePass {
404 const SPIRVTargetMachine &TM;
405
406public:
407 static char ID;
408 SPIRVEmitIntrinsicsLegacy(const SPIRVTargetMachine &TM)
409 : ModulePass(ID), TM(TM) {}
410
411 StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
412
413 bool runOnModule(Module &M) override {
414 return SPIRVEmitIntrinsicsImpl(TM).runOnModule(M);
415 }
416};
417
418bool isConvergenceIntrinsic(const Instruction *I) {
419 return match(V: I, P: m_AnyIntrinsic<Intrinsic::experimental_convergence_entry,
420 Intrinsic::experimental_convergence_loop,
421 Intrinsic::experimental_convergence_anchor>());
422}
423
424bool expectIgnoredInIRTranslation(const Instruction *I) {
425 return match(V: I, P: m_AnyIntrinsic<Intrinsic::invariant_start,
426 Intrinsic::spv_resource_handlefrombinding,
427 Intrinsic::spv_resource_getbasepointer,
428 Intrinsic::spv_resource_getpointer>());
429}
430
431// Returns the source pointer from `I` ignoring intermediate ptrcast.
432Value *getPointerRoot(Value *I) {
433 Value *V;
434 if (match(V: I, P: m_Intrinsic<Intrinsic::spv_ptrcast>(Ops: m_Value(V))))
435 return getPointerRoot(I: V);
436 return I;
437}
438
439} // namespace
440
441char SPIRVEmitIntrinsicsLegacy::ID = 0;
442
443INITIALIZE_PASS(SPIRVEmitIntrinsicsLegacy, "spirv-emit-intrinsics",
444 "SPIRV emit intrinsics", false, false)
445
446static inline bool isAssignTypeInstr(const Instruction *I) {
447 return match(V: I, P: m_Intrinsic<Intrinsic::spv_assign_type>());
448}
449
450static bool isMemInstrToReplace(Instruction *I) {
451 return isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I) || isa<InsertValueInst>(Val: I) ||
452 isa<ExtractValueInst>(Val: I) || isa<AtomicCmpXchgInst>(Val: I);
453}
454
455static bool isAggrConstForceInt32(const Value *V) {
456 bool IsAggrZero =
457 isa<ConstantAggregateZero>(Val: V) && !V->getType()->isVectorTy();
458 bool IsUndefAggregate = isa<UndefValue>(Val: V) && V->getType()->isAggregateType();
459 return isa<ConstantArray>(Val: V) || isa<ConstantStruct>(Val: V) ||
460 isa<ConstantDataArray>(Val: V) || IsAggrZero || IsUndefAggregate;
461}
462
463static bool isSpvAggrPlaceholder(const Value *V) {
464 return match(
465 V,
466 P: m_AnyIntrinsic<Intrinsic::spv_undef, Intrinsic::spv_const_composite>());
467}
468
469static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
470 if (isa<PHINode>(Val: I))
471 B.SetInsertPoint(I->getParent()->getFirstNonPHIOrDbgOrAlloca());
472 else
473 B.SetInsertPoint(I);
474}
475
476static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I) {
477 B.SetCurrentDebugLocation(I->getDebugLoc());
478 if (I->getType()->isVoidTy())
479 B.SetInsertPoint(I->getNextNode());
480 else
481 B.SetInsertPoint(*I->getInsertionPointAfterDef());
482}
483
484static bool requireAssignType(Instruction *I) {
485 return !match(
486 V: I,
487 P: m_AnyIntrinsic<Intrinsic::invariant_start, Intrinsic::invariant_end>());
488}
489
490static inline void reportFatalOnTokenType(const Instruction *I) {
491 if (I->getType()->isTokenTy())
492 report_fatal_error(reason: "A token is encountered but SPIR-V without extensions "
493 "does not support token type",
494 gen_crash_diag: false);
495}
496
497static void emitAssignName(Instruction *I, IRBuilder<> &B) {
498 if (!I->hasName() || I->getType()->isAggregateType() ||
499 expectIgnoredInIRTranslation(I))
500 return;
501
502 // We want to be conservative when adding the names because they can interfere
503 // with later optimizations.
504 bool KeepName = SpirvEmitOpNames;
505 if (!KeepName) {
506 if (isa<AllocaInst>(Val: I)) {
507 KeepName = true;
508 } else if (auto *CI = dyn_cast<CallBase>(Val: I)) {
509 Function *F = CI->getCalledFunction();
510 if (F && F->getName().starts_with(Prefix: "llvm.spv.alloca"))
511 KeepName = true;
512 }
513 }
514
515 if (!KeepName)
516 return;
517
518 reportFatalOnTokenType(I);
519 setInsertPointAfterDef(B, I);
520 LLVMContext &Ctx = I->getContext();
521 std::vector<Value *> Args = {
522 I, MetadataAsValue::get(
523 Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: I->getName())))};
524 B.CreateIntrinsic(ID: Intrinsic::spv_assign_name, OverloadTypes: {I->getType()}, Args);
525}
526
527void SPIRVEmitIntrinsicsImpl::replaceAllUsesWith(Value *Src, Value *Dest,
528 bool DeleteOld) {
529 GR->replaceAllUsesWith(Old: Src, New: Dest, DeleteOld);
530 // Update uncomplete type records if any
531 if (isTodoType(Op: Src)) {
532 if (DeleteOld)
533 eraseTodoType(Op: Src);
534 insertTodoType(Op: Dest);
535 }
536}
537
538void SPIRVEmitIntrinsicsImpl::replaceAllUsesWithAndErase(IRBuilder<> &B,
539 Instruction *Src,
540 Instruction *Dest,
541 bool DeleteOld) {
542 replaceAllUsesWith(Src, Dest, DeleteOld);
543 std::string Name = Src->hasName() ? Src->getName().str() : "";
544 Src->eraseFromParent();
545 if (!Name.empty()) {
546 Dest->setName(Name);
547 if (Named.insert(Ptr: Dest).second)
548 emitAssignName(I: Dest, B);
549 }
550}
551
552static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
553 return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
554 isPointerTy(T: SI->getValueOperand()->getType()) &&
555 isa<Argument>(Val: SI->getValueOperand());
556}
557
558// A pointer-typed local holds a pointer, so its deduced pointee must stay a
559// pointer.
560static bool tracesToPointerAlloca(Value *V) {
561 using namespace PatternMatch;
562 V = V->stripPointerCasts();
563 if (auto *AI = dyn_cast<AllocaInst>(Val: V))
564 return isUntypedPointerTy(T: AI->getAllocatedType());
565 return match(
566 V, P: m_AnyIntrinsic<Intrinsic::spv_alloca, Intrinsic::spv_alloca_array>());
567}
568
569// Maybe restore original function return type.
570static inline Type *restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I,
571 Type *Ty) {
572 CallInst *CI = dyn_cast<CallInst>(Val: I);
573 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
574 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
575 return Ty;
576 if (Type *OriginalTy = GR->findMutated(Val: CI->getCalledFunction()))
577 return OriginalTy;
578 return Ty;
579}
580
581// Reconstruct type with nested element types according to deduced type info.
582// Return nullptr if no detailed type info is available.
583Type *SPIRVEmitIntrinsicsImpl::reconstructType(Value *Op,
584 bool UnknownElemTypeI8,
585 bool IsPostprocessing) {
586 Type *Ty = Op->getType();
587 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
588 Ty = restoreMutatedType(GR, I: OpI, Ty);
589 if (auto It = AggrConstTypes.find(Val: OpI); It != AggrConstTypes.end())
590 Ty = It->second;
591 }
592 if (!isUntypedPointerTy(T: Ty))
593 return Ty;
594 // try to find the pointee type
595 if (Type *NestedTy = GR->findDeducedElementType(Val: Op))
596 return getTypedPointerWrapper(ElemTy: NestedTy, AS: getPointerAddressSpace(T: Ty));
597 // not a pointer according to the type info (e.g., Event object)
598 CallInst *CI = GR->findAssignPtrTypeInstr(Val: Op);
599 if (CI) {
600 MetadataAsValue *MD = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
601 return cast<ConstantAsMetadata>(Val: MD->getMetadata())->getType();
602 }
603 if (UnknownElemTypeI8) {
604 if (!IsPostprocessing)
605 insertTodoType(Op);
606 return getTypedPointerWrapper(ElemTy: IntegerType::getInt8Ty(C&: Op->getContext()),
607 AS: getPointerAddressSpace(T: Ty));
608 }
609 return nullptr;
610}
611
612CallInst *SPIRVEmitIntrinsicsImpl::buildSpvPtrcast(Function *F, Value *Op,
613 Type *ElemTy) {
614 IRBuilder<> B(Op->getContext());
615 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
616 // spv_ptrcast's argument Op denotes an instruction that generates
617 // a value, and we may use getInsertionPointAfterDef()
618 setInsertPointAfterDef(B, I: OpI);
619 } else if (auto *OpA = dyn_cast<Argument>(Val: Op)) {
620 B.SetInsertPointPastAllocas(OpA->getParent());
621 B.SetCurrentDebugLocation(DebugLoc());
622 } else {
623 B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
624 }
625 Type *OpTy = Op->getType();
626 SmallVector<Type *, 2> Types = {OpTy, OpTy};
627 SmallVector<Value *, 2> Args = {
628 Op, buildMD(Arg: getNormalizedPoisonValue(Ty: ElemTy, CanUseAnyVectorRank)),
629 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
630 CallInst *PtrCasted =
631 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
632 GR->buildAssignPtr(B, ElemTy, Arg: PtrCasted);
633 return PtrCasted;
634}
635
636void SPIRVEmitIntrinsicsImpl::replaceUsesOfWithSpvPtrcast(
637 Value *Op, Type *ElemTy, Instruction *I,
638 DenseMap<Function *, CallInst *> Ptrcasts) {
639 Function *F = I->getParent()->getParent();
640 CallInst *PtrCastedI = nullptr;
641 auto It = Ptrcasts.find(Val: F);
642 if (It == Ptrcasts.end()) {
643 PtrCastedI = buildSpvPtrcast(F, Op, ElemTy);
644 Ptrcasts[F] = PtrCastedI;
645 } else {
646 PtrCastedI = It->second;
647 }
648 I->replaceUsesOfWith(From: Op, To: PtrCastedI);
649}
650
651void SPIRVEmitIntrinsicsImpl::propagateElemType(
652 Value *Op, Type *ElemTy,
653 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
654 DenseMap<Function *, CallInst *> Ptrcasts;
655 SmallVector<User *> Users(Op->users());
656 for (auto *U : Users) {
657 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
658 continue;
659 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
660 continue;
661 Instruction *UI = dyn_cast<Instruction>(Val: U);
662 // If the instruction was validated already, we need to keep it valid by
663 // keeping current Op type.
664 if (isaGEP(V: UI) || TypeValidated.find(Ptr: UI) != TypeValidated.end())
665 replaceUsesOfWithSpvPtrcast(Op, ElemTy, I: UI, Ptrcasts);
666 }
667}
668
669void SPIRVEmitIntrinsicsImpl::propagateElemTypeRec(
670 Value *Op, Type *PtrElemTy, Type *CastElemTy,
671 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
672 SmallPtrSet<Value *, 0> Visited;
673 DenseMap<Function *, CallInst *> Ptrcasts;
674 propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited,
675 Ptrcasts: std::move(Ptrcasts));
676}
677
678void SPIRVEmitIntrinsicsImpl::propagateElemTypeRec(
679 Value *Op, Type *PtrElemTy, Type *CastElemTy,
680 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
681 SmallPtrSetImpl<Value *> &Visited,
682 DenseMap<Function *, CallInst *> Ptrcasts) {
683 if (!Visited.insert(Ptr: Op).second)
684 return;
685 SmallVector<User *> Users(Op->users());
686 for (auto *U : Users) {
687 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
688 continue;
689 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
690 continue;
691 Instruction *UI = dyn_cast<Instruction>(Val: U);
692 // If the instruction was validated already, we need to keep it valid by
693 // keeping current Op type.
694 if (isaGEP(V: UI) || TypeValidated.find(Ptr: UI) != TypeValidated.end())
695 replaceUsesOfWithSpvPtrcast(Op, ElemTy: CastElemTy, I: UI, Ptrcasts);
696 }
697}
698
699// Set element pointer type to the given value of ValueTy and tries to
700// specify this type further (recursively) by Operand value, if needed.
701
702Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByValueDeep(
703 Type *ValueTy, Value *Operand, bool UnknownElemTypeI8) {
704 SmallPtrSet<Value *, 0> Visited;
705 return deduceElementTypeByValueDeep(ValueTy, Operand, Visited,
706 UnknownElemTypeI8);
707}
708
709Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByValueDeep(
710 Type *ValueTy, Value *Operand, SmallPtrSetImpl<Value *> &Visited,
711 bool UnknownElemTypeI8) {
712 Type *Ty = ValueTy;
713 if (Operand) {
714 if (auto *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
715 if (Type *NestedTy =
716 deduceElementTypeHelper(I: Operand, Visited, UnknownElemTypeI8))
717 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
718 } else {
719 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Operand), Ty, Visited,
720 UnknownElemTypeI8);
721 }
722 }
723 return Ty;
724}
725
726// Traverse User instructions to deduce an element pointer type of the operand.
727Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByUsersDeep(
728 Value *Op, SmallPtrSetImpl<Value *> &Visited, bool UnknownElemTypeI8) {
729 if (!Op || !isPointerTy(T: Op->getType()) || isa<ConstantPointerNull>(Val: Op) ||
730 isa<UndefValue>(Val: Op))
731 return nullptr;
732
733 if (auto ElemTy = getPointeeType(Ty: Op->getType()))
734 return ElemTy;
735
736 // maybe we already know operand's element type
737 if (Type *KnownTy = GR->findDeducedElementType(Val: Op))
738 return KnownTy;
739
740 for (User *OpU : Op->users()) {
741 if (Instruction *Inst = dyn_cast<Instruction>(Val: OpU)) {
742 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8))
743 return Ty;
744 }
745 }
746 return nullptr;
747}
748
749// Implements what we know in advance about intrinsics and builtin calls
750// TODO: consider feasibility of this particular case to be generalized by
751// encoding knowledge about intrinsics and builtin calls by corresponding
752// specification rules
753static Type *getPointeeTypeByCallInst(StringRef DemangledName,
754 Function *CalledF, unsigned OpIdx) {
755 if ((DemangledName.starts_with(Prefix: "__spirv_ocl_printf(") ||
756 DemangledName.starts_with(Prefix: "printf(")) &&
757 OpIdx == 0)
758 return IntegerType::getInt8Ty(C&: CalledF->getContext());
759 return nullptr;
760}
761
762// Deduce and return a successfully deduced Type of the Instruction,
763// or nullptr otherwise.
764Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeHelper(Value *I,
765 bool UnknownElemTypeI8) {
766 SmallPtrSet<Value *, 0> Visited;
767 return deduceElementTypeHelper(I, Visited, UnknownElemTypeI8);
768}
769
770void SPIRVEmitIntrinsicsImpl::maybeAssignPtrType(Type *&Ty, Value *Op,
771 Type *RefTy,
772 bool UnknownElemTypeI8) {
773 if (isUntypedPointerTy(T: RefTy)) {
774 if (!UnknownElemTypeI8)
775 return;
776 insertTodoType(Op);
777 if (isa<IntToPtrInst>(Val: Op))
778 return;
779 }
780 Ty = RefTy;
781}
782
783bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChainDynamic(
784 Type *CurType, Value *Operand, uint64_t Multiplier,
785 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
786 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing) {
787 // Dynamic indexing into a struct is not possible.
788 // We know that we must be accessing the first element
789 // of the struct if the current type is a struct.
790 // Try to find the first array type that is at offset 0 in the struct.
791 while (auto *ST = dyn_cast<StructType>(Val: CurType)) {
792 if (ST->getNumElements() == 0)
793 break;
794 CurType = ST->getElementType(N: 0);
795 OnLiteralIndexing(CurType, 0);
796 }
797
798 assert(CurType);
799 ArrayType *AT = dyn_cast<ArrayType>(Val: CurType);
800 // Operand is not constant. Either we have an array and accept it, or we
801 // give up.
802 if (AT)
803 OnDynamicIndexing(AT->getElementType(), Operand, Multiplier);
804 return AT == nullptr;
805}
806
807bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChainConstant(
808 Type *CurType, uint64_t Offset,
809 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing) {
810 auto &DL = CurrF->getDataLayout();
811
812 do {
813 if (ArrayType *AT = dyn_cast<ArrayType>(Val: CurType)) {
814 uint64_t EltTypeSize = DL.getTypeAllocSize(Ty: AT->getElementType());
815 assert(Offset < AT->getNumElements() * EltTypeSize);
816 uint64_t Index = Offset / EltTypeSize;
817 Offset = Offset - (Index * EltTypeSize);
818 CurType = AT->getElementType();
819 OnLiteralIndexing(CurType, Index);
820 } else if (StructType *ST = dyn_cast<StructType>(Val: CurType)) {
821 uint32_t StructSize = DL.getTypeSizeInBits(Ty: ST) / 8;
822 assert(Offset < StructSize);
823 (void)StructSize;
824 const auto &STL = DL.getStructLayout(Ty: ST);
825 unsigned Element = STL->getElementContainingOffset(FixedOffset: Offset);
826 Offset -= STL->getElementOffset(Idx: Element);
827 CurType = ST->getElementType(N: Element);
828 OnLiteralIndexing(CurType, Element);
829 } else if (auto *VT = dyn_cast<FixedVectorType>(Val: CurType)) {
830 Type *EltTy = VT->getElementType();
831 TypeSize EltSizeBits = DL.getTypeSizeInBits(Ty: EltTy);
832 assert(EltSizeBits % 8 == 0 &&
833 "Element type size in bits must be a multiple of 8.");
834 uint32_t EltTypeSize = EltSizeBits / 8;
835 assert(Offset < VT->getNumElements() * EltTypeSize);
836 uint64_t Index = Offset / EltTypeSize;
837 Offset -= Index * EltTypeSize;
838 CurType = EltTy;
839 OnLiteralIndexing(CurType, Index);
840 } else {
841 // Unknown composite kind; give up.
842 return true;
843 }
844 } while (Offset > 0);
845
846 return false;
847}
848
849bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChain(
850 GetElementPtrInst &GEP,
851 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
852 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing) {
853 // We only rewrite byte-addressing GEP. Other should be left as-is.
854 // Valid byte-addressing GEP must always have a single index.
855 std::optional<uint64_t> MultiplierOpt =
856 getByteAddressingMultiplier(Ty: GEP.getSourceElementType());
857 assert(MultiplierOpt && "We only rewrite byte-addressing GEP");
858 uint64_t Multiplier = *MultiplierOpt;
859 assert(GEP.getNumIndices() == 1);
860
861 Value *Src = getPointerRoot(I: GEP.getPointerOperand());
862 Type *CurType = deduceElementType(I: Src, UnknownElemTypeI8: true);
863
864 Value *Operand = *GEP.idx_begin();
865 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Operand))
866 return walkLogicalAccessChainConstant(
867 CurType, Offset: CI->getZExtValue() * Multiplier, OnLiteralIndexing);
868
869 return walkLogicalAccessChainDynamic(CurType, Operand, Multiplier,
870 OnLiteralIndexing, OnDynamicIndexing);
871}
872
873Instruction *SPIRVEmitIntrinsicsImpl::buildLogicalAccessChainFromGEP(
874 GetElementPtrInst &GEP) {
875 auto &DL = CurrF->getDataLayout();
876 IRBuilder<> B(GEP.getParent());
877 B.SetInsertPoint(&GEP);
878
879 std::vector<Value *> Indices;
880 Indices.push_back(x: ConstantInt::get(
881 Ty: IntegerType::getInt32Ty(C&: CurrF->getContext()), V: 0, /* Signed= */ IsSigned: false));
882 walkLogicalAccessChain(
883 GEP,
884 OnLiteralIndexing: [&Indices, &B](Type *EltType, uint64_t Index) {
885 Indices.push_back(
886 x: ConstantInt::get(Ty: B.getInt64Ty(), V: Index, /* Signed= */ IsSigned: false));
887 },
888 OnDynamicIndexing: [&Indices, &B, &DL, this](Type *EltType, Value *Offset,
889 uint64_t Multiplier) {
890 Value *Index = nullptr;
891 uint32_t EltTypeSize = DL.getTypeSizeInBits(Ty: EltType) / 8;
892 assert(Multiplier != 0);
893 if (Multiplier == EltTypeSize) {
894 Index = Offset;
895 } else if (EltTypeSize % Multiplier == 0) {
896 Index =
897 B.CreateUDiv(LHS: Offset, RHS: ConstantInt::get(Ty: Offset->getType(),
898 V: EltTypeSize / Multiplier,
899 /* Signed= */ IsSigned: false));
900 } else {
901 Index = B.CreateMul(LHS: Offset,
902 RHS: ConstantInt::get(Ty: Offset->getType(), V: Multiplier,
903 /* Signed= */ IsSigned: false));
904 insertAssignTypeIntrs(I: cast<Instruction>(Val: Index), B);
905 Index = B.CreateUDiv(LHS: Index,
906 RHS: ConstantInt::get(Ty: Offset->getType(), V: EltTypeSize,
907 /* Signed= */ IsSigned: false));
908 }
909 insertAssignTypeIntrs(I: cast<Instruction>(Val: Index), B);
910 Indices.push_back(x: Index);
911 });
912
913 SmallVector<Type *, 2> Types = {GEP.getType(), GEP.getOperand(i_nocapture: 0)->getType()};
914 SmallVector<Value *, 4> Args;
915 Args.push_back(Elt: B.getInt1(V: GEP.isInBounds()));
916 Args.push_back(Elt: GEP.getOperand(i_nocapture: 0));
917 llvm::append_range(C&: Args, R&: Indices);
918 Instruction *NewI =
919 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
920 replaceAllUsesWithAndErase(B, Src: &GEP, Dest: NewI);
921 return NewI;
922}
923
924Type *SPIRVEmitIntrinsicsImpl::getGEPTypeLogical(GetElementPtrInst *GEP) {
925
926 Type *CurType = GEP->getResultElementType();
927
928 bool Interrupted = walkLogicalAccessChain(
929 GEP&: *GEP, OnLiteralIndexing: [&CurType](Type *EltType, uint64_t Index) { CurType = EltType; },
930 OnDynamicIndexing: [&CurType](Type *EltType, Value *Index, uint64_t) { CurType = EltType; });
931
932 return Interrupted ? GEP->getResultElementType() : CurType;
933}
934
935Type *SPIRVEmitIntrinsicsImpl::getGEPType(GetElementPtrInst *Ref) {
936 if (getByteAddressingMultiplier(Ty: Ref->getSourceElementType()) &&
937 TM.getSubtargetImpl()->isLogicalSPIRV()) {
938 return getGEPTypeLogical(GEP: Ref);
939 }
940
941 Type *Ty = nullptr;
942 // TODO: not sure if GetElementPtrInst::getTypeAtIndex() does anything
943 // useful here
944 if (isNestedPointer(Ty: Ref->getSourceElementType())) {
945 Ty = Ref->getSourceElementType();
946 for (Use &U : drop_begin(RangeOrContainer: Ref->indices()))
947 Ty = GetElementPtrInst::getTypeAtIndex(Ty, Idx: U.get());
948 } else {
949 Ty = Ref->getResultElementType();
950 }
951 return Ty;
952}
953
954Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeHelper(
955 Value *I, SmallPtrSetImpl<Value *> &Visited, bool UnknownElemTypeI8,
956 bool IgnoreKnownType) {
957 // allow to pass nullptr as an argument
958 if (!I)
959 return nullptr;
960
961 // maybe already known
962 if (!IgnoreKnownType)
963 if (Type *KnownTy = GR->findDeducedElementType(Val: I))
964 return KnownTy;
965
966 // maybe a cycle
967 if (!Visited.insert(Ptr: I).second)
968 return nullptr;
969
970 // fallback value in case when we fail to deduce a type
971 Type *Ty = nullptr;
972 // look for known basic patterns of type inference
973 if (auto *Ref = dyn_cast<AllocaInst>(Val: I)) {
974 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getAllocatedType(), UnknownElemTypeI8);
975 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
976 Ty = getGEPType(Ref);
977 } else if (auto *SGEP = dyn_cast<StructuredGEPInst>(Val: I)) {
978 Ty = SGEP->getResultElementType();
979 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
980 Value *Op = Ref->getPointerOperand();
981 Type *KnownTy = GR->findDeducedElementType(Val: Op);
982 if (!KnownTy)
983 KnownTy = Op->getType();
984 if (Type *ElemTy = getPointeeType(Ty: KnownTy))
985 maybeAssignPtrType(Ty, Op: I, RefTy: ElemTy, UnknownElemTypeI8);
986 } else if (auto *Ref = dyn_cast<GlobalValue>(Val: I)) {
987 if (auto *Fn = dyn_cast<Function>(Val: Ref)) {
988 Ty = SPIRV::getOriginalFunctionType(F: *Fn);
989 GR->addDeducedElementType(Val: I, Ty);
990 } else {
991 Ty = deduceElementTypeByValueDeep(
992 ValueTy: Ref->getValueType(),
993 Operand: Ref->getNumOperands() > 0 ? Ref->getOperand(i: 0) : nullptr, Visited,
994 UnknownElemTypeI8);
995 }
996 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
997 Type *RefTy = deduceElementTypeHelper(I: Ref->getPointerOperand(), Visited,
998 UnknownElemTypeI8);
999 maybeAssignPtrType(Ty, Op: I, RefTy, UnknownElemTypeI8);
1000 } else if (auto *Ref = dyn_cast<IntToPtrInst>(Val: I)) {
1001 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getDestTy(), UnknownElemTypeI8);
1002 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1003 if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
1004 isPointerTy(T: Src) && isPointerTy(T: Dest))
1005 Ty = deduceElementTypeHelper(I: Ref->getOperand(i_nocapture: 0), Visited,
1006 UnknownElemTypeI8);
1007 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1008 Value *Op = Ref->getNewValOperand();
1009 if (isPointerTy(T: Op->getType()))
1010 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
1011 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1012 Value *Op = Ref->getValOperand();
1013 if (isPointerTy(T: Op->getType()))
1014 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
1015 } else if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1016 Type *BestTy = nullptr;
1017 unsigned MaxN = 1;
1018 DenseMap<Type *, unsigned> PhiTys;
1019 for (int i = Ref->getNumIncomingValues() - 1; i >= 0; --i) {
1020 Ty = deduceElementTypeByUsersDeep(Op: Ref->getIncomingValue(i), Visited,
1021 UnknownElemTypeI8);
1022 if (!Ty)
1023 continue;
1024 auto It = PhiTys.try_emplace(Key: Ty, Args: 1);
1025 if (!It.second) {
1026 ++It.first->second;
1027 if (It.first->second > MaxN) {
1028 MaxN = It.first->second;
1029 BestTy = Ty;
1030 }
1031 }
1032 }
1033 if (BestTy)
1034 Ty = BestTy;
1035 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1036 for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) {
1037 // A function pointer operand carries its function type directly. Other
1038 // operands are deduced from their uses.
1039 Ty = isa<Function>(Val: Op)
1040 ? deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8)
1041 : deduceElementTypeByUsersDeep(Op, Visited, UnknownElemTypeI8);
1042 if (Ty)
1043 break;
1044 }
1045 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
1046 static StringMap<unsigned> ResTypeByArg = {
1047 {"to_global", 0},
1048 {"to_local", 0},
1049 {"to_private", 0},
1050 {"__spirv_GenericCastToPtr_ToGlobal", 0},
1051 {"__spirv_GenericCastToPtr_ToLocal", 0},
1052 {"__spirv_GenericCastToPtr_ToPrivate", 0},
1053 {"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
1054 {"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
1055 {"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
1056 // TODO: maybe improve performance by caching demangled names
1057
1058 auto *II = dyn_cast<IntrinsicInst>(Val: I);
1059 if (II && (II->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer ||
1060 II->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) {
1061 auto *HandleType = cast<TargetExtType>(Val: II->getOperand(i_nocapture: 0)->getType());
1062 if (HandleType->getTargetExtName() == "spirv.Image" ||
1063 HandleType->getTargetExtName() == "spirv.SignedImage") {
1064 for (User *U : II->users()) {
1065 Ty = cast<Instruction>(Val: U)->getAccessType();
1066 if (Ty)
1067 break;
1068 }
1069 } else if (HandleType->getTargetExtName() == "spirv.VulkanBuffer") {
1070 // This call is supposed to index into an array
1071 Ty = HandleType->getTypeParameter(i: 0);
1072 if (II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1073 if (Ty->isArrayTy())
1074 Ty = Ty->getArrayElementType();
1075 else {
1076 assert(Ty && Ty->isStructTy());
1077 uint32_t Index =
1078 cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1))->getZExtValue();
1079 Ty = cast<StructType>(Val: Ty)->getElementType(N: Index);
1080 }
1081 }
1082 Ty = reconstitutePeeledArrayType(Ty);
1083 } else {
1084 llvm_unreachable("Unknown handle type for spv_resource_getpointer.");
1085 }
1086 } else if (II && II->getIntrinsicID() ==
1087 Intrinsic::spv_generic_cast_to_ptr_explicit) {
1088 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: 0), Visited,
1089 UnknownElemTypeI8);
1090 } else if (Function *CalledF = CI->getCalledFunction()) {
1091 std::string DemangledName =
1092 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1093 if (DemangledName.length() > 0)
1094 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName);
1095 auto AsArgIt = ResTypeByArg.find(Key: DemangledName);
1096 if (AsArgIt != ResTypeByArg.end())
1097 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: AsArgIt->second),
1098 Visited, UnknownElemTypeI8);
1099 else if (Type *KnownRetTy = GR->findDeducedElementType(Val: CalledF))
1100 Ty = KnownRetTy;
1101 }
1102 }
1103
1104 // remember the found relationship
1105 if (Ty && !IgnoreKnownType) {
1106 // specify nested types if needed, otherwise return unchanged
1107 GR->addDeducedElementType(Val: I, Ty: normalizeType(Ty, CanUseAnyVectorRank));
1108 }
1109
1110 return Ty;
1111}
1112
1113// Re-create a type of the value if it has untyped pointer fields, also nested.
1114// Return the original value type if no corrections of untyped pointer
1115// information is found or needed.
1116Type *SPIRVEmitIntrinsicsImpl::deduceNestedTypeHelper(User *U,
1117 bool UnknownElemTypeI8) {
1118 SmallPtrSet<Value *, 0> Visited;
1119 return deduceNestedTypeHelper(U, Ty: U->getType(), Visited, UnknownElemTypeI8);
1120}
1121
1122Type *SPIRVEmitIntrinsicsImpl::deduceNestedTypeHelper(
1123 User *U, Type *OrigTy, SmallPtrSetImpl<Value *> &Visited,
1124 bool UnknownElemTypeI8) {
1125 if (!U)
1126 return OrigTy;
1127
1128 // maybe already known
1129 if (Type *KnownTy = GR->findDeducedCompositeType(Val: U))
1130 return KnownTy;
1131
1132 // maybe a cycle
1133 if (!Visited.insert(Ptr: U).second)
1134 return OrigTy;
1135
1136 if (isa<StructType>(Val: OrigTy)) {
1137 SmallVector<Type *> Tys;
1138 bool Change = false;
1139 for (unsigned i = 0; i < U->getNumOperands(); ++i) {
1140 Value *Op = U->getOperand(i);
1141 assert(Op && "Operands should not be null.");
1142 Type *OpTy = Op->getType();
1143 Type *Ty = OpTy;
1144 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1145 if (Type *NestedTy =
1146 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1147 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1148 } else {
1149 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1150 UnknownElemTypeI8);
1151 }
1152 Tys.push_back(Elt: Ty);
1153 Change |= Ty != OpTy;
1154 }
1155 if (Change) {
1156 Type *NewTy = StructType::create(Elements: Tys);
1157 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1158 return NewTy;
1159 }
1160 } else if (auto *ArrTy = dyn_cast<ArrayType>(Val: OrigTy)) {
1161 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1162 Type *OpTy = ArrTy->getElementType();
1163 Type *Ty = OpTy;
1164 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1165 if (Type *NestedTy =
1166 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1167 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1168 } else {
1169 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1170 UnknownElemTypeI8);
1171 }
1172 if (Ty != OpTy) {
1173 Type *NewTy = ArrayType::get(ElementType: Ty, NumElements: ArrTy->getNumElements());
1174 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1175 return NewTy;
1176 }
1177 }
1178 } else if (auto *VecTy = dyn_cast<VectorType>(Val: OrigTy)) {
1179 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1180 Type *OpTy = VecTy->getElementType();
1181 Type *Ty = OpTy;
1182 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1183 if (Type *NestedTy =
1184 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1185 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1186 } else {
1187 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1188 UnknownElemTypeI8);
1189 }
1190 if (Ty != OpTy) {
1191 Type *NewTy = VectorType::get(ElementType: Ty, EC: VecTy->getElementCount());
1192 GR->addDeducedCompositeType(Val: U,
1193 Ty: normalizeType(Ty: NewTy, CanUseAnyVectorRank));
1194 return NewTy;
1195 }
1196 }
1197 }
1198
1199 return OrigTy;
1200}
1201
1202Type *SPIRVEmitIntrinsicsImpl::deduceElementType(Value *I,
1203 bool UnknownElemTypeI8) {
1204 if (Type *Ty = deduceElementTypeHelper(I, UnknownElemTypeI8))
1205 return Ty;
1206 if (!UnknownElemTypeI8)
1207 return nullptr;
1208 insertTodoType(Op: I);
1209 return IntegerType::getInt8Ty(C&: I->getContext());
1210}
1211
1212static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I,
1213 Value *PointerOperand) {
1214 Type *PointeeTy = GR->findDeducedElementType(Val: PointerOperand);
1215 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1216 return nullptr;
1217 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType());
1218 if (!PtrTy)
1219 return I->getType();
1220 if (Type *NestedTy = GR->findDeducedElementType(Val: I))
1221 return getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1222 return nullptr;
1223}
1224
1225// Try to deduce element type for a call base. Returns false if this is an
1226// indirect function invocation, and true otherwise.
1227bool SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeCalledFunction(
1228 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1229 Type *&KnownElemTy, bool &Incomplete) {
1230 Function *CalledF = CI->getCalledFunction();
1231 if (!CalledF)
1232 return false;
1233 std::string DemangledName =
1234 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1235 if (DemangledName.length() > 0 &&
1236 !StringRef(DemangledName).starts_with(Prefix: "llvm.")) {
1237 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *CalledF);
1238 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
1239 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
1240 if (Opcode == SPIRV::OpGroupAsyncCopy) {
1241 for (unsigned i = 0, PtrCnt = 0; i < CI->arg_size() && PtrCnt < 2; ++i) {
1242 Value *Op = CI->getArgOperand(i);
1243 if (!isPointerTy(T: Op->getType()))
1244 continue;
1245 ++PtrCnt;
1246 if (Type *ElemTy = GR->findDeducedElementType(Val: Op))
1247 KnownElemTy = ElemTy; // src will rewrite dest if both are defined
1248 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1249 }
1250 } else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) {
1251 if (CI->arg_size() == 0)
1252 return true;
1253 Value *Op = CI->getArgOperand(i: 0);
1254 if (!isPointerTy(T: Op->getType()))
1255 return true;
1256 switch (Opcode) {
1257 case SPIRV::OpAtomicFAddEXT:
1258 case SPIRV::OpAtomicFMinEXT:
1259 case SPIRV::OpAtomicFMaxEXT:
1260 case SPIRV::OpAtomicLoad:
1261 case SPIRV::OpAtomicCompareExchangeWeak:
1262 case SPIRV::OpAtomicCompareExchange:
1263 case SPIRV::OpAtomicExchange:
1264 case SPIRV::OpAtomicIAdd:
1265 case SPIRV::OpAtomicISub:
1266 case SPIRV::OpAtomicOr:
1267 case SPIRV::OpAtomicXor:
1268 case SPIRV::OpAtomicAnd:
1269 case SPIRV::OpAtomicUMin:
1270 case SPIRV::OpAtomicUMax:
1271 case SPIRV::OpAtomicSMin:
1272 case SPIRV::OpAtomicSMax: {
1273 KnownElemTy = isPointerTy(T: CI->getType()) ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1274 : CI->getType();
1275 if (!KnownElemTy)
1276 return true;
1277 Incomplete = isTodoType(Op);
1278 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1279 } break;
1280 case SPIRV::OpAtomicStore: {
1281 if (CI->arg_size() < 4)
1282 return true;
1283 Value *ValOp = CI->getArgOperand(i: 3);
1284 KnownElemTy = isPointerTy(T: ValOp->getType())
1285 ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1286 : ValOp->getType();
1287 if (!KnownElemTy)
1288 return true;
1289 Incomplete = isTodoType(Op);
1290 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1291 } break;
1292 }
1293 }
1294 }
1295 return true;
1296}
1297
1298// Try to deduce element type for a function pointer.
1299void SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeFunctionPointer(
1300 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1301 Type *&KnownElemTy, bool IsPostprocessing) {
1302 Value *Op = CI->getCalledOperand();
1303 if (!Op || !isPointerTy(T: Op->getType()))
1304 return;
1305 Ops.push_back(Elt: std::make_pair(x&: Op, y: std::numeric_limits<unsigned>::max()));
1306 FunctionType *FTy = SPIRV::getOriginalFunctionType(CB: *CI);
1307 bool IsNewFTy = false, IsIncomplete = false;
1308 SmallVector<Type *, 4> ArgTys;
1309 for (auto &&[ParmIdx, Arg] : llvm::enumerate(First: CI->args())) {
1310 Type *ArgTy = Arg->getType();
1311 if (ArgTy->isPointerTy()) {
1312 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg)) {
1313 IsNewFTy = true;
1314 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
1315 if (isTodoType(Op: Arg))
1316 IsIncomplete = true;
1317 } else {
1318 IsIncomplete = true;
1319 }
1320 } else {
1321 ArgTy = FTy->getFunctionParamType(i: ParmIdx);
1322 }
1323 ArgTys.push_back(Elt: ArgTy);
1324 }
1325 Type *RetTy = FTy->getReturnType();
1326 if (CI->getType()->isPointerTy()) {
1327 if (Type *ElemTy = GR->findDeducedElementType(Val: CI)) {
1328 IsNewFTy = true;
1329 RetTy =
1330 getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: CI->getType()));
1331 if (isTodoType(Op: CI))
1332 IsIncomplete = true;
1333 } else {
1334 IsIncomplete = true;
1335 }
1336 }
1337 if (!IsPostprocessing && IsIncomplete)
1338 insertTodoType(Op);
1339 KnownElemTy =
1340 IsNewFTy ? FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: FTy->isVarArg()) : FTy;
1341}
1342
1343bool SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeFunctionRet(
1344 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
1345 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing,
1346 Type *&KnownElemTy, Value *Op, Function *F) {
1347 KnownElemTy = GR->findDeducedElementType(Val: F);
1348 if (KnownElemTy)
1349 return false;
1350 if (Type *OpElemTy = GR->findDeducedElementType(Val: Op)) {
1351 OpElemTy = normalizeType(Ty: OpElemTy, CanUseAnyVectorRank);
1352 GR->addDeducedElementType(Val: F, Ty: OpElemTy);
1353 GR->addReturnType(
1354 ArgF: F, DerivedTy: TypedPointerType::get(ElementType: OpElemTy,
1355 AddressSpace: getPointerAddressSpace(T: F->getReturnType())));
1356 // non-recursive update of types in function uses
1357 DenseSet<std::pair<Value *, Value *>> VisitedSubst{std::make_pair(x&: I, y&: Op)};
1358 for (User *U : F->users()) {
1359 CallInst *CI = dyn_cast<CallInst>(Val: U);
1360 if (!CI || CI->getCalledFunction() != F)
1361 continue;
1362 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: CI)) {
1363 if (Type *PrevElemTy = GR->findDeducedElementType(Val: CI)) {
1364 GR->updateAssignType(
1365 AssignCI, Arg: CI,
1366 OfType: getNormalizedPoisonValue(Ty: OpElemTy, CanUseAnyVectorRank));
1367 propagateElemType(Op: CI, ElemTy: PrevElemTy, VisitedSubst);
1368 }
1369 }
1370 }
1371 // Non-recursive update of types in the function uncomplete returns.
1372 // This may happen just once per a function, the latch is a pair of
1373 // findDeducedElementType(F) / addDeducedElementType(F, ...).
1374 // With or without the latch it is a non-recursive call due to
1375 // IncompleteRets set to nullptr in this call.
1376 if (IncompleteRets)
1377 for (Instruction *IncompleteRetI : *IncompleteRets)
1378 deduceOperandElementType(I: IncompleteRetI, IncompleteRets: nullptr, AskOps,
1379 IsPostprocessing);
1380 } else if (IncompleteRets) {
1381 IncompleteRets->insert(Ptr: I);
1382 }
1383 TypeValidated.insert(Ptr: I);
1384 return true;
1385}
1386
1387// If the Instruction has Pointer operands with unresolved types, this function
1388// tries to deduce them. If the Instruction has Pointer operands with known
1389// types which differ from expected, this function tries to insert a bitcast to
1390// resolve the issue.
1391void SPIRVEmitIntrinsicsImpl::deduceOperandElementType(
1392 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
1393 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing) {
1394 SmallVector<std::pair<Value *, unsigned>> Ops;
1395 Type *KnownElemTy = nullptr;
1396 bool Incomplete = false;
1397 // look for known basic patterns of type inference
1398 if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1399 if (!isPointerTy(T: I->getType()) ||
1400 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1401 return;
1402 Incomplete = isTodoType(Op: I);
1403 for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
1404 Value *Op = Ref->getIncomingValue(i);
1405 if (isPointerTy(T: Op->getType()))
1406 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1407 }
1408 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1409 KnownElemTy = GR->findDeducedElementType(Val: I);
1410 if (!KnownElemTy)
1411 return;
1412 Incomplete = isTodoType(Op: I);
1413 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(), y: 0));
1414 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1415 if (!isPointerTy(T: I->getType()))
1416 return;
1417 KnownElemTy = GR->findDeducedElementType(Val: I);
1418 if (!KnownElemTy)
1419 return;
1420 Incomplete = isTodoType(Op: I);
1421 Ops.push_back(Elt: std::make_pair(x: Ref->getOperand(i_nocapture: 0), y: 0));
1422 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
1423 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1424 return;
1425 KnownElemTy = Ref->getSourceElementType();
1426 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1427 y: GetElementPtrInst::getPointerOperandIndex()));
1428 } else if (auto *Ref = dyn_cast<StructuredGEPInst>(Val: I)) {
1429 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1430 return;
1431 KnownElemTy = Ref->getBaseType();
1432 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1433 y: StructuredGEPInst::getPointerOperandIndex()));
1434 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
1435 KnownElemTy = I->getType();
1436 if (isUntypedPointerTy(T: KnownElemTy)) {
1437 // A T** loaded back from its alloca comes out opaque, dropping type info.
1438 // When the load is a pointer-to-pointer, type the alloca as that pointer.
1439 Type *LoadedElemTy = GR->findDeducedElementType(Val: I);
1440 if (!LoadedElemTy || !isPointerTyOrWrapper(Ty: LoadedElemTy))
1441 return;
1442 Value *Root = Ref->getPointerOperand()->stripPointerCasts();
1443 if (!isa<AllocaInst>(Val: Root))
1444 return;
1445 KnownElemTy = getTypedPointerWrapper(ElemTy: LoadedElemTy,
1446 AS: getPointerAddressSpace(T: KnownElemTy));
1447 }
1448 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1449 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1450 return;
1451 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1452 y: LoadInst::getPointerOperandIndex()));
1453 } else if (auto *Ref = dyn_cast<StoreInst>(Val: I)) {
1454 if (!(KnownElemTy =
1455 reconstructType(Op: Ref->getValueOperand(), UnknownElemTypeI8: false, IsPostprocessing)))
1456 return;
1457 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1458 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1459 return;
1460 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1461 y: StoreInst::getPointerOperandIndex()));
1462 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1463 KnownElemTy = isPointerTy(T: I->getType())
1464 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1465 : I->getType();
1466 if (!KnownElemTy)
1467 return;
1468 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1469 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1470 y: AtomicCmpXchgInst::getPointerOperandIndex()));
1471 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1472 KnownElemTy = isPointerTy(T: I->getType())
1473 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1474 : I->getType();
1475 if (!KnownElemTy)
1476 return;
1477 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1478 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1479 y: AtomicRMWInst::getPointerOperandIndex()));
1480 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1481 if (!isPointerTy(T: I->getType()) ||
1482 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1483 return;
1484 Incomplete = isTodoType(Op: I);
1485 for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
1486 Value *Op = Ref->getOperand(i_nocapture: i);
1487 if (isPointerTy(T: Op->getType()))
1488 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1489 }
1490 } else if (auto *Ref = dyn_cast<ReturnInst>(Val: I)) {
1491 if (!isPointerTy(T: CurrF->getReturnType()))
1492 return;
1493 Value *Op = Ref->getReturnValue();
1494 if (!Op)
1495 return;
1496 if (deduceOperandElementTypeFunctionRet(I, IncompleteRets, AskOps,
1497 IsPostprocessing, KnownElemTy, Op,
1498 F: CurrF))
1499 return;
1500 Incomplete = isTodoType(Op: CurrF);
1501 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1502 } else if (auto *Ref = dyn_cast<ICmpInst>(Val: I)) {
1503 if (!isPointerTy(T: Ref->getOperand(i_nocapture: 0)->getType()))
1504 return;
1505 Value *Op0 = Ref->getOperand(i_nocapture: 0);
1506 Value *Op1 = Ref->getOperand(i_nocapture: 1);
1507 bool Incomplete0 = isTodoType(Op: Op0);
1508 bool Incomplete1 = isTodoType(Op: Op1);
1509 Type *ElemTy1 = GR->findDeducedElementType(Val: Op1);
1510 Type *ElemTy0 = (Incomplete0 && !Incomplete1 && ElemTy1)
1511 ? nullptr
1512 : GR->findDeducedElementType(Val: Op0);
1513 if (ElemTy0) {
1514 KnownElemTy = ElemTy0;
1515 Incomplete = Incomplete0;
1516 Ops.push_back(Elt: std::make_pair(x&: Op1, y: 1));
1517 } else if (ElemTy1) {
1518 KnownElemTy = ElemTy1;
1519 Incomplete = Incomplete1;
1520 Ops.push_back(Elt: std::make_pair(x&: Op0, y: 0));
1521 }
1522 } else if (CallInst *CI = dyn_cast<CallInst>(Val: I)) {
1523 if (!CI->isIndirectCall())
1524 deduceOperandElementTypeCalledFunction(CI, Ops, KnownElemTy, Incomplete);
1525 else if (HaveFunPtrs)
1526 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy,
1527 IsPostprocessing);
1528 }
1529
1530 // There is no enough info to deduce types or all is valid.
1531 if (!KnownElemTy || Ops.size() == 0)
1532 return;
1533
1534 LLVMContext &Ctx = CurrF->getContext();
1535 IRBuilder<> B(Ctx);
1536 for (auto &OpIt : Ops) {
1537 Value *Op = OpIt.first;
1538 if (AskOps && !AskOps->contains(Ptr: Op))
1539 continue;
1540 Type *AskTy = nullptr;
1541 CallInst *AskCI = nullptr;
1542 if (IsPostprocessing && AskOps) {
1543 AskTy = GR->findDeducedElementType(Val: Op);
1544 AskCI = GR->findAssignPtrTypeInstr(Val: Op);
1545 assert(AskTy && AskCI);
1546 }
1547 Type *Ty = AskTy ? AskTy : GR->findDeducedElementType(Val: Op);
1548 if (Ty == KnownElemTy)
1549 continue;
1550 Value *OpTyVal = getNormalizedPoisonValue(Ty: KnownElemTy, CanUseAnyVectorRank);
1551 Type *OpTy = Op->getType();
1552 // Do not let a non-pointer element type clobber an already-deduced pointer
1553 // element type for the same operand.
1554 bool WouldClobberPtrWithNonPtr = Ty && isPointerTyOrWrapper(Ty) &&
1555 !isPointerTyOrWrapper(Ty: KnownElemTy) &&
1556 tracesToPointerAlloca(V: Op);
1557 if (Op->hasUseList() && !WouldClobberPtrWithNonPtr &&
1558 (!Ty || AskTy || isUntypedPointerTy(T: Ty) || isTodoType(Op))) {
1559 Type *PrevElemTy = GR->findDeducedElementType(Val: Op);
1560 GR->addDeducedElementType(
1561 Val: Op, Ty: normalizeType(Ty: KnownElemTy, CanUseAnyVectorRank));
1562 // check if KnownElemTy is complete
1563 if (!Incomplete)
1564 eraseTodoType(Op);
1565 else if (!IsPostprocessing)
1566 insertTodoType(Op);
1567 // check if there is existing Intrinsic::spv_assign_ptr_type instruction
1568 CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Val: Op);
1569 if (AssignCI == nullptr) {
1570 Instruction *User = dyn_cast<Instruction>(Val: Op->use_begin()->get());
1571 setInsertPointSkippingPhis(B, I: User ? User->getNextNode() : I);
1572 CallInst *CI =
1573 buildIntrWithMD(IntrID: Intrinsic::spv_assign_ptr_type, Types: {OpTy}, Arg: OpTyVal, Arg2: Op,
1574 Imms: {B.getInt32(C: getPointerAddressSpace(T: OpTy))}, B);
1575 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: CI);
1576 } else {
1577 GR->updateAssignType(AssignCI, Arg: Op, OfType: OpTyVal);
1578 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1579 std::make_pair(x&: I, y&: Op)};
1580 propagateElemTypeRec(Op, PtrElemTy: KnownElemTy, CastElemTy: PrevElemTy, VisitedSubst);
1581 }
1582 } else {
1583 eraseTodoType(Op);
1584 CallInst *PtrCastI =
1585 buildSpvPtrcast(F: I->getParent()->getParent(), Op, ElemTy: KnownElemTy);
1586 if (OpIt.second == std::numeric_limits<unsigned>::max())
1587 dyn_cast<CallInst>(Val: I)->setCalledOperand(PtrCastI);
1588 else
1589 I->setOperand(i: OpIt.second, Val: PtrCastI);
1590 }
1591 }
1592 TypeValidated.insert(Ptr: I);
1593}
1594
1595void SPIRVEmitIntrinsicsImpl::replaceMemInstrUses(Instruction *Old,
1596 Instruction *New,
1597 IRBuilder<> &B) {
1598 while (!Old->user_empty()) {
1599 auto *U = Old->user_back();
1600 if (isAssignTypeInstr(I: U)) {
1601 B.SetInsertPoint(U);
1602 SmallVector<Value *, 2> Args = {New, U->getOperand(i: 1)};
1603 CallInst *AssignCI = B.CreateIntrinsicWithoutFolding(
1604 ID: Intrinsic::spv_assign_type, OverloadTypes: {New->getType()}, Args);
1605 GR->addAssignPtrTypeInstr(Val: New, AssignPtrTyCI: AssignCI);
1606 U->eraseFromParent();
1607 } else if (isMemInstrToReplace(I: U) || isa<ReturnInst>(Val: U) ||
1608 isa<CallInst>(Val: U)) {
1609 U->replaceUsesOfWith(From: Old, To: New);
1610 // For a `llvm.spv.abort` call whose composite message argument was
1611 // rewritten to a value-id (i32), also retarget the call to a matching
1612 // intrinsic declaration so the IR verifier is satisfied. The SPIR-V
1613 // type of the value is tracked via the GlobalRegistry, so the selector
1614 // still emits OpAbortKHR with the original composite type.
1615 if (auto *CI = dyn_cast<CallInst>(Val: U);
1616 CI && CI->getIntrinsicID() == Intrinsic::spv_abort) {
1617 Type *NewArgTy = New->getType();
1618 Type *ExpectedArgTy = CI->getFunctionType()->getParamType(i: 0);
1619 if (NewArgTy != ExpectedArgTy) {
1620 Module *M = CI->getModule();
1621 Function *NewF = Intrinsic::getOrInsertDeclaration(
1622 M, id: Intrinsic::spv_abort, OverloadTys: {NewArgTy});
1623 CI->setCalledFunction(NewF);
1624 }
1625 }
1626 } else if (isa<PHINode>(Val: U) || isa<SelectInst>(Val: U) || isa<FreezeInst>(Val: U)) {
1627 // Aggregate-typed PHIs, selects and freezes have already been mutated to
1628 // the i32 value-id type up front in runOnFunction, so only the operand
1629 // needs replacing here; their extractvalue users are lowered to
1630 // spv_extractv by visitExtractValueInst.
1631 assert(U->getType() == New->getType() &&
1632 "aggregate PHI/select/freeze should have been mutated to value-id "
1633 "type");
1634 U->replaceUsesOfWith(From: Old, To: New);
1635 } else {
1636 llvm_unreachable("illegal aggregate intrinsic user");
1637 }
1638 }
1639 New->copyMetadata(SrcInst: *Old);
1640 Old->eraseFromParent();
1641}
1642
1643// Lower a poison or undef Op to its placeholder intrinsic.
1644Value *SPIRVEmitIntrinsicsImpl::lowerUndefOrPoison(Value *Op, IRBuilder<> &B,
1645 bool HasPoisonExt) {
1646 auto *UV = dyn_cast<UndefValue>(Val: Op);
1647 if (!UV)
1648 return nullptr;
1649
1650 bool AsPoison = HasPoisonExt && isa<PoisonValue>(Val: UV);
1651 if (isa<PoisonValue>(Val: UV) && !HasPoisonExt)
1652 LLVM_DEBUG(dbgs() << "SPV_KHR_poison_freeze is not enabled. Poison is "
1653 "lowered as undef\n");
1654
1655 Intrinsic::ID IID = AsPoison ? Intrinsic::spv_poison : Intrinsic::spv_undef;
1656 Type *Ty = UV->getType();
1657
1658 // Aggregates use an i32-result placeholder with the real type kept in
1659 // AggrConstTypes and scalar poison uses a type-overloaded one.
1660 if (Ty->isAggregateType()) {
1661 auto *Call =
1662 AsPoison ? B.CreateIntrinsicWithoutFolding(ID: IID, OverloadTypes: {B.getInt32Ty()}, Args: {})
1663 : B.CreateIntrinsicWithoutFolding(ID: IID, Args: {});
1664 AggrConsts[Call] = UV;
1665 AggrConstTypes[Call] = Ty;
1666 return Call;
1667 }
1668
1669 if (AsPoison)
1670 return B.CreateIntrinsic(ID: IID, OverloadTypes: {Ty}, Args: {});
1671 return nullptr;
1672}
1673
1674// Replace aggregate undef or poison operands and extension-enabled scalar
1675// poison operands with placeholder intrinsics. Scalar undef is left as is. See
1676// lowerUndefOrPoison.
1677void SPIRVEmitIntrinsicsImpl::preprocessUndefsAndPoisons(IRBuilder<> &B) {
1678 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*CurrF);
1679 bool HasPoisonExt =
1680 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
1681
1682 SmallVector<Instruction *, 16> Insts;
1683 for (auto &I : instructions(F: CurrF))
1684 Insts.push_back(Elt: &I);
1685
1686 for (Instruction *I : Insts) {
1687 bool BPrepared = false;
1688 auto *Phi = dyn_cast<PHINode>(Val: I);
1689 for (unsigned Idx = 0; Idx < I->getNumOperands(); ++Idx) {
1690 Value *Op = I->getOperand(i: Idx);
1691 if (!isa<UndefValue>(Val: Op) || Op->getType()->isMetadataTy())
1692 continue;
1693 bool IsScalar = !Op->getType()->isAggregateType();
1694 bool AsPoison = HasPoisonExt && isa<PoisonValue>(Val: Op);
1695 // Scalar undef or extensionless scalar poison is directly translatable.
1696 if (IsScalar && !AsPoison)
1697 continue;
1698 // Scalar poison in a phi materializes in the incoming block. Everything
1699 // else materializes right before I.
1700 if (IsScalar && Phi)
1701 B.SetInsertPoint(Phi->getIncomingBlock(i: Idx)->getTerminator());
1702 else if (!BPrepared) {
1703 setInsertPointSkippingPhis(B, I);
1704 BPrepared = true;
1705 }
1706 if (Value *Repl = lowerUndefOrPoison(Op, B, HasPoisonExt))
1707 I->setOperand(i: Idx, Val: Repl);
1708 }
1709 }
1710}
1711
1712// Simplify addrspacecast(null) instructions to ConstantPointerNull of the
1713// target type. Casting null always yields null, and this avoids SPIR-V
1714// lowering issues where the null gets typed as an integer instead of a
1715// pointer.
1716void SPIRVEmitIntrinsicsImpl::simplifyNullAddrSpaceCasts() {
1717 for (Instruction &I : make_early_inc_range(Range: instructions(F: CurrF)))
1718 if (auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: &I))
1719 if (isa<ConstantPointerNull>(Val: ASC->getPointerOperand())) {
1720 ASC->replaceAllUsesWith(
1721 V: ConstantPointerNull::get(T: cast<PointerType>(Val: ASC->getType())));
1722 ASC->eraseFromParent();
1723 }
1724}
1725
1726// True for an aggregate value the legalizer splits into a multi-result op
1727// (with.overflow -> G_UADDO, frexp/sincos/modf -> G_FFREXP/...). These keep a
1728// genuine multi-register result; all other aggregates become a single value-id.
1729static bool isMultiRegisterAggregate(Value *V) {
1730 if (!V->getType()->isAggregateType())
1731 return false;
1732 return isa<IntrinsicInst>(Val: V) && !isSpvIntrinsic(Arg: V);
1733}
1734
1735// True for an aggregate PHI/select/freeze, which is lowered to a single
1736// value-id.
1737static bool isAggregateValueIdInstr(const Instruction &I) {
1738 return (isa<PHINode>(Val: I) || isa<SelectInst>(Val: I) || isa<FreezeInst>(Val: I)) &&
1739 I.getType()->isAggregateType();
1740}
1741
1742// Give each multi-register aggregate arm of an aggregate PHI/select/freeze a
1743// single value-id by reassembling it with extractvalue + insertvalue, so the
1744// arm matches the result once it is mutated to a value-id.
1745void SPIRVEmitIntrinsicsImpl::insertCompositeAggregateArms(Instruction *I,
1746 IRBuilder<> &B) {
1747 auto *Phi = dyn_cast<PHINode>(Val: I);
1748 for (Use &U : I->operands()) {
1749 Value *Op = U.get();
1750 if (!isMultiRegisterAggregate(V: Op))
1751 continue;
1752 // A PHI arm materializes in its incoming block, everything else after the
1753 // producer.
1754 if (Phi)
1755 B.SetInsertPoint(Phi->getIncomingBlock(U)->getTerminator());
1756 else
1757 setInsertPointAfterDef(B, I: cast<Instruction>(Val: Op));
1758 auto *AggrTy = cast<StructType>(Val: Op->getType());
1759 Value *Composite = PoisonValue::get(T: AggrTy);
1760 for (unsigned Idx = 0, E = AggrTy->getNumElements(); Idx != E; ++Idx) {
1761 Value *Field = B.CreateExtractValue(Agg: Op, Idxs: Idx);
1762 Composite = B.CreateInsertValue(Agg: Composite, Val: Field, Idxs: Idx);
1763 }
1764 U.set(Composite);
1765 }
1766}
1767
1768void SPIRVEmitIntrinsicsImpl::preprocessCompositeConstants(IRBuilder<> &B) {
1769 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*CurrF);
1770 bool HasPoisonExt =
1771 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
1772 std::queue<Instruction *> Worklist;
1773 for (auto &I : instructions(F: CurrF))
1774 Worklist.push(x: &I);
1775
1776 while (!Worklist.empty()) {
1777 auto *I = Worklist.front();
1778 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
1779 assert(I);
1780 bool KeepInst = false;
1781 for (const auto &Op : I->operands()) {
1782 Constant *AggrConst = nullptr;
1783 Type *ResTy = nullptr;
1784 if (auto *COp = dyn_cast<ConstantVector>(Val: Op)) {
1785 AggrConst = COp;
1786 ResTy = COp->getType();
1787 } else if (auto *COp = dyn_cast<ConstantArray>(Val: Op)) {
1788 AggrConst = COp;
1789 ResTy = B.getInt32Ty();
1790 } else if (auto *COp = dyn_cast<ConstantStruct>(Val: Op)) {
1791 AggrConst = COp;
1792 ResTy = B.getInt32Ty();
1793 } else if (auto *COp = dyn_cast<ConstantDataArray>(Val: Op)) {
1794 AggrConst = COp;
1795 ResTy = B.getInt32Ty();
1796 } else if (auto *COp = dyn_cast<ConstantAggregateZero>(Val: Op)) {
1797 AggrConst = COp;
1798 ResTy = Op->getType()->isVectorTy() ? COp->getType() : B.getInt32Ty();
1799 }
1800 if (AggrConst) {
1801 auto PrepareInsert = [&]() {
1802 if (BPrepared)
1803 return;
1804 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
1805 : B.SetInsertPoint(I);
1806 BPrepared = true;
1807 };
1808 SmallVector<Value *> Args;
1809 if (auto *COp = dyn_cast<ConstantDataSequential>(Val: Op))
1810 for (unsigned i = 0; i < COp->getNumElements(); ++i)
1811 Args.push_back(Elt: COp->getElementAsConstant(i));
1812 else
1813 for (Value *Op : AggrConst->operands()) {
1814 // Simplify addrspacecast(null) to null in the target address space
1815 // so that null pointers get the correct pointer type when lowered.
1816 if (auto *CE = dyn_cast<ConstantExpr>(Val: Op);
1817 CE && CE->getOpcode() == Instruction::AddrSpaceCast &&
1818 isa<ConstantPointerNull>(Val: CE->getOperand(i_nocapture: 0)))
1819 Op = ConstantPointerNull::get(T: cast<PointerType>(Val: CE->getType()));
1820 // Undef or poison nested in a constant aggregate is not a direct
1821 // instruction operand, so preprocessUndefsAndPoisons() misses it.
1822 // An unlowered aggregate one would reach IRTranslator as an
1823 // untranslatable spv_const_composite operand.
1824 if (isa<UndefValue>(Val: Op)) {
1825 PrepareInsert();
1826 if (Value *Repl = lowerUndefOrPoison(Op, B, HasPoisonExt))
1827 Op = Repl;
1828 }
1829 Args.push_back(Elt: Op);
1830 }
1831 PrepareInsert();
1832 auto *CI = B.CreateIntrinsicWithoutFolding(
1833 ID: Intrinsic::spv_const_composite, OverloadTypes: {ResTy}, Args: {Args});
1834 Worklist.push(x: CI);
1835 I->replaceUsesOfWith(From: Op, To: CI);
1836 KeepInst = true;
1837 AggrConsts[CI] = AggrConst;
1838 AggrConstTypes[CI] = deduceNestedTypeHelper(U: AggrConst, UnknownElemTypeI8: false);
1839 }
1840 }
1841 if (!KeepInst)
1842 Worklist.pop();
1843 }
1844}
1845
1846static void createDecorationIntrinsic(Instruction *I, MDNode *Node,
1847 IRBuilder<> &B) {
1848 LLVMContext &Ctx = I->getContext();
1849 setInsertPointAfterDef(B, I);
1850 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
1851 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: {Node}))});
1852}
1853
1854static void createRoundingModeDecoration(Instruction *I,
1855 unsigned RoundingModeDeco,
1856 IRBuilder<> &B) {
1857 LLVMContext &Ctx = I->getContext();
1858 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1859 MDNode *RoundingModeNode = MDNode::get(
1860 Context&: Ctx,
1861 MDs: {ConstantAsMetadata::get(
1862 C: ConstantInt::get(Ty: Int32Ty, V: SPIRV::Decoration::FPRoundingMode)),
1863 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: RoundingModeDeco))});
1864 createDecorationIntrinsic(I, Node: RoundingModeNode, B);
1865}
1866
1867static void createSaturatedConversionDecoration(Instruction *I,
1868 IRBuilder<> &B) {
1869 LLVMContext &Ctx = I->getContext();
1870 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1871 MDNode *SaturatedConversionNode =
1872 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1873 Ty: Int32Ty, V: SPIRV::Decoration::SaturatedConversion))});
1874 createDecorationIntrinsic(I, Node: SaturatedConversionNode, B);
1875}
1876
1877static void addSaturatedDecorationToIntrinsic(Instruction *I, IRBuilder<> &B) {
1878 if (match(V: I, P: m_AnyIntrinsic<Intrinsic::fptosi_sat, Intrinsic::fptoui_sat>()))
1879 createSaturatedConversionDecoration(I, B);
1880}
1881
1882Instruction *SPIRVEmitIntrinsicsImpl::visitCallInst(CallInst &Call) {
1883 if (!Call.isInlineAsm())
1884 return &Call;
1885
1886 LLVMContext &Ctx = CurrF->getContext();
1887 // TODO: this does not retain elementtype info for memory constraints, which
1888 // in turn means that we lower them into pointers to i8, rather than
1889 // pointers to elementtype; this can be fixed during reverse translation
1890 // but we should correct it here, possibly by tweaking the function
1891 // type to take TypedPointerType args.
1892 Constant *TyC = UndefValue::get(T: SPIRV::getOriginalFunctionType(CB: Call));
1893 MDString *ConstraintString =
1894 MDString::get(Context&: Ctx, Str: SPIRV::getOriginalAsmConstraints(CB: Call));
1895 SmallVector<Value *> Args = {
1896 buildMD(Arg: TyC),
1897 MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: ConstraintString))};
1898 for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++)
1899 Args.push_back(Elt: Call.getArgOperand(i: OpIdx));
1900
1901 IRBuilder<> B(Call.getParent());
1902 B.SetInsertPoint(&Call);
1903 B.CreateIntrinsic(ID: Intrinsic::spv_inline_asm, Args: {Args});
1904 return &Call;
1905}
1906
1907// Use a tip about rounding mode to create a decoration.
1908void SPIRVEmitIntrinsicsImpl::useRoundingMode(ConstrainedFPIntrinsic *FPI,
1909 IRBuilder<> &B) {
1910 std::optional<RoundingMode> RM = FPI->getRoundingMode();
1911 if (!RM.has_value())
1912 return;
1913 unsigned RoundingModeDeco = std::numeric_limits<unsigned>::max();
1914 switch (RM.value()) {
1915 default:
1916 // ignore unknown rounding modes
1917 break;
1918 case RoundingMode::NearestTiesToEven:
1919 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTE;
1920 break;
1921 case RoundingMode::TowardNegative:
1922 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTN;
1923 break;
1924 case RoundingMode::TowardPositive:
1925 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTP;
1926 break;
1927 case RoundingMode::TowardZero:
1928 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTZ;
1929 break;
1930 case RoundingMode::Dynamic:
1931 case RoundingMode::NearestTiesToAway:
1932 // TODO: check if supported
1933 break;
1934 }
1935 if (RoundingModeDeco == std::numeric_limits<unsigned>::max())
1936 return;
1937 // Convert the tip about rounding mode into a decoration record.
1938 createRoundingModeDecoration(I: FPI, RoundingModeDeco, B);
1939}
1940
1941Instruction *SPIRVEmitIntrinsicsImpl::visitSwitchInst(SwitchInst &I) {
1942 BasicBlock *ParentBB = I.getParent();
1943 Function *F = ParentBB->getParent();
1944 IRBuilder<> B(ParentBB);
1945 B.SetInsertPoint(&I);
1946 SmallVector<Value *, 4> Args;
1947 SmallVector<BasicBlock *> BBCases;
1948 Args.push_back(Elt: I.getCondition());
1949 BBCases.push_back(Elt: I.getDefaultDest());
1950 Args.push_back(Elt: BlockAddress::get(F, BB: I.getDefaultDest()));
1951 for (auto &Case : I.cases()) {
1952 Args.push_back(Elt: Case.getCaseValue());
1953 BBCases.push_back(Elt: Case.getCaseSuccessor());
1954 Args.push_back(Elt: BlockAddress::get(F, BB: Case.getCaseSuccessor()));
1955 }
1956 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
1957 ID: Intrinsic::spv_switch, OverloadTypes: {I.getOperand(i_nocapture: 0)->getType()}, Args: {Args});
1958 // remove switch to avoid its unneeded and undesirable unwrap into branches
1959 // and conditions
1960 replaceAllUsesWith(Src: &I, Dest: NewI);
1961 I.eraseFromParent();
1962 // insert artificial and temporary instruction to preserve valid CFG,
1963 // it will be removed after IR translation pass
1964 B.SetInsertPoint(ParentBB);
1965 IndirectBrInst *BrI = B.CreateIndirectBr(
1966 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: ParentBB->getContext())),
1967 NumDests: BBCases.size());
1968 for (BasicBlock *BBCase : BBCases)
1969 BrI->addDestination(Dest: BBCase);
1970 return BrI;
1971}
1972
1973static bool isFirstIndexZero(const GetElementPtrInst *GEP) {
1974 return GEP->getNumIndices() > 0 && match(V: GEP->getOperand(i_nocapture: 1), P: m_Zero());
1975}
1976
1977Instruction *SPIRVEmitIntrinsicsImpl::visitIntrinsicInst(IntrinsicInst &I) {
1978 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
1979 if (!SGEP)
1980 return &I;
1981
1982 IRBuilder<> B(I.getParent());
1983 B.SetInsertPoint(&I);
1984 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
1985 SmallVector<Value *, 4> Args;
1986 Args.push_back(/* inBounds= */ Elt: B.getInt1(V: true));
1987 Args.push_back(Elt: I.getOperand(i_nocapture: 0));
1988 Args.push_back(/* zero index */ Elt: B.getInt32(C: 0));
1989 for (unsigned J = 0; J < SGEP->getNumIndices(); ++J)
1990 Args.push_back(Elt: SGEP->getIndexOperand(Index: J));
1991
1992 Instruction *NewI =
1993 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: Types, Args);
1994 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1995 return NewI;
1996}
1997
1998Instruction *
1999SPIRVEmitIntrinsicsImpl::visitGetElementPtrInst(GetElementPtrInst &I) {
2000 IRBuilder<> B(I.getParent());
2001 B.SetInsertPoint(&I);
2002
2003 // OpPtrAccessChain requires a scalar pointer result; scalarize per-lane
2004 // GEPs that return <N x ptr> and rebuild the vector via insertelement.
2005 if (auto *RetVTy = dyn_cast<FixedVectorType>(Val: I.getType())) {
2006 unsigned N = RetVTy->getNumElements();
2007 Value *PtrOp = I.getPointerOperand();
2008 bool PtrIsVec = isa<VectorType>(Val: PtrOp->getType());
2009 Type *ResultPtrTy = RetVTy->getElementType();
2010 Type *ScalarPtrTy = PtrOp->getType()->getScalarType();
2011 SmallVector<Type *, 2> GepTypes = {ResultPtrTy, ScalarPtrTy};
2012 Value *InBounds = B.getInt1(V: I.isInBounds());
2013 Type *LanePointeeTy = getGEPType(Ref: &I);
2014 Type *SrcElemTy = I.getSourceElementType();
2015
2016 // Pin the lane pointee type on the vector operand and on each extracted
2017 // lane so the prelegalizer wraps them as OpTypeVector/OpTypePointer of
2018 // the right element type instead of defaulting to i8.
2019 if (PtrIsVec)
2020 GR->buildAssignPtr(B, ElemTy: SrcElemTy, Arg: PtrOp);
2021
2022 Value *VecResult = PoisonValue::get(T: RetVTy);
2023 for (unsigned Lane = 0; Lane < N; ++Lane) {
2024 Value *LaneIdx = B.getInt32(C: Lane);
2025 Value *ScalarPtr = PtrOp;
2026 if (PtrIsVec) {
2027 SmallVector<Type *, 3> ExtractTypes = {ScalarPtrTy, PtrOp->getType(),
2028 LaneIdx->getType()};
2029 ScalarPtr = B.CreateIntrinsic(ID: Intrinsic::spv_extractelt, OverloadTypes: {ExtractTypes},
2030 Args: {PtrOp, LaneIdx});
2031 GR->buildAssignPtr(B, ElemTy: SrcElemTy, Arg: ScalarPtr);
2032 }
2033 SmallVector<Value *, 4> Args;
2034 Args.push_back(Elt: InBounds);
2035 Args.push_back(Elt: ScalarPtr);
2036 for (Value *Idx : I.indices()) {
2037 if (isa<VectorType>(Val: Idx->getType())) {
2038 // We cannot use the builder here as for splat-ed / constant vectors
2039 // it will fold to the scalar, and then it becomes impossible to
2040 // retrieve / retain the vectorness.
2041 auto *EI =
2042 ExtractElementInst::Create(Vec: Idx, Idx: LaneIdx, NameStr: "", InsertBefore: B.GetInsertPoint());
2043 if (isVector1(Ty: Idx->getType())) // IRTranslator clobbers <1 x T>.
2044 Args.push_back(Elt: visitExtractElementInst(I&: *EI));
2045 else
2046 Args.push_back(Elt: EI);
2047 } else {
2048 Args.push_back(Elt: Idx);
2049 }
2050 }
2051 Value *ScalarGep = B.CreateIntrinsic(ID: Intrinsic::spv_gep, OverloadTypes: GepTypes, Args);
2052 GR->buildAssignPtr(B, ElemTy: LanePointeeTy, Arg: ScalarGep);
2053 VecResult = B.CreateInsertElement(Vec: VecResult, NewElt: ScalarGep, Idx: LaneIdx);
2054 }
2055
2056 auto *NewI = cast<Instruction>(Val: VecResult);
2057 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2058
2059 if (CallInst *Old = GR->findAssignPtrTypeInstr(Val: NewI)) {
2060 Old->eraseFromParent();
2061 GR->addAssignPtrTypeInstr(Val: NewI, AssignPtrTyCI: nullptr);
2062 }
2063 setInsertPointAfterDef(B, I: NewI);
2064 GR->buildAssignPtr(B, ElemTy: LanePointeeTy, Arg: NewI);
2065
2066 return NewI;
2067 }
2068
2069 if (TM.getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: &I)) {
2070 // Logical SPIR-V cannot use the OpPtrAccessChain instruction. If the first
2071 // index of the GEP is not 0, then we need to try to adjust it.
2072 //
2073 // If the GEP is doing byte addressing, try to rebuild the full access chain
2074 // from the type of the pointer.
2075 if (getByteAddressingMultiplier(Ty: I.getSourceElementType())) {
2076 return buildLogicalAccessChainFromGEP(GEP&: I);
2077 }
2078
2079 // Look for the array-to-pointer decay. If this is the pattern
2080 // we can adjust the types, and prepend a 0 to the indices.
2081 Value *PtrOp = I.getPointerOperand();
2082 Type *SrcElemTy = I.getSourceElementType();
2083 Type *DeducedPointeeTy = deduceElementType(I: PtrOp, UnknownElemTypeI8: true);
2084
2085 if (auto *ArrTy = dyn_cast<ArrayType>(Val: DeducedPointeeTy)) {
2086 if (ArrTy->getElementType() == SrcElemTy) {
2087 SmallVector<Value *> NewIndices;
2088 Type *FirstIdxType = I.getOperand(i_nocapture: 1)->getType();
2089 NewIndices.push_back(Elt: ConstantInt::get(Ty: FirstIdxType, V: 0));
2090 for (Value *Idx : I.indices())
2091 NewIndices.push_back(Elt: Idx);
2092
2093 SmallVector<Type *, 2> Types = {I.getType(), I.getPointerOperandType()};
2094 SmallVector<Value *, 4> Args;
2095 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
2096 Args.push_back(Elt: I.getPointerOperand());
2097 Args.append(in_start: NewIndices.begin(), in_end: NewIndices.end());
2098
2099 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep,
2100 OverloadTypes: {Types}, Args: {Args});
2101 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2102 return NewI;
2103 }
2104 }
2105 }
2106
2107 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
2108 SmallVector<Value *, 4> Args;
2109 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
2110 llvm::append_range(C&: Args, R: I.operands());
2111 Instruction *NewI =
2112 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
2113 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2114 return NewI;
2115}
2116
2117Instruction *SPIRVEmitIntrinsicsImpl::visitBitCastInst(BitCastInst &I) {
2118 IRBuilder<> B(I.getParent());
2119 B.SetInsertPoint(&I);
2120 Value *Source = I.getOperand(i_nocapture: 0);
2121
2122 // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of
2123 // varying element types. In case of IR coming from older versions of LLVM
2124 // such bitcasts do not provide sufficient information, should be just skipped
2125 // here, and handled in insertPtrCastOrAssignTypeInstr.
2126 if (isPointerTy(T: I.getType())) {
2127 replaceAllUsesWith(Src: &I, Dest: Source);
2128 I.eraseFromParent();
2129 return nullptr;
2130 }
2131
2132 SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
2133 SmallVector<Value *> Args(I.op_begin(), I.op_end());
2134 Instruction *NewI =
2135 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_bitcast, OverloadTypes: {Types}, Args: {Args});
2136 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2137 return NewI;
2138}
2139
2140void SPIRVEmitIntrinsicsImpl::insertAssignPtrTypeTargetExt(
2141 TargetExtType *AssignedType, Value *V, IRBuilder<> &B) {
2142 Type *VTy = V->getType();
2143
2144 // A couple of sanity checks.
2145 assert((isPointerTy(VTy)) && "Expect a pointer type!");
2146 if (Type *ElemTy = getPointeeType(Ty: VTy))
2147 if (ElemTy != AssignedType)
2148 report_fatal_error(reason: "Unexpected pointer element type!");
2149
2150 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: V);
2151 if (!AssignCI) {
2152 GR->buildAssignType(B, Ty: AssignedType, Arg: V, CanUseAnyVectorRank);
2153 return;
2154 }
2155
2156 Type *CurrentType =
2157 dyn_cast<ConstantAsMetadata>(
2158 Val: cast<MetadataAsValue>(Val: AssignCI->getOperand(i_nocapture: 1))->getMetadata())
2159 ->getType();
2160 if (CurrentType == AssignedType)
2161 return;
2162
2163 // Builtin types cannot be redeclared or casted.
2164 if (CurrentType->isTargetExtTy())
2165 report_fatal_error(reason: "Type mismatch " + CurrentType->getTargetExtName() +
2166 "/" + AssignedType->getTargetExtName() +
2167 " for value " + V->getName(),
2168 gen_crash_diag: false);
2169
2170 // Our previous guess about the type seems to be wrong, let's update
2171 // inferred type according to a new, more precise type information.
2172 GR->updateAssignType(
2173 AssignCI, Arg: V, OfType: getNormalizedPoisonValue(Ty: AssignedType, CanUseAnyVectorRank));
2174}
2175
2176void SPIRVEmitIntrinsicsImpl::replacePointerOperandWithPtrCast(
2177 Instruction *I, Value *Pointer, Type *ExpectedElementType,
2178 unsigned OperandToReplace, IRBuilder<> &B) {
2179 TypeValidated.insert(Ptr: I);
2180
2181 // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType
2182 Type *PointerElemTy = deduceElementTypeHelper(I: Pointer, UnknownElemTypeI8: false);
2183 if (PointerElemTy == ExpectedElementType ||
2184 isEquivalentTypes(Ty1: PointerElemTy, Ty2: ExpectedElementType))
2185 return;
2186
2187 setInsertPointSkippingPhis(B, I);
2188 Value *ExpectedElementVal =
2189 getNormalizedPoisonValue(Ty: ExpectedElementType, CanUseAnyVectorRank);
2190 MetadataAsValue *VMD = buildMD(Arg: ExpectedElementVal);
2191 unsigned AddressSpace = getPointerAddressSpace(T: Pointer->getType());
2192 bool FirstPtrCastOrAssignPtrType = true;
2193
2194 // Do not emit new spv_ptrcast if equivalent one already exists or when
2195 // spv_assign_ptr_type already targets this pointer with the same element
2196 // type.
2197 if (Pointer->hasUseList()) {
2198 for (auto User : Pointer->users()) {
2199 auto *II = dyn_cast<IntrinsicInst>(Val: User);
2200 if (!II ||
2201 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
2202 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
2203 II->getOperand(i_nocapture: 0) != Pointer)
2204 continue;
2205
2206 // There is some spv_ptrcast/spv_assign_ptr_type already targeting this
2207 // pointer.
2208 FirstPtrCastOrAssignPtrType = false;
2209 if (II->getOperand(i_nocapture: 1) != VMD ||
2210 dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 2))->getSExtValue() !=
2211 AddressSpace)
2212 continue;
2213
2214 // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the
2215 // same element type and address space.
2216 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
2217 return;
2218
2219 // This must be a spv_ptrcast, do not emit new if this one has the same BB
2220 // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type.
2221 if (II->getParent() != I->getParent())
2222 continue;
2223
2224 I->setOperand(i: OperandToReplace, Val: II);
2225 return;
2226 }
2227 }
2228
2229 // Never replace an already-deduced pointer element type with a non-pointer
2230 // one. The conflicting use comes from a mis-deduced expected type. Leave the
2231 // operand untouched rather than emitting a ptrcast that re-introduces the
2232 // collapsed type at the use site.
2233 if (PointerElemTy && isPointerTyOrWrapper(Ty: PointerElemTy) &&
2234 !isPointerTyOrWrapper(Ty: ExpectedElementType) &&
2235 tracesToPointerAlloca(V: Pointer))
2236 return;
2237
2238 if (isa<Instruction>(Val: Pointer) || isa<Argument>(Val: Pointer)) {
2239 if (FirstPtrCastOrAssignPtrType) {
2240 // If this would be the first spv_ptrcast, do not emit spv_ptrcast and
2241 // emit spv_assign_ptr_type instead.
2242 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
2243 return;
2244 } else if (isTodoType(Op: Pointer)) {
2245 eraseTodoType(Op: Pointer);
2246 if (!isa<CallInst>(Val: Pointer) && !isaGEP(V: Pointer) &&
2247 !isa<AllocaInst>(Val: Pointer)) {
2248 // If this wouldn't be the first spv_ptrcast but existing type info is
2249 // uncomplete, update spv_assign_ptr_type arguments.
2250 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Pointer)) {
2251 Type *PrevElemTy = GR->findDeducedElementType(Val: Pointer);
2252 assert(PrevElemTy);
2253 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
2254 std::make_pair(x&: I, y&: Pointer)};
2255 GR->updateAssignType(AssignCI, Arg: Pointer, OfType: ExpectedElementVal);
2256 propagateElemType(Op: Pointer, ElemTy: PrevElemTy, VisitedSubst);
2257 } else {
2258 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
2259 }
2260 return;
2261 }
2262 }
2263 }
2264
2265 // Emit spv_ptrcast
2266 SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()};
2267 SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(C: AddressSpace)};
2268 auto *PtrCastI = B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
2269 I->setOperand(i: OperandToReplace, Val: PtrCastI);
2270 // We need to set up a pointee type for the newly created spv_ptrcast.
2271 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: PtrCastI);
2272}
2273
2274void SPIRVEmitIntrinsicsImpl::insertPtrCastOrAssignTypeInstr(Instruction *I,
2275 IRBuilder<> &B) {
2276 // Handle basic instructions:
2277 StoreInst *SI = dyn_cast<StoreInst>(Val: I);
2278 if (IsKernelArgInt8(F: CurrF, SI)) {
2279 replacePointerOperandWithPtrCast(
2280 I, Pointer: SI->getValueOperand(), ExpectedElementType: IntegerType::getInt8Ty(C&: CurrF->getContext()),
2281 OperandToReplace: 0, B);
2282 }
2283 if (SI) {
2284 Value *Op = SI->getValueOperand();
2285 Value *Pointer = SI->getPointerOperand();
2286 Type *OpTy = Op->getType();
2287 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
2288 OpTy = restoreMutatedType(GR, I: OpI, Ty: OpTy);
2289 if (auto It = AggrConstTypes.find(Val: OpI); It != AggrConstTypes.end())
2290 OpTy = It->second;
2291 }
2292 if (OpTy == Op->getType())
2293 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: Op, UnknownElemTypeI8: false);
2294 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 1, B);
2295 return;
2296 }
2297 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
2298 Value *Pointer = LI->getPointerOperand();
2299 Type *OpTy = LI->getType();
2300 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
2301 if (Type *ElemTy = GR->findDeducedElementType(Val: LI)) {
2302 OpTy = getTypedPointerWrapper(ElemTy, AS: PtrTy->getAddressSpace());
2303 } else {
2304 Type *NewOpTy = OpTy;
2305 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: LI, UnknownElemTypeI8: false);
2306 if (OpTy == NewOpTy)
2307 insertTodoType(Op: Pointer);
2308 }
2309 }
2310 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
2311 return;
2312 }
2313 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(Val: I)) {
2314 Value *Pointer = GEPI->getPointerOperand();
2315 Type *OpTy = nullptr;
2316
2317 // Logical SPIR-V is not allowed to use Op*PtrAccessChain instructions. If
2318 // the first index is 0, then we can trivially lower to OpAccessChain. If
2319 // not we need to try to rewrite the GEP. We avoid adding a pointer cast at
2320 // this time, and will rewrite the GEP when visiting it.
2321 if (TM.getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: GEPI)) {
2322 return;
2323 }
2324
2325 // In all cases, fall back to the GEP type if type scavenging failed.
2326 if (!OpTy)
2327 OpTy = GEPI->getSourceElementType();
2328
2329 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
2330 if (isNestedPointer(Ty: OpTy))
2331 insertTodoType(Op: Pointer);
2332 return;
2333 }
2334
2335 // TODO: review and merge with existing logics:
2336 // Handle calls to builtins (non-intrinsics):
2337 CallInst *CI = dyn_cast<CallInst>(Val: I);
2338 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
2339 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
2340 return;
2341
2342 // collect information about formal parameter types
2343 std::string DemangledName =
2344 getOclOrSpirvBuiltinDemangledName(Name: CI->getCalledFunction()->getName());
2345 Function *CalledF = CI->getCalledFunction();
2346 SmallVector<Type *, 4> CalledArgTys;
2347 bool HaveTypes = false;
2348 for (unsigned OpIdx = 0; OpIdx < CalledF->arg_size(); ++OpIdx) {
2349 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2350 Type *ArgType = CalledArg->getType();
2351 if (!isPointerTy(T: ArgType)) {
2352 CalledArgTys.push_back(Elt: nullptr);
2353 } else if (Type *ArgTypeElem = getPointeeType(Ty: ArgType)) {
2354 CalledArgTys.push_back(Elt: ArgTypeElem);
2355 HaveTypes = true;
2356 } else {
2357 Type *ElemTy = GR->findDeducedElementType(Val: CalledArg);
2358 if (!ElemTy && hasPointeeTypeAttr(Arg: CalledArg))
2359 ElemTy = getPointeeTypeByAttr(Arg: CalledArg);
2360 if (!ElemTy) {
2361 ElemTy = getPointeeTypeByCallInst(DemangledName, CalledF, OpIdx);
2362 if (ElemTy) {
2363 GR->addDeducedElementType(Val: CalledArg,
2364 Ty: normalizeType(Ty: ElemTy, CanUseAnyVectorRank));
2365 } else {
2366 for (User *U : CalledArg->users()) {
2367 if (Instruction *Inst = dyn_cast<Instruction>(Val: U)) {
2368 if ((ElemTy = deduceElementTypeHelper(I: Inst, UnknownElemTypeI8: false)) != nullptr)
2369 break;
2370 }
2371 }
2372 }
2373 }
2374 HaveTypes |= ElemTy != nullptr;
2375 CalledArgTys.push_back(Elt: ElemTy);
2376 }
2377 }
2378
2379 if (DemangledName.empty() && !HaveTypes)
2380 return;
2381
2382 for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) {
2383 Value *ArgOperand = CI->getArgOperand(i: OpIdx);
2384 if (!isPointerTy(T: ArgOperand->getType()))
2385 continue;
2386
2387 // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs()
2388 if (!isa<Instruction>(Val: ArgOperand) && !isa<Argument>(Val: ArgOperand)) {
2389 // However, we may have assumptions about the formal argument's type and
2390 // may have a need to insert a ptr cast for the actual parameter of this
2391 // call.
2392 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2393 if (!GR->findDeducedElementType(Val: CalledArg))
2394 continue;
2395 }
2396
2397 Type *ExpectedType =
2398 OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr;
2399 if (!ExpectedType && !DemangledName.empty())
2400 ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType(
2401 DemangledCall: DemangledName, ArgIdx: OpIdx, Ctx&: I->getContext());
2402 if (!ExpectedType || ExpectedType->isVoidTy())
2403 continue;
2404
2405 if (ExpectedType->isTargetExtTy() &&
2406 !isTypedPointerWrapper(ExtTy: cast<TargetExtType>(Val: ExpectedType)))
2407 insertAssignPtrTypeTargetExt(AssignedType: cast<TargetExtType>(Val: ExpectedType),
2408 V: ArgOperand, B);
2409 else
2410 replacePointerOperandWithPtrCast(I: CI, Pointer: ArgOperand, ExpectedElementType: ExpectedType, OperandToReplace: OpIdx, B);
2411 }
2412}
2413
2414Instruction *
2415SPIRVEmitIntrinsicsImpl::visitInsertElementInst(InsertElementInst &I) {
2416 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2417 // type in LLT and IRTranslator will replace it by the scalar.
2418 if (isVector1(Ty: I.getType()) && !CanUseAnyVectorRank)
2419 return &I;
2420
2421 SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType(),
2422 I.getOperand(i_nocapture: 1)->getType(),
2423 I.getOperand(i_nocapture: 2)->getType()};
2424 IRBuilder<> B(I.getParent());
2425 B.SetInsertPoint(&I);
2426 SmallVector<Value *> Args(I.op_begin(), I.op_end());
2427 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_insertelt,
2428 OverloadTypes: {Types}, Args: {Args});
2429 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2430 return NewI;
2431}
2432
2433Instruction *
2434SPIRVEmitIntrinsicsImpl::visitExtractElementInst(ExtractElementInst &I) {
2435 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2436 // type in LLT and IRTranslator will replace it by the scalar.
2437 if (isVector1(Ty: I.getVectorOperandType()) && !CanUseAnyVectorRank)
2438 return &I;
2439
2440 IRBuilder<> B(I.getParent());
2441 B.SetInsertPoint(&I);
2442 SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
2443 I.getIndexOperand()->getType()};
2444 SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
2445 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_extractelt,
2446 OverloadTypes: {Types}, Args: {Args});
2447 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2448 return NewI;
2449}
2450
2451Instruction *SPIRVEmitIntrinsicsImpl::visitInsertValueInst(InsertValueInst &I) {
2452 IRBuilder<> B(I.getParent());
2453 B.SetInsertPoint(&I);
2454 SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
2455 SmallVector<Value *> Args;
2456 Value *AggregateOp = I.getAggregateOperand();
2457 if (isa<UndefValue>(Val: AggregateOp))
2458 Args.push_back(Elt: UndefValue::get(T: B.getInt32Ty()));
2459 else
2460 Args.push_back(Elt: AggregateOp);
2461 Args.push_back(Elt: I.getInsertedValueOperand());
2462 for (auto &Op : I.indices())
2463 Args.push_back(Elt: B.getInt32(C: Op));
2464 Instruction *NewI =
2465 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_insertv, OverloadTypes: {Types}, Args: {Args});
2466 replaceMemInstrUses(Old: &I, New: NewI, B);
2467 return NewI;
2468}
2469
2470Instruction *
2471SPIRVEmitIntrinsicsImpl::visitExtractValueInst(ExtractValueInst &I) {
2472 IRBuilder<> B(I.getParent());
2473 B.SetInsertPoint(&I);
2474 if (I.getAggregateOperand()->getType()->isAggregateType()) {
2475 // Mutate an aggregate-returning spv_extractv producer to i32 so
2476 // IRTranslator does not see a multi-register value.
2477 CallBase *CB = dyn_cast<CallBase>(Val: I.getAggregateOperand());
2478 if (!CB || CB->getIntrinsicID() != Intrinsic::spv_extractv)
2479 return &I;
2480 CB->mutateType(Ty: B.getInt32Ty());
2481 }
2482 SmallVector<Value *> Args(I.operands());
2483 for (auto &Op : I.indices())
2484 Args.push_back(Elt: B.getInt32(C: Op));
2485 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_extractv,
2486 OverloadTypes: {I.getType()}, Args: {Args});
2487 // If this aggregate extract feeds another insertvalue, the extracted
2488 // composite is used as a SPIR-V value-id by llvm.spv.insertv. Keep the real
2489 // aggregate type in metadata, but expose the value itself as i32 so the
2490 // intrinsic signature remains valid.
2491 if (NewI->getType()->isAggregateType() &&
2492 any_of(Range: I.users(), P: [](User *U) { return isa<InsertValueInst>(Val: U); })) {
2493 AggrConstTypes[NewI] = I.getType();
2494 NewI->mutateType(Ty: B.getInt32Ty());
2495 replaceMemInstrUses(Old: &I, New: NewI, B);
2496 return NewI;
2497 }
2498 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2499 // If the aggregate result feeds a return or callsite whose type was rewritten
2500 // to an i32 value-id by SPIRVPrepareFunctions, mutate it to match.
2501 if (NewI->getType()->isAggregateType()) {
2502 for (const Use &U : NewI->uses()) {
2503 User *Usr = U.getUser();
2504 if (auto *RI = dyn_cast<ReturnInst>(Val: Usr)) {
2505 if (RI->getFunction()->getReturnType() != NewI->getType()) {
2506 NewI->mutateType(Ty: B.getInt32Ty());
2507 break;
2508 }
2509 continue;
2510 }
2511 auto *CB = dyn_cast<CallBase>(Val: Usr);
2512 if (!CB || !CB->isArgOperand(U: &U))
2513 continue;
2514 unsigned ArgNo = CB->getArgOperandNo(U: &U);
2515 FunctionType *FT = CB->getFunctionType();
2516 if (ArgNo < FT->getNumParams() &&
2517 !FT->getParamType(i: ArgNo)->isAggregateType()) {
2518 NewI->mutateType(Ty: B.getInt32Ty());
2519 break;
2520 }
2521 }
2522 }
2523 return NewI;
2524}
2525
2526Instruction *SPIRVEmitIntrinsicsImpl::visitLoadInst(LoadInst &I) {
2527 if (!I.getType()->isAggregateType())
2528 return &I;
2529 IRBuilder<> B(I.getParent());
2530 B.SetInsertPoint(&I);
2531 TrackConstants = false;
2532 const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
2533 MachineMemOperand::Flags Flags =
2534 TLI->getLoadMemOperandFlags(LI: I, DL: CurrF->getDataLayout());
2535
2536 unsigned IntrinsicId;
2537 SmallVector<Value *, 4> Args = {I.getPointerOperand(), B.getInt16(C: Flags)};
2538 if (!I.isAtomic()) {
2539 IntrinsicId = Intrinsic::spv_load;
2540 Args.push_back(Elt: B.getInt32(C: I.getAlign().value()));
2541 } else {
2542 IntrinsicId = Intrinsic::spv_atomic_load;
2543 Args.push_back(Elt: B.getInt8(C: static_cast<uint8_t>(I.getOrdering())));
2544 }
2545 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
2546 ID: IntrinsicId, OverloadTypes: {I.getOperand(i_nocapture: 0)->getType()}, Args);
2547
2548 replaceMemInstrUses(Old: &I, New: NewI, B);
2549 return NewI;
2550}
2551
2552Instruction *SPIRVEmitIntrinsicsImpl::visitStoreInst(StoreInst &I) {
2553 if (!AggrStores.contains(Ptr: &I))
2554 return &I;
2555 IRBuilder<> B(I.getParent());
2556 B.SetInsertPoint(&I);
2557 TrackConstants = false;
2558 const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
2559 MachineMemOperand::Flags Flags =
2560 TLI->getStoreMemOperandFlags(SI: I, DL: CurrF->getDataLayout());
2561 auto *PtrOp = I.getPointerOperand();
2562
2563 if (I.getValueOperand()->getType()->isAggregateType()) {
2564 // It is possible that what used to be an ExtractValueInst has been replaced
2565 // with a call to the spv_extractv intrinsic, and that said call hasn't
2566 // had its return type replaced with i32 during the dedicated pass (because
2567 // it was emitted later); we have to handle this here, because IRTranslator
2568 // cannot deal with multi-register types at the moment.
2569 CallBase *CB = dyn_cast<CallBase>(Val: I.getValueOperand());
2570 assert(CB && CB->getIntrinsicID() == Intrinsic::spv_extractv &&
2571 "Unexpected argument of aggregate type, should be spv_extractv!");
2572 CB->mutateType(Ty: B.getInt32Ty());
2573 }
2574
2575 unsigned IntrinsicId;
2576 SmallVector<Value *, 4> Args = {I.getValueOperand(), PtrOp,
2577 B.getInt16(C: Flags)};
2578 if (!I.isAtomic()) {
2579 IntrinsicId = Intrinsic::spv_store;
2580 Args.push_back(Elt: B.getInt32(C: I.getAlign().value()));
2581 } else {
2582 IntrinsicId = Intrinsic::spv_atomic_store;
2583 Args.push_back(Elt: B.getInt8(C: static_cast<uint8_t>(I.getOrdering())));
2584 }
2585 Instruction *NewI = B.CreateIntrinsicWithoutFolding(
2586 ID: IntrinsicId, OverloadTypes: {I.getValueOperand()->getType(), PtrOp->getType()}, Args);
2587 NewI->copyMetadata(SrcInst: I);
2588 I.eraseFromParent();
2589 return NewI;
2590}
2591
2592Instruction *SPIRVEmitIntrinsicsImpl::visitAllocaInst(AllocaInst &I) {
2593 Value *ArraySize = nullptr;
2594 if (I.isArrayAllocation()) {
2595 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I.getFunction());
2596 if (!STI->canUseExtension(
2597 E: SPIRV::Extension::SPV_INTEL_variable_length_array))
2598 report_fatal_error(
2599 reason: "array allocation: this instruction requires the following "
2600 "SPIR-V extension: SPV_INTEL_variable_length_array",
2601 gen_crash_diag: false);
2602 ArraySize = I.getArraySize();
2603 }
2604 IRBuilder<> B(I.getParent());
2605 B.SetInsertPoint(&I);
2606 TrackConstants = false;
2607 Type *PtrTy = I.getType();
2608 Instruction *NewI =
2609 ArraySize
2610 ? B.CreateIntrinsicWithoutFolding(
2611 ID: Intrinsic::spv_alloca_array, OverloadTypes: {PtrTy, ArraySize->getType()},
2612 Args: {ArraySize, B.getInt32(C: I.getAlign().value())})
2613 : B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_alloca, OverloadTypes: {PtrTy},
2614 Args: {B.getInt32(C: I.getAlign().value())});
2615 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2616 return NewI;
2617}
2618
2619Instruction *
2620SPIRVEmitIntrinsicsImpl::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2621 assert(I.getType()->isAggregateType() && "Aggregate result is expected");
2622 IRBuilder<> B(I.getParent());
2623 B.SetInsertPoint(&I);
2624 SmallVector<Value *> Args(I.operands());
2625 const Triple &TT = TM.getTargetTriple();
2626 Args.push_back(Elt: B.getInt32(C: static_cast<uint32_t>(
2627 getMemScope(TT, Ctx&: I.getContext(), Id: I.getSyncScopeID()))));
2628 // Per SPIR-V spec atomic ops must combine the ordering bits with the
2629 // storage-class bit.
2630 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2631 unsigned AS = I.getPointerOperand()->getType()->getPointerAddressSpace();
2632 uint32_t ScSem = static_cast<uint32_t>(
2633 getMemSemanticsForStorageClass(SC: addressSpaceToStorageClass(AddrSpace: AS, STI: ST)));
2634 Args.push_back(Elt: B.getInt32(C: getMemSemanticsWithStorageClass(
2635 TT, OrderSem: static_cast<uint32_t>(getMemSemantics(Ord: I.getSuccessOrdering())),
2636 StorageClassSem: ScSem)));
2637 Args.push_back(Elt: B.getInt32(C: getMemSemanticsWithStorageClass(
2638 TT, OrderSem: static_cast<uint32_t>(getMemSemantics(Ord: I.getFailureOrdering())),
2639 StorageClassSem: ScSem)));
2640 Instruction *NewI = B.CreateIntrinsicWithoutFolding(
2641 ID: Intrinsic::spv_cmpxchg, OverloadTypes: {I.getPointerOperand()->getType()}, Args: {Args});
2642 replaceMemInstrUses(Old: &I, New: NewI, B);
2643 return NewI;
2644}
2645
2646static bool isAbortCall(const Instruction &I, const SPIRVSubtarget &ST) {
2647 auto *CI = dyn_cast<CallInst>(Val: &I);
2648 if (!CI)
2649 return false;
2650 switch (CI->getIntrinsicID()) {
2651 case Intrinsic::spv_abort:
2652 return true;
2653 case Intrinsic::trap:
2654 case Intrinsic::ubsantrap:
2655 // When the extension is enabled, selection lowers these to OpAbortKHR.
2656 return ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_abort);
2657 default:
2658 return false;
2659 }
2660}
2661
2662// The OpAbortKHR instruction itself is a block terminator, so we don't need to
2663// emit an extra OpUnreachable instruction.
2664static bool precededByAbortIntrinsic(const UnreachableInst &I,
2665 const SPIRVSubtarget &ST) {
2666 // Find a previous non-debug instruction.
2667 const Instruction *Prev = I.getPrevNode();
2668 while (Prev && Prev->isDebugOrPseudoInst())
2669 Prev = Prev->getPrevNode();
2670
2671 if (Prev && isAbortCall(I: *Prev, ST))
2672 return true;
2673
2674 assert(llvm::none_of(
2675 *I.getParent(),
2676 [&ST](const Instruction &II) { return isAbortCall(II, ST); }) &&
2677 "abort-like call must be the last non-debug instruction before its "
2678 "block's terminator");
2679 return false;
2680}
2681
2682Instruction *SPIRVEmitIntrinsicsImpl::visitUnreachableInst(UnreachableInst &I) {
2683 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2684 if (precededByAbortIntrinsic(I, ST))
2685 return &I;
2686 IRBuilder<> B(&I);
2687 B.CreateIntrinsic(ID: Intrinsic::spv_unreachable, Args: {});
2688 return &I;
2689}
2690
2691// llvm.compiler.used and llvm.used hold use-list entries that protect their
2692// referenced globals from DCE without participating in code generation.
2693static bool isUseListGlobal(StringRef Name) {
2694 return Name == "llvm.compiler.used" || Name == "llvm.used";
2695}
2696
2697// Returns true for module-level globals that should not have SPIR-V intrinsics
2698// emitted (use-list globals plus llvm.global.annotations).
2699static bool isArtificialGlobal(StringRef Name) {
2700 return isUseListGlobal(Name) || Name == "llvm.global.annotations";
2701}
2702
2703// Returns true if every use of GV traces back to llvm.compiler.used or
2704// llvm.used.
2705static bool hasOnlyArtificialUses(const GlobalVariable &GV) {
2706 SmallPtrSet<const Value *, 8> Visited;
2707 SmallVector<const Value *> Stack(GV.users());
2708 while (!Stack.empty()) {
2709 const Value *V = Stack.pop_back_val();
2710 if (!Visited.insert(Ptr: V).second)
2711 continue;
2712 if (const auto *GVUser = dyn_cast<GlobalVariable>(Val: V)) {
2713 if (!isUseListGlobal(Name: GVUser->getName()))
2714 return false;
2715 continue;
2716 }
2717 if (const auto *C = dyn_cast<Constant>(Val: V)) {
2718 Stack.append(in_start: C->user_begin(), in_end: C->user_end());
2719 continue;
2720 }
2721 return false;
2722 }
2723 return true;
2724}
2725
2726static bool
2727shouldEmitIntrinsicsForGlobalValue(const GlobalVariableUsers &GVUsers,
2728 const GlobalVariable &GV,
2729 const Function *F) {
2730 // Skip special artificial variables.
2731 if (isArtificialGlobal(Name: GV.getName()))
2732 return false;
2733
2734 auto &UserFunctions = GVUsers.getTransitiveUserFunctions(GV);
2735 if (UserFunctions.contains(Ptr: F))
2736 return true;
2737
2738 // Do not emit the intrinsics in this function, it's going to be emitted on
2739 // the functions that reference it.
2740 if (!UserFunctions.empty())
2741 return false;
2742
2743 // Emit definitions for globals that are not referenced by any function on the
2744 // first function definition.
2745 const Module &M = *F->getParent();
2746 const Function &FirstDefinition = *M.getFunctionDefs().begin();
2747 return F == &FirstDefinition;
2748}
2749
2750Value *SPIRVEmitIntrinsicsImpl::buildSpvUndefComposite(Type *AggrTy,
2751 IRBuilder<> &B) {
2752 auto MakeLeaf = [&](Type *ElemTy) -> Instruction * {
2753 CallInst *Leaf = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_undef, Args: {});
2754 AggrConsts[Leaf] = PoisonValue::get(T: ElemTy);
2755 AggrConstTypes[Leaf] = ElemTy;
2756 return Leaf;
2757 };
2758 SmallVector<Value *, 4> Elems;
2759 if (auto *ArrTy = dyn_cast<ArrayType>(Val: AggrTy)) {
2760 Elems.assign(NumElts: ArrTy->getNumElements(), Elt: MakeLeaf(ArrTy->getElementType()));
2761 } else {
2762 auto *StructTy = cast<StructType>(Val: AggrTy);
2763 DenseMap<Type *, Instruction *> LeafByType;
2764 for (unsigned I = 0; I < StructTy->getNumElements(); ++I) {
2765 Type *ElemTy = StructTy->getContainedType(i: I);
2766 auto &Entry = LeafByType[ElemTy];
2767 if (!Entry)
2768 Entry = MakeLeaf(ElemTy);
2769 Elems.push_back(Elt: Entry);
2770 }
2771 }
2772 CallInst *Composite = B.CreateIntrinsicWithoutFolding(
2773 ID: Intrinsic::spv_const_composite, OverloadTypes: {B.getInt32Ty()}, Args: Elems);
2774 AggrConsts[Composite] = PoisonValue::get(T: AggrTy);
2775 AggrConstTypes[Composite] = AggrTy;
2776 return Composite;
2777}
2778
2779// If a function directly returns an aggregate-typed call result,
2780// the ReturnInst carries an aggregate while the function signature
2781// was rewritten to i32 by SPIRVPrepareFunctions. Rebuild the return value
2782// via extractvalue/insertvalue so the regular spv_extractv/spv_insertv
2783// lowering produces a valid OpReturnValue.
2784void SPIRVEmitIntrinsicsImpl::reconstructAggregateReturns(Function &Func,
2785 IRBuilder<> &B) {
2786 Type *OrigRetTy = GR->findMutated(Val: &Func);
2787 if (!OrigRetTy || !OrigRetTy->isAggregateType())
2788 return;
2789 for (BasicBlock &BB : Func) {
2790 auto *RI = dyn_cast<ReturnInst>(Val: BB.getTerminator());
2791 if (!RI)
2792 continue;
2793 Value *RetVal = RI->getReturnValue();
2794 if (!RetVal || RetVal->getType() != OrigRetTy || !isa<CallBase>(Val: RetVal))
2795 continue;
2796 Type *AggrTy = RetVal->getType();
2797 uint64_t NumElts = isa<StructType>(Val: AggrTy)
2798 ? cast<StructType>(Val: AggrTy)->getNumElements()
2799 : cast<ArrayType>(Val: AggrTy)->getNumElements();
2800 B.SetInsertPoint(RI);
2801 Value *Rebuilt = PoisonValue::get(T: AggrTy);
2802 for (uint64_t I = 0; I < NumElts; ++I) {
2803 Value *Elt = B.CreateExtractValue(Agg: RetVal, Idxs: I);
2804 Rebuilt = B.CreateInsertValue(Agg: Rebuilt, Val: Elt, Idxs: I);
2805 }
2806 RI->setOperand(i_nocapture: 0, Val_nocapture: Rebuilt);
2807 }
2808}
2809
2810void SPIRVEmitIntrinsicsImpl::processGlobalValue(GlobalVariable &GV,
2811 IRBuilder<> &B) {
2812
2813 if (!shouldEmitIntrinsicsForGlobalValue(GVUsers, GV, F: CurrF))
2814 return;
2815
2816 // Record the pointee type for every global, not only initialized ones, so an
2817 // undef non-constant aggregate global is not later collapsed to its element
2818 // type. Result is ignored, because TypedPointerType is not supported
2819 // by llvm IR general logic.
2820 deduceElementTypeHelper(I: &GV, UnknownElemTypeI8: false);
2821
2822 Constant *Init = nullptr;
2823 if (hasInitializer(GV: &GV)) {
2824 Init = GV.getInitializer();
2825 Value *InitOp = Init;
2826 if (isa<UndefValue>(Val: Init) && Init->getType()->isAggregateType()) {
2827 const SPIRVSubtarget *STI = TM.getSubtargetImpl();
2828 bool UsePoison =
2829 isa<PoisonValue>(Val: Init) &&
2830 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
2831 if (UsePoison) {
2832 CallInst *Call = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_poison,
2833 OverloadTypes: {B.getInt32Ty()}, Args: {});
2834 AggrConsts[Call] = cast<PoisonValue>(Val: Init);
2835 AggrConstTypes[Call] = Init->getType();
2836 InitOp = Call;
2837 } else {
2838 InitOp = buildSpvUndefComposite(AggrTy: Init->getType(), B);
2839 }
2840 }
2841 Type *Ty = isAggrConstForceInt32(V: Init) ? B.getInt32Ty() : Init->getType();
2842 Constant *Const = isAggrConstForceInt32(V: Init) ? B.getInt32(C: 1) : Init;
2843 CallInst *InitInst = B.CreateIntrinsicWithoutFolding(
2844 ID: Intrinsic::spv_init_global, OverloadTypes: {GV.getType(), Ty}, Args: {&GV, Const});
2845 InitInst->setArgOperand(i: 1, v: InitOp);
2846 }
2847 // Globals with only use-list references have no real function uses. Emit
2848 // spv_unref_global so buildGlobalVariable is called for them.
2849 if (!Init && hasOnlyArtificialUses(GV))
2850 B.CreateIntrinsic(ID: Intrinsic::spv_unref_global, OverloadTypes: GV.getType(), Args: &GV);
2851}
2852
2853// Return true, if we can't decide what is the pointee type now and will get
2854// back to the question later. Return false is spv_assign_ptr_type is not needed
2855// or can be inserted immediately.
2856bool SPIRVEmitIntrinsicsImpl::insertAssignPtrTypeIntrs(Instruction *I,
2857 IRBuilder<> &B,
2858 bool UnknownElemTypeI8) {
2859 reportFatalOnTokenType(I);
2860 if (!isPointerTy(T: I->getType()) || !requireAssignType(I))
2861 return false;
2862
2863 setInsertPointAfterDef(B, I);
2864 if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) {
2865 GR->buildAssignPtr(B, ElemTy, Arg: I);
2866 return false;
2867 }
2868 return true;
2869}
2870
2871void SPIRVEmitIntrinsicsImpl::insertAssignTypeIntrs(Instruction *I,
2872 IRBuilder<> &B) {
2873 // TODO: extend the list of functions with known result types
2874 static StringMap<unsigned> ResTypeWellKnown = {
2875 {"async_work_group_copy", WellKnownTypes::Event},
2876 {"async_work_group_strided_copy", WellKnownTypes::Event},
2877 {"__spirv_GroupAsyncCopy", WellKnownTypes::Event}};
2878
2879 reportFatalOnTokenType(I);
2880
2881 bool IsKnown = false;
2882 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
2883 if (!CI->isIndirectCall() && !CI->isInlineAsm() &&
2884 CI->getCalledFunction() && !CI->getCalledFunction()->isIntrinsic()) {
2885 Function *CalledF = CI->getCalledFunction();
2886 std::string DemangledName =
2887 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
2888 FPDecorationId DecorationId = FPDecorationId::NONE;
2889 if (DemangledName.length() > 0)
2890 DemangledName =
2891 SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName, DecorationId: &DecorationId);
2892 auto ResIt = ResTypeWellKnown.find(Key: DemangledName);
2893 if (ResIt != ResTypeWellKnown.end()) {
2894 IsKnown = true;
2895 setInsertPointAfterDef(B, I);
2896 switch (ResIt->second) {
2897 case WellKnownTypes::Event:
2898 GR->buildAssignType(
2899 B, Ty: TargetExtType::get(Context&: I->getContext(), Name: "spirv.Event"), Arg: I,
2900 CanUseAnyVectorRank);
2901 break;
2902 }
2903 }
2904 // check if a floating rounding mode or saturation info is present
2905 switch (DecorationId) {
2906 default:
2907 break;
2908 case FPDecorationId::SAT:
2909 createSaturatedConversionDecoration(I: CI, B);
2910 break;
2911 case FPDecorationId::RTE:
2912 createRoundingModeDecoration(
2913 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTE, B);
2914 break;
2915 case FPDecorationId::RTZ:
2916 createRoundingModeDecoration(
2917 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTZ, B);
2918 break;
2919 case FPDecorationId::RTP:
2920 createRoundingModeDecoration(
2921 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTP, B);
2922 break;
2923 case FPDecorationId::RTN:
2924 createRoundingModeDecoration(
2925 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTN, B);
2926 break;
2927 }
2928 }
2929 }
2930
2931 Type *Ty = I->getType();
2932 if (!IsKnown && !Ty->isVoidTy() && !isPointerTy(T: Ty) && requireAssignType(I)) {
2933 setInsertPointAfterDef(B, I);
2934 Type *TypeToAssign = Ty;
2935 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2936 if (isSpvAggrPlaceholder(V: II)) {
2937 auto It = AggrConstTypes.find(Val: II);
2938 if (It == AggrConstTypes.end())
2939 report_fatal_error(reason: "Unknown composite intrinsic type");
2940 TypeToAssign = It->second;
2941 } else if (II->getIntrinsicID() == Intrinsic::spv_poison) {
2942 if (auto It = AggrConstTypes.find(Val: II); It != AggrConstTypes.end())
2943 TypeToAssign = It->second;
2944 }
2945 } else if (auto It = AggrConstTypes.find(Val: I); It != AggrConstTypes.end())
2946 TypeToAssign = It->second;
2947 TypeToAssign = restoreMutatedType(GR, I, Ty: TypeToAssign);
2948 GR->buildAssignType(B, Ty: TypeToAssign, Arg: I, CanUseAnyVectorRank);
2949 }
2950 for (const auto &Op : I->operands()) {
2951 if (isa<ConstantPointerNull>(Val: Op) || isa<UndefValue>(Val: Op) ||
2952 isVector1(Ty: Op->getType()) || // <1 x T> gets clobbered ty IRTranslator.
2953 // Check GetElementPtrConstantExpr case.
2954 (isa<ConstantExpr>(Val: Op) &&
2955 (isa<GEPOperator>(Val: Op) ||
2956 (cast<ConstantExpr>(Val: Op)->getOpcode() == CastInst::IntToPtr)))) {
2957 setInsertPointSkippingPhis(B, I);
2958 Type *OpTy = Op->getType();
2959 if (isa<UndefValue>(Val: Op) && OpTy->isAggregateType()) {
2960 CallInst *AssignCI =
2961 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {B.getInt32Ty()}, Arg: Op,
2962 Arg2: UndefValue::get(T: B.getInt32Ty()), Imms: {}, B);
2963 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: AssignCI);
2964 } else if (!isa<Instruction>(Val: Op)) {
2965 Type *OpTy = Op->getType();
2966 Type *OpTyElem = getPointeeType(Ty: OpTy);
2967 if (OpTyElem) {
2968 GR->buildAssignPtr(B, ElemTy: OpTyElem, Arg: Op);
2969 } else if (isPointerTy(T: OpTy)) {
2970 Type *ElemTy = GR->findDeducedElementType(Val: Op);
2971 GR->buildAssignPtr(B, ElemTy: ElemTy ? ElemTy : deduceElementType(I: Op, UnknownElemTypeI8: true),
2972 Arg: Op);
2973 } else {
2974 Value *OpTyVal = Op;
2975 if (OpTy->isTargetExtTy()) {
2976 // We need to do this in order to be consistent with how target ext
2977 // types are handled in `processInstrAfterVisit`
2978 OpTyVal = getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank);
2979 }
2980 CallInst *AssignCI = buildIntrWithMD(
2981 IntrID: Intrinsic::spv_assign_type, Types: {OpTy},
2982 Arg: getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank), Arg2: OpTyVal, Imms: {},
2983 B);
2984 GR->addAssignPtrTypeInstr(Val: OpTyVal, AssignPtrTyCI: AssignCI);
2985 }
2986 }
2987 }
2988 }
2989}
2990
2991bool SPIRVEmitIntrinsicsImpl::shouldTryToAddMemAliasingDecoration(
2992 Instruction *Inst) {
2993 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*Inst->getFunction());
2994 if (!STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing))
2995 return false;
2996 // Add aliasing decorations to internal load and store intrinsics.
2997 // Do not attach them to store atomic or load atomic intrinsics / instructions
2998 // since the extension is inconsistent at the moment (we cannot add the
2999 // decoration to atomic stores because they do not have an id).
3000 return match(V: Inst,
3001 P: m_AnyIntrinsic<Intrinsic::spv_load, Intrinsic::spv_store>());
3002}
3003
3004void SPIRVEmitIntrinsicsImpl::insertSpirvDecorations(Instruction *I,
3005 IRBuilder<> &B) {
3006 if (MDNode *MD = I->getMetadata(Kind: "spirv.Decorations")) {
3007 setInsertPointAfterDef(B, I);
3008 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
3009 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
3010 }
3011 // Lower alias.scope/noalias metadata
3012 {
3013 auto processMemAliasingDecoration = [&](unsigned Kind) {
3014 if (MDNode *AliasListMD = I->getMetadata(KindID: Kind)) {
3015 if (shouldTryToAddMemAliasingDecoration(Inst: I)) {
3016 uint32_t Dec = Kind == LLVMContext::MD_alias_scope
3017 ? SPIRV::Decoration::AliasScopeINTEL
3018 : SPIRV::Decoration::NoAliasINTEL;
3019 SmallVector<Value *, 3> Args = {
3020 I, ConstantInt::get(Ty: B.getInt32Ty(), V: Dec),
3021 MetadataAsValue::get(Context&: I->getContext(), MD: AliasListMD)};
3022 setInsertPointAfterDef(B, I);
3023 B.CreateIntrinsic(ID: Intrinsic::spv_assign_aliasing_decoration,
3024 OverloadTypes: {I->getType()}, Args: {Args});
3025 }
3026 }
3027 };
3028 processMemAliasingDecoration(LLVMContext::MD_alias_scope);
3029 processMemAliasingDecoration(LLVMContext::MD_noalias);
3030 }
3031 // MD_fpmath
3032 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_fpmath)) {
3033 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I->getFunction());
3034 bool AllowFPMaxError =
3035 STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_fp_max_error);
3036 if (!AllowFPMaxError)
3037 return;
3038
3039 setInsertPointAfterDef(B, I);
3040 B.CreateIntrinsic(ID: Intrinsic::spv_assign_fpmaxerror_decoration,
3041 OverloadTypes: {I->getType()},
3042 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
3043 }
3044 if (I->getModule()->getTargetTriple().getVendor() == Triple::AMD &&
3045 isa<AtomicRMWInst>(Val: I)) {
3046 // If present, we encode AMDGPU atomic metadata as UserSemantic string
3047 // decorations, which will be parsed during reverse translation.
3048 auto &Ctx = B.getContext();
3049 auto *US = ConstantAsMetadata::get(
3050 C: ConstantInt::get(Ty: B.getInt32Ty(), V: SPIRV::Decoration::UserSemantic));
3051
3052 SmallVector<Metadata *> MDs;
3053 if (I->hasMetadata(Kind: "amdgpu.no.fine.grained.memory"))
3054 MDs.push_back(Elt: MDNode::get(
3055 Context&: Ctx, MDs: {US, MDString::get(Context&: Ctx, Str: "amdgpu.no.fine.grained.memory")}));
3056 if (I->hasMetadata(Kind: "amdgpu.no.remote.memory"))
3057 MDs.push_back(Elt: MDNode::get(
3058 Context&: Ctx, MDs: {US, MDString::get(Context&: Ctx, Str: "amdgpu.no.remote.memory")}));
3059 if (I->hasMetadata(Kind: "amdgpu.ignore.denormal.mode"))
3060 MDs.push_back(Elt: MDNode::get(
3061 Context&: Ctx, MDs: {US, MDString::get(Context&: Ctx, Str: "amdgpu.ignore.denormal.mode")}));
3062 if (!MDs.empty())
3063 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
3064 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs))});
3065 }
3066}
3067
3068static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
3069 const Module &M,
3070 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
3071 &FPFastMathDefaultInfoMap,
3072 Function *F) {
3073 auto it = FPFastMathDefaultInfoMap.find(Val: F);
3074 if (it != FPFastMathDefaultInfoMap.end())
3075 return it->second;
3076
3077 // If the map does not contain the entry, create a new one. Initialize it to
3078 // contain all 3 elements sorted by bit width of target type: {half, float,
3079 // double}.
3080 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
3081 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
3082 Args: SPIRV::FPFastMathMode::None);
3083 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
3084 Args: SPIRV::FPFastMathMode::None);
3085 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
3086 Args: SPIRV::FPFastMathMode::None);
3087 return FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
3088}
3089
3090static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
3091 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
3092 const Type *Ty) {
3093 size_t BitWidth = Ty->getScalarSizeInBits();
3094 int Index =
3095 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
3096 BitWidth);
3097 assert(Index >= 0 && Index < 3 &&
3098 "Expected FPFastMathDefaultInfo for half, float, or double");
3099 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3100 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3101 return FPFastMathDefaultInfoVec[Index];
3102}
3103
3104void SPIRVEmitIntrinsicsImpl::insertConstantsForFPFastMathDefault(Module &M) {
3105 const SPIRVSubtarget *ST = TM.getSubtargetImpl();
3106 if (!ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
3107 return;
3108
3109 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
3110 // We need the entry point (function) as the key, and the target
3111 // type and flags as the value.
3112 // We also need to check ContractionOff and SignedZeroInfNanPreserve
3113 // execution modes, as they are now deprecated and must be replaced
3114 // with FPFastMathDefaultInfo.
3115 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
3116 if (!Node) {
3117 if (!M.getNamedMetadata(Name: "opencl.enable.FP_CONTRACT")) {
3118 // This requires emitting ContractionOff. However, because
3119 // ContractionOff is now deprecated, we need to replace it with
3120 // FPFastMathDefaultInfo with FP Fast Math Mode bitmask set to all 0.
3121 // We need to create the constant for that.
3122
3123 // Create constant instruction with the bitmask flags.
3124 Constant *InitValue =
3125 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: 0);
3126 // TODO: Reuse constant if there is one already with the required
3127 // value.
3128 [[maybe_unused]] GlobalVariable *GV =
3129 new GlobalVariable(M, // Module
3130 Type::getInt32Ty(C&: M.getContext()), // Type
3131 true, // isConstant
3132 GlobalValue::InternalLinkage, // Linkage
3133 InitValue // Initializer
3134 );
3135 }
3136 return;
3137 }
3138
3139 // The table maps function pointers to their default FP fast math info. It
3140 // can be assumed that the SmallVector is sorted by the bit width of the
3141 // type. The first element is the smallest bit width, and the last element
3142 // is the largest bit width, therefore, we will have {half, float, double}
3143 // in the order of their bit widths.
3144 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
3145 FPFastMathDefaultInfoMap;
3146
3147 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
3148 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
3149 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
3150 Function *F = cast<Function>(
3151 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
3152 const auto EM =
3153 cast<ConstantInt>(
3154 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
3155 ->getZExtValue();
3156 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
3157 assert(MDN->getNumOperands() == 4 &&
3158 "Expected 4 operands for FPFastMathDefault");
3159 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
3160 unsigned Flags =
3161 cast<ConstantInt>(
3162 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
3163 ->getZExtValue();
3164 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3165 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3166 SPIRV::FPFastMathDefaultInfo &Info =
3167 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
3168 Info.FastMathFlags = Flags;
3169 Info.FPFastMathDefault = true;
3170 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
3171 assert(MDN->getNumOperands() == 2 &&
3172 "Expected no operands for ContractionOff");
3173
3174 // We need to save this info for every possible FP type, i.e. {half,
3175 // float, double, fp128}.
3176 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3177 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3178 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3179 Info.ContractionOff = true;
3180 }
3181 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
3182 assert(MDN->getNumOperands() == 3 &&
3183 "Expected 1 operand for SignedZeroInfNanPreserve");
3184 unsigned TargetWidth =
3185 cast<ConstantInt>(
3186 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
3187 ->getZExtValue();
3188 // We need to save this info only for the FP type with TargetWidth.
3189 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3190 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3191 int Index = SPIRV::FPFastMathDefaultInfoVector::
3192 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
3193 assert(Index >= 0 && Index < 3 &&
3194 "Expected FPFastMathDefaultInfo for half, float, or double");
3195 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3196 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3197 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
3198 }
3199 }
3200
3201 DenseMap<unsigned, GlobalVariable *> GlobalVars;
3202 for (auto &[Func, FPFastMathDefaultInfoVec] : FPFastMathDefaultInfoMap) {
3203 if (FPFastMathDefaultInfoVec.empty())
3204 continue;
3205
3206 for (const SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3207 assert(Info.Ty && "Expected target type for FPFastMathDefaultInfo");
3208 // Skip if none of the execution modes was used.
3209 unsigned Flags = Info.FastMathFlags;
3210 if (Flags == SPIRV::FPFastMathMode::None && !Info.ContractionOff &&
3211 !Info.SignedZeroInfNanPreserve && !Info.FPFastMathDefault)
3212 continue;
3213
3214 // Check if flags are compatible.
3215 if (Info.ContractionOff && (Flags & SPIRV::FPFastMathMode::AllowContract))
3216 report_fatal_error(reason: "Conflicting FPFastMathFlags: ContractionOff "
3217 "and AllowContract");
3218
3219 if (Info.SignedZeroInfNanPreserve &&
3220 !(Flags &
3221 (SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
3222 SPIRV::FPFastMathMode::NSZ))) {
3223 if (Info.FPFastMathDefault)
3224 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
3225 "SignedZeroInfNanPreserve but at least one of "
3226 "NotNaN/NotInf/NSZ is enabled.");
3227 }
3228
3229 if ((Flags & SPIRV::FPFastMathMode::AllowTransform) &&
3230 !((Flags & SPIRV::FPFastMathMode::AllowReassoc) &&
3231 (Flags & SPIRV::FPFastMathMode::AllowContract))) {
3232 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
3233 "AllowTransform requires AllowReassoc and "
3234 "AllowContract to be set.");
3235 }
3236
3237 auto it = GlobalVars.find(Val: Flags);
3238 GlobalVariable *GV = nullptr;
3239 if (it != GlobalVars.end()) {
3240 // Reuse existing global variable.
3241 GV = it->second;
3242 } else {
3243 // Create constant instruction with the bitmask flags.
3244 Constant *InitValue =
3245 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: Flags);
3246 // TODO: Reuse constant if there is one already with the required
3247 // value.
3248 GV = new GlobalVariable(M, // Module
3249 Type::getInt32Ty(C&: M.getContext()), // Type
3250 true, // isConstant
3251 GlobalValue::InternalLinkage, // Linkage
3252 InitValue // Initializer
3253 );
3254 GlobalVars[Flags] = GV;
3255 }
3256 }
3257 }
3258}
3259
3260void SPIRVEmitIntrinsicsImpl::processInstrAfterVisit(Instruction *I,
3261 IRBuilder<> &B) {
3262 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3263 bool IsConstComposite =
3264 II && II->getIntrinsicID() == Intrinsic::spv_const_composite;
3265 if (IsConstComposite && TrackConstants) {
3266 setInsertPointAfterDef(B, I);
3267 auto t = AggrConsts.find(Val: I);
3268 assert(t != AggrConsts.end());
3269 auto *NewOp =
3270 buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
3271 Types: {II->getType(), II->getType()}, Arg: t->second, Arg2: I, Imms: {}, B);
3272 replaceAllUsesWith(Src: I, Dest: NewOp, DeleteOld: false);
3273 NewOp->setArgOperand(i: 0, v: I);
3274 }
3275 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
3276 for (const auto &Op : I->operands()) {
3277 if (isa<PHINode>(Val: I) || isa<SwitchInst>(Val: I) ||
3278 !(isa<ConstantData>(Val: Op) || isa<ConstantExpr>(Val: Op)))
3279 continue;
3280 unsigned OpNo = Op.getOperandNo();
3281 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
3282 (!II->isBundleOperand(Idx: OpNo) &&
3283 II->paramHasAttr(ArgNo: OpNo, Kind: Attribute::ImmArg))))
3284 continue;
3285
3286 if (!BPrepared) {
3287 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
3288 : B.SetInsertPoint(I);
3289 BPrepared = true;
3290 }
3291 Type *OpTy = Op->getType();
3292 Type *OpElemTy = GR->findDeducedElementType(Val: Op);
3293 Value *NewOp = Op;
3294 if (OpTy->isTargetExtTy()) {
3295 // Since this value is replaced by poison, we need to do the same in
3296 // `insertAssignTypeIntrs`.
3297 Value *OpTyVal = getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank);
3298 NewOp = buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
3299 Types: {OpTy, OpTyVal->getType()}, Arg: Op, Arg2: OpTyVal, Imms: {}, B);
3300 }
3301 if (!IsConstComposite && isPointerTy(T: OpTy) && OpElemTy != nullptr &&
3302 OpElemTy != IntegerType::getInt8Ty(C&: I->getContext())) {
3303 SmallVector<Type *, 2> Types = {OpTy, OpTy};
3304 SmallVector<Value *, 2> Args = {
3305 NewOp,
3306 buildMD(Arg: getNormalizedPoisonValue(Ty: OpElemTy, CanUseAnyVectorRank)),
3307 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
3308 CallInst *PtrCasted = B.CreateIntrinsicWithoutFolding(
3309 ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
3310 GR->buildAssignPtr(B, ElemTy: OpElemTy, Arg: PtrCasted);
3311 NewOp = PtrCasted;
3312 }
3313 if (NewOp != Op)
3314 I->setOperand(i: OpNo, Val: NewOp);
3315 }
3316 if (Named.insert(Ptr: I).second)
3317 emitAssignName(I, B);
3318}
3319
3320Type *SPIRVEmitIntrinsicsImpl::deduceFunParamElementType(Function *F,
3321 unsigned OpIdx) {
3322 SmallPtrSet<Function *, 0> FVisited;
3323 return deduceFunParamElementType(F, OpIdx, FVisited);
3324}
3325
3326Type *SPIRVEmitIntrinsicsImpl::deduceFunParamElementType(
3327 Function *F, unsigned OpIdx, SmallPtrSetImpl<Function *> &FVisited) {
3328 // maybe a cycle
3329 if (!FVisited.insert(Ptr: F).second)
3330 return nullptr;
3331
3332 SmallPtrSet<Value *, 0> Visited;
3333 SmallVector<std::pair<Function *, unsigned>> Lookup;
3334 // search in function's call sites
3335 for (User *U : F->users()) {
3336 CallInst *CI = dyn_cast<CallInst>(Val: U);
3337 if (!CI || OpIdx >= CI->arg_size())
3338 continue;
3339 Value *OpArg = CI->getArgOperand(i: OpIdx);
3340 if (!isPointerTy(T: OpArg->getType()))
3341 continue;
3342 // maybe we already know operand's element type
3343 if (Type *KnownTy = GR->findDeducedElementType(Val: OpArg))
3344 return KnownTy;
3345 // try to deduce from the operand itself
3346 Visited.clear();
3347 if (Type *Ty = deduceElementTypeHelper(I: OpArg, Visited, UnknownElemTypeI8: false))
3348 return Ty;
3349 // search in actual parameter's users
3350 for (User *OpU : OpArg->users()) {
3351 Instruction *Inst = dyn_cast<Instruction>(Val: OpU);
3352 if (!Inst || Inst == CI)
3353 continue;
3354 Visited.clear();
3355 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8: false))
3356 return Ty;
3357 }
3358 // check if it's a formal parameter of the outer function
3359 if (!CI->getParent() || !CI->getParent()->getParent())
3360 continue;
3361 Function *OuterF = CI->getParent()->getParent();
3362 if (FVisited.find(Ptr: OuterF) != FVisited.end())
3363 continue;
3364 for (unsigned i = 0; i < OuterF->arg_size(); ++i) {
3365 if (OuterF->getArg(i) == OpArg) {
3366 Lookup.push_back(Elt: std::make_pair(x&: OuterF, y&: i));
3367 break;
3368 }
3369 }
3370 }
3371
3372 // search in function parameters
3373 for (auto &Pair : Lookup) {
3374 if (Type *Ty = deduceFunParamElementType(F: Pair.first, OpIdx: Pair.second, FVisited))
3375 return Ty;
3376 }
3377
3378 return nullptr;
3379}
3380
3381void SPIRVEmitIntrinsicsImpl::processParamTypesByFunHeader(Function *F,
3382 IRBuilder<> &B) {
3383 B.SetInsertPointPastAllocas(F);
3384 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
3385 Argument *Arg = F->getArg(i: OpIdx);
3386 // Vector-of-pointers arg: deduce pointee from a GEP user so the function
3387 // type isn't emitted with the default i8 pointee.
3388 if (isUntypedPointerVectorTy(T: Arg->getType()) &&
3389 !GR->findDeducedElementType(Val: Arg)) {
3390 for (User *U : Arg->users()) {
3391 auto *GEP = dyn_cast<GetElementPtrInst>(Val: U);
3392 if (GEP && GEP->getPointerOperand() == Arg) {
3393 GR->buildAssignPtr(B, ElemTy: GEP->getSourceElementType(), Arg);
3394 break;
3395 }
3396 }
3397 continue;
3398 }
3399 if (!isUntypedPointerTy(T: Arg->getType()))
3400 continue;
3401 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
3402 if (ElemTy)
3403 continue;
3404 if (hasPointeeTypeAttr(Arg) &&
3405 (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) {
3406 GR->buildAssignPtr(B, ElemTy, Arg);
3407 continue;
3408 }
3409 // search in function's call sites
3410 for (User *U : F->users()) {
3411 CallInst *CI = dyn_cast<CallInst>(Val: U);
3412 if (!CI || OpIdx >= CI->arg_size())
3413 continue;
3414 Value *OpArg = CI->getArgOperand(i: OpIdx);
3415 if (!isPointerTy(T: OpArg->getType()))
3416 continue;
3417 // maybe we already know operand's element type
3418 if ((ElemTy = GR->findDeducedElementType(Val: OpArg)) != nullptr)
3419 break;
3420 }
3421 if (ElemTy) {
3422 GR->buildAssignPtr(B, ElemTy, Arg);
3423 continue;
3424 }
3425 if (HaveFunPtrs) {
3426 for (User *U : Arg->users()) {
3427 CallInst *CI = dyn_cast<CallInst>(Val: U);
3428 if (CI && !isa<IntrinsicInst>(Val: CI) && CI->isIndirectCall() &&
3429 CI->getCalledOperand() == Arg &&
3430 CI->getParent()->getParent() == CurrF) {
3431 SmallVector<std::pair<Value *, unsigned>> Ops;
3432 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy&: ElemTy, IsPostprocessing: false);
3433 if (ElemTy) {
3434 GR->buildAssignPtr(B, ElemTy, Arg);
3435 break;
3436 }
3437 }
3438 }
3439 }
3440 }
3441}
3442
3443void SPIRVEmitIntrinsicsImpl::processParamTypes(Function *F, IRBuilder<> &B) {
3444 B.SetInsertPointPastAllocas(F);
3445 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
3446 Argument *Arg = F->getArg(i: OpIdx);
3447 if (!isUntypedPointerTy(T: Arg->getType()))
3448 continue;
3449 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
3450 if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) {
3451 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Arg)) {
3452 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3453 GR->updateAssignType(
3454 AssignCI, Arg,
3455 OfType: getNormalizedPoisonValue(Ty: ElemTy, CanUseAnyVectorRank));
3456 propagateElemType(Op: Arg, ElemTy: IntegerType::getInt8Ty(C&: F->getContext()),
3457 VisitedSubst);
3458 } else {
3459 GR->buildAssignPtr(B, ElemTy, Arg);
3460 }
3461 }
3462 }
3463}
3464
3465static FunctionType *getFunctionPointerElemType(Function *F,
3466 SPIRVGlobalRegistry *GR) {
3467 FunctionType *FTy = F->getFunctionType();
3468 bool IsNewFTy = false;
3469 SmallVector<Type *, 4> ArgTys;
3470 for (Argument &Arg : F->args()) {
3471 Type *ArgTy = Arg.getType();
3472 if (ArgTy->isPointerTy())
3473 if (Type *ElemTy = GR->findDeducedElementType(Val: &Arg)) {
3474 IsNewFTy = true;
3475 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
3476 }
3477 ArgTys.push_back(Elt: ArgTy);
3478 }
3479 return IsNewFTy
3480 ? FunctionType::get(Result: FTy->getReturnType(), Params: ArgTys, isVarArg: FTy->isVarArg())
3481 : FTy;
3482}
3483
3484bool SPIRVEmitIntrinsicsImpl::processFunctionPointers(Module &M) {
3485 SmallVector<Function *> Worklist;
3486 for (auto &F : M) {
3487 if (F.isIntrinsic())
3488 continue;
3489 if (F.isDeclaration()) {
3490 for (User *U : F.users()) {
3491 CallInst *CI = dyn_cast<CallInst>(Val: U);
3492 if (!CI || CI->getCalledFunction() != &F) {
3493 Worklist.push_back(Elt: &F);
3494 break;
3495 }
3496 }
3497 } else {
3498 if (F.user_empty())
3499 continue;
3500 Type *FPElemTy = GR->findDeducedElementType(Val: &F);
3501 if (!FPElemTy)
3502 FPElemTy = getFunctionPointerElemType(F: &F, GR);
3503 for (User *U : F.users()) {
3504 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
3505 if (!II || II->arg_size() != 3 || II->getOperand(i_nocapture: 0) != &F)
3506 continue;
3507 if (II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type ||
3508 II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
3509 GR->updateAssignType(
3510 AssignCI: II, Arg: &F, OfType: getNormalizedPoisonValue(Ty: FPElemTy, CanUseAnyVectorRank));
3511 break;
3512 }
3513 }
3514 }
3515 }
3516 if (Worklist.empty())
3517 return false;
3518
3519 LLVMContext &Ctx = M.getContext();
3520 Function *SF = getOrCreateBackendServiceFunction(M);
3521 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: SF);
3522 IRBuilder<> IRB(BB);
3523
3524 for (Function *F : Worklist) {
3525 SmallVector<Value *> Args;
3526 for (const auto &Arg : F->args())
3527 Args.push_back(
3528 Elt: getNormalizedPoisonValue(Ty: Arg.getType(), CanUseAnyVectorRank));
3529 IRB.CreateCall(Callee: F, Args);
3530 }
3531 IRB.CreateRetVoid();
3532
3533 return true;
3534}
3535
3536// Apply types parsed from demangled function declarations.
3537void SPIRVEmitIntrinsicsImpl::applyDemangledPtrArgTypes(IRBuilder<> &B) {
3538 DenseMap<Function *, CallInst *> Ptrcasts;
3539 for (auto It : FDeclPtrTys) {
3540 Function *F = It.first;
3541 for (auto *U : F->users()) {
3542 CallInst *CI = dyn_cast<CallInst>(Val: U);
3543 if (!CI || CI->getCalledFunction() != F)
3544 continue;
3545 unsigned Sz = CI->arg_size();
3546 for (auto [Idx, ElemTy] : It.second) {
3547 if (Idx >= Sz)
3548 continue;
3549 Value *Param = CI->getArgOperand(i: Idx);
3550 if (GR->findDeducedElementType(Val: Param) || isa<GlobalValue>(Val: Param))
3551 continue;
3552 if (Argument *Arg = dyn_cast<Argument>(Val: Param)) {
3553 if (!hasPointeeTypeAttr(Arg)) {
3554 B.SetInsertPointPastAllocas(Arg->getParent());
3555 B.SetCurrentDebugLocation(DebugLoc());
3556 GR->buildAssignPtr(B, ElemTy, Arg);
3557 }
3558 } else if (isaGEP(V: Param)) {
3559 replaceUsesOfWithSpvPtrcast(
3560 Op: Param, ElemTy: normalizeType(Ty: ElemTy, CanUseAnyVectorRank), I: CI, Ptrcasts);
3561 } else if (isa<Instruction>(Val: Param)) {
3562 GR->addDeducedElementType(Val: Param,
3563 Ty: normalizeType(Ty: ElemTy, CanUseAnyVectorRank));
3564 // insertAssignTypeIntrs() will complete buildAssignPtr()
3565 } else {
3566 B.SetInsertPoint(CI->getParent()
3567 ->getParent()
3568 ->getEntryBlock()
3569 .getFirstNonPHIOrDbgOrAlloca());
3570 GR->buildAssignPtr(B, ElemTy, Arg: Param);
3571 }
3572 CallInst *Ref = dyn_cast<CallInst>(Val: Param);
3573 if (!Ref)
3574 continue;
3575 Function *RefF = Ref->getCalledFunction();
3576 if (!RefF || !isPointerTy(T: RefF->getReturnType()) ||
3577 GR->findDeducedElementType(Val: RefF))
3578 continue;
3579 ElemTy = normalizeType(Ty: ElemTy, CanUseAnyVectorRank);
3580 GR->addDeducedElementType(Val: RefF, Ty: ElemTy);
3581 GR->addReturnType(
3582 ArgF: RefF, DerivedTy: TypedPointerType::get(
3583 ElementType: ElemTy, AddressSpace: getPointerAddressSpace(T: RefF->getReturnType())));
3584 }
3585 }
3586 }
3587}
3588
3589GetElementPtrInst *SPIRVEmitIntrinsicsImpl::simplifyZeroLengthArrayGepInst(
3590 GetElementPtrInst *GEP) {
3591 // getelementptr [0 x T], P, 0 (zero), I -> getelementptr T, P, I.
3592 // If type is 0-length array and first index is 0 (zero), drop both the
3593 // 0-length array type and the first index. This is a common pattern in
3594 // the IR, e.g. when using a zero-length array as a placeholder for a
3595 // flexible array such as unbound arrays.
3596 assert(GEP && "GEP is null");
3597 Type *SrcTy = GEP->getSourceElementType();
3598 SmallVector<Value *, 8> Indices(GEP->indices());
3599 ArrayType *ArrTy = dyn_cast<ArrayType>(Val: SrcTy);
3600 if (ArrTy && ArrTy->getNumElements() == 0 && match(V: Indices[0], P: m_Zero())) {
3601 Indices.erase(CI: Indices.begin());
3602 SrcTy = ArrTy->getElementType();
3603 return GetElementPtrInst::Create(PointeeType: SrcTy, Ptr: GEP->getPointerOperand(), IdxList: Indices,
3604 NW: GEP->getNoWrapFlags(), NameStr: "",
3605 InsertBefore: GEP->getIterator());
3606 }
3607 return nullptr;
3608}
3609
3610void SPIRVEmitIntrinsicsImpl::emitUnstructuredLoopControls(Function &F,
3611 IRBuilder<> &B) {
3612 const SPIRVSubtarget *ST = TM.getSubtargetImpl(F);
3613 // Shaders use SPIRVStructurizer which emits OpLoopMerge via spv_loop_merge.
3614 if (ST->isShader())
3615 return;
3616
3617 if (ST->canUseExtension(
3618 E: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls)) {
3619 for (BasicBlock &BB : F) {
3620 Instruction *Term = BB.getTerminator();
3621 MDNode *LoopMD = Term->getMetadata(KindID: LLVMContext::MD_loop);
3622 if (!LoopMD)
3623 continue;
3624
3625 SmallVector<unsigned, 1> Ops =
3626 getSpirvLoopControlOperandsFromLoopMetadata(LoopMD);
3627 unsigned LC = Ops[0];
3628 if (LC == SPIRV::LoopControl::None)
3629 continue;
3630
3631 // Emit intrinsic: loop control mask + optional parameters.
3632 B.SetInsertPoint(Term);
3633 SmallVector<Value *, 4> IntrArgs;
3634 for (unsigned Op : Ops)
3635 IntrArgs.push_back(Elt: B.getInt32(C: Op));
3636 B.CreateIntrinsic(ID: Intrinsic::spv_loop_control_intel, Args: IntrArgs);
3637 }
3638 return;
3639 }
3640
3641 // For non-shader targets without the Intel extension, emit OpLoopMerge
3642 // using spv_loop_merge intrinsics, mirroring the structurizer approach.
3643 LoopInfo LI;
3644 LI.analyze(F: &F);
3645 if (LI.empty())
3646 return;
3647
3648 for (Loop *L : LI.getLoopsInPreorder()) {
3649 BasicBlock *Latch = L->getLoopLatch();
3650 if (!Latch)
3651 continue;
3652 BasicBlock *MergeBlock = L->getUniqueExitBlock();
3653 if (!MergeBlock)
3654 continue;
3655
3656 // Check for loop unroll metadata on the latch terminator.
3657 SmallVector<unsigned, 1> LoopControlOps =
3658 getSpirvLoopControlOperandsFromLoopMetadata(L);
3659 if (LoopControlOps[0] == SPIRV::LoopControl::None)
3660 continue;
3661
3662 BasicBlock *Header = L->getHeader();
3663 B.SetInsertPoint(Header->getTerminator());
3664 auto *MergeAddress = BlockAddress::get(F: &F, BB: MergeBlock);
3665 auto *ContinueAddress = BlockAddress::get(F: &F, BB: Latch);
3666 SmallVector<Value *, 4> Args = {MergeAddress, ContinueAddress};
3667 for (unsigned Imm : LoopControlOps)
3668 Args.emplace_back(Args: B.getInt32(C: Imm));
3669 B.CreateIntrinsic(ID: Intrinsic::spv_loop_merge, Args: {Args});
3670 }
3671}
3672
3673bool SPIRVEmitIntrinsicsImpl::runOnFunction(Function &Func) {
3674 if (Func.isDeclaration())
3675 return false;
3676
3677 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: Func);
3678 GR = ST.getSPIRVGlobalRegistry();
3679
3680 if (!CurrF)
3681 HaveFunPtrs =
3682 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers);
3683
3684 CanUseAnyVectorRank =
3685 ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector);
3686 CurrF = &Func;
3687 IRBuilder<> B(Func.getContext());
3688 AggrConsts.clear();
3689 AggrConstTypes.clear();
3690 AggrStores.clear();
3691
3692 processParamTypesByFunHeader(F: CurrF, B);
3693
3694 // Fix GEP result types ahead of inference, and simplify if possible.
3695 // Data structure for dead instructions that were simplified and replaced.
3696 SmallPtrSet<Instruction *, 4> DeadInsts;
3697 for (auto &I : instructions(F&: Func)) {
3698 if (StoreInst *SI = dyn_cast<StoreInst>(Val: &I)) {
3699 Type *ElTy = SI->getValueOperand()->getType();
3700 if (ElTy->isAggregateType() || ElTy->isVectorTy())
3701 AggrStores.insert(Ptr: &I);
3702 continue;
3703 }
3704
3705 auto *GEP = dyn_cast<GetElementPtrInst>(Val: &I);
3706 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
3707
3708 if ((!GEP && !SGEP) || GR->findDeducedElementType(Val: &I))
3709 continue;
3710
3711 if (SGEP) {
3712 GR->addDeducedElementType(
3713 Val: SGEP,
3714 Ty: normalizeType(Ty: SGEP->getResultElementType(), CanUseAnyVectorRank));
3715 continue;
3716 }
3717
3718 GetElementPtrInst *NewGEP = simplifyZeroLengthArrayGepInst(GEP);
3719 if (NewGEP) {
3720 GEP->replaceAllUsesWith(V: NewGEP);
3721 DeadInsts.insert(Ptr: GEP);
3722 GEP = NewGEP;
3723 }
3724 if (Type *GepTy = getGEPType(Ref: GEP))
3725 GR->addDeducedElementType(Val: GEP, Ty: normalizeType(Ty: GepTy, CanUseAnyVectorRank));
3726 }
3727 // Remove dead instructions that were simplified and replaced.
3728 for (auto *I : DeadInsts) {
3729 assert(I->use_empty() && "Dead instruction should not have any uses left");
3730 I->eraseFromParent();
3731 }
3732
3733 B.SetInsertPoint(TheBB: &Func.getEntryBlock(), IP: Func.getEntryBlock().begin());
3734 for (auto &GV : Func.getParent()->globals())
3735 processGlobalValue(GV, B);
3736
3737 reconstructAggregateReturns(Func, B);
3738 preprocessUndefsAndPoisons(B);
3739 simplifyNullAddrSpaceCasts();
3740 preprocessCompositeConstants(B);
3741
3742 // A PHINode, SelectInst or FreezeInst takes its result type from its
3743 // operands. Aggregate arms are lowered to i32 value-ids (composite constants
3744 // here, loads and other producers during the visitor pass below), so mutate
3745 // an aggregate PHI, select or freeze to match. The original type is tracked
3746 // in AggrConstTypes (used to assign the SPIR-V type) and its extractvalue
3747 // users are lowered to spv_extractv.
3748 Type *I32Ty = B.getInt32Ty();
3749 for (Instruction &I : instructions(F&: Func)) {
3750 if (!isAggregateValueIdInstr(I))
3751 continue;
3752 // Give multi-register arms a value-id first, before the result is mutated.
3753 insertCompositeAggregateArms(I: &I, B);
3754 AggrConstTypes[&I] = I.getType();
3755 I.mutateType(Ty: I32Ty);
3756 }
3757
3758 preprocessBoolVectorBitcasts(F&: Func);
3759 SmallVector<Instruction *> Worklist(
3760 llvm::make_pointer_range(Range: instructions(F&: Func)));
3761
3762 applyDemangledPtrArgTypes(B);
3763
3764 // Pass forward: use operand to deduce instructions result.
3765 for (auto &I : Worklist) {
3766 // Don't emit intrinsincs for convergence intrinsics.
3767 if (isConvergenceIntrinsic(I))
3768 continue;
3769
3770 bool Postpone = insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: false);
3771 // if Postpone is true, we can't decide on pointee type yet
3772 insertAssignTypeIntrs(I, B);
3773 insertPtrCastOrAssignTypeInstr(I, B);
3774 insertSpirvDecorations(I, B);
3775 // if instruction requires a pointee type set, let's check if we know it
3776 // already, and force it to be i8 if not
3777 if (Postpone && !GR->findAssignPtrTypeInstr(Val: I))
3778 insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: true);
3779
3780 if (auto *FPI = dyn_cast<ConstrainedFPIntrinsic>(Val: I))
3781 useRoundingMode(FPI, B);
3782 }
3783
3784 // Pass backward: use instructions results to specify/update/cast operands
3785 // where needed.
3786 SmallPtrSet<Instruction *, 4> IncompleteRets;
3787 for (auto &I : llvm::reverse(C: instructions(F&: Func)))
3788 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets);
3789
3790 // Pass forward for PHIs only, their operands are not preceed the
3791 // instruction in meaning of `instructions(Func)`.
3792 for (BasicBlock &BB : Func)
3793 for (PHINode &Phi : BB.phis())
3794 if (isPointerTy(T: Phi.getType()))
3795 deduceOperandElementType(I: &Phi, IncompleteRets: nullptr);
3796
3797 for (auto *I : Worklist) {
3798 TrackConstants = true;
3799 if (!I->getType()->isVoidTy() || isa<StoreInst>(Val: I))
3800 setInsertPointAfterDef(B, I);
3801 // Visitors return either the original/newly created instruction for
3802 // further processing, nullptr otherwise.
3803 I = visit(I&: *I);
3804 if (!I)
3805 continue;
3806
3807 // Don't emit intrinsics for convergence operations.
3808 if (isConvergenceIntrinsic(I))
3809 continue;
3810
3811 addSaturatedDecorationToIntrinsic(I, B);
3812 processInstrAfterVisit(I, B);
3813 }
3814
3815 emitUnstructuredLoopControls(F&: Func, B);
3816
3817 return true;
3818}
3819
3820// Try to deduce a better type for pointers to untyped ptr.
3821bool SPIRVEmitIntrinsicsImpl::postprocessTypes(Module &M) {
3822 if (!GR || TodoTypeSz == 0)
3823 return false;
3824
3825 unsigned SzTodo = TodoTypeSz;
3826 DenseMap<Value *, SmallPtrSet<Value *, 4>> ToProcess;
3827 for (auto [Op, Enabled] : TodoType) {
3828 // TODO: add isa<CallInst>(Op) to continue
3829 if (!Enabled || isaGEP(V: Op))
3830 continue;
3831 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Op);
3832 Type *KnownTy = GR->findDeducedElementType(Val: Op);
3833 if (!KnownTy || !AssignCI)
3834 continue;
3835 assert(Op == AssignCI->getArgOperand(0));
3836 // Try to improve the type deduced after all Functions are processed.
3837 if (auto *CI = dyn_cast<Instruction>(Val: Op)) {
3838 CurrF = CI->getParent()->getParent();
3839 SmallPtrSet<Value *, 0> Visited;
3840 if (Type *ElemTy = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8: false, IgnoreKnownType: true)) {
3841 if (ElemTy != KnownTy) {
3842 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3843 propagateElemType(Op: CI, ElemTy, VisitedSubst);
3844 eraseTodoType(Op);
3845 continue;
3846 }
3847 }
3848 }
3849
3850 if (Op->hasUseList()) {
3851 for (User *U : Op->users()) {
3852 Instruction *Inst = dyn_cast<Instruction>(Val: U);
3853 if (Inst && !isa<IntrinsicInst>(Val: Inst))
3854 ToProcess[Inst].insert(Ptr: Op);
3855 }
3856 }
3857 }
3858 if (TodoTypeSz == 0)
3859 return true;
3860
3861 for (auto &F : M) {
3862 CurrF = &F;
3863 SmallPtrSet<Instruction *, 4> IncompleteRets;
3864 for (auto &I : llvm::reverse(C: instructions(F))) {
3865 auto It = ToProcess.find(Val: &I);
3866 if (It == ToProcess.end())
3867 continue;
3868 It->second.remove_if(P: [this](Value *V) { return !isTodoType(Op: V); });
3869 if (It->second.size() == 0)
3870 continue;
3871 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets, AskOps: &It->second, IsPostprocessing: true);
3872 if (TodoTypeSz == 0)
3873 return true;
3874 }
3875 }
3876
3877 return SzTodo > TodoTypeSz;
3878}
3879
3880// Parse and store argument types of function declarations where needed.
3881void SPIRVEmitIntrinsicsImpl::parseFunDeclarations(Module &M) {
3882 for (auto &F : M) {
3883 if (!F.isDeclaration() || F.isIntrinsic())
3884 continue;
3885 // get the demangled name
3886 std::string DemangledName = getOclOrSpirvBuiltinDemangledName(Name: F.getName());
3887 if (DemangledName.empty())
3888 continue;
3889 // allow only OpGroupAsyncCopy use case at the moment
3890 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F);
3891 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
3892 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
3893 if (Opcode != SPIRV::OpGroupAsyncCopy)
3894 continue;
3895 // find pointer arguments
3896 SmallVector<unsigned> Idxs;
3897 for (unsigned OpIdx = 0; OpIdx < F.arg_size(); ++OpIdx) {
3898 Argument *Arg = F.getArg(i: OpIdx);
3899 if (isPointerTy(T: Arg->getType()) && !hasPointeeTypeAttr(Arg))
3900 Idxs.push_back(Elt: OpIdx);
3901 }
3902 if (!Idxs.size())
3903 continue;
3904 // parse function arguments
3905 LLVMContext &Ctx = F.getContext();
3906 SmallVector<StringRef, 10> TypeStrs;
3907 SPIRV::parseBuiltinTypeStr(BuiltinArgsTypeStrs&: TypeStrs, DemangledCall: DemangledName, Ctx);
3908 if (!TypeStrs.size())
3909 continue;
3910 // find type info for pointer arguments
3911 for (unsigned Idx : Idxs) {
3912 if (Idx >= TypeStrs.size())
3913 continue;
3914 if (Type *ElemTy =
3915 SPIRV::parseBuiltinCallArgumentType(TypeStr: TypeStrs[Idx].trim(), Ctx))
3916 if (TypedPointerType::isValidElementType(ElemTy) &&
3917 !ElemTy->isTargetExtTy())
3918 FDeclPtrTys[&F].push_back(Elt: std::make_pair(x&: Idx, y&: ElemTy));
3919 }
3920 }
3921}
3922
3923bool SPIRVEmitIntrinsicsImpl::processMaskedMemIntrinsic(IntrinsicInst &I) {
3924 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
3925
3926 if (I.getIntrinsicID() == Intrinsic::masked_gather) {
3927 if (!ST.canUseExtension(
3928 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
3929 I.getContext().emitError(
3930 I: &I, ErrorStr: "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter "
3931 "extension");
3932 // Replace with poison to allow compilation to continue and report error.
3933 I.replaceAllUsesWith(V: PoisonValue::get(T: I.getType()));
3934 I.eraseFromParent();
3935 return true;
3936 }
3937
3938 IRBuilder<> B(&I);
3939
3940 Value *Ptrs = I.getArgOperand(i: 0);
3941 Value *Mask = I.getArgOperand(i: 1);
3942 Value *Passthru = I.getArgOperand(i: 2);
3943
3944 // Alignment is stored as a parameter attribute, not as a regular parameter.
3945 uint32_t Alignment = I.getParamAlign(ArgNo: 0).valueOrOne().value();
3946
3947 SmallVector<Value *, 4> Args = {Ptrs, B.getInt32(C: Alignment), Mask,
3948 Passthru};
3949 SmallVector<Type *, 4> Types = {I.getType(), Ptrs->getType(),
3950 Mask->getType(), Passthru->getType()};
3951
3952 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_masked_gather, OverloadTypes: Types, Args);
3953 I.replaceAllUsesWith(V: NewI);
3954 I.eraseFromParent();
3955 return true;
3956 }
3957
3958 if (I.getIntrinsicID() == Intrinsic::masked_scatter) {
3959 if (!ST.canUseExtension(
3960 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
3961 I.getContext().emitError(
3962 I: &I, ErrorStr: "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter "
3963 "extension");
3964 // Erase the intrinsic to allow compilation to continue and report error.
3965 I.eraseFromParent();
3966 return true;
3967 }
3968
3969 IRBuilder<> B(&I);
3970
3971 Value *Values = I.getArgOperand(i: 0);
3972 Value *Ptrs = I.getArgOperand(i: 1);
3973 Value *Mask = I.getArgOperand(i: 2);
3974
3975 // Alignment is stored as a parameter attribute on the ptrs parameter (arg
3976 // 1).
3977 uint32_t Alignment = I.getParamAlign(ArgNo: 1).valueOrOne().value();
3978
3979 SmallVector<Value *, 4> Args = {Values, Ptrs, B.getInt32(C: Alignment), Mask};
3980 SmallVector<Type *, 3> Types = {Values->getType(), Ptrs->getType(),
3981 Mask->getType()};
3982
3983 B.CreateIntrinsic(ID: Intrinsic::spv_masked_scatter, OverloadTypes: Types, Args);
3984 I.eraseFromParent();
3985 return true;
3986 }
3987
3988 return false;
3989}
3990
3991// SPIR-V doesn't support bitcasts involving vector boolean type. Decompose such
3992// bitcasts into element-wise operations before building instructions
3993// worklist, so new instructions are properly visited and converted to
3994// SPIR-V intrinsics.
3995void SPIRVEmitIntrinsicsImpl::preprocessBoolVectorBitcasts(Function &F) {
3996 struct BoolVecBitcast {
3997 BitCastInst *BC;
3998 FixedVectorType *BoolVecTy;
3999 bool SrcIsBoolVec;
4000 };
4001
4002 auto getAsBoolVec = [](Type *Ty) -> FixedVectorType * {
4003 auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
4004 return (VTy && VTy->getElementType()->isIntegerTy(BitWidth: 1)) ? VTy : nullptr;
4005 };
4006
4007 SmallVector<BoolVecBitcast, 4> ToReplace;
4008 for (auto &I : instructions(F)) {
4009 auto *BC = dyn_cast<BitCastInst>(Val: &I);
4010 if (!BC)
4011 continue;
4012 if (auto *BVTy = getAsBoolVec(BC->getSrcTy()))
4013 ToReplace.push_back(Elt: {.BC: BC, .BoolVecTy: BVTy, .SrcIsBoolVec: true});
4014 else if (auto *BVTy = getAsBoolVec(BC->getDestTy()))
4015 ToReplace.push_back(Elt: {.BC: BC, .BoolVecTy: BVTy, .SrcIsBoolVec: false});
4016 }
4017
4018 for (auto &[BC, BoolVecTy, SrcIsBoolVec] : ToReplace) {
4019 IRBuilder<> B(BC);
4020 Value *Src = BC->getOperand(i_nocapture: 0);
4021 unsigned BoolVecN = BoolVecTy->getNumElements();
4022 // Use iN as the scalar intermediate type for the bool vector side.
4023 Type *IntTy = B.getIntNTy(N: BoolVecN);
4024
4025 // Convert source to scalar integer.
4026 Value *IntVal;
4027 if (SrcIsBoolVec) {
4028 // Extract each bool, zext, shift, and OR.
4029 IntVal = ConstantInt::get(Ty: IntTy, V: 0);
4030 for (unsigned I = 0; I < BoolVecN; ++I) {
4031 Value *Elem = B.CreateExtractElement(Vec: Src, Idx: B.getInt32(C: I));
4032 Value *Ext = B.CreateZExt(V: Elem, DestTy: IntTy);
4033 if (I > 0)
4034 Ext = B.CreateShl(LHS: Ext, RHS: ConstantInt::get(Ty: IntTy, V: I));
4035 IntVal = B.CreateOr(LHS: IntVal, RHS: Ext);
4036 }
4037 } else {
4038 // Source is a non-bool type. If it's already a scalar integer, use it
4039 // directly, otherwise bitcast to iN first.
4040 IntVal = Src;
4041 if (!Src->getType()->isIntegerTy())
4042 IntVal = B.CreateBitCast(V: Src, DestTy: IntTy);
4043 }
4044
4045 // Convert scalar integer to destination type.
4046 Value *Result;
4047 if (!SrcIsBoolVec) {
4048 // Test each bit with AND + icmp.
4049 Result = PoisonValue::get(T: BoolVecTy);
4050 for (unsigned I = 0; I < BoolVecN; ++I) {
4051 Value *Mask = ConstantInt::get(Ty: IntTy, V: APInt::getOneBitSet(numBits: BoolVecN, BitNo: I));
4052 Value *And = B.CreateAnd(LHS: IntVal, RHS: Mask);
4053 Value *Cmp = B.CreateICmpNE(LHS: And, RHS: ConstantInt::get(Ty: IntTy, V: 0));
4054 Result = B.CreateInsertElement(Vec: Result, NewElt: Cmp, Idx: B.getInt32(C: I));
4055 }
4056 } else {
4057 // Destination is a non-bool type. If it's a scalar integer, use IntVal
4058 // directly, otherwise bitcast from iN.
4059 Result = IntVal;
4060 if (!BC->getDestTy()->isIntegerTy())
4061 Result = B.CreateBitCast(V: IntVal, DestTy: BC->getDestTy());
4062 }
4063
4064 BC->replaceAllUsesWith(V: Result);
4065 BC->eraseFromParent();
4066 }
4067}
4068
4069bool SPIRVEmitIntrinsicsImpl::convertMaskedMemIntrinsics(Module &M) {
4070 bool Changed = false;
4071
4072 for (Function &F : make_early_inc_range(Range&: M)) {
4073 if (!F.isIntrinsic())
4074 continue;
4075 Intrinsic::ID IID = F.getIntrinsicID();
4076 if (IID != Intrinsic::masked_gather && IID != Intrinsic::masked_scatter)
4077 continue;
4078
4079 for (User *U : make_early_inc_range(Range: F.users())) {
4080 if (auto *II = dyn_cast<IntrinsicInst>(Val: U))
4081 Changed |= processMaskedMemIntrinsic(I&: *II);
4082 }
4083
4084 if (F.use_empty())
4085 F.eraseFromParent();
4086 }
4087
4088 return Changed;
4089}
4090
4091bool SPIRVEmitIntrinsicsImpl::runOnModule(Module &M) {
4092 bool Changed = false;
4093
4094 Changed |= convertMaskedMemIntrinsics(M);
4095
4096 parseFunDeclarations(M);
4097 insertConstantsForFPFastMathDefault(M);
4098 GVUsers.init(M);
4099
4100 TodoType.clear();
4101 for (auto &F : M)
4102 Changed |= runOnFunction(Func&: F);
4103
4104 // Specify function parameters after all functions were processed.
4105 for (auto &F : M) {
4106 // check if function parameter types are set
4107 CurrF = &F;
4108 if (!F.isDeclaration() && !F.isIntrinsic()) {
4109 IRBuilder<> B(F.getContext());
4110 processParamTypes(F: &F, B);
4111 }
4112 }
4113
4114 CanTodoType = false;
4115 Changed |= postprocessTypes(M);
4116
4117 if (HaveFunPtrs)
4118 Changed |= processFunctionPointers(M);
4119
4120 return Changed;
4121}
4122
4123PreservedAnalyses
4124llvm::SPIRVEmitIntrinsicsPass::run(Module &M, ModuleAnalysisManager &AM) {
4125 if (SPIRVEmitIntrinsicsImpl(TM).runOnModule(M))
4126 return PreservedAnalyses::none();
4127 return PreservedAnalyses::all();
4128}
4129
4130ModulePass *llvm::createSPIRVEmitIntrinsicsPass(const SPIRVTargetMachine &TM) {
4131 return new SPIRVEmitIntrinsicsLegacy(TM);
4132}
4133