1//===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass emits SPIRV intrinsics keeping essential high-level information for
10// the translation of LLVM IR to SPIR-V.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRVEmitIntrinsics.h"
15#include "SPIRV.h"
16#include "SPIRVBuiltins.h"
17#include "SPIRVSubtarget.h"
18#include "SPIRVTargetMachine.h"
19#include "SPIRVUtils.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/DenseSet.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/ADT/StringSet.h"
24#include "llvm/Analysis/LoopInfo.h"
25#include "llvm/IR/Dominators.h"
26#include "llvm/IR/IRBuilder.h"
27#include "llvm/IR/InstIterator.h"
28#include "llvm/IR/InstVisitor.h"
29#include "llvm/IR/IntrinsicsSPIRV.h"
30#include "llvm/IR/PatternMatch.h"
31#include "llvm/IR/TypedPointerType.h"
32#include "llvm/IR/Value.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Transforms/Utils/Local.h"
36
37#include <cassert>
38#include <optional>
39#include <queue>
40
41// This pass performs the following transformation on LLVM IR level required
42// for the following translation to SPIR-V:
43// - replaces direct usages of aggregate constants with target-specific
44// intrinsics;
45// - replaces aggregates-related instructions (extract/insert, ld/st, etc)
46// with a target-specific intrinsics;
47// - emits intrinsics for the global variable initializers since IRTranslator
48// doesn't handle them and it's not very convenient to translate them
49// ourselves;
50// - emits intrinsics to keep track of the string names assigned to the values;
51// - emits intrinsics to keep track of constants (this is necessary to have an
52// LLVM IR constant after the IRTranslation is completed) for their further
53// deduplication;
54// - emits intrinsics to keep track of original LLVM types of the values
55// to be able to emit proper SPIR-V types eventually.
56//
57// TODO: consider removing spv.track.constant in favor of spv.assign.type.
58
59using namespace llvm;
60using namespace llvm::PatternMatch;
61
62#define DEBUG_TYPE "spirv-emit-intrinsics"
63
64static cl::opt<bool>
65 SpirvEmitOpNames("spirv-emit-op-names",
66 cl::desc("Emit OpName for all instructions"),
67 cl::init(Val: false));
68
69namespace llvm::SPIRV {
70#define GET_BuiltinGroup_DECL
71#include "SPIRVGenTables.inc"
72} // namespace llvm::SPIRV
73
74namespace {
75// This class keeps track of which functions reference which global variables.
76class GlobalVariableUsers {
77 template <typename T1, typename T2>
78 using OneToManyMapTy = DenseMap<T1, SmallPtrSet<T2, 4>>;
79
80 OneToManyMapTy<const GlobalVariable *, const Function *> GlobalIsUsedByFun;
81
82 void collectGlobalUsers(
83 const GlobalVariable *GV,
84 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
85 &GlobalIsUsedByGlobal) {
86 SmallVector<const Value *> Stack = {GV->user_begin(), GV->user_end()};
87 while (!Stack.empty()) {
88 const Value *V = Stack.pop_back_val();
89
90 if (const Instruction *I = dyn_cast<Instruction>(Val: V)) {
91 GlobalIsUsedByFun[GV].insert(Ptr: I->getFunction());
92 continue;
93 }
94
95 if (const GlobalVariable *UserGV = dyn_cast<GlobalVariable>(Val: V)) {
96 GlobalIsUsedByGlobal[GV].insert(Ptr: UserGV);
97 continue;
98 }
99
100 if (const Constant *C = dyn_cast<Constant>(Val: V))
101 Stack.append(in_start: C->user_begin(), in_end: C->user_end());
102 }
103 }
104
105 bool propagateGlobalToGlobalUsers(
106 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
107 &GlobalIsUsedByGlobal) {
108 SmallVector<const GlobalVariable *> OldUsersGlobals;
109 bool Changed = false;
110 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
111 OldUsersGlobals.assign(in_start: UserGlobals.begin(), in_end: UserGlobals.end());
112 for (const GlobalVariable *UserGV : OldUsersGlobals) {
113 auto It = GlobalIsUsedByGlobal.find(Val: UserGV);
114 if (It == GlobalIsUsedByGlobal.end())
115 continue;
116 Changed |= set_union(S1&: UserGlobals, S2: It->second);
117 }
118 }
119 return Changed;
120 }
121
122 void propagateGlobalToFunctionReferences(
123 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
124 &GlobalIsUsedByGlobal) {
125 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
126 auto &UserFunctions = GlobalIsUsedByFun[GV];
127 for (const GlobalVariable *UserGV : UserGlobals) {
128 auto It = GlobalIsUsedByFun.find(Val: UserGV);
129 if (It == GlobalIsUsedByFun.end())
130 continue;
131 set_union(S1&: UserFunctions, S2: It->second);
132 }
133 }
134 }
135
136public:
137 void init(Module &M) {
138 // Collect which global variables are referenced by which global variables
139 // and which functions reference each global variables.
140 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
141 GlobalIsUsedByGlobal;
142 GlobalIsUsedByFun.clear();
143 for (GlobalVariable &GV : M.globals())
144 collectGlobalUsers(GV: &GV, GlobalIsUsedByGlobal);
145
146 // Compute indirect references by iterating until a fixed point is reached.
147 while (propagateGlobalToGlobalUsers(GlobalIsUsedByGlobal))
148 (void)0;
149
150 propagateGlobalToFunctionReferences(GlobalIsUsedByGlobal);
151 }
152
153 using FunctionSetType = typename decltype(GlobalIsUsedByFun)::mapped_type;
154 const FunctionSetType &
155 getTransitiveUserFunctions(const GlobalVariable &GV) const {
156 auto It = GlobalIsUsedByFun.find(Val: &GV);
157 if (It != GlobalIsUsedByFun.end())
158 return It->second;
159
160 static const FunctionSetType Empty{};
161 return Empty;
162 }
163};
164
165static bool isaGEP(const Value *V) {
166 return isa<StructuredGEPInst>(Val: V) || isa<GetElementPtrInst>(Val: V);
167}
168
169// If Ty is a byte-addressing type, return the multiplier for the offset.
170// Otherwise return std::nullopt.
171static std::optional<uint64_t> getByteAddressingMultiplier(Type *Ty) {
172 if (Ty == IntegerType::getInt8Ty(C&: Ty->getContext())) {
173 return 1;
174 }
175 if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
176 if (AT->getElementType() == IntegerType::getInt8Ty(C&: Ty->getContext())) {
177 return AT->getNumElements();
178 }
179 }
180 return std::nullopt;
181}
182
183class SPIRVEmitIntrinsicsImpl
184 : public InstVisitor<SPIRVEmitIntrinsicsImpl, Instruction *> {
185 const SPIRVTargetMachine &TM;
186 SPIRVGlobalRegistry *GR = nullptr;
187 Function *CurrF = nullptr;
188 bool TrackConstants = true;
189 bool HaveFunPtrs = false;
190 DenseMap<Instruction *, Constant *> AggrConsts;
191 DenseMap<Instruction *, Type *> AggrConstTypes;
192 SmallPtrSet<Instruction *, 0> AggrStores;
193 GlobalVariableUsers GVUsers;
194 SmallPtrSet<Value *, 0> Named;
195
196 // map of function declarations to <pointer arg index => element type>
197 DenseMap<Function *, SmallVector<std::pair<unsigned, Type *>>> FDeclPtrTys;
198
199 // a register of Instructions that don't have a complete type definition
200 bool CanTodoType = true;
201 unsigned TodoTypeSz = 0;
202 DenseMap<Value *, bool> TodoType;
203 void insertTodoType(Value *Op) {
204 // TODO: add isa<CallInst>(Op) to no-insert
205 if (CanTodoType && !isaGEP(V: Op)) {
206 auto It = TodoType.try_emplace(Key: Op, Args: true);
207 if (It.second)
208 ++TodoTypeSz;
209 }
210 }
211 void eraseTodoType(Value *Op) {
212 auto It = TodoType.find(Val: Op);
213 if (It != TodoType.end() && It->second) {
214 It->second = false;
215 --TodoTypeSz;
216 }
217 }
218 bool isTodoType(Value *Op) {
219 if (isaGEP(V: Op))
220 return false;
221 auto It = TodoType.find(Val: Op);
222 return It != TodoType.end() && It->second;
223 }
224 // a register of Instructions that were visited by deduceOperandElementType()
225 // to validate operand types with an instruction
226 SmallPtrSet<Instruction *, 0> TypeValidated;
227
228 // well known result types of builtins
229 enum WellKnownTypes { Event };
230
231 // deduce element type of untyped pointers
232 Type *deduceElementType(Value *I, bool UnknownElemTypeI8);
233 Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8);
234 Type *deduceElementTypeHelper(Value *I, SmallPtrSetImpl<Value *> &Visited,
235 bool UnknownElemTypeI8,
236 bool IgnoreKnownType = false);
237 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
238 bool UnknownElemTypeI8);
239 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
240 SmallPtrSetImpl<Value *> &Visited,
241 bool UnknownElemTypeI8);
242 Type *deduceElementTypeByUsersDeep(Value *Op,
243 SmallPtrSetImpl<Value *> &Visited,
244 bool UnknownElemTypeI8);
245 void maybeAssignPtrType(Type *&Ty, Value *I, Type *RefTy,
246 bool UnknownElemTypeI8);
247
248 // deduce nested types of composites
249 Type *deduceNestedTypeHelper(User *U, bool UnknownElemTypeI8);
250 Type *deduceNestedTypeHelper(User *U, Type *Ty,
251 SmallPtrSetImpl<Value *> &Visited,
252 bool UnknownElemTypeI8);
253
254 // deduce Types of operands of the Instruction if possible
255 void
256 deduceOperandElementType(Instruction *I,
257 SmallPtrSetImpl<Instruction *> *IncompleteRets,
258 const SmallPtrSetImpl<Value *> *AskOps = nullptr,
259 bool IsPostprocessing = false);
260
261 void preprocessCompositeConstants(IRBuilder<> &B);
262 Value *lowerUndefOrPoison(Value *Op, IRBuilder<> &B, bool HasPoisonExt);
263 void preprocessUndefsAndPoisons(IRBuilder<> &B);
264 void insertCompositeAggregateArms(Instruction *I, IRBuilder<> &B);
265 void simplifyNullAddrSpaceCasts();
266
267 Type *reconstructType(Value *Op, bool UnknownElemTypeI8,
268 bool IsPostprocessing);
269
270 void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
271 void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
272 bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
273 bool UnknownElemTypeI8);
274 void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B);
275 void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V,
276 IRBuilder<> &B);
277 void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer,
278 Type *ExpectedElementType,
279 unsigned OperandToReplace,
280 IRBuilder<> &B);
281 void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
282 bool shouldTryToAddMemAliasingDecoration(Instruction *Inst);
283 void insertSpirvDecorations(Instruction *I, IRBuilder<> &B);
284 void insertConstantsForFPFastMathDefault(Module &M);
285 Value *buildSpvUndefComposite(Type *AggrTy, IRBuilder<> &B);
286 void reconstructAggregateReturns(Function &Func, IRBuilder<> &B);
287 void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
288 void processParamTypes(Function *F, IRBuilder<> &B);
289 void processParamTypesByFunHeader(Function *F, IRBuilder<> &B);
290 Type *deduceFunParamElementType(Function *F, unsigned OpIdx);
291 Type *deduceFunParamElementType(Function *F, unsigned OpIdx,
292 SmallPtrSetImpl<Function *> &FVisited);
293
294 bool deduceOperandElementTypeCalledFunction(
295 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
296 Type *&KnownElemTy, bool &Incomplete);
297 void deduceOperandElementTypeFunctionPointer(
298 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
299 Type *&KnownElemTy, bool IsPostprocessing);
300 bool deduceOperandElementTypeFunctionRet(
301 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
302 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing,
303 Type *&KnownElemTy, Value *Op, Function *F);
304
305 CallInst *buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy);
306 void replaceUsesOfWithSpvPtrcast(Value *Op, Type *ElemTy, Instruction *I,
307 DenseMap<Function *, CallInst *> Ptrcasts);
308 void propagateElemType(Value *Op, Type *ElemTy,
309 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
310 void
311 propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
312 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
313 void propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
314 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
315 SmallPtrSetImpl<Value *> &Visited,
316 DenseMap<Function *, CallInst *> Ptrcasts);
317
318 void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
319 void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src,
320 Instruction *Dest, bool DeleteOld = true);
321
322 void applyDemangledPtrArgTypes(IRBuilder<> &B);
323
324 GetElementPtrInst *simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP);
325
326 bool runOnFunction(Function &F);
327 bool postprocessTypes(Module &M);
328 bool processFunctionPointers(Module &M);
329 void parseFunDeclarations(Module &M);
330 void useRoundingMode(ConstrainedFPIntrinsic *FPI, IRBuilder<> &B);
331 bool processMaskedMemIntrinsic(IntrinsicInst &I);
332 bool convertMaskedMemIntrinsics(Module &M);
333 void preprocessBoolVectorBitcasts(Function &F);
334
335 void emitUnstructuredLoopControls(Function &F, IRBuilder<> &B);
336
337 // Tries to walk the type accessed by the given GEP instruction.
338 // For each nested type access, one of the 2 callbacks is called:
339 // - OnLiteralIndexing when the index is a known constant value.
340 // Parameters:
341 // PointedType: the pointed type resulting of this indexing.
342 // If the parent type is an array, this is the index in the array.
343 // If the parent type is a struct, this is the field index.
344 // Index: index of the element in the parent type.
345 // - OnDynamnicIndexing when the index is a non-constant value.
346 // This callback is only called when indexing into an array.
347 // Parameters:
348 // ElementType: the type of the elements stored in the parent array.
349 // Offset: the Value* containing the byte offset into the array.
350 // Multiplier: a scaling factor for the offset.
351 // Return true if an error occurred during the walk, false otherwise.
352 bool walkLogicalAccessChain(
353 GetElementPtrInst &GEP,
354 const std::function<void(Type *PointedType, uint64_t Index)>
355 &OnLiteralIndexing,
356 const std::function<void(Type *ElementType, Value *Offset,
357 uint64_t Multiplier)> &OnDynamicIndexing);
358
359 bool walkLogicalAccessChainDynamic(
360 Type *CurType, Value *Operand, uint64_t Multiplier,
361 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
362 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing);
363
364 bool walkLogicalAccessChainConstant(
365 Type *CurType, uint64_t Offset,
366 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing);
367
368 // Returns the type accessed using the given GEP instruction by relying
369 // on the GEP type.
370 // FIXME: GEP types are not supposed to be used to retrieve the pointed
371 // type. This must be fixed.
372 Type *getGEPType(GetElementPtrInst *GEP);
373
374 // Returns the type accessed using the given GEP instruction by walking
375 // the source type using the GEP indices.
376 // FIXME: without help from the frontend, this method cannot reliably retrieve
377 // the stored type, nor can robustly determine the depth of the type
378 // we are accessing.
379 Type *getGEPTypeLogical(GetElementPtrInst *GEP);
380
381 Instruction *buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP);
382
383public:
384 SPIRVEmitIntrinsicsImpl(const SPIRVTargetMachine &TM) : TM(TM) {}
385 Instruction *visitInstruction(Instruction &I) { return &I; }
386 Instruction *visitSwitchInst(SwitchInst &I);
387 Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
388 Instruction *visitIntrinsicInst(IntrinsicInst &I);
389 Instruction *visitBitCastInst(BitCastInst &I);
390 Instruction *visitInsertElementInst(InsertElementInst &I);
391 Instruction *visitExtractElementInst(ExtractElementInst &I);
392 Instruction *visitInsertValueInst(InsertValueInst &I);
393 Instruction *visitExtractValueInst(ExtractValueInst &I);
394 Instruction *visitLoadInst(LoadInst &I);
395 Instruction *visitStoreInst(StoreInst &I);
396 Instruction *visitAllocaInst(AllocaInst &I);
397 Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
398 Instruction *visitUnreachableInst(UnreachableInst &I);
399 Instruction *visitCallInst(CallInst &I);
400
401 bool runOnModule(Module &M);
402};
403
404class SPIRVEmitIntrinsicsLegacy : public ModulePass {
405 const SPIRVTargetMachine &TM;
406
407public:
408 static char ID;
409 SPIRVEmitIntrinsicsLegacy(const SPIRVTargetMachine &TM)
410 : ModulePass(ID), TM(TM) {}
411
412 StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
413
414 bool runOnModule(Module &M) override {
415 return SPIRVEmitIntrinsicsImpl(TM).runOnModule(M);
416 }
417};
418
419bool isConvergenceIntrinsic(const Instruction *I) {
420 return match(V: I, P: m_AnyIntrinsic<Intrinsic::experimental_convergence_entry,
421 Intrinsic::experimental_convergence_loop,
422 Intrinsic::experimental_convergence_anchor>());
423}
424
425bool expectIgnoredInIRTranslation(const Instruction *I) {
426 return match(V: I, P: m_AnyIntrinsic<Intrinsic::invariant_start,
427 Intrinsic::spv_resource_handlefrombinding,
428 Intrinsic::spv_resource_getbasepointer,
429 Intrinsic::spv_resource_getpointer>());
430}
431
432// Returns the source pointer from `I` ignoring intermediate ptrcast.
433Value *getPointerRoot(Value *I) {
434 Value *V;
435 if (match(V: I, P: m_Intrinsic<Intrinsic::spv_ptrcast>(Ops: m_Value(V))))
436 return getPointerRoot(I: V);
437 return I;
438}
439
440} // namespace
441
442char SPIRVEmitIntrinsicsLegacy::ID = 0;
443
444INITIALIZE_PASS(SPIRVEmitIntrinsicsLegacy, "spirv-emit-intrinsics",
445 "SPIRV emit intrinsics", false, false)
446
447static inline bool isAssignTypeInstr(const Instruction *I) {
448 return match(V: I, P: m_Intrinsic<Intrinsic::spv_assign_type>());
449}
450
451static bool isMemInstrToReplace(Instruction *I) {
452 return isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I) || isa<InsertValueInst>(Val: I) ||
453 isa<ExtractValueInst>(Val: I) || isa<AtomicCmpXchgInst>(Val: I);
454}
455
456static bool isAggrConstForceInt32(const Value *V) {
457 bool IsAggrZero =
458 isa<ConstantAggregateZero>(Val: V) && !V->getType()->isVectorTy();
459 bool IsUndefAggregate = isa<UndefValue>(Val: V) && V->getType()->isAggregateType();
460 return isa<ConstantArray>(Val: V) || isa<ConstantStruct>(Val: V) ||
461 isa<ConstantDataArray>(Val: V) || IsAggrZero || IsUndefAggregate;
462}
463
464static bool isSpvAggrPlaceholder(const Value *V) {
465 return match(
466 V,
467 P: m_AnyIntrinsic<Intrinsic::spv_undef, Intrinsic::spv_const_composite>());
468}
469
470static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
471 if (isa<PHINode>(Val: I))
472 B.SetInsertPoint(I->getParent()->getFirstNonPHIOrDbgOrAlloca());
473 else
474 B.SetInsertPoint(I);
475}
476
477static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I) {
478 B.SetCurrentDebugLocation(I->getDebugLoc());
479 if (I->getType()->isVoidTy())
480 B.SetInsertPoint(I->getNextNode());
481 else
482 B.SetInsertPoint(*I->getInsertionPointAfterDef());
483}
484
485static bool requireAssignType(Instruction *I) {
486 return !match(
487 V: I,
488 P: m_AnyIntrinsic<Intrinsic::invariant_start, Intrinsic::invariant_end>());
489}
490
491static inline void reportFatalOnTokenType(const Instruction *I) {
492 if (I->getType()->isTokenTy())
493 report_fatal_error(reason: "A token is encountered but SPIR-V without extensions "
494 "does not support token type",
495 gen_crash_diag: false);
496}
497
498static void emitAssignName(Instruction *I, IRBuilder<> &B) {
499 if (!I->hasName() || I->getType()->isAggregateType() ||
500 expectIgnoredInIRTranslation(I))
501 return;
502
503 // We want to be conservative when adding the names because they can interfere
504 // with later optimizations.
505 bool KeepName = SpirvEmitOpNames;
506 if (!KeepName) {
507 if (isa<AllocaInst>(Val: I)) {
508 KeepName = true;
509 } else if (auto *CI = dyn_cast<CallBase>(Val: I)) {
510 Function *F = CI->getCalledFunction();
511 if (F && F->getName().starts_with(Prefix: "llvm.spv.alloca"))
512 KeepName = true;
513 }
514 }
515
516 if (!KeepName)
517 return;
518
519 reportFatalOnTokenType(I);
520 setInsertPointAfterDef(B, I);
521 LLVMContext &Ctx = I->getContext();
522 std::vector<Value *> Args = {
523 I, MetadataAsValue::get(
524 Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: I->getName())))};
525 B.CreateIntrinsic(ID: Intrinsic::spv_assign_name, OverloadTypes: {I->getType()}, Args);
526}
527
528void SPIRVEmitIntrinsicsImpl::replaceAllUsesWith(Value *Src, Value *Dest,
529 bool DeleteOld) {
530 GR->replaceAllUsesWith(Old: Src, New: Dest, DeleteOld);
531 // Update uncomplete type records if any
532 if (isTodoType(Op: Src)) {
533 if (DeleteOld)
534 eraseTodoType(Op: Src);
535 insertTodoType(Op: Dest);
536 }
537}
538
539void SPIRVEmitIntrinsicsImpl::replaceAllUsesWithAndErase(IRBuilder<> &B,
540 Instruction *Src,
541 Instruction *Dest,
542 bool DeleteOld) {
543 replaceAllUsesWith(Src, Dest, DeleteOld);
544 std::string Name = Src->hasName() ? Src->getName().str() : "";
545 Src->eraseFromParent();
546 if (!Name.empty()) {
547 Dest->setName(Name);
548 if (Named.insert(Ptr: Dest).second)
549 emitAssignName(I: Dest, B);
550 }
551}
552
553static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
554 return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
555 isPointerTy(T: SI->getValueOperand()->getType()) &&
556 isa<Argument>(Val: SI->getValueOperand());
557}
558
559// A pointer-typed local holds a pointer, so its deduced pointee must stay a
560// pointer.
561static bool tracesToPointerAlloca(Value *V) {
562 using namespace PatternMatch;
563 V = V->stripPointerCasts();
564 if (auto *AI = dyn_cast<AllocaInst>(Val: V))
565 return isUntypedPointerTy(T: AI->getAllocatedType());
566 return match(
567 V, P: m_AnyIntrinsic<Intrinsic::spv_alloca, Intrinsic::spv_alloca_array>());
568}
569
570// Maybe restore original function return type.
571static inline Type *restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I,
572 Type *Ty) {
573 CallInst *CI = dyn_cast<CallInst>(Val: I);
574 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
575 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
576 return Ty;
577 if (Type *OriginalTy = GR->findMutated(Val: CI->getCalledFunction()))
578 return OriginalTy;
579 return Ty;
580}
581
582// Reconstruct type with nested element types according to deduced type info.
583// Return nullptr if no detailed type info is available.
584Type *SPIRVEmitIntrinsicsImpl::reconstructType(Value *Op,
585 bool UnknownElemTypeI8,
586 bool IsPostprocessing) {
587 Type *Ty = Op->getType();
588 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
589 Ty = restoreMutatedType(GR, I: OpI, Ty);
590 if (auto It = AggrConstTypes.find(Val: OpI); It != AggrConstTypes.end())
591 Ty = It->second;
592 }
593 if (!isUntypedPointerTy(T: Ty))
594 return Ty;
595 // try to find the pointee type
596 if (Type *NestedTy = GR->findDeducedElementType(Val: Op))
597 return getTypedPointerWrapper(ElemTy: NestedTy, AS: getPointerAddressSpace(T: Ty));
598 // not a pointer according to the type info (e.g., Event object)
599 CallInst *CI = GR->findAssignPtrTypeInstr(Val: Op);
600 if (CI) {
601 MetadataAsValue *MD = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
602 return cast<ConstantAsMetadata>(Val: MD->getMetadata())->getType();
603 }
604 if (UnknownElemTypeI8) {
605 if (!IsPostprocessing)
606 insertTodoType(Op);
607 return getTypedPointerWrapper(ElemTy: IntegerType::getInt8Ty(C&: Op->getContext()),
608 AS: getPointerAddressSpace(T: Ty));
609 }
610 return nullptr;
611}
612
613CallInst *SPIRVEmitIntrinsicsImpl::buildSpvPtrcast(Function *F, Value *Op,
614 Type *ElemTy) {
615 IRBuilder<> B(Op->getContext());
616 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
617 // spv_ptrcast's argument Op denotes an instruction that generates
618 // a value, and we may use getInsertionPointAfterDef()
619 setInsertPointAfterDef(B, I: OpI);
620 } else if (auto *OpA = dyn_cast<Argument>(Val: Op)) {
621 B.SetInsertPointPastAllocas(OpA->getParent());
622 B.SetCurrentDebugLocation(DebugLoc());
623 } else {
624 B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
625 }
626 Type *OpTy = Op->getType();
627 SmallVector<Type *, 2> Types = {OpTy, OpTy};
628 SmallVector<Value *, 2> Args = {Op, buildMD(Arg: getNormalizedPoisonValue(Ty: ElemTy)),
629 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
630 CallInst *PtrCasted =
631 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
632 GR->buildAssignPtr(B, ElemTy, Arg: PtrCasted);
633 return PtrCasted;
634}
635
636void SPIRVEmitIntrinsicsImpl::replaceUsesOfWithSpvPtrcast(
637 Value *Op, Type *ElemTy, Instruction *I,
638 DenseMap<Function *, CallInst *> Ptrcasts) {
639 Function *F = I->getParent()->getParent();
640 CallInst *PtrCastedI = nullptr;
641 auto It = Ptrcasts.find(Val: F);
642 if (It == Ptrcasts.end()) {
643 PtrCastedI = buildSpvPtrcast(F, Op, ElemTy);
644 Ptrcasts[F] = PtrCastedI;
645 } else {
646 PtrCastedI = It->second;
647 }
648 I->replaceUsesOfWith(From: Op, To: PtrCastedI);
649}
650
651void SPIRVEmitIntrinsicsImpl::propagateElemType(
652 Value *Op, Type *ElemTy,
653 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
654 DenseMap<Function *, CallInst *> Ptrcasts;
655 SmallVector<User *> Users(Op->users());
656 for (auto *U : Users) {
657 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
658 continue;
659 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
660 continue;
661 Instruction *UI = dyn_cast<Instruction>(Val: U);
662 // If the instruction was validated already, we need to keep it valid by
663 // keeping current Op type.
664 if (isaGEP(V: UI) || TypeValidated.find(Ptr: UI) != TypeValidated.end())
665 replaceUsesOfWithSpvPtrcast(Op, ElemTy, I: UI, Ptrcasts);
666 }
667}
668
669void SPIRVEmitIntrinsicsImpl::propagateElemTypeRec(
670 Value *Op, Type *PtrElemTy, Type *CastElemTy,
671 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
672 SmallPtrSet<Value *, 0> Visited;
673 DenseMap<Function *, CallInst *> Ptrcasts;
674 propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited,
675 Ptrcasts: std::move(Ptrcasts));
676}
677
678void SPIRVEmitIntrinsicsImpl::propagateElemTypeRec(
679 Value *Op, Type *PtrElemTy, Type *CastElemTy,
680 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
681 SmallPtrSetImpl<Value *> &Visited,
682 DenseMap<Function *, CallInst *> Ptrcasts) {
683 if (!Visited.insert(Ptr: Op).second)
684 return;
685 SmallVector<User *> Users(Op->users());
686 for (auto *U : Users) {
687 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
688 continue;
689 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
690 continue;
691 Instruction *UI = dyn_cast<Instruction>(Val: U);
692 // If the instruction was validated already, we need to keep it valid by
693 // keeping current Op type.
694 if (isaGEP(V: UI) || TypeValidated.find(Ptr: UI) != TypeValidated.end())
695 replaceUsesOfWithSpvPtrcast(Op, ElemTy: CastElemTy, I: UI, Ptrcasts);
696 }
697}
698
699// Set element pointer type to the given value of ValueTy and tries to
700// specify this type further (recursively) by Operand value, if needed.
701
702Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByValueDeep(
703 Type *ValueTy, Value *Operand, bool UnknownElemTypeI8) {
704 SmallPtrSet<Value *, 0> Visited;
705 return deduceElementTypeByValueDeep(ValueTy, Operand, Visited,
706 UnknownElemTypeI8);
707}
708
709Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByValueDeep(
710 Type *ValueTy, Value *Operand, SmallPtrSetImpl<Value *> &Visited,
711 bool UnknownElemTypeI8) {
712 Type *Ty = ValueTy;
713 if (Operand) {
714 if (auto *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
715 if (Type *NestedTy =
716 deduceElementTypeHelper(I: Operand, Visited, UnknownElemTypeI8))
717 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
718 } else {
719 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Operand), Ty, Visited,
720 UnknownElemTypeI8);
721 }
722 }
723 return Ty;
724}
725
726// Traverse User instructions to deduce an element pointer type of the operand.
727Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByUsersDeep(
728 Value *Op, SmallPtrSetImpl<Value *> &Visited, bool UnknownElemTypeI8) {
729 if (!Op || !isPointerTy(T: Op->getType()) || isa<ConstantPointerNull>(Val: Op) ||
730 isa<UndefValue>(Val: Op))
731 return nullptr;
732
733 if (auto ElemTy = getPointeeType(Ty: Op->getType()))
734 return ElemTy;
735
736 // maybe we already know operand's element type
737 if (Type *KnownTy = GR->findDeducedElementType(Val: Op))
738 return KnownTy;
739
740 for (User *OpU : Op->users()) {
741 if (Instruction *Inst = dyn_cast<Instruction>(Val: OpU)) {
742 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8))
743 return Ty;
744 }
745 }
746 return nullptr;
747}
748
749// Implements what we know in advance about intrinsics and builtin calls
750// TODO: consider feasibility of this particular case to be generalized by
751// encoding knowledge about intrinsics and builtin calls by corresponding
752// specification rules
753static Type *getPointeeTypeByCallInst(StringRef DemangledName,
754 Function *CalledF, unsigned OpIdx) {
755 if ((DemangledName.starts_with(Prefix: "__spirv_ocl_printf(") ||
756 DemangledName.starts_with(Prefix: "printf(")) &&
757 OpIdx == 0)
758 return IntegerType::getInt8Ty(C&: CalledF->getContext());
759 return nullptr;
760}
761
762// Deduce and return a successfully deduced Type of the Instruction,
763// or nullptr otherwise.
764Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeHelper(Value *I,
765 bool UnknownElemTypeI8) {
766 SmallPtrSet<Value *, 0> Visited;
767 return deduceElementTypeHelper(I, Visited, UnknownElemTypeI8);
768}
769
770void SPIRVEmitIntrinsicsImpl::maybeAssignPtrType(Type *&Ty, Value *Op,
771 Type *RefTy,
772 bool UnknownElemTypeI8) {
773 if (isUntypedPointerTy(T: RefTy)) {
774 if (!UnknownElemTypeI8)
775 return;
776 insertTodoType(Op);
777 if (isa<IntToPtrInst>(Val: Op))
778 return;
779 }
780 Ty = RefTy;
781}
782
783bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChainDynamic(
784 Type *CurType, Value *Operand, uint64_t Multiplier,
785 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
786 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing) {
787 // Dynamic indexing into a struct is not possible.
788 // We know that we must be accessing the first element
789 // of the struct if the current type is a struct.
790 // Try to find the first array type that is at offset 0 in the struct.
791 while (auto *ST = dyn_cast<StructType>(Val: CurType)) {
792 if (ST->getNumElements() == 0)
793 break;
794 CurType = ST->getElementType(N: 0);
795 OnLiteralIndexing(CurType, 0);
796 }
797
798 assert(CurType);
799 ArrayType *AT = dyn_cast<ArrayType>(Val: CurType);
800 // Operand is not constant. Either we have an array and accept it, or we
801 // give up.
802 if (AT)
803 OnDynamicIndexing(AT->getElementType(), Operand, Multiplier);
804 return AT == nullptr;
805}
806
807bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChainConstant(
808 Type *CurType, uint64_t Offset,
809 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing) {
810 auto &DL = CurrF->getDataLayout();
811
812 do {
813 if (ArrayType *AT = dyn_cast<ArrayType>(Val: CurType)) {
814 uint64_t EltTypeSize = DL.getTypeAllocSize(Ty: AT->getElementType());
815 assert(Offset < AT->getNumElements() * EltTypeSize);
816 uint64_t Index = Offset / EltTypeSize;
817 Offset = Offset - (Index * EltTypeSize);
818 CurType = AT->getElementType();
819 OnLiteralIndexing(CurType, Index);
820 } else if (StructType *ST = dyn_cast<StructType>(Val: CurType)) {
821 uint32_t StructSize = DL.getTypeSizeInBits(Ty: ST) / 8;
822 assert(Offset < StructSize);
823 (void)StructSize;
824 const auto &STL = DL.getStructLayout(Ty: ST);
825 unsigned Element = STL->getElementContainingOffset(FixedOffset: Offset);
826 Offset -= STL->getElementOffset(Idx: Element);
827 CurType = ST->getElementType(N: Element);
828 OnLiteralIndexing(CurType, Element);
829 } else if (auto *VT = dyn_cast<FixedVectorType>(Val: CurType)) {
830 Type *EltTy = VT->getElementType();
831 TypeSize EltSizeBits = DL.getTypeSizeInBits(Ty: EltTy);
832 assert(EltSizeBits % 8 == 0 &&
833 "Element type size in bits must be a multiple of 8.");
834 uint32_t EltTypeSize = EltSizeBits / 8;
835 assert(Offset < VT->getNumElements() * EltTypeSize);
836 uint64_t Index = Offset / EltTypeSize;
837 Offset -= Index * EltTypeSize;
838 CurType = EltTy;
839 OnLiteralIndexing(CurType, Index);
840 } else {
841 // Unknown composite kind; give up.
842 return true;
843 }
844 } while (Offset > 0);
845
846 return false;
847}
848
849bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChain(
850 GetElementPtrInst &GEP,
851 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
852 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing) {
853 // We only rewrite byte-addressing GEP. Other should be left as-is.
854 // Valid byte-addressing GEP must always have a single index.
855 std::optional<uint64_t> MultiplierOpt =
856 getByteAddressingMultiplier(Ty: GEP.getSourceElementType());
857 assert(MultiplierOpt && "We only rewrite byte-addressing GEP");
858 uint64_t Multiplier = *MultiplierOpt;
859 assert(GEP.getNumIndices() == 1);
860
861 Value *Src = getPointerRoot(I: GEP.getPointerOperand());
862 Type *CurType = deduceElementType(I: Src, UnknownElemTypeI8: true);
863
864 Value *Operand = *GEP.idx_begin();
865 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Operand))
866 return walkLogicalAccessChainConstant(
867 CurType, Offset: CI->getZExtValue() * Multiplier, OnLiteralIndexing);
868
869 return walkLogicalAccessChainDynamic(CurType, Operand, Multiplier,
870 OnLiteralIndexing, OnDynamicIndexing);
871}
872
873Instruction *SPIRVEmitIntrinsicsImpl::buildLogicalAccessChainFromGEP(
874 GetElementPtrInst &GEP) {
875 auto &DL = CurrF->getDataLayout();
876 IRBuilder<> B(GEP.getParent());
877 B.SetInsertPoint(&GEP);
878
879 std::vector<Value *> Indices;
880 Indices.push_back(x: ConstantInt::get(
881 Ty: IntegerType::getInt32Ty(C&: CurrF->getContext()), V: 0, /* Signed= */ IsSigned: false));
882 walkLogicalAccessChain(
883 GEP,
884 OnLiteralIndexing: [&Indices, &B](Type *EltType, uint64_t Index) {
885 Indices.push_back(
886 x: ConstantInt::get(Ty: B.getInt64Ty(), V: Index, /* Signed= */ IsSigned: false));
887 },
888 OnDynamicIndexing: [&Indices, &B, &DL, this](Type *EltType, Value *Offset,
889 uint64_t Multiplier) {
890 Value *Index = nullptr;
891 uint32_t EltTypeSize = DL.getTypeSizeInBits(Ty: EltType) / 8;
892 assert(Multiplier != 0);
893 if (Multiplier == EltTypeSize) {
894 Index = Offset;
895 } else if (EltTypeSize % Multiplier == 0) {
896 Index =
897 B.CreateUDiv(LHS: Offset, RHS: ConstantInt::get(Ty: Offset->getType(),
898 V: EltTypeSize / Multiplier,
899 /* Signed= */ IsSigned: false));
900 } else {
901 Index = B.CreateMul(LHS: Offset,
902 RHS: ConstantInt::get(Ty: Offset->getType(), V: Multiplier,
903 /* Signed= */ IsSigned: false));
904 insertAssignTypeIntrs(I: cast<Instruction>(Val: Index), B);
905 Index = B.CreateUDiv(LHS: Index,
906 RHS: ConstantInt::get(Ty: Offset->getType(), V: EltTypeSize,
907 /* Signed= */ IsSigned: false));
908 }
909 insertAssignTypeIntrs(I: cast<Instruction>(Val: Index), B);
910 Indices.push_back(x: Index);
911 });
912
913 SmallVector<Type *, 2> Types = {GEP.getType(), GEP.getOperand(i_nocapture: 0)->getType()};
914 SmallVector<Value *, 4> Args;
915 Args.push_back(Elt: B.getInt1(V: GEP.isInBounds()));
916 Args.push_back(Elt: GEP.getOperand(i_nocapture: 0));
917 llvm::append_range(C&: Args, R&: Indices);
918 Instruction *NewI =
919 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
920 replaceAllUsesWithAndErase(B, Src: &GEP, Dest: NewI);
921 return NewI;
922}
923
924Type *SPIRVEmitIntrinsicsImpl::getGEPTypeLogical(GetElementPtrInst *GEP) {
925
926 Type *CurType = GEP->getResultElementType();
927
928 bool Interrupted = walkLogicalAccessChain(
929 GEP&: *GEP, OnLiteralIndexing: [&CurType](Type *EltType, uint64_t Index) { CurType = EltType; },
930 OnDynamicIndexing: [&CurType](Type *EltType, Value *Index, uint64_t) { CurType = EltType; });
931
932 return Interrupted ? GEP->getResultElementType() : CurType;
933}
934
935Type *SPIRVEmitIntrinsicsImpl::getGEPType(GetElementPtrInst *Ref) {
936 if (getByteAddressingMultiplier(Ty: Ref->getSourceElementType()) &&
937 TM.getSubtargetImpl()->isLogicalSPIRV()) {
938 return getGEPTypeLogical(GEP: Ref);
939 }
940
941 Type *Ty = nullptr;
942 // TODO: not sure if GetElementPtrInst::getTypeAtIndex() does anything
943 // useful here
944 if (isNestedPointer(Ty: Ref->getSourceElementType())) {
945 Ty = Ref->getSourceElementType();
946 for (Use &U : drop_begin(RangeOrContainer: Ref->indices()))
947 Ty = GetElementPtrInst::getTypeAtIndex(Ty, Idx: U.get());
948 } else {
949 Ty = Ref->getResultElementType();
950 }
951 return Ty;
952}
953
954Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeHelper(
955 Value *I, SmallPtrSetImpl<Value *> &Visited, bool UnknownElemTypeI8,
956 bool IgnoreKnownType) {
957 // allow to pass nullptr as an argument
958 if (!I)
959 return nullptr;
960
961 // maybe already known
962 if (!IgnoreKnownType)
963 if (Type *KnownTy = GR->findDeducedElementType(Val: I))
964 return KnownTy;
965
966 // maybe a cycle
967 if (!Visited.insert(Ptr: I).second)
968 return nullptr;
969
970 // fallback value in case when we fail to deduce a type
971 Type *Ty = nullptr;
972 // look for known basic patterns of type inference
973 if (auto *Ref = dyn_cast<AllocaInst>(Val: I)) {
974 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getAllocatedType(), UnknownElemTypeI8);
975 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
976 Ty = getGEPType(Ref);
977 } else if (auto *SGEP = dyn_cast<StructuredGEPInst>(Val: I)) {
978 Ty = SGEP->getResultElementType();
979 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
980 Value *Op = Ref->getPointerOperand();
981 Type *KnownTy = GR->findDeducedElementType(Val: Op);
982 if (!KnownTy)
983 KnownTy = Op->getType();
984 if (Type *ElemTy = getPointeeType(Ty: KnownTy))
985 maybeAssignPtrType(Ty, Op: I, RefTy: ElemTy, UnknownElemTypeI8);
986 } else if (auto *Ref = dyn_cast<GlobalValue>(Val: I)) {
987 if (auto *Fn = dyn_cast<Function>(Val: Ref)) {
988 Ty = SPIRV::getOriginalFunctionType(F: *Fn);
989 GR->addDeducedElementType(Val: I, Ty);
990 } else {
991 Ty = deduceElementTypeByValueDeep(
992 ValueTy: Ref->getValueType(),
993 Operand: Ref->getNumOperands() > 0 ? Ref->getOperand(i: 0) : nullptr, Visited,
994 UnknownElemTypeI8);
995 }
996 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
997 Type *RefTy = deduceElementTypeHelper(I: Ref->getPointerOperand(), Visited,
998 UnknownElemTypeI8);
999 maybeAssignPtrType(Ty, Op: I, RefTy, UnknownElemTypeI8);
1000 } else if (auto *Ref = dyn_cast<IntToPtrInst>(Val: I)) {
1001 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getDestTy(), UnknownElemTypeI8);
1002 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1003 if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
1004 isPointerTy(T: Src) && isPointerTy(T: Dest))
1005 Ty = deduceElementTypeHelper(I: Ref->getOperand(i_nocapture: 0), Visited,
1006 UnknownElemTypeI8);
1007 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1008 Value *Op = Ref->getNewValOperand();
1009 if (isPointerTy(T: Op->getType()))
1010 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
1011 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1012 Value *Op = Ref->getValOperand();
1013 if (isPointerTy(T: Op->getType()))
1014 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
1015 } else if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1016 Type *BestTy = nullptr;
1017 unsigned MaxN = 1;
1018 DenseMap<Type *, unsigned> PhiTys;
1019 for (int i = Ref->getNumIncomingValues() - 1; i >= 0; --i) {
1020 Ty = deduceElementTypeByUsersDeep(Op: Ref->getIncomingValue(i), Visited,
1021 UnknownElemTypeI8);
1022 if (!Ty)
1023 continue;
1024 auto It = PhiTys.try_emplace(Key: Ty, Args: 1);
1025 if (!It.second) {
1026 ++It.first->second;
1027 if (It.first->second > MaxN) {
1028 MaxN = It.first->second;
1029 BestTy = Ty;
1030 }
1031 }
1032 }
1033 if (BestTy)
1034 Ty = BestTy;
1035 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1036 for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) {
1037 Ty = deduceElementTypeByUsersDeep(Op, Visited, UnknownElemTypeI8);
1038 if (Ty)
1039 break;
1040 }
1041 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
1042 static StringMap<unsigned> ResTypeByArg = {
1043 {"to_global", 0},
1044 {"to_local", 0},
1045 {"to_private", 0},
1046 {"__spirv_GenericCastToPtr_ToGlobal", 0},
1047 {"__spirv_GenericCastToPtr_ToLocal", 0},
1048 {"__spirv_GenericCastToPtr_ToPrivate", 0},
1049 {"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
1050 {"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
1051 {"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
1052 // TODO: maybe improve performance by caching demangled names
1053
1054 auto *II = dyn_cast<IntrinsicInst>(Val: I);
1055 if (II && (II->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer ||
1056 II->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) {
1057 auto *HandleType = cast<TargetExtType>(Val: II->getOperand(i_nocapture: 0)->getType());
1058 if (HandleType->getTargetExtName() == "spirv.Image" ||
1059 HandleType->getTargetExtName() == "spirv.SignedImage") {
1060 for (User *U : II->users()) {
1061 Ty = cast<Instruction>(Val: U)->getAccessType();
1062 if (Ty)
1063 break;
1064 }
1065 } else if (HandleType->getTargetExtName() == "spirv.VulkanBuffer") {
1066 // This call is supposed to index into an array
1067 Ty = HandleType->getTypeParameter(i: 0);
1068 if (II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1069 if (Ty->isArrayTy())
1070 Ty = Ty->getArrayElementType();
1071 else {
1072 assert(Ty && Ty->isStructTy());
1073 uint32_t Index =
1074 cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1))->getZExtValue();
1075 Ty = cast<StructType>(Val: Ty)->getElementType(N: Index);
1076 }
1077 }
1078 Ty = reconstitutePeeledArrayType(Ty);
1079 } else {
1080 llvm_unreachable("Unknown handle type for spv_resource_getpointer.");
1081 }
1082 } else if (II && II->getIntrinsicID() ==
1083 Intrinsic::spv_generic_cast_to_ptr_explicit) {
1084 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: 0), Visited,
1085 UnknownElemTypeI8);
1086 } else if (Function *CalledF = CI->getCalledFunction()) {
1087 std::string DemangledName =
1088 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1089 if (DemangledName.length() > 0)
1090 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName);
1091 auto AsArgIt = ResTypeByArg.find(Key: DemangledName);
1092 if (AsArgIt != ResTypeByArg.end())
1093 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: AsArgIt->second),
1094 Visited, UnknownElemTypeI8);
1095 else if (Type *KnownRetTy = GR->findDeducedElementType(Val: CalledF))
1096 Ty = KnownRetTy;
1097 }
1098 }
1099
1100 // remember the found relationship
1101 if (Ty && !IgnoreKnownType) {
1102 // specify nested types if needed, otherwise return unchanged
1103 GR->addDeducedElementType(Val: I, Ty: normalizeType(Ty));
1104 }
1105
1106 return Ty;
1107}
1108
1109// Re-create a type of the value if it has untyped pointer fields, also nested.
1110// Return the original value type if no corrections of untyped pointer
1111// information is found or needed.
1112Type *SPIRVEmitIntrinsicsImpl::deduceNestedTypeHelper(User *U,
1113 bool UnknownElemTypeI8) {
1114 SmallPtrSet<Value *, 0> Visited;
1115 return deduceNestedTypeHelper(U, Ty: U->getType(), Visited, UnknownElemTypeI8);
1116}
1117
1118Type *SPIRVEmitIntrinsicsImpl::deduceNestedTypeHelper(
1119 User *U, Type *OrigTy, SmallPtrSetImpl<Value *> &Visited,
1120 bool UnknownElemTypeI8) {
1121 if (!U)
1122 return OrigTy;
1123
1124 // maybe already known
1125 if (Type *KnownTy = GR->findDeducedCompositeType(Val: U))
1126 return KnownTy;
1127
1128 // maybe a cycle
1129 if (!Visited.insert(Ptr: U).second)
1130 return OrigTy;
1131
1132 if (isa<StructType>(Val: OrigTy)) {
1133 SmallVector<Type *> Tys;
1134 bool Change = false;
1135 for (unsigned i = 0; i < U->getNumOperands(); ++i) {
1136 Value *Op = U->getOperand(i);
1137 assert(Op && "Operands should not be null.");
1138 Type *OpTy = Op->getType();
1139 Type *Ty = OpTy;
1140 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1141 if (Type *NestedTy =
1142 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1143 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1144 } else {
1145 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1146 UnknownElemTypeI8);
1147 }
1148 Tys.push_back(Elt: Ty);
1149 Change |= Ty != OpTy;
1150 }
1151 if (Change) {
1152 Type *NewTy = StructType::create(Elements: Tys);
1153 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1154 return NewTy;
1155 }
1156 } else if (auto *ArrTy = dyn_cast<ArrayType>(Val: OrigTy)) {
1157 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1158 Type *OpTy = ArrTy->getElementType();
1159 Type *Ty = OpTy;
1160 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1161 if (Type *NestedTy =
1162 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1163 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1164 } else {
1165 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1166 UnknownElemTypeI8);
1167 }
1168 if (Ty != OpTy) {
1169 Type *NewTy = ArrayType::get(ElementType: Ty, NumElements: ArrTy->getNumElements());
1170 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1171 return NewTy;
1172 }
1173 }
1174 } else if (auto *VecTy = dyn_cast<VectorType>(Val: OrigTy)) {
1175 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1176 Type *OpTy = VecTy->getElementType();
1177 Type *Ty = OpTy;
1178 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1179 if (Type *NestedTy =
1180 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1181 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1182 } else {
1183 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1184 UnknownElemTypeI8);
1185 }
1186 if (Ty != OpTy) {
1187 Type *NewTy = VectorType::get(ElementType: Ty, EC: VecTy->getElementCount());
1188 GR->addDeducedCompositeType(Val: U, Ty: normalizeType(Ty: NewTy));
1189 return NewTy;
1190 }
1191 }
1192 }
1193
1194 return OrigTy;
1195}
1196
1197Type *SPIRVEmitIntrinsicsImpl::deduceElementType(Value *I,
1198 bool UnknownElemTypeI8) {
1199 if (Type *Ty = deduceElementTypeHelper(I, UnknownElemTypeI8))
1200 return Ty;
1201 if (!UnknownElemTypeI8)
1202 return nullptr;
1203 insertTodoType(Op: I);
1204 return IntegerType::getInt8Ty(C&: I->getContext());
1205}
1206
1207static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I,
1208 Value *PointerOperand) {
1209 Type *PointeeTy = GR->findDeducedElementType(Val: PointerOperand);
1210 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1211 return nullptr;
1212 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType());
1213 if (!PtrTy)
1214 return I->getType();
1215 if (Type *NestedTy = GR->findDeducedElementType(Val: I))
1216 return getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1217 return nullptr;
1218}
1219
1220// Try to deduce element type for a call base. Returns false if this is an
1221// indirect function invocation, and true otherwise.
1222bool SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeCalledFunction(
1223 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1224 Type *&KnownElemTy, bool &Incomplete) {
1225 Function *CalledF = CI->getCalledFunction();
1226 if (!CalledF)
1227 return false;
1228 std::string DemangledName =
1229 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1230 if (DemangledName.length() > 0 &&
1231 !StringRef(DemangledName).starts_with(Prefix: "llvm.")) {
1232 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *CalledF);
1233 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
1234 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
1235 if (Opcode == SPIRV::OpGroupAsyncCopy) {
1236 for (unsigned i = 0, PtrCnt = 0; i < CI->arg_size() && PtrCnt < 2; ++i) {
1237 Value *Op = CI->getArgOperand(i);
1238 if (!isPointerTy(T: Op->getType()))
1239 continue;
1240 ++PtrCnt;
1241 if (Type *ElemTy = GR->findDeducedElementType(Val: Op))
1242 KnownElemTy = ElemTy; // src will rewrite dest if both are defined
1243 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1244 }
1245 } else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) {
1246 if (CI->arg_size() == 0)
1247 return true;
1248 Value *Op = CI->getArgOperand(i: 0);
1249 if (!isPointerTy(T: Op->getType()))
1250 return true;
1251 switch (Opcode) {
1252 case SPIRV::OpAtomicFAddEXT:
1253 case SPIRV::OpAtomicFMinEXT:
1254 case SPIRV::OpAtomicFMaxEXT:
1255 case SPIRV::OpAtomicLoad:
1256 case SPIRV::OpAtomicCompareExchangeWeak:
1257 case SPIRV::OpAtomicCompareExchange:
1258 case SPIRV::OpAtomicExchange:
1259 case SPIRV::OpAtomicIAdd:
1260 case SPIRV::OpAtomicISub:
1261 case SPIRV::OpAtomicOr:
1262 case SPIRV::OpAtomicXor:
1263 case SPIRV::OpAtomicAnd:
1264 case SPIRV::OpAtomicUMin:
1265 case SPIRV::OpAtomicUMax:
1266 case SPIRV::OpAtomicSMin:
1267 case SPIRV::OpAtomicSMax: {
1268 KnownElemTy = isPointerTy(T: CI->getType()) ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1269 : CI->getType();
1270 if (!KnownElemTy)
1271 return true;
1272 Incomplete = isTodoType(Op);
1273 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1274 } break;
1275 case SPIRV::OpAtomicStore: {
1276 if (CI->arg_size() < 4)
1277 return true;
1278 Value *ValOp = CI->getArgOperand(i: 3);
1279 KnownElemTy = isPointerTy(T: ValOp->getType())
1280 ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1281 : ValOp->getType();
1282 if (!KnownElemTy)
1283 return true;
1284 Incomplete = isTodoType(Op);
1285 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1286 } break;
1287 }
1288 }
1289 }
1290 return true;
1291}
1292
1293// Try to deduce element type for a function pointer.
1294void SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeFunctionPointer(
1295 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1296 Type *&KnownElemTy, bool IsPostprocessing) {
1297 Value *Op = CI->getCalledOperand();
1298 if (!Op || !isPointerTy(T: Op->getType()))
1299 return;
1300 Ops.push_back(Elt: std::make_pair(x&: Op, y: std::numeric_limits<unsigned>::max()));
1301 FunctionType *FTy = SPIRV::getOriginalFunctionType(CB: *CI);
1302 bool IsNewFTy = false, IsIncomplete = false;
1303 SmallVector<Type *, 4> ArgTys;
1304 for (auto &&[ParmIdx, Arg] : llvm::enumerate(First: CI->args())) {
1305 Type *ArgTy = Arg->getType();
1306 if (ArgTy->isPointerTy()) {
1307 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg)) {
1308 IsNewFTy = true;
1309 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
1310 if (isTodoType(Op: Arg))
1311 IsIncomplete = true;
1312 } else {
1313 IsIncomplete = true;
1314 }
1315 } else {
1316 ArgTy = FTy->getFunctionParamType(i: ParmIdx);
1317 }
1318 ArgTys.push_back(Elt: ArgTy);
1319 }
1320 Type *RetTy = FTy->getReturnType();
1321 if (CI->getType()->isPointerTy()) {
1322 if (Type *ElemTy = GR->findDeducedElementType(Val: CI)) {
1323 IsNewFTy = true;
1324 RetTy =
1325 getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: CI->getType()));
1326 if (isTodoType(Op: CI))
1327 IsIncomplete = true;
1328 } else {
1329 IsIncomplete = true;
1330 }
1331 }
1332 if (!IsPostprocessing && IsIncomplete)
1333 insertTodoType(Op);
1334 KnownElemTy =
1335 IsNewFTy ? FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: FTy->isVarArg()) : FTy;
1336}
1337
1338bool SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeFunctionRet(
1339 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
1340 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing,
1341 Type *&KnownElemTy, Value *Op, Function *F) {
1342 KnownElemTy = GR->findDeducedElementType(Val: F);
1343 if (KnownElemTy)
1344 return false;
1345 if (Type *OpElemTy = GR->findDeducedElementType(Val: Op)) {
1346 OpElemTy = normalizeType(Ty: OpElemTy);
1347 GR->addDeducedElementType(Val: F, Ty: OpElemTy);
1348 GR->addReturnType(
1349 ArgF: F, DerivedTy: TypedPointerType::get(ElementType: OpElemTy,
1350 AddressSpace: getPointerAddressSpace(T: F->getReturnType())));
1351 // non-recursive update of types in function uses
1352 DenseSet<std::pair<Value *, Value *>> VisitedSubst{std::make_pair(x&: I, y&: Op)};
1353 for (User *U : F->users()) {
1354 CallInst *CI = dyn_cast<CallInst>(Val: U);
1355 if (!CI || CI->getCalledFunction() != F)
1356 continue;
1357 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: CI)) {
1358 if (Type *PrevElemTy = GR->findDeducedElementType(Val: CI)) {
1359 GR->updateAssignType(AssignCI, Arg: CI,
1360 OfType: getNormalizedPoisonValue(Ty: OpElemTy));
1361 propagateElemType(Op: CI, ElemTy: PrevElemTy, VisitedSubst);
1362 }
1363 }
1364 }
1365 // Non-recursive update of types in the function uncomplete returns.
1366 // This may happen just once per a function, the latch is a pair of
1367 // findDeducedElementType(F) / addDeducedElementType(F, ...).
1368 // With or without the latch it is a non-recursive call due to
1369 // IncompleteRets set to nullptr in this call.
1370 if (IncompleteRets)
1371 for (Instruction *IncompleteRetI : *IncompleteRets)
1372 deduceOperandElementType(I: IncompleteRetI, IncompleteRets: nullptr, AskOps,
1373 IsPostprocessing);
1374 } else if (IncompleteRets) {
1375 IncompleteRets->insert(Ptr: I);
1376 }
1377 TypeValidated.insert(Ptr: I);
1378 return true;
1379}
1380
1381// If the Instruction has Pointer operands with unresolved types, this function
1382// tries to deduce them. If the Instruction has Pointer operands with known
1383// types which differ from expected, this function tries to insert a bitcast to
1384// resolve the issue.
1385void SPIRVEmitIntrinsicsImpl::deduceOperandElementType(
1386 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
1387 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing) {
1388 SmallVector<std::pair<Value *, unsigned>> Ops;
1389 Type *KnownElemTy = nullptr;
1390 bool Incomplete = false;
1391 // look for known basic patterns of type inference
1392 if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1393 if (!isPointerTy(T: I->getType()) ||
1394 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1395 return;
1396 Incomplete = isTodoType(Op: I);
1397 for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
1398 Value *Op = Ref->getIncomingValue(i);
1399 if (isPointerTy(T: Op->getType()))
1400 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1401 }
1402 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1403 KnownElemTy = GR->findDeducedElementType(Val: I);
1404 if (!KnownElemTy)
1405 return;
1406 Incomplete = isTodoType(Op: I);
1407 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(), y: 0));
1408 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1409 if (!isPointerTy(T: I->getType()))
1410 return;
1411 KnownElemTy = GR->findDeducedElementType(Val: I);
1412 if (!KnownElemTy)
1413 return;
1414 Incomplete = isTodoType(Op: I);
1415 Ops.push_back(Elt: std::make_pair(x: Ref->getOperand(i_nocapture: 0), y: 0));
1416 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
1417 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1418 return;
1419 KnownElemTy = Ref->getSourceElementType();
1420 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1421 y: GetElementPtrInst::getPointerOperandIndex()));
1422 } else if (auto *Ref = dyn_cast<StructuredGEPInst>(Val: I)) {
1423 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1424 return;
1425 KnownElemTy = Ref->getBaseType();
1426 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1427 y: StructuredGEPInst::getPointerOperandIndex()));
1428 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
1429 KnownElemTy = I->getType();
1430 if (isUntypedPointerTy(T: KnownElemTy)) {
1431 // A T** loaded back from its alloca comes out opaque, dropping type info.
1432 // When the load is a pointer-to-pointer, type the alloca as that pointer.
1433 Type *LoadedElemTy = GR->findDeducedElementType(Val: I);
1434 if (!LoadedElemTy || !isPointerTyOrWrapper(Ty: LoadedElemTy))
1435 return;
1436 Value *Root = Ref->getPointerOperand()->stripPointerCasts();
1437 if (!isa<AllocaInst>(Val: Root))
1438 return;
1439 KnownElemTy = getTypedPointerWrapper(ElemTy: LoadedElemTy,
1440 AS: getPointerAddressSpace(T: KnownElemTy));
1441 }
1442 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1443 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1444 return;
1445 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1446 y: LoadInst::getPointerOperandIndex()));
1447 } else if (auto *Ref = dyn_cast<StoreInst>(Val: I)) {
1448 if (!(KnownElemTy =
1449 reconstructType(Op: Ref->getValueOperand(), UnknownElemTypeI8: false, IsPostprocessing)))
1450 return;
1451 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1452 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1453 return;
1454 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1455 y: StoreInst::getPointerOperandIndex()));
1456 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1457 KnownElemTy = isPointerTy(T: I->getType())
1458 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1459 : I->getType();
1460 if (!KnownElemTy)
1461 return;
1462 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1463 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1464 y: AtomicCmpXchgInst::getPointerOperandIndex()));
1465 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1466 KnownElemTy = isPointerTy(T: I->getType())
1467 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1468 : I->getType();
1469 if (!KnownElemTy)
1470 return;
1471 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1472 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1473 y: AtomicRMWInst::getPointerOperandIndex()));
1474 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1475 if (!isPointerTy(T: I->getType()) ||
1476 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1477 return;
1478 Incomplete = isTodoType(Op: I);
1479 for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
1480 Value *Op = Ref->getOperand(i_nocapture: i);
1481 if (isPointerTy(T: Op->getType()))
1482 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1483 }
1484 } else if (auto *Ref = dyn_cast<ReturnInst>(Val: I)) {
1485 if (!isPointerTy(T: CurrF->getReturnType()))
1486 return;
1487 Value *Op = Ref->getReturnValue();
1488 if (!Op)
1489 return;
1490 if (deduceOperandElementTypeFunctionRet(I, IncompleteRets, AskOps,
1491 IsPostprocessing, KnownElemTy, Op,
1492 F: CurrF))
1493 return;
1494 Incomplete = isTodoType(Op: CurrF);
1495 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1496 } else if (auto *Ref = dyn_cast<ICmpInst>(Val: I)) {
1497 if (!isPointerTy(T: Ref->getOperand(i_nocapture: 0)->getType()))
1498 return;
1499 Value *Op0 = Ref->getOperand(i_nocapture: 0);
1500 Value *Op1 = Ref->getOperand(i_nocapture: 1);
1501 bool Incomplete0 = isTodoType(Op: Op0);
1502 bool Incomplete1 = isTodoType(Op: Op1);
1503 Type *ElemTy1 = GR->findDeducedElementType(Val: Op1);
1504 Type *ElemTy0 = (Incomplete0 && !Incomplete1 && ElemTy1)
1505 ? nullptr
1506 : GR->findDeducedElementType(Val: Op0);
1507 if (ElemTy0) {
1508 KnownElemTy = ElemTy0;
1509 Incomplete = Incomplete0;
1510 Ops.push_back(Elt: std::make_pair(x&: Op1, y: 1));
1511 } else if (ElemTy1) {
1512 KnownElemTy = ElemTy1;
1513 Incomplete = Incomplete1;
1514 Ops.push_back(Elt: std::make_pair(x&: Op0, y: 0));
1515 }
1516 } else if (CallInst *CI = dyn_cast<CallInst>(Val: I)) {
1517 if (!CI->isIndirectCall())
1518 deduceOperandElementTypeCalledFunction(CI, Ops, KnownElemTy, Incomplete);
1519 else if (HaveFunPtrs)
1520 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy,
1521 IsPostprocessing);
1522 }
1523
1524 // There is no enough info to deduce types or all is valid.
1525 if (!KnownElemTy || Ops.size() == 0)
1526 return;
1527
1528 LLVMContext &Ctx = CurrF->getContext();
1529 IRBuilder<> B(Ctx);
1530 for (auto &OpIt : Ops) {
1531 Value *Op = OpIt.first;
1532 if (AskOps && !AskOps->contains(Ptr: Op))
1533 continue;
1534 Type *AskTy = nullptr;
1535 CallInst *AskCI = nullptr;
1536 if (IsPostprocessing && AskOps) {
1537 AskTy = GR->findDeducedElementType(Val: Op);
1538 AskCI = GR->findAssignPtrTypeInstr(Val: Op);
1539 assert(AskTy && AskCI);
1540 }
1541 Type *Ty = AskTy ? AskTy : GR->findDeducedElementType(Val: Op);
1542 if (Ty == KnownElemTy)
1543 continue;
1544 Value *OpTyVal = getNormalizedPoisonValue(Ty: KnownElemTy);
1545 Type *OpTy = Op->getType();
1546 // Do not let a non-pointer element type clobber an already-deduced pointer
1547 // pointee.
1548 bool WouldClobberPtrWithNonPtr = Ty && isPointerTyOrWrapper(Ty) &&
1549 !isPointerTyOrWrapper(Ty: KnownElemTy) &&
1550 tracesToPointerAlloca(V: Op);
1551 if (Op->hasUseList() && !WouldClobberPtrWithNonPtr &&
1552 (!Ty || AskTy || isUntypedPointerTy(T: Ty) || isTodoType(Op))) {
1553 Type *PrevElemTy = GR->findDeducedElementType(Val: Op);
1554 GR->addDeducedElementType(Val: Op, Ty: normalizeType(Ty: KnownElemTy));
1555 // check if KnownElemTy is complete
1556 if (!Incomplete)
1557 eraseTodoType(Op);
1558 else if (!IsPostprocessing)
1559 insertTodoType(Op);
1560 // check if there is existing Intrinsic::spv_assign_ptr_type instruction
1561 CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Val: Op);
1562 if (AssignCI == nullptr) {
1563 Instruction *User = dyn_cast<Instruction>(Val: Op->use_begin()->get());
1564 setInsertPointSkippingPhis(B, I: User ? User->getNextNode() : I);
1565 CallInst *CI =
1566 buildIntrWithMD(IntrID: Intrinsic::spv_assign_ptr_type, Types: {OpTy}, Arg: OpTyVal, Arg2: Op,
1567 Imms: {B.getInt32(C: getPointerAddressSpace(T: OpTy))}, B);
1568 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: CI);
1569 } else {
1570 GR->updateAssignType(AssignCI, Arg: Op, OfType: OpTyVal);
1571 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1572 std::make_pair(x&: I, y&: Op)};
1573 propagateElemTypeRec(Op, PtrElemTy: KnownElemTy, CastElemTy: PrevElemTy, VisitedSubst);
1574 }
1575 } else {
1576 eraseTodoType(Op);
1577 CallInst *PtrCastI =
1578 buildSpvPtrcast(F: I->getParent()->getParent(), Op, ElemTy: KnownElemTy);
1579 if (OpIt.second == std::numeric_limits<unsigned>::max())
1580 dyn_cast<CallInst>(Val: I)->setCalledOperand(PtrCastI);
1581 else
1582 I->setOperand(i: OpIt.second, Val: PtrCastI);
1583 }
1584 }
1585 TypeValidated.insert(Ptr: I);
1586}
1587
1588void SPIRVEmitIntrinsicsImpl::replaceMemInstrUses(Instruction *Old,
1589 Instruction *New,
1590 IRBuilder<> &B) {
1591 while (!Old->user_empty()) {
1592 auto *U = Old->user_back();
1593 if (isAssignTypeInstr(I: U)) {
1594 B.SetInsertPoint(U);
1595 SmallVector<Value *, 2> Args = {New, U->getOperand(i: 1)};
1596 CallInst *AssignCI = B.CreateIntrinsicWithoutFolding(
1597 ID: Intrinsic::spv_assign_type, OverloadTypes: {New->getType()}, Args);
1598 GR->addAssignPtrTypeInstr(Val: New, AssignPtrTyCI: AssignCI);
1599 U->eraseFromParent();
1600 } else if (isMemInstrToReplace(I: U) || isa<ReturnInst>(Val: U) ||
1601 isa<CallInst>(Val: U)) {
1602 U->replaceUsesOfWith(From: Old, To: New);
1603 // For a `llvm.spv.abort` call whose composite message argument was
1604 // rewritten to a value-id (i32), also retarget the call to a matching
1605 // intrinsic declaration so the IR verifier is satisfied. The SPIR-V
1606 // type of the value is tracked via the GlobalRegistry, so the selector
1607 // still emits OpAbortKHR with the original composite type.
1608 if (auto *CI = dyn_cast<CallInst>(Val: U);
1609 CI && CI->getIntrinsicID() == Intrinsic::spv_abort) {
1610 Type *NewArgTy = New->getType();
1611 Type *ExpectedArgTy = CI->getFunctionType()->getParamType(i: 0);
1612 if (NewArgTy != ExpectedArgTy) {
1613 Module *M = CI->getModule();
1614 Function *NewF = Intrinsic::getOrInsertDeclaration(
1615 M, id: Intrinsic::spv_abort, OverloadTys: {NewArgTy});
1616 CI->setCalledFunction(NewF);
1617 }
1618 }
1619 } else if (isa<PHINode>(Val: U) || isa<SelectInst>(Val: U) || isa<FreezeInst>(Val: U)) {
1620 // Aggregate-typed PHIs, selects and freezes have already been mutated to
1621 // the i32 value-id type up front in runOnFunction, so only the operand
1622 // needs replacing here; their extractvalue users are lowered to
1623 // spv_extractv by visitExtractValueInst.
1624 assert(U->getType() == New->getType() &&
1625 "aggregate PHI/select/freeze should have been mutated to value-id "
1626 "type");
1627 U->replaceUsesOfWith(From: Old, To: New);
1628 } else {
1629 llvm_unreachable("illegal aggregate intrinsic user");
1630 }
1631 }
1632 New->copyMetadata(SrcInst: *Old);
1633 Old->eraseFromParent();
1634}
1635
1636// Lower a poison or undef Op to its placeholder intrinsic.
1637Value *SPIRVEmitIntrinsicsImpl::lowerUndefOrPoison(Value *Op, IRBuilder<> &B,
1638 bool HasPoisonExt) {
1639 auto *UV = dyn_cast<UndefValue>(Val: Op);
1640 if (!UV)
1641 return nullptr;
1642
1643 bool AsPoison = HasPoisonExt && isa<PoisonValue>(Val: UV);
1644 if (isa<PoisonValue>(Val: UV) && !HasPoisonExt)
1645 LLVM_DEBUG(dbgs() << "SPV_KHR_poison_freeze is not enabled. Poison is "
1646 "lowered as undef\n");
1647
1648 Intrinsic::ID IID = AsPoison ? Intrinsic::spv_poison : Intrinsic::spv_undef;
1649 Type *Ty = UV->getType();
1650
1651 // Aggregates use an i32-result placeholder with the real type kept in
1652 // AggrConstTypes and scalar poison uses a type-overloaded one.
1653 if (Ty->isAggregateType()) {
1654 auto *Call =
1655 AsPoison ? B.CreateIntrinsicWithoutFolding(ID: IID, OverloadTypes: {B.getInt32Ty()}, Args: {})
1656 : B.CreateIntrinsicWithoutFolding(ID: IID, Args: {});
1657 AggrConsts[Call] = UV;
1658 AggrConstTypes[Call] = Ty;
1659 return Call;
1660 }
1661
1662 if (AsPoison)
1663 return B.CreateIntrinsic(ID: IID, OverloadTypes: {Ty}, Args: {});
1664 return nullptr;
1665}
1666
1667// Replace aggregate undef or poison operands and extension-enabled scalar
1668// poison operands with placeholder intrinsics. Scalar undef is left as is. See
1669// lowerUndefOrPoison.
1670void SPIRVEmitIntrinsicsImpl::preprocessUndefsAndPoisons(IRBuilder<> &B) {
1671 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*CurrF);
1672 bool HasPoisonExt =
1673 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
1674
1675 SmallVector<Instruction *, 16> Insts;
1676 for (auto &I : instructions(F: CurrF))
1677 Insts.push_back(Elt: &I);
1678
1679 for (Instruction *I : Insts) {
1680 bool BPrepared = false;
1681 auto *Phi = dyn_cast<PHINode>(Val: I);
1682 for (unsigned Idx = 0; Idx < I->getNumOperands(); ++Idx) {
1683 Value *Op = I->getOperand(i: Idx);
1684 if (!isa<UndefValue>(Val: Op) || Op->getType()->isMetadataTy())
1685 continue;
1686 bool IsScalar = !Op->getType()->isAggregateType();
1687 bool AsPoison = HasPoisonExt && isa<PoisonValue>(Val: Op);
1688 // Scalar undef or extensionless scalar poison is directly translatable.
1689 if (IsScalar && !AsPoison)
1690 continue;
1691 // Scalar poison in a phi materializes in the incoming block. Everything
1692 // else materializes right before I.
1693 if (IsScalar && Phi)
1694 B.SetInsertPoint(Phi->getIncomingBlock(i: Idx)->getTerminator());
1695 else if (!BPrepared) {
1696 setInsertPointSkippingPhis(B, I);
1697 BPrepared = true;
1698 }
1699 if (Value *Repl = lowerUndefOrPoison(Op, B, HasPoisonExt))
1700 I->setOperand(i: Idx, Val: Repl);
1701 }
1702 }
1703}
1704
1705// Simplify addrspacecast(null) instructions to ConstantPointerNull of the
1706// target type. Casting null always yields null, and this avoids SPIR-V
1707// lowering issues where the null gets typed as an integer instead of a
1708// pointer.
1709void SPIRVEmitIntrinsicsImpl::simplifyNullAddrSpaceCasts() {
1710 for (Instruction &I : make_early_inc_range(Range: instructions(F: CurrF)))
1711 if (auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: &I))
1712 if (isa<ConstantPointerNull>(Val: ASC->getPointerOperand())) {
1713 ASC->replaceAllUsesWith(
1714 V: ConstantPointerNull::get(T: cast<PointerType>(Val: ASC->getType())));
1715 ASC->eraseFromParent();
1716 }
1717}
1718
1719// True for an aggregate value the legalizer splits into a multi-result op
1720// (with.overflow -> G_UADDO, frexp/sincos/modf -> G_FFREXP/...). These keep a
1721// genuine multi-register result; all other aggregates become a single value-id.
1722static bool isMultiRegisterAggregate(Value *V) {
1723 if (!V->getType()->isAggregateType())
1724 return false;
1725 return isa<IntrinsicInst>(Val: V) && !isSpvIntrinsic(Arg: V);
1726}
1727
1728// True for an aggregate PHI/select/freeze, which is lowered to a single
1729// value-id.
1730static bool isAggregateValueIdInstr(const Instruction &I) {
1731 return (isa<PHINode>(Val: I) || isa<SelectInst>(Val: I) || isa<FreezeInst>(Val: I)) &&
1732 I.getType()->isAggregateType();
1733}
1734
1735// Give each multi-register aggregate arm of an aggregate PHI/select/freeze a
1736// single value-id by reassembling it with extractvalue + insertvalue, so the
1737// arm matches the result once it is mutated to a value-id.
1738void SPIRVEmitIntrinsicsImpl::insertCompositeAggregateArms(Instruction *I,
1739 IRBuilder<> &B) {
1740 auto *Phi = dyn_cast<PHINode>(Val: I);
1741 for (Use &U : I->operands()) {
1742 Value *Op = U.get();
1743 if (!isMultiRegisterAggregate(V: Op))
1744 continue;
1745 // A PHI arm materializes in its incoming block, everything else after the
1746 // producer.
1747 if (Phi)
1748 B.SetInsertPoint(Phi->getIncomingBlock(U)->getTerminator());
1749 else
1750 setInsertPointAfterDef(B, I: cast<Instruction>(Val: Op));
1751 auto *AggrTy = cast<StructType>(Val: Op->getType());
1752 Value *Composite = PoisonValue::get(T: AggrTy);
1753 for (unsigned Idx = 0, E = AggrTy->getNumElements(); Idx != E; ++Idx) {
1754 Value *Field = B.CreateExtractValue(Agg: Op, Idxs: Idx);
1755 Composite = B.CreateInsertValue(Agg: Composite, Val: Field, Idxs: Idx);
1756 }
1757 U.set(Composite);
1758 }
1759}
1760
1761void SPIRVEmitIntrinsicsImpl::preprocessCompositeConstants(IRBuilder<> &B) {
1762 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*CurrF);
1763 bool HasPoisonExt =
1764 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
1765 std::queue<Instruction *> Worklist;
1766 for (auto &I : instructions(F: CurrF))
1767 Worklist.push(x: &I);
1768
1769 while (!Worklist.empty()) {
1770 auto *I = Worklist.front();
1771 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
1772 assert(I);
1773 bool KeepInst = false;
1774 for (const auto &Op : I->operands()) {
1775 Constant *AggrConst = nullptr;
1776 Type *ResTy = nullptr;
1777 if (auto *COp = dyn_cast<ConstantVector>(Val: Op)) {
1778 AggrConst = COp;
1779 ResTy = COp->getType();
1780 } else if (auto *COp = dyn_cast<ConstantArray>(Val: Op)) {
1781 AggrConst = COp;
1782 ResTy = B.getInt32Ty();
1783 } else if (auto *COp = dyn_cast<ConstantStruct>(Val: Op)) {
1784 AggrConst = COp;
1785 ResTy = B.getInt32Ty();
1786 } else if (auto *COp = dyn_cast<ConstantDataArray>(Val: Op)) {
1787 AggrConst = COp;
1788 ResTy = B.getInt32Ty();
1789 } else if (auto *COp = dyn_cast<ConstantAggregateZero>(Val: Op)) {
1790 AggrConst = COp;
1791 ResTy = Op->getType()->isVectorTy() ? COp->getType() : B.getInt32Ty();
1792 }
1793 if (AggrConst) {
1794 auto PrepareInsert = [&]() {
1795 if (BPrepared)
1796 return;
1797 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
1798 : B.SetInsertPoint(I);
1799 BPrepared = true;
1800 };
1801 SmallVector<Value *> Args;
1802 if (auto *COp = dyn_cast<ConstantDataSequential>(Val: Op))
1803 for (unsigned i = 0; i < COp->getNumElements(); ++i)
1804 Args.push_back(Elt: COp->getElementAsConstant(i));
1805 else
1806 for (Value *Op : AggrConst->operands()) {
1807 // Simplify addrspacecast(null) to null in the target address space
1808 // so that null pointers get the correct pointer type when lowered.
1809 if (auto *CE = dyn_cast<ConstantExpr>(Val: Op);
1810 CE && CE->getOpcode() == Instruction::AddrSpaceCast &&
1811 isa<ConstantPointerNull>(Val: CE->getOperand(i_nocapture: 0)))
1812 Op = ConstantPointerNull::get(T: cast<PointerType>(Val: CE->getType()));
1813 // Undef or poison nested in a constant aggregate is not a direct
1814 // instruction operand, so preprocessUndefsAndPoisons() misses it.
1815 // An unlowered aggregate one would reach IRTranslator as an
1816 // untranslatable spv_const_composite operand.
1817 if (isa<UndefValue>(Val: Op)) {
1818 PrepareInsert();
1819 if (Value *Repl = lowerUndefOrPoison(Op, B, HasPoisonExt))
1820 Op = Repl;
1821 }
1822 Args.push_back(Elt: Op);
1823 }
1824 PrepareInsert();
1825 auto *CI = B.CreateIntrinsicWithoutFolding(
1826 ID: Intrinsic::spv_const_composite, OverloadTypes: {ResTy}, Args: {Args});
1827 Worklist.push(x: CI);
1828 I->replaceUsesOfWith(From: Op, To: CI);
1829 KeepInst = true;
1830 AggrConsts[CI] = AggrConst;
1831 AggrConstTypes[CI] = deduceNestedTypeHelper(U: AggrConst, UnknownElemTypeI8: false);
1832 }
1833 }
1834 if (!KeepInst)
1835 Worklist.pop();
1836 }
1837}
1838
1839static void createDecorationIntrinsic(Instruction *I, MDNode *Node,
1840 IRBuilder<> &B) {
1841 LLVMContext &Ctx = I->getContext();
1842 setInsertPointAfterDef(B, I);
1843 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
1844 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: {Node}))});
1845}
1846
1847static void createRoundingModeDecoration(Instruction *I,
1848 unsigned RoundingModeDeco,
1849 IRBuilder<> &B) {
1850 LLVMContext &Ctx = I->getContext();
1851 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1852 MDNode *RoundingModeNode = MDNode::get(
1853 Context&: Ctx,
1854 MDs: {ConstantAsMetadata::get(
1855 C: ConstantInt::get(Ty: Int32Ty, V: SPIRV::Decoration::FPRoundingMode)),
1856 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: RoundingModeDeco))});
1857 createDecorationIntrinsic(I, Node: RoundingModeNode, B);
1858}
1859
1860static void createSaturatedConversionDecoration(Instruction *I,
1861 IRBuilder<> &B) {
1862 LLVMContext &Ctx = I->getContext();
1863 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1864 MDNode *SaturatedConversionNode =
1865 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1866 Ty: Int32Ty, V: SPIRV::Decoration::SaturatedConversion))});
1867 createDecorationIntrinsic(I, Node: SaturatedConversionNode, B);
1868}
1869
1870static void addSaturatedDecorationToIntrinsic(Instruction *I, IRBuilder<> &B) {
1871 if (match(V: I, P: m_AnyIntrinsic<Intrinsic::fptosi_sat, Intrinsic::fptoui_sat>()))
1872 createSaturatedConversionDecoration(I, B);
1873}
1874
1875Instruction *SPIRVEmitIntrinsicsImpl::visitCallInst(CallInst &Call) {
1876 if (!Call.isInlineAsm())
1877 return &Call;
1878
1879 LLVMContext &Ctx = CurrF->getContext();
1880 // TODO: this does not retain elementtype info for memory constraints, which
1881 // in turn means that we lower them into pointers to i8, rather than
1882 // pointers to elementtype; this can be fixed during reverse translation
1883 // but we should correct it here, possibly by tweaking the function
1884 // type to take TypedPointerType args.
1885 Constant *TyC = UndefValue::get(T: SPIRV::getOriginalFunctionType(CB: Call));
1886 MDString *ConstraintString =
1887 MDString::get(Context&: Ctx, Str: SPIRV::getOriginalAsmConstraints(CB: Call));
1888 SmallVector<Value *> Args = {
1889 buildMD(Arg: TyC),
1890 MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: ConstraintString))};
1891 for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++)
1892 Args.push_back(Elt: Call.getArgOperand(i: OpIdx));
1893
1894 IRBuilder<> B(Call.getParent());
1895 B.SetInsertPoint(&Call);
1896 B.CreateIntrinsic(ID: Intrinsic::spv_inline_asm, Args: {Args});
1897 return &Call;
1898}
1899
1900// Use a tip about rounding mode to create a decoration.
1901void SPIRVEmitIntrinsicsImpl::useRoundingMode(ConstrainedFPIntrinsic *FPI,
1902 IRBuilder<> &B) {
1903 std::optional<RoundingMode> RM = FPI->getRoundingMode();
1904 if (!RM.has_value())
1905 return;
1906 unsigned RoundingModeDeco = std::numeric_limits<unsigned>::max();
1907 switch (RM.value()) {
1908 default:
1909 // ignore unknown rounding modes
1910 break;
1911 case RoundingMode::NearestTiesToEven:
1912 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTE;
1913 break;
1914 case RoundingMode::TowardNegative:
1915 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTN;
1916 break;
1917 case RoundingMode::TowardPositive:
1918 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTP;
1919 break;
1920 case RoundingMode::TowardZero:
1921 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTZ;
1922 break;
1923 case RoundingMode::Dynamic:
1924 case RoundingMode::NearestTiesToAway:
1925 // TODO: check if supported
1926 break;
1927 }
1928 if (RoundingModeDeco == std::numeric_limits<unsigned>::max())
1929 return;
1930 // Convert the tip about rounding mode into a decoration record.
1931 createRoundingModeDecoration(I: FPI, RoundingModeDeco, B);
1932}
1933
1934Instruction *SPIRVEmitIntrinsicsImpl::visitSwitchInst(SwitchInst &I) {
1935 BasicBlock *ParentBB = I.getParent();
1936 Function *F = ParentBB->getParent();
1937 IRBuilder<> B(ParentBB);
1938 B.SetInsertPoint(&I);
1939 SmallVector<Value *, 4> Args;
1940 SmallVector<BasicBlock *> BBCases;
1941 Args.push_back(Elt: I.getCondition());
1942 BBCases.push_back(Elt: I.getDefaultDest());
1943 Args.push_back(Elt: BlockAddress::get(F, BB: I.getDefaultDest()));
1944 for (auto &Case : I.cases()) {
1945 Args.push_back(Elt: Case.getCaseValue());
1946 BBCases.push_back(Elt: Case.getCaseSuccessor());
1947 Args.push_back(Elt: BlockAddress::get(F, BB: Case.getCaseSuccessor()));
1948 }
1949 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
1950 ID: Intrinsic::spv_switch, OverloadTypes: {I.getOperand(i_nocapture: 0)->getType()}, Args: {Args});
1951 // remove switch to avoid its unneeded and undesirable unwrap into branches
1952 // and conditions
1953 replaceAllUsesWith(Src: &I, Dest: NewI);
1954 I.eraseFromParent();
1955 // insert artificial and temporary instruction to preserve valid CFG,
1956 // it will be removed after IR translation pass
1957 B.SetInsertPoint(ParentBB);
1958 IndirectBrInst *BrI = B.CreateIndirectBr(
1959 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: ParentBB->getContext())),
1960 NumDests: BBCases.size());
1961 for (BasicBlock *BBCase : BBCases)
1962 BrI->addDestination(Dest: BBCase);
1963 return BrI;
1964}
1965
1966static bool isFirstIndexZero(const GetElementPtrInst *GEP) {
1967 return GEP->getNumIndices() > 0 && match(V: GEP->getOperand(i_nocapture: 1), P: m_Zero());
1968}
1969
1970Instruction *SPIRVEmitIntrinsicsImpl::visitIntrinsicInst(IntrinsicInst &I) {
1971 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
1972 if (!SGEP)
1973 return &I;
1974
1975 IRBuilder<> B(I.getParent());
1976 B.SetInsertPoint(&I);
1977 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
1978 SmallVector<Value *, 4> Args;
1979 Args.push_back(/* inBounds= */ Elt: B.getInt1(V: true));
1980 Args.push_back(Elt: I.getOperand(i_nocapture: 0));
1981 Args.push_back(/* zero index */ Elt: B.getInt32(C: 0));
1982 for (unsigned J = 0; J < SGEP->getNumIndices(); ++J)
1983 Args.push_back(Elt: SGEP->getIndexOperand(Index: J));
1984
1985 Instruction *NewI =
1986 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: Types, Args);
1987 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
1988 return NewI;
1989}
1990
1991Instruction *
1992SPIRVEmitIntrinsicsImpl::visitGetElementPtrInst(GetElementPtrInst &I) {
1993 IRBuilder<> B(I.getParent());
1994 B.SetInsertPoint(&I);
1995
1996 // OpPtrAccessChain requires a scalar pointer result; scalarize per-lane
1997 // GEPs that return <N x ptr> and rebuild the vector via insertelement.
1998 if (auto *RetVTy = dyn_cast<FixedVectorType>(Val: I.getType())) {
1999 unsigned N = RetVTy->getNumElements();
2000 Value *PtrOp = I.getPointerOperand();
2001 bool PtrIsVec = isa<VectorType>(Val: PtrOp->getType());
2002 Type *ResultPtrTy = RetVTy->getElementType();
2003 Type *ScalarPtrTy = PtrOp->getType()->getScalarType();
2004 SmallVector<Type *, 2> GepTypes = {ResultPtrTy, ScalarPtrTy};
2005 Value *InBounds = B.getInt1(V: I.isInBounds());
2006 Type *LanePointeeTy = getGEPType(Ref: &I);
2007 Type *SrcElemTy = I.getSourceElementType();
2008
2009 // Pin the lane pointee type on the vector operand and on each extracted
2010 // lane so the prelegalizer wraps them as OpTypeVector/OpTypePointer of
2011 // the right element type instead of defaulting to i8.
2012 if (PtrIsVec)
2013 GR->buildAssignPtr(B, ElemTy: SrcElemTy, Arg: PtrOp);
2014
2015 Value *VecResult = PoisonValue::get(T: RetVTy);
2016 for (unsigned Lane = 0; Lane < N; ++Lane) {
2017 Value *LaneIdx = B.getInt32(C: Lane);
2018 Value *ScalarPtr = PtrOp;
2019 if (PtrIsVec) {
2020 SmallVector<Type *, 3> ExtractTypes = {ScalarPtrTy, PtrOp->getType(),
2021 LaneIdx->getType()};
2022 ScalarPtr = B.CreateIntrinsic(ID: Intrinsic::spv_extractelt, OverloadTypes: {ExtractTypes},
2023 Args: {PtrOp, LaneIdx});
2024 GR->buildAssignPtr(B, ElemTy: SrcElemTy, Arg: ScalarPtr);
2025 }
2026 SmallVector<Value *, 4> Args;
2027 Args.push_back(Elt: InBounds);
2028 Args.push_back(Elt: ScalarPtr);
2029 for (Value *Idx : I.indices()) {
2030 if (isa<VectorType>(Val: Idx->getType()))
2031 Args.push_back(Elt: B.CreateExtractElement(Vec: Idx, Idx: LaneIdx));
2032 else
2033 Args.push_back(Elt: Idx);
2034 }
2035 Value *ScalarGep = B.CreateIntrinsic(ID: Intrinsic::spv_gep, OverloadTypes: GepTypes, Args);
2036 GR->buildAssignPtr(B, ElemTy: LanePointeeTy, Arg: ScalarGep);
2037 VecResult = B.CreateInsertElement(Vec: VecResult, NewElt: ScalarGep, Idx: LaneIdx);
2038 }
2039
2040 auto *NewI = cast<Instruction>(Val: VecResult);
2041 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2042
2043 if (CallInst *Old = GR->findAssignPtrTypeInstr(Val: NewI)) {
2044 Old->eraseFromParent();
2045 GR->addAssignPtrTypeInstr(Val: NewI, AssignPtrTyCI: nullptr);
2046 }
2047 setInsertPointAfterDef(B, I: NewI);
2048 GR->buildAssignPtr(B, ElemTy: LanePointeeTy, Arg: NewI);
2049
2050 return NewI;
2051 }
2052
2053 if (TM.getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: &I)) {
2054 // Logical SPIR-V cannot use the OpPtrAccessChain instruction. If the first
2055 // index of the GEP is not 0, then we need to try to adjust it.
2056 //
2057 // If the GEP is doing byte addressing, try to rebuild the full access chain
2058 // from the type of the pointer.
2059 if (getByteAddressingMultiplier(Ty: I.getSourceElementType())) {
2060 return buildLogicalAccessChainFromGEP(GEP&: I);
2061 }
2062
2063 // Look for the array-to-pointer decay. If this is the pattern
2064 // we can adjust the types, and prepend a 0 to the indices.
2065 Value *PtrOp = I.getPointerOperand();
2066 Type *SrcElemTy = I.getSourceElementType();
2067 Type *DeducedPointeeTy = deduceElementType(I: PtrOp, UnknownElemTypeI8: true);
2068
2069 if (auto *ArrTy = dyn_cast<ArrayType>(Val: DeducedPointeeTy)) {
2070 if (ArrTy->getElementType() == SrcElemTy) {
2071 SmallVector<Value *> NewIndices;
2072 Type *FirstIdxType = I.getOperand(i_nocapture: 1)->getType();
2073 NewIndices.push_back(Elt: ConstantInt::get(Ty: FirstIdxType, V: 0));
2074 for (Value *Idx : I.indices())
2075 NewIndices.push_back(Elt: Idx);
2076
2077 SmallVector<Type *, 2> Types = {I.getType(), I.getPointerOperandType()};
2078 SmallVector<Value *, 4> Args;
2079 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
2080 Args.push_back(Elt: I.getPointerOperand());
2081 Args.append(in_start: NewIndices.begin(), in_end: NewIndices.end());
2082
2083 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep,
2084 OverloadTypes: {Types}, Args: {Args});
2085 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2086 return NewI;
2087 }
2088 }
2089 }
2090
2091 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
2092 SmallVector<Value *, 4> Args;
2093 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
2094 llvm::append_range(C&: Args, R: I.operands());
2095 Instruction *NewI =
2096 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
2097 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2098 return NewI;
2099}
2100
2101Instruction *SPIRVEmitIntrinsicsImpl::visitBitCastInst(BitCastInst &I) {
2102 IRBuilder<> B(I.getParent());
2103 B.SetInsertPoint(&I);
2104 Value *Source = I.getOperand(i_nocapture: 0);
2105
2106 // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of
2107 // varying element types. In case of IR coming from older versions of LLVM
2108 // such bitcasts do not provide sufficient information, should be just skipped
2109 // here, and handled in insertPtrCastOrAssignTypeInstr.
2110 if (isPointerTy(T: I.getType())) {
2111 replaceAllUsesWith(Src: &I, Dest: Source);
2112 I.eraseFromParent();
2113 return nullptr;
2114 }
2115
2116 SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
2117 SmallVector<Value *> Args(I.op_begin(), I.op_end());
2118 Instruction *NewI =
2119 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_bitcast, OverloadTypes: {Types}, Args: {Args});
2120 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2121 return NewI;
2122}
2123
2124void SPIRVEmitIntrinsicsImpl::insertAssignPtrTypeTargetExt(
2125 TargetExtType *AssignedType, Value *V, IRBuilder<> &B) {
2126 Type *VTy = V->getType();
2127
2128 // A couple of sanity checks.
2129 assert((isPointerTy(VTy)) && "Expect a pointer type!");
2130 if (Type *ElemTy = getPointeeType(Ty: VTy))
2131 if (ElemTy != AssignedType)
2132 report_fatal_error(reason: "Unexpected pointer element type!");
2133
2134 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: V);
2135 if (!AssignCI) {
2136 GR->buildAssignType(B, Ty: AssignedType, Arg: V);
2137 return;
2138 }
2139
2140 Type *CurrentType =
2141 dyn_cast<ConstantAsMetadata>(
2142 Val: cast<MetadataAsValue>(Val: AssignCI->getOperand(i_nocapture: 1))->getMetadata())
2143 ->getType();
2144 if (CurrentType == AssignedType)
2145 return;
2146
2147 // Builtin types cannot be redeclared or casted.
2148 if (CurrentType->isTargetExtTy())
2149 report_fatal_error(reason: "Type mismatch " + CurrentType->getTargetExtName() +
2150 "/" + AssignedType->getTargetExtName() +
2151 " for value " + V->getName(),
2152 gen_crash_diag: false);
2153
2154 // Our previous guess about the type seems to be wrong, let's update
2155 // inferred type according to a new, more precise type information.
2156 GR->updateAssignType(AssignCI, Arg: V, OfType: getNormalizedPoisonValue(Ty: AssignedType));
2157}
2158
2159void SPIRVEmitIntrinsicsImpl::replacePointerOperandWithPtrCast(
2160 Instruction *I, Value *Pointer, Type *ExpectedElementType,
2161 unsigned OperandToReplace, IRBuilder<> &B) {
2162 TypeValidated.insert(Ptr: I);
2163
2164 // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType
2165 Type *PointerElemTy = deduceElementTypeHelper(I: Pointer, UnknownElemTypeI8: false);
2166 if (PointerElemTy == ExpectedElementType ||
2167 isEquivalentTypes(Ty1: PointerElemTy, Ty2: ExpectedElementType))
2168 return;
2169
2170 setInsertPointSkippingPhis(B, I);
2171 Value *ExpectedElementVal = getNormalizedPoisonValue(Ty: ExpectedElementType);
2172 MetadataAsValue *VMD = buildMD(Arg: ExpectedElementVal);
2173 unsigned AddressSpace = getPointerAddressSpace(T: Pointer->getType());
2174 bool FirstPtrCastOrAssignPtrType = true;
2175
2176 // Do not emit new spv_ptrcast if equivalent one already exists or when
2177 // spv_assign_ptr_type already targets this pointer with the same element
2178 // type.
2179 if (Pointer->hasUseList()) {
2180 for (auto User : Pointer->users()) {
2181 auto *II = dyn_cast<IntrinsicInst>(Val: User);
2182 if (!II ||
2183 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
2184 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
2185 II->getOperand(i_nocapture: 0) != Pointer)
2186 continue;
2187
2188 // There is some spv_ptrcast/spv_assign_ptr_type already targeting this
2189 // pointer.
2190 FirstPtrCastOrAssignPtrType = false;
2191 if (II->getOperand(i_nocapture: 1) != VMD ||
2192 dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 2))->getSExtValue() !=
2193 AddressSpace)
2194 continue;
2195
2196 // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the
2197 // same element type and address space.
2198 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
2199 return;
2200
2201 // This must be a spv_ptrcast, do not emit new if this one has the same BB
2202 // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type.
2203 if (II->getParent() != I->getParent())
2204 continue;
2205
2206 I->setOperand(i: OperandToReplace, Val: II);
2207 return;
2208 }
2209 }
2210
2211 // Never replace an already-deduced pointer pointee with a non-pointer one.
2212 // The conflicting use comes from a mis-deduced expected type. Leave the
2213 // operand untouched rather than emitting a ptrcast that re-introduces
2214 // the collapsed type at the use site.
2215 if (PointerElemTy && isPointerTyOrWrapper(Ty: PointerElemTy) &&
2216 !isPointerTyOrWrapper(Ty: ExpectedElementType) &&
2217 tracesToPointerAlloca(V: Pointer))
2218 return;
2219
2220 if (isa<Instruction>(Val: Pointer) || isa<Argument>(Val: Pointer)) {
2221 if (FirstPtrCastOrAssignPtrType) {
2222 // If this would be the first spv_ptrcast, do not emit spv_ptrcast and
2223 // emit spv_assign_ptr_type instead.
2224 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
2225 return;
2226 } else if (isTodoType(Op: Pointer)) {
2227 eraseTodoType(Op: Pointer);
2228 if (!isa<CallInst>(Val: Pointer) && !isaGEP(V: Pointer) &&
2229 !isa<AllocaInst>(Val: Pointer)) {
2230 // If this wouldn't be the first spv_ptrcast but existing type info is
2231 // uncomplete, update spv_assign_ptr_type arguments.
2232 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Pointer)) {
2233 Type *PrevElemTy = GR->findDeducedElementType(Val: Pointer);
2234 assert(PrevElemTy);
2235 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
2236 std::make_pair(x&: I, y&: Pointer)};
2237 GR->updateAssignType(AssignCI, Arg: Pointer, OfType: ExpectedElementVal);
2238 propagateElemType(Op: Pointer, ElemTy: PrevElemTy, VisitedSubst);
2239 } else {
2240 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
2241 }
2242 return;
2243 }
2244 }
2245 }
2246
2247 // Emit spv_ptrcast
2248 SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()};
2249 SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(C: AddressSpace)};
2250 auto *PtrCastI = B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
2251 I->setOperand(i: OperandToReplace, Val: PtrCastI);
2252 // We need to set up a pointee type for the newly created spv_ptrcast.
2253 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: PtrCastI);
2254}
2255
2256void SPIRVEmitIntrinsicsImpl::insertPtrCastOrAssignTypeInstr(Instruction *I,
2257 IRBuilder<> &B) {
2258 // Handle basic instructions:
2259 StoreInst *SI = dyn_cast<StoreInst>(Val: I);
2260 if (IsKernelArgInt8(F: CurrF, SI)) {
2261 replacePointerOperandWithPtrCast(
2262 I, Pointer: SI->getValueOperand(), ExpectedElementType: IntegerType::getInt8Ty(C&: CurrF->getContext()),
2263 OperandToReplace: 0, B);
2264 }
2265 if (SI) {
2266 Value *Op = SI->getValueOperand();
2267 Value *Pointer = SI->getPointerOperand();
2268 Type *OpTy = Op->getType();
2269 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
2270 OpTy = restoreMutatedType(GR, I: OpI, Ty: OpTy);
2271 if (auto It = AggrConstTypes.find(Val: OpI); It != AggrConstTypes.end())
2272 OpTy = It->second;
2273 }
2274 if (OpTy == Op->getType())
2275 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: Op, UnknownElemTypeI8: false);
2276 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 1, B);
2277 return;
2278 }
2279 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
2280 Value *Pointer = LI->getPointerOperand();
2281 Type *OpTy = LI->getType();
2282 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
2283 if (Type *ElemTy = GR->findDeducedElementType(Val: LI)) {
2284 OpTy = getTypedPointerWrapper(ElemTy, AS: PtrTy->getAddressSpace());
2285 } else {
2286 Type *NewOpTy = OpTy;
2287 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: LI, UnknownElemTypeI8: false);
2288 if (OpTy == NewOpTy)
2289 insertTodoType(Op: Pointer);
2290 }
2291 }
2292 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
2293 return;
2294 }
2295 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(Val: I)) {
2296 Value *Pointer = GEPI->getPointerOperand();
2297 Type *OpTy = nullptr;
2298
2299 // Logical SPIR-V is not allowed to use Op*PtrAccessChain instructions. If
2300 // the first index is 0, then we can trivially lower to OpAccessChain. If
2301 // not we need to try to rewrite the GEP. We avoid adding a pointer cast at
2302 // this time, and will rewrite the GEP when visiting it.
2303 if (TM.getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: GEPI)) {
2304 return;
2305 }
2306
2307 // In all cases, fall back to the GEP type if type scavenging failed.
2308 if (!OpTy)
2309 OpTy = GEPI->getSourceElementType();
2310
2311 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
2312 if (isNestedPointer(Ty: OpTy))
2313 insertTodoType(Op: Pointer);
2314 return;
2315 }
2316
2317 // TODO: review and merge with existing logics:
2318 // Handle calls to builtins (non-intrinsics):
2319 CallInst *CI = dyn_cast<CallInst>(Val: I);
2320 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
2321 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
2322 return;
2323
2324 // collect information about formal parameter types
2325 std::string DemangledName =
2326 getOclOrSpirvBuiltinDemangledName(Name: CI->getCalledFunction()->getName());
2327 Function *CalledF = CI->getCalledFunction();
2328 SmallVector<Type *, 4> CalledArgTys;
2329 bool HaveTypes = false;
2330 for (unsigned OpIdx = 0; OpIdx < CalledF->arg_size(); ++OpIdx) {
2331 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2332 Type *ArgType = CalledArg->getType();
2333 if (!isPointerTy(T: ArgType)) {
2334 CalledArgTys.push_back(Elt: nullptr);
2335 } else if (Type *ArgTypeElem = getPointeeType(Ty: ArgType)) {
2336 CalledArgTys.push_back(Elt: ArgTypeElem);
2337 HaveTypes = true;
2338 } else {
2339 Type *ElemTy = GR->findDeducedElementType(Val: CalledArg);
2340 if (!ElemTy && hasPointeeTypeAttr(Arg: CalledArg))
2341 ElemTy = getPointeeTypeByAttr(Arg: CalledArg);
2342 if (!ElemTy) {
2343 ElemTy = getPointeeTypeByCallInst(DemangledName, CalledF, OpIdx);
2344 if (ElemTy) {
2345 GR->addDeducedElementType(Val: CalledArg, Ty: normalizeType(Ty: ElemTy));
2346 } else {
2347 for (User *U : CalledArg->users()) {
2348 if (Instruction *Inst = dyn_cast<Instruction>(Val: U)) {
2349 if ((ElemTy = deduceElementTypeHelper(I: Inst, UnknownElemTypeI8: false)) != nullptr)
2350 break;
2351 }
2352 }
2353 }
2354 }
2355 HaveTypes |= ElemTy != nullptr;
2356 CalledArgTys.push_back(Elt: ElemTy);
2357 }
2358 }
2359
2360 if (DemangledName.empty() && !HaveTypes)
2361 return;
2362
2363 for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) {
2364 Value *ArgOperand = CI->getArgOperand(i: OpIdx);
2365 if (!isPointerTy(T: ArgOperand->getType()))
2366 continue;
2367
2368 // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs()
2369 if (!isa<Instruction>(Val: ArgOperand) && !isa<Argument>(Val: ArgOperand)) {
2370 // However, we may have assumptions about the formal argument's type and
2371 // may have a need to insert a ptr cast for the actual parameter of this
2372 // call.
2373 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2374 if (!GR->findDeducedElementType(Val: CalledArg))
2375 continue;
2376 }
2377
2378 Type *ExpectedType =
2379 OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr;
2380 if (!ExpectedType && !DemangledName.empty())
2381 ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType(
2382 DemangledCall: DemangledName, ArgIdx: OpIdx, Ctx&: I->getContext());
2383 if (!ExpectedType || ExpectedType->isVoidTy())
2384 continue;
2385
2386 if (ExpectedType->isTargetExtTy() &&
2387 !isTypedPointerWrapper(ExtTy: cast<TargetExtType>(Val: ExpectedType)))
2388 insertAssignPtrTypeTargetExt(AssignedType: cast<TargetExtType>(Val: ExpectedType),
2389 V: ArgOperand, B);
2390 else
2391 replacePointerOperandWithPtrCast(I: CI, Pointer: ArgOperand, ExpectedElementType: ExpectedType, OperandToReplace: OpIdx, B);
2392 }
2393}
2394
2395Instruction *
2396SPIRVEmitIntrinsicsImpl::visitInsertElementInst(InsertElementInst &I) {
2397 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2398 // type in LLT and IRTranslator will replace it by the scalar.
2399 if (isVector1(Ty: I.getType()))
2400 return &I;
2401
2402 SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType(),
2403 I.getOperand(i_nocapture: 1)->getType(),
2404 I.getOperand(i_nocapture: 2)->getType()};
2405 IRBuilder<> B(I.getParent());
2406 B.SetInsertPoint(&I);
2407 SmallVector<Value *> Args(I.op_begin(), I.op_end());
2408 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_insertelt,
2409 OverloadTypes: {Types}, Args: {Args});
2410 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2411 return NewI;
2412}
2413
2414Instruction *
2415SPIRVEmitIntrinsicsImpl::visitExtractElementInst(ExtractElementInst &I) {
2416 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2417 // type in LLT and IRTranslator will replace it by the scalar.
2418 if (isVector1(Ty: I.getVectorOperandType()))
2419 return &I;
2420
2421 IRBuilder<> B(I.getParent());
2422 B.SetInsertPoint(&I);
2423 SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
2424 I.getIndexOperand()->getType()};
2425 SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
2426 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_extractelt,
2427 OverloadTypes: {Types}, Args: {Args});
2428 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2429 return NewI;
2430}
2431
2432Instruction *SPIRVEmitIntrinsicsImpl::visitInsertValueInst(InsertValueInst &I) {
2433 IRBuilder<> B(I.getParent());
2434 B.SetInsertPoint(&I);
2435 SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
2436 SmallVector<Value *> Args;
2437 Value *AggregateOp = I.getAggregateOperand();
2438 if (isa<UndefValue>(Val: AggregateOp))
2439 Args.push_back(Elt: UndefValue::get(T: B.getInt32Ty()));
2440 else
2441 Args.push_back(Elt: AggregateOp);
2442 Args.push_back(Elt: I.getInsertedValueOperand());
2443 for (auto &Op : I.indices())
2444 Args.push_back(Elt: B.getInt32(C: Op));
2445 Instruction *NewI =
2446 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_insertv, OverloadTypes: {Types}, Args: {Args});
2447 replaceMemInstrUses(Old: &I, New: NewI, B);
2448 return NewI;
2449}
2450
2451Instruction *
2452SPIRVEmitIntrinsicsImpl::visitExtractValueInst(ExtractValueInst &I) {
2453 IRBuilder<> B(I.getParent());
2454 B.SetInsertPoint(&I);
2455 if (I.getAggregateOperand()->getType()->isAggregateType()) {
2456 // Mutate an aggregate-returning spv_extractv producer to i32 so
2457 // IRTranslator does not see a multi-register value.
2458 CallBase *CB = dyn_cast<CallBase>(Val: I.getAggregateOperand());
2459 if (!CB || CB->getIntrinsicID() != Intrinsic::spv_extractv)
2460 return &I;
2461 CB->mutateType(Ty: B.getInt32Ty());
2462 }
2463 SmallVector<Value *> Args(I.operands());
2464 for (auto &Op : I.indices())
2465 Args.push_back(Elt: B.getInt32(C: Op));
2466 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_extractv,
2467 OverloadTypes: {I.getType()}, Args: {Args});
2468 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2469 // If the aggregate result feeds a callsite whose aggregate params were
2470 // rewritten to i32 value-ids by SPIRVPrepareFunctions, mutate it to match.
2471 if (NewI->getType()->isAggregateType()) {
2472 for (const Use &U : NewI->uses()) {
2473 auto *CB = dyn_cast<CallBase>(Val: U.getUser());
2474 if (!CB || !CB->isArgOperand(U: &U))
2475 continue;
2476 unsigned ArgNo = CB->getArgOperandNo(U: &U);
2477 FunctionType *FT = CB->getFunctionType();
2478 if (ArgNo < FT->getNumParams() &&
2479 !FT->getParamType(i: ArgNo)->isAggregateType()) {
2480 NewI->mutateType(Ty: B.getInt32Ty());
2481 break;
2482 }
2483 }
2484 }
2485 return NewI;
2486}
2487
2488Instruction *SPIRVEmitIntrinsicsImpl::visitLoadInst(LoadInst &I) {
2489 if (!I.getType()->isAggregateType())
2490 return &I;
2491 IRBuilder<> B(I.getParent());
2492 B.SetInsertPoint(&I);
2493 TrackConstants = false;
2494 const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
2495 MachineMemOperand::Flags Flags =
2496 TLI->getLoadMemOperandFlags(LI: I, DL: CurrF->getDataLayout());
2497
2498 unsigned IntrinsicId;
2499 SmallVector<Value *, 4> Args = {I.getPointerOperand(), B.getInt16(C: Flags)};
2500 if (!I.isAtomic()) {
2501 IntrinsicId = Intrinsic::spv_load;
2502 Args.push_back(Elt: B.getInt32(C: I.getAlign().value()));
2503 } else {
2504 IntrinsicId = Intrinsic::spv_atomic_load;
2505 Args.push_back(Elt: B.getInt8(C: static_cast<uint8_t>(I.getOrdering())));
2506 }
2507 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
2508 ID: IntrinsicId, OverloadTypes: {I.getOperand(i_nocapture: 0)->getType()}, Args);
2509
2510 replaceMemInstrUses(Old: &I, New: NewI, B);
2511 return NewI;
2512}
2513
2514Instruction *SPIRVEmitIntrinsicsImpl::visitStoreInst(StoreInst &I) {
2515 if (!AggrStores.contains(Ptr: &I))
2516 return &I;
2517 IRBuilder<> B(I.getParent());
2518 B.SetInsertPoint(&I);
2519 TrackConstants = false;
2520 const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
2521 MachineMemOperand::Flags Flags =
2522 TLI->getStoreMemOperandFlags(SI: I, DL: CurrF->getDataLayout());
2523 auto *PtrOp = I.getPointerOperand();
2524
2525 if (I.getValueOperand()->getType()->isAggregateType()) {
2526 // It is possible that what used to be an ExtractValueInst has been replaced
2527 // with a call to the spv_extractv intrinsic, and that said call hasn't
2528 // had its return type replaced with i32 during the dedicated pass (because
2529 // it was emitted later); we have to handle this here, because IRTranslator
2530 // cannot deal with multi-register types at the moment.
2531 CallBase *CB = dyn_cast<CallBase>(Val: I.getValueOperand());
2532 assert(CB && CB->getIntrinsicID() == Intrinsic::spv_extractv &&
2533 "Unexpected argument of aggregate type, should be spv_extractv!");
2534 CB->mutateType(Ty: B.getInt32Ty());
2535 }
2536
2537 unsigned IntrinsicId;
2538 SmallVector<Value *, 4> Args = {I.getValueOperand(), PtrOp,
2539 B.getInt16(C: Flags)};
2540 if (!I.isAtomic()) {
2541 IntrinsicId = Intrinsic::spv_store;
2542 Args.push_back(Elt: B.getInt32(C: I.getAlign().value()));
2543 } else {
2544 IntrinsicId = Intrinsic::spv_atomic_store;
2545 Args.push_back(Elt: B.getInt8(C: static_cast<uint8_t>(I.getOrdering())));
2546 }
2547 Instruction *NewI = B.CreateIntrinsicWithoutFolding(
2548 ID: IntrinsicId, OverloadTypes: {I.getValueOperand()->getType(), PtrOp->getType()}, Args);
2549 NewI->copyMetadata(SrcInst: I);
2550 I.eraseFromParent();
2551 return NewI;
2552}
2553
2554Instruction *SPIRVEmitIntrinsicsImpl::visitAllocaInst(AllocaInst &I) {
2555 Value *ArraySize = nullptr;
2556 if (I.isArrayAllocation()) {
2557 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I.getFunction());
2558 if (!STI->canUseExtension(
2559 E: SPIRV::Extension::SPV_INTEL_variable_length_array))
2560 report_fatal_error(
2561 reason: "array allocation: this instruction requires the following "
2562 "SPIR-V extension: SPV_INTEL_variable_length_array",
2563 gen_crash_diag: false);
2564 ArraySize = I.getArraySize();
2565 }
2566 IRBuilder<> B(I.getParent());
2567 B.SetInsertPoint(&I);
2568 TrackConstants = false;
2569 Type *PtrTy = I.getType();
2570 Instruction *NewI =
2571 ArraySize
2572 ? B.CreateIntrinsicWithoutFolding(
2573 ID: Intrinsic::spv_alloca_array, OverloadTypes: {PtrTy, ArraySize->getType()},
2574 Args: {ArraySize, B.getInt32(C: I.getAlign().value())})
2575 : B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_alloca, OverloadTypes: {PtrTy},
2576 Args: {B.getInt32(C: I.getAlign().value())});
2577 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2578 return NewI;
2579}
2580
2581Instruction *
2582SPIRVEmitIntrinsicsImpl::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2583 assert(I.getType()->isAggregateType() && "Aggregate result is expected");
2584 IRBuilder<> B(I.getParent());
2585 B.SetInsertPoint(&I);
2586 SmallVector<Value *> Args(I.operands());
2587 Args.push_back(Elt: B.getInt32(
2588 C: static_cast<uint32_t>(getMemScope(Ctx&: I.getContext(), Id: I.getSyncScopeID()))));
2589 // Per SPIR-V spec atomic ops must combine the ordering bits with the
2590 // storage-class bit.
2591 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2592 unsigned AS = I.getPointerOperand()->getType()->getPointerAddressSpace();
2593 uint32_t ScSem = static_cast<uint32_t>(
2594 getMemSemanticsForStorageClass(SC: addressSpaceToStorageClass(AddrSpace: AS, STI: ST)));
2595 Args.push_back(Elt: B.getInt32(
2596 C: static_cast<uint32_t>(getMemSemantics(Ord: I.getSuccessOrdering())) | ScSem));
2597 Args.push_back(Elt: B.getInt32(
2598 C: static_cast<uint32_t>(getMemSemantics(Ord: I.getFailureOrdering())) | ScSem));
2599 Instruction *NewI = B.CreateIntrinsicWithoutFolding(
2600 ID: Intrinsic::spv_cmpxchg, OverloadTypes: {I.getPointerOperand()->getType()}, Args: {Args});
2601 replaceMemInstrUses(Old: &I, New: NewI, B);
2602 return NewI;
2603}
2604
2605static bool isAbortCall(const Instruction &I, const SPIRVSubtarget &ST) {
2606 auto *CI = dyn_cast<CallInst>(Val: &I);
2607 if (!CI)
2608 return false;
2609 switch (CI->getIntrinsicID()) {
2610 case Intrinsic::spv_abort:
2611 return true;
2612 case Intrinsic::trap:
2613 case Intrinsic::ubsantrap:
2614 // When the extension is enabled, selection lowers these to OpAbortKHR.
2615 return ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_abort);
2616 default:
2617 return false;
2618 }
2619}
2620
2621// The OpAbortKHR instruction itself is a block terminator, so we don't need to
2622// emit an extra OpUnreachable instruction.
2623static bool precededByAbortIntrinsic(const UnreachableInst &I,
2624 const SPIRVSubtarget &ST) {
2625 // Find a previous non-debug instruction.
2626 const Instruction *Prev = I.getPrevNode();
2627 while (Prev && Prev->isDebugOrPseudoInst())
2628 Prev = Prev->getPrevNode();
2629
2630 if (Prev && isAbortCall(I: *Prev, ST))
2631 return true;
2632
2633 assert(llvm::none_of(
2634 *I.getParent(),
2635 [&ST](const Instruction &II) { return isAbortCall(II, ST); }) &&
2636 "abort-like call must be the last non-debug instruction before its "
2637 "block's terminator");
2638 return false;
2639}
2640
2641Instruction *SPIRVEmitIntrinsicsImpl::visitUnreachableInst(UnreachableInst &I) {
2642 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2643 if (precededByAbortIntrinsic(I, ST))
2644 return &I;
2645 IRBuilder<> B(&I);
2646 B.CreateIntrinsic(ID: Intrinsic::spv_unreachable, Args: {});
2647 return &I;
2648}
2649
2650// llvm.compiler.used and llvm.used hold use-list entries that protect their
2651// referenced globals from DCE without participating in code generation.
2652static bool isUseListGlobal(StringRef Name) {
2653 return Name == "llvm.compiler.used" || Name == "llvm.used";
2654}
2655
2656// Returns true for module-level globals that should not have SPIR-V intrinsics
2657// emitted (use-list globals plus llvm.global.annotations).
2658static bool isArtificialGlobal(StringRef Name) {
2659 return isUseListGlobal(Name) || Name == "llvm.global.annotations";
2660}
2661
2662// Returns true if every use of GV traces back to llvm.compiler.used or
2663// llvm.used.
2664static bool hasOnlyArtificialUses(const GlobalVariable &GV) {
2665 SmallPtrSet<const Value *, 8> Visited;
2666 SmallVector<const Value *> Stack(GV.users());
2667 while (!Stack.empty()) {
2668 const Value *V = Stack.pop_back_val();
2669 if (!Visited.insert(Ptr: V).second)
2670 continue;
2671 if (const auto *GVUser = dyn_cast<GlobalVariable>(Val: V)) {
2672 if (!isUseListGlobal(Name: GVUser->getName()))
2673 return false;
2674 continue;
2675 }
2676 if (const auto *C = dyn_cast<Constant>(Val: V)) {
2677 Stack.append(in_start: C->user_begin(), in_end: C->user_end());
2678 continue;
2679 }
2680 return false;
2681 }
2682 return true;
2683}
2684
2685static bool
2686shouldEmitIntrinsicsForGlobalValue(const GlobalVariableUsers &GVUsers,
2687 const GlobalVariable &GV,
2688 const Function *F) {
2689 // Skip special artificial variables.
2690 if (isArtificialGlobal(Name: GV.getName()))
2691 return false;
2692
2693 auto &UserFunctions = GVUsers.getTransitiveUserFunctions(GV);
2694 if (UserFunctions.contains(Ptr: F))
2695 return true;
2696
2697 // Do not emit the intrinsics in this function, it's going to be emitted on
2698 // the functions that reference it.
2699 if (!UserFunctions.empty())
2700 return false;
2701
2702 // Emit definitions for globals that are not referenced by any function on the
2703 // first function definition.
2704 const Module &M = *F->getParent();
2705 const Function &FirstDefinition = *M.getFunctionDefs().begin();
2706 return F == &FirstDefinition;
2707}
2708
2709Value *SPIRVEmitIntrinsicsImpl::buildSpvUndefComposite(Type *AggrTy,
2710 IRBuilder<> &B) {
2711 auto MakeLeaf = [&](Type *ElemTy) -> Instruction * {
2712 CallInst *Leaf = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_undef, Args: {});
2713 AggrConsts[Leaf] = PoisonValue::get(T: ElemTy);
2714 AggrConstTypes[Leaf] = ElemTy;
2715 return Leaf;
2716 };
2717 SmallVector<Value *, 4> Elems;
2718 if (auto *ArrTy = dyn_cast<ArrayType>(Val: AggrTy)) {
2719 Elems.assign(NumElts: ArrTy->getNumElements(), Elt: MakeLeaf(ArrTy->getElementType()));
2720 } else {
2721 auto *StructTy = cast<StructType>(Val: AggrTy);
2722 DenseMap<Type *, Instruction *> LeafByType;
2723 for (unsigned I = 0; I < StructTy->getNumElements(); ++I) {
2724 Type *ElemTy = StructTy->getContainedType(i: I);
2725 auto &Entry = LeafByType[ElemTy];
2726 if (!Entry)
2727 Entry = MakeLeaf(ElemTy);
2728 Elems.push_back(Elt: Entry);
2729 }
2730 }
2731 CallInst *Composite = B.CreateIntrinsicWithoutFolding(
2732 ID: Intrinsic::spv_const_composite, OverloadTypes: {B.getInt32Ty()}, Args: Elems);
2733 AggrConsts[Composite] = PoisonValue::get(T: AggrTy);
2734 AggrConstTypes[Composite] = AggrTy;
2735 return Composite;
2736}
2737
2738// If a function directly returns an aggregate-typed call result,
2739// the ReturnInst carries an aggregate while the function signature
2740// was rewritten to i32 by SPIRVPrepareFunctions. Rebuild the return value
2741// via extractvalue/insertvalue so the regular spv_extractv/spv_insertv
2742// lowering produces a valid OpReturnValue.
2743void SPIRVEmitIntrinsicsImpl::reconstructAggregateReturns(Function &Func,
2744 IRBuilder<> &B) {
2745 Type *OrigRetTy = GR->findMutated(Val: &Func);
2746 if (!OrigRetTy || !OrigRetTy->isAggregateType())
2747 return;
2748 for (BasicBlock &BB : Func) {
2749 auto *RI = dyn_cast<ReturnInst>(Val: BB.getTerminator());
2750 if (!RI)
2751 continue;
2752 Value *RetVal = RI->getReturnValue();
2753 if (!RetVal || RetVal->getType() != OrigRetTy || !isa<CallBase>(Val: RetVal))
2754 continue;
2755 Type *AggrTy = RetVal->getType();
2756 uint64_t NumElts = isa<StructType>(Val: AggrTy)
2757 ? cast<StructType>(Val: AggrTy)->getNumElements()
2758 : cast<ArrayType>(Val: AggrTy)->getNumElements();
2759 B.SetInsertPoint(RI);
2760 Value *Rebuilt = PoisonValue::get(T: AggrTy);
2761 for (uint64_t I = 0; I < NumElts; ++I) {
2762 Value *Elt = B.CreateExtractValue(Agg: RetVal, Idxs: I);
2763 Rebuilt = B.CreateInsertValue(Agg: Rebuilt, Val: Elt, Idxs: I);
2764 }
2765 RI->setOperand(i_nocapture: 0, Val_nocapture: Rebuilt);
2766 }
2767}
2768
2769void SPIRVEmitIntrinsicsImpl::processGlobalValue(GlobalVariable &GV,
2770 IRBuilder<> &B) {
2771
2772 if (!shouldEmitIntrinsicsForGlobalValue(GVUsers, GV, F: CurrF))
2773 return;
2774
2775 // Record the pointee type for every global, not only initialized ones, so an
2776 // undef non-constant aggregate global is not later collapsed to its element
2777 // type. Result is ignored, because TypedPointerType is not supported
2778 // by llvm IR general logic.
2779 deduceElementTypeHelper(I: &GV, UnknownElemTypeI8: false);
2780
2781 Constant *Init = nullptr;
2782 if (hasInitializer(GV: &GV)) {
2783 Init = GV.getInitializer();
2784 Value *InitOp = Init;
2785 if (isa<UndefValue>(Val: Init) && Init->getType()->isAggregateType()) {
2786 const SPIRVSubtarget *STI = TM.getSubtargetImpl();
2787 bool UsePoison =
2788 isa<PoisonValue>(Val: Init) &&
2789 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
2790 if (UsePoison) {
2791 CallInst *Call = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_poison,
2792 OverloadTypes: {B.getInt32Ty()}, Args: {});
2793 AggrConsts[Call] = cast<PoisonValue>(Val: Init);
2794 AggrConstTypes[Call] = Init->getType();
2795 InitOp = Call;
2796 } else {
2797 InitOp = buildSpvUndefComposite(AggrTy: Init->getType(), B);
2798 }
2799 }
2800 Type *Ty = isAggrConstForceInt32(V: Init) ? B.getInt32Ty() : Init->getType();
2801 Constant *Const = isAggrConstForceInt32(V: Init) ? B.getInt32(C: 1) : Init;
2802 CallInst *InitInst = B.CreateIntrinsicWithoutFolding(
2803 ID: Intrinsic::spv_init_global, OverloadTypes: {GV.getType(), Ty}, Args: {&GV, Const});
2804 InitInst->setArgOperand(i: 1, v: InitOp);
2805 }
2806 // Globals with only use-list references have no real function uses. Emit
2807 // spv_unref_global so buildGlobalVariable is called for them.
2808 if (!Init && hasOnlyArtificialUses(GV))
2809 B.CreateIntrinsic(ID: Intrinsic::spv_unref_global, OverloadTypes: GV.getType(), Args: &GV);
2810}
2811
2812// Return true, if we can't decide what is the pointee type now and will get
2813// back to the question later. Return false is spv_assign_ptr_type is not needed
2814// or can be inserted immediately.
2815bool SPIRVEmitIntrinsicsImpl::insertAssignPtrTypeIntrs(Instruction *I,
2816 IRBuilder<> &B,
2817 bool UnknownElemTypeI8) {
2818 reportFatalOnTokenType(I);
2819 if (!isPointerTy(T: I->getType()) || !requireAssignType(I))
2820 return false;
2821
2822 setInsertPointAfterDef(B, I);
2823 if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) {
2824 GR->buildAssignPtr(B, ElemTy, Arg: I);
2825 return false;
2826 }
2827 return true;
2828}
2829
2830void SPIRVEmitIntrinsicsImpl::insertAssignTypeIntrs(Instruction *I,
2831 IRBuilder<> &B) {
2832 // TODO: extend the list of functions with known result types
2833 static StringMap<unsigned> ResTypeWellKnown = {
2834 {"async_work_group_copy", WellKnownTypes::Event},
2835 {"async_work_group_strided_copy", WellKnownTypes::Event},
2836 {"__spirv_GroupAsyncCopy", WellKnownTypes::Event}};
2837
2838 reportFatalOnTokenType(I);
2839
2840 bool IsKnown = false;
2841 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
2842 if (!CI->isIndirectCall() && !CI->isInlineAsm() &&
2843 CI->getCalledFunction() && !CI->getCalledFunction()->isIntrinsic()) {
2844 Function *CalledF = CI->getCalledFunction();
2845 std::string DemangledName =
2846 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
2847 FPDecorationId DecorationId = FPDecorationId::NONE;
2848 if (DemangledName.length() > 0)
2849 DemangledName =
2850 SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName, DecorationId: &DecorationId);
2851 auto ResIt = ResTypeWellKnown.find(Key: DemangledName);
2852 if (ResIt != ResTypeWellKnown.end()) {
2853 IsKnown = true;
2854 setInsertPointAfterDef(B, I);
2855 switch (ResIt->second) {
2856 case WellKnownTypes::Event:
2857 GR->buildAssignType(
2858 B, Ty: TargetExtType::get(Context&: I->getContext(), Name: "spirv.Event"), Arg: I);
2859 break;
2860 }
2861 }
2862 // check if a floating rounding mode or saturation info is present
2863 switch (DecorationId) {
2864 default:
2865 break;
2866 case FPDecorationId::SAT:
2867 createSaturatedConversionDecoration(I: CI, B);
2868 break;
2869 case FPDecorationId::RTE:
2870 createRoundingModeDecoration(
2871 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTE, B);
2872 break;
2873 case FPDecorationId::RTZ:
2874 createRoundingModeDecoration(
2875 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTZ, B);
2876 break;
2877 case FPDecorationId::RTP:
2878 createRoundingModeDecoration(
2879 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTP, B);
2880 break;
2881 case FPDecorationId::RTN:
2882 createRoundingModeDecoration(
2883 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTN, B);
2884 break;
2885 }
2886 }
2887 }
2888
2889 Type *Ty = I->getType();
2890 if (!IsKnown && !Ty->isVoidTy() && !isPointerTy(T: Ty) && requireAssignType(I)) {
2891 setInsertPointAfterDef(B, I);
2892 Type *TypeToAssign = Ty;
2893 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2894 if (isSpvAggrPlaceholder(V: II)) {
2895 auto It = AggrConstTypes.find(Val: II);
2896 if (It == AggrConstTypes.end())
2897 report_fatal_error(reason: "Unknown composite intrinsic type");
2898 TypeToAssign = It->second;
2899 } else if (II->getIntrinsicID() == Intrinsic::spv_poison) {
2900 if (auto It = AggrConstTypes.find(Val: II); It != AggrConstTypes.end())
2901 TypeToAssign = It->second;
2902 }
2903 } else if (auto It = AggrConstTypes.find(Val: I); It != AggrConstTypes.end())
2904 TypeToAssign = It->second;
2905 TypeToAssign = restoreMutatedType(GR, I, Ty: TypeToAssign);
2906 GR->buildAssignType(B, Ty: TypeToAssign, Arg: I);
2907 }
2908 for (const auto &Op : I->operands()) {
2909 if (isa<ConstantPointerNull>(Val: Op) || isa<UndefValue>(Val: Op) ||
2910 // Check GetElementPtrConstantExpr case.
2911 (isa<ConstantExpr>(Val: Op) &&
2912 (isa<GEPOperator>(Val: Op) ||
2913 (cast<ConstantExpr>(Val: Op)->getOpcode() == CastInst::IntToPtr)))) {
2914 setInsertPointSkippingPhis(B, I);
2915 Type *OpTy = Op->getType();
2916 if (isa<UndefValue>(Val: Op) && OpTy->isAggregateType()) {
2917 CallInst *AssignCI =
2918 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {B.getInt32Ty()}, Arg: Op,
2919 Arg2: UndefValue::get(T: B.getInt32Ty()), Imms: {}, B);
2920 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: AssignCI);
2921 } else if (!isa<Instruction>(Val: Op)) {
2922 Type *OpTy = Op->getType();
2923 Type *OpTyElem = getPointeeType(Ty: OpTy);
2924 if (OpTyElem) {
2925 GR->buildAssignPtr(B, ElemTy: OpTyElem, Arg: Op);
2926 } else if (isPointerTy(T: OpTy)) {
2927 Type *ElemTy = GR->findDeducedElementType(Val: Op);
2928 GR->buildAssignPtr(B, ElemTy: ElemTy ? ElemTy : deduceElementType(I: Op, UnknownElemTypeI8: true),
2929 Arg: Op);
2930 } else {
2931 Value *OpTyVal = Op;
2932 if (OpTy->isTargetExtTy()) {
2933 // We need to do this in order to be consistent with how target ext
2934 // types are handled in `processInstrAfterVisit`
2935 OpTyVal = getNormalizedPoisonValue(Ty: OpTy);
2936 }
2937 CallInst *AssignCI =
2938 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {OpTy},
2939 Arg: getNormalizedPoisonValue(Ty: OpTy), Arg2: OpTyVal, Imms: {}, B);
2940 GR->addAssignPtrTypeInstr(Val: OpTyVal, AssignPtrTyCI: AssignCI);
2941 }
2942 }
2943 }
2944 }
2945}
2946
2947bool SPIRVEmitIntrinsicsImpl::shouldTryToAddMemAliasingDecoration(
2948 Instruction *Inst) {
2949 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*Inst->getFunction());
2950 if (!STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing))
2951 return false;
2952 // Add aliasing decorations to internal load and store intrinsics.
2953 // Do not attach them to store atomic or load atomic intrinsics / instructions
2954 // since the extension is inconsistent at the moment (we cannot add the
2955 // decoration to atomic stores because they do not have an id).
2956 return match(V: Inst,
2957 P: m_AnyIntrinsic<Intrinsic::spv_load, Intrinsic::spv_store>());
2958}
2959
2960void SPIRVEmitIntrinsicsImpl::insertSpirvDecorations(Instruction *I,
2961 IRBuilder<> &B) {
2962 if (MDNode *MD = I->getMetadata(Kind: "spirv.Decorations")) {
2963 setInsertPointAfterDef(B, I);
2964 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
2965 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
2966 }
2967 // Lower alias.scope/noalias metadata
2968 {
2969 auto processMemAliasingDecoration = [&](unsigned Kind) {
2970 if (MDNode *AliasListMD = I->getMetadata(KindID: Kind)) {
2971 if (shouldTryToAddMemAliasingDecoration(Inst: I)) {
2972 uint32_t Dec = Kind == LLVMContext::MD_alias_scope
2973 ? SPIRV::Decoration::AliasScopeINTEL
2974 : SPIRV::Decoration::NoAliasINTEL;
2975 SmallVector<Value *, 3> Args = {
2976 I, ConstantInt::get(Ty: B.getInt32Ty(), V: Dec),
2977 MetadataAsValue::get(Context&: I->getContext(), MD: AliasListMD)};
2978 setInsertPointAfterDef(B, I);
2979 B.CreateIntrinsic(ID: Intrinsic::spv_assign_aliasing_decoration,
2980 OverloadTypes: {I->getType()}, Args: {Args});
2981 }
2982 }
2983 };
2984 processMemAliasingDecoration(LLVMContext::MD_alias_scope);
2985 processMemAliasingDecoration(LLVMContext::MD_noalias);
2986 }
2987 // MD_fpmath
2988 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_fpmath)) {
2989 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I->getFunction());
2990 bool AllowFPMaxError =
2991 STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_fp_max_error);
2992 if (!AllowFPMaxError)
2993 return;
2994
2995 setInsertPointAfterDef(B, I);
2996 B.CreateIntrinsic(ID: Intrinsic::spv_assign_fpmaxerror_decoration,
2997 OverloadTypes: {I->getType()},
2998 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
2999 }
3000 if (I->getModule()->getTargetTriple().getVendor() == Triple::AMD &&
3001 isa<AtomicRMWInst>(Val: I)) {
3002 // If present, we encode AMDGPU atomic metadata as UserSemantic string
3003 // decorations, which will be parsed during reverse translation.
3004 auto &Ctx = B.getContext();
3005 auto *US = ConstantAsMetadata::get(
3006 C: ConstantInt::get(Ty: B.getInt32Ty(), V: SPIRV::Decoration::UserSemantic));
3007
3008 SmallVector<Metadata *> MDs;
3009 if (I->hasMetadata(Kind: "amdgpu.no.fine.grained.memory"))
3010 MDs.push_back(Elt: MDNode::get(
3011 Context&: Ctx, MDs: {US, MDString::get(Context&: Ctx, Str: "amdgpu.no.fine.grained.memory")}));
3012 if (I->hasMetadata(Kind: "amdgpu.no.remote.memory"))
3013 MDs.push_back(Elt: MDNode::get(
3014 Context&: Ctx, MDs: {US, MDString::get(Context&: Ctx, Str: "amdgpu.no.remote.memory")}));
3015 if (I->hasMetadata(Kind: "amdgpu.ignore.denormal.mode"))
3016 MDs.push_back(Elt: MDNode::get(
3017 Context&: Ctx, MDs: {US, MDString::get(Context&: Ctx, Str: "amdgpu.ignore.denormal.mode")}));
3018 if (!MDs.empty())
3019 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
3020 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs))});
3021 }
3022}
3023
3024static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
3025 const Module &M,
3026 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
3027 &FPFastMathDefaultInfoMap,
3028 Function *F) {
3029 auto it = FPFastMathDefaultInfoMap.find(Val: F);
3030 if (it != FPFastMathDefaultInfoMap.end())
3031 return it->second;
3032
3033 // If the map does not contain the entry, create a new one. Initialize it to
3034 // contain all 3 elements sorted by bit width of target type: {half, float,
3035 // double}.
3036 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
3037 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
3038 Args: SPIRV::FPFastMathMode::None);
3039 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
3040 Args: SPIRV::FPFastMathMode::None);
3041 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
3042 Args: SPIRV::FPFastMathMode::None);
3043 return FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
3044}
3045
3046static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
3047 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
3048 const Type *Ty) {
3049 size_t BitWidth = Ty->getScalarSizeInBits();
3050 int Index =
3051 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
3052 BitWidth);
3053 assert(Index >= 0 && Index < 3 &&
3054 "Expected FPFastMathDefaultInfo for half, float, or double");
3055 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3056 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3057 return FPFastMathDefaultInfoVec[Index];
3058}
3059
3060void SPIRVEmitIntrinsicsImpl::insertConstantsForFPFastMathDefault(Module &M) {
3061 const SPIRVSubtarget *ST = TM.getSubtargetImpl();
3062 if (!ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
3063 return;
3064
3065 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
3066 // We need the entry point (function) as the key, and the target
3067 // type and flags as the value.
3068 // We also need to check ContractionOff and SignedZeroInfNanPreserve
3069 // execution modes, as they are now deprecated and must be replaced
3070 // with FPFastMathDefaultInfo.
3071 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
3072 if (!Node) {
3073 if (!M.getNamedMetadata(Name: "opencl.enable.FP_CONTRACT")) {
3074 // This requires emitting ContractionOff. However, because
3075 // ContractionOff is now deprecated, we need to replace it with
3076 // FPFastMathDefaultInfo with FP Fast Math Mode bitmask set to all 0.
3077 // We need to create the constant for that.
3078
3079 // Create constant instruction with the bitmask flags.
3080 Constant *InitValue =
3081 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: 0);
3082 // TODO: Reuse constant if there is one already with the required
3083 // value.
3084 [[maybe_unused]] GlobalVariable *GV =
3085 new GlobalVariable(M, // Module
3086 Type::getInt32Ty(C&: M.getContext()), // Type
3087 true, // isConstant
3088 GlobalValue::InternalLinkage, // Linkage
3089 InitValue // Initializer
3090 );
3091 }
3092 return;
3093 }
3094
3095 // The table maps function pointers to their default FP fast math info. It
3096 // can be assumed that the SmallVector is sorted by the bit width of the
3097 // type. The first element is the smallest bit width, and the last element
3098 // is the largest bit width, therefore, we will have {half, float, double}
3099 // in the order of their bit widths.
3100 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
3101 FPFastMathDefaultInfoMap;
3102
3103 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
3104 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
3105 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
3106 Function *F = cast<Function>(
3107 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
3108 const auto EM =
3109 cast<ConstantInt>(
3110 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
3111 ->getZExtValue();
3112 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
3113 assert(MDN->getNumOperands() == 4 &&
3114 "Expected 4 operands for FPFastMathDefault");
3115 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
3116 unsigned Flags =
3117 cast<ConstantInt>(
3118 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
3119 ->getZExtValue();
3120 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3121 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3122 SPIRV::FPFastMathDefaultInfo &Info =
3123 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
3124 Info.FastMathFlags = Flags;
3125 Info.FPFastMathDefault = true;
3126 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
3127 assert(MDN->getNumOperands() == 2 &&
3128 "Expected no operands for ContractionOff");
3129
3130 // We need to save this info for every possible FP type, i.e. {half,
3131 // float, double, fp128}.
3132 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3133 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3134 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3135 Info.ContractionOff = true;
3136 }
3137 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
3138 assert(MDN->getNumOperands() == 3 &&
3139 "Expected 1 operand for SignedZeroInfNanPreserve");
3140 unsigned TargetWidth =
3141 cast<ConstantInt>(
3142 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
3143 ->getZExtValue();
3144 // We need to save this info only for the FP type with TargetWidth.
3145 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3146 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3147 int Index = SPIRV::FPFastMathDefaultInfoVector::
3148 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
3149 assert(Index >= 0 && Index < 3 &&
3150 "Expected FPFastMathDefaultInfo for half, float, or double");
3151 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3152 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3153 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
3154 }
3155 }
3156
3157 DenseMap<unsigned, GlobalVariable *> GlobalVars;
3158 for (auto &[Func, FPFastMathDefaultInfoVec] : FPFastMathDefaultInfoMap) {
3159 if (FPFastMathDefaultInfoVec.empty())
3160 continue;
3161
3162 for (const SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3163 assert(Info.Ty && "Expected target type for FPFastMathDefaultInfo");
3164 // Skip if none of the execution modes was used.
3165 unsigned Flags = Info.FastMathFlags;
3166 if (Flags == SPIRV::FPFastMathMode::None && !Info.ContractionOff &&
3167 !Info.SignedZeroInfNanPreserve && !Info.FPFastMathDefault)
3168 continue;
3169
3170 // Check if flags are compatible.
3171 if (Info.ContractionOff && (Flags & SPIRV::FPFastMathMode::AllowContract))
3172 report_fatal_error(reason: "Conflicting FPFastMathFlags: ContractionOff "
3173 "and AllowContract");
3174
3175 if (Info.SignedZeroInfNanPreserve &&
3176 !(Flags &
3177 (SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
3178 SPIRV::FPFastMathMode::NSZ))) {
3179 if (Info.FPFastMathDefault)
3180 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
3181 "SignedZeroInfNanPreserve but at least one of "
3182 "NotNaN/NotInf/NSZ is enabled.");
3183 }
3184
3185 if ((Flags & SPIRV::FPFastMathMode::AllowTransform) &&
3186 !((Flags & SPIRV::FPFastMathMode::AllowReassoc) &&
3187 (Flags & SPIRV::FPFastMathMode::AllowContract))) {
3188 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
3189 "AllowTransform requires AllowReassoc and "
3190 "AllowContract to be set.");
3191 }
3192
3193 auto it = GlobalVars.find(Val: Flags);
3194 GlobalVariable *GV = nullptr;
3195 if (it != GlobalVars.end()) {
3196 // Reuse existing global variable.
3197 GV = it->second;
3198 } else {
3199 // Create constant instruction with the bitmask flags.
3200 Constant *InitValue =
3201 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: Flags);
3202 // TODO: Reuse constant if there is one already with the required
3203 // value.
3204 GV = new GlobalVariable(M, // Module
3205 Type::getInt32Ty(C&: M.getContext()), // Type
3206 true, // isConstant
3207 GlobalValue::InternalLinkage, // Linkage
3208 InitValue // Initializer
3209 );
3210 GlobalVars[Flags] = GV;
3211 }
3212 }
3213 }
3214}
3215
3216void SPIRVEmitIntrinsicsImpl::processInstrAfterVisit(Instruction *I,
3217 IRBuilder<> &B) {
3218 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3219 bool IsConstComposite =
3220 II && II->getIntrinsicID() == Intrinsic::spv_const_composite;
3221 if (IsConstComposite && TrackConstants) {
3222 setInsertPointAfterDef(B, I);
3223 auto t = AggrConsts.find(Val: I);
3224 assert(t != AggrConsts.end());
3225 auto *NewOp =
3226 buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
3227 Types: {II->getType(), II->getType()}, Arg: t->second, Arg2: I, Imms: {}, B);
3228 replaceAllUsesWith(Src: I, Dest: NewOp, DeleteOld: false);
3229 NewOp->setArgOperand(i: 0, v: I);
3230 }
3231 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
3232 for (const auto &Op : I->operands()) {
3233 if (isa<PHINode>(Val: I) || isa<SwitchInst>(Val: I) ||
3234 !(isa<ConstantData>(Val: Op) || isa<ConstantExpr>(Val: Op)))
3235 continue;
3236 unsigned OpNo = Op.getOperandNo();
3237 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
3238 (!II->isBundleOperand(Idx: OpNo) &&
3239 II->paramHasAttr(ArgNo: OpNo, Kind: Attribute::ImmArg))))
3240 continue;
3241
3242 if (!BPrepared) {
3243 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
3244 : B.SetInsertPoint(I);
3245 BPrepared = true;
3246 }
3247 Type *OpTy = Op->getType();
3248 Type *OpElemTy = GR->findDeducedElementType(Val: Op);
3249 Value *NewOp = Op;
3250 if (OpTy->isTargetExtTy()) {
3251 // Since this value is replaced by poison, we need to do the same in
3252 // `insertAssignTypeIntrs`.
3253 Value *OpTyVal = getNormalizedPoisonValue(Ty: OpTy);
3254 NewOp = buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
3255 Types: {OpTy, OpTyVal->getType()}, Arg: Op, Arg2: OpTyVal, Imms: {}, B);
3256 }
3257 if (!IsConstComposite && isPointerTy(T: OpTy) && OpElemTy != nullptr &&
3258 OpElemTy != IntegerType::getInt8Ty(C&: I->getContext())) {
3259 SmallVector<Type *, 2> Types = {OpTy, OpTy};
3260 SmallVector<Value *, 2> Args = {
3261 NewOp, buildMD(Arg: getNormalizedPoisonValue(Ty: OpElemTy)),
3262 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
3263 CallInst *PtrCasted = B.CreateIntrinsicWithoutFolding(
3264 ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
3265 GR->buildAssignPtr(B, ElemTy: OpElemTy, Arg: PtrCasted);
3266 NewOp = PtrCasted;
3267 }
3268 if (NewOp != Op)
3269 I->setOperand(i: OpNo, Val: NewOp);
3270 }
3271 if (Named.insert(Ptr: I).second)
3272 emitAssignName(I, B);
3273}
3274
3275Type *SPIRVEmitIntrinsicsImpl::deduceFunParamElementType(Function *F,
3276 unsigned OpIdx) {
3277 SmallPtrSet<Function *, 0> FVisited;
3278 return deduceFunParamElementType(F, OpIdx, FVisited);
3279}
3280
3281Type *SPIRVEmitIntrinsicsImpl::deduceFunParamElementType(
3282 Function *F, unsigned OpIdx, SmallPtrSetImpl<Function *> &FVisited) {
3283 // maybe a cycle
3284 if (!FVisited.insert(Ptr: F).second)
3285 return nullptr;
3286
3287 SmallPtrSet<Value *, 0> Visited;
3288 SmallVector<std::pair<Function *, unsigned>> Lookup;
3289 // search in function's call sites
3290 for (User *U : F->users()) {
3291 CallInst *CI = dyn_cast<CallInst>(Val: U);
3292 if (!CI || OpIdx >= CI->arg_size())
3293 continue;
3294 Value *OpArg = CI->getArgOperand(i: OpIdx);
3295 if (!isPointerTy(T: OpArg->getType()))
3296 continue;
3297 // maybe we already know operand's element type
3298 if (Type *KnownTy = GR->findDeducedElementType(Val: OpArg))
3299 return KnownTy;
3300 // try to deduce from the operand itself
3301 Visited.clear();
3302 if (Type *Ty = deduceElementTypeHelper(I: OpArg, Visited, UnknownElemTypeI8: false))
3303 return Ty;
3304 // search in actual parameter's users
3305 for (User *OpU : OpArg->users()) {
3306 Instruction *Inst = dyn_cast<Instruction>(Val: OpU);
3307 if (!Inst || Inst == CI)
3308 continue;
3309 Visited.clear();
3310 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8: false))
3311 return Ty;
3312 }
3313 // check if it's a formal parameter of the outer function
3314 if (!CI->getParent() || !CI->getParent()->getParent())
3315 continue;
3316 Function *OuterF = CI->getParent()->getParent();
3317 if (FVisited.find(Ptr: OuterF) != FVisited.end())
3318 continue;
3319 for (unsigned i = 0; i < OuterF->arg_size(); ++i) {
3320 if (OuterF->getArg(i) == OpArg) {
3321 Lookup.push_back(Elt: std::make_pair(x&: OuterF, y&: i));
3322 break;
3323 }
3324 }
3325 }
3326
3327 // search in function parameters
3328 for (auto &Pair : Lookup) {
3329 if (Type *Ty = deduceFunParamElementType(F: Pair.first, OpIdx: Pair.second, FVisited))
3330 return Ty;
3331 }
3332
3333 return nullptr;
3334}
3335
3336void SPIRVEmitIntrinsicsImpl::processParamTypesByFunHeader(Function *F,
3337 IRBuilder<> &B) {
3338 B.SetInsertPointPastAllocas(F);
3339 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
3340 Argument *Arg = F->getArg(i: OpIdx);
3341 // Vector-of-pointers arg: deduce pointee from a GEP user so the function
3342 // type isn't emitted with the default i8 pointee.
3343 if (isUntypedPointerVectorTy(T: Arg->getType()) &&
3344 !GR->findDeducedElementType(Val: Arg)) {
3345 for (User *U : Arg->users()) {
3346 auto *GEP = dyn_cast<GetElementPtrInst>(Val: U);
3347 if (GEP && GEP->getPointerOperand() == Arg) {
3348 GR->buildAssignPtr(B, ElemTy: GEP->getSourceElementType(), Arg);
3349 break;
3350 }
3351 }
3352 continue;
3353 }
3354 if (!isUntypedPointerTy(T: Arg->getType()))
3355 continue;
3356 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
3357 if (ElemTy)
3358 continue;
3359 if (hasPointeeTypeAttr(Arg) &&
3360 (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) {
3361 GR->buildAssignPtr(B, ElemTy, Arg);
3362 continue;
3363 }
3364 // search in function's call sites
3365 for (User *U : F->users()) {
3366 CallInst *CI = dyn_cast<CallInst>(Val: U);
3367 if (!CI || OpIdx >= CI->arg_size())
3368 continue;
3369 Value *OpArg = CI->getArgOperand(i: OpIdx);
3370 if (!isPointerTy(T: OpArg->getType()))
3371 continue;
3372 // maybe we already know operand's element type
3373 if ((ElemTy = GR->findDeducedElementType(Val: OpArg)) != nullptr)
3374 break;
3375 }
3376 if (ElemTy) {
3377 GR->buildAssignPtr(B, ElemTy, Arg);
3378 continue;
3379 }
3380 if (HaveFunPtrs) {
3381 for (User *U : Arg->users()) {
3382 CallInst *CI = dyn_cast<CallInst>(Val: U);
3383 if (CI && !isa<IntrinsicInst>(Val: CI) && CI->isIndirectCall() &&
3384 CI->getCalledOperand() == Arg &&
3385 CI->getParent()->getParent() == CurrF) {
3386 SmallVector<std::pair<Value *, unsigned>> Ops;
3387 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy&: ElemTy, IsPostprocessing: false);
3388 if (ElemTy) {
3389 GR->buildAssignPtr(B, ElemTy, Arg);
3390 break;
3391 }
3392 }
3393 }
3394 }
3395 }
3396}
3397
3398void SPIRVEmitIntrinsicsImpl::processParamTypes(Function *F, IRBuilder<> &B) {
3399 B.SetInsertPointPastAllocas(F);
3400 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
3401 Argument *Arg = F->getArg(i: OpIdx);
3402 if (!isUntypedPointerTy(T: Arg->getType()))
3403 continue;
3404 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
3405 if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) {
3406 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Arg)) {
3407 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3408 GR->updateAssignType(AssignCI, Arg, OfType: getNormalizedPoisonValue(Ty: ElemTy));
3409 propagateElemType(Op: Arg, ElemTy: IntegerType::getInt8Ty(C&: F->getContext()),
3410 VisitedSubst);
3411 } else {
3412 GR->buildAssignPtr(B, ElemTy, Arg);
3413 }
3414 }
3415 }
3416}
3417
3418static FunctionType *getFunctionPointerElemType(Function *F,
3419 SPIRVGlobalRegistry *GR) {
3420 FunctionType *FTy = F->getFunctionType();
3421 bool IsNewFTy = false;
3422 SmallVector<Type *, 4> ArgTys;
3423 for (Argument &Arg : F->args()) {
3424 Type *ArgTy = Arg.getType();
3425 if (ArgTy->isPointerTy())
3426 if (Type *ElemTy = GR->findDeducedElementType(Val: &Arg)) {
3427 IsNewFTy = true;
3428 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
3429 }
3430 ArgTys.push_back(Elt: ArgTy);
3431 }
3432 return IsNewFTy
3433 ? FunctionType::get(Result: FTy->getReturnType(), Params: ArgTys, isVarArg: FTy->isVarArg())
3434 : FTy;
3435}
3436
3437bool SPIRVEmitIntrinsicsImpl::processFunctionPointers(Module &M) {
3438 SmallVector<Function *> Worklist;
3439 for (auto &F : M) {
3440 if (F.isIntrinsic())
3441 continue;
3442 if (F.isDeclaration()) {
3443 for (User *U : F.users()) {
3444 CallInst *CI = dyn_cast<CallInst>(Val: U);
3445 if (!CI || CI->getCalledFunction() != &F) {
3446 Worklist.push_back(Elt: &F);
3447 break;
3448 }
3449 }
3450 } else {
3451 if (F.user_empty())
3452 continue;
3453 Type *FPElemTy = GR->findDeducedElementType(Val: &F);
3454 if (!FPElemTy)
3455 FPElemTy = getFunctionPointerElemType(F: &F, GR);
3456 for (User *U : F.users()) {
3457 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
3458 if (!II || II->arg_size() != 3 || II->getOperand(i_nocapture: 0) != &F)
3459 continue;
3460 if (II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type ||
3461 II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
3462 GR->updateAssignType(AssignCI: II, Arg: &F, OfType: getNormalizedPoisonValue(Ty: FPElemTy));
3463 break;
3464 }
3465 }
3466 }
3467 }
3468 if (Worklist.empty())
3469 return false;
3470
3471 LLVMContext &Ctx = M.getContext();
3472 Function *SF = getOrCreateBackendServiceFunction(M);
3473 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: SF);
3474 IRBuilder<> IRB(BB);
3475
3476 for (Function *F : Worklist) {
3477 SmallVector<Value *> Args;
3478 for (const auto &Arg : F->args())
3479 Args.push_back(Elt: getNormalizedPoisonValue(Ty: Arg.getType()));
3480 IRB.CreateCall(Callee: F, Args);
3481 }
3482 IRB.CreateRetVoid();
3483
3484 return true;
3485}
3486
3487// Apply types parsed from demangled function declarations.
3488void SPIRVEmitIntrinsicsImpl::applyDemangledPtrArgTypes(IRBuilder<> &B) {
3489 DenseMap<Function *, CallInst *> Ptrcasts;
3490 for (auto It : FDeclPtrTys) {
3491 Function *F = It.first;
3492 for (auto *U : F->users()) {
3493 CallInst *CI = dyn_cast<CallInst>(Val: U);
3494 if (!CI || CI->getCalledFunction() != F)
3495 continue;
3496 unsigned Sz = CI->arg_size();
3497 for (auto [Idx, ElemTy] : It.second) {
3498 if (Idx >= Sz)
3499 continue;
3500 Value *Param = CI->getArgOperand(i: Idx);
3501 if (GR->findDeducedElementType(Val: Param) || isa<GlobalValue>(Val: Param))
3502 continue;
3503 if (Argument *Arg = dyn_cast<Argument>(Val: Param)) {
3504 if (!hasPointeeTypeAttr(Arg)) {
3505 B.SetInsertPointPastAllocas(Arg->getParent());
3506 B.SetCurrentDebugLocation(DebugLoc());
3507 GR->buildAssignPtr(B, ElemTy, Arg);
3508 }
3509 } else if (isaGEP(V: Param)) {
3510 replaceUsesOfWithSpvPtrcast(Op: Param, ElemTy: normalizeType(Ty: ElemTy), I: CI,
3511 Ptrcasts);
3512 } else if (isa<Instruction>(Val: Param)) {
3513 GR->addDeducedElementType(Val: Param, Ty: normalizeType(Ty: ElemTy));
3514 // insertAssignTypeIntrs() will complete buildAssignPtr()
3515 } else {
3516 B.SetInsertPoint(CI->getParent()
3517 ->getParent()
3518 ->getEntryBlock()
3519 .getFirstNonPHIOrDbgOrAlloca());
3520 GR->buildAssignPtr(B, ElemTy, Arg: Param);
3521 }
3522 CallInst *Ref = dyn_cast<CallInst>(Val: Param);
3523 if (!Ref)
3524 continue;
3525 Function *RefF = Ref->getCalledFunction();
3526 if (!RefF || !isPointerTy(T: RefF->getReturnType()) ||
3527 GR->findDeducedElementType(Val: RefF))
3528 continue;
3529 ElemTy = normalizeType(Ty: ElemTy);
3530 GR->addDeducedElementType(Val: RefF, Ty: ElemTy);
3531 GR->addReturnType(
3532 ArgF: RefF, DerivedTy: TypedPointerType::get(
3533 ElementType: ElemTy, AddressSpace: getPointerAddressSpace(T: RefF->getReturnType())));
3534 }
3535 }
3536 }
3537}
3538
3539GetElementPtrInst *SPIRVEmitIntrinsicsImpl::simplifyZeroLengthArrayGepInst(
3540 GetElementPtrInst *GEP) {
3541 // getelementptr [0 x T], P, 0 (zero), I -> getelementptr T, P, I.
3542 // If type is 0-length array and first index is 0 (zero), drop both the
3543 // 0-length array type and the first index. This is a common pattern in
3544 // the IR, e.g. when using a zero-length array as a placeholder for a
3545 // flexible array such as unbound arrays.
3546 assert(GEP && "GEP is null");
3547 Type *SrcTy = GEP->getSourceElementType();
3548 SmallVector<Value *, 8> Indices(GEP->indices());
3549 ArrayType *ArrTy = dyn_cast<ArrayType>(Val: SrcTy);
3550 if (ArrTy && ArrTy->getNumElements() == 0 && match(V: Indices[0], P: m_Zero())) {
3551 Indices.erase(CI: Indices.begin());
3552 SrcTy = ArrTy->getElementType();
3553 return GetElementPtrInst::Create(PointeeType: SrcTy, Ptr: GEP->getPointerOperand(), IdxList: Indices,
3554 NW: GEP->getNoWrapFlags(), NameStr: "",
3555 InsertBefore: GEP->getIterator());
3556 }
3557 return nullptr;
3558}
3559
3560void SPIRVEmitIntrinsicsImpl::emitUnstructuredLoopControls(Function &F,
3561 IRBuilder<> &B) {
3562 const SPIRVSubtarget *ST = TM.getSubtargetImpl(F);
3563 // Shaders use SPIRVStructurizer which emits OpLoopMerge via spv_loop_merge.
3564 if (ST->isShader())
3565 return;
3566
3567 if (ST->canUseExtension(
3568 E: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls)) {
3569 for (BasicBlock &BB : F) {
3570 Instruction *Term = BB.getTerminator();
3571 MDNode *LoopMD = Term->getMetadata(KindID: LLVMContext::MD_loop);
3572 if (!LoopMD)
3573 continue;
3574
3575 SmallVector<unsigned, 1> Ops =
3576 getSpirvLoopControlOperandsFromLoopMetadata(LoopMD);
3577 unsigned LC = Ops[0];
3578 if (LC == SPIRV::LoopControl::None)
3579 continue;
3580
3581 // Emit intrinsic: loop control mask + optional parameters.
3582 B.SetInsertPoint(Term);
3583 SmallVector<Value *, 4> IntrArgs;
3584 for (unsigned Op : Ops)
3585 IntrArgs.push_back(Elt: B.getInt32(C: Op));
3586 B.CreateIntrinsic(ID: Intrinsic::spv_loop_control_intel, Args: IntrArgs);
3587 }
3588 return;
3589 }
3590
3591 // For non-shader targets without the Intel extension, emit OpLoopMerge
3592 // using spv_loop_merge intrinsics, mirroring the structurizer approach.
3593 DominatorTree DT(F);
3594 LoopInfo LI(DT);
3595 if (LI.empty())
3596 return;
3597
3598 for (Loop *L : LI.getLoopsInPreorder()) {
3599 BasicBlock *Latch = L->getLoopLatch();
3600 if (!Latch)
3601 continue;
3602 BasicBlock *MergeBlock = L->getUniqueExitBlock();
3603 if (!MergeBlock)
3604 continue;
3605
3606 // Check for loop unroll metadata on the latch terminator.
3607 SmallVector<unsigned, 1> LoopControlOps =
3608 getSpirvLoopControlOperandsFromLoopMetadata(L);
3609 if (LoopControlOps[0] == SPIRV::LoopControl::None)
3610 continue;
3611
3612 BasicBlock *Header = L->getHeader();
3613 B.SetInsertPoint(Header->getTerminator());
3614 auto *MergeAddress = BlockAddress::get(F: &F, BB: MergeBlock);
3615 auto *ContinueAddress = BlockAddress::get(F: &F, BB: Latch);
3616 SmallVector<Value *, 4> Args = {MergeAddress, ContinueAddress};
3617 for (unsigned Imm : LoopControlOps)
3618 Args.emplace_back(Args: B.getInt32(C: Imm));
3619 B.CreateIntrinsic(ID: Intrinsic::spv_loop_merge, Args: {Args});
3620 }
3621}
3622
3623bool SPIRVEmitIntrinsicsImpl::runOnFunction(Function &Func) {
3624 if (Func.isDeclaration())
3625 return false;
3626
3627 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: Func);
3628 GR = ST.getSPIRVGlobalRegistry();
3629
3630 if (!CurrF)
3631 HaveFunPtrs =
3632 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers);
3633
3634 CurrF = &Func;
3635 IRBuilder<> B(Func.getContext());
3636 AggrConsts.clear();
3637 AggrConstTypes.clear();
3638 AggrStores.clear();
3639
3640 processParamTypesByFunHeader(F: CurrF, B);
3641
3642 // Fix GEP result types ahead of inference, and simplify if possible.
3643 // Data structure for dead instructions that were simplified and replaced.
3644 SmallPtrSet<Instruction *, 4> DeadInsts;
3645 for (auto &I : instructions(F&: Func)) {
3646 auto *GEP = dyn_cast<GetElementPtrInst>(Val: &I);
3647 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
3648
3649 if ((!GEP && !SGEP) || GR->findDeducedElementType(Val: &I))
3650 continue;
3651
3652 if (SGEP) {
3653 GR->addDeducedElementType(Val: SGEP,
3654 Ty: normalizeType(Ty: SGEP->getResultElementType()));
3655 continue;
3656 }
3657
3658 GetElementPtrInst *NewGEP = simplifyZeroLengthArrayGepInst(GEP);
3659 if (NewGEP) {
3660 GEP->replaceAllUsesWith(V: NewGEP);
3661 DeadInsts.insert(Ptr: GEP);
3662 GEP = NewGEP;
3663 }
3664 if (Type *GepTy = getGEPType(Ref: GEP))
3665 GR->addDeducedElementType(Val: GEP, Ty: normalizeType(Ty: GepTy));
3666 }
3667 // Remove dead instructions that were simplified and replaced.
3668 for (auto *I : DeadInsts) {
3669 assert(I->use_empty() && "Dead instruction should not have any uses left");
3670 I->eraseFromParent();
3671 }
3672
3673 // StoreInst's operand type can be changed during the next
3674 // transformations, so we need to store it in the set. Also store already
3675 // transformed types.
3676 for (auto &I : instructions(F&: Func)) {
3677 StoreInst *SI = dyn_cast<StoreInst>(Val: &I);
3678 if (!SI)
3679 continue;
3680 Type *ElTy = SI->getValueOperand()->getType();
3681 if (ElTy->isAggregateType() || ElTy->isVectorTy())
3682 AggrStores.insert(Ptr: &I);
3683 }
3684
3685 B.SetInsertPoint(TheBB: &Func.getEntryBlock(), IP: Func.getEntryBlock().begin());
3686 for (auto &GV : Func.getParent()->globals())
3687 processGlobalValue(GV, B);
3688
3689 reconstructAggregateReturns(Func, B);
3690 preprocessUndefsAndPoisons(B);
3691 simplifyNullAddrSpaceCasts();
3692 preprocessCompositeConstants(B);
3693
3694 // A PHINode, SelectInst or FreezeInst takes its result type from its
3695 // operands. Aggregate arms are lowered to i32 value-ids (composite constants
3696 // here, loads and other producers during the visitor pass below), so mutate
3697 // an aggregate PHI, select or freeze to match. The original type is tracked
3698 // in AggrConstTypes (used to assign the SPIR-V type) and its extractvalue
3699 // users are lowered to spv_extractv.
3700 Type *I32Ty = B.getInt32Ty();
3701 for (Instruction &I : instructions(F&: Func)) {
3702 if (!isAggregateValueIdInstr(I))
3703 continue;
3704 // Give multi-register arms a value-id first, before the result is mutated.
3705 insertCompositeAggregateArms(I: &I, B);
3706 AggrConstTypes[&I] = I.getType();
3707 I.mutateType(Ty: I32Ty);
3708 }
3709
3710 preprocessBoolVectorBitcasts(F&: Func);
3711 SmallVector<Instruction *> Worklist(
3712 llvm::make_pointer_range(Range: instructions(F&: Func)));
3713
3714 applyDemangledPtrArgTypes(B);
3715
3716 // Pass forward: use operand to deduce instructions result.
3717 for (auto &I : Worklist) {
3718 // Don't emit intrinsincs for convergence intrinsics.
3719 if (isConvergenceIntrinsic(I))
3720 continue;
3721
3722 bool Postpone = insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: false);
3723 // if Postpone is true, we can't decide on pointee type yet
3724 insertAssignTypeIntrs(I, B);
3725 insertPtrCastOrAssignTypeInstr(I, B);
3726 insertSpirvDecorations(I, B);
3727 // if instruction requires a pointee type set, let's check if we know it
3728 // already, and force it to be i8 if not
3729 if (Postpone && !GR->findAssignPtrTypeInstr(Val: I))
3730 insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: true);
3731
3732 if (auto *FPI = dyn_cast<ConstrainedFPIntrinsic>(Val: I))
3733 useRoundingMode(FPI, B);
3734 }
3735
3736 // Pass backward: use instructions results to specify/update/cast operands
3737 // where needed.
3738 SmallPtrSet<Instruction *, 4> IncompleteRets;
3739 for (auto &I : llvm::reverse(C: instructions(F&: Func)))
3740 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets);
3741
3742 // Pass forward for PHIs only, their operands are not preceed the
3743 // instruction in meaning of `instructions(Func)`.
3744 for (BasicBlock &BB : Func)
3745 for (PHINode &Phi : BB.phis())
3746 if (isPointerTy(T: Phi.getType()))
3747 deduceOperandElementType(I: &Phi, IncompleteRets: nullptr);
3748
3749 for (auto *I : Worklist) {
3750 TrackConstants = true;
3751 if (!I->getType()->isVoidTy() || isa<StoreInst>(Val: I))
3752 setInsertPointAfterDef(B, I);
3753 // Visitors return either the original/newly created instruction for
3754 // further processing, nullptr otherwise.
3755 I = visit(I&: *I);
3756 if (!I)
3757 continue;
3758
3759 // Don't emit intrinsics for convergence operations.
3760 if (isConvergenceIntrinsic(I))
3761 continue;
3762
3763 addSaturatedDecorationToIntrinsic(I, B);
3764 processInstrAfterVisit(I, B);
3765 }
3766
3767 emitUnstructuredLoopControls(F&: Func, B);
3768
3769 return true;
3770}
3771
3772// Try to deduce a better type for pointers to untyped ptr.
3773bool SPIRVEmitIntrinsicsImpl::postprocessTypes(Module &M) {
3774 if (!GR || TodoTypeSz == 0)
3775 return false;
3776
3777 unsigned SzTodo = TodoTypeSz;
3778 DenseMap<Value *, SmallPtrSet<Value *, 4>> ToProcess;
3779 for (auto [Op, Enabled] : TodoType) {
3780 // TODO: add isa<CallInst>(Op) to continue
3781 if (!Enabled || isaGEP(V: Op))
3782 continue;
3783 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Op);
3784 Type *KnownTy = GR->findDeducedElementType(Val: Op);
3785 if (!KnownTy || !AssignCI)
3786 continue;
3787 assert(Op == AssignCI->getArgOperand(0));
3788 // Try to improve the type deduced after all Functions are processed.
3789 if (auto *CI = dyn_cast<Instruction>(Val: Op)) {
3790 CurrF = CI->getParent()->getParent();
3791 SmallPtrSet<Value *, 0> Visited;
3792 if (Type *ElemTy = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8: false, IgnoreKnownType: true)) {
3793 if (ElemTy != KnownTy) {
3794 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3795 propagateElemType(Op: CI, ElemTy, VisitedSubst);
3796 eraseTodoType(Op);
3797 continue;
3798 }
3799 }
3800 }
3801
3802 if (Op->hasUseList()) {
3803 for (User *U : Op->users()) {
3804 Instruction *Inst = dyn_cast<Instruction>(Val: U);
3805 if (Inst && !isa<IntrinsicInst>(Val: Inst))
3806 ToProcess[Inst].insert(Ptr: Op);
3807 }
3808 }
3809 }
3810 if (TodoTypeSz == 0)
3811 return true;
3812
3813 for (auto &F : M) {
3814 CurrF = &F;
3815 SmallPtrSet<Instruction *, 4> IncompleteRets;
3816 for (auto &I : llvm::reverse(C: instructions(F))) {
3817 auto It = ToProcess.find(Val: &I);
3818 if (It == ToProcess.end())
3819 continue;
3820 It->second.remove_if(P: [this](Value *V) { return !isTodoType(Op: V); });
3821 if (It->second.size() == 0)
3822 continue;
3823 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets, AskOps: &It->second, IsPostprocessing: true);
3824 if (TodoTypeSz == 0)
3825 return true;
3826 }
3827 }
3828
3829 return SzTodo > TodoTypeSz;
3830}
3831
3832// Parse and store argument types of function declarations where needed.
3833void SPIRVEmitIntrinsicsImpl::parseFunDeclarations(Module &M) {
3834 for (auto &F : M) {
3835 if (!F.isDeclaration() || F.isIntrinsic())
3836 continue;
3837 // get the demangled name
3838 std::string DemangledName = getOclOrSpirvBuiltinDemangledName(Name: F.getName());
3839 if (DemangledName.empty())
3840 continue;
3841 // allow only OpGroupAsyncCopy use case at the moment
3842 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F);
3843 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
3844 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
3845 if (Opcode != SPIRV::OpGroupAsyncCopy)
3846 continue;
3847 // find pointer arguments
3848 SmallVector<unsigned> Idxs;
3849 for (unsigned OpIdx = 0; OpIdx < F.arg_size(); ++OpIdx) {
3850 Argument *Arg = F.getArg(i: OpIdx);
3851 if (isPointerTy(T: Arg->getType()) && !hasPointeeTypeAttr(Arg))
3852 Idxs.push_back(Elt: OpIdx);
3853 }
3854 if (!Idxs.size())
3855 continue;
3856 // parse function arguments
3857 LLVMContext &Ctx = F.getContext();
3858 SmallVector<StringRef, 10> TypeStrs;
3859 SPIRV::parseBuiltinTypeStr(BuiltinArgsTypeStrs&: TypeStrs, DemangledCall: DemangledName, Ctx);
3860 if (!TypeStrs.size())
3861 continue;
3862 // find type info for pointer arguments
3863 for (unsigned Idx : Idxs) {
3864 if (Idx >= TypeStrs.size())
3865 continue;
3866 if (Type *ElemTy =
3867 SPIRV::parseBuiltinCallArgumentType(TypeStr: TypeStrs[Idx].trim(), Ctx))
3868 if (TypedPointerType::isValidElementType(ElemTy) &&
3869 !ElemTy->isTargetExtTy())
3870 FDeclPtrTys[&F].push_back(Elt: std::make_pair(x&: Idx, y&: ElemTy));
3871 }
3872 }
3873}
3874
3875bool SPIRVEmitIntrinsicsImpl::processMaskedMemIntrinsic(IntrinsicInst &I) {
3876 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
3877
3878 if (I.getIntrinsicID() == Intrinsic::masked_gather) {
3879 if (!ST.canUseExtension(
3880 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
3881 I.getContext().emitError(
3882 I: &I, ErrorStr: "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter "
3883 "extension");
3884 // Replace with poison to allow compilation to continue and report error.
3885 I.replaceAllUsesWith(V: PoisonValue::get(T: I.getType()));
3886 I.eraseFromParent();
3887 return true;
3888 }
3889
3890 IRBuilder<> B(&I);
3891
3892 Value *Ptrs = I.getArgOperand(i: 0);
3893 Value *Mask = I.getArgOperand(i: 1);
3894 Value *Passthru = I.getArgOperand(i: 2);
3895
3896 // Alignment is stored as a parameter attribute, not as a regular parameter.
3897 uint32_t Alignment = I.getParamAlign(ArgNo: 0).valueOrOne().value();
3898
3899 SmallVector<Value *, 4> Args = {Ptrs, B.getInt32(C: Alignment), Mask,
3900 Passthru};
3901 SmallVector<Type *, 4> Types = {I.getType(), Ptrs->getType(),
3902 Mask->getType(), Passthru->getType()};
3903
3904 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_masked_gather, OverloadTypes: Types, Args);
3905 I.replaceAllUsesWith(V: NewI);
3906 I.eraseFromParent();
3907 return true;
3908 }
3909
3910 if (I.getIntrinsicID() == Intrinsic::masked_scatter) {
3911 if (!ST.canUseExtension(
3912 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
3913 I.getContext().emitError(
3914 I: &I, ErrorStr: "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter "
3915 "extension");
3916 // Erase the intrinsic to allow compilation to continue and report error.
3917 I.eraseFromParent();
3918 return true;
3919 }
3920
3921 IRBuilder<> B(&I);
3922
3923 Value *Values = I.getArgOperand(i: 0);
3924 Value *Ptrs = I.getArgOperand(i: 1);
3925 Value *Mask = I.getArgOperand(i: 2);
3926
3927 // Alignment is stored as a parameter attribute on the ptrs parameter (arg
3928 // 1).
3929 uint32_t Alignment = I.getParamAlign(ArgNo: 1).valueOrOne().value();
3930
3931 SmallVector<Value *, 4> Args = {Values, Ptrs, B.getInt32(C: Alignment), Mask};
3932 SmallVector<Type *, 3> Types = {Values->getType(), Ptrs->getType(),
3933 Mask->getType()};
3934
3935 B.CreateIntrinsic(ID: Intrinsic::spv_masked_scatter, OverloadTypes: Types, Args);
3936 I.eraseFromParent();
3937 return true;
3938 }
3939
3940 return false;
3941}
3942
3943// SPIR-V doesn't support bitcasts involving vector boolean type. Decompose such
3944// bitcasts into element-wise operations before building instructions
3945// worklist, so new instructions are properly visited and converted to
3946// SPIR-V intrinsics.
3947void SPIRVEmitIntrinsicsImpl::preprocessBoolVectorBitcasts(Function &F) {
3948 struct BoolVecBitcast {
3949 BitCastInst *BC;
3950 FixedVectorType *BoolVecTy;
3951 bool SrcIsBoolVec;
3952 };
3953
3954 auto getAsBoolVec = [](Type *Ty) -> FixedVectorType * {
3955 auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
3956 return (VTy && VTy->getElementType()->isIntegerTy(BitWidth: 1)) ? VTy : nullptr;
3957 };
3958
3959 SmallVector<BoolVecBitcast, 4> ToReplace;
3960 for (auto &I : instructions(F)) {
3961 auto *BC = dyn_cast<BitCastInst>(Val: &I);
3962 if (!BC)
3963 continue;
3964 if (auto *BVTy = getAsBoolVec(BC->getSrcTy()))
3965 ToReplace.push_back(Elt: {.BC: BC, .BoolVecTy: BVTy, .SrcIsBoolVec: true});
3966 else if (auto *BVTy = getAsBoolVec(BC->getDestTy()))
3967 ToReplace.push_back(Elt: {.BC: BC, .BoolVecTy: BVTy, .SrcIsBoolVec: false});
3968 }
3969
3970 for (auto &[BC, BoolVecTy, SrcIsBoolVec] : ToReplace) {
3971 IRBuilder<> B(BC);
3972 Value *Src = BC->getOperand(i_nocapture: 0);
3973 unsigned BoolVecN = BoolVecTy->getNumElements();
3974 // Use iN as the scalar intermediate type for the bool vector side.
3975 Type *IntTy = B.getIntNTy(N: BoolVecN);
3976
3977 // Convert source to scalar integer.
3978 Value *IntVal;
3979 if (SrcIsBoolVec) {
3980 // Extract each bool, zext, shift, and OR.
3981 IntVal = ConstantInt::get(Ty: IntTy, V: 0);
3982 for (unsigned I = 0; I < BoolVecN; ++I) {
3983 Value *Elem = B.CreateExtractElement(Vec: Src, Idx: B.getInt32(C: I));
3984 Value *Ext = B.CreateZExt(V: Elem, DestTy: IntTy);
3985 if (I > 0)
3986 Ext = B.CreateShl(LHS: Ext, RHS: ConstantInt::get(Ty: IntTy, V: I));
3987 IntVal = B.CreateOr(LHS: IntVal, RHS: Ext);
3988 }
3989 } else {
3990 // Source is a non-bool type. If it's already a scalar integer, use it
3991 // directly, otherwise bitcast to iN first.
3992 IntVal = Src;
3993 if (!Src->getType()->isIntegerTy())
3994 IntVal = B.CreateBitCast(V: Src, DestTy: IntTy);
3995 }
3996
3997 // Convert scalar integer to destination type.
3998 Value *Result;
3999 if (!SrcIsBoolVec) {
4000 // Test each bit with AND + icmp.
4001 Result = PoisonValue::get(T: BoolVecTy);
4002 for (unsigned I = 0; I < BoolVecN; ++I) {
4003 Value *Mask = ConstantInt::get(Ty: IntTy, V: APInt::getOneBitSet(numBits: BoolVecN, BitNo: I));
4004 Value *And = B.CreateAnd(LHS: IntVal, RHS: Mask);
4005 Value *Cmp = B.CreateICmpNE(LHS: And, RHS: ConstantInt::get(Ty: IntTy, V: 0));
4006 Result = B.CreateInsertElement(Vec: Result, NewElt: Cmp, Idx: B.getInt32(C: I));
4007 }
4008 } else {
4009 // Destination is a non-bool type. If it's a scalar integer, use IntVal
4010 // directly, otherwise bitcast from iN.
4011 Result = IntVal;
4012 if (!BC->getDestTy()->isIntegerTy())
4013 Result = B.CreateBitCast(V: IntVal, DestTy: BC->getDestTy());
4014 }
4015
4016 BC->replaceAllUsesWith(V: Result);
4017 BC->eraseFromParent();
4018 }
4019}
4020
4021bool SPIRVEmitIntrinsicsImpl::convertMaskedMemIntrinsics(Module &M) {
4022 bool Changed = false;
4023
4024 for (Function &F : make_early_inc_range(Range&: M)) {
4025 if (!F.isIntrinsic())
4026 continue;
4027 Intrinsic::ID IID = F.getIntrinsicID();
4028 if (IID != Intrinsic::masked_gather && IID != Intrinsic::masked_scatter)
4029 continue;
4030
4031 for (User *U : make_early_inc_range(Range: F.users())) {
4032 if (auto *II = dyn_cast<IntrinsicInst>(Val: U))
4033 Changed |= processMaskedMemIntrinsic(I&: *II);
4034 }
4035
4036 if (F.use_empty())
4037 F.eraseFromParent();
4038 }
4039
4040 return Changed;
4041}
4042
4043bool SPIRVEmitIntrinsicsImpl::runOnModule(Module &M) {
4044 bool Changed = false;
4045
4046 Changed |= convertMaskedMemIntrinsics(M);
4047
4048 parseFunDeclarations(M);
4049 insertConstantsForFPFastMathDefault(M);
4050 GVUsers.init(M);
4051
4052 TodoType.clear();
4053 for (auto &F : M)
4054 Changed |= runOnFunction(Func&: F);
4055
4056 // Specify function parameters after all functions were processed.
4057 for (auto &F : M) {
4058 // check if function parameter types are set
4059 CurrF = &F;
4060 if (!F.isDeclaration() && !F.isIntrinsic()) {
4061 IRBuilder<> B(F.getContext());
4062 processParamTypes(F: &F, B);
4063 }
4064 }
4065
4066 CanTodoType = false;
4067 Changed |= postprocessTypes(M);
4068
4069 if (HaveFunPtrs)
4070 Changed |= processFunctionPointers(M);
4071
4072 return Changed;
4073}
4074
4075PreservedAnalyses
4076llvm::SPIRVEmitIntrinsicsPass::run(Module &M, ModuleAnalysisManager &AM) {
4077 if (SPIRVEmitIntrinsicsImpl(TM).runOnModule(M))
4078 return PreservedAnalyses::none();
4079 return PreservedAnalyses::all();
4080}
4081
4082ModulePass *llvm::createSPIRVEmitIntrinsicsPass(const SPIRVTargetMachine &TM) {
4083 return new SPIRVEmitIntrinsicsLegacy(TM);
4084}
4085