1//===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass emits SPIRV intrinsics keeping essential high-level information for
10// the translation of LLVM IR to SPIR-V.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRV.h"
15#include "SPIRVBuiltins.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVTargetMachine.h"
18#include "SPIRVUtils.h"
19#include "llvm/ADT/DenseMap.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/Analysis/LoopInfo.h"
24#include "llvm/IR/IRBuilder.h"
25#include "llvm/IR/InstIterator.h"
26#include "llvm/IR/InstVisitor.h"
27#include "llvm/IR/IntrinsicsSPIRV.h"
28#include "llvm/IR/LLVMContext.h"
29#include "llvm/IR/PatternMatch.h"
30#include "llvm/IR/TypedPointerType.h"
31#include "llvm/IR/Value.h"
32#include "llvm/Support/CommandLine.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/raw_ostream.h"
35#include "llvm/Transforms/Utils/Local.h"
36
37#include <cassert>
38#include <optional>
39#include <queue>
40
41// This pass performs the following transformation on LLVM IR level required
42// for the following translation to SPIR-V:
43// - replaces direct usages of aggregate constants with target-specific
44// intrinsics;
45// - replaces aggregates-related instructions (extract/insert, ld/st, etc)
46// with a target-specific intrinsics;
47// - emits intrinsics for the global variable initializers since IRTranslator
48// doesn't handle them and it's not very convenient to translate them
49// ourselves;
50// - emits intrinsics to keep track of the string names assigned to the values;
51// - emits intrinsics to keep track of constants (this is necessary to have an
52// LLVM IR constant after the IRTranslation is completed) for their further
53// deduplication;
54// - emits intrinsics to keep track of original LLVM types of the values
55// to be able to emit proper SPIR-V types eventually.
56//
57// TODO: consider removing spv.track.constant in favor of spv.assign.type.
58
59using namespace llvm;
60using namespace llvm::PatternMatch;
61
62#define DEBUG_TYPE "spirv-emit-intrinsics"
63
64static cl::opt<bool>
65 SpirvEmitOpNames("spirv-emit-op-names",
66 cl::desc("Emit OpName for all instructions"),
67 cl::init(Val: false));
68
69namespace llvm::SPIRV {
70#define GET_BuiltinGroup_DECL
71#include "SPIRVGenTables.inc"
72} // namespace llvm::SPIRV
73
74namespace {
75// This class keeps track of which functions reference which global variables.
76class GlobalVariableUsers {
77 template <typename T1, typename T2>
78 using OneToManyMapTy = DenseMap<T1, SmallPtrSet<T2, 4>>;
79
80 OneToManyMapTy<const GlobalVariable *, const Function *> GlobalIsUsedByFun;
81
82 void collectGlobalUsers(
83 const GlobalVariable *GV,
84 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
85 &GlobalIsUsedByGlobal) {
86 SmallVector<const Value *> Stack = {GV->user_begin(), GV->user_end()};
87 while (!Stack.empty()) {
88 const Value *V = Stack.pop_back_val();
89
90 if (const Instruction *I = dyn_cast<Instruction>(Val: V)) {
91 GlobalIsUsedByFun[GV].insert(Ptr: I->getFunction());
92 continue;
93 }
94
95 if (const GlobalVariable *UserGV = dyn_cast<GlobalVariable>(Val: V)) {
96 GlobalIsUsedByGlobal[GV].insert(Ptr: UserGV);
97 continue;
98 }
99
100 if (const Constant *C = dyn_cast<Constant>(Val: V))
101 Stack.append(in_start: C->user_begin(), in_end: C->user_end());
102 }
103 }
104
105 bool propagateGlobalToGlobalUsers(
106 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
107 &GlobalIsUsedByGlobal) {
108 SmallVector<const GlobalVariable *> OldUsersGlobals;
109 bool Changed = false;
110 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
111 OldUsersGlobals.assign(in_start: UserGlobals.begin(), in_end: UserGlobals.end());
112 for (const GlobalVariable *UserGV : OldUsersGlobals) {
113 auto It = GlobalIsUsedByGlobal.find(Val: UserGV);
114 if (It == GlobalIsUsedByGlobal.end())
115 continue;
116 Changed |= set_union(S1&: UserGlobals, S2: It->second);
117 }
118 }
119 return Changed;
120 }
121
122 void propagateGlobalToFunctionReferences(
123 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
124 &GlobalIsUsedByGlobal) {
125 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
126 auto &UserFunctions = GlobalIsUsedByFun[GV];
127 for (const GlobalVariable *UserGV : UserGlobals) {
128 auto It = GlobalIsUsedByFun.find(Val: UserGV);
129 if (It == GlobalIsUsedByFun.end())
130 continue;
131 set_union(S1&: UserFunctions, S2: It->second);
132 }
133 }
134 }
135
136public:
137 void init(Module &M) {
138 // Collect which global variables are referenced by which global variables
139 // and which functions reference each global variables.
140 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
141 GlobalIsUsedByGlobal;
142 GlobalIsUsedByFun.clear();
143 for (GlobalVariable &GV : M.globals())
144 collectGlobalUsers(GV: &GV, GlobalIsUsedByGlobal);
145
146 // Compute indirect references by iterating until a fixed point is reached.
147 while (propagateGlobalToGlobalUsers(GlobalIsUsedByGlobal))
148 (void)0;
149
150 propagateGlobalToFunctionReferences(GlobalIsUsedByGlobal);
151 }
152
153 using FunctionSetType = typename decltype(GlobalIsUsedByFun)::mapped_type;
154 const FunctionSetType &
155 getTransitiveUserFunctions(const GlobalVariable &GV) const {
156 auto It = GlobalIsUsedByFun.find(Val: &GV);
157 if (It != GlobalIsUsedByFun.end())
158 return It->second;
159
160 static const FunctionSetType Empty{};
161 return Empty;
162 }
163};
164
165static bool isaGEP(const Value *V) {
166 return isa<StructuredGEPInst>(Val: V) || isa<GetElementPtrInst>(Val: V);
167}
168
169// If Ty is a byte-addressing type, return the multiplier for the offset.
170// Otherwise return std::nullopt.
171static std::optional<uint64_t> getByteAddressingMultiplier(Type *Ty) {
172 if (Ty == IntegerType::getInt8Ty(C&: Ty->getContext())) {
173 return 1;
174 }
175 if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
176 if (AT->getElementType() == IntegerType::getInt8Ty(C&: Ty->getContext())) {
177 return AT->getNumElements();
178 }
179 }
180 return std::nullopt;
181}
182
183class SPIRVEmitIntrinsicsImpl
184 : public InstVisitor<SPIRVEmitIntrinsicsImpl, Instruction *> {
185 const SPIRVTargetMachine &TM;
186 SPIRVGlobalRegistry *GR = nullptr;
187 Function *CurrF = nullptr;
188 bool TrackConstants = true;
189 bool HaveFunPtrs = false;
190 bool CanUseAnyVectorRank = false;
191 DenseMap<Instruction *, Constant *> AggrConsts;
192 DenseMap<Instruction *, Type *> AggrConstTypes;
193 SmallPtrSet<Instruction *, 0> AggrStores;
194 GlobalVariableUsers GVUsers;
195 SmallPtrSet<Value *, 0> Named;
196
197 // map of function declarations to <pointer arg index => element type>
198 DenseMap<Function *, SmallVector<std::pair<unsigned, Type *>>> FDeclPtrTys;
199
200 // a register of Instructions that don't have a complete type definition
201 bool CanTodoType = true;
202 unsigned TodoTypeSz = 0;
203 DenseMap<Value *, bool> TodoType;
204 void insertTodoType(Value *Op) {
205 // TODO: add isa<CallInst>(Op) to no-insert
206 if (CanTodoType && !isaGEP(V: Op)) {
207 auto It = TodoType.try_emplace(Key: Op, Args: true);
208 if (It.second)
209 ++TodoTypeSz;
210 }
211 }
212 void eraseTodoType(Value *Op) {
213 auto It = TodoType.find(Val: Op);
214 if (It != TodoType.end() && It->second) {
215 It->second = false;
216 --TodoTypeSz;
217 }
218 }
219 bool isTodoType(Value *Op) {
220 if (isaGEP(V: Op))
221 return false;
222 auto It = TodoType.find(Val: Op);
223 return It != TodoType.end() && It->second;
224 }
225 // a register of Instructions that were visited by deduceOperandElementType()
226 // to validate operand types with an instruction
227 SmallPtrSet<Instruction *, 0> TypeValidated;
228
229 // well known result types of builtins
230 enum WellKnownTypes { Event };
231
232 // deduce element type of untyped pointers
233 Type *deduceElementType(Value *I, bool UnknownElemTypeI8);
234 Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8);
235 Type *deduceElementTypeHelper(Value *I, SmallPtrSetImpl<Value *> &Visited,
236 bool UnknownElemTypeI8,
237 bool IgnoreKnownType = false);
238 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
239 bool UnknownElemTypeI8);
240 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
241 SmallPtrSetImpl<Value *> &Visited,
242 bool UnknownElemTypeI8);
243 Type *deduceElementTypeByUsersDeep(Value *Op,
244 SmallPtrSetImpl<Value *> &Visited,
245 bool UnknownElemTypeI8);
246 void maybeAssignPtrType(Type *&Ty, Value *I, Type *RefTy,
247 bool UnknownElemTypeI8);
248
249 // deduce nested types of composites
250 Type *deduceNestedTypeHelper(User *U, bool UnknownElemTypeI8);
251 Type *deduceNestedTypeHelper(User *U, Type *Ty,
252 SmallPtrSetImpl<Value *> &Visited,
253 bool UnknownElemTypeI8);
254
255 // deduce Types of operands of the Instruction if possible
256 void
257 deduceOperandElementType(Instruction *I,
258 SmallPtrSetImpl<Instruction *> *IncompleteRets,
259 const SmallPtrSetImpl<Value *> *AskOps = nullptr,
260 bool IsPostprocessing = false);
261
262 void preprocessCompositeConstants(IRBuilder<> &B);
263 Value *lowerUndefOrPoison(Value *Op, IRBuilder<> &B, bool HasPoisonExt);
264 void preprocessUndefsAndPoisons(IRBuilder<> &B);
265 void insertCompositeAggregateArms(Instruction *I, IRBuilder<> &B);
266 void simplifyNullAddrSpaceCasts();
267
268 Type *reconstructType(Value *Op, bool UnknownElemTypeI8,
269 bool IsPostprocessing);
270
271 void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
272 void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
273 bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
274 bool UnknownElemTypeI8);
275 void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B);
276 void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V,
277 IRBuilder<> &B);
278 void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer,
279 Type *ExpectedElementType,
280 unsigned OperandToReplace,
281 IRBuilder<> &B);
282 void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
283 bool shouldTryToAddMemAliasingDecoration(Instruction *Inst);
284 void insertSpirvDecorations(Instruction *I, IRBuilder<> &B);
285 void insertConstantsForFPFastMathDefault(Module &M);
286 Value *buildSpvUndefComposite(Type *AggrTy, IRBuilder<> &B);
287 void reconstructAggregateReturns(Function &Func, IRBuilder<> &B);
288 void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
289 void processParamTypes(Function *F, IRBuilder<> &B);
290 void processParamTypesByFunHeader(Function *F, IRBuilder<> &B);
291 Type *deduceFunParamElementType(Function *F, unsigned OpIdx);
292 Type *deduceFunParamElementType(Function *F, unsigned OpIdx,
293 SmallPtrSetImpl<Function *> &FVisited);
294
295 bool deduceOperandElementTypeCalledFunction(
296 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
297 Type *&KnownElemTy, bool &Incomplete);
298 void deduceOperandElementTypeFunctionPointer(
299 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
300 Type *&KnownElemTy, bool IsPostprocessing);
301 bool deduceOperandElementTypeFunctionRet(
302 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
303 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing,
304 Type *&KnownElemTy, Value *Op, Function *F);
305
306 CallInst *buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy);
307 void replaceUsesOfWithSpvPtrcast(Value *Op, Type *ElemTy, Instruction *I,
308 DenseMap<Function *, CallInst *> Ptrcasts);
309 void propagateElemType(Value *Op, Type *ElemTy,
310 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
311 void
312 propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
313 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
314 void propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
315 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
316 SmallPtrSetImpl<Value *> &Visited,
317 DenseMap<Function *, CallInst *> Ptrcasts);
318
319 void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
320 void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src,
321 Instruction *Dest, bool DeleteOld = true);
322
323 void applyDemangledPtrArgTypes(IRBuilder<> &B);
324
325 GetElementPtrInst *simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP);
326
327 bool runOnFunction(Function &F);
328 bool postprocessTypes(Module &M);
329 bool processFunctionPointers(Module &M);
330 void parseFunDeclarations(Module &M);
331 void useRoundingMode(ConstrainedFPIntrinsic *FPI, IRBuilder<> &B);
332 bool processMaskedMemIntrinsic(IntrinsicInst &I);
333 bool convertMaskedMemIntrinsics(Module &M);
334 void preprocessBoolVectorBitcasts(Function &F);
335
336 void emitUnstructuredLoopControls(Function &F, IRBuilder<> &B);
337
338 // Tries to walk the type accessed by the given GEP instruction.
339 // For each nested type access, one of the 2 callbacks is called:
340 // - OnLiteralIndexing when the index is a known constant value.
341 // Parameters:
342 // PointedType: the pointed type resulting of this indexing.
343 // If the parent type is an array, this is the index in the array.
344 // If the parent type is a struct, this is the field index.
345 // Index: index of the element in the parent type.
346 // - OnDynamnicIndexing when the index is a non-constant value.
347 // This callback is only called when indexing into an array.
348 // Parameters:
349 // ElementType: the type of the elements stored in the parent array.
350 // Offset: the Value* containing the byte offset into the array.
351 // Multiplier: a scaling factor for the offset.
352 // Return true if an error occurred during the walk, false otherwise.
353 bool walkLogicalAccessChain(
354 GetElementPtrInst &GEP,
355 const std::function<void(Type *PointedType, uint64_t Index)>
356 &OnLiteralIndexing,
357 const std::function<void(Type *ElementType, Value *Offset,
358 uint64_t Multiplier)> &OnDynamicIndexing);
359
360 bool walkLogicalAccessChainDynamic(
361 Type *CurType, Value *Operand, uint64_t Multiplier,
362 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
363 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing);
364
365 bool walkLogicalAccessChainConstant(
366 Type *CurType, uint64_t Offset,
367 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing);
368
369 // Returns the type accessed using the given GEP instruction by relying
370 // on the GEP type.
371 // FIXME: GEP types are not supposed to be used to retrieve the pointed
372 // type. This must be fixed.
373 Type *getGEPType(GetElementPtrInst *GEP);
374
375 // Returns the type accessed using the given GEP instruction by walking
376 // the source type using the GEP indices.
377 // FIXME: without help from the frontend, this method cannot reliably retrieve
378 // the stored type, nor can robustly determine the depth of the type
379 // we are accessing.
380 Type *getGEPTypeLogical(GetElementPtrInst *GEP);
381
382 Instruction *buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP);
383
384public:
385 SPIRVEmitIntrinsicsImpl(const SPIRVTargetMachine &TM) : TM(TM) {}
386 Instruction *visitInstruction(Instruction &I) { return &I; }
387 Instruction *visitSwitchInst(SwitchInst &I);
388 Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
389 Instruction *visitIntrinsicInst(IntrinsicInst &I);
390 Instruction *visitBitCastInst(BitCastInst &I);
391 Instruction *visitInsertElementInst(InsertElementInst &I);
392 Instruction *visitExtractElementInst(ExtractElementInst &I);
393 Instruction *visitInsertValueInst(InsertValueInst &I);
394 Instruction *visitExtractValueInst(ExtractValueInst &I);
395 Instruction *visitLoadInst(LoadInst &I);
396 Instruction *visitStoreInst(StoreInst &I);
397 Instruction *visitAllocaInst(AllocaInst &I);
398 Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
399 Instruction *visitAtomicRMWInst(AtomicRMWInst &I);
400 Instruction *visitUnreachableInst(UnreachableInst &I);
401 Instruction *visitCallInst(CallInst &I);
402
403 bool runOnModule(Module &M);
404};
405
406class SPIRVEmitIntrinsicsLegacy : public ModulePass {
407 const SPIRVTargetMachine &TM;
408
409public:
410 static char ID;
411 SPIRVEmitIntrinsicsLegacy(const SPIRVTargetMachine &TM)
412 : ModulePass(ID), TM(TM) {}
413
414 StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
415
416 bool runOnModule(Module &M) override {
417 return SPIRVEmitIntrinsicsImpl(TM).runOnModule(M);
418 }
419};
420
421bool isConvergenceIntrinsic(const Instruction *I) {
422 return match(V: I, P: m_AnyIntrinsic<Intrinsic::experimental_convergence_entry,
423 Intrinsic::experimental_convergence_loop,
424 Intrinsic::experimental_convergence_anchor>());
425}
426
427bool expectIgnoredInIRTranslation(const Instruction *I) {
428 return match(V: I, P: m_AnyIntrinsic<Intrinsic::invariant_start,
429 Intrinsic::spv_resource_handlefrombinding,
430 Intrinsic::spv_resource_getbasepointer,
431 Intrinsic::spv_resource_getpointer>());
432}
433
434// Returns the source pointer from `I` ignoring intermediate ptrcast.
435Value *getPointerRoot(Value *I) {
436 Value *V;
437 if (match(V: I, P: m_Intrinsic<Intrinsic::spv_ptrcast>(Ops: m_Value(V))))
438 return getPointerRoot(I: V);
439 return I;
440}
441
442} // namespace
443
444char SPIRVEmitIntrinsicsLegacy::ID = 0;
445
446INITIALIZE_PASS(SPIRVEmitIntrinsicsLegacy, "spirv-emit-intrinsics",
447 "SPIRV emit intrinsics", false, false)
448
449static inline bool isAssignTypeInstr(const Instruction *I) {
450 return match(V: I, P: m_Intrinsic<Intrinsic::spv_assign_type>());
451}
452
453static bool isMemInstrToReplace(Instruction *I) {
454 return isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I) || isa<InsertValueInst>(Val: I) ||
455 isa<ExtractValueInst>(Val: I) || isa<AtomicCmpXchgInst>(Val: I);
456}
457
458static bool isAggrConstForceInt32(const Value *V) {
459 bool IsAggrZero =
460 isa<ConstantAggregateZero>(Val: V) && !V->getType()->isVectorTy();
461 bool IsUndefAggregate = isa<UndefValue>(Val: V) && V->getType()->isAggregateType();
462 return isa<ConstantArray>(Val: V) || isa<ConstantStruct>(Val: V) ||
463 isa<ConstantDataArray>(Val: V) || IsAggrZero || IsUndefAggregate;
464}
465
466static bool isSpvAggrPlaceholder(const Value *V) {
467 return match(
468 V,
469 P: m_AnyIntrinsic<Intrinsic::spv_undef, Intrinsic::spv_const_composite>());
470}
471
472static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
473 if (isa<PHINode>(Val: I))
474 B.SetInsertPoint(I->getParent()->getFirstNonPHIOrDbgOrAlloca());
475 else
476 B.SetInsertPoint(I);
477}
478
479static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I) {
480 B.SetCurrentDebugLocation(I->getDebugLoc());
481 if (I->getType()->isVoidTy())
482 B.SetInsertPoint(I->getNextNode());
483 else
484 B.SetInsertPoint(*I->getInsertionPointAfterDef());
485}
486
487static bool requireAssignType(Instruction *I) {
488 return !match(
489 V: I,
490 P: m_AnyIntrinsic<Intrinsic::invariant_start, Intrinsic::invariant_end>());
491}
492
493static inline void reportFatalOnTokenType(const Instruction *I) {
494 if (I->getType()->isTokenTy())
495 report_fatal_error(reason: "A token is encountered but SPIR-V without extensions "
496 "does not support token type",
497 gen_crash_diag: false);
498}
499
500static void emitAssignName(Instruction *I, IRBuilder<> &B) {
501 if (!I->hasName() || I->getType()->isAggregateType() ||
502 expectIgnoredInIRTranslation(I))
503 return;
504
505 // We want to be conservative when adding the names because they can interfere
506 // with later optimizations.
507 bool KeepName = SpirvEmitOpNames;
508 if (!KeepName) {
509 if (isa<AllocaInst>(Val: I)) {
510 KeepName = true;
511 } else if (auto *CI = dyn_cast<CallBase>(Val: I)) {
512 Function *F = CI->getCalledFunction();
513 if (F && F->getName().starts_with(Prefix: "llvm.spv.alloca"))
514 KeepName = true;
515 }
516 }
517
518 if (!KeepName)
519 return;
520
521 reportFatalOnTokenType(I);
522 setInsertPointAfterDef(B, I);
523 LLVMContext &Ctx = I->getContext();
524 std::vector<Value *> Args = {
525 I, MetadataAsValue::get(
526 Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: I->getName())))};
527 B.CreateIntrinsic(ID: Intrinsic::spv_assign_name, OverloadTypes: {I->getType()}, Args);
528}
529
530void SPIRVEmitIntrinsicsImpl::replaceAllUsesWith(Value *Src, Value *Dest,
531 bool DeleteOld) {
532 GR->replaceAllUsesWith(Old: Src, New: Dest, DeleteOld);
533 // Update uncomplete type records if any
534 if (isTodoType(Op: Src)) {
535 if (DeleteOld)
536 eraseTodoType(Op: Src);
537 insertTodoType(Op: Dest);
538 }
539}
540
541void SPIRVEmitIntrinsicsImpl::replaceAllUsesWithAndErase(IRBuilder<> &B,
542 Instruction *Src,
543 Instruction *Dest,
544 bool DeleteOld) {
545 replaceAllUsesWith(Src, Dest, DeleteOld);
546 std::string Name = Src->hasName() ? Src->getName().str() : "";
547 Src->eraseFromParent();
548 if (!Name.empty()) {
549 Dest->setName(Name);
550 if (Named.insert(Ptr: Dest).second)
551 emitAssignName(I: Dest, B);
552 }
553}
554
555static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
556 return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
557 isPointerTy(T: SI->getValueOperand()->getType()) &&
558 isa<Argument>(Val: SI->getValueOperand());
559}
560
561// A pointer-typed local holds a pointer, so its deduced pointee must stay a
562// pointer.
563static bool tracesToPointerAlloca(Value *V) {
564 using namespace PatternMatch;
565 V = V->stripPointerCasts();
566 if (auto *AI = dyn_cast<AllocaInst>(Val: V))
567 return isUntypedPointerTy(T: AI->getAllocatedType());
568 return match(
569 V, P: m_AnyIntrinsic<Intrinsic::spv_alloca, Intrinsic::spv_alloca_array>());
570}
571
572// Maybe restore original function return type.
573static inline Type *restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I,
574 Type *Ty) {
575 CallInst *CI = dyn_cast<CallInst>(Val: I);
576 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
577 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
578 return Ty;
579 if (Type *OriginalTy = GR->findMutated(Val: CI->getCalledFunction()))
580 return OriginalTy;
581 return Ty;
582}
583
584// Reconstruct type with nested element types according to deduced type info.
585// Return nullptr if no detailed type info is available.
586Type *SPIRVEmitIntrinsicsImpl::reconstructType(Value *Op,
587 bool UnknownElemTypeI8,
588 bool IsPostprocessing) {
589 Type *Ty = Op->getType();
590 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
591 Ty = restoreMutatedType(GR, I: OpI, Ty);
592 if (auto It = AggrConstTypes.find(Val: OpI); It != AggrConstTypes.end())
593 Ty = It->second;
594 }
595 if (!isUntypedPointerTy(T: Ty))
596 return Ty;
597 // try to find the pointee type
598 if (Type *NestedTy = GR->findDeducedElementType(Val: Op))
599 return getTypedPointerWrapper(ElemTy: NestedTy, AS: getPointerAddressSpace(T: Ty));
600 // not a pointer according to the type info (e.g., Event object)
601 CallInst *CI = GR->findAssignPtrTypeInstr(Val: Op);
602 if (CI) {
603 MetadataAsValue *MD = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
604 return cast<ConstantAsMetadata>(Val: MD->getMetadata())->getType();
605 }
606 if (UnknownElemTypeI8) {
607 if (!IsPostprocessing)
608 insertTodoType(Op);
609 return getTypedPointerWrapper(ElemTy: IntegerType::getInt8Ty(C&: Op->getContext()),
610 AS: getPointerAddressSpace(T: Ty));
611 }
612 return nullptr;
613}
614
615CallInst *SPIRVEmitIntrinsicsImpl::buildSpvPtrcast(Function *F, Value *Op,
616 Type *ElemTy) {
617 IRBuilder<> B(Op->getContext());
618 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
619 // spv_ptrcast's argument Op denotes an instruction that generates
620 // a value, and we may use getInsertionPointAfterDef()
621 setInsertPointAfterDef(B, I: OpI);
622 } else if (auto *OpA = dyn_cast<Argument>(Val: Op)) {
623 B.SetInsertPointPastAllocas(OpA->getParent());
624 B.SetCurrentDebugLocation(DebugLoc());
625 } else {
626 B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
627 }
628 Type *OpTy = Op->getType();
629 SmallVector<Type *, 2> Types = {OpTy, OpTy};
630 SmallVector<Value *, 2> Args = {
631 Op, buildMD(Arg: getNormalizedPoisonValue(Ty: ElemTy, CanUseAnyVectorRank)),
632 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
633 CallInst *PtrCasted =
634 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
635 GR->buildAssignPtr(B, ElemTy, Arg: PtrCasted);
636 return PtrCasted;
637}
638
639void SPIRVEmitIntrinsicsImpl::replaceUsesOfWithSpvPtrcast(
640 Value *Op, Type *ElemTy, Instruction *I,
641 DenseMap<Function *, CallInst *> Ptrcasts) {
642 Function *F = I->getParent()->getParent();
643 CallInst *PtrCastedI = nullptr;
644 auto It = Ptrcasts.find(Val: F);
645 if (It == Ptrcasts.end()) {
646 PtrCastedI = buildSpvPtrcast(F, Op, ElemTy);
647 Ptrcasts[F] = PtrCastedI;
648 } else {
649 PtrCastedI = It->second;
650 }
651 I->replaceUsesOfWith(From: Op, To: PtrCastedI);
652}
653
654void SPIRVEmitIntrinsicsImpl::propagateElemType(
655 Value *Op, Type *ElemTy,
656 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
657 DenseMap<Function *, CallInst *> Ptrcasts;
658 SmallVector<User *> Users(Op->users());
659 for (auto *U : Users) {
660 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
661 continue;
662 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
663 continue;
664 Instruction *UI = dyn_cast<Instruction>(Val: U);
665 // If the instruction was validated already, we need to keep it valid by
666 // keeping current Op type.
667 if (isaGEP(V: UI) || TypeValidated.find(Ptr: UI) != TypeValidated.end())
668 replaceUsesOfWithSpvPtrcast(Op, ElemTy, I: UI, Ptrcasts);
669 }
670}
671
672void SPIRVEmitIntrinsicsImpl::propagateElemTypeRec(
673 Value *Op, Type *PtrElemTy, Type *CastElemTy,
674 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
675 SmallPtrSet<Value *, 0> Visited;
676 DenseMap<Function *, CallInst *> Ptrcasts;
677 propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited,
678 Ptrcasts: std::move(Ptrcasts));
679}
680
681void SPIRVEmitIntrinsicsImpl::propagateElemTypeRec(
682 Value *Op, Type *PtrElemTy, Type *CastElemTy,
683 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
684 SmallPtrSetImpl<Value *> &Visited,
685 DenseMap<Function *, CallInst *> Ptrcasts) {
686 if (!Visited.insert(Ptr: Op).second)
687 return;
688 SmallVector<User *> Users(Op->users());
689 for (auto *U : Users) {
690 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
691 continue;
692 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
693 continue;
694 Instruction *UI = dyn_cast<Instruction>(Val: U);
695 // If the instruction was validated already, we need to keep it valid by
696 // keeping current Op type.
697 if (isaGEP(V: UI) || TypeValidated.find(Ptr: UI) != TypeValidated.end())
698 replaceUsesOfWithSpvPtrcast(Op, ElemTy: CastElemTy, I: UI, Ptrcasts);
699 }
700}
701
702// Set element pointer type to the given value of ValueTy and tries to
703// specify this type further (recursively) by Operand value, if needed.
704
705Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByValueDeep(
706 Type *ValueTy, Value *Operand, bool UnknownElemTypeI8) {
707 SmallPtrSet<Value *, 0> Visited;
708 return deduceElementTypeByValueDeep(ValueTy, Operand, Visited,
709 UnknownElemTypeI8);
710}
711
712Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByValueDeep(
713 Type *ValueTy, Value *Operand, SmallPtrSetImpl<Value *> &Visited,
714 bool UnknownElemTypeI8) {
715 Type *Ty = ValueTy;
716 if (Operand) {
717 if (auto *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
718 if (Type *NestedTy =
719 deduceElementTypeHelper(I: Operand, Visited, UnknownElemTypeI8))
720 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
721 } else {
722 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Operand), Ty, Visited,
723 UnknownElemTypeI8);
724 }
725 }
726 return Ty;
727}
728
729// Traverse User instructions to deduce an element pointer type of the operand.
730Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByUsersDeep(
731 Value *Op, SmallPtrSetImpl<Value *> &Visited, bool UnknownElemTypeI8) {
732 if (!Op || !isPointerTy(T: Op->getType()) || isa<ConstantPointerNull>(Val: Op) ||
733 isa<UndefValue>(Val: Op))
734 return nullptr;
735
736 if (auto ElemTy = getPointeeType(Ty: Op->getType()))
737 return ElemTy;
738
739 // maybe we already know operand's element type
740 if (Type *KnownTy = GR->findDeducedElementType(Val: Op))
741 return KnownTy;
742
743 for (User *OpU : Op->users()) {
744 if (Instruction *Inst = dyn_cast<Instruction>(Val: OpU)) {
745 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8))
746 return Ty;
747 }
748 }
749 return nullptr;
750}
751
752// Implements what we know in advance about intrinsics and builtin calls
753// TODO: consider feasibility of this particular case to be generalized by
754// encoding knowledge about intrinsics and builtin calls by corresponding
755// specification rules
756static Type *getPointeeTypeByCallInst(StringRef DemangledName,
757 Function *CalledF, unsigned OpIdx) {
758 if ((DemangledName.starts_with(Prefix: "__spirv_ocl_printf(") ||
759 DemangledName.starts_with(Prefix: "printf(")) &&
760 OpIdx == 0)
761 return IntegerType::getInt8Ty(C&: CalledF->getContext());
762 return nullptr;
763}
764
765// Deduce and return a successfully deduced Type of the Instruction,
766// or nullptr otherwise.
767Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeHelper(Value *I,
768 bool UnknownElemTypeI8) {
769 SmallPtrSet<Value *, 0> Visited;
770 return deduceElementTypeHelper(I, Visited, UnknownElemTypeI8);
771}
772
773void SPIRVEmitIntrinsicsImpl::maybeAssignPtrType(Type *&Ty, Value *Op,
774 Type *RefTy,
775 bool UnknownElemTypeI8) {
776 if (isUntypedPointerTy(T: RefTy)) {
777 if (!UnknownElemTypeI8)
778 return;
779 insertTodoType(Op);
780 if (isa<IntToPtrInst>(Val: Op))
781 return;
782 }
783 Ty = RefTy;
784}
785
786bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChainDynamic(
787 Type *CurType, Value *Operand, uint64_t Multiplier,
788 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
789 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing) {
790 // Dynamic indexing into a struct is not possible.
791 // We know that we must be accessing the first element
792 // of the struct if the current type is a struct.
793 // Try to find the first array type that is at offset 0 in the struct.
794 while (auto *ST = dyn_cast<StructType>(Val: CurType)) {
795 if (ST->getNumElements() == 0)
796 break;
797 CurType = ST->getElementType(N: 0);
798 OnLiteralIndexing(CurType, 0);
799 }
800
801 assert(CurType);
802 ArrayType *AT = dyn_cast<ArrayType>(Val: CurType);
803 // Operand is not constant. Either we have an array and accept it, or we
804 // give up.
805 if (AT)
806 OnDynamicIndexing(AT->getElementType(), Operand, Multiplier);
807 return AT == nullptr;
808}
809
810bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChainConstant(
811 Type *CurType, uint64_t Offset,
812 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing) {
813 auto &DL = CurrF->getDataLayout();
814
815 do {
816 if (ArrayType *AT = dyn_cast<ArrayType>(Val: CurType)) {
817 uint64_t EltTypeSize = DL.getTypeAllocSize(Ty: AT->getElementType());
818 assert(Offset < AT->getNumElements() * EltTypeSize);
819 uint64_t Index = Offset / EltTypeSize;
820 Offset = Offset - (Index * EltTypeSize);
821 CurType = AT->getElementType();
822 OnLiteralIndexing(CurType, Index);
823 } else if (StructType *ST = dyn_cast<StructType>(Val: CurType)) {
824 uint32_t StructSize = DL.getTypeSizeInBits(Ty: ST) / 8;
825 assert(Offset < StructSize);
826 (void)StructSize;
827 const auto &STL = DL.getStructLayout(Ty: ST);
828 unsigned Element = STL->getElementContainingOffset(FixedOffset: Offset);
829 Offset -= STL->getElementOffset(Idx: Element);
830 CurType = ST->getElementType(N: Element);
831 OnLiteralIndexing(CurType, Element);
832 } else if (auto *VT = dyn_cast<FixedVectorType>(Val: CurType)) {
833 Type *EltTy = VT->getElementType();
834 TypeSize EltSizeBits = DL.getTypeSizeInBits(Ty: EltTy);
835 assert(EltSizeBits % 8 == 0 &&
836 "Element type size in bits must be a multiple of 8.");
837 uint32_t EltTypeSize = EltSizeBits / 8;
838 assert(Offset < VT->getNumElements() * EltTypeSize);
839 uint64_t Index = Offset / EltTypeSize;
840 Offset -= Index * EltTypeSize;
841 CurType = EltTy;
842 OnLiteralIndexing(CurType, Index);
843 } else {
844 // Unknown composite kind; give up.
845 return true;
846 }
847 } while (Offset > 0);
848
849 return false;
850}
851
852bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChain(
853 GetElementPtrInst &GEP,
854 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
855 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing) {
856 // We only rewrite byte-addressing GEP. Other should be left as-is.
857 // Valid byte-addressing GEP must always have a single index.
858 std::optional<uint64_t> MultiplierOpt =
859 getByteAddressingMultiplier(Ty: GEP.getSourceElementType());
860 assert(MultiplierOpt && "We only rewrite byte-addressing GEP");
861 uint64_t Multiplier = *MultiplierOpt;
862 assert(GEP.getNumIndices() == 1);
863
864 Value *Src = getPointerRoot(I: GEP.getPointerOperand());
865 Type *CurType = deduceElementType(I: Src, UnknownElemTypeI8: true);
866
867 Value *Operand = *GEP.idx_begin();
868 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Operand))
869 return walkLogicalAccessChainConstant(
870 CurType, Offset: CI->getZExtValue() * Multiplier, OnLiteralIndexing);
871
872 return walkLogicalAccessChainDynamic(CurType, Operand, Multiplier,
873 OnLiteralIndexing, OnDynamicIndexing);
874}
875
876Instruction *SPIRVEmitIntrinsicsImpl::buildLogicalAccessChainFromGEP(
877 GetElementPtrInst &GEP) {
878 auto &DL = CurrF->getDataLayout();
879 IRBuilder<> B(GEP.getParent());
880 B.SetInsertPoint(&GEP);
881
882 std::vector<Value *> Indices;
883 Indices.push_back(x: ConstantInt::get(
884 Ty: IntegerType::getInt32Ty(C&: CurrF->getContext()), V: 0, /* Signed= */ IsSigned: false));
885 walkLogicalAccessChain(
886 GEP,
887 OnLiteralIndexing: [&Indices, &B](Type *EltType, uint64_t Index) {
888 Indices.push_back(
889 x: ConstantInt::get(Ty: B.getInt64Ty(), V: Index, /* Signed= */ IsSigned: false));
890 },
891 OnDynamicIndexing: [&Indices, &B, &DL, this](Type *EltType, Value *Offset,
892 uint64_t Multiplier) {
893 Value *Index = nullptr;
894 uint32_t EltTypeSize = DL.getTypeSizeInBits(Ty: EltType) / 8;
895 assert(Multiplier != 0);
896 if (Multiplier == EltTypeSize) {
897 Index = Offset;
898 } else if (EltTypeSize % Multiplier == 0) {
899 Index =
900 B.CreateUDiv(LHS: Offset, RHS: ConstantInt::get(Ty: Offset->getType(),
901 V: EltTypeSize / Multiplier,
902 /* Signed= */ IsSigned: false));
903 } else {
904 Index = B.CreateMul(LHS: Offset,
905 RHS: ConstantInt::get(Ty: Offset->getType(), V: Multiplier,
906 /* Signed= */ IsSigned: false));
907 insertAssignTypeIntrs(I: cast<Instruction>(Val: Index), B);
908 Index = B.CreateUDiv(LHS: Index,
909 RHS: ConstantInt::get(Ty: Offset->getType(), V: EltTypeSize,
910 /* Signed= */ IsSigned: false));
911 }
912 insertAssignTypeIntrs(I: cast<Instruction>(Val: Index), B);
913 Indices.push_back(x: Index);
914 });
915
916 SmallVector<Type *, 2> Types = {GEP.getType(), GEP.getOperand(i_nocapture: 0)->getType()};
917 SmallVector<Value *, 4> Args;
918 Args.push_back(Elt: B.getInt1(V: GEP.isInBounds()));
919 Args.push_back(Elt: GEP.getOperand(i_nocapture: 0));
920 llvm::append_range(C&: Args, R&: Indices);
921 Instruction *NewI =
922 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
923 replaceAllUsesWithAndErase(B, Src: &GEP, Dest: NewI);
924 return NewI;
925}
926
927Type *SPIRVEmitIntrinsicsImpl::getGEPTypeLogical(GetElementPtrInst *GEP) {
928
929 Type *CurType = GEP->getResultElementType();
930
931 bool Interrupted = walkLogicalAccessChain(
932 GEP&: *GEP, OnLiteralIndexing: [&CurType](Type *EltType, uint64_t Index) { CurType = EltType; },
933 OnDynamicIndexing: [&CurType](Type *EltType, Value *Index, uint64_t) { CurType = EltType; });
934
935 return Interrupted ? GEP->getResultElementType() : CurType;
936}
937
938Type *SPIRVEmitIntrinsicsImpl::getGEPType(GetElementPtrInst *Ref) {
939 if (getByteAddressingMultiplier(Ty: Ref->getSourceElementType()) &&
940 TM.getSubtargetImpl()->isLogicalSPIRV()) {
941 return getGEPTypeLogical(GEP: Ref);
942 }
943
944 Type *Ty = nullptr;
945 // TODO: not sure if GetElementPtrInst::getTypeAtIndex() does anything
946 // useful here
947 if (isNestedPointer(Ty: Ref->getSourceElementType())) {
948 Ty = Ref->getSourceElementType();
949 for (Use &U : drop_begin(RangeOrContainer: Ref->indices()))
950 Ty = GetElementPtrInst::getTypeAtIndex(Ty, Idx: U.get());
951 } else {
952 Ty = Ref->getResultElementType();
953 }
954 return Ty;
955}
956
957Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeHelper(
958 Value *I, SmallPtrSetImpl<Value *> &Visited, bool UnknownElemTypeI8,
959 bool IgnoreKnownType) {
960 // allow to pass nullptr as an argument
961 if (!I)
962 return nullptr;
963
964 // maybe already known
965 if (!IgnoreKnownType)
966 if (Type *KnownTy = GR->findDeducedElementType(Val: I))
967 return KnownTy;
968
969 // maybe a cycle
970 if (!Visited.insert(Ptr: I).second)
971 return nullptr;
972
973 // fallback value in case when we fail to deduce a type
974 Type *Ty = nullptr;
975 // look for known basic patterns of type inference
976 if (auto *Ref = dyn_cast<AllocaInst>(Val: I)) {
977 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getAllocatedType(), UnknownElemTypeI8);
978 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
979 Ty = getGEPType(Ref);
980 } else if (auto *SGEP = dyn_cast<StructuredGEPInst>(Val: I)) {
981 Ty = SGEP->getResultElementType();
982 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
983 Value *Op = Ref->getPointerOperand();
984 Type *KnownTy = GR->findDeducedElementType(Val: Op);
985 if (!KnownTy)
986 KnownTy = Op->getType();
987 if (Type *ElemTy = getPointeeType(Ty: KnownTy))
988 maybeAssignPtrType(Ty, Op: I, RefTy: ElemTy, UnknownElemTypeI8);
989 } else if (auto *Ref = dyn_cast<GlobalValue>(Val: I)) {
990 if (auto *Fn = dyn_cast<Function>(Val: Ref)) {
991 Ty = SPIRV::getOriginalFunctionType(F: *Fn);
992 GR->addDeducedElementType(Val: I, Ty);
993 } else {
994 Ty = deduceElementTypeByValueDeep(
995 ValueTy: Ref->getValueType(),
996 Operand: Ref->getNumOperands() > 0 ? Ref->getOperand(i: 0) : nullptr, Visited,
997 UnknownElemTypeI8);
998 }
999 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1000 Type *RefTy = deduceElementTypeHelper(I: Ref->getPointerOperand(), Visited,
1001 UnknownElemTypeI8);
1002 maybeAssignPtrType(Ty, Op: I, RefTy, UnknownElemTypeI8);
1003 } else if (auto *Ref = dyn_cast<IntToPtrInst>(Val: I)) {
1004 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getDestTy(), UnknownElemTypeI8);
1005 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1006 if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
1007 isPointerTy(T: Src) && isPointerTy(T: Dest))
1008 Ty = deduceElementTypeHelper(I: Ref->getOperand(i_nocapture: 0), Visited,
1009 UnknownElemTypeI8);
1010 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1011 Value *Op = Ref->getNewValOperand();
1012 if (isPointerTy(T: Op->getType()))
1013 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
1014 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1015 Value *Op = Ref->getValOperand();
1016 if (isPointerTy(T: Op->getType()))
1017 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
1018 } else if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1019 Type *BestTy = nullptr;
1020 unsigned MaxN = 1;
1021 DenseMap<Type *, unsigned> PhiTys;
1022 for (int i = Ref->getNumIncomingValues() - 1; i >= 0; --i) {
1023 Ty = deduceElementTypeByUsersDeep(Op: Ref->getIncomingValue(i), Visited,
1024 UnknownElemTypeI8);
1025 if (!Ty)
1026 continue;
1027 auto It = PhiTys.try_emplace(Key: Ty, Args: 1);
1028 if (!It.second) {
1029 ++It.first->second;
1030 if (It.first->second > MaxN) {
1031 MaxN = It.first->second;
1032 BestTy = Ty;
1033 }
1034 }
1035 }
1036 if (BestTy)
1037 Ty = BestTy;
1038 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1039 for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) {
1040 // A function pointer operand carries its function type directly. Other
1041 // operands are deduced from their uses.
1042 Ty = isa<Function>(Val: Op)
1043 ? deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8)
1044 : deduceElementTypeByUsersDeep(Op, Visited, UnknownElemTypeI8);
1045 if (Ty)
1046 break;
1047 }
1048 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
1049 static StringMap<unsigned> ResTypeByArg = {
1050 {"to_global", 0},
1051 {"to_local", 0},
1052 {"to_private", 0},
1053 {"__spirv_GenericCastToPtr_ToGlobal", 0},
1054 {"__spirv_GenericCastToPtr_ToLocal", 0},
1055 {"__spirv_GenericCastToPtr_ToPrivate", 0},
1056 {"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
1057 {"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
1058 {"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
1059 // TODO: maybe improve performance by caching demangled names
1060
1061 auto *II = dyn_cast<IntrinsicInst>(Val: I);
1062 if (II && (II->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer ||
1063 II->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) {
1064 auto *HandleType = cast<TargetExtType>(Val: II->getOperand(i_nocapture: 0)->getType());
1065 if (HandleType->getTargetExtName() == "spirv.Image" ||
1066 HandleType->getTargetExtName() == "spirv.SignedImage") {
1067 for (User *U : II->users()) {
1068 Ty = cast<Instruction>(Val: U)->getAccessType();
1069 if (Ty)
1070 break;
1071 }
1072 } else if (HandleType->getTargetExtName() == "spirv.VulkanBuffer") {
1073 // This call is supposed to index into an array
1074 Ty = HandleType->getTypeParameter(i: 0);
1075 if (II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1076 if (Ty->isArrayTy())
1077 Ty = Ty->getArrayElementType();
1078 else {
1079 assert(Ty && Ty->isStructTy());
1080 uint32_t Index =
1081 cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1))->getZExtValue();
1082 Ty = cast<StructType>(Val: Ty)->getElementType(N: Index);
1083 }
1084 }
1085 Ty = reconstitutePeeledArrayType(Ty);
1086 } else {
1087 llvm_unreachable("Unknown handle type for spv_resource_getpointer.");
1088 }
1089 } else if (II && II->getIntrinsicID() ==
1090 Intrinsic::spv_generic_cast_to_ptr_explicit) {
1091 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: 0), Visited,
1092 UnknownElemTypeI8);
1093 } else if (Function *CalledF = CI->getCalledFunction()) {
1094 std::string DemangledName =
1095 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1096 if (DemangledName.length() > 0)
1097 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName);
1098 auto AsArgIt = ResTypeByArg.find(Key: DemangledName);
1099 if (AsArgIt != ResTypeByArg.end())
1100 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: AsArgIt->second),
1101 Visited, UnknownElemTypeI8);
1102 else if (Type *KnownRetTy = GR->findDeducedElementType(Val: CalledF))
1103 Ty = KnownRetTy;
1104 }
1105 }
1106
1107 // remember the found relationship
1108 if (Ty && !IgnoreKnownType) {
1109 // specify nested types if needed, otherwise return unchanged
1110 GR->addDeducedElementType(Val: I, Ty: normalizeType(Ty, CanUseAnyVectorRank));
1111 }
1112
1113 return Ty;
1114}
1115
1116// Re-create a type of the value if it has untyped pointer fields, also nested.
1117// Return the original value type if no corrections of untyped pointer
1118// information is found or needed.
1119Type *SPIRVEmitIntrinsicsImpl::deduceNestedTypeHelper(User *U,
1120 bool UnknownElemTypeI8) {
1121 SmallPtrSet<Value *, 0> Visited;
1122 return deduceNestedTypeHelper(U, Ty: U->getType(), Visited, UnknownElemTypeI8);
1123}
1124
1125Type *SPIRVEmitIntrinsicsImpl::deduceNestedTypeHelper(
1126 User *U, Type *OrigTy, SmallPtrSetImpl<Value *> &Visited,
1127 bool UnknownElemTypeI8) {
1128 if (!U)
1129 return OrigTy;
1130
1131 // maybe already known
1132 if (Type *KnownTy = GR->findDeducedCompositeType(Val: U))
1133 return KnownTy;
1134
1135 // maybe a cycle
1136 if (!Visited.insert(Ptr: U).second)
1137 return OrigTy;
1138
1139 if (auto *OrigStructTy = dyn_cast<StructType>(Val: OrigTy)) {
1140 SmallVector<Type *> Tys;
1141 bool Change = false;
1142 for (unsigned i = 0; i < U->getNumOperands(); ++i) {
1143 Value *Op = U->getOperand(i);
1144 assert(Op && "Operands should not be null.");
1145 Type *OpTy = Op->getType();
1146 Type *Ty = OpTy;
1147 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1148 if (Type *NestedTy =
1149 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1150 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1151 } else {
1152 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1153 UnknownElemTypeI8);
1154 }
1155 Tys.push_back(Elt: Ty);
1156 Change |= Ty != OpTy;
1157 }
1158 if (Change) {
1159 Type *NewTy = StructType::create(
1160 Elements: Tys, Name: OrigStructTy->isLiteral() ? "" : OrigStructTy->getName(),
1161 isPacked: OrigStructTy->isPacked());
1162 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1163 return NewTy;
1164 }
1165 } else if (auto *ArrTy = dyn_cast<ArrayType>(Val: OrigTy)) {
1166 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1167 Type *OpTy = ArrTy->getElementType();
1168 Type *Ty = OpTy;
1169 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1170 if (Type *NestedTy =
1171 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1172 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1173 } else {
1174 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1175 UnknownElemTypeI8);
1176 }
1177 if (Ty != OpTy) {
1178 Type *NewTy = ArrayType::get(ElementType: Ty, NumElements: ArrTy->getNumElements());
1179 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1180 return NewTy;
1181 }
1182 }
1183 } else if (auto *VecTy = dyn_cast<VectorType>(Val: OrigTy)) {
1184 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1185 Type *OpTy = VecTy->getElementType();
1186 Type *Ty = OpTy;
1187 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1188 if (Type *NestedTy =
1189 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1190 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1191 } else {
1192 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1193 UnknownElemTypeI8);
1194 }
1195 if (Ty != OpTy) {
1196 Type *NewTy = VectorType::get(ElementType: Ty, EC: VecTy->getElementCount());
1197 GR->addDeducedCompositeType(Val: U,
1198 Ty: normalizeType(Ty: NewTy, CanUseAnyVectorRank));
1199 return NewTy;
1200 }
1201 }
1202 }
1203
1204 return OrigTy;
1205}
1206
1207Type *SPIRVEmitIntrinsicsImpl::deduceElementType(Value *I,
1208 bool UnknownElemTypeI8) {
1209 if (Type *Ty = deduceElementTypeHelper(I, UnknownElemTypeI8))
1210 return Ty;
1211 if (!UnknownElemTypeI8)
1212 return nullptr;
1213 insertTodoType(Op: I);
1214 return IntegerType::getInt8Ty(C&: I->getContext());
1215}
1216
1217static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I,
1218 Value *PointerOperand) {
1219 Type *PointeeTy = GR->findDeducedElementType(Val: PointerOperand);
1220 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1221 return nullptr;
1222 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType());
1223 if (!PtrTy)
1224 return I->getType();
1225 if (Type *NestedTy = GR->findDeducedElementType(Val: I))
1226 return getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1227 return nullptr;
1228}
1229
1230// Try to deduce element type for a call base. Returns false if this is an
1231// indirect function invocation, and true otherwise.
1232bool SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeCalledFunction(
1233 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1234 Type *&KnownElemTy, bool &Incomplete) {
1235 Function *CalledF = CI->getCalledFunction();
1236 if (!CalledF)
1237 return false;
1238 std::string DemangledName =
1239 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1240 if (DemangledName.length() > 0 &&
1241 !StringRef(DemangledName).starts_with(Prefix: "llvm.")) {
1242 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *CalledF);
1243 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
1244 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
1245 if (Opcode == SPIRV::OpGroupAsyncCopy) {
1246 for (unsigned i = 0, PtrCnt = 0; i < CI->arg_size() && PtrCnt < 2; ++i) {
1247 Value *Op = CI->getArgOperand(i);
1248 if (!isPointerTy(T: Op->getType()))
1249 continue;
1250 ++PtrCnt;
1251 if (Type *ElemTy = GR->findDeducedElementType(Val: Op))
1252 KnownElemTy = ElemTy; // src will rewrite dest if both are defined
1253 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1254 }
1255 } else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) {
1256 if (CI->arg_size() == 0)
1257 return true;
1258 Value *Op = CI->getArgOperand(i: 0);
1259 if (!isPointerTy(T: Op->getType()))
1260 return true;
1261 switch (Opcode) {
1262 case SPIRV::OpAtomicFAddEXT:
1263 case SPIRV::OpAtomicFMinEXT:
1264 case SPIRV::OpAtomicFMaxEXT:
1265 case SPIRV::OpAtomicLoad:
1266 case SPIRV::OpAtomicCompareExchangeWeak:
1267 case SPIRV::OpAtomicCompareExchange:
1268 case SPIRV::OpAtomicExchange:
1269 case SPIRV::OpAtomicIAdd:
1270 case SPIRV::OpAtomicISub:
1271 case SPIRV::OpAtomicOr:
1272 case SPIRV::OpAtomicXor:
1273 case SPIRV::OpAtomicAnd:
1274 case SPIRV::OpAtomicUMin:
1275 case SPIRV::OpAtomicUMax:
1276 case SPIRV::OpAtomicSMin:
1277 case SPIRV::OpAtomicSMax: {
1278 KnownElemTy = isPointerTy(T: CI->getType()) ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1279 : CI->getType();
1280 if (!KnownElemTy)
1281 return true;
1282 Incomplete = isTodoType(Op);
1283 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1284 } break;
1285 case SPIRV::OpAtomicStore: {
1286 if (CI->arg_size() < 4)
1287 return true;
1288 Value *ValOp = CI->getArgOperand(i: 3);
1289 KnownElemTy = isPointerTy(T: ValOp->getType())
1290 ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1291 : ValOp->getType();
1292 if (!KnownElemTy)
1293 return true;
1294 Incomplete = isTodoType(Op);
1295 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1296 } break;
1297 }
1298 }
1299 }
1300 return true;
1301}
1302
1303// Try to deduce element type for a function pointer.
1304void SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeFunctionPointer(
1305 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1306 Type *&KnownElemTy, bool IsPostprocessing) {
1307 Value *Op = CI->getCalledOperand();
1308 if (!Op || !isPointerTy(T: Op->getType()))
1309 return;
1310 Ops.push_back(Elt: std::make_pair(x&: Op, y: std::numeric_limits<unsigned>::max()));
1311 FunctionType *FTy = SPIRV::getOriginalFunctionType(CB: *CI);
1312 bool IsNewFTy = false, IsIncomplete = false;
1313 SmallVector<Type *, 4> ArgTys;
1314 for (auto &&[ParmIdx, Arg] : llvm::enumerate(First: CI->args())) {
1315 Type *ArgTy = Arg->getType();
1316 if (ArgTy->isPointerTy()) {
1317 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg)) {
1318 IsNewFTy = true;
1319 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
1320 if (isTodoType(Op: Arg))
1321 IsIncomplete = true;
1322 } else {
1323 IsIncomplete = true;
1324 }
1325 } else {
1326 ArgTy = FTy->getFunctionParamType(i: ParmIdx);
1327 }
1328 ArgTys.push_back(Elt: ArgTy);
1329 }
1330 Type *RetTy = FTy->getReturnType();
1331 if (CI->getType()->isPointerTy()) {
1332 if (Type *ElemTy = GR->findDeducedElementType(Val: CI)) {
1333 IsNewFTy = true;
1334 RetTy =
1335 getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: CI->getType()));
1336 if (isTodoType(Op: CI))
1337 IsIncomplete = true;
1338 } else {
1339 IsIncomplete = true;
1340 }
1341 }
1342 if (!IsPostprocessing && IsIncomplete)
1343 insertTodoType(Op);
1344 KnownElemTy =
1345 IsNewFTy ? FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: FTy->isVarArg()) : FTy;
1346}
1347
1348bool SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeFunctionRet(
1349 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
1350 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing,
1351 Type *&KnownElemTy, Value *Op, Function *F) {
1352 KnownElemTy = GR->findDeducedElementType(Val: F);
1353 if (KnownElemTy)
1354 return false;
1355 if (Type *OpElemTy = GR->findDeducedElementType(Val: Op)) {
1356 OpElemTy = normalizeType(Ty: OpElemTy, CanUseAnyVectorRank);
1357 GR->addDeducedElementType(Val: F, Ty: OpElemTy);
1358 GR->addReturnType(
1359 ArgF: F, DerivedTy: TypedPointerType::get(ElementType: OpElemTy,
1360 AddressSpace: getPointerAddressSpace(T: F->getReturnType())));
1361 // non-recursive update of types in function uses
1362 DenseSet<std::pair<Value *, Value *>> VisitedSubst{std::make_pair(x&: I, y&: Op)};
1363 for (User *U : F->users()) {
1364 CallInst *CI = dyn_cast<CallInst>(Val: U);
1365 if (!CI || CI->getCalledFunction() != F)
1366 continue;
1367 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: CI)) {
1368 if (Type *PrevElemTy = GR->findDeducedElementType(Val: CI)) {
1369 GR->updateAssignType(
1370 AssignCI, Arg: CI,
1371 OfType: getNormalizedPoisonValue(Ty: OpElemTy, CanUseAnyVectorRank));
1372 propagateElemType(Op: CI, ElemTy: PrevElemTy, VisitedSubst);
1373 }
1374 }
1375 }
1376 // Non-recursive update of types in the function uncomplete returns.
1377 // This may happen just once per a function, the latch is a pair of
1378 // findDeducedElementType(F) / addDeducedElementType(F, ...).
1379 // With or without the latch it is a non-recursive call due to
1380 // IncompleteRets set to nullptr in this call.
1381 if (IncompleteRets)
1382 for (Instruction *IncompleteRetI : *IncompleteRets)
1383 deduceOperandElementType(I: IncompleteRetI, IncompleteRets: nullptr, AskOps,
1384 IsPostprocessing);
1385 } else if (IncompleteRets) {
1386 IncompleteRets->insert(Ptr: I);
1387 }
1388 TypeValidated.insert(Ptr: I);
1389 return true;
1390}
1391
1392// If the Instruction has Pointer operands with unresolved types, this function
1393// tries to deduce them. If the Instruction has Pointer operands with known
1394// types which differ from expected, this function tries to insert a bitcast to
1395// resolve the issue.
1396void SPIRVEmitIntrinsicsImpl::deduceOperandElementType(
1397 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
1398 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing) {
1399 SmallVector<std::pair<Value *, unsigned>> Ops;
1400 Type *KnownElemTy = nullptr;
1401 bool Incomplete = false;
1402 // look for known basic patterns of type inference
1403 if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1404 if (!isPointerTy(T: I->getType()) ||
1405 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1406 return;
1407 Incomplete = isTodoType(Op: I);
1408 for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
1409 Value *Op = Ref->getIncomingValue(i);
1410 if (isPointerTy(T: Op->getType()))
1411 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1412 }
1413 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1414 KnownElemTy = GR->findDeducedElementType(Val: I);
1415 if (!KnownElemTy)
1416 return;
1417 Incomplete = isTodoType(Op: I);
1418 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(), y: 0));
1419 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1420 if (!isPointerTy(T: I->getType()))
1421 return;
1422 KnownElemTy = GR->findDeducedElementType(Val: I);
1423 if (!KnownElemTy)
1424 return;
1425 Incomplete = isTodoType(Op: I);
1426 Ops.push_back(Elt: std::make_pair(x: Ref->getOperand(i_nocapture: 0), y: 0));
1427 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
1428 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1429 return;
1430 KnownElemTy = Ref->getSourceElementType();
1431 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1432 y: GetElementPtrInst::getPointerOperandIndex()));
1433 } else if (auto *Ref = dyn_cast<StructuredGEPInst>(Val: I)) {
1434 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1435 return;
1436 KnownElemTy = Ref->getBaseType();
1437 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1438 y: StructuredGEPInst::getPointerOperandIndex()));
1439 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
1440 KnownElemTy = I->getType();
1441 if (isUntypedPointerTy(T: KnownElemTy)) {
1442 // A T** loaded back from its alloca comes out opaque, dropping type info.
1443 // When the load is a pointer-to-pointer, type the alloca as that pointer.
1444 Type *LoadedElemTy = GR->findDeducedElementType(Val: I);
1445 if (!LoadedElemTy || !isPointerTyOrWrapper(Ty: LoadedElemTy))
1446 return;
1447 Value *Root = Ref->getPointerOperand()->stripPointerCasts();
1448 if (!isa<AllocaInst>(Val: Root))
1449 return;
1450 KnownElemTy = getTypedPointerWrapper(ElemTy: LoadedElemTy,
1451 AS: getPointerAddressSpace(T: KnownElemTy));
1452 }
1453 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1454 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1455 return;
1456 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1457 y: LoadInst::getPointerOperandIndex()));
1458 } else if (auto *Ref = dyn_cast<StoreInst>(Val: I)) {
1459 if (!(KnownElemTy =
1460 reconstructType(Op: Ref->getValueOperand(), UnknownElemTypeI8: false, IsPostprocessing)))
1461 return;
1462 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1463 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1464 return;
1465 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1466 y: StoreInst::getPointerOperandIndex()));
1467 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1468 KnownElemTy = isPointerTy(T: I->getType())
1469 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1470 : I->getType();
1471 if (!KnownElemTy)
1472 return;
1473 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1474 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1475 y: AtomicCmpXchgInst::getPointerOperandIndex()));
1476 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1477 KnownElemTy = isPointerTy(T: I->getType())
1478 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1479 : I->getType();
1480 if (!KnownElemTy)
1481 return;
1482 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1483 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1484 y: AtomicRMWInst::getPointerOperandIndex()));
1485 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1486 if (!isPointerTy(T: I->getType()) ||
1487 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1488 return;
1489 Incomplete = isTodoType(Op: I);
1490 for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
1491 Value *Op = Ref->getOperand(i_nocapture: i);
1492 if (isPointerTy(T: Op->getType()))
1493 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1494 }
1495 } else if (auto *Ref = dyn_cast<ReturnInst>(Val: I)) {
1496 if (!isPointerTy(T: CurrF->getReturnType()))
1497 return;
1498 Value *Op = Ref->getReturnValue();
1499 if (!Op)
1500 return;
1501 if (deduceOperandElementTypeFunctionRet(I, IncompleteRets, AskOps,
1502 IsPostprocessing, KnownElemTy, Op,
1503 F: CurrF))
1504 return;
1505 Incomplete = isTodoType(Op: CurrF);
1506 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1507 } else if (auto *Ref = dyn_cast<ICmpInst>(Val: I)) {
1508 if (!isPointerTy(T: Ref->getOperand(i_nocapture: 0)->getType()))
1509 return;
1510 Value *Op0 = Ref->getOperand(i_nocapture: 0);
1511 Value *Op1 = Ref->getOperand(i_nocapture: 1);
1512 bool Incomplete0 = isTodoType(Op: Op0);
1513 bool Incomplete1 = isTodoType(Op: Op1);
1514 Type *ElemTy1 = GR->findDeducedElementType(Val: Op1);
1515 Type *ElemTy0 = (Incomplete0 && !Incomplete1 && ElemTy1)
1516 ? nullptr
1517 : GR->findDeducedElementType(Val: Op0);
1518 if (ElemTy0) {
1519 KnownElemTy = ElemTy0;
1520 Incomplete = Incomplete0;
1521 Ops.push_back(Elt: std::make_pair(x&: Op1, y: 1));
1522 } else if (ElemTy1) {
1523 KnownElemTy = ElemTy1;
1524 Incomplete = Incomplete1;
1525 Ops.push_back(Elt: std::make_pair(x&: Op0, y: 0));
1526 }
1527 } else if (CallInst *CI = dyn_cast<CallInst>(Val: I)) {
1528 if (!CI->isIndirectCall())
1529 deduceOperandElementTypeCalledFunction(CI, Ops, KnownElemTy, Incomplete);
1530 else if (HaveFunPtrs)
1531 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy,
1532 IsPostprocessing);
1533 }
1534
1535 // There is no enough info to deduce types or all is valid.
1536 if (!KnownElemTy || Ops.size() == 0)
1537 return;
1538
1539 LLVMContext &Ctx = CurrF->getContext();
1540 IRBuilder<> B(Ctx);
1541 for (auto &OpIt : Ops) {
1542 Value *Op = OpIt.first;
1543 if (AskOps && !AskOps->contains(Ptr: Op))
1544 continue;
1545 Type *AskTy = nullptr;
1546 CallInst *AskCI = nullptr;
1547 if (IsPostprocessing && AskOps) {
1548 AskTy = GR->findDeducedElementType(Val: Op);
1549 AskCI = GR->findAssignPtrTypeInstr(Val: Op);
1550 assert(AskTy && AskCI);
1551 }
1552 Type *Ty = AskTy ? AskTy : GR->findDeducedElementType(Val: Op);
1553 if (Ty == KnownElemTy)
1554 continue;
1555 Value *OpTyVal = getNormalizedPoisonValue(Ty: KnownElemTy, CanUseAnyVectorRank);
1556 Type *OpTy = Op->getType();
1557 // Do not let a non-pointer element type clobber an already-deduced pointer
1558 // element type for the same operand.
1559 bool WouldClobberPtrWithNonPtr = Ty && isPointerTyOrWrapper(Ty) &&
1560 !isPointerTyOrWrapper(Ty: KnownElemTy) &&
1561 tracesToPointerAlloca(V: Op);
1562 if (Op->hasUseList() && !WouldClobberPtrWithNonPtr &&
1563 (!Ty || AskTy || isUntypedPointerTy(T: Ty) || isTodoType(Op))) {
1564 Type *PrevElemTy = GR->findDeducedElementType(Val: Op);
1565 GR->addDeducedElementType(
1566 Val: Op, Ty: normalizeType(Ty: KnownElemTy, CanUseAnyVectorRank));
1567 // check if KnownElemTy is complete
1568 if (!Incomplete)
1569 eraseTodoType(Op);
1570 else if (!IsPostprocessing)
1571 insertTodoType(Op);
1572 // check if there is existing Intrinsic::spv_assign_ptr_type instruction
1573 CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Val: Op);
1574 if (AssignCI == nullptr) {
1575 Instruction *User = dyn_cast<Instruction>(Val: Op->use_begin()->get());
1576 setInsertPointSkippingPhis(B, I: User ? User->getNextNode() : I);
1577 CallInst *CI =
1578 buildIntrWithMD(IntrID: Intrinsic::spv_assign_ptr_type, Types: {OpTy}, Arg: OpTyVal, Arg2: Op,
1579 Imms: {B.getInt32(C: getPointerAddressSpace(T: OpTy))}, B);
1580 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: CI);
1581 } else {
1582 GR->updateAssignType(AssignCI, Arg: Op, OfType: OpTyVal);
1583 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1584 std::make_pair(x&: I, y&: Op)};
1585 propagateElemTypeRec(Op, PtrElemTy: KnownElemTy, CastElemTy: PrevElemTy, VisitedSubst);
1586 }
1587 } else {
1588 eraseTodoType(Op);
1589 CallInst *PtrCastI =
1590 buildSpvPtrcast(F: I->getParent()->getParent(), Op, ElemTy: KnownElemTy);
1591 if (OpIt.second == std::numeric_limits<unsigned>::max())
1592 dyn_cast<CallInst>(Val: I)->setCalledOperand(PtrCastI);
1593 else
1594 I->setOperand(i: OpIt.second, Val: PtrCastI);
1595 }
1596 }
1597 TypeValidated.insert(Ptr: I);
1598}
1599
1600void SPIRVEmitIntrinsicsImpl::replaceMemInstrUses(Instruction *Old,
1601 Instruction *New,
1602 IRBuilder<> &B) {
1603 while (!Old->user_empty()) {
1604 auto *U = Old->user_back();
1605 if (isAssignTypeInstr(I: U)) {
1606 B.SetInsertPoint(U);
1607 SmallVector<Value *, 2> Args = {New, U->getOperand(i: 1)};
1608 CallInst *AssignCI = B.CreateIntrinsicWithoutFolding(
1609 ID: Intrinsic::spv_assign_type, OverloadTypes: {New->getType()}, Args);
1610 GR->addAssignPtrTypeInstr(Val: New, AssignPtrTyCI: AssignCI);
1611 U->eraseFromParent();
1612 } else if (isMemInstrToReplace(I: U) || isa<ReturnInst>(Val: U) ||
1613 isa<CallInst>(Val: U)) {
1614 U->replaceUsesOfWith(From: Old, To: New);
1615 // For a `llvm.spv.abort` call whose composite message argument was
1616 // rewritten to a value-id (i32), also retarget the call to a matching
1617 // intrinsic declaration so the IR verifier is satisfied. The SPIR-V
1618 // type of the value is tracked via the GlobalRegistry, so the selector
1619 // still emits OpAbortKHR with the original composite type.
1620 if (auto *CI = dyn_cast<CallInst>(Val: U);
1621 CI && CI->getIntrinsicID() == Intrinsic::spv_abort) {
1622 Type *NewArgTy = New->getType();
1623 Type *ExpectedArgTy = CI->getFunctionType()->getParamType(i: 0);
1624 if (NewArgTy != ExpectedArgTy) {
1625 Module *M = CI->getModule();
1626 Function *NewF = Intrinsic::getOrInsertDeclaration(
1627 M, id: Intrinsic::spv_abort, OverloadTys: {NewArgTy});
1628 CI->setCalledFunction(NewF);
1629 }
1630 }
1631 } else if (isa<PHINode>(Val: U) || isa<SelectInst>(Val: U) || isa<FreezeInst>(Val: U)) {
1632 // Aggregate-typed PHIs, selects and freezes have already been mutated to
1633 // the i32 value-id type up front in runOnFunction, so only the operand
1634 // needs replacing here; their extractvalue users are lowered to
1635 // spv_extractv by visitExtractValueInst.
1636 assert(U->getType() == New->getType() &&
1637 "aggregate PHI/select/freeze should have been mutated to value-id "
1638 "type");
1639 U->replaceUsesOfWith(From: Old, To: New);
1640 } else {
1641 llvm_unreachable("illegal aggregate intrinsic user");
1642 }
1643 }
1644 New->copyMetadata(SrcInst: *Old);
1645 Old->eraseFromParent();
1646}
1647
1648// Lower a poison or undef Op to its placeholder intrinsic.
1649Value *SPIRVEmitIntrinsicsImpl::lowerUndefOrPoison(Value *Op, IRBuilder<> &B,
1650 bool HasPoisonExt) {
1651 auto *UV = dyn_cast<UndefValue>(Val: Op);
1652 if (!UV)
1653 return nullptr;
1654
1655 bool AsPoison = HasPoisonExt && isa<PoisonValue>(Val: UV);
1656 if (isa<PoisonValue>(Val: UV) && !HasPoisonExt)
1657 LLVM_DEBUG(dbgs() << "SPV_KHR_poison_freeze is not enabled. Poison is "
1658 "lowered as undef\n");
1659
1660 Intrinsic::ID IID = AsPoison ? Intrinsic::spv_poison : Intrinsic::spv_undef;
1661 Type *Ty = UV->getType();
1662
1663 // Aggregates use an i32-result placeholder with the real type kept in
1664 // AggrConstTypes and scalar poison uses a type-overloaded one.
1665 if (Ty->isAggregateType()) {
1666 auto *Call =
1667 AsPoison ? B.CreateIntrinsicWithoutFolding(ID: IID, OverloadTypes: {B.getInt32Ty()}, Args: {})
1668 : B.CreateIntrinsicWithoutFolding(ID: IID, Args: {});
1669 AggrConsts[Call] = UV;
1670 AggrConstTypes[Call] = Ty;
1671 return Call;
1672 }
1673
1674 if (AsPoison)
1675 return B.CreateIntrinsic(ID: IID, OverloadTypes: {Ty}, Args: {});
1676 return nullptr;
1677}
1678
1679// Replace aggregate undef or poison operands and extension-enabled scalar
1680// poison operands with placeholder intrinsics. Scalar undef is left as is. See
1681// lowerUndefOrPoison.
1682void SPIRVEmitIntrinsicsImpl::preprocessUndefsAndPoisons(IRBuilder<> &B) {
1683 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*CurrF);
1684 bool HasPoisonExt =
1685 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
1686
1687 SmallVector<Instruction *, 16> Insts;
1688 for (auto &I : instructions(F: CurrF))
1689 Insts.push_back(Elt: &I);
1690
1691 for (Instruction *I : Insts) {
1692 bool BPrepared = false;
1693 auto *Phi = dyn_cast<PHINode>(Val: I);
1694 for (unsigned Idx = 0; Idx < I->getNumOperands(); ++Idx) {
1695 Value *Op = I->getOperand(i: Idx);
1696 if (!isa<UndefValue>(Val: Op) || Op->getType()->isMetadataTy())
1697 continue;
1698 bool IsScalar = !Op->getType()->isAggregateType();
1699 bool AsPoison = HasPoisonExt && isa<PoisonValue>(Val: Op);
1700 // Scalar undef or extensionless scalar poison is directly translatable.
1701 if (IsScalar && !AsPoison)
1702 continue;
1703 // Scalar poison in a phi materializes in the incoming block. Everything
1704 // else materializes right before I.
1705 if (IsScalar && Phi)
1706 B.SetInsertPoint(Phi->getIncomingBlock(i: Idx)->getTerminator());
1707 else if (!BPrepared) {
1708 setInsertPointSkippingPhis(B, I);
1709 BPrepared = true;
1710 }
1711 if (Value *Repl = lowerUndefOrPoison(Op, B, HasPoisonExt))
1712 I->setOperand(i: Idx, Val: Repl);
1713 }
1714 }
1715}
1716
1717// Simplify addrspacecast(null) instructions to ConstantPointerNull of the
1718// target type. Casting null always yields null, and this avoids SPIR-V
1719// lowering issues where the null gets typed as an integer instead of a
1720// pointer.
1721void SPIRVEmitIntrinsicsImpl::simplifyNullAddrSpaceCasts() {
1722 for (Instruction &I : make_early_inc_range(Range: instructions(F: CurrF)))
1723 if (auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: &I))
1724 if (isa<ConstantPointerNull>(Val: ASC->getPointerOperand())) {
1725 ASC->replaceAllUsesWith(
1726 V: ConstantPointerNull::get(T: cast<PointerType>(Val: ASC->getType())));
1727 ASC->eraseFromParent();
1728 }
1729}
1730
1731// True for an aggregate value the legalizer splits into a multi-result op
1732// (with.overflow -> G_UADDO, frexp/sincos/modf -> G_FFREXP/...). These keep a
1733// genuine multi-register result; all other aggregates become a single value-id.
1734static bool isMultiRegisterAggregate(Value *V) {
1735 if (!V->getType()->isAggregateType())
1736 return false;
1737 return isa<IntrinsicInst>(Val: V) && !isSpvIntrinsic(Arg: V);
1738}
1739
1740// True for an aggregate PHI/select/freeze, which is lowered to a single
1741// value-id.
1742static bool isAggregateValueIdInstr(const Instruction &I) {
1743 return (isa<PHINode>(Val: I) || isa<SelectInst>(Val: I) || isa<FreezeInst>(Val: I)) &&
1744 I.getType()->isAggregateType();
1745}
1746
1747// Give each multi-register aggregate arm of an aggregate PHI/select/freeze a
1748// single value-id by reassembling it with extractvalue + insertvalue, so the
1749// arm matches the result once it is mutated to a value-id.
1750void SPIRVEmitIntrinsicsImpl::insertCompositeAggregateArms(Instruction *I,
1751 IRBuilder<> &B) {
1752 auto *Phi = dyn_cast<PHINode>(Val: I);
1753 for (Use &U : I->operands()) {
1754 Value *Op = U.get();
1755 if (!isMultiRegisterAggregate(V: Op))
1756 continue;
1757 // A PHI arm materializes in its incoming block, everything else after the
1758 // producer.
1759 if (Phi)
1760 B.SetInsertPoint(Phi->getIncomingBlock(U)->getTerminator());
1761 else
1762 setInsertPointAfterDef(B, I: cast<Instruction>(Val: Op));
1763 auto *AggrTy = cast<StructType>(Val: Op->getType());
1764 Value *Composite = PoisonValue::get(T: AggrTy);
1765 for (unsigned Idx = 0, E = AggrTy->getNumElements(); Idx != E; ++Idx) {
1766 Value *Field = B.CreateExtractValue(Agg: Op, Idxs: Idx);
1767 Composite = B.CreateInsertValue(Agg: Composite, Val: Field, Idxs: Idx);
1768 }
1769 U.set(Composite);
1770 }
1771}
1772
1773void SPIRVEmitIntrinsicsImpl::preprocessCompositeConstants(IRBuilder<> &B) {
1774 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*CurrF);
1775 bool HasPoisonExt =
1776 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
1777 std::queue<Instruction *> Worklist;
1778 for (auto &I : instructions(F: CurrF))
1779 Worklist.push(x: &I);
1780
1781 while (!Worklist.empty()) {
1782 auto *I = Worklist.front();
1783 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
1784 assert(I);
1785 bool KeepInst = false;
1786 for (const auto &Op : I->operands()) {
1787 Constant *AggrConst = nullptr;
1788 Type *ResTy = nullptr;
1789 if (auto *COp = dyn_cast<ConstantVector>(Val: Op)) {
1790 AggrConst = COp;
1791 ResTy = COp->getType();
1792 } else if (auto *COp = dyn_cast<ConstantArray>(Val: Op)) {
1793 AggrConst = COp;
1794 ResTy = B.getInt32Ty();
1795 } else if (auto *COp = dyn_cast<ConstantStruct>(Val: Op)) {
1796 AggrConst = COp;
1797 ResTy = B.getInt32Ty();
1798 } else if (auto *COp = dyn_cast<ConstantDataArray>(Val: Op)) {
1799 AggrConst = COp;
1800 ResTy = B.getInt32Ty();
1801 } else if (auto *COp = dyn_cast<ConstantAggregateZero>(Val: Op)) {
1802 AggrConst = COp;
1803 ResTy = Op->getType()->isVectorTy() ? COp->getType() : B.getInt32Ty();
1804 }
1805 if (AggrConst) {
1806 auto PrepareInsert = [&]() {
1807 if (BPrepared)
1808 return;
1809 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
1810 : B.SetInsertPoint(I);
1811 BPrepared = true;
1812 };
1813 SmallVector<Value *> Args;
1814 if (auto *COp = dyn_cast<ConstantDataSequential>(Val: Op))
1815 for (unsigned i = 0; i < COp->getNumElements(); ++i)
1816 Args.push_back(Elt: COp->getElementAsConstant(i));
1817 else
1818 for (Value *Op : AggrConst->operands()) {
1819 // Simplify addrspacecast(null) to null in the target address space
1820 // so that null pointers get the correct pointer type when lowered.
1821 if (auto *CE = dyn_cast<ConstantExpr>(Val: Op);
1822 CE && CE->getOpcode() == Instruction::AddrSpaceCast &&
1823 isa<ConstantPointerNull>(Val: CE->getOperand(i_nocapture: 0)))
1824 Op = ConstantPointerNull::get(T: cast<PointerType>(Val: CE->getType()));
1825 // Undef or poison nested in a constant aggregate is not a direct
1826 // instruction operand, so preprocessUndefsAndPoisons() misses it.
1827 // An unlowered aggregate one would reach IRTranslator as an
1828 // untranslatable spv_const_composite operand.
1829 if (isa<UndefValue>(Val: Op)) {
1830 PrepareInsert();
1831 if (Value *Repl = lowerUndefOrPoison(Op, B, HasPoisonExt))
1832 Op = Repl;
1833 }
1834 Args.push_back(Elt: Op);
1835 }
1836 PrepareInsert();
1837 auto *CI = B.CreateIntrinsicWithoutFolding(
1838 ID: Intrinsic::spv_const_composite, OverloadTypes: {ResTy}, Args: {Args});
1839 Worklist.push(x: CI);
1840 I->replaceUsesOfWith(From: Op, To: CI);
1841 KeepInst = true;
1842 AggrConsts[CI] = AggrConst;
1843 AggrConstTypes[CI] = deduceNestedTypeHelper(U: AggrConst, UnknownElemTypeI8: false);
1844 }
1845 }
1846 if (!KeepInst)
1847 Worklist.pop();
1848 }
1849}
1850
1851static void createDecorationIntrinsic(Instruction *I, MDNode *Node,
1852 IRBuilder<> &B) {
1853 LLVMContext &Ctx = I->getContext();
1854 setInsertPointAfterDef(B, I);
1855 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
1856 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: {Node}))});
1857}
1858
1859static void createRoundingModeDecoration(Instruction *I,
1860 unsigned RoundingModeDeco,
1861 IRBuilder<> &B) {
1862 LLVMContext &Ctx = I->getContext();
1863 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1864 MDNode *RoundingModeNode = MDNode::get(
1865 Context&: Ctx,
1866 MDs: {ConstantAsMetadata::get(
1867 C: ConstantInt::get(Ty: Int32Ty, V: SPIRV::Decoration::FPRoundingMode)),
1868 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: RoundingModeDeco))});
1869 createDecorationIntrinsic(I, Node: RoundingModeNode, B);
1870}
1871
1872static void createSaturatedConversionDecoration(Instruction *I,
1873 IRBuilder<> &B) {
1874 LLVMContext &Ctx = I->getContext();
1875 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1876 MDNode *SaturatedConversionNode =
1877 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1878 Ty: Int32Ty, V: SPIRV::Decoration::SaturatedConversion))});
1879 createDecorationIntrinsic(I, Node: SaturatedConversionNode, B);
1880}
1881
1882static void addSaturatedDecorationToIntrinsic(Instruction *I, IRBuilder<> &B) {
1883 if (match(V: I, P: m_AnyIntrinsic<Intrinsic::fptosi_sat, Intrinsic::fptoui_sat>()))
1884 createSaturatedConversionDecoration(I, B);
1885}
1886
1887Instruction *SPIRVEmitIntrinsicsImpl::visitCallInst(CallInst &Call) {
1888 if (!Call.isInlineAsm())
1889 return &Call;
1890
1891 LLVMContext &Ctx = CurrF->getContext();
1892 // TODO: this does not retain elementtype info for memory constraints, which
1893 // in turn means that we lower them into pointers to i8, rather than
1894 // pointers to elementtype; this can be fixed during reverse translation
1895 // but we should correct it here, possibly by tweaking the function
1896 // type to take TypedPointerType args.
1897 Constant *TyC = UndefValue::get(T: SPIRV::getOriginalFunctionType(CB: Call));
1898 MDString *ConstraintString =
1899 MDString::get(Context&: Ctx, Str: SPIRV::getOriginalAsmConstraints(CB: Call));
1900 SmallVector<Value *> Args = {
1901 buildMD(Arg: TyC),
1902 MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: ConstraintString))};
1903 for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++)
1904 Args.push_back(Elt: Call.getArgOperand(i: OpIdx));
1905
1906 IRBuilder<> B(Call.getParent());
1907 B.SetInsertPoint(&Call);
1908 B.CreateIntrinsic(ID: Intrinsic::spv_inline_asm, Args: {Args});
1909 return &Call;
1910}
1911
1912// Use a tip about rounding mode to create a decoration.
1913void SPIRVEmitIntrinsicsImpl::useRoundingMode(ConstrainedFPIntrinsic *FPI,
1914 IRBuilder<> &B) {
1915 std::optional<RoundingMode> RM = FPI->getRoundingMode();
1916 if (!RM.has_value())
1917 return;
1918 unsigned RoundingModeDeco = std::numeric_limits<unsigned>::max();
1919 switch (RM.value()) {
1920 default:
1921 // ignore unknown rounding modes
1922 break;
1923 case RoundingMode::NearestTiesToEven:
1924 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTE;
1925 break;
1926 case RoundingMode::TowardNegative:
1927 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTN;
1928 break;
1929 case RoundingMode::TowardPositive:
1930 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTP;
1931 break;
1932 case RoundingMode::TowardZero:
1933 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTZ;
1934 break;
1935 case RoundingMode::Dynamic:
1936 case RoundingMode::NearestTiesToAway:
1937 // TODO: check if supported
1938 break;
1939 }
1940 if (RoundingModeDeco == std::numeric_limits<unsigned>::max())
1941 return;
1942 // Convert the tip about rounding mode into a decoration record.
1943 createRoundingModeDecoration(I: FPI, RoundingModeDeco, B);
1944}
1945
1946Instruction *SPIRVEmitIntrinsicsImpl::visitSwitchInst(SwitchInst &I) {
1947 BasicBlock *ParentBB = I.getParent();
1948 Function *F = ParentBB->getParent();
1949 IRBuilder<> B(ParentBB);
1950 B.SetInsertPoint(&I);
1951 SmallVector<Value *, 4> Args;
1952 SmallVector<BasicBlock *> BBCases;
1953 Args.push_back(Elt: I.getCondition());
1954 BBCases.push_back(Elt: I.getDefaultDest());
1955 Args.push_back(Elt: BlockAddress::get(F, BB: I.getDefaultDest()));
1956 for (auto &Case : I.cases()) {
1957 Args.push_back(Elt: Case.getCaseValue());
1958 BBCases.push_back(Elt: Case.getCaseSuccessor());
1959 Args.push_back(Elt: BlockAddress::get(F, BB: Case.getCaseSuccessor()));
1960 }
1961 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
1962 ID: Intrinsic::spv_switch, OverloadTypes: {I.getOperand(i_nocapture: 0)->getType()}, Args: {Args});
1963 // remove switch to avoid its unneeded and undesirable unwrap into branches
1964 // and conditions
1965 replaceAllUsesWith(Src: &I, Dest: NewI);
1966 I.eraseFromParent();
1967 // insert artificial and temporary instruction to preserve valid CFG,
1968 // it will be removed after IR translation pass
1969 B.SetInsertPoint(ParentBB);
1970 IndirectBrInst *BrI = B.CreateIndirectBr(
1971 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: ParentBB->getContext())),
1972 NumDests: BBCases.size());
1973 for (BasicBlock *BBCase : BBCases)
1974 BrI->addDestination(Dest: BBCase);
1975 return BrI;
1976}
1977
1978static bool isFirstIndexZero(const GetElementPtrInst *GEP) {
1979 return GEP->getNumIndices() > 0 && match(V: GEP->getOperand(i_nocapture: 1), P: m_Zero());
1980}
1981
1982Instruction *SPIRVEmitIntrinsicsImpl::visitIntrinsicInst(IntrinsicInst &I) {
1983 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
1984 if (!SGEP)
1985 return &I;
1986
1987 IRBuilder<> B(I.getParent());
1988 B.SetInsertPoint(&I);
1989 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
1990 SmallVector<Value *, 4> Args;
1991 Args.push_back(/* inBounds= */ Elt: B.getInt1(V: true));
1992 Args.push_back(Elt: I.getOperand(i_nocapture: 0));
1993 Args.push_back(/* zero index */ Elt: B.getInt32(C: 0));
1994 for (unsigned J = 0; J < SGEP->getNumIndices(); ++J)
1995 Args.push_back(Elt: SGEP->getIndexOperand(Index: J));
1996
1997 Instruction *NewI =
1998 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: Types, Args);
1999 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2000 return NewI;
2001}
2002
2003Instruction *
2004SPIRVEmitIntrinsicsImpl::visitGetElementPtrInst(GetElementPtrInst &I) {
2005 IRBuilder<> B(I.getParent());
2006 B.SetInsertPoint(&I);
2007
2008 // OpPtrAccessChain requires a scalar pointer result; scalarize per-lane
2009 // GEPs that return <N x ptr> and rebuild the vector via insertelement.
2010 if (auto *RetVTy = dyn_cast<FixedVectorType>(Val: I.getType())) {
2011 unsigned N = RetVTy->getNumElements();
2012 Value *PtrOp = I.getPointerOperand();
2013 bool PtrIsVec = isa<VectorType>(Val: PtrOp->getType());
2014 Type *ResultPtrTy = RetVTy->getElementType();
2015 Type *ScalarPtrTy = PtrOp->getType()->getScalarType();
2016 SmallVector<Type *, 2> GepTypes = {ResultPtrTy, ScalarPtrTy};
2017 Value *InBounds = B.getInt1(V: I.isInBounds());
2018 Type *LanePointeeTy = getGEPType(Ref: &I);
2019 Type *SrcElemTy = I.getSourceElementType();
2020
2021 // Pin the lane pointee type on the vector operand and on each extracted
2022 // lane so the prelegalizer wraps them as OpTypeVector/OpTypePointer of
2023 // the right element type instead of defaulting to i8.
2024 if (PtrIsVec)
2025 GR->buildAssignPtr(B, ElemTy: SrcElemTy, Arg: PtrOp);
2026
2027 Value *VecResult = PoisonValue::get(T: RetVTy);
2028 for (unsigned Lane = 0; Lane < N; ++Lane) {
2029 Value *LaneIdx = B.getInt32(C: Lane);
2030 Value *ScalarPtr = PtrOp;
2031 if (PtrIsVec) {
2032 SmallVector<Type *, 3> ExtractTypes = {ScalarPtrTy, PtrOp->getType(),
2033 LaneIdx->getType()};
2034 ScalarPtr = B.CreateIntrinsic(ID: Intrinsic::spv_extractelt, OverloadTypes: {ExtractTypes},
2035 Args: {PtrOp, LaneIdx});
2036 GR->buildAssignPtr(B, ElemTy: SrcElemTy, Arg: ScalarPtr);
2037 }
2038 SmallVector<Value *, 4> Args;
2039 Args.push_back(Elt: InBounds);
2040 Args.push_back(Elt: ScalarPtr);
2041 for (Value *Idx : I.indices()) {
2042 if (isa<VectorType>(Val: Idx->getType())) {
2043 // We cannot use the builder here as for splat-ed / constant vectors
2044 // it will fold to the scalar, and then it becomes impossible to
2045 // retrieve / retain the vectorness.
2046 auto *EI =
2047 ExtractElementInst::Create(Vec: Idx, Idx: LaneIdx, NameStr: "", InsertBefore: B.GetInsertPoint());
2048 if (isVector1(Ty: Idx->getType())) // IRTranslator clobbers <1 x T>.
2049 Args.push_back(Elt: visitExtractElementInst(I&: *EI));
2050 else
2051 Args.push_back(Elt: EI);
2052 } else {
2053 Args.push_back(Elt: Idx);
2054 }
2055 }
2056 Value *ScalarGep = B.CreateIntrinsic(ID: Intrinsic::spv_gep, OverloadTypes: GepTypes, Args);
2057 GR->buildAssignPtr(B, ElemTy: LanePointeeTy, Arg: ScalarGep);
2058 VecResult = B.CreateInsertElement(Vec: VecResult, NewElt: ScalarGep, Idx: LaneIdx);
2059 }
2060
2061 auto *NewI = cast<Instruction>(Val: VecResult);
2062 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2063
2064 if (CallInst *Old = GR->findAssignPtrTypeInstr(Val: NewI)) {
2065 Old->eraseFromParent();
2066 GR->addAssignPtrTypeInstr(Val: NewI, AssignPtrTyCI: nullptr);
2067 }
2068 setInsertPointAfterDef(B, I: NewI);
2069 GR->buildAssignPtr(B, ElemTy: LanePointeeTy, Arg: NewI);
2070
2071 return NewI;
2072 }
2073
2074 if (TM.getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: &I)) {
2075 // Logical SPIR-V cannot use the OpPtrAccessChain instruction. If the first
2076 // index of the GEP is not 0, then we need to try to adjust it.
2077 //
2078 // If the GEP is doing byte addressing, try to rebuild the full access chain
2079 // from the type of the pointer.
2080 if (getByteAddressingMultiplier(Ty: I.getSourceElementType())) {
2081 return buildLogicalAccessChainFromGEP(GEP&: I);
2082 }
2083
2084 // Look for the array-to-pointer decay. If this is the pattern
2085 // we can adjust the types, and prepend a 0 to the indices.
2086 Value *PtrOp = I.getPointerOperand();
2087 Type *SrcElemTy = I.getSourceElementType();
2088 Type *DeducedPointeeTy = deduceElementType(I: PtrOp, UnknownElemTypeI8: true);
2089
2090 if (auto *ArrTy = dyn_cast<ArrayType>(Val: DeducedPointeeTy)) {
2091 if (ArrTy->getElementType() == SrcElemTy) {
2092 SmallVector<Value *> NewIndices;
2093 Type *FirstIdxType = I.getOperand(i_nocapture: 1)->getType();
2094 NewIndices.push_back(Elt: ConstantInt::get(Ty: FirstIdxType, V: 0));
2095 for (Value *Idx : I.indices())
2096 NewIndices.push_back(Elt: Idx);
2097
2098 SmallVector<Type *, 2> Types = {I.getType(), I.getPointerOperandType()};
2099 SmallVector<Value *, 4> Args;
2100 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
2101 Args.push_back(Elt: I.getPointerOperand());
2102 Args.append(in_start: NewIndices.begin(), in_end: NewIndices.end());
2103
2104 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep,
2105 OverloadTypes: {Types}, Args: {Args});
2106 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2107 return NewI;
2108 }
2109 }
2110 }
2111
2112 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
2113 SmallVector<Value *, 4> Args;
2114 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
2115 llvm::append_range(C&: Args, R: I.operands());
2116 Instruction *NewI =
2117 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
2118 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2119 return NewI;
2120}
2121
2122Instruction *SPIRVEmitIntrinsicsImpl::visitBitCastInst(BitCastInst &I) {
2123 IRBuilder<> B(I.getParent());
2124 B.SetInsertPoint(&I);
2125 Value *Source = I.getOperand(i_nocapture: 0);
2126
2127 // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of
2128 // varying element types. In case of IR coming from older versions of LLVM
2129 // such bitcasts do not provide sufficient information, should be just skipped
2130 // here, and handled in insertPtrCastOrAssignTypeInstr.
2131 if (isPointerTy(T: I.getType())) {
2132 replaceAllUsesWith(Src: &I, Dest: Source);
2133 I.eraseFromParent();
2134 return nullptr;
2135 }
2136
2137 SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
2138 SmallVector<Value *> Args(I.op_begin(), I.op_end());
2139 Instruction *NewI =
2140 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_bitcast, OverloadTypes: {Types}, Args: {Args});
2141 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2142 return NewI;
2143}
2144
2145void SPIRVEmitIntrinsicsImpl::insertAssignPtrTypeTargetExt(
2146 TargetExtType *AssignedType, Value *V, IRBuilder<> &B) {
2147 Type *VTy = V->getType();
2148
2149 // A couple of sanity checks.
2150 assert((isPointerTy(VTy)) && "Expect a pointer type!");
2151 if (Type *ElemTy = getPointeeType(Ty: VTy))
2152 if (ElemTy != AssignedType)
2153 report_fatal_error(reason: "Unexpected pointer element type!");
2154
2155 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: V);
2156 if (!AssignCI) {
2157 GR->buildAssignType(B, Ty: AssignedType, Arg: V, CanUseAnyVectorRank);
2158 return;
2159 }
2160
2161 Type *CurrentType =
2162 dyn_cast<ConstantAsMetadata>(
2163 Val: cast<MetadataAsValue>(Val: AssignCI->getOperand(i_nocapture: 1))->getMetadata())
2164 ->getType();
2165 if (CurrentType == AssignedType)
2166 return;
2167
2168 // Builtin types cannot be redeclared or casted.
2169 if (CurrentType->isTargetExtTy())
2170 report_fatal_error(reason: "Type mismatch " + CurrentType->getTargetExtName() +
2171 "/" + AssignedType->getTargetExtName() +
2172 " for value " + V->getName(),
2173 gen_crash_diag: false);
2174
2175 // Our previous guess about the type seems to be wrong, let's update
2176 // inferred type according to a new, more precise type information.
2177 GR->updateAssignType(
2178 AssignCI, Arg: V, OfType: getNormalizedPoisonValue(Ty: AssignedType, CanUseAnyVectorRank));
2179}
2180
2181void SPIRVEmitIntrinsicsImpl::replacePointerOperandWithPtrCast(
2182 Instruction *I, Value *Pointer, Type *ExpectedElementType,
2183 unsigned OperandToReplace, IRBuilder<> &B) {
2184 TypeValidated.insert(Ptr: I);
2185
2186 // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType
2187 Type *PointerElemTy = deduceElementTypeHelper(I: Pointer, UnknownElemTypeI8: false);
2188 if (PointerElemTy == ExpectedElementType ||
2189 isEquivalentTypes(Ty1: PointerElemTy, Ty2: ExpectedElementType))
2190 return;
2191
2192 setInsertPointSkippingPhis(B, I);
2193 Value *ExpectedElementVal =
2194 getNormalizedPoisonValue(Ty: ExpectedElementType, CanUseAnyVectorRank);
2195 MetadataAsValue *VMD = buildMD(Arg: ExpectedElementVal);
2196 unsigned AddressSpace = getPointerAddressSpace(T: Pointer->getType());
2197 bool FirstPtrCastOrAssignPtrType = true;
2198
2199 // Do not emit new spv_ptrcast if equivalent one already exists or when
2200 // spv_assign_ptr_type already targets this pointer with the same element
2201 // type.
2202 if (Pointer->hasUseList()) {
2203 for (auto User : Pointer->users()) {
2204 auto *II = dyn_cast<IntrinsicInst>(Val: User);
2205 if (!II ||
2206 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
2207 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
2208 II->getOperand(i_nocapture: 0) != Pointer)
2209 continue;
2210
2211 // There is some spv_ptrcast/spv_assign_ptr_type already targeting this
2212 // pointer.
2213 FirstPtrCastOrAssignPtrType = false;
2214 if (II->getOperand(i_nocapture: 1) != VMD ||
2215 dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 2))->getSExtValue() !=
2216 AddressSpace)
2217 continue;
2218
2219 // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the
2220 // same element type and address space.
2221 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
2222 return;
2223
2224 // This must be a spv_ptrcast, do not emit new if this one has the same BB
2225 // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type.
2226 if (II->getParent() != I->getParent())
2227 continue;
2228
2229 I->setOperand(i: OperandToReplace, Val: II);
2230 return;
2231 }
2232 }
2233
2234 // Never replace an already-deduced pointer element type with a non-pointer
2235 // one. The conflicting use comes from a mis-deduced expected type. Leave the
2236 // operand untouched rather than emitting a ptrcast that re-introduces the
2237 // collapsed type at the use site.
2238 if (PointerElemTy && isPointerTyOrWrapper(Ty: PointerElemTy) &&
2239 !isPointerTyOrWrapper(Ty: ExpectedElementType) &&
2240 tracesToPointerAlloca(V: Pointer))
2241 return;
2242
2243 if (isa<Instruction>(Val: Pointer) || isa<Argument>(Val: Pointer)) {
2244 if (FirstPtrCastOrAssignPtrType) {
2245 // If this would be the first spv_ptrcast, do not emit spv_ptrcast and
2246 // emit spv_assign_ptr_type instead.
2247 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
2248 return;
2249 } else if (isTodoType(Op: Pointer)) {
2250 eraseTodoType(Op: Pointer);
2251 if (!isa<CallInst>(Val: Pointer) && !isaGEP(V: Pointer) &&
2252 !isa<AllocaInst>(Val: Pointer)) {
2253 // If this wouldn't be the first spv_ptrcast but existing type info is
2254 // uncomplete, update spv_assign_ptr_type arguments.
2255 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Pointer)) {
2256 Type *PrevElemTy = GR->findDeducedElementType(Val: Pointer);
2257 assert(PrevElemTy);
2258 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
2259 std::make_pair(x&: I, y&: Pointer)};
2260 GR->updateAssignType(AssignCI, Arg: Pointer, OfType: ExpectedElementVal);
2261 propagateElemType(Op: Pointer, ElemTy: PrevElemTy, VisitedSubst);
2262 } else {
2263 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
2264 }
2265 return;
2266 }
2267 }
2268 }
2269
2270 // Emit spv_ptrcast
2271 SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()};
2272 SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(C: AddressSpace)};
2273 auto *PtrCastI = B.CreateIntrinsic(ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
2274 I->setOperand(i: OperandToReplace, Val: PtrCastI);
2275 // We need to set up a pointee type for the newly created spv_ptrcast.
2276 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: PtrCastI);
2277}
2278
2279void SPIRVEmitIntrinsicsImpl::insertPtrCastOrAssignTypeInstr(Instruction *I,
2280 IRBuilder<> &B) {
2281 // Handle basic instructions:
2282 StoreInst *SI = dyn_cast<StoreInst>(Val: I);
2283 if (IsKernelArgInt8(F: CurrF, SI)) {
2284 replacePointerOperandWithPtrCast(
2285 I, Pointer: SI->getValueOperand(), ExpectedElementType: IntegerType::getInt8Ty(C&: CurrF->getContext()),
2286 OperandToReplace: 0, B);
2287 }
2288 if (SI) {
2289 Value *Op = SI->getValueOperand();
2290 Value *Pointer = SI->getPointerOperand();
2291 Type *OpTy = Op->getType();
2292 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
2293 OpTy = restoreMutatedType(GR, I: OpI, Ty: OpTy);
2294 if (auto It = AggrConstTypes.find(Val: OpI); It != AggrConstTypes.end())
2295 OpTy = It->second;
2296 }
2297 if (OpTy == Op->getType())
2298 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: Op, UnknownElemTypeI8: false);
2299 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 1, B);
2300 return;
2301 }
2302 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
2303 Value *Pointer = LI->getPointerOperand();
2304 Type *OpTy = LI->getType();
2305 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
2306 if (Type *ElemTy = GR->findDeducedElementType(Val: LI)) {
2307 OpTy = getTypedPointerWrapper(ElemTy, AS: PtrTy->getAddressSpace());
2308 } else {
2309 Type *NewOpTy = OpTy;
2310 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: LI, UnknownElemTypeI8: false);
2311 if (OpTy == NewOpTy)
2312 insertTodoType(Op: Pointer);
2313 }
2314 }
2315 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
2316 return;
2317 }
2318 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(Val: I)) {
2319 Value *Pointer = GEPI->getPointerOperand();
2320 Type *OpTy = nullptr;
2321
2322 // Logical SPIR-V is not allowed to use Op*PtrAccessChain instructions. If
2323 // the first index is 0, then we can trivially lower to OpAccessChain. If
2324 // not we need to try to rewrite the GEP. We avoid adding a pointer cast at
2325 // this time, and will rewrite the GEP when visiting it.
2326 if (TM.getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: GEPI)) {
2327 return;
2328 }
2329
2330 // In all cases, fall back to the GEP type if type scavenging failed.
2331 if (!OpTy)
2332 OpTy = GEPI->getSourceElementType();
2333
2334 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
2335 if (isNestedPointer(Ty: OpTy))
2336 insertTodoType(Op: Pointer);
2337 return;
2338 }
2339
2340 // TODO: review and merge with existing logics:
2341 // Handle calls to builtins (non-intrinsics):
2342 CallInst *CI = dyn_cast<CallInst>(Val: I);
2343 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
2344 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
2345 return;
2346
2347 // collect information about formal parameter types
2348 std::string DemangledName =
2349 getOclOrSpirvBuiltinDemangledName(Name: CI->getCalledFunction()->getName());
2350 Function *CalledF = CI->getCalledFunction();
2351 SmallVector<Type *, 4> CalledArgTys;
2352 bool HaveTypes = false;
2353 for (unsigned OpIdx = 0; OpIdx < CalledF->arg_size(); ++OpIdx) {
2354 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2355 Type *ArgType = CalledArg->getType();
2356 if (!isPointerTy(T: ArgType)) {
2357 CalledArgTys.push_back(Elt: nullptr);
2358 } else if (Type *ArgTypeElem = getPointeeType(Ty: ArgType)) {
2359 CalledArgTys.push_back(Elt: ArgTypeElem);
2360 HaveTypes = true;
2361 } else {
2362 Type *ElemTy = GR->findDeducedElementType(Val: CalledArg);
2363 if (!ElemTy && hasPointeeTypeAttr(Arg: CalledArg))
2364 ElemTy = getPointeeTypeByAttr(Arg: CalledArg);
2365 if (!ElemTy) {
2366 ElemTy = getPointeeTypeByCallInst(DemangledName, CalledF, OpIdx);
2367 if (ElemTy) {
2368 GR->addDeducedElementType(Val: CalledArg,
2369 Ty: normalizeType(Ty: ElemTy, CanUseAnyVectorRank));
2370 } else {
2371 for (User *U : CalledArg->users()) {
2372 if (Instruction *Inst = dyn_cast<Instruction>(Val: U)) {
2373 if ((ElemTy = deduceElementTypeHelper(I: Inst, UnknownElemTypeI8: false)) != nullptr)
2374 break;
2375 }
2376 }
2377 }
2378 }
2379 HaveTypes |= ElemTy != nullptr;
2380 CalledArgTys.push_back(Elt: ElemTy);
2381 }
2382 }
2383
2384 if (DemangledName.empty() && !HaveTypes)
2385 return;
2386
2387 for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) {
2388 Value *ArgOperand = CI->getArgOperand(i: OpIdx);
2389 if (!isPointerTy(T: ArgOperand->getType()))
2390 continue;
2391
2392 // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs()
2393 if (!isa<Instruction>(Val: ArgOperand) && !isa<Argument>(Val: ArgOperand)) {
2394 // However, we may have assumptions about the formal argument's type and
2395 // may have a need to insert a ptr cast for the actual parameter of this
2396 // call.
2397 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2398 if (!GR->findDeducedElementType(Val: CalledArg))
2399 continue;
2400 }
2401
2402 Type *ExpectedType =
2403 OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr;
2404 if (!ExpectedType && !DemangledName.empty())
2405 ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType(
2406 DemangledCall: DemangledName, ArgIdx: OpIdx, Ctx&: I->getContext());
2407 if (!ExpectedType || ExpectedType->isVoidTy())
2408 continue;
2409
2410 if (ExpectedType->isTargetExtTy() &&
2411 !isTypedPointerWrapper(ExtTy: cast<TargetExtType>(Val: ExpectedType)))
2412 insertAssignPtrTypeTargetExt(AssignedType: cast<TargetExtType>(Val: ExpectedType),
2413 V: ArgOperand, B);
2414 else
2415 replacePointerOperandWithPtrCast(I: CI, Pointer: ArgOperand, ExpectedElementType: ExpectedType, OperandToReplace: OpIdx, B);
2416 }
2417}
2418
2419Instruction *
2420SPIRVEmitIntrinsicsImpl::visitInsertElementInst(InsertElementInst &I) {
2421 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2422 // type in LLT and IRTranslator will replace it by the scalar.
2423 if (isVector1(Ty: I.getType()) && !CanUseAnyVectorRank)
2424 return &I;
2425
2426 SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType(),
2427 I.getOperand(i_nocapture: 1)->getType(),
2428 I.getOperand(i_nocapture: 2)->getType()};
2429 IRBuilder<> B(I.getParent());
2430 B.SetInsertPoint(&I);
2431 SmallVector<Value *> Args(I.op_begin(), I.op_end());
2432 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_insertelt,
2433 OverloadTypes: {Types}, Args: {Args});
2434 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2435 return NewI;
2436}
2437
2438Instruction *
2439SPIRVEmitIntrinsicsImpl::visitExtractElementInst(ExtractElementInst &I) {
2440 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2441 // type in LLT and IRTranslator will replace it by the scalar.
2442 if (isVector1(Ty: I.getVectorOperandType()) && !CanUseAnyVectorRank)
2443 return &I;
2444
2445 IRBuilder<> B(I.getParent());
2446 B.SetInsertPoint(&I);
2447 SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
2448 I.getIndexOperand()->getType()};
2449 SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
2450 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_extractelt,
2451 OverloadTypes: {Types}, Args: {Args});
2452 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2453 return NewI;
2454}
2455
2456Instruction *SPIRVEmitIntrinsicsImpl::visitInsertValueInst(InsertValueInst &I) {
2457 IRBuilder<> B(I.getParent());
2458 B.SetInsertPoint(&I);
2459 SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
2460 SmallVector<Value *> Args;
2461 Value *AggregateOp = I.getAggregateOperand();
2462 if (isa<UndefValue>(Val: AggregateOp))
2463 Args.push_back(Elt: UndefValue::get(T: B.getInt32Ty()));
2464 else
2465 Args.push_back(Elt: AggregateOp);
2466 Args.push_back(Elt: I.getInsertedValueOperand());
2467 for (auto &Op : I.indices())
2468 Args.push_back(Elt: B.getInt32(C: Op));
2469 Instruction *NewI =
2470 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_insertv, OverloadTypes: {Types}, Args: {Args});
2471 replaceMemInstrUses(Old: &I, New: NewI, B);
2472 return NewI;
2473}
2474
2475Instruction *
2476SPIRVEmitIntrinsicsImpl::visitExtractValueInst(ExtractValueInst &I) {
2477 IRBuilder<> B(I.getParent());
2478 B.SetInsertPoint(&I);
2479 if (I.getAggregateOperand()->getType()->isAggregateType()) {
2480 // Mutate an aggregate-returning spv_extractv producer to i32 so
2481 // IRTranslator does not see a multi-register value.
2482 CallBase *CB = dyn_cast<CallBase>(Val: I.getAggregateOperand());
2483 if (!CB || CB->getIntrinsicID() != Intrinsic::spv_extractv)
2484 return &I;
2485 CB->mutateType(Ty: B.getInt32Ty());
2486 }
2487 SmallVector<Value *> Args(I.operands());
2488 for (auto &Op : I.indices())
2489 Args.push_back(Elt: B.getInt32(C: Op));
2490 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_extractv,
2491 OverloadTypes: {I.getType()}, Args: {Args});
2492 // If this aggregate extract feeds another insertvalue, the extracted
2493 // composite is used as a SPIR-V value-id by llvm.spv.insertv. Keep the real
2494 // aggregate type in metadata, but expose the value itself as i32 so the
2495 // intrinsic signature remains valid.
2496 if (NewI->getType()->isAggregateType() &&
2497 any_of(Range: I.users(), P: [](User *U) { return isa<InsertValueInst>(Val: U); })) {
2498 AggrConstTypes[NewI] = I.getType();
2499 NewI->mutateType(Ty: B.getInt32Ty());
2500 replaceMemInstrUses(Old: &I, New: NewI, B);
2501 return NewI;
2502 }
2503 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2504 // If the aggregate result feeds a return or callsite whose type was rewritten
2505 // to an i32 value-id by SPIRVPrepareFunctions, mutate it to match.
2506 if (NewI->getType()->isAggregateType()) {
2507 for (const Use &U : NewI->uses()) {
2508 User *Usr = U.getUser();
2509 if (auto *RI = dyn_cast<ReturnInst>(Val: Usr)) {
2510 if (RI->getFunction()->getReturnType() != NewI->getType()) {
2511 NewI->mutateType(Ty: B.getInt32Ty());
2512 break;
2513 }
2514 continue;
2515 }
2516 auto *CB = dyn_cast<CallBase>(Val: Usr);
2517 if (!CB || !CB->isArgOperand(U: &U))
2518 continue;
2519 unsigned ArgNo = CB->getArgOperandNo(U: &U);
2520 FunctionType *FT = CB->getFunctionType();
2521 if (ArgNo < FT->getNumParams() &&
2522 !FT->getParamType(i: ArgNo)->isAggregateType()) {
2523 NewI->mutateType(Ty: B.getInt32Ty());
2524 break;
2525 }
2526 }
2527 }
2528 return NewI;
2529}
2530
2531Instruction *SPIRVEmitIntrinsicsImpl::visitLoadInst(LoadInst &I) {
2532 if (!I.getType()->isAggregateType())
2533 return &I;
2534 IRBuilder<> B(I.getParent());
2535 B.SetInsertPoint(&I);
2536 TrackConstants = false;
2537 const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
2538 MachineMemOperand::Flags Flags =
2539 TLI->getLoadMemOperandFlags(LI: I, DL: CurrF->getDataLayout());
2540
2541 unsigned IntrinsicId;
2542 SmallVector<Value *, 4> Args = {I.getPointerOperand(), B.getInt16(C: Flags)};
2543 if (!I.isAtomic()) {
2544 IntrinsicId = Intrinsic::spv_load;
2545 Args.push_back(Elt: B.getInt32(C: I.getAlign().value()));
2546 } else {
2547 IntrinsicId = Intrinsic::spv_atomic_load;
2548 Args.push_back(Elt: B.getInt8(C: static_cast<uint8_t>(I.getOrdering())));
2549 }
2550 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
2551 ID: IntrinsicId, OverloadTypes: {I.getOperand(i_nocapture: 0)->getType()}, Args);
2552
2553 replaceMemInstrUses(Old: &I, New: NewI, B);
2554 return NewI;
2555}
2556
2557Instruction *SPIRVEmitIntrinsicsImpl::visitStoreInst(StoreInst &I) {
2558 if (!AggrStores.contains(Ptr: &I))
2559 return &I;
2560 IRBuilder<> B(I.getParent());
2561 B.SetInsertPoint(&I);
2562 TrackConstants = false;
2563 const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
2564 MachineMemOperand::Flags Flags =
2565 TLI->getStoreMemOperandFlags(SI: I, DL: CurrF->getDataLayout());
2566 auto *PtrOp = I.getPointerOperand();
2567
2568 if (I.getValueOperand()->getType()->isAggregateType()) {
2569 // It is possible that what used to be an ExtractValueInst has been replaced
2570 // with a call to the spv_extractv intrinsic, and that said call hasn't
2571 // had its return type replaced with i32 during the dedicated pass (because
2572 // it was emitted later); we have to handle this here, because IRTranslator
2573 // cannot deal with multi-register types at the moment.
2574 CallBase *CB = dyn_cast<CallBase>(Val: I.getValueOperand());
2575 assert(CB && CB->getIntrinsicID() == Intrinsic::spv_extractv &&
2576 "Unexpected argument of aggregate type, should be spv_extractv!");
2577 CB->mutateType(Ty: B.getInt32Ty());
2578 }
2579
2580 unsigned IntrinsicId;
2581 SmallVector<Value *, 4> Args = {I.getValueOperand(), PtrOp,
2582 B.getInt16(C: Flags)};
2583 if (!I.isAtomic()) {
2584 IntrinsicId = Intrinsic::spv_store;
2585 Args.push_back(Elt: B.getInt32(C: I.getAlign().value()));
2586 } else {
2587 IntrinsicId = Intrinsic::spv_atomic_store;
2588 Args.push_back(Elt: B.getInt8(C: static_cast<uint8_t>(I.getOrdering())));
2589 }
2590 Instruction *NewI = B.CreateIntrinsicWithoutFolding(
2591 ID: IntrinsicId, OverloadTypes: {I.getValueOperand()->getType(), PtrOp->getType()}, Args);
2592 NewI->copyMetadata(SrcInst: I);
2593 I.eraseFromParent();
2594 return NewI;
2595}
2596
2597Instruction *SPIRVEmitIntrinsicsImpl::visitAllocaInst(AllocaInst &I) {
2598 Value *ArraySize = nullptr;
2599 if (I.isArrayAllocation()) {
2600 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I.getFunction());
2601 if (!STI->canUseExtension(
2602 E: SPIRV::Extension::SPV_INTEL_variable_length_array))
2603 report_fatal_error(
2604 reason: "array allocation: this instruction requires the following "
2605 "SPIR-V extension: SPV_INTEL_variable_length_array",
2606 gen_crash_diag: false);
2607 ArraySize = I.getArraySize();
2608 }
2609 IRBuilder<> B(I.getParent());
2610 B.SetInsertPoint(&I);
2611 TrackConstants = false;
2612 Type *PtrTy = I.getType();
2613 Instruction *NewI =
2614 ArraySize
2615 ? B.CreateIntrinsicWithoutFolding(
2616 ID: Intrinsic::spv_alloca_array, OverloadTypes: {PtrTy, ArraySize->getType()},
2617 Args: {ArraySize, B.getInt32(C: I.getAlign().value())})
2618 : B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_alloca, OverloadTypes: {PtrTy},
2619 Args: {B.getInt32(C: I.getAlign().value())});
2620 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2621 return NewI;
2622}
2623
2624Instruction *
2625SPIRVEmitIntrinsicsImpl::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2626 assert(I.getType()->isAggregateType() && "Aggregate result is expected");
2627 IRBuilder<> B(I.getParent());
2628 B.SetInsertPoint(&I);
2629 SmallVector<Value *> Args(I.operands());
2630 const Triple &TT = TM.getTargetTriple();
2631 Args.push_back(Elt: B.getInt32(C: static_cast<uint32_t>(
2632 getMemScope(TT, Ctx&: I.getContext(), Id: I.getSyncScopeID()))));
2633 // Per SPIR-V spec atomic ops must combine the ordering bits with the
2634 // storage-class bit.
2635 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2636 unsigned AS = I.getPointerOperand()->getType()->getPointerAddressSpace();
2637 uint32_t ScSem = static_cast<uint32_t>(
2638 getMemSemanticsForStorageClass(SC: addressSpaceToStorageClass(AddrSpace: AS, STI: ST)));
2639 Args.push_back(Elt: B.getInt32(C: getMemSemanticsWithStorageClass(
2640 TT, OrderSem: static_cast<uint32_t>(getMemSemantics(Ord: I.getSuccessOrdering())),
2641 StorageClassSem: ScSem)));
2642 Args.push_back(Elt: B.getInt32(C: getMemSemanticsWithStorageClass(
2643 TT, OrderSem: static_cast<uint32_t>(getMemSemantics(Ord: I.getFailureOrdering())),
2644 StorageClassSem: ScSem)));
2645 Instruction *NewI = B.CreateIntrinsicWithoutFolding(
2646 ID: Intrinsic::spv_cmpxchg, OverloadTypes: {I.getPointerOperand()->getType()}, Args: {Args});
2647 replaceMemInstrUses(Old: &I, New: NewI, B);
2648 return NewI;
2649}
2650
2651Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
2652 auto Op = I.getOperation();
2653 if (Op != AtomicRMWInst::UIncWrap && Op != AtomicRMWInst::UDecWrap)
2654 return &I;
2655
2656 // No SPIR-V opcode exists for these, so lowering to an imported helper is an
2657 // AMD extension. Other targets keep the generic expansion.
2658 if (TM.getTargetTriple().getVendor() != Triple::AMD)
2659 return &I;
2660
2661 Module *M = I.getModule();
2662 IRBuilder<> B(I.getParent());
2663 B.SetInsertPoint(&I);
2664
2665 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2666 unsigned AS = I.getPointerOperand()->getType()->getPointerAddressSpace();
2667
2668 uint32_t Scope = static_cast<uint32_t>(
2669 getMemScope(TT: TM.getTargetTriple(), Ctx&: I.getContext(), Id: I.getSyncScopeID()));
2670 uint32_t ScSem = static_cast<uint32_t>(
2671 getMemSemanticsForStorageClass(SC: addressSpaceToStorageClass(AddrSpace: AS, STI: ST)));
2672 uint32_t MemSem = getMemSemanticsWithStorageClass(
2673 TT: TM.getTargetTriple(),
2674 OrderSem: static_cast<uint32_t>(getMemSemantics(Ord: I.getOrdering())), StorageClassSem: ScSem);
2675
2676 SmallString<64> FuncName(Op == AtomicRMWInst::UIncWrap
2677 ? "__translate_spirv_atomic_uinc_wrap"
2678 : "__translate_spirv_atomic_udec_wrap");
2679
2680 Type *ValTy = I.getValOperand()->getType();
2681 Type *PtrTy = I.getPointerOperand()->getType();
2682 // Encode the address space and value type in the name for overload.
2683 raw_svector_ostream OS(FuncName);
2684 OS << "_p" << AS << "_";
2685 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: ValTy))
2686 OS << "v" << VecTy->getNumElements();
2687 OS << "i" << ValTy->getScalarSizeInBits();
2688
2689 Type *Int32Ty = B.getInt32Ty();
2690 Type *BoolTy = B.getInt1Ty();
2691 SmallVector<Type *, 6> ArgTys = {PtrTy, Int32Ty, Int32Ty,
2692 ValTy, BoolTy, BoolTy};
2693 FunctionType *FT = FunctionType::get(Result: ValTy, Params: ArgTys, isVarArg: false);
2694 FunctionCallee FC = M->getOrInsertFunction(Name: FuncName, T: FT);
2695 cast<Function>(Val: FC.getCallee())->setCallingConv(CallingConv::SPIR_FUNC);
2696
2697 SmallVector<Value *, 6> Args = {
2698 I.getPointerOperand(), B.getInt32(C: Scope),
2699 B.getInt32(C: MemSem), I.getValOperand(),
2700 B.getInt1(V: I.isVolatile()), B.getInt1(V: I.isElementwise())};
2701 CallInst *CI = B.CreateCall(Callee: FC, Args);
2702 CI->setCallingConv(CallingConv::SPIR_FUNC);
2703 // SPIRVCallLowering reads alias.scope/noalias off the call to build the
2704 // aliasing decorations and runs after this pass, so preserve the metadata.
2705 CI->copyMetadata(SrcInst: I);
2706
2707 replaceAllUsesWithAndErase(B, Src: &I, Dest: CI);
2708 return CI;
2709}
2710
2711static bool isAbortCall(const Instruction &I, const SPIRVSubtarget &ST) {
2712 auto *CI = dyn_cast<CallInst>(Val: &I);
2713 if (!CI)
2714 return false;
2715 switch (CI->getIntrinsicID()) {
2716 case Intrinsic::spv_abort:
2717 return true;
2718 case Intrinsic::trap:
2719 case Intrinsic::ubsantrap:
2720 // When the extension is enabled, selection lowers these to OpAbortKHR.
2721 return ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_abort);
2722 default:
2723 return false;
2724 }
2725}
2726
2727// The OpAbortKHR instruction itself is a block terminator, so we don't need to
2728// emit an extra OpUnreachable instruction.
2729static bool precededByAbortIntrinsic(const UnreachableInst &I,
2730 const SPIRVSubtarget &ST) {
2731 // Find a previous non-debug instruction.
2732 const Instruction *Prev = I.getPrevNode();
2733 while (Prev && Prev->isDebugOrPseudoInst())
2734 Prev = Prev->getPrevNode();
2735
2736 if (Prev && isAbortCall(I: *Prev, ST))
2737 return true;
2738
2739 assert(llvm::none_of(
2740 *I.getParent(),
2741 [&ST](const Instruction &II) { return isAbortCall(II, ST); }) &&
2742 "abort-like call must be the last non-debug instruction before its "
2743 "block's terminator");
2744 return false;
2745}
2746
2747Instruction *SPIRVEmitIntrinsicsImpl::visitUnreachableInst(UnreachableInst &I) {
2748 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2749 if (precededByAbortIntrinsic(I, ST))
2750 return &I;
2751 IRBuilder<> B(&I);
2752 B.CreateIntrinsic(ID: Intrinsic::spv_unreachable, Args: {});
2753 return &I;
2754}
2755
2756// llvm.compiler.used and llvm.used hold use-list entries that protect their
2757// referenced globals from DCE without participating in code generation.
2758static bool isUseListGlobal(StringRef Name) {
2759 return Name == "llvm.compiler.used" || Name == "llvm.used";
2760}
2761
2762// Returns true for module-level globals that should not have SPIR-V intrinsics
2763// emitted (use-list globals plus llvm.global.annotations).
2764static bool isArtificialGlobal(StringRef Name) {
2765 return isUseListGlobal(Name) || Name == "llvm.global.annotations";
2766}
2767
2768// Returns true if every use of GV traces back to llvm.compiler.used or
2769// llvm.used.
2770static bool hasOnlyArtificialUses(const GlobalVariable &GV) {
2771 SmallPtrSet<const Value *, 8> Visited;
2772 SmallVector<const Value *> Stack(GV.users());
2773 while (!Stack.empty()) {
2774 const Value *V = Stack.pop_back_val();
2775 if (!Visited.insert(Ptr: V).second)
2776 continue;
2777 if (const auto *GVUser = dyn_cast<GlobalVariable>(Val: V)) {
2778 if (!isUseListGlobal(Name: GVUser->getName()))
2779 return false;
2780 continue;
2781 }
2782 if (const auto *C = dyn_cast<Constant>(Val: V)) {
2783 Stack.append(in_start: C->user_begin(), in_end: C->user_end());
2784 continue;
2785 }
2786 return false;
2787 }
2788 return true;
2789}
2790
2791static bool
2792shouldEmitIntrinsicsForGlobalValue(const GlobalVariableUsers &GVUsers,
2793 const GlobalVariable &GV,
2794 const Function *F) {
2795 // Skip special artificial variables.
2796 if (isArtificialGlobal(Name: GV.getName()))
2797 return false;
2798
2799 auto &UserFunctions = GVUsers.getTransitiveUserFunctions(GV);
2800 if (UserFunctions.contains(Ptr: F))
2801 return true;
2802
2803 // Do not emit the intrinsics in this function, it's going to be emitted on
2804 // the functions that reference it.
2805 if (!UserFunctions.empty())
2806 return false;
2807
2808 // Emit definitions for globals that are not referenced by any function on the
2809 // first function definition.
2810 const Module &M = *F->getParent();
2811 const Function &FirstDefinition = *M.getFunctionDefs().begin();
2812 return F == &FirstDefinition;
2813}
2814
2815Value *SPIRVEmitIntrinsicsImpl::buildSpvUndefComposite(Type *AggrTy,
2816 IRBuilder<> &B) {
2817 auto MakeLeaf = [&](Type *ElemTy) -> Instruction * {
2818 CallInst *Leaf = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_undef, Args: {});
2819 AggrConsts[Leaf] = PoisonValue::get(T: ElemTy);
2820 AggrConstTypes[Leaf] = ElemTy;
2821 return Leaf;
2822 };
2823 SmallVector<Value *, 4> Elems;
2824 if (auto *ArrTy = dyn_cast<ArrayType>(Val: AggrTy)) {
2825 Elems.assign(NumElts: ArrTy->getNumElements(), Elt: MakeLeaf(ArrTy->getElementType()));
2826 } else {
2827 auto *StructTy = cast<StructType>(Val: AggrTy);
2828 DenseMap<Type *, Instruction *> LeafByType;
2829 for (unsigned I = 0; I < StructTy->getNumElements(); ++I) {
2830 Type *ElemTy = StructTy->getContainedType(i: I);
2831 auto &Entry = LeafByType[ElemTy];
2832 if (!Entry)
2833 Entry = MakeLeaf(ElemTy);
2834 Elems.push_back(Elt: Entry);
2835 }
2836 }
2837 CallInst *Composite = B.CreateIntrinsicWithoutFolding(
2838 ID: Intrinsic::spv_const_composite, OverloadTypes: {B.getInt32Ty()}, Args: Elems);
2839 AggrConsts[Composite] = PoisonValue::get(T: AggrTy);
2840 AggrConstTypes[Composite] = AggrTy;
2841 return Composite;
2842}
2843
2844// If a function directly returns an aggregate-typed call result,
2845// the ReturnInst carries an aggregate while the function signature
2846// was rewritten to i32 by SPIRVPrepareFunctions. Rebuild the return value
2847// via extractvalue/insertvalue so the regular spv_extractv/spv_insertv
2848// lowering produces a valid OpReturnValue.
2849void SPIRVEmitIntrinsicsImpl::reconstructAggregateReturns(Function &Func,
2850 IRBuilder<> &B) {
2851 Type *OrigRetTy = GR->findMutated(Val: &Func);
2852 if (!OrigRetTy || !OrigRetTy->isAggregateType())
2853 return;
2854 for (BasicBlock &BB : Func) {
2855 auto *RI = dyn_cast<ReturnInst>(Val: BB.getTerminator());
2856 if (!RI)
2857 continue;
2858 Value *RetVal = RI->getReturnValue();
2859 if (!RetVal || RetVal->getType() != OrigRetTy || !isa<CallBase>(Val: RetVal))
2860 continue;
2861 Type *AggrTy = RetVal->getType();
2862 uint64_t NumElts = isa<StructType>(Val: AggrTy)
2863 ? cast<StructType>(Val: AggrTy)->getNumElements()
2864 : cast<ArrayType>(Val: AggrTy)->getNumElements();
2865 B.SetInsertPoint(RI);
2866 Value *Rebuilt = PoisonValue::get(T: AggrTy);
2867 for (uint64_t I = 0; I < NumElts; ++I) {
2868 Value *Elt = B.CreateExtractValue(Agg: RetVal, Idxs: I);
2869 Rebuilt = B.CreateInsertValue(Agg: Rebuilt, Val: Elt, Idxs: I);
2870 }
2871 RI->setOperand(i_nocapture: 0, Val_nocapture: Rebuilt);
2872 }
2873}
2874
2875void SPIRVEmitIntrinsicsImpl::processGlobalValue(GlobalVariable &GV,
2876 IRBuilder<> &B) {
2877
2878 if (!shouldEmitIntrinsicsForGlobalValue(GVUsers, GV, F: CurrF))
2879 return;
2880
2881 // Record the pointee type for every global, not only initialized ones, so an
2882 // undef non-constant aggregate global is not later collapsed to its element
2883 // type. Result is ignored, because TypedPointerType is not supported
2884 // by llvm IR general logic.
2885 deduceElementTypeHelper(I: &GV, UnknownElemTypeI8: false);
2886
2887 Constant *Init = nullptr;
2888 if (hasInitializer(GV: &GV)) {
2889 Init = GV.getInitializer();
2890 Value *InitOp = Init;
2891 if (isa<UndefValue>(Val: Init) && Init->getType()->isAggregateType()) {
2892 const SPIRVSubtarget *STI = TM.getSubtargetImpl();
2893 bool UsePoison =
2894 isa<PoisonValue>(Val: Init) &&
2895 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
2896 if (UsePoison) {
2897 CallInst *Call = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_poison,
2898 OverloadTypes: {B.getInt32Ty()}, Args: {});
2899 AggrConsts[Call] = cast<PoisonValue>(Val: Init);
2900 AggrConstTypes[Call] = Init->getType();
2901 InitOp = Call;
2902 } else {
2903 InitOp = buildSpvUndefComposite(AggrTy: Init->getType(), B);
2904 }
2905 }
2906 Type *Ty = isAggrConstForceInt32(V: Init) ? B.getInt32Ty() : Init->getType();
2907 Constant *Const = isAggrConstForceInt32(V: Init) ? B.getInt32(C: 1) : Init;
2908 CallInst *InitInst = B.CreateIntrinsicWithoutFolding(
2909 ID: Intrinsic::spv_init_global, OverloadTypes: {GV.getType(), Ty}, Args: {&GV, Const});
2910 InitInst->setArgOperand(i: 1, v: InitOp);
2911 }
2912 // Globals with only use-list references have no real function uses. Emit
2913 // spv_unref_global so buildGlobalVariable is called for them.
2914 if (!Init && hasOnlyArtificialUses(GV))
2915 B.CreateIntrinsic(ID: Intrinsic::spv_unref_global, OverloadTypes: GV.getType(), Args: &GV);
2916}
2917
2918// Return true, if we can't decide what is the pointee type now and will get
2919// back to the question later. Return false is spv_assign_ptr_type is not needed
2920// or can be inserted immediately.
2921bool SPIRVEmitIntrinsicsImpl::insertAssignPtrTypeIntrs(Instruction *I,
2922 IRBuilder<> &B,
2923 bool UnknownElemTypeI8) {
2924 reportFatalOnTokenType(I);
2925 if (!isPointerTy(T: I->getType()) || !requireAssignType(I))
2926 return false;
2927
2928 setInsertPointAfterDef(B, I);
2929 if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) {
2930 GR->buildAssignPtr(B, ElemTy, Arg: I);
2931 return false;
2932 }
2933 return true;
2934}
2935
2936void SPIRVEmitIntrinsicsImpl::insertAssignTypeIntrs(Instruction *I,
2937 IRBuilder<> &B) {
2938 // TODO: extend the list of functions with known result types
2939 static StringMap<unsigned> ResTypeWellKnown = {
2940 {"async_work_group_copy", WellKnownTypes::Event},
2941 {"async_work_group_strided_copy", WellKnownTypes::Event},
2942 {"__spirv_GroupAsyncCopy", WellKnownTypes::Event}};
2943
2944 reportFatalOnTokenType(I);
2945
2946 bool IsKnown = false;
2947 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
2948 if (!CI->isIndirectCall() && !CI->isInlineAsm() &&
2949 CI->getCalledFunction() && !CI->getCalledFunction()->isIntrinsic()) {
2950 Function *CalledF = CI->getCalledFunction();
2951 std::string DemangledName =
2952 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
2953 FPDecorationId DecorationId = FPDecorationId::NONE;
2954 if (DemangledName.length() > 0)
2955 DemangledName =
2956 SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName, DecorationId: &DecorationId);
2957 auto ResIt = ResTypeWellKnown.find(Key: DemangledName);
2958 if (ResIt != ResTypeWellKnown.end()) {
2959 IsKnown = true;
2960 setInsertPointAfterDef(B, I);
2961 switch (ResIt->second) {
2962 case WellKnownTypes::Event:
2963 GR->buildAssignType(
2964 B, Ty: TargetExtType::get(Context&: I->getContext(), Name: "spirv.Event"), Arg: I,
2965 CanUseAnyVectorRank);
2966 break;
2967 }
2968 }
2969 // check if a floating rounding mode or saturation info is present
2970 switch (DecorationId) {
2971 default:
2972 break;
2973 case FPDecorationId::SAT:
2974 createSaturatedConversionDecoration(I: CI, B);
2975 break;
2976 case FPDecorationId::RTE:
2977 createRoundingModeDecoration(
2978 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTE, B);
2979 break;
2980 case FPDecorationId::RTZ:
2981 createRoundingModeDecoration(
2982 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTZ, B);
2983 break;
2984 case FPDecorationId::RTP:
2985 createRoundingModeDecoration(
2986 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTP, B);
2987 break;
2988 case FPDecorationId::RTN:
2989 createRoundingModeDecoration(
2990 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTN, B);
2991 break;
2992 }
2993 }
2994 }
2995
2996 Type *Ty = I->getType();
2997 if (!IsKnown && !Ty->isVoidTy() && !isPointerTy(T: Ty) && requireAssignType(I)) {
2998 setInsertPointAfterDef(B, I);
2999 Type *TypeToAssign = Ty;
3000 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
3001 if (isSpvAggrPlaceholder(V: II)) {
3002 auto It = AggrConstTypes.find(Val: II);
3003 if (It == AggrConstTypes.end())
3004 report_fatal_error(reason: "Unknown composite intrinsic type");
3005 TypeToAssign = It->second;
3006 } else if (II->getIntrinsicID() == Intrinsic::spv_poison) {
3007 if (auto It = AggrConstTypes.find(Val: II); It != AggrConstTypes.end())
3008 TypeToAssign = It->second;
3009 }
3010 } else if (auto It = AggrConstTypes.find(Val: I); It != AggrConstTypes.end())
3011 TypeToAssign = It->second;
3012 TypeToAssign = restoreMutatedType(GR, I, Ty: TypeToAssign);
3013 GR->buildAssignType(B, Ty: TypeToAssign, Arg: I, CanUseAnyVectorRank);
3014 }
3015 for (const auto &Op : I->operands()) {
3016 if (isa<ConstantPointerNull>(Val: Op) || isa<UndefValue>(Val: Op) ||
3017 isVector1(Ty: Op->getType()) || // <1 x T> gets clobbered ty IRTranslator.
3018 // Check GetElementPtrConstantExpr case.
3019 (isa<ConstantExpr>(Val: Op) &&
3020 (isa<GEPOperator>(Val: Op) ||
3021 (cast<ConstantExpr>(Val: Op)->getOpcode() == CastInst::IntToPtr)))) {
3022 setInsertPointSkippingPhis(B, I);
3023 Type *OpTy = Op->getType();
3024 if (isa<UndefValue>(Val: Op) && OpTy->isAggregateType()) {
3025 CallInst *AssignCI =
3026 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {B.getInt32Ty()}, Arg: Op,
3027 Arg2: UndefValue::get(T: B.getInt32Ty()), Imms: {}, B);
3028 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: AssignCI);
3029 } else if (!isa<Instruction>(Val: Op)) {
3030 Type *OpTy = Op->getType();
3031 Type *OpTyElem = getPointeeType(Ty: OpTy);
3032 if (OpTyElem) {
3033 GR->buildAssignPtr(B, ElemTy: OpTyElem, Arg: Op);
3034 } else if (isPointerTy(T: OpTy)) {
3035 Type *ElemTy = GR->findDeducedElementType(Val: Op);
3036 GR->buildAssignPtr(B, ElemTy: ElemTy ? ElemTy : deduceElementType(I: Op, UnknownElemTypeI8: true),
3037 Arg: Op);
3038 } else {
3039 Value *OpTyVal = Op;
3040 if (OpTy->isTargetExtTy()) {
3041 // We need to do this in order to be consistent with how target ext
3042 // types are handled in `processInstrAfterVisit`
3043 OpTyVal = getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank);
3044 }
3045 CallInst *AssignCI = buildIntrWithMD(
3046 IntrID: Intrinsic::spv_assign_type, Types: {OpTy},
3047 Arg: getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank), Arg2: OpTyVal, Imms: {},
3048 B);
3049 GR->addAssignPtrTypeInstr(Val: OpTyVal, AssignPtrTyCI: AssignCI);
3050 }
3051 }
3052 }
3053 }
3054}
3055
3056bool SPIRVEmitIntrinsicsImpl::shouldTryToAddMemAliasingDecoration(
3057 Instruction *Inst) {
3058 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*Inst->getFunction());
3059 if (!STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing))
3060 return false;
3061 // Add aliasing decorations to internal load and store intrinsics.
3062 // Do not attach them to store atomic or load atomic intrinsics / instructions
3063 // since the extension is inconsistent at the moment (we cannot add the
3064 // decoration to atomic stores because they do not have an id).
3065 return match(V: Inst,
3066 P: m_AnyIntrinsic<Intrinsic::spv_load, Intrinsic::spv_store>());
3067}
3068
3069void SPIRVEmitIntrinsicsImpl::insertSpirvDecorations(Instruction *I,
3070 IRBuilder<> &B) {
3071 if (MDNode *MD = I->getMetadata(Kind: "spirv.Decorations")) {
3072 setInsertPointAfterDef(B, I);
3073 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
3074 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
3075 }
3076 // Lower alias.scope/noalias metadata
3077 {
3078 auto processMemAliasingDecoration = [&](unsigned Kind) {
3079 if (MDNode *AliasListMD = I->getMetadata(KindID: Kind)) {
3080 if (shouldTryToAddMemAliasingDecoration(Inst: I)) {
3081 uint32_t Dec = Kind == LLVMContext::MD_alias_scope
3082 ? SPIRV::Decoration::AliasScopeINTEL
3083 : SPIRV::Decoration::NoAliasINTEL;
3084 SmallVector<Value *, 3> Args = {
3085 I, ConstantInt::get(Ty: B.getInt32Ty(), V: Dec),
3086 MetadataAsValue::get(Context&: I->getContext(), MD: AliasListMD)};
3087 setInsertPointAfterDef(B, I);
3088 B.CreateIntrinsic(ID: Intrinsic::spv_assign_aliasing_decoration,
3089 OverloadTypes: {I->getType()}, Args: {Args});
3090 }
3091 }
3092 };
3093 processMemAliasingDecoration(LLVMContext::MD_alias_scope);
3094 processMemAliasingDecoration(LLVMContext::MD_noalias);
3095 }
3096 // MD_fpmath
3097 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_fpmath)) {
3098 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I->getFunction());
3099 bool AllowFPMaxError =
3100 STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_fp_max_error);
3101 if (!AllowFPMaxError)
3102 return;
3103
3104 setInsertPointAfterDef(B, I);
3105 B.CreateIntrinsic(ID: Intrinsic::spv_assign_fpmaxerror_decoration,
3106 OverloadTypes: {I->getType()},
3107 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
3108 }
3109 if (I->getModule()->getTargetTriple().getVendor() == Triple::AMD &&
3110 isa<AtomicRMWInst>(Val: I)) {
3111 // If present, we encode AMDGPU atomic metadata as UserSemantic string
3112 // decorations, which will be parsed during reverse translation.
3113 auto &Ctx = B.getContext();
3114 auto *US = ConstantAsMetadata::get(
3115 C: ConstantInt::get(Ty: B.getInt32Ty(), V: SPIRV::Decoration::UserSemantic));
3116
3117 SmallVector<Metadata *> MDs;
3118 if (I->hasMetadata(Kind: "amdgpu.no.fine.grained.memory"))
3119 MDs.push_back(Elt: MDNode::get(
3120 Context&: Ctx, MDs: {US, MDString::get(Context&: Ctx, Str: "amdgpu.no.fine.grained.memory")}));
3121 if (I->hasMetadata(Kind: "amdgpu.no.remote.memory"))
3122 MDs.push_back(Elt: MDNode::get(
3123 Context&: Ctx, MDs: {US, MDString::get(Context&: Ctx, Str: "amdgpu.no.remote.memory")}));
3124 if (I->hasMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode))
3125 MDs.push_back(Elt: MDNode::get(
3126 Context&: Ctx, MDs: {US, MDString::get(Context&: Ctx, Str: "atomic.ignore.denormal.mode")}));
3127 if (!MDs.empty())
3128 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
3129 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs))});
3130 }
3131}
3132
3133static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
3134 const Module &M,
3135 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
3136 &FPFastMathDefaultInfoMap,
3137 Function *F) {
3138 auto it = FPFastMathDefaultInfoMap.find(Val: F);
3139 if (it != FPFastMathDefaultInfoMap.end())
3140 return it->second;
3141
3142 // If the map does not contain the entry, create a new one. Initialize it to
3143 // contain all 3 elements sorted by bit width of target type: {half, float,
3144 // double}.
3145 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
3146 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
3147 Args: SPIRV::FPFastMathMode::None);
3148 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
3149 Args: SPIRV::FPFastMathMode::None);
3150 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
3151 Args: SPIRV::FPFastMathMode::None);
3152 return FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
3153}
3154
3155static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
3156 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
3157 const Type *Ty) {
3158 size_t BitWidth = Ty->getScalarSizeInBits();
3159 int Index =
3160 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
3161 BitWidth);
3162 assert(Index >= 0 && Index < 3 &&
3163 "Expected FPFastMathDefaultInfo for half, float, or double");
3164 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3165 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3166 return FPFastMathDefaultInfoVec[Index];
3167}
3168
3169void SPIRVEmitIntrinsicsImpl::insertConstantsForFPFastMathDefault(Module &M) {
3170 const SPIRVSubtarget *ST = TM.getSubtargetImpl();
3171 if (!ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
3172 return;
3173
3174 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
3175 // We need the entry point (function) as the key, and the target
3176 // type and flags as the value.
3177 // We also need to check ContractionOff and SignedZeroInfNanPreserve
3178 // execution modes, as they are now deprecated and must be replaced
3179 // with FPFastMathDefaultInfo.
3180 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
3181 if (!Node) {
3182 if (!M.getNamedMetadata(Name: "opencl.enable.FP_CONTRACT")) {
3183 // This requires emitting ContractionOff. However, because
3184 // ContractionOff is now deprecated, we need to replace it with
3185 // FPFastMathDefaultInfo with FP Fast Math Mode bitmask set to all 0.
3186 // We need to create the constant for that.
3187
3188 // Create constant instruction with the bitmask flags.
3189 Constant *InitValue =
3190 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: 0);
3191 // TODO: Reuse constant if there is one already with the required
3192 // value.
3193 [[maybe_unused]] GlobalVariable *GV =
3194 new GlobalVariable(M, // Module
3195 Type::getInt32Ty(C&: M.getContext()), // Type
3196 true, // isConstant
3197 GlobalValue::InternalLinkage, // Linkage
3198 InitValue // Initializer
3199 );
3200 }
3201 return;
3202 }
3203
3204 // The table maps function pointers to their default FP fast math info. It
3205 // can be assumed that the SmallVector is sorted by the bit width of the
3206 // type. The first element is the smallest bit width, and the last element
3207 // is the largest bit width, therefore, we will have {half, float, double}
3208 // in the order of their bit widths.
3209 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
3210 FPFastMathDefaultInfoMap;
3211
3212 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
3213 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
3214 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
3215 Function *F = cast<Function>(
3216 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
3217 const auto EM =
3218 cast<ConstantInt>(
3219 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
3220 ->getZExtValue();
3221 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
3222 assert(MDN->getNumOperands() == 4 &&
3223 "Expected 4 operands for FPFastMathDefault");
3224 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
3225 unsigned Flags =
3226 cast<ConstantInt>(
3227 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
3228 ->getZExtValue();
3229 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3230 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3231 SPIRV::FPFastMathDefaultInfo &Info =
3232 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
3233 Info.FastMathFlags = Flags;
3234 Info.FPFastMathDefault = true;
3235 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
3236 assert(MDN->getNumOperands() == 2 &&
3237 "Expected no operands for ContractionOff");
3238
3239 // We need to save this info for every possible FP type, i.e. {half,
3240 // float, double, fp128}.
3241 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3242 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3243 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3244 Info.ContractionOff = true;
3245 }
3246 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
3247 assert(MDN->getNumOperands() == 3 &&
3248 "Expected 1 operand for SignedZeroInfNanPreserve");
3249 unsigned TargetWidth =
3250 cast<ConstantInt>(
3251 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
3252 ->getZExtValue();
3253 // We need to save this info only for the FP type with TargetWidth.
3254 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3255 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3256 int Index = SPIRV::FPFastMathDefaultInfoVector::
3257 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
3258 assert(Index >= 0 && Index < 3 &&
3259 "Expected FPFastMathDefaultInfo for half, float, or double");
3260 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3261 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3262 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
3263 }
3264 }
3265
3266 DenseMap<unsigned, GlobalVariable *> GlobalVars;
3267 for (auto &[Func, FPFastMathDefaultInfoVec] : FPFastMathDefaultInfoMap) {
3268 if (FPFastMathDefaultInfoVec.empty())
3269 continue;
3270
3271 for (const SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3272 assert(Info.Ty && "Expected target type for FPFastMathDefaultInfo");
3273 // Skip if none of the execution modes was used.
3274 unsigned Flags = Info.FastMathFlags;
3275 if (Flags == SPIRV::FPFastMathMode::None && !Info.ContractionOff &&
3276 !Info.SignedZeroInfNanPreserve && !Info.FPFastMathDefault)
3277 continue;
3278
3279 // Check if flags are compatible.
3280 if (Info.ContractionOff && (Flags & SPIRV::FPFastMathMode::AllowContract))
3281 report_fatal_error(reason: "Conflicting FPFastMathFlags: ContractionOff "
3282 "and AllowContract");
3283
3284 if (Info.SignedZeroInfNanPreserve &&
3285 !(Flags &
3286 (SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
3287 SPIRV::FPFastMathMode::NSZ))) {
3288 if (Info.FPFastMathDefault)
3289 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
3290 "SignedZeroInfNanPreserve but at least one of "
3291 "NotNaN/NotInf/NSZ is enabled.");
3292 }
3293
3294 if ((Flags & SPIRV::FPFastMathMode::AllowTransform) &&
3295 !((Flags & SPIRV::FPFastMathMode::AllowReassoc) &&
3296 (Flags & SPIRV::FPFastMathMode::AllowContract))) {
3297 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
3298 "AllowTransform requires AllowReassoc and "
3299 "AllowContract to be set.");
3300 }
3301
3302 auto it = GlobalVars.find(Val: Flags);
3303 GlobalVariable *GV = nullptr;
3304 if (it != GlobalVars.end()) {
3305 // Reuse existing global variable.
3306 GV = it->second;
3307 } else {
3308 // Create constant instruction with the bitmask flags.
3309 Constant *InitValue =
3310 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: Flags);
3311 // TODO: Reuse constant if there is one already with the required
3312 // value.
3313 GV = new GlobalVariable(M, // Module
3314 Type::getInt32Ty(C&: M.getContext()), // Type
3315 true, // isConstant
3316 GlobalValue::InternalLinkage, // Linkage
3317 InitValue // Initializer
3318 );
3319 GlobalVars[Flags] = GV;
3320 }
3321 }
3322 }
3323}
3324
3325void SPIRVEmitIntrinsicsImpl::processInstrAfterVisit(Instruction *I,
3326 IRBuilder<> &B) {
3327 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3328 bool IsConstComposite =
3329 II && II->getIntrinsicID() == Intrinsic::spv_const_composite;
3330 if (IsConstComposite && TrackConstants) {
3331 setInsertPointAfterDef(B, I);
3332 auto t = AggrConsts.find(Val: I);
3333 assert(t != AggrConsts.end());
3334 auto *NewOp =
3335 buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
3336 Types: {II->getType(), II->getType()}, Arg: t->second, Arg2: I, Imms: {}, B);
3337 replaceAllUsesWith(Src: I, Dest: NewOp, DeleteOld: false);
3338 NewOp->setArgOperand(i: 0, v: I);
3339 }
3340 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
3341 for (const auto &Op : I->operands()) {
3342 if (isa<PHINode>(Val: I) || isa<SwitchInst>(Val: I) ||
3343 !(isa<ConstantData>(Val: Op) || isa<ConstantExpr>(Val: Op)))
3344 continue;
3345 unsigned OpNo = Op.getOperandNo();
3346 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
3347 (!II->isBundleOperand(Idx: OpNo) &&
3348 II->paramHasAttr(ArgNo: OpNo, Kind: Attribute::ImmArg))))
3349 continue;
3350
3351 if (!BPrepared) {
3352 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
3353 : B.SetInsertPoint(I);
3354 BPrepared = true;
3355 }
3356 Type *OpTy = Op->getType();
3357 Type *OpElemTy = GR->findDeducedElementType(Val: Op);
3358 Value *NewOp = Op;
3359 if (OpTy->isTargetExtTy()) {
3360 // Since this value is replaced by poison, we need to do the same in
3361 // `insertAssignTypeIntrs`.
3362 Value *OpTyVal = getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank);
3363 NewOp = buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
3364 Types: {OpTy, OpTyVal->getType()}, Arg: Op, Arg2: OpTyVal, Imms: {}, B);
3365 }
3366 if (!IsConstComposite && isPointerTy(T: OpTy) && OpElemTy != nullptr &&
3367 OpElemTy != IntegerType::getInt8Ty(C&: I->getContext())) {
3368 SmallVector<Type *, 2> Types = {OpTy, OpTy};
3369 SmallVector<Value *, 2> Args = {
3370 NewOp,
3371 buildMD(Arg: getNormalizedPoisonValue(Ty: OpElemTy, CanUseAnyVectorRank)),
3372 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
3373 CallInst *PtrCasted = B.CreateIntrinsicWithoutFolding(
3374 ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
3375 GR->buildAssignPtr(B, ElemTy: OpElemTy, Arg: PtrCasted);
3376 NewOp = PtrCasted;
3377 }
3378 if (NewOp != Op)
3379 I->setOperand(i: OpNo, Val: NewOp);
3380 }
3381 if (Named.insert(Ptr: I).second)
3382 emitAssignName(I, B);
3383}
3384
3385Type *SPIRVEmitIntrinsicsImpl::deduceFunParamElementType(Function *F,
3386 unsigned OpIdx) {
3387 SmallPtrSet<Function *, 0> FVisited;
3388 return deduceFunParamElementType(F, OpIdx, FVisited);
3389}
3390
3391Type *SPIRVEmitIntrinsicsImpl::deduceFunParamElementType(
3392 Function *F, unsigned OpIdx, SmallPtrSetImpl<Function *> &FVisited) {
3393 // maybe a cycle
3394 if (!FVisited.insert(Ptr: F).second)
3395 return nullptr;
3396
3397 SmallPtrSet<Value *, 0> Visited;
3398 SmallVector<std::pair<Function *, unsigned>> Lookup;
3399 // search in function's call sites
3400 for (User *U : F->users()) {
3401 CallInst *CI = dyn_cast<CallInst>(Val: U);
3402 if (!CI || OpIdx >= CI->arg_size())
3403 continue;
3404 Value *OpArg = CI->getArgOperand(i: OpIdx);
3405 if (!isPointerTy(T: OpArg->getType()))
3406 continue;
3407 // maybe we already know operand's element type
3408 if (Type *KnownTy = GR->findDeducedElementType(Val: OpArg))
3409 return KnownTy;
3410 // try to deduce from the operand itself
3411 Visited.clear();
3412 if (Type *Ty = deduceElementTypeHelper(I: OpArg, Visited, UnknownElemTypeI8: false))
3413 return Ty;
3414 // search in actual parameter's users
3415 for (User *OpU : OpArg->users()) {
3416 Instruction *Inst = dyn_cast<Instruction>(Val: OpU);
3417 if (!Inst || Inst == CI)
3418 continue;
3419 Visited.clear();
3420 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8: false))
3421 return Ty;
3422 }
3423 // check if it's a formal parameter of the outer function
3424 if (!CI->getParent() || !CI->getParent()->getParent())
3425 continue;
3426 Function *OuterF = CI->getParent()->getParent();
3427 if (FVisited.find(Ptr: OuterF) != FVisited.end())
3428 continue;
3429 for (unsigned i = 0; i < OuterF->arg_size(); ++i) {
3430 if (OuterF->getArg(i) == OpArg) {
3431 Lookup.push_back(Elt: std::make_pair(x&: OuterF, y&: i));
3432 break;
3433 }
3434 }
3435 }
3436
3437 // search in function parameters
3438 for (auto &Pair : Lookup) {
3439 if (Type *Ty = deduceFunParamElementType(F: Pair.first, OpIdx: Pair.second, FVisited))
3440 return Ty;
3441 }
3442
3443 return nullptr;
3444}
3445
3446void SPIRVEmitIntrinsicsImpl::processParamTypesByFunHeader(Function *F,
3447 IRBuilder<> &B) {
3448 B.SetInsertPointPastAllocas(F);
3449 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
3450 Argument *Arg = F->getArg(i: OpIdx);
3451 // Vector-of-pointers arg: deduce pointee from a GEP user so the function
3452 // type isn't emitted with the default i8 pointee.
3453 if (isUntypedPointerVectorTy(T: Arg->getType()) &&
3454 !GR->findDeducedElementType(Val: Arg)) {
3455 for (User *U : Arg->users()) {
3456 auto *GEP = dyn_cast<GetElementPtrInst>(Val: U);
3457 if (GEP && GEP->getPointerOperand() == Arg) {
3458 GR->buildAssignPtr(B, ElemTy: GEP->getSourceElementType(), Arg);
3459 break;
3460 }
3461 }
3462 continue;
3463 }
3464 if (!isUntypedPointerTy(T: Arg->getType()))
3465 continue;
3466 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
3467 if (ElemTy)
3468 continue;
3469 if (hasPointeeTypeAttr(Arg) &&
3470 (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) {
3471 GR->buildAssignPtr(B, ElemTy, Arg);
3472 continue;
3473 }
3474 // search in function's call sites
3475 for (User *U : F->users()) {
3476 CallInst *CI = dyn_cast<CallInst>(Val: U);
3477 if (!CI || OpIdx >= CI->arg_size())
3478 continue;
3479 Value *OpArg = CI->getArgOperand(i: OpIdx);
3480 if (!isPointerTy(T: OpArg->getType()))
3481 continue;
3482 // maybe we already know operand's element type
3483 if ((ElemTy = GR->findDeducedElementType(Val: OpArg)) != nullptr)
3484 break;
3485 }
3486 if (ElemTy) {
3487 GR->buildAssignPtr(B, ElemTy, Arg);
3488 continue;
3489 }
3490 if (HaveFunPtrs) {
3491 for (User *U : Arg->users()) {
3492 CallInst *CI = dyn_cast<CallInst>(Val: U);
3493 if (CI && !isa<IntrinsicInst>(Val: CI) && CI->isIndirectCall() &&
3494 CI->getCalledOperand() == Arg &&
3495 CI->getParent()->getParent() == CurrF) {
3496 SmallVector<std::pair<Value *, unsigned>> Ops;
3497 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy&: ElemTy, IsPostprocessing: false);
3498 if (ElemTy) {
3499 GR->buildAssignPtr(B, ElemTy, Arg);
3500 break;
3501 }
3502 }
3503 }
3504 }
3505 }
3506}
3507
3508void SPIRVEmitIntrinsicsImpl::processParamTypes(Function *F, IRBuilder<> &B) {
3509 B.SetInsertPointPastAllocas(F);
3510 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
3511 Argument *Arg = F->getArg(i: OpIdx);
3512 if (!isUntypedPointerTy(T: Arg->getType()))
3513 continue;
3514 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
3515 if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) {
3516 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Arg)) {
3517 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3518 GR->updateAssignType(
3519 AssignCI, Arg,
3520 OfType: getNormalizedPoisonValue(Ty: ElemTy, CanUseAnyVectorRank));
3521 propagateElemType(Op: Arg, ElemTy: IntegerType::getInt8Ty(C&: F->getContext()),
3522 VisitedSubst);
3523 } else {
3524 GR->buildAssignPtr(B, ElemTy, Arg);
3525 }
3526 }
3527 }
3528}
3529
3530static FunctionType *getFunctionPointerElemType(Function *F,
3531 SPIRVGlobalRegistry *GR) {
3532 FunctionType *FTy = F->getFunctionType();
3533 bool IsNewFTy = false;
3534 SmallVector<Type *, 4> ArgTys;
3535 for (Argument &Arg : F->args()) {
3536 Type *ArgTy = Arg.getType();
3537 if (ArgTy->isPointerTy())
3538 if (Type *ElemTy = GR->findDeducedElementType(Val: &Arg)) {
3539 IsNewFTy = true;
3540 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
3541 }
3542 ArgTys.push_back(Elt: ArgTy);
3543 }
3544 return IsNewFTy
3545 ? FunctionType::get(Result: FTy->getReturnType(), Params: ArgTys, isVarArg: FTy->isVarArg())
3546 : FTy;
3547}
3548
3549bool SPIRVEmitIntrinsicsImpl::processFunctionPointers(Module &M) {
3550 SmallVector<Function *> Worklist;
3551 for (auto &F : M) {
3552 if (F.isIntrinsic())
3553 continue;
3554 if (F.isDeclaration()) {
3555 for (User *U : F.users()) {
3556 CallInst *CI = dyn_cast<CallInst>(Val: U);
3557 if (!CI || CI->getCalledFunction() != &F) {
3558 Worklist.push_back(Elt: &F);
3559 break;
3560 }
3561 }
3562 } else {
3563 if (F.user_empty())
3564 continue;
3565 Type *FPElemTy = GR->findDeducedElementType(Val: &F);
3566 if (!FPElemTy)
3567 FPElemTy = getFunctionPointerElemType(F: &F, GR);
3568 for (User *U : F.users()) {
3569 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
3570 if (!II || II->arg_size() != 3 || II->getOperand(i_nocapture: 0) != &F)
3571 continue;
3572 if (II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type ||
3573 II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
3574 GR->updateAssignType(
3575 AssignCI: II, Arg: &F, OfType: getNormalizedPoisonValue(Ty: FPElemTy, CanUseAnyVectorRank));
3576 break;
3577 }
3578 }
3579 }
3580 }
3581 if (Worklist.empty())
3582 return false;
3583
3584 LLVMContext &Ctx = M.getContext();
3585 Function *SF = getOrCreateBackendServiceFunction(M);
3586 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: SF);
3587 IRBuilder<> IRB(BB);
3588
3589 for (Function *F : Worklist) {
3590 SmallVector<Value *> Args;
3591 for (const auto &Arg : F->args())
3592 Args.push_back(
3593 Elt: getNormalizedPoisonValue(Ty: Arg.getType(), CanUseAnyVectorRank));
3594 IRB.CreateCall(Callee: F, Args);
3595 }
3596 IRB.CreateRetVoid();
3597
3598 return true;
3599}
3600
3601// Apply types parsed from demangled function declarations.
3602void SPIRVEmitIntrinsicsImpl::applyDemangledPtrArgTypes(IRBuilder<> &B) {
3603 DenseMap<Function *, CallInst *> Ptrcasts;
3604 for (auto It : FDeclPtrTys) {
3605 Function *F = It.first;
3606 for (auto *U : F->users()) {
3607 CallInst *CI = dyn_cast<CallInst>(Val: U);
3608 if (!CI || CI->getCalledFunction() != F)
3609 continue;
3610 unsigned Sz = CI->arg_size();
3611 for (auto [Idx, ElemTy] : It.second) {
3612 if (Idx >= Sz)
3613 continue;
3614 Value *Param = CI->getArgOperand(i: Idx);
3615 if (GR->findDeducedElementType(Val: Param) || isa<GlobalValue>(Val: Param))
3616 continue;
3617 if (Argument *Arg = dyn_cast<Argument>(Val: Param)) {
3618 if (!hasPointeeTypeAttr(Arg)) {
3619 B.SetInsertPointPastAllocas(Arg->getParent());
3620 B.SetCurrentDebugLocation(DebugLoc());
3621 GR->buildAssignPtr(B, ElemTy, Arg);
3622 }
3623 } else if (isaGEP(V: Param)) {
3624 replaceUsesOfWithSpvPtrcast(
3625 Op: Param, ElemTy: normalizeType(Ty: ElemTy, CanUseAnyVectorRank), I: CI, Ptrcasts);
3626 } else if (isa<Instruction>(Val: Param)) {
3627 GR->addDeducedElementType(Val: Param,
3628 Ty: normalizeType(Ty: ElemTy, CanUseAnyVectorRank));
3629 // insertAssignTypeIntrs() will complete buildAssignPtr()
3630 } else {
3631 B.SetInsertPoint(CI->getParent()
3632 ->getParent()
3633 ->getEntryBlock()
3634 .getFirstNonPHIOrDbgOrAlloca());
3635 GR->buildAssignPtr(B, ElemTy, Arg: Param);
3636 }
3637 CallInst *Ref = dyn_cast<CallInst>(Val: Param);
3638 if (!Ref)
3639 continue;
3640 Function *RefF = Ref->getCalledFunction();
3641 if (!RefF || !isPointerTy(T: RefF->getReturnType()) ||
3642 GR->findDeducedElementType(Val: RefF))
3643 continue;
3644 ElemTy = normalizeType(Ty: ElemTy, CanUseAnyVectorRank);
3645 GR->addDeducedElementType(Val: RefF, Ty: ElemTy);
3646 GR->addReturnType(
3647 ArgF: RefF, DerivedTy: TypedPointerType::get(
3648 ElementType: ElemTy, AddressSpace: getPointerAddressSpace(T: RefF->getReturnType())));
3649 }
3650 }
3651 }
3652}
3653
3654GetElementPtrInst *SPIRVEmitIntrinsicsImpl::simplifyZeroLengthArrayGepInst(
3655 GetElementPtrInst *GEP) {
3656 // getelementptr [0 x T], P, 0 (zero), I -> getelementptr T, P, I.
3657 // If type is 0-length array and first index is 0 (zero), drop both the
3658 // 0-length array type and the first index. This is a common pattern in
3659 // the IR, e.g. when using a zero-length array as a placeholder for a
3660 // flexible array such as unbound arrays.
3661 assert(GEP && "GEP is null");
3662 Type *SrcTy = GEP->getSourceElementType();
3663 SmallVector<Value *, 8> Indices(GEP->indices());
3664 ArrayType *ArrTy = dyn_cast<ArrayType>(Val: SrcTy);
3665 if (ArrTy && ArrTy->getNumElements() == 0 && match(V: Indices[0], P: m_Zero())) {
3666 Indices.erase(CI: Indices.begin());
3667 SrcTy = ArrTy->getElementType();
3668 return GetElementPtrInst::Create(PointeeType: SrcTy, Ptr: GEP->getPointerOperand(), IdxList: Indices,
3669 NW: GEP->getNoWrapFlags(), NameStr: "",
3670 InsertBefore: GEP->getIterator());
3671 }
3672 return nullptr;
3673}
3674
3675void SPIRVEmitIntrinsicsImpl::emitUnstructuredLoopControls(Function &F,
3676 IRBuilder<> &B) {
3677 const SPIRVSubtarget *ST = TM.getSubtargetImpl(F);
3678 // Shaders use SPIRVStructurizer which emits OpLoopMerge via spv_loop_merge.
3679 if (ST->isShader())
3680 return;
3681
3682 if (ST->canUseExtension(
3683 E: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls)) {
3684 for (BasicBlock &BB : F) {
3685 Instruction *Term = BB.getTerminator();
3686 MDNode *LoopMD = Term->getMetadata(KindID: LLVMContext::MD_loop);
3687 if (!LoopMD)
3688 continue;
3689
3690 SmallVector<unsigned, 1> Ops =
3691 getSpirvLoopControlOperandsFromLoopMetadata(LoopMD);
3692 unsigned LC = Ops[0];
3693 if (LC == SPIRV::LoopControl::None)
3694 continue;
3695
3696 // Emit intrinsic: loop control mask + optional parameters.
3697 B.SetInsertPoint(Term);
3698 SmallVector<Value *, 4> IntrArgs;
3699 for (unsigned Op : Ops)
3700 IntrArgs.push_back(Elt: B.getInt32(C: Op));
3701 B.CreateIntrinsic(ID: Intrinsic::spv_loop_control_intel, Args: IntrArgs);
3702 }
3703 return;
3704 }
3705
3706 // For non-shader targets without the Intel extension, emit OpLoopMerge
3707 // using spv_loop_merge intrinsics, mirroring the structurizer approach.
3708 LoopInfo LI;
3709 LI.analyze(F: &F);
3710 if (LI.empty())
3711 return;
3712
3713 for (Loop *L : LI.getLoopsInPreorder()) {
3714 BasicBlock *Latch = L->getLoopLatch();
3715 if (!Latch)
3716 continue;
3717 BasicBlock *MergeBlock = L->getUniqueExitBlock();
3718 if (!MergeBlock)
3719 continue;
3720
3721 // Check for loop unroll metadata on the latch terminator.
3722 SmallVector<unsigned, 1> LoopControlOps =
3723 getSpirvLoopControlOperandsFromLoopMetadata(L);
3724 if (LoopControlOps[0] == SPIRV::LoopControl::None)
3725 continue;
3726
3727 BasicBlock *Header = L->getHeader();
3728 B.SetInsertPoint(Header->getTerminator());
3729 auto *MergeAddress = BlockAddress::get(F: &F, BB: MergeBlock);
3730 auto *ContinueAddress = BlockAddress::get(F: &F, BB: Latch);
3731 SmallVector<Value *, 4> Args = {MergeAddress, ContinueAddress};
3732 for (unsigned Imm : LoopControlOps)
3733 Args.emplace_back(Args: B.getInt32(C: Imm));
3734 B.CreateIntrinsic(ID: Intrinsic::spv_loop_merge, Args: {Args});
3735 }
3736}
3737
3738bool SPIRVEmitIntrinsicsImpl::runOnFunction(Function &Func) {
3739 if (Func.isDeclaration())
3740 return false;
3741
3742 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: Func);
3743 GR = ST.getSPIRVGlobalRegistry();
3744
3745 if (!CurrF)
3746 HaveFunPtrs =
3747 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers);
3748
3749 CanUseAnyVectorRank =
3750 ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector);
3751 CurrF = &Func;
3752 IRBuilder<> B(Func.getContext());
3753 AggrConsts.clear();
3754 AggrConstTypes.clear();
3755 AggrStores.clear();
3756
3757 processParamTypesByFunHeader(F: CurrF, B);
3758
3759 // Fix GEP result types ahead of inference, and simplify if possible.
3760 // Data structure for dead instructions that were simplified and replaced.
3761 SmallPtrSet<Instruction *, 4> DeadInsts;
3762 for (auto &I : instructions(F&: Func)) {
3763 if (StoreInst *SI = dyn_cast<StoreInst>(Val: &I)) {
3764 Type *ElTy = SI->getValueOperand()->getType();
3765 if (ElTy->isAggregateType() || ElTy->isVectorTy())
3766 AggrStores.insert(Ptr: &I);
3767 continue;
3768 }
3769
3770 auto *GEP = dyn_cast<GetElementPtrInst>(Val: &I);
3771 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
3772
3773 if ((!GEP && !SGEP) || GR->findDeducedElementType(Val: &I))
3774 continue;
3775
3776 if (SGEP) {
3777 GR->addDeducedElementType(
3778 Val: SGEP,
3779 Ty: normalizeType(Ty: SGEP->getResultElementType(), CanUseAnyVectorRank));
3780 continue;
3781 }
3782
3783 GetElementPtrInst *NewGEP = simplifyZeroLengthArrayGepInst(GEP);
3784 if (NewGEP) {
3785 GEP->replaceAllUsesWith(V: NewGEP);
3786 DeadInsts.insert(Ptr: GEP);
3787 GEP = NewGEP;
3788 }
3789 if (Type *GepTy = getGEPType(Ref: GEP))
3790 GR->addDeducedElementType(Val: GEP, Ty: normalizeType(Ty: GepTy, CanUseAnyVectorRank));
3791 }
3792 // Remove dead instructions that were simplified and replaced.
3793 for (auto *I : DeadInsts) {
3794 assert(I->use_empty() && "Dead instruction should not have any uses left");
3795 I->eraseFromParent();
3796 }
3797
3798 B.SetInsertPoint(TheBB: &Func.getEntryBlock(), IP: Func.getEntryBlock().begin());
3799 for (auto &GV : Func.getParent()->globals())
3800 processGlobalValue(GV, B);
3801
3802 reconstructAggregateReturns(Func, B);
3803 preprocessUndefsAndPoisons(B);
3804 simplifyNullAddrSpaceCasts();
3805 preprocessCompositeConstants(B);
3806
3807 // A PHINode, SelectInst or FreezeInst takes its result type from its
3808 // operands. Aggregate arms are lowered to i32 value-ids (composite constants
3809 // here, loads and other producers during the visitor pass below), so mutate
3810 // an aggregate PHI, select or freeze to match. The original type is tracked
3811 // in AggrConstTypes (used to assign the SPIR-V type) and its extractvalue
3812 // users are lowered to spv_extractv.
3813 Type *I32Ty = B.getInt32Ty();
3814 for (Instruction &I : instructions(F&: Func)) {
3815 if (!isAggregateValueIdInstr(I))
3816 continue;
3817 // Give multi-register arms a value-id first, before the result is mutated.
3818 insertCompositeAggregateArms(I: &I, B);
3819 AggrConstTypes[&I] = I.getType();
3820 I.mutateType(Ty: I32Ty);
3821 }
3822
3823 preprocessBoolVectorBitcasts(F&: Func);
3824 SmallVector<Instruction *> Worklist(
3825 llvm::make_pointer_range(Range: instructions(F&: Func)));
3826
3827 applyDemangledPtrArgTypes(B);
3828
3829 // Pass forward: use operand to deduce instructions result.
3830 for (auto &I : Worklist) {
3831 // Don't emit intrinsincs for convergence intrinsics.
3832 if (isConvergenceIntrinsic(I))
3833 continue;
3834
3835 bool Postpone = insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: false);
3836 // if Postpone is true, we can't decide on pointee type yet
3837 insertAssignTypeIntrs(I, B);
3838 insertPtrCastOrAssignTypeInstr(I, B);
3839 insertSpirvDecorations(I, B);
3840 // if instruction requires a pointee type set, let's check if we know it
3841 // already, and force it to be i8 if not
3842 if (Postpone && !GR->findAssignPtrTypeInstr(Val: I))
3843 insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: true);
3844
3845 if (auto *FPI = dyn_cast<ConstrainedFPIntrinsic>(Val: I))
3846 useRoundingMode(FPI, B);
3847 }
3848
3849 // Pass backward: use instructions results to specify/update/cast operands
3850 // where needed.
3851 SmallPtrSet<Instruction *, 4> IncompleteRets;
3852 for (auto &I : llvm::reverse(C: instructions(F&: Func)))
3853 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets);
3854
3855 // Pass forward for PHIs only, their operands are not preceed the
3856 // instruction in meaning of `instructions(Func)`.
3857 for (BasicBlock &BB : Func)
3858 for (PHINode &Phi : BB.phis())
3859 if (isPointerTy(T: Phi.getType()))
3860 deduceOperandElementType(I: &Phi, IncompleteRets: nullptr);
3861
3862 for (auto *I : Worklist) {
3863 TrackConstants = true;
3864 if (!I->getType()->isVoidTy() || isa<StoreInst>(Val: I))
3865 setInsertPointAfterDef(B, I);
3866 // Visitors return either the original/newly created instruction for
3867 // further processing, nullptr otherwise.
3868 I = visit(I&: *I);
3869 if (!I)
3870 continue;
3871
3872 // Don't emit intrinsics for convergence operations.
3873 if (isConvergenceIntrinsic(I))
3874 continue;
3875
3876 addSaturatedDecorationToIntrinsic(I, B);
3877 processInstrAfterVisit(I, B);
3878 }
3879
3880 emitUnstructuredLoopControls(F&: Func, B);
3881
3882 return true;
3883}
3884
3885// Try to deduce a better type for pointers to untyped ptr.
3886bool SPIRVEmitIntrinsicsImpl::postprocessTypes(Module &M) {
3887 if (!GR || TodoTypeSz == 0)
3888 return false;
3889
3890 unsigned SzTodo = TodoTypeSz;
3891 DenseMap<Value *, SmallPtrSet<Value *, 4>> ToProcess;
3892 for (auto [Op, Enabled] : TodoType) {
3893 // TODO: add isa<CallInst>(Op) to continue
3894 if (!Enabled || isaGEP(V: Op))
3895 continue;
3896 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Op);
3897 Type *KnownTy = GR->findDeducedElementType(Val: Op);
3898 if (!KnownTy || !AssignCI)
3899 continue;
3900 assert(Op == AssignCI->getArgOperand(0));
3901 // Try to improve the type deduced after all Functions are processed.
3902 if (auto *CI = dyn_cast<Instruction>(Val: Op)) {
3903 CurrF = CI->getParent()->getParent();
3904 SmallPtrSet<Value *, 0> Visited;
3905 if (Type *ElemTy = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8: false, IgnoreKnownType: true)) {
3906 if (ElemTy != KnownTy) {
3907 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3908 propagateElemType(Op: CI, ElemTy, VisitedSubst);
3909 eraseTodoType(Op);
3910 continue;
3911 }
3912 }
3913 }
3914
3915 if (Op->hasUseList()) {
3916 for (User *U : Op->users()) {
3917 Instruction *Inst = dyn_cast<Instruction>(Val: U);
3918 if (Inst && !isa<IntrinsicInst>(Val: Inst))
3919 ToProcess[Inst].insert(Ptr: Op);
3920 }
3921 }
3922 }
3923 if (TodoTypeSz == 0)
3924 return true;
3925
3926 for (auto &F : M) {
3927 CurrF = &F;
3928 SmallPtrSet<Instruction *, 4> IncompleteRets;
3929 for (auto &I : llvm::reverse(C: instructions(F))) {
3930 auto It = ToProcess.find(Val: &I);
3931 if (It == ToProcess.end())
3932 continue;
3933 It->second.remove_if(P: [this](Value *V) { return !isTodoType(Op: V); });
3934 if (It->second.size() == 0)
3935 continue;
3936 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets, AskOps: &It->second, IsPostprocessing: true);
3937 if (TodoTypeSz == 0)
3938 return true;
3939 }
3940 }
3941
3942 return SzTodo > TodoTypeSz;
3943}
3944
3945// Parse and store argument types of function declarations where needed.
3946void SPIRVEmitIntrinsicsImpl::parseFunDeclarations(Module &M) {
3947 for (auto &F : M) {
3948 if (!F.isDeclaration() || F.isIntrinsic())
3949 continue;
3950 // get the demangled name
3951 std::string DemangledName = getOclOrSpirvBuiltinDemangledName(Name: F.getName());
3952 if (DemangledName.empty())
3953 continue;
3954 // allow only OpGroupAsyncCopy use case at the moment
3955 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F);
3956 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
3957 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
3958 if (Opcode != SPIRV::OpGroupAsyncCopy)
3959 continue;
3960 // find pointer arguments
3961 SmallVector<unsigned> Idxs;
3962 for (unsigned OpIdx = 0; OpIdx < F.arg_size(); ++OpIdx) {
3963 Argument *Arg = F.getArg(i: OpIdx);
3964 if (isPointerTy(T: Arg->getType()) && !hasPointeeTypeAttr(Arg))
3965 Idxs.push_back(Elt: OpIdx);
3966 }
3967 if (!Idxs.size())
3968 continue;
3969 // parse function arguments
3970 LLVMContext &Ctx = F.getContext();
3971 SmallVector<StringRef, 10> TypeStrs;
3972 SPIRV::parseBuiltinTypeStr(BuiltinArgsTypeStrs&: TypeStrs, DemangledCall: DemangledName, Ctx);
3973 if (!TypeStrs.size())
3974 continue;
3975 // find type info for pointer arguments
3976 for (unsigned Idx : Idxs) {
3977 if (Idx >= TypeStrs.size())
3978 continue;
3979 if (Type *ElemTy =
3980 SPIRV::parseBuiltinCallArgumentType(TypeStr: TypeStrs[Idx].trim(), Ctx))
3981 if (TypedPointerType::isValidElementType(ElemTy) &&
3982 !ElemTy->isTargetExtTy())
3983 FDeclPtrTys[&F].push_back(Elt: std::make_pair(x&: Idx, y&: ElemTy));
3984 }
3985 }
3986}
3987
3988bool SPIRVEmitIntrinsicsImpl::processMaskedMemIntrinsic(IntrinsicInst &I) {
3989 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
3990
3991 if (I.getIntrinsicID() == Intrinsic::masked_gather) {
3992 if (!ST.canUseExtension(
3993 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
3994 I.getContext().emitError(
3995 I: &I, ErrorStr: "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter "
3996 "extension");
3997 // Replace with poison to allow compilation to continue and report error.
3998 I.replaceAllUsesWith(V: PoisonValue::get(T: I.getType()));
3999 I.eraseFromParent();
4000 return true;
4001 }
4002
4003 IRBuilder<> B(&I);
4004
4005 Value *Ptrs = I.getArgOperand(i: 0);
4006 Value *Mask = I.getArgOperand(i: 1);
4007 Value *Passthru = I.getArgOperand(i: 2);
4008
4009 // Alignment is stored as a parameter attribute, not as a regular parameter.
4010 uint32_t Alignment = I.getParamAlign(ArgNo: 0).valueOrOne().value();
4011
4012 SmallVector<Value *, 4> Args = {Ptrs, B.getInt32(C: Alignment), Mask,
4013 Passthru};
4014 SmallVector<Type *, 4> Types = {I.getType(), Ptrs->getType(),
4015 Mask->getType(), Passthru->getType()};
4016
4017 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_masked_gather, OverloadTypes: Types, Args);
4018 I.replaceAllUsesWith(V: NewI);
4019 I.eraseFromParent();
4020 return true;
4021 }
4022
4023 if (I.getIntrinsicID() == Intrinsic::masked_scatter) {
4024 if (!ST.canUseExtension(
4025 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
4026 I.getContext().emitError(
4027 I: &I, ErrorStr: "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter "
4028 "extension");
4029 // Erase the intrinsic to allow compilation to continue and report error.
4030 I.eraseFromParent();
4031 return true;
4032 }
4033
4034 IRBuilder<> B(&I);
4035
4036 Value *Values = I.getArgOperand(i: 0);
4037 Value *Ptrs = I.getArgOperand(i: 1);
4038 Value *Mask = I.getArgOperand(i: 2);
4039
4040 // Alignment is stored as a parameter attribute on the ptrs parameter (arg
4041 // 1).
4042 uint32_t Alignment = I.getParamAlign(ArgNo: 1).valueOrOne().value();
4043
4044 SmallVector<Value *, 4> Args = {Values, Ptrs, B.getInt32(C: Alignment), Mask};
4045 SmallVector<Type *, 3> Types = {Values->getType(), Ptrs->getType(),
4046 Mask->getType()};
4047
4048 B.CreateIntrinsic(ID: Intrinsic::spv_masked_scatter, OverloadTypes: Types, Args);
4049 I.eraseFromParent();
4050 return true;
4051 }
4052
4053 return false;
4054}
4055
4056// SPIR-V doesn't support bitcasts involving vector boolean type. Decompose such
4057// bitcasts into element-wise operations before building instructions
4058// worklist, so new instructions are properly visited and converted to
4059// SPIR-V intrinsics.
4060void SPIRVEmitIntrinsicsImpl::preprocessBoolVectorBitcasts(Function &F) {
4061 struct BoolVecBitcast {
4062 BitCastInst *BC;
4063 FixedVectorType *BoolVecTy;
4064 bool SrcIsBoolVec;
4065 };
4066
4067 auto getAsBoolVec = [](Type *Ty) -> FixedVectorType * {
4068 auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
4069 return (VTy && VTy->getElementType()->isIntegerTy(BitWidth: 1)) ? VTy : nullptr;
4070 };
4071
4072 SmallVector<BoolVecBitcast, 4> ToReplace;
4073 for (auto &I : instructions(F)) {
4074 auto *BC = dyn_cast<BitCastInst>(Val: &I);
4075 if (!BC)
4076 continue;
4077 if (auto *BVTy = getAsBoolVec(BC->getSrcTy()))
4078 ToReplace.push_back(Elt: {.BC: BC, .BoolVecTy: BVTy, .SrcIsBoolVec: true});
4079 else if (auto *BVTy = getAsBoolVec(BC->getDestTy()))
4080 ToReplace.push_back(Elt: {.BC: BC, .BoolVecTy: BVTy, .SrcIsBoolVec: false});
4081 }
4082
4083 for (auto &[BC, BoolVecTy, SrcIsBoolVec] : ToReplace) {
4084 IRBuilder<> B(BC);
4085 Value *Src = BC->getOperand(i_nocapture: 0);
4086 unsigned BoolVecN = BoolVecTy->getNumElements();
4087 // Use iN as the scalar intermediate type for the bool vector side.
4088 Type *IntTy = B.getIntNTy(N: BoolVecN);
4089
4090 // Convert source to scalar integer.
4091 Value *IntVal;
4092 if (SrcIsBoolVec) {
4093 // Extract each bool, zext, shift, and OR.
4094 IntVal = ConstantInt::get(Ty: IntTy, V: 0);
4095 for (unsigned I = 0; I < BoolVecN; ++I) {
4096 Value *Elem = B.CreateExtractElement(Vec: Src, Idx: B.getInt32(C: I));
4097 Value *Ext = B.CreateZExt(V: Elem, DestTy: IntTy);
4098 if (I > 0)
4099 Ext = B.CreateShl(LHS: Ext, RHS: ConstantInt::get(Ty: IntTy, V: I));
4100 IntVal = B.CreateOr(LHS: IntVal, RHS: Ext);
4101 }
4102 } else {
4103 // Source is a non-bool type. If it's already a scalar integer, use it
4104 // directly, otherwise bitcast to iN first.
4105 IntVal = Src;
4106 if (!Src->getType()->isIntegerTy())
4107 IntVal = B.CreateBitCast(V: Src, DestTy: IntTy);
4108 }
4109
4110 // Convert scalar integer to destination type.
4111 Value *Result;
4112 if (!SrcIsBoolVec) {
4113 // Test each bit with AND + icmp.
4114 Result = PoisonValue::get(T: BoolVecTy);
4115 for (unsigned I = 0; I < BoolVecN; ++I) {
4116 Value *Mask = ConstantInt::get(Ty: IntTy, V: APInt::getOneBitSet(numBits: BoolVecN, BitNo: I));
4117 Value *And = B.CreateAnd(LHS: IntVal, RHS: Mask);
4118 Value *Cmp = B.CreateICmpNE(LHS: And, RHS: ConstantInt::get(Ty: IntTy, V: 0));
4119 Result = B.CreateInsertElement(Vec: Result, NewElt: Cmp, Idx: B.getInt32(C: I));
4120 }
4121 } else {
4122 // Destination is a non-bool type. If it's a scalar integer, use IntVal
4123 // directly, otherwise bitcast from iN.
4124 Result = IntVal;
4125 if (!BC->getDestTy()->isIntegerTy())
4126 Result = B.CreateBitCast(V: IntVal, DestTy: BC->getDestTy());
4127 }
4128
4129 BC->replaceAllUsesWith(V: Result);
4130 BC->eraseFromParent();
4131 }
4132}
4133
4134bool SPIRVEmitIntrinsicsImpl::convertMaskedMemIntrinsics(Module &M) {
4135 bool Changed = false;
4136
4137 for (Function &F : make_early_inc_range(Range&: M)) {
4138 if (!F.isIntrinsic())
4139 continue;
4140 Intrinsic::ID IID = F.getIntrinsicID();
4141 if (IID != Intrinsic::masked_gather && IID != Intrinsic::masked_scatter)
4142 continue;
4143
4144 for (User *U : make_early_inc_range(Range: F.users())) {
4145 if (auto *II = dyn_cast<IntrinsicInst>(Val: U))
4146 Changed |= processMaskedMemIntrinsic(I&: *II);
4147 }
4148
4149 if (F.use_empty())
4150 F.eraseFromParent();
4151 }
4152
4153 return Changed;
4154}
4155
4156bool SPIRVEmitIntrinsicsImpl::runOnModule(Module &M) {
4157 bool Changed = false;
4158
4159 Changed |= convertMaskedMemIntrinsics(M);
4160
4161 parseFunDeclarations(M);
4162 insertConstantsForFPFastMathDefault(M);
4163 GVUsers.init(M);
4164
4165 TodoType.clear();
4166 for (auto &F : M)
4167 Changed |= runOnFunction(Func&: F);
4168
4169 // Specify function parameters after all functions were processed.
4170 for (auto &F : M) {
4171 // check if function parameter types are set
4172 CurrF = &F;
4173 if (!F.isDeclaration() && !F.isIntrinsic()) {
4174 IRBuilder<> B(F.getContext());
4175 processParamTypes(F: &F, B);
4176 }
4177 }
4178
4179 CanTodoType = false;
4180 Changed |= postprocessTypes(M);
4181
4182 if (HaveFunPtrs)
4183 Changed |= processFunctionPointers(M);
4184
4185 return Changed;
4186}
4187
4188PreservedAnalyses
4189llvm::SPIRVEmitIntrinsicsPass::run(Module &M, ModuleAnalysisManager &AM) {
4190 if (SPIRVEmitIntrinsicsImpl(TM).runOnModule(M))
4191 return PreservedAnalyses::none();
4192 return PreservedAnalyses::all();
4193}
4194
4195ModulePass *llvm::createSPIRVEmitIntrinsicsPass(const SPIRVTargetMachine &TM) {
4196 return new SPIRVEmitIntrinsicsLegacy(TM);
4197}
4198