1//===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass emits SPIRV intrinsics keeping essential high-level information for
10// the translation of LLVM IR to SPIR-V.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRV.h"
15#include "SPIRVAuxDataHandler.h"
16#include "SPIRVBuiltins.h"
17#include "SPIRVSubtarget.h"
18#include "SPIRVTargetMachine.h"
19#include "SPIRVUtils.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/DenseSet.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/ADT/SmallString.h"
24#include "llvm/ADT/StringSwitch.h"
25#include "llvm/Analysis/LoopInfo.h"
26#include "llvm/IR/IRBuilder.h"
27#include "llvm/IR/InstIterator.h"
28#include "llvm/IR/InstVisitor.h"
29#include "llvm/IR/IntrinsicsSPIRV.h"
30#include "llvm/IR/LLVMContext.h"
31#include "llvm/IR/PatternMatch.h"
32#include "llvm/IR/TypedPointerType.h"
33#include "llvm/IR/Value.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/raw_ostream.h"
37#include "llvm/Transforms/Utils/Local.h"
38
39#include <cassert>
40#include <optional>
41#include <queue>
42
43// This pass performs the following transformation on LLVM IR level required
44// for the following translation to SPIR-V:
45// - replaces direct usages of aggregate constants with target-specific
46// intrinsics;
47// - replaces aggregates-related instructions (extract/insert, ld/st, etc)
48// with a target-specific intrinsics;
49// - emits intrinsics for the global variable initializers since IRTranslator
50// doesn't handle them and it's not very convenient to translate them
51// ourselves;
52// - emits intrinsics to keep track of the string names assigned to the values;
53// - emits intrinsics to keep track of constants (this is necessary to have an
54// LLVM IR constant after the IRTranslation is completed) for their further
55// deduplication;
56// - emits intrinsics to keep track of original LLVM types of the values
57// to be able to emit proper SPIR-V types eventually.
58//
59// TODO: consider removing spv.track.constant in favor of spv.assign.type.
60
61using namespace llvm;
62using namespace llvm::PatternMatch;
63
64#define DEBUG_TYPE "spirv-emit-intrinsics"
65
66static cl::opt<bool>
67 SpirvEmitOpNames("spirv-emit-op-names",
68 cl::desc("Emit OpName for all instructions"),
69 cl::init(Val: false));
70
71namespace llvm::SPIRV {
72#define GET_BuiltinGroup_DECL
73#include "SPIRVGenTables.inc"
74} // namespace llvm::SPIRV
75
76namespace {
77// This class keeps track of which functions reference which global variables.
78class GlobalVariableUsers {
79 template <typename T1, typename T2>
80 using OneToManyMapTy = DenseMap<T1, SmallPtrSet<T2, 4>>;
81
82 OneToManyMapTy<const GlobalVariable *, const Function *> GlobalIsUsedByFun;
83
84 void collectGlobalUsers(
85 const GlobalVariable *GV,
86 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
87 &GlobalIsUsedByGlobal) {
88 SmallVector<const Value *> Stack = {GV->user_begin(), GV->user_end()};
89 while (!Stack.empty()) {
90 const Value *V = Stack.pop_back_val();
91
92 if (const Instruction *I = dyn_cast<Instruction>(Val: V)) {
93 GlobalIsUsedByFun[GV].insert(Ptr: I->getFunction());
94 continue;
95 }
96
97 if (const GlobalVariable *UserGV = dyn_cast<GlobalVariable>(Val: V)) {
98 GlobalIsUsedByGlobal[GV].insert(Ptr: UserGV);
99 continue;
100 }
101
102 if (const Constant *C = dyn_cast<Constant>(Val: V))
103 Stack.append(in_start: C->user_begin(), in_end: C->user_end());
104 }
105 }
106
107 bool propagateGlobalToGlobalUsers(
108 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
109 &GlobalIsUsedByGlobal) {
110 SmallVector<const GlobalVariable *> OldUsersGlobals;
111 bool Changed = false;
112 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
113 OldUsersGlobals.assign(in_start: UserGlobals.begin(), in_end: UserGlobals.end());
114 for (const GlobalVariable *UserGV : OldUsersGlobals) {
115 auto It = GlobalIsUsedByGlobal.find(Val: UserGV);
116 if (It == GlobalIsUsedByGlobal.end())
117 continue;
118 Changed |= set_union(S1&: UserGlobals, S2: It->second);
119 }
120 }
121 return Changed;
122 }
123
124 void propagateGlobalToFunctionReferences(
125 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
126 &GlobalIsUsedByGlobal) {
127 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
128 auto &UserFunctions = GlobalIsUsedByFun[GV];
129 for (const GlobalVariable *UserGV : UserGlobals) {
130 auto It = GlobalIsUsedByFun.find(Val: UserGV);
131 if (It == GlobalIsUsedByFun.end())
132 continue;
133 set_union(S1&: UserFunctions, S2: It->second);
134 }
135 }
136 }
137
138public:
139 void init(Module &M) {
140 // Collect which global variables are referenced by which global variables
141 // and which functions reference each global variables.
142 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
143 GlobalIsUsedByGlobal;
144 GlobalIsUsedByFun.clear();
145 for (GlobalVariable &GV : M.globals())
146 collectGlobalUsers(GV: &GV, GlobalIsUsedByGlobal);
147
148 // Compute indirect references by iterating until a fixed point is reached.
149 while (propagateGlobalToGlobalUsers(GlobalIsUsedByGlobal))
150 (void)0;
151
152 propagateGlobalToFunctionReferences(GlobalIsUsedByGlobal);
153 }
154
155 using FunctionSetType = typename decltype(GlobalIsUsedByFun)::mapped_type;
156 const FunctionSetType &
157 getTransitiveUserFunctions(const GlobalVariable &GV) const {
158 auto It = GlobalIsUsedByFun.find(Val: &GV);
159 if (It != GlobalIsUsedByFun.end())
160 return It->second;
161
162 static const FunctionSetType Empty{};
163 return Empty;
164 }
165};
166
167static bool isaGEP(const Value *V) {
168 return isa<StructuredGEPInst>(Val: V) || isa<GetElementPtrInst>(Val: V);
169}
170
171// If Ty is a byte-addressing type, return the multiplier for the offset.
172// Otherwise return std::nullopt.
173static std::optional<uint64_t> getByteAddressingMultiplier(Type *Ty) {
174 if (Ty == IntegerType::getInt8Ty(C&: Ty->getContext())) {
175 return 1;
176 }
177 if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
178 if (AT->getElementType() == IntegerType::getInt8Ty(C&: Ty->getContext())) {
179 return AT->getNumElements();
180 }
181 }
182 return std::nullopt;
183}
184
185class SPIRVEmitIntrinsicsImpl
186 : public InstVisitor<SPIRVEmitIntrinsicsImpl, Instruction *> {
187 const SPIRVTargetMachine &TM;
188 SPIRVGlobalRegistry *GR = nullptr;
189 Function *CurrF = nullptr;
190 bool TrackConstants = true;
191 bool HaveFunPtrs = false;
192 bool CanUseAnyVectorRank = false;
193 DenseMap<Instruction *, Constant *> AggrConsts;
194 DenseMap<Instruction *, Type *> AggrConstTypes;
195 SmallPtrSet<Instruction *, 0> AggrStores;
196 GlobalVariableUsers GVUsers;
197 SmallPtrSet<Value *, 0> Named;
198
199 // map of function declarations to <pointer arg index => element type>
200 DenseMap<Function *, SmallVector<std::pair<unsigned, Type *>>> FDeclPtrTys;
201
202 // a register of Instructions that don't have a complete type definition
203 bool CanTodoType = true;
204 unsigned TodoTypeSz = 0;
205 DenseMap<Value *, bool> TodoType;
206 void insertTodoType(Value *Op) {
207 // TODO: add isa<CallInst>(Op) to no-insert
208 if (CanTodoType && !isaGEP(V: Op)) {
209 auto It = TodoType.try_emplace(Key: Op, Args: true);
210 if (It.second)
211 ++TodoTypeSz;
212 }
213 }
214 void eraseTodoType(Value *Op) {
215 auto It = TodoType.find(Val: Op);
216 if (It != TodoType.end() && It->second) {
217 It->second = false;
218 --TodoTypeSz;
219 }
220 }
221 bool isTodoType(Value *Op) {
222 if (isaGEP(V: Op))
223 return false;
224 auto It = TodoType.find(Val: Op);
225 return It != TodoType.end() && It->second;
226 }
227 // a register of Instructions that were visited by deduceOperandElementType()
228 // to validate operand types with an instruction
229 SmallPtrSet<Instruction *, 0> TypeValidated;
230
231 // well known result types of builtins
232 enum WellKnownTypes { Event };
233
234 // deduce element type of untyped pointers
235 Type *deduceElementType(Value *I, bool UnknownElemTypeI8);
236 Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8);
237 Type *deduceElementTypeHelper(Value *I, SmallPtrSetImpl<Value *> &Visited,
238 bool UnknownElemTypeI8,
239 bool IgnoreKnownType = false);
240 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
241 bool UnknownElemTypeI8);
242 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
243 SmallPtrSetImpl<Value *> &Visited,
244 bool UnknownElemTypeI8,
245 std::optional<unsigned> AddrSpace = {});
246 Type *deduceElementTypeByUsersDeep(Value *Op,
247 SmallPtrSetImpl<Value *> &Visited,
248 bool UnknownElemTypeI8);
249 void maybeAssignPtrType(Type *&Ty, Value *I, Type *RefTy,
250 bool UnknownElemTypeI8);
251
252 // deduce nested types of composites
253 Type *deduceNestedTypeHelper(User *U, bool UnknownElemTypeI8);
254 Type *deduceNestedTypeHelper(User *U, Type *Ty,
255 SmallPtrSetImpl<Value *> &Visited,
256 bool UnknownElemTypeI8);
257
258 // deduce Types of operands of the Instruction if possible
259 void
260 deduceOperandElementType(Instruction *I,
261 SmallPtrSetImpl<Instruction *> *IncompleteRets,
262 const SmallPtrSetImpl<Value *> *AskOps = nullptr,
263 bool IsPostprocessing = false);
264
265 void preprocessCompositeConstants(IRBuilder<> &B);
266 Value *lowerUndefOrPoison(Value *Op, IRBuilder<> &B, bool HasPoisonExt);
267 void preprocessUndefsAndPoisons(IRBuilder<> &B);
268 void insertCompositeAggregateArms(Instruction *I, IRBuilder<> &B);
269 void simplifyNullAddrSpaceCasts();
270
271 Type *reconstructType(Value *Op, bool UnknownElemTypeI8,
272 bool IsPostprocessing);
273
274 void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
275 void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
276 bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
277 bool UnknownElemTypeI8);
278 void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B);
279 void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V,
280 IRBuilder<> &B);
281 void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer,
282 Type *ExpectedElementType,
283 unsigned OperandToReplace,
284 IRBuilder<> &B);
285 void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
286 bool shouldTryToAddMemAliasingDecoration(Instruction *Inst);
287 void insertSpirvDecorations(Instruction *I, IRBuilder<> &B);
288 void insertConstantsForFPFastMathDefault(Module &M);
289 Value *buildSpvUndefComposite(Type *AggrTy, IRBuilder<> &B);
290 void reconstructAggregateReturns(Function &Func, IRBuilder<> &B);
291 void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
292 void processParamTypes(Function *F, IRBuilder<> &B);
293 void processParamTypesByFunHeader(Function *F, IRBuilder<> &B);
294 Type *deduceFunParamElementType(Function *F, unsigned OpIdx);
295 Type *deduceFunParamElementType(Function *F, unsigned OpIdx,
296 SmallPtrSetImpl<Function *> &FVisited);
297
298 bool deduceOperandElementTypeCalledFunction(
299 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
300 Type *&KnownElemTy, bool &Incomplete);
301 void deduceOperandElementTypeFunctionPointer(
302 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
303 Type *&KnownElemTy, bool IsPostprocessing);
304 bool deduceOperandElementTypeFunctionRet(
305 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
306 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing,
307 Type *&KnownElemTy, Value *Op, Function *F);
308
309 CallInst *buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy);
310 void replaceUsesOfWithSpvPtrcast(Value *Op, Type *ElemTy, Instruction *I,
311 DenseMap<Function *, CallInst *> Ptrcasts);
312 void propagateElemType(Value *Op, Type *ElemTy,
313 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
314 void
315 propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
316 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
317 void propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
318 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
319 SmallPtrSetImpl<Value *> &Visited,
320 DenseMap<Function *, CallInst *> Ptrcasts);
321
322 void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
323 void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src,
324 Instruction *Dest, bool DeleteOld = true);
325
326 void applyDemangledPtrArgTypes(IRBuilder<> &B);
327
328 GetElementPtrInst *simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP);
329
330 bool runOnFunction(Function &F);
331 bool postprocessTypes(Module &M);
332 bool processFunctionPointers(Module &M);
333 void parseFunDeclarations(Module &M);
334 void useRoundingMode(ConstrainedFPIntrinsic *FPI, IRBuilder<> &B);
335 bool processMaskedMemIntrinsic(IntrinsicInst &I);
336 bool convertMaskedMemIntrinsics(Module &M);
337 void preprocessBoolVectorBitcasts(Function &F);
338
339 void emitUnstructuredLoopControls(Function &F, IRBuilder<> &B);
340
341 // Tries to walk the type accessed by the given GEP instruction.
342 // For each nested type access, one of the 2 callbacks is called:
343 // - OnLiteralIndexing when the index is a known constant value.
344 // Parameters:
345 // PointedType: the pointed type resulting of this indexing.
346 // If the parent type is an array, this is the index in the array.
347 // If the parent type is a struct, this is the field index.
348 // Index: index of the element in the parent type.
349 // - OnDynamnicIndexing when the index is a non-constant value.
350 // This callback is only called when indexing into an array.
351 // Parameters:
352 // ElementType: the type of the elements stored in the parent array.
353 // Offset: the Value* containing the byte offset into the array.
354 // Multiplier: a scaling factor for the offset.
355 // Return true if an error occurred during the walk, false otherwise.
356 bool walkLogicalAccessChain(
357 GetElementPtrInst &GEP,
358 const std::function<void(Type *PointedType, uint64_t Index)>
359 &OnLiteralIndexing,
360 const std::function<void(Type *ElementType, Value *Offset,
361 uint64_t Multiplier)> &OnDynamicIndexing);
362
363 bool walkLogicalAccessChainDynamic(
364 Type *CurType, Value *Operand, uint64_t Multiplier,
365 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
366 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing);
367
368 bool walkLogicalAccessChainConstant(
369 Type *CurType, uint64_t Offset,
370 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing);
371
372 // Returns the type accessed using the given GEP instruction by relying
373 // on the GEP type.
374 // FIXME: GEP types are not supposed to be used to retrieve the pointed
375 // type. This must be fixed.
376 Type *getGEPType(GetElementPtrInst *GEP);
377
378 // Returns the type accessed using the given GEP instruction by walking
379 // the source type using the GEP indices.
380 // FIXME: without help from the frontend, this method cannot reliably retrieve
381 // the stored type, nor can robustly determine the depth of the type
382 // we are accessing.
383 Type *getGEPTypeLogical(GetElementPtrInst *GEP);
384
385 Instruction *buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP);
386
387public:
388 SPIRVEmitIntrinsicsImpl(const SPIRVTargetMachine &TM) : TM(TM) {}
389 Instruction *visitInstruction(Instruction &I) { return &I; }
390 Instruction *visitSwitchInst(SwitchInst &I);
391 Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
392 Instruction *visitIntrinsicInst(IntrinsicInst &I);
393 Instruction *visitBitCastInst(BitCastInst &I);
394 Instruction *visitInsertElementInst(InsertElementInst &I);
395 Instruction *visitExtractElementInst(ExtractElementInst &I);
396 Instruction *visitInsertValueInst(InsertValueInst &I);
397 Instruction *visitExtractValueInst(ExtractValueInst &I);
398 Instruction *visitLoadInst(LoadInst &I);
399 Instruction *visitStoreInst(StoreInst &I);
400 Instruction *visitAllocaInst(AllocaInst &I);
401 Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
402 Instruction *visitAtomicRMWInst(AtomicRMWInst &I);
403 Instruction *visitUnreachableInst(UnreachableInst &I);
404 Instruction *visitCallInst(CallInst &I);
405
406 bool runOnModule(Module &M);
407};
408
409class SPIRVEmitIntrinsicsLegacy : public ModulePass {
410 const SPIRVTargetMachine &TM;
411
412public:
413 static char ID;
414 SPIRVEmitIntrinsicsLegacy(const SPIRVTargetMachine &TM)
415 : ModulePass(ID), TM(TM) {}
416
417 StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
418
419 bool runOnModule(Module &M) override {
420 return SPIRVEmitIntrinsicsImpl(TM).runOnModule(M);
421 }
422};
423
424bool isConvergenceIntrinsic(const Instruction *I) {
425 return match(V: I, P: m_AnyIntrinsic<Intrinsic::experimental_convergence_entry,
426 Intrinsic::experimental_convergence_loop,
427 Intrinsic::experimental_convergence_anchor>());
428}
429
430bool expectIgnoredInIRTranslation(const Instruction *I) {
431 return match(V: I, P: m_AnyIntrinsic<Intrinsic::invariant_start,
432 Intrinsic::spv_resource_handlefrombinding,
433 Intrinsic::spv_resource_getbasepointer,
434 Intrinsic::spv_resource_getpointer>());
435}
436
437// Returns the source pointer from `I` ignoring intermediate ptrcast.
438Value *getPointerRoot(Value *I) {
439 Value *V;
440 if (match(V: I, P: m_Intrinsic<Intrinsic::spv_ptrcast>(Ops: m_Value(V))))
441 return getPointerRoot(I: V);
442 return I;
443}
444
445} // namespace
446
447char SPIRVEmitIntrinsicsLegacy::ID = 0;
448
449INITIALIZE_PASS(SPIRVEmitIntrinsicsLegacy, "spirv-emit-intrinsics",
450 "SPIRV emit intrinsics", false, false)
451
452static inline bool isAssignTypeInstr(const Instruction *I) {
453 return match(V: I, P: m_Intrinsic<Intrinsic::spv_assign_type>());
454}
455
456static bool isMemInstrToReplace(Instruction *I) {
457 return isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I) || isa<InsertValueInst>(Val: I) ||
458 isa<ExtractValueInst>(Val: I) || isa<AtomicCmpXchgInst>(Val: I);
459}
460
461static bool isAggrConstForceInt32(const Value *V) {
462 bool IsAggrZero =
463 isa<ConstantAggregateZero>(Val: V) && !V->getType()->isVectorTy();
464 bool IsUndefAggregate = isa<UndefValue>(Val: V) && V->getType()->isAggregateType();
465 return isa<ConstantArray>(Val: V) || isa<ConstantStruct>(Val: V) ||
466 isa<ConstantDataArray>(Val: V) || IsAggrZero || IsUndefAggregate;
467}
468
469static bool isSpvAggrPlaceholder(const Value *V) {
470 return match(
471 V,
472 P: m_AnyIntrinsic<Intrinsic::spv_undef, Intrinsic::spv_const_composite>());
473}
474
475static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
476 if (isa<PHINode>(Val: I))
477 B.SetInsertPoint(I->getParent()->getFirstNonPHIOrAlloca());
478 else
479 B.SetInsertPoint(I);
480}
481
482static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I) {
483 B.SetCurrentDebugLocation(I->getDebugLoc());
484 if (I->getType()->isVoidTy())
485 B.SetInsertPoint(I->getNextNode());
486 else
487 B.SetInsertPoint(*I->getInsertionPointAfterDef());
488}
489
490static bool requireAssignType(Instruction *I) {
491 return !match(
492 V: I,
493 P: m_AnyIntrinsic<Intrinsic::invariant_start, Intrinsic::invariant_end>());
494}
495
496static inline void reportFatalOnTokenType(const Instruction *I) {
497 if (I->getType()->isTokenTy())
498 report_fatal_error(reason: "A token is encountered but SPIR-V without extensions "
499 "does not support token type",
500 gen_crash_diag: false);
501}
502
503static void emitAssignName(Instruction *I, IRBuilder<> &B) {
504 if (!I->hasName() || I->getType()->isAggregateType() ||
505 expectIgnoredInIRTranslation(I))
506 return;
507
508 // We want to be conservative when adding the names because they can interfere
509 // with later optimizations.
510 bool KeepName = SpirvEmitOpNames;
511 if (!KeepName) {
512 if (isa<AllocaInst>(Val: I)) {
513 KeepName = true;
514 } else if (auto *CI = dyn_cast<CallBase>(Val: I)) {
515 Function *F = CI->getCalledFunction();
516 if (F && F->getName().starts_with(Prefix: "llvm.spv.alloca"))
517 KeepName = true;
518 }
519 }
520
521 if (!KeepName)
522 return;
523
524 reportFatalOnTokenType(I);
525 setInsertPointAfterDef(B, I);
526 LLVMContext &Ctx = I->getContext();
527 std::vector<Value *> Args = {
528 I, MetadataAsValue::get(
529 Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: I->getName())))};
530 B.CreateIntrinsic(ID: Intrinsic::spv_assign_name, OverloadTypes: {I->getType()}, Args);
531}
532
533void SPIRVEmitIntrinsicsImpl::replaceAllUsesWith(Value *Src, Value *Dest,
534 bool DeleteOld) {
535 GR->replaceAllUsesWith(Old: Src, New: Dest, DeleteOld);
536 // Update uncomplete type records if any
537 if (isTodoType(Op: Src)) {
538 if (DeleteOld)
539 eraseTodoType(Op: Src);
540 insertTodoType(Op: Dest);
541 }
542}
543
544void SPIRVEmitIntrinsicsImpl::replaceAllUsesWithAndErase(IRBuilder<> &B,
545 Instruction *Src,
546 Instruction *Dest,
547 bool DeleteOld) {
548 replaceAllUsesWith(Src, Dest, DeleteOld);
549 std::string Name = Src->hasName() ? Src->getName().str() : "";
550 Src->eraseFromParent();
551 if (!Name.empty()) {
552 Dest->setName(Name);
553 if (Named.insert(Ptr: Dest).second)
554 emitAssignName(I: Dest, B);
555 }
556}
557
558static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
559 return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
560 isPointerTy(T: SI->getValueOperand()->getType()) &&
561 isa<Argument>(Val: SI->getValueOperand());
562}
563
564// A pointer-typed local holds a pointer, so its deduced pointee must stay a
565// pointer.
566static bool tracesToPointerAlloca(Value *V) {
567 using namespace PatternMatch;
568 V = V->stripPointerCasts();
569 if (auto *AI = dyn_cast<AllocaInst>(Val: V))
570 return isUntypedPointerTy(T: AI->getAllocatedType());
571 return match(
572 V, P: m_AnyIntrinsic<Intrinsic::spv_alloca, Intrinsic::spv_alloca_array>());
573}
574
575// Maybe restore original function return type.
576static inline Type *restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I,
577 Type *Ty) {
578 CallInst *CI = dyn_cast<CallInst>(Val: I);
579 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
580 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
581 return Ty;
582 if (Type *OriginalTy = GR->findMutated(Val: CI->getCalledFunction()))
583 return OriginalTy;
584 return Ty;
585}
586
587// Reconstruct type with nested element types according to deduced type info.
588// Return nullptr if no detailed type info is available.
589Type *SPIRVEmitIntrinsicsImpl::reconstructType(Value *Op,
590 bool UnknownElemTypeI8,
591 bool IsPostprocessing) {
592 Type *Ty = Op->getType();
593 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
594 Ty = restoreMutatedType(GR, I: OpI, Ty);
595 if (auto It = AggrConstTypes.find(Val: OpI); It != AggrConstTypes.end())
596 Ty = It->second;
597 }
598 if (!isUntypedPointerTy(T: Ty))
599 return Ty;
600 // try to find the pointee type
601 if (Type *NestedTy = GR->findDeducedElementType(Val: Op))
602 return getTypedPointerWrapper(ElemTy: NestedTy, AS: getPointerAddressSpace(T: Ty));
603 // not a pointer according to the type info (e.g., Event object)
604 CallInst *CI = GR->findAssignPtrTypeInstr(Val: Op);
605 if (CI) {
606 MetadataAsValue *MD = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
607 return cast<ConstantAsMetadata>(Val: MD->getMetadata())->getType();
608 }
609 if (UnknownElemTypeI8) {
610 if (!IsPostprocessing)
611 insertTodoType(Op);
612 return getTypedPointerWrapper(ElemTy: IntegerType::getInt8Ty(C&: Op->getContext()),
613 AS: getPointerAddressSpace(T: Ty));
614 }
615 return nullptr;
616}
617
618CallInst *SPIRVEmitIntrinsicsImpl::buildSpvPtrcast(Function *F, Value *Op,
619 Type *ElemTy) {
620 IRBuilder<> B(*F->getParent());
621 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
622 // spv_ptrcast's argument Op denotes an instruction that generates
623 // a value, and we may use getInsertionPointAfterDef()
624 setInsertPointAfterDef(B, I: OpI);
625 } else if (auto *OpA = dyn_cast<Argument>(Val: Op)) {
626 B.SetInsertPointPastAllocas(OpA->getParent());
627 B.SetCurrentDebugLocation(DebugLoc());
628 } else {
629 B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrAlloca());
630 }
631 Type *OpTy = Op->getType();
632 SmallVector<Type *, 2> Types = {OpTy, OpTy};
633 SmallVector<Value *, 2> Args = {
634 Op, buildMD(Arg: getNormalizedPoisonValue(Ty: ElemTy, CanUseAnyVectorRank)),
635 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
636 CallInst *PtrCasted =
637 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
638 GR->buildAssignPtr(B, ElemTy, Arg: PtrCasted);
639 return PtrCasted;
640}
641
642void SPIRVEmitIntrinsicsImpl::replaceUsesOfWithSpvPtrcast(
643 Value *Op, Type *ElemTy, Instruction *I,
644 DenseMap<Function *, CallInst *> Ptrcasts) {
645 Function *F = I->getParent()->getParent();
646 CallInst *PtrCastedI = nullptr;
647 auto It = Ptrcasts.find(Val: F);
648 if (It == Ptrcasts.end()) {
649 PtrCastedI = buildSpvPtrcast(F, Op, ElemTy);
650 Ptrcasts[F] = PtrCastedI;
651 } else {
652 PtrCastedI = It->second;
653 }
654 I->replaceUsesOfWith(From: Op, To: PtrCastedI);
655}
656
657void SPIRVEmitIntrinsicsImpl::propagateElemType(
658 Value *Op, Type *ElemTy,
659 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
660 DenseMap<Function *, CallInst *> Ptrcasts;
661 SmallVector<User *> Users(Op->users());
662 for (auto *U : Users) {
663 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
664 continue;
665 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
666 continue;
667 Instruction *UI = dyn_cast<Instruction>(Val: U);
668 // If the instruction was validated already, we need to keep it valid by
669 // keeping current Op type.
670 if (isaGEP(V: UI) || TypeValidated.find(Ptr: UI) != TypeValidated.end())
671 replaceUsesOfWithSpvPtrcast(Op, ElemTy, I: UI, Ptrcasts);
672 }
673}
674
675void SPIRVEmitIntrinsicsImpl::propagateElemTypeRec(
676 Value *Op, Type *PtrElemTy, Type *CastElemTy,
677 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
678 SmallPtrSet<Value *, 0> Visited;
679 DenseMap<Function *, CallInst *> Ptrcasts;
680 propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited,
681 Ptrcasts: std::move(Ptrcasts));
682}
683
684void SPIRVEmitIntrinsicsImpl::propagateElemTypeRec(
685 Value *Op, Type *PtrElemTy, Type *CastElemTy,
686 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
687 SmallPtrSetImpl<Value *> &Visited,
688 DenseMap<Function *, CallInst *> Ptrcasts) {
689 if (!Visited.insert(Ptr: Op).second)
690 return;
691 SmallVector<User *> Users(Op->users());
692 for (auto *U : Users) {
693 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
694 continue;
695 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
696 continue;
697 Instruction *UI = dyn_cast<Instruction>(Val: U);
698 // If the instruction was validated already, we need to keep it valid by
699 // keeping current Op type.
700 if (isaGEP(V: UI) || TypeValidated.find(Ptr: UI) != TypeValidated.end())
701 replaceUsesOfWithSpvPtrcast(Op, ElemTy: CastElemTy, I: UI, Ptrcasts);
702 }
703}
704
705// Set element pointer type to the given value of ValueTy and tries to
706// specify this type further (recursively) by Operand value, if needed.
707
708Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByValueDeep(
709 Type *ValueTy, Value *Operand, bool UnknownElemTypeI8) {
710 SmallPtrSet<Value *, 0> Visited;
711 return deduceElementTypeByValueDeep(ValueTy, Operand, Visited,
712 UnknownElemTypeI8);
713}
714
715Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByValueDeep(
716 Type *ValueTy, Value *Operand, SmallPtrSetImpl<Value *> &Visited,
717 bool UnknownElemTypeI8, std::optional<unsigned> AddrSpace) {
718 Type *Ty = ValueTy;
719 if (Operand) {
720 if (auto *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
721 if (Type *NestedTy =
722 deduceElementTypeHelper(I: Operand, Visited, UnknownElemTypeI8))
723 Ty = getTypedPointerWrapper(
724 ElemTy: NestedTy, AS: AddrSpace.value_or(u: PtrTy->getAddressSpace()));
725 } else {
726 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Operand), Ty, Visited,
727 UnknownElemTypeI8);
728 }
729 }
730 return Ty;
731}
732
733// Traverse User instructions to deduce an element pointer type of the operand.
734Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByUsersDeep(
735 Value *Op, SmallPtrSetImpl<Value *> &Visited, bool UnknownElemTypeI8) {
736 if (!Op || !isPointerTy(T: Op->getType()) || isa<ConstantPointerNull>(Val: Op) ||
737 isa<UndefValue>(Val: Op))
738 return nullptr;
739
740 if (auto ElemTy = getPointeeType(Ty: Op->getType()))
741 return ElemTy;
742
743 // maybe we already know operand's element type
744 if (Type *KnownTy = GR->findDeducedElementType(Val: Op))
745 return KnownTy;
746
747 for (User *OpU : Op->users()) {
748 if (Instruction *Inst = dyn_cast<Instruction>(Val: OpU)) {
749 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8))
750 return Ty;
751 }
752 }
753 return nullptr;
754}
755
756// Implements what we know in advance about intrinsics and builtin calls
757// TODO: consider feasibility of this particular case to be generalized by
758// encoding knowledge about intrinsics and builtin calls by corresponding
759// specification rules
760static Type *getPointeeTypeByCallInst(StringRef DemangledName,
761 Function *CalledF, unsigned OpIdx) {
762 // OpenCL.std printf takes its format string as a pointer to i8. Match the
763 // bare builtin name, as lowering does, to also cover unmangled `printf`.
764 if (OpIdx == 0 && SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName) == "printf")
765 return IntegerType::getInt8Ty(C&: CalledF->getContext());
766 return nullptr;
767}
768
769// Deduce and return a successfully deduced Type of the Instruction,
770// or nullptr otherwise.
771Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeHelper(Value *I,
772 bool UnknownElemTypeI8) {
773 SmallPtrSet<Value *, 0> Visited;
774 return deduceElementTypeHelper(I, Visited, UnknownElemTypeI8);
775}
776
777void SPIRVEmitIntrinsicsImpl::maybeAssignPtrType(Type *&Ty, Value *Op,
778 Type *RefTy,
779 bool UnknownElemTypeI8) {
780 if (isUntypedPointerTy(T: RefTy)) {
781 if (!UnknownElemTypeI8)
782 return;
783 insertTodoType(Op);
784 if (isa<IntToPtrInst>(Val: Op))
785 return;
786 }
787 Ty = RefTy;
788}
789
790bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChainDynamic(
791 Type *CurType, Value *Operand, uint64_t Multiplier,
792 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
793 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing) {
794 // Dynamic indexing into a struct is not possible.
795 // We know that we must be accessing the first element
796 // of the struct if the current type is a struct.
797 // Try to find the first array type that is at offset 0 in the struct.
798 while (auto *ST = dyn_cast<StructType>(Val: CurType)) {
799 if (ST->getNumElements() == 0)
800 break;
801 CurType = ST->getElementType(N: 0);
802 OnLiteralIndexing(CurType, 0);
803 }
804
805 assert(CurType);
806 ArrayType *AT = dyn_cast<ArrayType>(Val: CurType);
807 // Operand is not constant. Either we have an array and accept it, or we
808 // give up.
809 if (AT)
810 OnDynamicIndexing(AT->getElementType(), Operand, Multiplier);
811 return AT == nullptr;
812}
813
814bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChainConstant(
815 Type *CurType, uint64_t Offset,
816 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing) {
817 auto &DL = CurrF->getDataLayout();
818
819 do {
820 if (ArrayType *AT = dyn_cast<ArrayType>(Val: CurType)) {
821 uint64_t EltTypeSize = DL.getTypeAllocSize(Ty: AT->getElementType());
822 assert(Offset < AT->getNumElements() * EltTypeSize);
823 uint64_t Index = Offset / EltTypeSize;
824 Offset = Offset - (Index * EltTypeSize);
825 CurType = AT->getElementType();
826 OnLiteralIndexing(CurType, Index);
827 } else if (StructType *ST = dyn_cast<StructType>(Val: CurType)) {
828 uint32_t StructSize = DL.getTypeSizeInBits(Ty: ST) / 8;
829 assert(Offset < StructSize);
830 (void)StructSize;
831 const auto &STL = DL.getStructLayout(Ty: ST);
832 unsigned Element = STL->getElementContainingOffset(FixedOffset: Offset);
833 Offset -= STL->getElementOffset(Idx: Element);
834 CurType = ST->getElementType(N: Element);
835 OnLiteralIndexing(CurType, Element);
836 } else if (auto *VT = dyn_cast<FixedVectorType>(Val: CurType)) {
837 Type *EltTy = VT->getElementType();
838 TypeSize EltSizeBits = DL.getTypeSizeInBits(Ty: EltTy);
839 assert(EltSizeBits % 8 == 0 &&
840 "Element type size in bits must be a multiple of 8.");
841 uint32_t EltTypeSize = EltSizeBits / 8;
842 assert(Offset < VT->getNumElements() * EltTypeSize);
843 uint64_t Index = Offset / EltTypeSize;
844 Offset -= Index * EltTypeSize;
845 CurType = EltTy;
846 OnLiteralIndexing(CurType, Index);
847 } else {
848 // Unknown composite kind; give up.
849 return true;
850 }
851 } while (Offset > 0);
852
853 return false;
854}
855
856bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChain(
857 GetElementPtrInst &GEP,
858 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
859 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing) {
860 // We only rewrite byte-addressing GEP. Other should be left as-is.
861 // Valid byte-addressing GEP must always have a single index.
862 std::optional<uint64_t> MultiplierOpt =
863 getByteAddressingMultiplier(Ty: GEP.getSourceElementType());
864 assert(MultiplierOpt && "We only rewrite byte-addressing GEP");
865 uint64_t Multiplier = *MultiplierOpt;
866 assert(GEP.getNumIndices() == 1);
867
868 Value *Src = getPointerRoot(I: GEP.getPointerOperand());
869 Type *CurType = deduceElementType(I: Src, UnknownElemTypeI8: true);
870
871 Value *Operand = *GEP.idx_begin();
872 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Operand))
873 return walkLogicalAccessChainConstant(
874 CurType, Offset: CI->getZExtValue() * Multiplier, OnLiteralIndexing);
875
876 return walkLogicalAccessChainDynamic(CurType, Operand, Multiplier,
877 OnLiteralIndexing, OnDynamicIndexing);
878}
879
880Instruction *SPIRVEmitIntrinsicsImpl::buildLogicalAccessChainFromGEP(
881 GetElementPtrInst &GEP) {
882 auto &DL = CurrF->getDataLayout();
883 IRBuilder<> B(GEP.getParent());
884 B.SetInsertPoint(&GEP);
885
886 std::vector<Value *> Indices;
887 Indices.push_back(x: ConstantInt::get(
888 Ty: IntegerType::getInt32Ty(C&: CurrF->getContext()), V: 0, /* Signed= */ IsSigned: false));
889 walkLogicalAccessChain(
890 GEP,
891 OnLiteralIndexing: [&Indices, &B](Type *EltType, uint64_t Index) {
892 Indices.push_back(
893 x: ConstantInt::get(Ty: B.getInt64Ty(), V: Index, /* Signed= */ IsSigned: false));
894 },
895 OnDynamicIndexing: [&Indices, &B, &DL, this](Type *EltType, Value *Offset,
896 uint64_t Multiplier) {
897 Value *Index = nullptr;
898 uint32_t EltTypeSize = DL.getTypeSizeInBits(Ty: EltType) / 8;
899 assert(Multiplier != 0);
900 if (Multiplier == EltTypeSize) {
901 Index = Offset;
902 } else if (EltTypeSize % Multiplier == 0) {
903 Index =
904 B.CreateUDiv(LHS: Offset, RHS: ConstantInt::get(Ty: Offset->getType(),
905 V: EltTypeSize / Multiplier,
906 /* Signed= */ IsSigned: false));
907 } else {
908 Index = B.CreateMul(LHS: Offset,
909 RHS: ConstantInt::get(Ty: Offset->getType(), V: Multiplier,
910 /* Signed= */ IsSigned: false));
911 insertAssignTypeIntrs(I: cast<Instruction>(Val: Index), B);
912 Index = B.CreateUDiv(LHS: Index,
913 RHS: ConstantInt::get(Ty: Offset->getType(), V: EltTypeSize,
914 /* Signed= */ IsSigned: false));
915 }
916 insertAssignTypeIntrs(I: cast<Instruction>(Val: Index), B);
917 Indices.push_back(x: Index);
918 });
919
920 SmallVector<Type *, 2> Types = {GEP.getType(), GEP.getOperand(i_nocapture: 0)->getType()};
921 SmallVector<Value *, 4> Args;
922 Args.push_back(Elt: B.getInt1(V: GEP.isInBounds()));
923 Args.push_back(Elt: GEP.getOperand(i_nocapture: 0));
924 llvm::append_range(C&: Args, R&: Indices);
925 Instruction *NewI =
926 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
927 replaceAllUsesWithAndErase(B, Src: &GEP, Dest: NewI);
928 return NewI;
929}
930
931Type *SPIRVEmitIntrinsicsImpl::getGEPTypeLogical(GetElementPtrInst *GEP) {
932
933 Type *CurType = GEP->getResultElementType();
934
935 bool Interrupted = walkLogicalAccessChain(
936 GEP&: *GEP, OnLiteralIndexing: [&CurType](Type *EltType, uint64_t Index) { CurType = EltType; },
937 OnDynamicIndexing: [&CurType](Type *EltType, Value *Index, uint64_t) { CurType = EltType; });
938
939 return Interrupted ? GEP->getResultElementType() : CurType;
940}
941
942Type *SPIRVEmitIntrinsicsImpl::getGEPType(GetElementPtrInst *Ref) {
943 if (getByteAddressingMultiplier(Ty: Ref->getSourceElementType()) &&
944 TM.getSubtargetImpl()->isLogicalSPIRV()) {
945 return getGEPTypeLogical(GEP: Ref);
946 }
947
948 Type *Ty = nullptr;
949 // TODO: not sure if GetElementPtrInst::getTypeAtIndex() does anything
950 // useful here
951 if (isNestedPointer(Ty: Ref->getSourceElementType())) {
952 Ty = Ref->getSourceElementType();
953 for (Use &U : drop_begin(RangeOrContainer: Ref->indices()))
954 Ty = GetElementPtrInst::getTypeAtIndex(Ty, Idx: U.get());
955 } else {
956 Ty = Ref->getResultElementType();
957 }
958 return Ty;
959}
960
961Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeHelper(
962 Value *I, SmallPtrSetImpl<Value *> &Visited, bool UnknownElemTypeI8,
963 bool IgnoreKnownType) {
964 // allow to pass nullptr as an argument
965 if (!I)
966 return nullptr;
967
968 // maybe already known
969 if (!IgnoreKnownType)
970 if (Type *KnownTy = GR->findDeducedElementType(Val: I))
971 return KnownTy;
972
973 // maybe a cycle
974 if (!Visited.insert(Ptr: I).second)
975 return nullptr;
976
977 // fallback value in case when we fail to deduce a type
978 Type *Ty = nullptr;
979 // look for known basic patterns of type inference
980 if (auto *Ref = dyn_cast<AllocaInst>(Val: I)) {
981 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getAllocatedType(), UnknownElemTypeI8);
982 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
983 Ty = getGEPType(Ref);
984 } else if (auto *SGEP = dyn_cast<StructuredGEPInst>(Val: I)) {
985 Ty = SGEP->getResultElementType();
986 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
987 Value *Op = Ref->getPointerOperand();
988 Type *KnownTy = GR->findDeducedElementType(Val: Op);
989 if (!KnownTy)
990 KnownTy = Op->getType();
991 if (Type *ElemTy = getPointeeType(Ty: KnownTy))
992 maybeAssignPtrType(Ty, Op: I, RefTy: ElemTy, UnknownElemTypeI8);
993 } else if (auto *Ref = dyn_cast<GlobalValue>(Val: I)) {
994 if (auto *Fn = dyn_cast<Function>(Val: Ref)) {
995 Ty = SPIRV::getOriginalFunctionType(F: *Fn);
996 GR->addDeducedElementType(Val: I, Ty);
997 } else {
998 Value *Op = Ref->getNumOperands() > 0 ? Ref->getOperand(i: 0) : nullptr;
999 // Code lives in the program address space, not in the address space of
1000 // the global. For known SPIR-V targets, program address space 0 maps
1001 // to Function/Private, which cannot hold code, so treat it as unset.
1002 std::optional<unsigned> ProgramAS;
1003 if (isa_and_nonnull<Function>(Val: Op)) {
1004 if (unsigned AS = CurrF->getDataLayout().getProgramAddressSpace())
1005 ProgramAS = AS;
1006 else if (TM.getSubtargetImpl()->canUseExtension(
1007 E: SPIRV::Extension::SPV_INTEL_function_pointers))
1008 ProgramAS =
1009 storageClassToAddressSpace(SC: SPIRV::StorageClass::CodeSectionINTEL);
1010 }
1011 Ty = deduceElementTypeByValueDeep(ValueTy: Ref->getValueType(), Operand: Op, Visited,
1012 UnknownElemTypeI8, AddrSpace: ProgramAS);
1013 }
1014 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1015 Type *RefTy = deduceElementTypeHelper(I: Ref->getPointerOperand(), Visited,
1016 UnknownElemTypeI8);
1017 maybeAssignPtrType(Ty, Op: I, RefTy, UnknownElemTypeI8);
1018 } else if (auto *Ref = dyn_cast<IntToPtrInst>(Val: I)) {
1019 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getDestTy(), UnknownElemTypeI8);
1020 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1021 if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
1022 isPointerTy(T: Src) && isPointerTy(T: Dest))
1023 Ty = deduceElementTypeHelper(I: Ref->getOperand(i_nocapture: 0), Visited,
1024 UnknownElemTypeI8);
1025 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1026 Value *Op = Ref->getNewValOperand();
1027 if (isPointerTy(T: Op->getType()))
1028 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
1029 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1030 Value *Op = Ref->getValOperand();
1031 if (isPointerTy(T: Op->getType()))
1032 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
1033 } else if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1034 Type *BestTy = nullptr;
1035 unsigned MaxN = 1;
1036 DenseMap<Type *, unsigned> PhiTys;
1037 for (int i = Ref->getNumIncomingValues() - 1; i >= 0; --i) {
1038 Ty = deduceElementTypeByUsersDeep(Op: Ref->getIncomingValue(i), Visited,
1039 UnknownElemTypeI8);
1040 if (!Ty)
1041 continue;
1042 auto It = PhiTys.try_emplace(Key: Ty, Args: 1);
1043 if (!It.second) {
1044 ++It.first->second;
1045 if (It.first->second > MaxN) {
1046 MaxN = It.first->second;
1047 BestTy = Ty;
1048 }
1049 }
1050 }
1051 if (BestTy)
1052 Ty = BestTy;
1053 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1054 for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) {
1055 // A function pointer operand carries its function type directly. Other
1056 // operands are deduced from their uses.
1057 Ty = isa<Function>(Val: Op)
1058 ? deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8)
1059 : deduceElementTypeByUsersDeep(Op, Visited, UnknownElemTypeI8);
1060 if (Ty)
1061 break;
1062 }
1063 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
1064 static StringMap<unsigned> ResTypeByArg = {
1065 {"to_global", 0},
1066 {"to_local", 0},
1067 {"to_private", 0},
1068 {"__to_global", 0},
1069 {"__to_local", 0},
1070 {"__to_private", 0},
1071 {"__spirv_GenericCastToPtr_ToGlobal", 0},
1072 {"__spirv_GenericCastToPtr_ToLocal", 0},
1073 {"__spirv_GenericCastToPtr_ToPrivate", 0},
1074 {"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
1075 {"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
1076 {"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
1077 // TODO: maybe improve performance by caching demangled names
1078
1079 auto *II = dyn_cast<IntrinsicInst>(Val: I);
1080 if (II && (II->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer ||
1081 II->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) {
1082 auto *HandleType = cast<TargetExtType>(Val: II->getOperand(i_nocapture: 0)->getType());
1083 if (HandleType->getTargetExtName() == "spirv.Image" ||
1084 HandleType->getTargetExtName() == "spirv.SignedImage") {
1085 for (User *U : II->users()) {
1086 Ty = cast<Instruction>(Val: U)->getAccessType();
1087 if (Ty)
1088 break;
1089 }
1090 } else if (HandleType->getTargetExtName() == "spirv.VulkanBuffer") {
1091 // This call is supposed to index into an array
1092 Ty = HandleType->getTypeParameter(i: 0);
1093 if (II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1094 if (Ty->isArrayTy())
1095 Ty = Ty->getArrayElementType();
1096 else {
1097 assert(Ty && Ty->isStructTy());
1098 uint32_t Index =
1099 cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1))->getZExtValue();
1100 Ty = cast<StructType>(Val: Ty)->getElementType(N: Index);
1101 }
1102 }
1103 Ty = reconstitutePeeledArrayType(Ty);
1104 } else {
1105 llvm_unreachable("Unknown handle type for spv_resource_getpointer.");
1106 }
1107 } else if (II && II->getIntrinsicID() ==
1108 Intrinsic::spv_generic_cast_to_ptr_explicit) {
1109 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: 0), Visited,
1110 UnknownElemTypeI8);
1111 } else if (Function *CalledF = CI->getCalledFunction()) {
1112 std::string DemangledName =
1113 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1114 if (DemangledName.length() > 0)
1115 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName);
1116 auto AsArgIt = ResTypeByArg.find(Key: DemangledName);
1117 if (AsArgIt != ResTypeByArg.end())
1118 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: AsArgIt->second),
1119 Visited, UnknownElemTypeI8);
1120 else if (Type *KnownRetTy = GR->findDeducedElementType(Val: CalledF))
1121 Ty = KnownRetTy;
1122 }
1123 }
1124
1125 // remember the found relationship
1126 if (Ty && !IgnoreKnownType) {
1127 // specify nested types if needed, otherwise return unchanged
1128 GR->addDeducedElementType(Val: I, Ty: normalizeType(Ty, CanUseAnyVectorRank));
1129 }
1130
1131 return Ty;
1132}
1133
1134// Re-create a type of the value if it has untyped pointer fields, also nested.
1135// Return the original value type if no corrections of untyped pointer
1136// information is found or needed.
1137Type *SPIRVEmitIntrinsicsImpl::deduceNestedTypeHelper(User *U,
1138 bool UnknownElemTypeI8) {
1139 SmallPtrSet<Value *, 0> Visited;
1140 return deduceNestedTypeHelper(U, Ty: U->getType(), Visited, UnknownElemTypeI8);
1141}
1142
1143Type *SPIRVEmitIntrinsicsImpl::deduceNestedTypeHelper(
1144 User *U, Type *OrigTy, SmallPtrSetImpl<Value *> &Visited,
1145 bool UnknownElemTypeI8) {
1146 if (!U)
1147 return OrigTy;
1148
1149 // maybe already known
1150 if (Type *KnownTy = GR->findDeducedCompositeType(Val: U))
1151 return KnownTy;
1152
1153 // maybe a cycle
1154 if (!Visited.insert(Ptr: U).second)
1155 return OrigTy;
1156
1157 if (auto *OrigStructTy = dyn_cast<StructType>(Val: OrigTy)) {
1158 SmallVector<Type *> Tys;
1159 bool Change = false;
1160 for (unsigned i = 0; i < U->getNumOperands(); ++i) {
1161 Value *Op = U->getOperand(i);
1162 assert(Op && "Operands should not be null.");
1163 Type *OpTy = Op->getType();
1164 Type *Ty = OpTy;
1165 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1166 if (Type *NestedTy =
1167 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1168 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1169 } else {
1170 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1171 UnknownElemTypeI8);
1172 }
1173 Tys.push_back(Elt: Ty);
1174 Change |= Ty != OpTy;
1175 }
1176 if (Change) {
1177 Type *NewTy = StructType::create(
1178 Elements: Tys, Name: OrigStructTy->isLiteral() ? "" : OrigStructTy->getName(),
1179 isPacked: OrigStructTy->isPacked());
1180 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1181 return NewTy;
1182 }
1183 } else if (auto *ArrTy = dyn_cast<ArrayType>(Val: OrigTy)) {
1184 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1185 Type *OpTy = ArrTy->getElementType();
1186 Type *Ty = OpTy;
1187 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1188 if (Type *NestedTy =
1189 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1190 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1191 } else {
1192 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1193 UnknownElemTypeI8);
1194 }
1195 if (Ty != OpTy) {
1196 Type *NewTy = ArrayType::get(ElementType: Ty, NumElements: ArrTy->getNumElements());
1197 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1198 return NewTy;
1199 }
1200 }
1201 } else if (auto *VecTy = dyn_cast<VectorType>(Val: OrigTy)) {
1202 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1203 Type *OpTy = VecTy->getElementType();
1204 Type *Ty = OpTy;
1205 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1206 if (Type *NestedTy =
1207 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1208 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1209 } else {
1210 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1211 UnknownElemTypeI8);
1212 }
1213 if (Ty != OpTy) {
1214 Type *NewTy = VectorType::get(ElementType: Ty, EC: VecTy->getElementCount());
1215 GR->addDeducedCompositeType(Val: U,
1216 Ty: normalizeType(Ty: NewTy, CanUseAnyVectorRank));
1217 return NewTy;
1218 }
1219 }
1220 }
1221
1222 return OrigTy;
1223}
1224
1225Type *SPIRVEmitIntrinsicsImpl::deduceElementType(Value *I,
1226 bool UnknownElemTypeI8) {
1227 if (Type *Ty = deduceElementTypeHelper(I, UnknownElemTypeI8))
1228 return Ty;
1229 if (!UnknownElemTypeI8)
1230 return nullptr;
1231 insertTodoType(Op: I);
1232 return IntegerType::getInt8Ty(C&: I->getContext());
1233}
1234
1235static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I,
1236 Value *PointerOperand) {
1237 Type *PointeeTy = GR->findDeducedElementType(Val: PointerOperand);
1238 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1239 return nullptr;
1240 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType());
1241 if (!PtrTy)
1242 return I->getType();
1243 if (Type *NestedTy = GR->findDeducedElementType(Val: I))
1244 return getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1245 return nullptr;
1246}
1247
1248// Try to deduce element type for a call base. Returns false if this is an
1249// indirect function invocation, and true otherwise.
1250bool SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeCalledFunction(
1251 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1252 Type *&KnownElemTy, bool &Incomplete) {
1253 Function *CalledF = CI->getCalledFunction();
1254 if (!CalledF)
1255 return false;
1256 std::string DemangledName =
1257 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1258 if (DemangledName.length() > 0 &&
1259 !StringRef(DemangledName).starts_with(Prefix: "llvm.")) {
1260 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *CalledF);
1261 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
1262 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
1263 if (Opcode == SPIRV::OpGroupAsyncCopy) {
1264 for (unsigned i = 0, PtrCnt = 0; i < CI->arg_size() && PtrCnt < 2; ++i) {
1265 Value *Op = CI->getArgOperand(i);
1266 if (!isPointerTy(T: Op->getType()))
1267 continue;
1268 ++PtrCnt;
1269 if (Type *ElemTy = GR->findDeducedElementType(Val: Op))
1270 KnownElemTy = ElemTy; // src will rewrite dest if both are defined
1271 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1272 }
1273 } else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) {
1274 if (CI->arg_size() == 0)
1275 return true;
1276 Value *Op = CI->getArgOperand(i: 0);
1277 if (!isPointerTy(T: Op->getType()))
1278 return true;
1279 switch (Opcode) {
1280 case SPIRV::OpAtomicFAddEXT:
1281 case SPIRV::OpAtomicFMinEXT:
1282 case SPIRV::OpAtomicFMaxEXT:
1283 case SPIRV::OpAtomicLoad:
1284 case SPIRV::OpAtomicCompareExchangeWeak:
1285 case SPIRV::OpAtomicCompareExchange:
1286 case SPIRV::OpAtomicExchange:
1287 case SPIRV::OpAtomicIAdd:
1288 case SPIRV::OpAtomicISub:
1289 case SPIRV::OpAtomicOr:
1290 case SPIRV::OpAtomicXor:
1291 case SPIRV::OpAtomicAnd:
1292 case SPIRV::OpAtomicUMin:
1293 case SPIRV::OpAtomicUMax:
1294 case SPIRV::OpAtomicSMin:
1295 case SPIRV::OpAtomicSMax: {
1296 KnownElemTy = isPointerTy(T: CI->getType()) ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1297 : CI->getType();
1298 if (!KnownElemTy)
1299 return true;
1300 Incomplete = isTodoType(Op);
1301 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1302 } break;
1303 case SPIRV::OpAtomicStore: {
1304 if (CI->arg_size() < 4)
1305 return true;
1306 Value *ValOp = CI->getArgOperand(i: 3);
1307 KnownElemTy = isPointerTy(T: ValOp->getType())
1308 ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1309 : ValOp->getType();
1310 if (!KnownElemTy)
1311 return true;
1312 Incomplete = isTodoType(Op);
1313 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1314 } break;
1315 }
1316 }
1317 }
1318 return true;
1319}
1320
1321// Try to deduce element type for a function pointer.
1322void SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeFunctionPointer(
1323 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1324 Type *&KnownElemTy, bool IsPostprocessing) {
1325 Value *Op = CI->getCalledOperand();
1326 if (!Op || !isPointerTy(T: Op->getType()))
1327 return;
1328 Ops.push_back(Elt: std::make_pair(x&: Op, y: std::numeric_limits<unsigned>::max()));
1329 FunctionType *FTy = SPIRV::getOriginalFunctionType(CB: *CI);
1330 bool IsNewFTy = false, IsIncomplete = false;
1331 SmallVector<Type *, 4> ArgTys;
1332 for (auto &&[ParmIdx, Arg] : llvm::enumerate(First: CI->args())) {
1333 Type *ArgTy = Arg->getType();
1334 if (ArgTy->isPointerTy()) {
1335 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg)) {
1336 IsNewFTy = true;
1337 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
1338 if (isTodoType(Op: Arg))
1339 IsIncomplete = true;
1340 } else {
1341 IsIncomplete = true;
1342 }
1343 } else {
1344 ArgTy = FTy->getFunctionParamType(i: ParmIdx);
1345 }
1346 ArgTys.push_back(Elt: ArgTy);
1347 }
1348 Type *RetTy = FTy->getReturnType();
1349 if (CI->getType()->isPointerTy()) {
1350 if (Type *ElemTy = GR->findDeducedElementType(Val: CI)) {
1351 IsNewFTy = true;
1352 RetTy =
1353 getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: CI->getType()));
1354 if (isTodoType(Op: CI))
1355 IsIncomplete = true;
1356 } else {
1357 IsIncomplete = true;
1358 }
1359 }
1360 if (!IsPostprocessing && IsIncomplete)
1361 insertTodoType(Op);
1362 KnownElemTy =
1363 IsNewFTy ? FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: FTy->isVarArg()) : FTy;
1364}
1365
1366bool SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeFunctionRet(
1367 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
1368 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing,
1369 Type *&KnownElemTy, Value *Op, Function *F) {
1370 KnownElemTy = GR->findDeducedElementType(Val: F);
1371 if (KnownElemTy)
1372 return false;
1373 if (Type *OpElemTy = GR->findDeducedElementType(Val: Op)) {
1374 OpElemTy = normalizeType(Ty: OpElemTy, CanUseAnyVectorRank);
1375 GR->addDeducedElementType(Val: F, Ty: OpElemTy);
1376 GR->addReturnType(
1377 ArgF: F, DerivedTy: TypedPointerType::get(ElementType: OpElemTy,
1378 AddressSpace: getPointerAddressSpace(T: F->getReturnType())));
1379 // non-recursive update of types in function uses
1380 DenseSet<std::pair<Value *, Value *>> VisitedSubst{std::make_pair(x&: I, y&: Op)};
1381 for (User *U : F->users()) {
1382 CallInst *CI = dyn_cast<CallInst>(Val: U);
1383 if (!CI || CI->getCalledFunction() != F)
1384 continue;
1385 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: CI)) {
1386 if (Type *PrevElemTy = GR->findDeducedElementType(Val: CI)) {
1387 GR->updateAssignType(
1388 AssignCI, Arg: CI,
1389 OfType: getNormalizedPoisonValue(Ty: OpElemTy, CanUseAnyVectorRank));
1390 propagateElemType(Op: CI, ElemTy: PrevElemTy, VisitedSubst);
1391 }
1392 }
1393 }
1394 // Non-recursive update of types in the function uncomplete returns.
1395 // This may happen just once per a function, the latch is a pair of
1396 // findDeducedElementType(F) / addDeducedElementType(F, ...).
1397 // With or without the latch it is a non-recursive call due to
1398 // IncompleteRets set to nullptr in this call.
1399 if (IncompleteRets)
1400 for (Instruction *IncompleteRetI : *IncompleteRets)
1401 deduceOperandElementType(I: IncompleteRetI, IncompleteRets: nullptr, AskOps,
1402 IsPostprocessing);
1403 } else if (IncompleteRets) {
1404 IncompleteRets->insert(Ptr: I);
1405 }
1406 TypeValidated.insert(Ptr: I);
1407 return true;
1408}
1409
1410// If the Instruction has Pointer operands with unresolved types, this function
1411// tries to deduce them. If the Instruction has Pointer operands with known
1412// types which differ from expected, this function tries to insert a bitcast to
1413// resolve the issue.
1414void SPIRVEmitIntrinsicsImpl::deduceOperandElementType(
1415 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
1416 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing) {
1417 SmallVector<std::pair<Value *, unsigned>> Ops;
1418 Type *KnownElemTy = nullptr;
1419 bool Incomplete = false;
1420 // look for known basic patterns of type inference
1421 if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1422 if (!isPointerTy(T: I->getType()) ||
1423 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1424 return;
1425 Incomplete = isTodoType(Op: I);
1426 for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
1427 Value *Op = Ref->getIncomingValue(i);
1428 if (isPointerTy(T: Op->getType()))
1429 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1430 }
1431 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1432 KnownElemTy = GR->findDeducedElementType(Val: I);
1433 if (!KnownElemTy)
1434 return;
1435 Incomplete = isTodoType(Op: I);
1436 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(), y: 0));
1437 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1438 if (!isPointerTy(T: I->getType()))
1439 return;
1440 KnownElemTy = GR->findDeducedElementType(Val: I);
1441 if (!KnownElemTy)
1442 return;
1443 Incomplete = isTodoType(Op: I);
1444 Ops.push_back(Elt: std::make_pair(x: Ref->getOperand(i_nocapture: 0), y: 0));
1445 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
1446 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1447 return;
1448 KnownElemTy = Ref->getSourceElementType();
1449 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1450 y: GetElementPtrInst::getPointerOperandIndex()));
1451 } else if (auto *Ref = dyn_cast<StructuredGEPInst>(Val: I)) {
1452 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1453 return;
1454 KnownElemTy = Ref->getBaseType();
1455 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1456 y: StructuredGEPInst::getPointerOperandIndex()));
1457 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
1458 KnownElemTy = I->getType();
1459 if (isUntypedPointerTy(T: KnownElemTy)) {
1460 // A T** loaded back from its alloca comes out opaque, dropping type info.
1461 // When the load is a pointer-to-pointer, type the alloca as that pointer.
1462 Type *LoadedElemTy = GR->findDeducedElementType(Val: I);
1463 if (!LoadedElemTy || !isPointerTyOrWrapper(Ty: LoadedElemTy))
1464 return;
1465 Value *Root = Ref->getPointerOperand()->stripPointerCasts();
1466 if (!isa<AllocaInst>(Val: Root))
1467 return;
1468 KnownElemTy = getTypedPointerWrapper(ElemTy: LoadedElemTy,
1469 AS: getPointerAddressSpace(T: KnownElemTy));
1470 }
1471 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1472 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1473 return;
1474 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1475 y: LoadInst::getPointerOperandIndex()));
1476 } else if (auto *Ref = dyn_cast<StoreInst>(Val: I)) {
1477 if (!(KnownElemTy =
1478 reconstructType(Op: Ref->getValueOperand(), UnknownElemTypeI8: false, IsPostprocessing)))
1479 return;
1480 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1481 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1482 return;
1483 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1484 y: StoreInst::getPointerOperandIndex()));
1485 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1486 KnownElemTy = isPointerTy(T: I->getType())
1487 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1488 : I->getType();
1489 if (!KnownElemTy)
1490 return;
1491 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1492 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1493 y: AtomicCmpXchgInst::getPointerOperandIndex()));
1494 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1495 KnownElemTy = isPointerTy(T: I->getType())
1496 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1497 : I->getType();
1498 if (!KnownElemTy)
1499 return;
1500 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1501 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1502 y: AtomicRMWInst::getPointerOperandIndex()));
1503 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1504 if (!isPointerTy(T: I->getType()) ||
1505 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1506 return;
1507 Incomplete = isTodoType(Op: I);
1508 for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
1509 Value *Op = Ref->getOperand(i_nocapture: i);
1510 if (isPointerTy(T: Op->getType()))
1511 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1512 }
1513 } else if (auto *Ref = dyn_cast<ReturnInst>(Val: I)) {
1514 if (!isPointerTy(T: CurrF->getReturnType()))
1515 return;
1516 Value *Op = Ref->getReturnValue();
1517 if (!Op)
1518 return;
1519 if (deduceOperandElementTypeFunctionRet(I, IncompleteRets, AskOps,
1520 IsPostprocessing, KnownElemTy, Op,
1521 F: CurrF))
1522 return;
1523 Incomplete = isTodoType(Op: CurrF);
1524 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1525 } else if (auto *Ref = dyn_cast<ICmpInst>(Val: I)) {
1526 if (!isPointerTy(T: Ref->getOperand(i_nocapture: 0)->getType()))
1527 return;
1528 Value *Op0 = Ref->getOperand(i_nocapture: 0);
1529 Value *Op1 = Ref->getOperand(i_nocapture: 1);
1530 bool Incomplete0 = isTodoType(Op: Op0);
1531 bool Incomplete1 = isTodoType(Op: Op1);
1532 Type *ElemTy1 = GR->findDeducedElementType(Val: Op1);
1533 Type *ElemTy0 = (Incomplete0 && !Incomplete1 && ElemTy1)
1534 ? nullptr
1535 : GR->findDeducedElementType(Val: Op0);
1536 if (ElemTy0) {
1537 KnownElemTy = ElemTy0;
1538 Incomplete = Incomplete0;
1539 Ops.push_back(Elt: std::make_pair(x&: Op1, y: 1));
1540 } else if (ElemTy1) {
1541 KnownElemTy = ElemTy1;
1542 Incomplete = Incomplete1;
1543 Ops.push_back(Elt: std::make_pair(x&: Op0, y: 0));
1544 }
1545 } else if (CallInst *CI = dyn_cast<CallInst>(Val: I)) {
1546 if (!CI->isIndirectCall())
1547 deduceOperandElementTypeCalledFunction(CI, Ops, KnownElemTy, Incomplete);
1548 else if (HaveFunPtrs)
1549 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy,
1550 IsPostprocessing);
1551 }
1552
1553 // There is no enough info to deduce types or all is valid.
1554 if (!KnownElemTy || Ops.size() == 0)
1555 return;
1556
1557 IRBuilder<> B(*CurrF->getParent());
1558 for (auto &OpIt : Ops) {
1559 Value *Op = OpIt.first;
1560 if (AskOps && !AskOps->contains(Ptr: Op))
1561 continue;
1562 Type *AskTy = nullptr;
1563 CallInst *AskCI = nullptr;
1564 if (IsPostprocessing && AskOps) {
1565 AskTy = GR->findDeducedElementType(Val: Op);
1566 AskCI = GR->findAssignPtrTypeInstr(Val: Op);
1567 assert(AskTy && AskCI);
1568 }
1569 Type *Ty = AskTy ? AskTy : GR->findDeducedElementType(Val: Op);
1570 if (Ty == KnownElemTy)
1571 continue;
1572 Value *OpTyVal = getNormalizedPoisonValue(Ty: KnownElemTy, CanUseAnyVectorRank);
1573 Type *OpTy = Op->getType();
1574 // Do not let a non-pointer element type clobber an already-deduced pointer
1575 // element type for the same operand.
1576 bool WouldClobberPtrWithNonPtr = Ty && isPointerTyOrWrapper(Ty) &&
1577 !isPointerTyOrWrapper(Ty: KnownElemTy) &&
1578 tracesToPointerAlloca(V: Op);
1579 if (Op->hasUseList() && !WouldClobberPtrWithNonPtr &&
1580 (!Ty || AskTy || isUntypedPointerTy(T: Ty) || isTodoType(Op))) {
1581 Type *PrevElemTy = GR->findDeducedElementType(Val: Op);
1582 GR->addDeducedElementType(
1583 Val: Op, Ty: normalizeType(Ty: KnownElemTy, CanUseAnyVectorRank));
1584 // check if KnownElemTy is complete
1585 if (!Incomplete)
1586 eraseTodoType(Op);
1587 else if (!IsPostprocessing)
1588 insertTodoType(Op);
1589 // check if there is existing Intrinsic::spv_assign_ptr_type instruction
1590 CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Val: Op);
1591 if (AssignCI == nullptr) {
1592 Instruction *User = dyn_cast<Instruction>(Val: Op->use_begin()->get());
1593 setInsertPointSkippingPhis(B, I: User ? User->getNextNode() : I);
1594 CallInst *CI =
1595 buildIntrWithMD(IntrID: Intrinsic::spv_assign_ptr_type, Types: {OpTy}, Arg: OpTyVal, Arg2: Op,
1596 Imms: {B.getInt32(C: getPointerAddressSpace(T: OpTy))}, B);
1597 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: CI);
1598 } else {
1599 GR->updateAssignType(AssignCI, Arg: Op, OfType: OpTyVal);
1600 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1601 std::make_pair(x&: I, y&: Op)};
1602 propagateElemTypeRec(Op, PtrElemTy: KnownElemTy, CastElemTy: PrevElemTy, VisitedSubst);
1603 }
1604 } else {
1605 eraseTodoType(Op);
1606 CallInst *PtrCastI =
1607 buildSpvPtrcast(F: I->getParent()->getParent(), Op, ElemTy: KnownElemTy);
1608 if (OpIt.second == std::numeric_limits<unsigned>::max())
1609 dyn_cast<CallInst>(Val: I)->setCalledOperand(PtrCastI);
1610 else
1611 I->setOperand(i: OpIt.second, Val: PtrCastI);
1612 }
1613 }
1614 TypeValidated.insert(Ptr: I);
1615}
1616
1617void SPIRVEmitIntrinsicsImpl::replaceMemInstrUses(Instruction *Old,
1618 Instruction *New,
1619 IRBuilder<> &B) {
1620 while (!Old->user_empty()) {
1621 auto *U = Old->user_back();
1622 if (isAssignTypeInstr(I: U)) {
1623 B.SetInsertPoint(U);
1624 SmallVector<Value *, 2> Args = {New, U->getOperand(i: 1)};
1625 CallInst *AssignCI = B.CreateIntrinsicWithoutFolding(
1626 ID: Intrinsic::spv_assign_type, OverloadTypes: {New->getType()}, Args);
1627 GR->addAssignPtrTypeInstr(Val: New, AssignPtrTyCI: AssignCI);
1628 U->eraseFromParent();
1629 } else if (isMemInstrToReplace(I: U) || isa<ReturnInst>(Val: U) ||
1630 isa<CallInst>(Val: U)) {
1631 U->replaceUsesOfWith(From: Old, To: New);
1632 // For a `llvm.spv.abort` call whose composite message argument was
1633 // rewritten to a value-id (i32), also retarget the call to a matching
1634 // intrinsic declaration so the IR verifier is satisfied. The SPIR-V
1635 // type of the value is tracked via the GlobalRegistry, so the selector
1636 // still emits OpAbortKHR with the original composite type.
1637 if (auto *CI = dyn_cast<CallInst>(Val: U);
1638 CI && CI->getIntrinsicID() == Intrinsic::spv_abort) {
1639 Type *NewArgTy = New->getType();
1640 Type *ExpectedArgTy = CI->getFunctionType()->getParamType(i: 0);
1641 if (NewArgTy != ExpectedArgTy) {
1642 Module *M = CI->getModule();
1643 Function *NewF = Intrinsic::getOrInsertDeclaration(
1644 M, id: Intrinsic::spv_abort, OverloadTys: {NewArgTy});
1645 CI->setCalledFunction(NewF);
1646 }
1647 }
1648 } else if (isa<PHINode>(Val: U) || isa<SelectInst>(Val: U) || isa<FreezeInst>(Val: U)) {
1649 // Aggregate-typed PHIs, selects and freezes have already been mutated to
1650 // the i32 value-id type up front in runOnFunction, so only the operand
1651 // needs replacing here; their extractvalue users are lowered to
1652 // spv_extractv by visitExtractValueInst.
1653 assert(U->getType() == New->getType() &&
1654 "aggregate PHI/select/freeze should have been mutated to value-id "
1655 "type");
1656 U->replaceUsesOfWith(From: Old, To: New);
1657 } else {
1658 llvm_unreachable("illegal aggregate intrinsic user");
1659 }
1660 }
1661 New->copyMetadata(SrcInst: *Old);
1662 Old->eraseFromParent();
1663}
1664
1665// Lower a poison or undef Op to its placeholder intrinsic.
1666Value *SPIRVEmitIntrinsicsImpl::lowerUndefOrPoison(Value *Op, IRBuilder<> &B,
1667 bool HasPoisonExt) {
1668 auto *UV = dyn_cast<UndefValue>(Val: Op);
1669 if (!UV)
1670 return nullptr;
1671
1672 bool AsPoison = HasPoisonExt && isa<PoisonValue>(Val: UV);
1673 if (isa<PoisonValue>(Val: UV) && !HasPoisonExt)
1674 LLVM_DEBUG(dbgs() << "SPV_KHR_poison_freeze is not enabled. Poison is "
1675 "lowered as undef\n");
1676
1677 Intrinsic::ID IID = AsPoison ? Intrinsic::spv_poison : Intrinsic::spv_undef;
1678 Type *Ty = UV->getType();
1679
1680 // Aggregates use an i32-result placeholder with the real type kept in
1681 // AggrConstTypes and scalar poison uses a type-overloaded one.
1682 if (Ty->isAggregateType()) {
1683 auto *Call =
1684 AsPoison ? B.CreateIntrinsicWithoutFolding(ID: IID, OverloadTypes: {B.getInt32Ty()}, Args: {})
1685 : B.CreateIntrinsicWithoutFolding(ID: IID, Args: {});
1686 AggrConsts[Call] = UV;
1687 AggrConstTypes[Call] = Ty;
1688 return Call;
1689 }
1690
1691 if (AsPoison)
1692 return B.CreateIntrinsic(ID: IID, OverloadTypes: {Ty}, Args: {});
1693 return nullptr;
1694}
1695
1696// Replace aggregate undef or poison operands and extension-enabled scalar
1697// poison operands with placeholder intrinsics. Scalar undef is left as is. See
1698// lowerUndefOrPoison.
1699void SPIRVEmitIntrinsicsImpl::preprocessUndefsAndPoisons(IRBuilder<> &B) {
1700 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*CurrF);
1701 bool HasPoisonExt =
1702 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
1703
1704 SmallVector<Instruction *, 16> Insts;
1705 for (auto &I : instructions(F: CurrF))
1706 Insts.push_back(Elt: &I);
1707
1708 for (Instruction *I : Insts) {
1709 bool BPrepared = false;
1710 auto *Phi = dyn_cast<PHINode>(Val: I);
1711 for (unsigned Idx = 0; Idx < I->getNumOperands(); ++Idx) {
1712 Value *Op = I->getOperand(i: Idx);
1713 if (!isa<UndefValue>(Val: Op) || Op->getType()->isMetadataTy())
1714 continue;
1715 bool IsScalar = !Op->getType()->isAggregateType();
1716 bool AsPoison = HasPoisonExt && isa<PoisonValue>(Val: Op);
1717 // Scalar undef or extensionless scalar poison is directly translatable.
1718 if (IsScalar && !AsPoison)
1719 continue;
1720 // Scalar poison in a phi materializes in the incoming block. Everything
1721 // else materializes right before I.
1722 if (IsScalar && Phi)
1723 B.SetInsertPoint(Phi->getIncomingBlock(i: Idx)->getTerminator());
1724 else if (!BPrepared) {
1725 setInsertPointSkippingPhis(B, I);
1726 BPrepared = true;
1727 }
1728 if (Value *Repl = lowerUndefOrPoison(Op, B, HasPoisonExt))
1729 I->setOperand(i: Idx, Val: Repl);
1730 }
1731 }
1732}
1733
1734// Simplify addrspacecast(null) instructions to ConstantPointerNull of the
1735// target type. Casting null always yields null, and this avoids SPIR-V
1736// lowering issues where the null gets typed as an integer instead of a
1737// pointer.
1738void SPIRVEmitIntrinsicsImpl::simplifyNullAddrSpaceCasts() {
1739 for (Instruction &I : make_early_inc_range(Range: instructions(F: CurrF)))
1740 if (auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: &I))
1741 if (isa<ConstantPointerNull>(Val: ASC->getPointerOperand())) {
1742 ASC->replaceAllUsesWith(
1743 V: ConstantPointerNull::get(T: cast<PointerType>(Val: ASC->getType())));
1744 ASC->eraseFromParent();
1745 }
1746}
1747
1748// True for an aggregate value the legalizer splits into a multi-result op
1749// (with.overflow -> G_UADDO, frexp/sincos/modf -> G_FFREXP/...). These keep a
1750// genuine multi-register result; all other aggregates become a single value-id.
1751static bool isMultiRegisterAggregate(Value *V) {
1752 if (!V->getType()->isAggregateType())
1753 return false;
1754 return isa<IntrinsicInst>(Val: V) && !isSpvIntrinsic(Arg: V);
1755}
1756
1757// True for an aggregate PHI/select/freeze, which is lowered to a single
1758// value-id.
1759static bool isAggregateValueIdInstr(const Instruction &I) {
1760 return (isa<PHINode>(Val: I) || isa<SelectInst>(Val: I) || isa<FreezeInst>(Val: I)) &&
1761 I.getType()->isAggregateType();
1762}
1763
1764// Give each multi-register aggregate arm of an aggregate PHI/select/freeze a
1765// single value-id by reassembling it with extractvalue + insertvalue, so the
1766// arm matches the result once it is mutated to a value-id.
1767void SPIRVEmitIntrinsicsImpl::insertCompositeAggregateArms(Instruction *I,
1768 IRBuilder<> &B) {
1769 auto *Phi = dyn_cast<PHINode>(Val: I);
1770 for (Use &U : I->operands()) {
1771 Value *Op = U.get();
1772 if (!isMultiRegisterAggregate(V: Op))
1773 continue;
1774 // A PHI arm materializes in its incoming block, everything else after the
1775 // producer.
1776 if (Phi)
1777 B.SetInsertPoint(Phi->getIncomingBlock(U)->getTerminator());
1778 else
1779 setInsertPointAfterDef(B, I: cast<Instruction>(Val: Op));
1780 auto *AggrTy = cast<StructType>(Val: Op->getType());
1781 Value *Composite = PoisonValue::get(T: AggrTy);
1782 for (unsigned Idx = 0, E = AggrTy->getNumElements(); Idx != E; ++Idx) {
1783 Value *Field = B.CreateExtractValue(Agg: Op, Idxs: Idx);
1784 Composite = B.CreateInsertValue(Agg: Composite, Val: Field, Idxs: Idx);
1785 }
1786 U.set(Composite);
1787 }
1788}
1789
1790void SPIRVEmitIntrinsicsImpl::preprocessCompositeConstants(IRBuilder<> &B) {
1791 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*CurrF);
1792 bool HasPoisonExt =
1793 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
1794 std::queue<Instruction *> Worklist;
1795 for (auto &I : instructions(F: CurrF))
1796 Worklist.push(x: &I);
1797
1798 while (!Worklist.empty()) {
1799 auto *I = Worklist.front();
1800 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
1801 assert(I);
1802 bool KeepInst = false;
1803 for (const auto &Op : I->operands()) {
1804 Constant *AggrConst = nullptr;
1805 Type *ResTy = nullptr;
1806 if (auto *COp = dyn_cast<ConstantVector>(Val: Op)) {
1807 AggrConst = COp;
1808 ResTy = COp->getType();
1809 } else if (auto *COp = dyn_cast<ConstantArray>(Val: Op)) {
1810 AggrConst = COp;
1811 ResTy = B.getInt32Ty();
1812 } else if (auto *COp = dyn_cast<ConstantStruct>(Val: Op)) {
1813 AggrConst = COp;
1814 ResTy = B.getInt32Ty();
1815 } else if (auto *COp = dyn_cast<ConstantDataArray>(Val: Op)) {
1816 AggrConst = COp;
1817 ResTy = B.getInt32Ty();
1818 } else if (auto *COp = dyn_cast<ConstantAggregateZero>(Val: Op)) {
1819 AggrConst = COp;
1820 ResTy = Op->getType()->isVectorTy() ? COp->getType() : B.getInt32Ty();
1821 }
1822 if (AggrConst) {
1823 auto PrepareInsert = [&]() {
1824 if (BPrepared)
1825 return;
1826 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
1827 : B.SetInsertPoint(I);
1828 BPrepared = true;
1829 };
1830 SmallVector<Value *> Args;
1831 if (auto *COp = dyn_cast<ConstantDataSequential>(Val: Op))
1832 for (unsigned i = 0; i < COp->getNumElements(); ++i)
1833 Args.push_back(Elt: COp->getElementAsConstant(i));
1834 else
1835 for (Value *Op : AggrConst->operands()) {
1836 // Simplify addrspacecast(null) to null in the target address space
1837 // so that null pointers get the correct pointer type when lowered.
1838 if (auto *CE = dyn_cast<ConstantExpr>(Val: Op);
1839 CE && CE->getOpcode() == Instruction::AddrSpaceCast &&
1840 isa<ConstantPointerNull>(Val: CE->getOperand(i_nocapture: 0)))
1841 Op = ConstantPointerNull::get(T: cast<PointerType>(Val: CE->getType()));
1842 // Undef or poison nested in a constant aggregate is not a direct
1843 // instruction operand, so preprocessUndefsAndPoisons() misses it.
1844 // An unlowered aggregate one would reach IRTranslator as an
1845 // untranslatable spv_const_composite operand.
1846 if (isa<UndefValue>(Val: Op)) {
1847 PrepareInsert();
1848 if (Value *Repl = lowerUndefOrPoison(Op, B, HasPoisonExt))
1849 Op = Repl;
1850 }
1851 Args.push_back(Elt: Op);
1852 }
1853 PrepareInsert();
1854 auto *CI = B.CreateIntrinsicWithoutFolding(
1855 ID: Intrinsic::spv_const_composite, OverloadTypes: {ResTy}, Args: {Args});
1856 Worklist.push(x: CI);
1857 I->replaceUsesOfWith(From: Op, To: CI);
1858 KeepInst = true;
1859 AggrConsts[CI] = AggrConst;
1860 AggrConstTypes[CI] = deduceNestedTypeHelper(U: AggrConst, UnknownElemTypeI8: false);
1861 }
1862 }
1863 if (!KeepInst)
1864 Worklist.pop();
1865 }
1866}
1867
1868static void createDecorationIntrinsic(Instruction *I, MDNode *Node,
1869 IRBuilder<> &B) {
1870 LLVMContext &Ctx = I->getContext();
1871 setInsertPointAfterDef(B, I);
1872 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
1873 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: {Node}))});
1874}
1875
1876static void createRoundingModeDecoration(Instruction *I,
1877 unsigned RoundingModeDeco,
1878 IRBuilder<> &B) {
1879 LLVMContext &Ctx = I->getContext();
1880 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1881 MDNode *RoundingModeNode = MDNode::get(
1882 Context&: Ctx,
1883 MDs: {ConstantAsMetadata::get(
1884 C: ConstantInt::get(Ty: Int32Ty, V: SPIRV::Decoration::FPRoundingMode)),
1885 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: RoundingModeDeco))});
1886 createDecorationIntrinsic(I, Node: RoundingModeNode, B);
1887}
1888
1889static void createSaturatedConversionDecoration(Instruction *I,
1890 IRBuilder<> &B) {
1891 LLVMContext &Ctx = I->getContext();
1892 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1893 MDNode *SaturatedConversionNode =
1894 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1895 Ty: Int32Ty, V: SPIRV::Decoration::SaturatedConversion))});
1896 createDecorationIntrinsic(I, Node: SaturatedConversionNode, B);
1897}
1898
1899static void addSaturatedDecorationToIntrinsic(Instruction *I, IRBuilder<> &B) {
1900 if (match(V: I, P: m_AnyIntrinsic<Intrinsic::fptosi_sat, Intrinsic::fptoui_sat>()))
1901 createSaturatedConversionDecoration(I, B);
1902}
1903
1904Instruction *SPIRVEmitIntrinsicsImpl::visitCallInst(CallInst &Call) {
1905 if (!Call.isInlineAsm())
1906 return &Call;
1907
1908 LLVMContext &Ctx = CurrF->getContext();
1909 // TODO: this does not retain elementtype info for memory constraints, which
1910 // in turn means that we lower them into pointers to i8, rather than
1911 // pointers to elementtype; this can be fixed during reverse translation
1912 // but we should correct it here, possibly by tweaking the function
1913 // type to take TypedPointerType args.
1914 Constant *TyC = UndefValue::get(T: SPIRV::getOriginalFunctionType(CB: Call));
1915 MDString *ConstraintString =
1916 MDString::get(Context&: Ctx, Str: SPIRV::getOriginalAsmConstraints(CB: Call));
1917 SmallVector<Value *> Args = {
1918 buildMD(Arg: TyC),
1919 MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: ConstraintString))};
1920 for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++)
1921 Args.push_back(Elt: Call.getArgOperand(i: OpIdx));
1922
1923 IRBuilder<> B(Call.getParent());
1924 B.SetInsertPoint(&Call);
1925 B.CreateIntrinsic(ID: Intrinsic::spv_inline_asm, Args: {Args});
1926 return &Call;
1927}
1928
1929// Use a tip about rounding mode to create a decoration.
1930void SPIRVEmitIntrinsicsImpl::useRoundingMode(ConstrainedFPIntrinsic *FPI,
1931 IRBuilder<> &B) {
1932 std::optional<RoundingMode> RM = FPI->getRoundingMode();
1933 if (!RM.has_value())
1934 return;
1935 unsigned RoundingModeDeco = std::numeric_limits<unsigned>::max();
1936 switch (RM.value()) {
1937 default:
1938 // ignore unknown rounding modes
1939 break;
1940 case RoundingMode::NearestTiesToEven:
1941 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTE;
1942 break;
1943 case RoundingMode::TowardNegative:
1944 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTN;
1945 break;
1946 case RoundingMode::TowardPositive:
1947 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTP;
1948 break;
1949 case RoundingMode::TowardZero:
1950 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTZ;
1951 break;
1952 case RoundingMode::Dynamic:
1953 case RoundingMode::NearestTiesToAway:
1954 // TODO: check if supported
1955 break;
1956 }
1957 if (RoundingModeDeco == std::numeric_limits<unsigned>::max())
1958 return;
1959 // Convert the tip about rounding mode into a decoration record.
1960 createRoundingModeDecoration(I: FPI, RoundingModeDeco, B);
1961}
1962
1963Instruction *SPIRVEmitIntrinsicsImpl::visitSwitchInst(SwitchInst &I) {
1964 BasicBlock *ParentBB = I.getParent();
1965 Function *F = ParentBB->getParent();
1966 IRBuilder<> B(ParentBB);
1967 B.SetInsertPoint(&I);
1968 SmallVector<Value *, 4> Args;
1969 SmallVector<BasicBlock *> BBCases;
1970 Args.push_back(Elt: I.getCondition());
1971 BBCases.push_back(Elt: I.getDefaultDest());
1972 Args.push_back(Elt: BlockAddress::get(F, BB: I.getDefaultDest()));
1973 for (auto &Case : I.cases()) {
1974 Args.push_back(Elt: Case.getCaseValue());
1975 BBCases.push_back(Elt: Case.getCaseSuccessor());
1976 Args.push_back(Elt: BlockAddress::get(F, BB: Case.getCaseSuccessor()));
1977 }
1978 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
1979 ID: Intrinsic::spv_switch, OverloadTypes: {I.getOperand(i_nocapture: 0)->getType()}, Args: {Args});
1980 // remove switch to avoid its unneeded and undesirable unwrap into branches
1981 // and conditions
1982 replaceAllUsesWith(Src: &I, Dest: NewI);
1983 I.eraseFromParent();
1984 // insert artificial and temporary instruction to preserve valid CFG,
1985 // it will be removed after IR translation pass
1986 B.SetInsertPoint(ParentBB);
1987 IndirectBrInst *BrI = B.CreateIndirectBr(
1988 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: ParentBB->getContext())),
1989 NumDests: BBCases.size());
1990 for (BasicBlock *BBCase : BBCases)
1991 BrI->addDestination(Dest: BBCase);
1992 return BrI;
1993}
1994
1995static bool isFirstIndexZero(const GetElementPtrInst *GEP) {
1996 return GEP->getNumIndices() > 0 && match(V: GEP->getOperand(i_nocapture: 1), P: m_Zero());
1997}
1998
1999Instruction *SPIRVEmitIntrinsicsImpl::visitIntrinsicInst(IntrinsicInst &I) {
2000 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
2001 if (!SGEP)
2002 return &I;
2003
2004 IRBuilder<> B(I.getParent());
2005 B.SetInsertPoint(&I);
2006 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
2007 SmallVector<Value *, 4> Args;
2008 Args.push_back(/* inBounds= */ Elt: B.getInt1(V: true));
2009 Args.push_back(Elt: I.getOperand(i_nocapture: 0));
2010 Args.push_back(/* zero index */ Elt: B.getInt32(C: 0));
2011 for (unsigned J = 0; J < SGEP->getNumIndices(); ++J)
2012 Args.push_back(Elt: SGEP->getIndexOperand(Index: J));
2013
2014 Instruction *NewI =
2015 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: Types, Args);
2016 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2017 return NewI;
2018}
2019
2020Instruction *
2021SPIRVEmitIntrinsicsImpl::visitGetElementPtrInst(GetElementPtrInst &I) {
2022 IRBuilder<> B(I.getParent());
2023 B.SetInsertPoint(&I);
2024
2025 // OpPtrAccessChain requires a scalar pointer result; scalarize per-lane
2026 // GEPs that return <N x ptr> and rebuild the vector via insertelement.
2027 if (auto *RetVTy = dyn_cast<FixedVectorType>(Val: I.getType())) {
2028 unsigned N = RetVTy->getNumElements();
2029 Value *PtrOp = I.getPointerOperand();
2030 bool PtrIsVec = isa<VectorType>(Val: PtrOp->getType());
2031 Type *ResultPtrTy = RetVTy->getElementType();
2032 Type *ScalarPtrTy = PtrOp->getType()->getScalarType();
2033 SmallVector<Type *, 2> GepTypes = {ResultPtrTy, ScalarPtrTy};
2034 Value *InBounds = B.getInt1(V: I.isInBounds());
2035 Type *LanePointeeTy = getGEPType(Ref: &I);
2036 Type *SrcElemTy = I.getSourceElementType();
2037
2038 // Pin the lane pointee type on the vector operand and on each extracted
2039 // lane so the prelegalizer wraps them as OpTypeVector/OpTypePointer of
2040 // the right element type instead of defaulting to i8.
2041 if (PtrIsVec)
2042 GR->buildAssignPtr(B, ElemTy: SrcElemTy, Arg: PtrOp);
2043
2044 Value *VecResult = PoisonValue::get(T: RetVTy);
2045 for (unsigned Lane = 0; Lane < N; ++Lane) {
2046 Value *LaneIdx = B.getInt32(C: Lane);
2047 Value *ScalarPtr = PtrOp;
2048 if (PtrIsVec) {
2049 SmallVector<Type *, 3> ExtractTypes = {ScalarPtrTy, PtrOp->getType(),
2050 LaneIdx->getType()};
2051 ScalarPtr = B.CreateIntrinsic(ID: Intrinsic::spv_extractelt, OverloadTypes: {ExtractTypes},
2052 Args: {PtrOp, LaneIdx});
2053 GR->buildAssignPtr(B, ElemTy: SrcElemTy, Arg: ScalarPtr);
2054 }
2055 SmallVector<Value *, 4> Args;
2056 Args.push_back(Elt: InBounds);
2057 Args.push_back(Elt: ScalarPtr);
2058 for (Value *Idx : I.indices()) {
2059 if (isa<VectorType>(Val: Idx->getType())) {
2060 // We cannot use the builder here as for splat-ed / constant vectors
2061 // it will fold to the scalar, and then it becomes impossible to
2062 // retrieve / retain the vectorness.
2063 auto *EI =
2064 ExtractElementInst::Create(Vec: Idx, Idx: LaneIdx, NameStr: "", InsertBefore: B.GetInsertPoint());
2065 if (isVector1(Ty: Idx->getType())) // IRTranslator clobbers <1 x T>.
2066 Args.push_back(Elt: visitExtractElementInst(I&: *EI));
2067 else
2068 Args.push_back(Elt: EI);
2069 } else {
2070 Args.push_back(Elt: Idx);
2071 }
2072 }
2073 Value *ScalarGep = B.CreateIntrinsic(ID: Intrinsic::spv_gep, OverloadTypes: GepTypes, Args);
2074 GR->buildAssignPtr(B, ElemTy: LanePointeeTy, Arg: ScalarGep);
2075 VecResult = B.CreateInsertElement(Vec: VecResult, NewElt: ScalarGep, Idx: LaneIdx);
2076 }
2077
2078 auto *NewI = cast<Instruction>(Val: VecResult);
2079 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2080
2081 if (CallInst *Old = GR->findAssignPtrTypeInstr(Val: NewI)) {
2082 Old->eraseFromParent();
2083 GR->addAssignPtrTypeInstr(Val: NewI, AssignPtrTyCI: nullptr);
2084 }
2085 setInsertPointAfterDef(B, I: NewI);
2086 GR->buildAssignPtr(B, ElemTy: LanePointeeTy, Arg: NewI);
2087
2088 return NewI;
2089 }
2090
2091 if (TM.getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: &I)) {
2092 // Logical SPIR-V cannot use the OpPtrAccessChain instruction. If the first
2093 // index of the GEP is not 0, then we need to try to adjust it.
2094 //
2095 // If the GEP is doing byte addressing, try to rebuild the full access chain
2096 // from the type of the pointer.
2097 if (getByteAddressingMultiplier(Ty: I.getSourceElementType())) {
2098 return buildLogicalAccessChainFromGEP(GEP&: I);
2099 }
2100
2101 // Look for the array-to-pointer decay. If this is the pattern
2102 // we can adjust the types, and prepend a 0 to the indices.
2103 Value *PtrOp = I.getPointerOperand();
2104 Type *SrcElemTy = I.getSourceElementType();
2105 Type *DeducedPointeeTy = deduceElementType(I: PtrOp, UnknownElemTypeI8: true);
2106
2107 if (auto *ArrTy = dyn_cast<ArrayType>(Val: DeducedPointeeTy)) {
2108 if (ArrTy->getElementType() == SrcElemTy) {
2109 SmallVector<Value *> NewIndices;
2110 Type *FirstIdxType = I.getOperand(i_nocapture: 1)->getType();
2111 NewIndices.push_back(Elt: ConstantInt::get(Ty: FirstIdxType, V: 0));
2112 for (Value *Idx : I.indices())
2113 NewIndices.push_back(Elt: Idx);
2114
2115 SmallVector<Type *, 2> Types = {I.getType(), I.getPointerOperandType()};
2116 SmallVector<Value *, 4> Args;
2117 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
2118 Args.push_back(Elt: I.getPointerOperand());
2119 Args.append(in_start: NewIndices.begin(), in_end: NewIndices.end());
2120
2121 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep,
2122 OverloadTypes: {Types}, Args: {Args});
2123 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2124 return NewI;
2125 }
2126 }
2127 }
2128
2129 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
2130 SmallVector<Value *, 4> Args;
2131 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
2132 llvm::append_range(C&: Args, R: I.operands());
2133 Instruction *NewI =
2134 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
2135 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2136 return NewI;
2137}
2138
2139Instruction *SPIRVEmitIntrinsicsImpl::visitBitCastInst(BitCastInst &I) {
2140 IRBuilder<> B(I.getParent());
2141 B.SetInsertPoint(&I);
2142 Value *Source = I.getOperand(i_nocapture: 0);
2143
2144 // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of
2145 // varying element types. In case of IR coming from older versions of LLVM
2146 // such bitcasts do not provide sufficient information, should be just skipped
2147 // here, and handled in insertPtrCastOrAssignTypeInstr.
2148 if (isPointerTy(T: I.getType())) {
2149 replaceAllUsesWith(Src: &I, Dest: Source);
2150 I.eraseFromParent();
2151 return nullptr;
2152 }
2153
2154 SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
2155 SmallVector<Value *> Args(I.op_begin(), I.op_end());
2156 Instruction *NewI =
2157 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_bitcast, OverloadTypes: {Types}, Args: {Args});
2158 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2159 return NewI;
2160}
2161
2162void SPIRVEmitIntrinsicsImpl::insertAssignPtrTypeTargetExt(
2163 TargetExtType *AssignedType, Value *V, IRBuilder<> &B) {
2164 Type *VTy = V->getType();
2165
2166 // A couple of sanity checks.
2167 assert((isPointerTy(VTy)) && "Expect a pointer type!");
2168 if (Type *ElemTy = getPointeeType(Ty: VTy))
2169 if (ElemTy != AssignedType)
2170 report_fatal_error(reason: "Unexpected pointer element type!");
2171
2172 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: V);
2173 if (!AssignCI) {
2174 GR->buildAssignType(B, Ty: AssignedType, Arg: V, CanUseAnyVectorRank);
2175 return;
2176 }
2177
2178 Type *CurrentType =
2179 dyn_cast<ConstantAsMetadata>(
2180 Val: cast<MetadataAsValue>(Val: AssignCI->getOperand(i_nocapture: 1))->getMetadata())
2181 ->getType();
2182 if (CurrentType == AssignedType)
2183 return;
2184
2185 // Builtin types cannot be redeclared or casted.
2186 if (CurrentType->isTargetExtTy())
2187 report_fatal_error(reason: "Type mismatch " + CurrentType->getTargetExtName() +
2188 "/" + AssignedType->getTargetExtName() +
2189 " for value " + V->getName(),
2190 gen_crash_diag: false);
2191
2192 // Our previous guess about the type seems to be wrong, let's update
2193 // inferred type according to a new, more precise type information.
2194 GR->updateAssignType(
2195 AssignCI, Arg: V, OfType: getNormalizedPoisonValue(Ty: AssignedType, CanUseAnyVectorRank));
2196}
2197
2198void SPIRVEmitIntrinsicsImpl::replacePointerOperandWithPtrCast(
2199 Instruction *I, Value *Pointer, Type *ExpectedElementType,
2200 unsigned OperandToReplace, IRBuilder<> &B) {
2201 TypeValidated.insert(Ptr: I);
2202
2203 // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType
2204 Type *PointerElemTy = deduceElementTypeHelper(I: Pointer, UnknownElemTypeI8: false);
2205 if (PointerElemTy == ExpectedElementType ||
2206 isEquivalentTypes(Ty1: PointerElemTy, Ty2: ExpectedElementType))
2207 return;
2208
2209 setInsertPointSkippingPhis(B, I);
2210 Value *ExpectedElementVal =
2211 getNormalizedPoisonValue(Ty: ExpectedElementType, CanUseAnyVectorRank);
2212 MetadataAsValue *VMD = buildMD(Arg: ExpectedElementVal);
2213 unsigned AddressSpace = getPointerAddressSpace(T: Pointer->getType());
2214 bool FirstPtrCastOrAssignPtrType = true;
2215
2216 // Do not emit new spv_ptrcast if equivalent one already exists or when
2217 // spv_assign_ptr_type already targets this pointer with the same element
2218 // type.
2219 if (Pointer->hasUseList()) {
2220 for (auto User : Pointer->users()) {
2221 auto *II = dyn_cast<IntrinsicInst>(Val: User);
2222 if (!II ||
2223 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
2224 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
2225 II->getOperand(i_nocapture: 0) != Pointer)
2226 continue;
2227
2228 // There is some spv_ptrcast/spv_assign_ptr_type already targeting this
2229 // pointer.
2230 FirstPtrCastOrAssignPtrType = false;
2231 if (II->getOperand(i_nocapture: 1) != VMD ||
2232 dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 2))->getSExtValue() !=
2233 AddressSpace)
2234 continue;
2235
2236 // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the
2237 // same element type and address space.
2238 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
2239 return;
2240
2241 // This must be a spv_ptrcast, do not emit new if this one has the same BB
2242 // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type.
2243 if (II->getParent() != I->getParent())
2244 continue;
2245
2246 I->setOperand(i: OperandToReplace, Val: II);
2247 return;
2248 }
2249 }
2250
2251 // Never replace an already-deduced pointer element type with a non-pointer
2252 // one. The conflicting use comes from a mis-deduced expected type. Leave the
2253 // operand untouched rather than emitting a ptrcast that re-introduces the
2254 // collapsed type at the use site.
2255 if (PointerElemTy && isPointerTyOrWrapper(Ty: PointerElemTy) &&
2256 !isPointerTyOrWrapper(Ty: ExpectedElementType) &&
2257 tracesToPointerAlloca(V: Pointer))
2258 return;
2259
2260 if (isa<Instruction>(Val: Pointer) || isa<Argument>(Val: Pointer)) {
2261 if (FirstPtrCastOrAssignPtrType) {
2262 // If this would be the first spv_ptrcast, do not emit spv_ptrcast and
2263 // emit spv_assign_ptr_type instead.
2264 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
2265 return;
2266 } else if (isTodoType(Op: Pointer)) {
2267 eraseTodoType(Op: Pointer);
2268 if (!isa<CallInst>(Val: Pointer) && !isaGEP(V: Pointer) &&
2269 !isa<AllocaInst>(Val: Pointer)) {
2270 // If this wouldn't be the first spv_ptrcast but existing type info is
2271 // uncomplete, update spv_assign_ptr_type arguments.
2272 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Pointer)) {
2273 Type *PrevElemTy = GR->findDeducedElementType(Val: Pointer);
2274 assert(PrevElemTy);
2275 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
2276 std::make_pair(x&: I, y&: Pointer)};
2277 GR->updateAssignType(AssignCI, Arg: Pointer, OfType: ExpectedElementVal);
2278 propagateElemType(Op: Pointer, ElemTy: PrevElemTy, VisitedSubst);
2279 } else {
2280 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
2281 }
2282 return;
2283 }
2284 }
2285 }
2286
2287 // Emit spv_ptrcast
2288 SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()};
2289 SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(C: AddressSpace)};
2290 auto *PtrCastI =
2291 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
2292 I->setOperand(i: OperandToReplace, Val: PtrCastI);
2293 // We need to set up a pointee type for the newly created spv_ptrcast.
2294 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: PtrCastI);
2295}
2296
2297void SPIRVEmitIntrinsicsImpl::insertPtrCastOrAssignTypeInstr(Instruction *I,
2298 IRBuilder<> &B) {
2299 // Handle basic instructions:
2300 StoreInst *SI = dyn_cast<StoreInst>(Val: I);
2301 if (IsKernelArgInt8(F: CurrF, SI)) {
2302 replacePointerOperandWithPtrCast(
2303 I, Pointer: SI->getValueOperand(), ExpectedElementType: IntegerType::getInt8Ty(C&: CurrF->getContext()),
2304 OperandToReplace: 0, B);
2305 }
2306 if (SI) {
2307 Value *Op = SI->getValueOperand();
2308 Value *Pointer = SI->getPointerOperand();
2309 Type *OpTy = Op->getType();
2310 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
2311 OpTy = restoreMutatedType(GR, I: OpI, Ty: OpTy);
2312 if (auto It = AggrConstTypes.find(Val: OpI); It != AggrConstTypes.end())
2313 OpTy = It->second;
2314 }
2315 if (OpTy == Op->getType())
2316 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: Op, UnknownElemTypeI8: false);
2317 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 1, B);
2318 return;
2319 }
2320 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
2321 Value *Pointer = LI->getPointerOperand();
2322 Type *OpTy = LI->getType();
2323 // If the loaded from pointer carries byref/byval, the dominant type is the
2324 // pointee type specified therein, and we should retain it.
2325 if (Argument *Arg;
2326 (Arg = dyn_cast<Argument>(Val: Pointer)) && hasPointeeTypeAttr(Arg))
2327 OpTy = getPointeeTypeByAttr(Arg);
2328
2329 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
2330 if (Type *ElemTy = GR->findDeducedElementType(Val: LI)) {
2331 OpTy = getTypedPointerWrapper(ElemTy, AS: PtrTy->getAddressSpace());
2332 } else {
2333 Type *NewOpTy = OpTy;
2334 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: LI, UnknownElemTypeI8: false);
2335 if (OpTy == NewOpTy)
2336 insertTodoType(Op: Pointer);
2337 }
2338 }
2339 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
2340 return;
2341 }
2342 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(Val: I)) {
2343 Value *Pointer = GEPI->getPointerOperand();
2344 Type *OpTy = nullptr;
2345
2346 // Logical SPIR-V is not allowed to use Op*PtrAccessChain instructions. If
2347 // the first index is 0, then we can trivially lower to OpAccessChain. If
2348 // not we need to try to rewrite the GEP. We avoid adding a pointer cast at
2349 // this time, and will rewrite the GEP when visiting it.
2350 if (TM.getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: GEPI)) {
2351 return;
2352 }
2353
2354 // In all cases, fall back to the GEP type if type scavenging failed.
2355 if (!OpTy)
2356 OpTy = GEPI->getSourceElementType();
2357
2358 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
2359 if (isNestedPointer(Ty: OpTy))
2360 insertTodoType(Op: Pointer);
2361 return;
2362 }
2363
2364 // TODO: review and merge with existing logics:
2365 // Handle calls to builtins (non-intrinsics):
2366 CallInst *CI = dyn_cast<CallInst>(Val: I);
2367 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
2368 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
2369 return;
2370
2371 // collect information about formal parameter types
2372 std::string DemangledName =
2373 getOclOrSpirvBuiltinDemangledName(Name: CI->getCalledFunction()->getName());
2374 Function *CalledF = CI->getCalledFunction();
2375 SmallVector<Type *, 4> CalledArgTys;
2376 bool HaveTypes = false;
2377 for (unsigned OpIdx = 0; OpIdx < CalledF->arg_size(); ++OpIdx) {
2378 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2379 Type *ArgType = CalledArg->getType();
2380 if (!isPointerTy(T: ArgType)) {
2381 CalledArgTys.push_back(Elt: nullptr);
2382 } else if (Type *ArgTypeElem = getPointeeType(Ty: ArgType)) {
2383 CalledArgTys.push_back(Elt: ArgTypeElem);
2384 HaveTypes = true;
2385 } else {
2386 Type *ElemTy = GR->findDeducedElementType(Val: CalledArg);
2387 if (!ElemTy && hasPointeeTypeAttr(Arg: CalledArg))
2388 ElemTy = getPointeeTypeByAttr(Arg: CalledArg);
2389 if (!ElemTy) {
2390 ElemTy = getPointeeTypeByCallInst(DemangledName, CalledF, OpIdx);
2391 if (ElemTy) {
2392 GR->addDeducedElementType(Val: CalledArg,
2393 Ty: normalizeType(Ty: ElemTy, CanUseAnyVectorRank));
2394 } else {
2395 for (User *U : CalledArg->users()) {
2396 if (Instruction *Inst = dyn_cast<Instruction>(Val: U)) {
2397 if ((ElemTy = deduceElementTypeHelper(I: Inst, UnknownElemTypeI8: false)) != nullptr)
2398 break;
2399 }
2400 }
2401 }
2402 }
2403 HaveTypes |= ElemTy != nullptr;
2404 CalledArgTys.push_back(Elt: ElemTy);
2405 }
2406 }
2407
2408 if (DemangledName.empty() && !HaveTypes)
2409 return;
2410
2411 auto Set =
2412 TM.getSubtarget<SPIRVSubtarget>(F: *CalledF).getPreferredInstructionSet();
2413 // Do not reparse rejected declaration types with the builtin parser.
2414 bool ParseBuiltinTypes =
2415 !DemangledName.empty() &&
2416 (!CalledF->isDeclaration() || SPIRV::isBuiltin(DemangledCall: DemangledName, Set));
2417 for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) {
2418 Value *ArgOperand = CI->getArgOperand(i: OpIdx);
2419 if (!isPointerTy(T: ArgOperand->getType()))
2420 continue;
2421
2422 // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs()
2423 if (!isa<Instruction>(Val: ArgOperand) && !isa<Argument>(Val: ArgOperand)) {
2424 // However, we may have assumptions about the formal argument's type and
2425 // may have a need to insert a ptr cast for the actual parameter of this
2426 // call.
2427 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2428 if (!GR->findDeducedElementType(Val: CalledArg))
2429 continue;
2430 }
2431
2432 Type *ExpectedType =
2433 OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr;
2434 if (!ExpectedType && ParseBuiltinTypes)
2435 ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType(
2436 DemangledCall: DemangledName, ArgIdx: OpIdx, Ctx&: I->getContext());
2437 if (!ExpectedType || ExpectedType->isVoidTy())
2438 continue;
2439
2440 if (ExpectedType->isTargetExtTy() &&
2441 !isTypedPointerWrapper(ExtTy: cast<TargetExtType>(Val: ExpectedType)))
2442 insertAssignPtrTypeTargetExt(AssignedType: cast<TargetExtType>(Val: ExpectedType),
2443 V: ArgOperand, B);
2444 else
2445 replacePointerOperandWithPtrCast(I: CI, Pointer: ArgOperand, ExpectedElementType: ExpectedType, OperandToReplace: OpIdx, B);
2446 }
2447}
2448
2449Instruction *
2450SPIRVEmitIntrinsicsImpl::visitInsertElementInst(InsertElementInst &I) {
2451 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2452 // type in LLT and IRTranslator will replace it by the scalar.
2453 if (isVector1(Ty: I.getType()) && !CanUseAnyVectorRank)
2454 return &I;
2455
2456 SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType(),
2457 I.getOperand(i_nocapture: 1)->getType(),
2458 I.getOperand(i_nocapture: 2)->getType()};
2459 IRBuilder<> B(I.getParent());
2460 B.SetInsertPoint(&I);
2461 SmallVector<Value *> Args(I.op_begin(), I.op_end());
2462 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_insertelt,
2463 OverloadTypes: {Types}, Args: {Args});
2464 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2465 return NewI;
2466}
2467
2468Instruction *
2469SPIRVEmitIntrinsicsImpl::visitExtractElementInst(ExtractElementInst &I) {
2470 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2471 // type in LLT and IRTranslator will replace it by the scalar.
2472 if (isVector1(Ty: I.getVectorOperandType()) && !CanUseAnyVectorRank)
2473 return &I;
2474
2475 IRBuilder<> B(I.getParent());
2476 B.SetInsertPoint(&I);
2477 SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
2478 I.getIndexOperand()->getType()};
2479 SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
2480 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_extractelt,
2481 OverloadTypes: {Types}, Args: {Args});
2482 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2483 return NewI;
2484}
2485
2486Instruction *SPIRVEmitIntrinsicsImpl::visitInsertValueInst(InsertValueInst &I) {
2487 IRBuilder<> B(I.getParent());
2488 B.SetInsertPoint(&I);
2489 SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
2490 SmallVector<Value *> Args;
2491 Value *AggregateOp = I.getAggregateOperand();
2492 if (isa<UndefValue>(Val: AggregateOp))
2493 Args.push_back(Elt: UndefValue::get(T: B.getInt32Ty()));
2494 else
2495 Args.push_back(Elt: AggregateOp);
2496 Args.push_back(Elt: I.getInsertedValueOperand());
2497 for (auto &Op : I.indices())
2498 Args.push_back(Elt: B.getInt32(C: Op));
2499 Instruction *NewI =
2500 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_insertv, OverloadTypes: {Types}, Args: {Args});
2501 replaceMemInstrUses(Old: &I, New: NewI, B);
2502 return NewI;
2503}
2504
2505Instruction *
2506SPIRVEmitIntrinsicsImpl::visitExtractValueInst(ExtractValueInst &I) {
2507 IRBuilder<> B(I.getParent());
2508 B.SetInsertPoint(&I);
2509 if (I.getAggregateOperand()->getType()->isAggregateType()) {
2510 // Mutate an aggregate-returning spv_extractv producer to i32 so
2511 // IRTranslator does not see a multi-register value.
2512 CallBase *CB = dyn_cast<CallBase>(Val: I.getAggregateOperand());
2513 if (!CB || CB->getIntrinsicID() != Intrinsic::spv_extractv)
2514 return &I;
2515 CB->mutateType(Ty: B.getInt32Ty());
2516 }
2517 SmallVector<Value *> Args(I.operands());
2518 for (auto &Op : I.indices())
2519 Args.push_back(Elt: B.getInt32(C: Op));
2520 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_extractv,
2521 OverloadTypes: {I.getType()}, Args: {Args});
2522 // If this aggregate extract feeds another insertvalue, the extracted
2523 // composite is used as a SPIR-V value-id by llvm.spv.insertv. Keep the real
2524 // aggregate type in metadata, but expose the value itself as i32 so the
2525 // intrinsic signature remains valid.
2526 if (NewI->getType()->isAggregateType() &&
2527 any_of(Range: I.users(), P: [](User *U) { return isa<InsertValueInst>(Val: U); })) {
2528 AggrConstTypes[NewI] = I.getType();
2529 NewI->mutateType(Ty: B.getInt32Ty());
2530 replaceMemInstrUses(Old: &I, New: NewI, B);
2531 return NewI;
2532 }
2533 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2534 // If the aggregate result feeds a return or callsite whose type was rewritten
2535 // to an i32 value-id by SPIRVPrepareFunctions, mutate it to match.
2536 if (NewI->getType()->isAggregateType()) {
2537 for (const Use &U : NewI->uses()) {
2538 User *Usr = U.getUser();
2539 if (auto *RI = dyn_cast<ReturnInst>(Val: Usr)) {
2540 if (RI->getFunction()->getReturnType() != NewI->getType()) {
2541 NewI->mutateType(Ty: B.getInt32Ty());
2542 break;
2543 }
2544 continue;
2545 }
2546 auto *CB = dyn_cast<CallBase>(Val: Usr);
2547 if (!CB || !CB->isArgOperand(U: &U))
2548 continue;
2549 unsigned ArgNo = CB->getArgOperandNo(U: &U);
2550 FunctionType *FT = CB->getFunctionType();
2551 if (ArgNo < FT->getNumParams() &&
2552 !FT->getParamType(i: ArgNo)->isAggregateType()) {
2553 NewI->mutateType(Ty: B.getInt32Ty());
2554 break;
2555 }
2556 }
2557 }
2558 return NewI;
2559}
2560
2561Instruction *SPIRVEmitIntrinsicsImpl::visitLoadInst(LoadInst &I) {
2562 if (!I.getType()->isAggregateType())
2563 return &I;
2564 IRBuilder<> B(I.getParent());
2565 B.SetInsertPoint(&I);
2566 TrackConstants = false;
2567 const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
2568 MachineMemOperand::Flags Flags =
2569 TLI->getLoadMemOperandFlags(LI: I, DL: CurrF->getDataLayout());
2570
2571 unsigned IntrinsicId;
2572 SmallVector<Value *, 4> Args = {I.getPointerOperand(), B.getInt16(C: Flags)};
2573 if (!I.isAtomic()) {
2574 IntrinsicId = Intrinsic::spv_load;
2575 Args.push_back(Elt: B.getInt32(C: I.getAlign().value()));
2576 } else {
2577 IntrinsicId = Intrinsic::spv_atomic_load;
2578 Args.push_back(Elt: B.getInt8(C: static_cast<uint8_t>(I.getOrdering())));
2579 }
2580 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
2581 ID: IntrinsicId, OverloadTypes: {I.getOperand(i_nocapture: 0)->getType()}, Args);
2582
2583 replaceMemInstrUses(Old: &I, New: NewI, B);
2584 return NewI;
2585}
2586
2587Instruction *SPIRVEmitIntrinsicsImpl::visitStoreInst(StoreInst &I) {
2588 if (!AggrStores.contains(Ptr: &I))
2589 return &I;
2590 IRBuilder<> B(I.getParent());
2591 B.SetInsertPoint(&I);
2592 TrackConstants = false;
2593 const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
2594 MachineMemOperand::Flags Flags =
2595 TLI->getStoreMemOperandFlags(SI: I, DL: CurrF->getDataLayout());
2596 auto *PtrOp = I.getPointerOperand();
2597
2598 if (I.getValueOperand()->getType()->isAggregateType()) {
2599 // It is possible that what used to be an ExtractValueInst has been replaced
2600 // with a call to the spv_extractv intrinsic, and that said call hasn't
2601 // had its return type replaced with i32 during the dedicated pass (because
2602 // it was emitted later); we have to handle this here, because IRTranslator
2603 // cannot deal with multi-register types at the moment.
2604 CallBase *CB = dyn_cast<CallBase>(Val: I.getValueOperand());
2605 assert(CB && CB->getIntrinsicID() == Intrinsic::spv_extractv &&
2606 "Unexpected argument of aggregate type, should be spv_extractv!");
2607 CB->mutateType(Ty: B.getInt32Ty());
2608 }
2609
2610 unsigned IntrinsicId;
2611 SmallVector<Value *, 4> Args = {I.getValueOperand(), PtrOp,
2612 B.getInt16(C: Flags)};
2613 if (!I.isAtomic()) {
2614 IntrinsicId = Intrinsic::spv_store;
2615 Args.push_back(Elt: B.getInt32(C: I.getAlign().value()));
2616 } else {
2617 IntrinsicId = Intrinsic::spv_atomic_store;
2618 Args.push_back(Elt: B.getInt8(C: static_cast<uint8_t>(I.getOrdering())));
2619 }
2620 Instruction *NewI = B.CreateIntrinsicWithoutFolding(
2621 ID: IntrinsicId, OverloadTypes: {I.getValueOperand()->getType(), PtrOp->getType()}, Args);
2622 NewI->copyMetadata(SrcInst: I);
2623 I.eraseFromParent();
2624 return NewI;
2625}
2626
2627Instruction *SPIRVEmitIntrinsicsImpl::visitAllocaInst(AllocaInst &I) {
2628 Value *ArraySize = nullptr;
2629 if (I.isArrayAllocation()) {
2630 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I.getFunction());
2631 if (!STI->canUseExtension(
2632 E: SPIRV::Extension::SPV_INTEL_variable_length_array))
2633 report_fatal_error(
2634 reason: "array allocation: this instruction requires the following "
2635 "SPIR-V extension: SPV_INTEL_variable_length_array",
2636 gen_crash_diag: false);
2637 ArraySize = I.getArraySize();
2638 }
2639 IRBuilder<> B(I.getParent());
2640 B.SetInsertPoint(&I);
2641 TrackConstants = false;
2642 Type *PtrTy = I.getType();
2643 Instruction *NewI =
2644 ArraySize
2645 ? B.CreateIntrinsicWithoutFolding(
2646 ID: Intrinsic::spv_alloca_array, OverloadTypes: {PtrTy, ArraySize->getType()},
2647 Args: {ArraySize, B.getInt32(C: I.getAlign().value())})
2648 : B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_alloca, OverloadTypes: {PtrTy},
2649 Args: {B.getInt32(C: I.getAlign().value())});
2650 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2651 return NewI;
2652}
2653
2654Instruction *
2655SPIRVEmitIntrinsicsImpl::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2656 assert(I.getType()->isAggregateType() && "Aggregate result is expected");
2657 IRBuilder<> B(I.getParent());
2658 B.SetInsertPoint(&I);
2659 SmallVector<Value *> Args(I.operands());
2660 const Triple &TT = TM.getTargetTriple();
2661 Args.push_back(Elt: B.getInt32(C: static_cast<uint32_t>(
2662 getMemScope(TT, Ctx&: I.getContext(), Id: I.getSyncScopeID()))));
2663 // Per SPIR-V spec atomic ops must combine the ordering bits with the
2664 // storage-class bit.
2665 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2666 unsigned AS = I.getPointerOperand()->getType()->getPointerAddressSpace();
2667 uint32_t ScSem = static_cast<uint32_t>(
2668 getMemSemanticsForStorageClass(SC: addressSpaceToStorageClass(AddrSpace: AS, STI: ST)));
2669 Args.push_back(Elt: B.getInt32(C: getMemSemanticsWithStorageClass(
2670 TT, OrderSem: static_cast<uint32_t>(getMemSemantics(Ord: I.getSuccessOrdering())),
2671 StorageClassSem: ScSem)));
2672 Args.push_back(Elt: B.getInt32(C: getMemSemanticsWithStorageClass(
2673 TT, OrderSem: static_cast<uint32_t>(getMemSemantics(Ord: I.getFailureOrdering())),
2674 StorageClassSem: ScSem)));
2675 Instruction *NewI = B.CreateIntrinsicWithoutFolding(
2676 ID: Intrinsic::spv_cmpxchg, OverloadTypes: {I.getPointerOperand()->getType()}, Args: {Args});
2677 replaceMemInstrUses(Old: &I, New: NewI, B);
2678 return NewI;
2679}
2680
2681Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
2682 auto Op = I.getOperation();
2683 if (Op != AtomicRMWInst::UIncWrap && Op != AtomicRMWInst::UDecWrap)
2684 return &I;
2685
2686 // No SPIR-V opcode exists for these, so lowering to an imported helper is an
2687 // AMD extension. Other targets keep the generic expansion.
2688 if (TM.getTargetTriple().getVendor() != Triple::AMD)
2689 return &I;
2690
2691 Module *M = I.getModule();
2692 IRBuilder<> B(I.getParent());
2693 B.SetInsertPoint(&I);
2694
2695 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2696 unsigned AS = I.getPointerOperand()->getType()->getPointerAddressSpace();
2697
2698 uint32_t Scope = static_cast<uint32_t>(
2699 getMemScope(TT: M->getTargetTriple(), Ctx&: I.getContext(), Id: I.getSyncScopeID()));
2700 uint32_t ScSem = static_cast<uint32_t>(
2701 getMemSemanticsForStorageClass(SC: addressSpaceToStorageClass(AddrSpace: AS, STI: ST)));
2702 uint32_t MemSem = getMemSemanticsWithStorageClass(
2703 TT: M->getTargetTriple(),
2704 OrderSem: static_cast<uint32_t>(getMemSemantics(Ord: I.getOrdering())), StorageClassSem: ScSem);
2705
2706 SmallString<64> FuncName(Op == AtomicRMWInst::UIncWrap
2707 ? "__translate_spirv_atomic_uinc_wrap"
2708 : "__translate_spirv_atomic_udec_wrap");
2709
2710 Type *ValTy = I.getValOperand()->getType();
2711 Type *PtrTy = I.getPointerOperand()->getType();
2712 // Encode the address space and value type in the name for overload.
2713 raw_svector_ostream OS(FuncName);
2714 OS << "_p" << AS << "_";
2715 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: ValTy))
2716 OS << "v" << VecTy->getNumElements();
2717 OS << "i" << ValTy->getScalarSizeInBits();
2718
2719 Type *Int32Ty = B.getInt32Ty();
2720 Type *BoolTy = B.getInt1Ty();
2721 SmallVector<Type *, 6> ArgTys = {PtrTy, Int32Ty, Int32Ty,
2722 ValTy, BoolTy, BoolTy};
2723 FunctionType *FT = FunctionType::get(Result: ValTy, Params: ArgTys, isVarArg: false);
2724 FunctionCallee FC = M->getOrInsertFunction(Name: FuncName, T: FT);
2725 cast<Function>(Val: FC.getCallee())->setCallingConv(CallingConv::SPIR_FUNC);
2726
2727 SmallVector<Value *, 6> Args = {
2728 I.getPointerOperand(), B.getInt32(C: Scope),
2729 B.getInt32(C: MemSem), I.getValOperand(),
2730 B.getInt1(V: I.isVolatile()), B.getInt1(V: I.isElementwise())};
2731 CallInst *CI = B.CreateCall(Callee: FC, Args);
2732 CI->setCallingConv(CallingConv::SPIR_FUNC);
2733 // SPIRVCallLowering reads alias.scope/noalias off the call to build the
2734 // aliasing decorations and runs after this pass, so preserve the metadata.
2735 CI->copyMetadata(SrcInst: I);
2736
2737 replaceAllUsesWithAndErase(B, Src: &I, Dest: CI);
2738 return CI;
2739}
2740
2741static bool isAbortCall(const Instruction &I, const SPIRVSubtarget &ST) {
2742 auto *CI = dyn_cast<CallInst>(Val: &I);
2743 if (!CI)
2744 return false;
2745 switch (CI->getIntrinsicID()) {
2746 case Intrinsic::spv_abort:
2747 return true;
2748 case Intrinsic::trap:
2749 case Intrinsic::ubsantrap:
2750 // When the extension is enabled, selection lowers these to OpAbortKHR.
2751 return ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_abort);
2752 default:
2753 return false;
2754 }
2755}
2756
2757// The OpAbortKHR instruction itself is a block terminator, so we don't need to
2758// emit an extra OpUnreachable instruction.
2759static bool precededByAbortIntrinsic(const UnreachableInst &I,
2760 const SPIRVSubtarget &ST) {
2761 const Instruction *Prev = I.getPrevNode();
2762 if (Prev && isAbortCall(I: *Prev, ST))
2763 return true;
2764
2765 assert(llvm::none_of(
2766 *I.getParent(),
2767 [&ST](const Instruction &II) { return isAbortCall(II, ST); }) &&
2768 "abort-like call must be the last non-debug instruction before its "
2769 "block's terminator");
2770 return false;
2771}
2772
2773Instruction *SPIRVEmitIntrinsicsImpl::visitUnreachableInst(UnreachableInst &I) {
2774 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2775 if (precededByAbortIntrinsic(I, ST))
2776 return &I;
2777 IRBuilder<> B(&I);
2778 B.CreateIntrinsic(ID: Intrinsic::spv_unreachable, Args: {});
2779 return &I;
2780}
2781
2782// llvm.compiler.used and llvm.used hold use-list entries that protect their
2783// referenced globals from DCE without participating in code generation.
2784static bool isUseListGlobal(StringRef Name) {
2785 return Name == "llvm.compiler.used" || Name == "llvm.used";
2786}
2787
2788// Returns true for module-level globals that should not have SPIR-V intrinsics
2789// emitted (use-list globals plus llvm.global.annotations).
2790static bool isArtificialGlobal(StringRef Name) {
2791 return isUseListGlobal(Name) || Name == "llvm.global.annotations";
2792}
2793
2794// Returns true if every use of GV traces back to llvm.compiler.used or
2795// llvm.used.
2796static bool hasOnlyArtificialUses(const GlobalVariable &GV) {
2797 SmallPtrSet<const Value *, 8> Visited;
2798 SmallVector<const Value *> Stack(GV.users());
2799 while (!Stack.empty()) {
2800 const Value *V = Stack.pop_back_val();
2801 if (!Visited.insert(Ptr: V).second)
2802 continue;
2803 if (const auto *GVUser = dyn_cast<GlobalVariable>(Val: V)) {
2804 if (!isUseListGlobal(Name: GVUser->getName()))
2805 return false;
2806 continue;
2807 }
2808 if (const auto *C = dyn_cast<Constant>(Val: V)) {
2809 Stack.append(in_start: C->user_begin(), in_end: C->user_end());
2810 continue;
2811 }
2812 return false;
2813 }
2814 return true;
2815}
2816
2817static bool
2818shouldEmitIntrinsicsForGlobalValue(const GlobalVariableUsers &GVUsers,
2819 const GlobalVariable &GV,
2820 const Function *F) {
2821 // Skip special artificial variables.
2822 if (isArtificialGlobal(Name: GV.getName()))
2823 return false;
2824
2825 auto &UserFunctions = GVUsers.getTransitiveUserFunctions(GV);
2826 if (UserFunctions.contains(Ptr: F))
2827 return true;
2828
2829 // Do not emit the intrinsics in this function, it's going to be emitted on
2830 // the functions that reference it.
2831 if (!UserFunctions.empty())
2832 return false;
2833
2834 // Emit definitions for globals that are not referenced by any function on the
2835 // first function definition.
2836 const Module &M = *F->getParent();
2837 const Function &FirstDefinition = *M.getFunctionDefs().begin();
2838 return F == &FirstDefinition;
2839}
2840
2841Value *SPIRVEmitIntrinsicsImpl::buildSpvUndefComposite(Type *AggrTy,
2842 IRBuilder<> &B) {
2843 auto MakeLeaf = [&](Type *ElemTy) -> Instruction * {
2844 CallInst *Leaf = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_undef, Args: {});
2845 AggrConsts[Leaf] = PoisonValue::get(T: ElemTy);
2846 AggrConstTypes[Leaf] = ElemTy;
2847 return Leaf;
2848 };
2849 SmallVector<Value *, 4> Elems;
2850 if (auto *ArrTy = dyn_cast<ArrayType>(Val: AggrTy)) {
2851 Elems.assign(NumElts: ArrTy->getNumElements(), Elt: MakeLeaf(ArrTy->getElementType()));
2852 } else {
2853 auto *StructTy = cast<StructType>(Val: AggrTy);
2854 DenseMap<Type *, Instruction *> LeafByType;
2855 for (unsigned I = 0; I < StructTy->getNumElements(); ++I) {
2856 Type *ElemTy = StructTy->getContainedType(i: I);
2857 auto &Entry = LeafByType[ElemTy];
2858 if (!Entry)
2859 Entry = MakeLeaf(ElemTy);
2860 Elems.push_back(Elt: Entry);
2861 }
2862 }
2863 CallInst *Composite = B.CreateIntrinsicWithoutFolding(
2864 ID: Intrinsic::spv_const_composite, OverloadTypes: {B.getInt32Ty()}, Args: Elems);
2865 AggrConsts[Composite] = PoisonValue::get(T: AggrTy);
2866 AggrConstTypes[Composite] = AggrTy;
2867 return Composite;
2868}
2869
2870// If a function directly returns an aggregate-typed call result,
2871// the ReturnInst carries an aggregate while the function signature
2872// was rewritten to i32 by SPIRVPrepareFunctions. Rebuild the return value
2873// via extractvalue/insertvalue so the regular spv_extractv/spv_insertv
2874// lowering produces a valid OpReturnValue.
2875void SPIRVEmitIntrinsicsImpl::reconstructAggregateReturns(Function &Func,
2876 IRBuilder<> &B) {
2877 Type *OrigRetTy = GR->findMutated(Val: &Func);
2878 if (!OrigRetTy || !OrigRetTy->isAggregateType())
2879 return;
2880 for (BasicBlock &BB : Func) {
2881 auto *RI = dyn_cast<ReturnInst>(Val: BB.getTerminator());
2882 if (!RI)
2883 continue;
2884 Value *RetVal = RI->getReturnValue();
2885 if (!RetVal || RetVal->getType() != OrigRetTy || !isa<CallBase>(Val: RetVal))
2886 continue;
2887 Type *AggrTy = RetVal->getType();
2888 uint64_t NumElts = isa<StructType>(Val: AggrTy)
2889 ? cast<StructType>(Val: AggrTy)->getNumElements()
2890 : cast<ArrayType>(Val: AggrTy)->getNumElements();
2891 B.SetInsertPoint(RI);
2892 Value *Rebuilt = PoisonValue::get(T: AggrTy);
2893 for (uint64_t I = 0; I < NumElts; ++I) {
2894 Value *Elt = B.CreateExtractValue(Agg: RetVal, Idxs: I);
2895 Rebuilt = B.CreateInsertValue(Agg: Rebuilt, Val: Elt, Idxs: I);
2896 }
2897 RI->setOperand(i_nocapture: 0, Val_nocapture: Rebuilt);
2898 }
2899}
2900
2901void SPIRVEmitIntrinsicsImpl::processGlobalValue(GlobalVariable &GV,
2902 IRBuilder<> &B) {
2903
2904 if (!shouldEmitIntrinsicsForGlobalValue(GVUsers, GV, F: CurrF))
2905 return;
2906
2907 // Record the pointee type for every global, not only initialized ones, so an
2908 // undef non-constant aggregate global is not later collapsed to its element
2909 // type. Result is ignored, because TypedPointerType is not supported
2910 // by llvm IR general logic.
2911 deduceElementTypeHelper(I: &GV, UnknownElemTypeI8: false);
2912
2913 Constant *Init = nullptr;
2914 if (hasInitializer(GV: &GV)) {
2915 Init = GV.getInitializer();
2916 Value *InitOp = Init;
2917 if (isa<UndefValue>(Val: Init) && Init->getType()->isAggregateType()) {
2918 const SPIRVSubtarget *STI = TM.getSubtargetImpl();
2919 bool UsePoison =
2920 isa<PoisonValue>(Val: Init) &&
2921 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
2922 if (UsePoison) {
2923 CallInst *Call = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_poison,
2924 OverloadTypes: {B.getInt32Ty()}, Args: {});
2925 AggrConsts[Call] = cast<PoisonValue>(Val: Init);
2926 AggrConstTypes[Call] = Init->getType();
2927 InitOp = Call;
2928 } else {
2929 InitOp = buildSpvUndefComposite(AggrTy: Init->getType(), B);
2930 }
2931 }
2932 Type *Ty = isAggrConstForceInt32(V: Init) ? B.getInt32Ty() : Init->getType();
2933 Constant *Const = isAggrConstForceInt32(V: Init) ? B.getInt32(C: 1) : Init;
2934 CallInst *InitInst = B.CreateIntrinsicWithoutFolding(
2935 ID: Intrinsic::spv_init_global, OverloadTypes: {GV.getType(), Ty}, Args: {&GV, Const});
2936 InitInst->setArgOperand(i: 1, v: InitOp);
2937 }
2938 // Globals with only use-list references have no real function uses. Emit
2939 // spv_unref_global so buildGlobalVariable is called for them.
2940 if (!Init && hasOnlyArtificialUses(GV))
2941 B.CreateIntrinsic(ID: Intrinsic::spv_unref_global, OverloadTypes: GV.getType(), Args: &GV);
2942}
2943
2944// Return true, if we can't decide what is the pointee type now and will get
2945// back to the question later. Return false is spv_assign_ptr_type is not needed
2946// or can be inserted immediately.
2947bool SPIRVEmitIntrinsicsImpl::insertAssignPtrTypeIntrs(Instruction *I,
2948 IRBuilder<> &B,
2949 bool UnknownElemTypeI8) {
2950 reportFatalOnTokenType(I);
2951 if (!isPointerTy(T: I->getType()) || !requireAssignType(I))
2952 return false;
2953
2954 setInsertPointAfterDef(B, I);
2955 if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) {
2956 GR->buildAssignPtr(B, ElemTy, Arg: I);
2957 return false;
2958 }
2959 return true;
2960}
2961
2962void SPIRVEmitIntrinsicsImpl::insertAssignTypeIntrs(Instruction *I,
2963 IRBuilder<> &B) {
2964 // TODO: extend the list of functions with known result types
2965 static StringMap<unsigned> ResTypeWellKnown = {
2966 {"async_work_group_copy", WellKnownTypes::Event},
2967 {"async_work_group_strided_copy", WellKnownTypes::Event},
2968 {"__spirv_GroupAsyncCopy", WellKnownTypes::Event}};
2969
2970 reportFatalOnTokenType(I);
2971
2972 bool IsKnown = false;
2973 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
2974 if (!CI->isIndirectCall() && !CI->isInlineAsm() &&
2975 CI->getCalledFunction() && !CI->getCalledFunction()->isIntrinsic()) {
2976 Function *CalledF = CI->getCalledFunction();
2977 std::string DemangledName =
2978 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
2979 FPDecorationId DecorationId = FPDecorationId::NONE;
2980 if (DemangledName.length() > 0)
2981 DemangledName =
2982 SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName, DecorationId: &DecorationId);
2983 auto ResIt = ResTypeWellKnown.find(Key: DemangledName);
2984 if (ResIt != ResTypeWellKnown.end()) {
2985 IsKnown = true;
2986 setInsertPointAfterDef(B, I);
2987 switch (ResIt->second) {
2988 case WellKnownTypes::Event:
2989 GR->buildAssignType(
2990 B, Ty: TargetExtType::get(Context&: I->getContext(), Name: "spirv.Event"), Arg: I,
2991 CanUseAnyVectorRank);
2992 break;
2993 }
2994 }
2995 // check if a floating rounding mode or saturation info is present
2996 switch (DecorationId) {
2997 default:
2998 break;
2999 case FPDecorationId::SAT:
3000 createSaturatedConversionDecoration(I: CI, B);
3001 break;
3002 case FPDecorationId::RTE:
3003 createRoundingModeDecoration(
3004 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTE, B);
3005 break;
3006 case FPDecorationId::RTZ:
3007 createRoundingModeDecoration(
3008 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTZ, B);
3009 break;
3010 case FPDecorationId::RTP:
3011 createRoundingModeDecoration(
3012 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTP, B);
3013 break;
3014 case FPDecorationId::RTN:
3015 createRoundingModeDecoration(
3016 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTN, B);
3017 break;
3018 }
3019 }
3020 }
3021
3022 Type *Ty = I->getType();
3023 if (!IsKnown && !Ty->isVoidTy() && !isPointerTy(T: Ty) && requireAssignType(I)) {
3024 setInsertPointAfterDef(B, I);
3025 Type *TypeToAssign = Ty;
3026 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
3027 if (isSpvAggrPlaceholder(V: II)) {
3028 auto It = AggrConstTypes.find(Val: II);
3029 if (It == AggrConstTypes.end())
3030 report_fatal_error(reason: "Unknown composite intrinsic type");
3031 TypeToAssign = It->second;
3032 } else if (II->getIntrinsicID() == Intrinsic::spv_poison) {
3033 if (auto It = AggrConstTypes.find(Val: II); It != AggrConstTypes.end())
3034 TypeToAssign = It->second;
3035 }
3036 } else if (auto It = AggrConstTypes.find(Val: I); It != AggrConstTypes.end())
3037 TypeToAssign = It->second;
3038 TypeToAssign = restoreMutatedType(GR, I, Ty: TypeToAssign);
3039 GR->buildAssignType(B, Ty: TypeToAssign, Arg: I, CanUseAnyVectorRank);
3040 }
3041 for (const auto &Op : I->operands()) {
3042 if (isa<ConstantPointerNull>(Val: Op) || isa<UndefValue>(Val: Op) ||
3043 isVector1(Ty: Op->getType()) || // <1 x T> gets clobbered ty IRTranslator.
3044 // Check GetElementPtrConstantExpr case.
3045 (isa<ConstantExpr>(Val: Op) &&
3046 (isa<GEPOperator>(Val: Op) ||
3047 (cast<ConstantExpr>(Val: Op)->getOpcode() == CastInst::IntToPtr)))) {
3048 setInsertPointSkippingPhis(B, I);
3049 Type *OpTy = Op->getType();
3050 if (isa<UndefValue>(Val: Op) && OpTy->isAggregateType()) {
3051 CallInst *AssignCI =
3052 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {B.getInt32Ty()}, Arg: Op,
3053 Arg2: UndefValue::get(T: B.getInt32Ty()), Imms: {}, B);
3054 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: AssignCI);
3055 } else if (!isa<Instruction>(Val: Op)) {
3056 Type *OpTy = Op->getType();
3057 Type *OpTyElem = getPointeeType(Ty: OpTy);
3058 if (OpTyElem) {
3059 GR->buildAssignPtr(B, ElemTy: OpTyElem, Arg: Op);
3060 } else if (isPointerTy(T: OpTy)) {
3061 Type *ElemTy = GR->findDeducedElementType(Val: Op);
3062 GR->buildAssignPtr(B, ElemTy: ElemTy ? ElemTy : deduceElementType(I: Op, UnknownElemTypeI8: true),
3063 Arg: Op);
3064 } else {
3065 Value *OpTyVal = Op;
3066 if (OpTy->isTargetExtTy()) {
3067 // We need to do this in order to be consistent with how target ext
3068 // types are handled in `processInstrAfterVisit`
3069 OpTyVal = getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank);
3070 }
3071 CallInst *AssignCI = buildIntrWithMD(
3072 IntrID: Intrinsic::spv_assign_type, Types: {OpTy},
3073 Arg: getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank), Arg2: OpTyVal, Imms: {},
3074 B);
3075 GR->addAssignPtrTypeInstr(Val: OpTyVal, AssignPtrTyCI: AssignCI);
3076 }
3077 }
3078 }
3079 }
3080}
3081
3082bool SPIRVEmitIntrinsicsImpl::shouldTryToAddMemAliasingDecoration(
3083 Instruction *Inst) {
3084 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*Inst->getFunction());
3085 if (!STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing))
3086 return false;
3087 // Add aliasing decorations to internal load and store intrinsics.
3088 // Do not attach them to store atomic or load atomic intrinsics / instructions
3089 // since the extension is inconsistent at the moment (we cannot add the
3090 // decoration to atomic stores because they do not have an id).
3091 return match(V: Inst,
3092 P: m_AnyIntrinsic<Intrinsic::spv_load, Intrinsic::spv_store>());
3093}
3094
3095void SPIRVEmitIntrinsicsImpl::insertSpirvDecorations(Instruction *I,
3096 IRBuilder<> &B) {
3097 if (MDNode *MD = I->getMetadata(Kind: "spirv.Decorations")) {
3098 setInsertPointAfterDef(B, I);
3099 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
3100 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
3101 }
3102 // Lower alias.scope/noalias metadata
3103 {
3104 auto processMemAliasingDecoration = [&](unsigned Kind) {
3105 if (MDNode *AliasListMD = I->getMetadata(KindID: Kind)) {
3106 if (shouldTryToAddMemAliasingDecoration(Inst: I)) {
3107 uint32_t Dec = Kind == LLVMContext::MD_alias_scope
3108 ? SPIRV::Decoration::AliasScopeINTEL
3109 : SPIRV::Decoration::NoAliasINTEL;
3110 SmallVector<Value *, 3> Args = {
3111 I, ConstantInt::get(Ty: B.getInt32Ty(), V: Dec),
3112 MetadataAsValue::get(Context&: I->getContext(), MD: AliasListMD)};
3113 setInsertPointAfterDef(B, I);
3114 B.CreateIntrinsic(ID: Intrinsic::spv_assign_aliasing_decoration,
3115 OverloadTypes: {I->getType()}, Args: {Args});
3116 }
3117 }
3118 };
3119 processMemAliasingDecoration(LLVMContext::MD_alias_scope);
3120 processMemAliasingDecoration(LLVMContext::MD_noalias);
3121 }
3122 // MD_fpmath
3123 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_fpmath)) {
3124 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I->getFunction());
3125 bool AllowFPMaxError =
3126 STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_fp_max_error);
3127 if (!AllowFPMaxError)
3128 return;
3129
3130 setInsertPointAfterDef(B, I);
3131 B.CreateIntrinsic(ID: Intrinsic::spv_assign_fpmaxerror_decoration,
3132 OverloadTypes: {I->getType()},
3133 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
3134 }
3135 if (spirvPreserveAuxData(TT: I->getModule()->getTargetTriple()) &&
3136 isa<AtomicRMWInst>(Val: I)) {
3137 LLVMContext &Ctx = B.getContext();
3138 auto *AuxMD = ConstantAsMetadata::get(C: ConstantInt::get(
3139 Ty: B.getInt32Ty(), V: SPIRV::Decoration::AuxDataInstructionMetadata));
3140
3141 SmallVector<Metadata *> MDs;
3142 if (I->hasMetadata(Kind: "amdgpu.no.fine.grained.memory"))
3143 MDs.push_back(Elt: MDNode::get(
3144 Context&: Ctx, MDs: {AuxMD, MDString::get(Context&: Ctx, Str: "amdgpu.no.fine.grained.memory")}));
3145 if (I->hasMetadata(Kind: "amdgpu.no.remote.memory"))
3146 MDs.push_back(Elt: MDNode::get(
3147 Context&: Ctx, MDs: {AuxMD, MDString::get(Context&: Ctx, Str: "amdgpu.no.remote.memory")}));
3148 if (I->hasMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode))
3149 MDs.push_back(Elt: MDNode::get(
3150 Context&: Ctx, MDs: {AuxMD, MDString::get(Context&: Ctx, Str: "atomic.ignore.denormal.mode")}));
3151 if (!MDs.empty()) {
3152 setInsertPointAfterDef(B, I);
3153 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
3154 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs))});
3155 }
3156 }
3157}
3158
3159static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
3160 const Module &M,
3161 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
3162 &FPFastMathDefaultInfoMap,
3163 Function *F) {
3164 auto it = FPFastMathDefaultInfoMap.find(Val: F);
3165 if (it != FPFastMathDefaultInfoMap.end())
3166 return it->second;
3167
3168 // If the map does not contain the entry, create a new one. Initialize it to
3169 // contain all 3 elements sorted by bit width of target type: {half, float,
3170 // double}.
3171 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
3172 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
3173 Args: SPIRV::FPFastMathMode::None);
3174 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
3175 Args: SPIRV::FPFastMathMode::None);
3176 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
3177 Args: SPIRV::FPFastMathMode::None);
3178 return FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
3179}
3180
3181static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
3182 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
3183 const Type *Ty) {
3184 size_t BitWidth = Ty->getScalarSizeInBits();
3185 int Index =
3186 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
3187 BitWidth);
3188 assert(Index >= 0 && Index < 3 &&
3189 "Expected FPFastMathDefaultInfo for half, float, or double");
3190 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3191 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3192 return FPFastMathDefaultInfoVec[Index];
3193}
3194
3195void SPIRVEmitIntrinsicsImpl::insertConstantsForFPFastMathDefault(Module &M) {
3196 const SPIRVSubtarget *ST = TM.getSubtargetImpl();
3197 if (!ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
3198 return;
3199
3200 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
3201 // We need the entry point (function) as the key, and the target
3202 // type and flags as the value.
3203 // We also need to check ContractionOff and SignedZeroInfNanPreserve
3204 // execution modes, as they are now deprecated and must be replaced
3205 // with FPFastMathDefaultInfo.
3206 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
3207 if (!Node) {
3208 if (!M.getNamedMetadata(Name: "opencl.enable.FP_CONTRACT")) {
3209 // This requires emitting ContractionOff. However, because
3210 // ContractionOff is now deprecated, we need to replace it with
3211 // FPFastMathDefaultInfo with FP Fast Math Mode bitmask set to all 0.
3212 // We need to create the constant for that.
3213
3214 // Create constant instruction with the bitmask flags.
3215 Constant *InitValue =
3216 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: 0);
3217 // TODO: Reuse constant if there is one already with the required
3218 // value.
3219 [[maybe_unused]] GlobalVariable *GV =
3220 new GlobalVariable(M, // Module
3221 Type::getInt32Ty(C&: M.getContext()), // Type
3222 true, // isConstant
3223 GlobalValue::InternalLinkage, // Linkage
3224 InitValue // Initializer
3225 );
3226 }
3227 return;
3228 }
3229
3230 // The table maps function pointers to their default FP fast math info. It
3231 // can be assumed that the SmallVector is sorted by the bit width of the
3232 // type. The first element is the smallest bit width, and the last element
3233 // is the largest bit width, therefore, we will have {half, float, double}
3234 // in the order of their bit widths.
3235 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
3236 FPFastMathDefaultInfoMap;
3237
3238 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
3239 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
3240 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
3241 Function *F = cast<Function>(
3242 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
3243 const auto EM =
3244 cast<ConstantInt>(
3245 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
3246 ->getZExtValue();
3247 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
3248 assert(MDN->getNumOperands() == 4 &&
3249 "Expected 4 operands for FPFastMathDefault");
3250 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
3251 unsigned Flags =
3252 cast<ConstantInt>(
3253 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
3254 ->getZExtValue();
3255 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3256 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3257 SPIRV::FPFastMathDefaultInfo &Info =
3258 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
3259 Info.FastMathFlags = Flags;
3260 Info.FPFastMathDefault = true;
3261 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
3262 assert(MDN->getNumOperands() == 2 &&
3263 "Expected no operands for ContractionOff");
3264
3265 // We need to save this info for every possible FP type, i.e. {half,
3266 // float, double, fp128}.
3267 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3268 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3269 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3270 Info.ContractionOff = true;
3271 }
3272 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
3273 assert(MDN->getNumOperands() == 3 &&
3274 "Expected 1 operand for SignedZeroInfNanPreserve");
3275 unsigned TargetWidth =
3276 cast<ConstantInt>(
3277 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
3278 ->getZExtValue();
3279 // We need to save this info only for the FP type with TargetWidth.
3280 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3281 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3282 int Index = SPIRV::FPFastMathDefaultInfoVector::
3283 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
3284 assert(Index >= 0 && Index < 3 &&
3285 "Expected FPFastMathDefaultInfo for half, float, or double");
3286 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3287 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3288 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
3289 }
3290 }
3291
3292 DenseMap<unsigned, GlobalVariable *> GlobalVars;
3293 for (auto &[Func, FPFastMathDefaultInfoVec] : FPFastMathDefaultInfoMap) {
3294 if (FPFastMathDefaultInfoVec.empty())
3295 continue;
3296
3297 for (const SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3298 assert(Info.Ty && "Expected target type for FPFastMathDefaultInfo");
3299 // Skip if none of the execution modes was used.
3300 unsigned Flags = Info.FastMathFlags;
3301 if (Flags == SPIRV::FPFastMathMode::None && !Info.ContractionOff &&
3302 !Info.SignedZeroInfNanPreserve && !Info.FPFastMathDefault)
3303 continue;
3304
3305 // Check if flags are compatible.
3306 if (Info.ContractionOff && (Flags & SPIRV::FPFastMathMode::AllowContract))
3307 report_fatal_error(reason: "Conflicting FPFastMathFlags: ContractionOff "
3308 "and AllowContract");
3309
3310 if (Info.SignedZeroInfNanPreserve &&
3311 !(Flags &
3312 (SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
3313 SPIRV::FPFastMathMode::NSZ))) {
3314 if (Info.FPFastMathDefault)
3315 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
3316 "SignedZeroInfNanPreserve but at least one of "
3317 "NotNaN/NotInf/NSZ is enabled.");
3318 }
3319
3320 if ((Flags & SPIRV::FPFastMathMode::AllowTransform) &&
3321 !((Flags & SPIRV::FPFastMathMode::AllowReassoc) &&
3322 (Flags & SPIRV::FPFastMathMode::AllowContract))) {
3323 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
3324 "AllowTransform requires AllowReassoc and "
3325 "AllowContract to be set.");
3326 }
3327
3328 auto it = GlobalVars.find(Val: Flags);
3329 GlobalVariable *GV = nullptr;
3330 if (it != GlobalVars.end()) {
3331 // Reuse existing global variable.
3332 GV = it->second;
3333 } else {
3334 // Create constant instruction with the bitmask flags.
3335 Constant *InitValue =
3336 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: Flags);
3337 // TODO: Reuse constant if there is one already with the required
3338 // value.
3339 GV = new GlobalVariable(M, // Module
3340 Type::getInt32Ty(C&: M.getContext()), // Type
3341 true, // isConstant
3342 GlobalValue::InternalLinkage, // Linkage
3343 InitValue // Initializer
3344 );
3345 GlobalVars[Flags] = GV;
3346 }
3347 }
3348 }
3349}
3350
3351void SPIRVEmitIntrinsicsImpl::processInstrAfterVisit(Instruction *I,
3352 IRBuilder<> &B) {
3353 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3354 bool IsConstComposite =
3355 II && II->getIntrinsicID() == Intrinsic::spv_const_composite;
3356 if (IsConstComposite && TrackConstants) {
3357 setInsertPointAfterDef(B, I);
3358 auto t = AggrConsts.find(Val: I);
3359 assert(t != AggrConsts.end());
3360 auto *NewOp =
3361 buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
3362 Types: {II->getType(), II->getType()}, Arg: t->second, Arg2: I, Imms: {}, B);
3363 replaceAllUsesWith(Src: I, Dest: NewOp, DeleteOld: false);
3364 NewOp->setArgOperand(i: 0, v: I);
3365 }
3366 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
3367 for (const auto &Op : I->operands()) {
3368 if (isa<PHINode>(Val: I) || isa<SwitchInst>(Val: I) ||
3369 !(isa<ConstantData>(Val: Op) || isa<ConstantExpr>(Val: Op)))
3370 continue;
3371 unsigned OpNo = Op.getOperandNo();
3372 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
3373 (!II->isBundleOperand(Idx: OpNo) &&
3374 II->paramHasAttr(ArgNo: OpNo, Kind: Attribute::ImmArg))))
3375 continue;
3376
3377 if (!BPrepared) {
3378 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
3379 : B.SetInsertPoint(I);
3380 BPrepared = true;
3381 }
3382 Type *OpTy = Op->getType();
3383 Type *OpElemTy = GR->findDeducedElementType(Val: Op);
3384 Value *NewOp = Op;
3385 if (OpTy->isTargetExtTy()) {
3386 // Since this value is replaced by poison, we need to do the same in
3387 // `insertAssignTypeIntrs`.
3388 Value *OpTyVal = getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank);
3389 NewOp = buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
3390 Types: {OpTy, OpTyVal->getType()}, Arg: Op, Arg2: OpTyVal, Imms: {}, B);
3391 }
3392 if (!IsConstComposite && isPointerTy(T: OpTy) && OpElemTy != nullptr &&
3393 OpElemTy != IntegerType::getInt8Ty(C&: I->getContext())) {
3394 SmallVector<Type *, 2> Types = {OpTy, OpTy};
3395 SmallVector<Value *, 2> Args = {
3396 NewOp,
3397 buildMD(Arg: getNormalizedPoisonValue(Ty: OpElemTy, CanUseAnyVectorRank)),
3398 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
3399 CallInst *PtrCasted = B.CreateIntrinsicWithoutFolding(
3400 ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
3401 GR->buildAssignPtr(B, ElemTy: OpElemTy, Arg: PtrCasted);
3402 NewOp = PtrCasted;
3403 }
3404 if (NewOp != Op)
3405 I->setOperand(i: OpNo, Val: NewOp);
3406 }
3407 if (Named.insert(Ptr: I).second)
3408 emitAssignName(I, B);
3409}
3410
3411Type *SPIRVEmitIntrinsicsImpl::deduceFunParamElementType(Function *F,
3412 unsigned OpIdx) {
3413 SmallPtrSet<Function *, 0> FVisited;
3414 return deduceFunParamElementType(F, OpIdx, FVisited);
3415}
3416
3417Type *SPIRVEmitIntrinsicsImpl::deduceFunParamElementType(
3418 Function *F, unsigned OpIdx, SmallPtrSetImpl<Function *> &FVisited) {
3419 // maybe a cycle
3420 if (!FVisited.insert(Ptr: F).second)
3421 return nullptr;
3422
3423 SmallPtrSet<Value *, 0> Visited;
3424 SmallVector<std::pair<Function *, unsigned>> Lookup;
3425 // search in function's call sites
3426 for (User *U : F->users()) {
3427 CallInst *CI = dyn_cast<CallInst>(Val: U);
3428 if (!CI || OpIdx >= CI->arg_size())
3429 continue;
3430 Value *OpArg = CI->getArgOperand(i: OpIdx);
3431 if (!isPointerTy(T: OpArg->getType()))
3432 continue;
3433 // maybe we already know operand's element type
3434 if (Type *KnownTy = GR->findDeducedElementType(Val: OpArg))
3435 return KnownTy;
3436 // try to deduce from the operand itself
3437 Visited.clear();
3438 if (Type *Ty = deduceElementTypeHelper(I: OpArg, Visited, UnknownElemTypeI8: false))
3439 return Ty;
3440 // search in actual parameter's users
3441 for (User *OpU : OpArg->users()) {
3442 Instruction *Inst = dyn_cast<Instruction>(Val: OpU);
3443 if (!Inst || Inst == CI)
3444 continue;
3445 Visited.clear();
3446 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8: false))
3447 return Ty;
3448 }
3449 // check if it's a formal parameter of the outer function
3450 if (!CI->getParent() || !CI->getParent()->getParent())
3451 continue;
3452 Function *OuterF = CI->getParent()->getParent();
3453 if (FVisited.find(Ptr: OuterF) != FVisited.end())
3454 continue;
3455 for (unsigned i = 0; i < OuterF->arg_size(); ++i) {
3456 if (OuterF->getArg(i) == OpArg) {
3457 Lookup.push_back(Elt: std::make_pair(x&: OuterF, y&: i));
3458 break;
3459 }
3460 }
3461 }
3462
3463 // search in function parameters
3464 for (auto &Pair : Lookup) {
3465 if (Type *Ty = deduceFunParamElementType(F: Pair.first, OpIdx: Pair.second, FVisited))
3466 return Ty;
3467 }
3468
3469 return nullptr;
3470}
3471
3472void SPIRVEmitIntrinsicsImpl::processParamTypesByFunHeader(Function *F,
3473 IRBuilder<> &B) {
3474 B.SetInsertPointPastAllocas(F);
3475 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
3476 Argument *Arg = F->getArg(i: OpIdx);
3477 // Vector-of-pointers arg: deduce pointee from a GEP user so the function
3478 // type isn't emitted with the default i8 pointee.
3479 if (isUntypedPointerVectorTy(T: Arg->getType()) &&
3480 !GR->findDeducedElementType(Val: Arg)) {
3481 for (User *U : Arg->users()) {
3482 auto *GEP = dyn_cast<GetElementPtrInst>(Val: U);
3483 if (GEP && GEP->getPointerOperand() == Arg) {
3484 GR->buildAssignPtr(B, ElemTy: GEP->getSourceElementType(), Arg);
3485 break;
3486 }
3487 }
3488 continue;
3489 }
3490 if (!isUntypedPointerTy(T: Arg->getType()))
3491 continue;
3492 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
3493 if (ElemTy)
3494 continue;
3495 if (hasPointeeTypeAttr(Arg) &&
3496 (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) {
3497 GR->buildAssignPtr(B, ElemTy, Arg);
3498 continue;
3499 }
3500 // search in function's call sites
3501 for (User *U : F->users()) {
3502 CallInst *CI = dyn_cast<CallInst>(Val: U);
3503 if (!CI || OpIdx >= CI->arg_size())
3504 continue;
3505 Value *OpArg = CI->getArgOperand(i: OpIdx);
3506 if (!isPointerTy(T: OpArg->getType()))
3507 continue;
3508 // maybe we already know operand's element type
3509 if ((ElemTy = GR->findDeducedElementType(Val: OpArg)) != nullptr)
3510 break;
3511 }
3512 if (ElemTy) {
3513 GR->buildAssignPtr(B, ElemTy, Arg);
3514 continue;
3515 }
3516 if (HaveFunPtrs) {
3517 for (User *U : Arg->users()) {
3518 CallInst *CI = dyn_cast<CallInst>(Val: U);
3519 if (CI && !isa<IntrinsicInst>(Val: CI) && CI->isIndirectCall() &&
3520 CI->getCalledOperand() == Arg &&
3521 CI->getParent()->getParent() == CurrF) {
3522 SmallVector<std::pair<Value *, unsigned>> Ops;
3523 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy&: ElemTy, IsPostprocessing: false);
3524 if (ElemTy) {
3525 GR->buildAssignPtr(B, ElemTy, Arg);
3526 break;
3527 }
3528 }
3529 }
3530 }
3531 }
3532}
3533
3534void SPIRVEmitIntrinsicsImpl::processParamTypes(Function *F, IRBuilder<> &B) {
3535 B.SetInsertPointPastAllocas(F);
3536 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
3537 Argument *Arg = F->getArg(i: OpIdx);
3538 if (!isUntypedPointerTy(T: Arg->getType()))
3539 continue;
3540 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
3541 if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) {
3542 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Arg)) {
3543 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3544 GR->updateAssignType(
3545 AssignCI, Arg,
3546 OfType: getNormalizedPoisonValue(Ty: ElemTy, CanUseAnyVectorRank));
3547 propagateElemType(Op: Arg, ElemTy: IntegerType::getInt8Ty(C&: F->getContext()),
3548 VisitedSubst);
3549 } else {
3550 GR->buildAssignPtr(B, ElemTy, Arg);
3551 }
3552 }
3553 }
3554}
3555
3556static FunctionType *getFunctionPointerElemType(Function *F,
3557 SPIRVGlobalRegistry *GR) {
3558 FunctionType *FTy = F->getFunctionType();
3559 bool IsNewFTy = false;
3560 SmallVector<Type *, 4> ArgTys;
3561 for (Argument &Arg : F->args()) {
3562 Type *ArgTy = Arg.getType();
3563 if (ArgTy->isPointerTy())
3564 if (Type *ElemTy = GR->findDeducedElementType(Val: &Arg)) {
3565 IsNewFTy = true;
3566 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
3567 }
3568 ArgTys.push_back(Elt: ArgTy);
3569 }
3570 return IsNewFTy
3571 ? FunctionType::get(Result: FTy->getReturnType(), Params: ArgTys, isVarArg: FTy->isVarArg())
3572 : FTy;
3573}
3574
3575bool SPIRVEmitIntrinsicsImpl::processFunctionPointers(Module &M) {
3576 SmallVector<Function *> Worklist;
3577 for (auto &F : M) {
3578 if (F.isIntrinsic())
3579 continue;
3580 if (F.isDeclaration()) {
3581 for (User *U : F.users()) {
3582 // SPIR-V has no personality, so it is not a function pointer use.
3583 if (auto *UF = dyn_cast<Function>(Val: U);
3584 UF && UF->hasPersonalityFn() && UF->getPersonalityFn() == &F)
3585 continue;
3586 CallInst *CI = dyn_cast<CallInst>(Val: U);
3587 if (!CI || CI->getCalledFunction() != &F) {
3588 Worklist.push_back(Elt: &F);
3589 break;
3590 }
3591 }
3592 } else {
3593 if (F.user_empty())
3594 continue;
3595 Type *FPElemTy = GR->findDeducedElementType(Val: &F);
3596 if (!FPElemTy)
3597 FPElemTy = getFunctionPointerElemType(F: &F, GR);
3598 for (User *U : F.users()) {
3599 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
3600 if (!II || II->arg_size() != 3 || II->getOperand(i_nocapture: 0) != &F)
3601 continue;
3602 if (II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type ||
3603 II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
3604 GR->updateAssignType(
3605 AssignCI: II, Arg: &F, OfType: getNormalizedPoisonValue(Ty: FPElemTy, CanUseAnyVectorRank));
3606 break;
3607 }
3608 }
3609 }
3610 }
3611 if (Worklist.empty())
3612 return false;
3613
3614 LLVMContext &Ctx = M.getContext();
3615 Function *SF = getOrCreateBackendServiceFunction(M);
3616 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: SF);
3617 IRBuilder<> IRB(BB);
3618
3619 for (Function *F : Worklist) {
3620 SmallVector<Value *> Args;
3621 for (const auto &Arg : F->args())
3622 Args.push_back(
3623 Elt: getNormalizedPoisonValue(Ty: Arg.getType(), CanUseAnyVectorRank));
3624 IRB.CreateCall(Callee: F, Args);
3625 }
3626 IRB.CreateRetVoid();
3627
3628 return true;
3629}
3630
3631// Apply types parsed from demangled function declarations.
3632void SPIRVEmitIntrinsicsImpl::applyDemangledPtrArgTypes(IRBuilder<> &B) {
3633 DenseMap<Function *, CallInst *> Ptrcasts;
3634 for (auto It : FDeclPtrTys) {
3635 Function *F = It.first;
3636 for (auto *U : F->users()) {
3637 CallInst *CI = dyn_cast<CallInst>(Val: U);
3638 if (!CI || CI->getCalledFunction() != F)
3639 continue;
3640 unsigned Sz = CI->arg_size();
3641 for (auto [Idx, ElemTy] : It.second) {
3642 if (Idx >= Sz)
3643 continue;
3644 Value *Param = CI->getArgOperand(i: Idx);
3645 if (GR->findDeducedElementType(Val: Param) || isa<GlobalValue>(Val: Param))
3646 continue;
3647 if (Argument *Arg = dyn_cast<Argument>(Val: Param)) {
3648 if (!hasPointeeTypeAttr(Arg)) {
3649 B.SetInsertPointPastAllocas(Arg->getParent());
3650 B.SetCurrentDebugLocation(DebugLoc());
3651 GR->buildAssignPtr(B, ElemTy, Arg);
3652 }
3653 } else if (isaGEP(V: Param)) {
3654 replaceUsesOfWithSpvPtrcast(
3655 Op: Param, ElemTy: normalizeType(Ty: ElemTy, CanUseAnyVectorRank), I: CI, Ptrcasts);
3656 } else if (isa<Instruction>(Val: Param)) {
3657 GR->addDeducedElementType(Val: Param,
3658 Ty: normalizeType(Ty: ElemTy, CanUseAnyVectorRank));
3659 // insertAssignTypeIntrs() will complete buildAssignPtr()
3660 } else {
3661 B.SetInsertPoint(CI->getParent()
3662 ->getParent()
3663 ->getEntryBlock()
3664 .getFirstNonPHIOrAlloca());
3665 GR->buildAssignPtr(B, ElemTy, Arg: Param);
3666 }
3667 CallInst *Ref = dyn_cast<CallInst>(Val: Param);
3668 if (!Ref)
3669 continue;
3670 Function *RefF = Ref->getCalledFunction();
3671 if (!RefF || !isPointerTy(T: RefF->getReturnType()) ||
3672 GR->findDeducedElementType(Val: RefF))
3673 continue;
3674 ElemTy = normalizeType(Ty: ElemTy, CanUseAnyVectorRank);
3675 GR->addDeducedElementType(Val: RefF, Ty: ElemTy);
3676 GR->addReturnType(
3677 ArgF: RefF, DerivedTy: TypedPointerType::get(
3678 ElementType: ElemTy, AddressSpace: getPointerAddressSpace(T: RefF->getReturnType())));
3679 }
3680 }
3681 }
3682 FDeclPtrTys.clear();
3683}
3684
3685GetElementPtrInst *SPIRVEmitIntrinsicsImpl::simplifyZeroLengthArrayGepInst(
3686 GetElementPtrInst *GEP) {
3687 // getelementptr [0 x T], P, 0 (zero), I -> getelementptr T, P, I.
3688 // If type is 0-length array and first index is 0 (zero), drop both the
3689 // 0-length array type and the first index. This is a common pattern in
3690 // the IR, e.g. when using a zero-length array as a placeholder for a
3691 // flexible array such as unbound arrays.
3692 assert(GEP && "GEP is null");
3693 Type *SrcTy = GEP->getSourceElementType();
3694 SmallVector<Value *, 8> Indices(GEP->indices());
3695 ArrayType *ArrTy = dyn_cast<ArrayType>(Val: SrcTy);
3696 if (ArrTy && ArrTy->getNumElements() == 0 && match(V: Indices[0], P: m_Zero())) {
3697 Indices.erase(CI: Indices.begin());
3698 SrcTy = ArrTy->getElementType();
3699 return GetElementPtrInst::Create(PointeeType: SrcTy, Ptr: GEP->getPointerOperand(), IdxList: Indices,
3700 NW: GEP->getNoWrapFlags(), NameStr: "",
3701 InsertBefore: GEP->getIterator());
3702 }
3703 return nullptr;
3704}
3705
3706void SPIRVEmitIntrinsicsImpl::emitUnstructuredLoopControls(Function &F,
3707 IRBuilder<> &B) {
3708 const SPIRVSubtarget *ST = TM.getSubtargetImpl(F);
3709 // Shaders use SPIRVStructurizer which emits OpLoopMerge via spv_loop_merge.
3710 if (ST->isShader())
3711 return;
3712
3713 if (ST->canUseExtension(
3714 E: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls)) {
3715 for (BasicBlock &BB : F) {
3716 Instruction *Term = BB.getTerminator();
3717 MDNode *LoopMD = Term->getMetadata(KindID: LLVMContext::MD_loop);
3718 if (!LoopMD)
3719 continue;
3720
3721 SmallVector<unsigned, 1> Ops =
3722 getSpirvLoopControlOperandsFromLoopMetadata(LoopMD);
3723 unsigned LC = Ops[0];
3724 if (LC == SPIRV::LoopControl::None)
3725 continue;
3726
3727 // Emit intrinsic: loop control mask + optional parameters.
3728 B.SetInsertPoint(Term);
3729 SmallVector<Value *, 4> IntrArgs;
3730 for (unsigned Op : Ops)
3731 IntrArgs.push_back(Elt: B.getInt32(C: Op));
3732 B.CreateIntrinsic(ID: Intrinsic::spv_loop_control_intel, Args: IntrArgs);
3733 }
3734 return;
3735 }
3736
3737 // For non-shader targets without the Intel extension, emit OpLoopMerge
3738 // using spv_loop_merge intrinsics, mirroring the structurizer approach.
3739 LoopInfo LI;
3740 LI.analyze(F: &F);
3741 if (LI.empty())
3742 return;
3743
3744 SmallPtrSet<BasicBlock *, 8> UsedMergeBlocks;
3745 for (Loop *L : LI.getLoopsInPreorder()) {
3746 BasicBlock *Latch = L->getLoopLatch();
3747 if (!Latch)
3748 continue;
3749
3750 // Check for loop unroll metadata on the latch terminator.
3751 SmallVector<unsigned, 1> LoopControlOps =
3752 getSpirvLoopControlOperandsFromLoopMetadata(L);
3753 if (LoopControlOps[0] == SPIRV::LoopControl::None)
3754 continue;
3755
3756 BasicBlock *Header = L->getHeader();
3757 // OpLoopMerge must immediately precede an OpBranch or OpBranchConditional.
3758 // Switches are already lowered to spv_switch + indirectbr at this point.
3759 if (!isa<UncondBrInst, CondBrInst>(Val: Header->getTerminator()))
3760 continue;
3761
3762 BasicBlock *MergeBlock = L->getUniqueExitBlock();
3763 // LoopSimplify does not guarantee a unique exit block. Try the normal exit
3764 // of a rotated loop (from the latch), then an unrotated loop (from the
3765 // header). A merge block cannot be shared by multiple loop headers.
3766 for (BasicBlock *BB : {Latch, Header}) {
3767 if (MergeBlock || !isa<CondBrInst>(Val: BB->getTerminator()))
3768 continue;
3769 for (BasicBlock *Succ : successors(BB))
3770 if (!L->contains(BB: Succ) && !UsedMergeBlocks.contains(Ptr: Succ))
3771 MergeBlock = Succ;
3772 }
3773 if (!MergeBlock || !UsedMergeBlocks.insert(Ptr: MergeBlock).second)
3774 continue;
3775
3776 B.SetInsertPoint(Header->getTerminator());
3777 auto *MergeAddress = BlockAddress::get(F: &F, BB: MergeBlock);
3778 auto *ContinueAddress = BlockAddress::get(F: &F, BB: Latch);
3779 SmallVector<Value *, 4> Args = {MergeAddress, ContinueAddress};
3780 for (unsigned Imm : LoopControlOps)
3781 Args.emplace_back(Args: B.getInt32(C: Imm));
3782 B.CreateIntrinsic(ID: Intrinsic::spv_loop_merge, Args: {Args});
3783 }
3784}
3785
3786bool SPIRVEmitIntrinsicsImpl::runOnFunction(Function &Func) {
3787 if (Func.isDeclaration())
3788 return false;
3789
3790 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: Func);
3791 // LoopSimplify runs after SPIRVPrepareFunctions sorted the blocks, and the
3792 // preheaders/dedicated exits it creates can end up before their dominator,
3793 // which SPIR-V forbids.
3794 if (!ST.isShader())
3795 sortBlocks(F&: Func);
3796 GR = ST.getSPIRVGlobalRegistry();
3797
3798 if (!CurrF)
3799 HaveFunPtrs =
3800 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers);
3801
3802 CanUseAnyVectorRank =
3803 ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector);
3804 CurrF = &Func;
3805 IRBuilder<> B(*Func.getParent());
3806 AggrConsts.clear();
3807 AggrConstTypes.clear();
3808 AggrStores.clear();
3809
3810 processParamTypesByFunHeader(F: CurrF, B);
3811
3812 // Fix GEP result types ahead of inference, and simplify if possible.
3813 // Data structure for dead instructions that were simplified and replaced.
3814 SmallPtrSet<Instruction *, 4> DeadInsts;
3815 for (auto &I : instructions(F&: Func)) {
3816 if (StoreInst *SI = dyn_cast<StoreInst>(Val: &I)) {
3817 Type *ElTy = SI->getValueOperand()->getType();
3818 if (ElTy->isAggregateType() || ElTy->isVectorTy())
3819 AggrStores.insert(Ptr: &I);
3820 continue;
3821 }
3822
3823 auto *GEP = dyn_cast<GetElementPtrInst>(Val: &I);
3824 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
3825
3826 if ((!GEP && !SGEP) || GR->findDeducedElementType(Val: &I))
3827 continue;
3828
3829 if (SGEP) {
3830 GR->addDeducedElementType(
3831 Val: SGEP,
3832 Ty: normalizeType(Ty: SGEP->getResultElementType(), CanUseAnyVectorRank));
3833 continue;
3834 }
3835
3836 GetElementPtrInst *NewGEP = simplifyZeroLengthArrayGepInst(GEP);
3837 if (NewGEP) {
3838 GEP->replaceAllUsesWith(V: NewGEP);
3839 DeadInsts.insert(Ptr: GEP);
3840 GEP = NewGEP;
3841 }
3842 if (Type *GepTy = getGEPType(Ref: GEP))
3843 GR->addDeducedElementType(Val: GEP, Ty: normalizeType(Ty: GepTy, CanUseAnyVectorRank));
3844 }
3845 // Remove dead instructions that were simplified and replaced.
3846 for (auto *I : DeadInsts) {
3847 assert(I->use_empty() && "Dead instruction should not have any uses left");
3848 I->eraseFromParent();
3849 }
3850
3851 B.SetInsertPoint(Func.getEntryBlock().begin());
3852 for (auto &GV : Func.getParent()->globals())
3853 processGlobalValue(GV, B);
3854
3855 reconstructAggregateReturns(Func, B);
3856 preprocessUndefsAndPoisons(B);
3857 simplifyNullAddrSpaceCasts();
3858 preprocessCompositeConstants(B);
3859
3860 // A PHINode, SelectInst or FreezeInst takes its result type from its
3861 // operands. Aggregate arms are lowered to i32 value-ids (composite constants
3862 // here, loads and other producers during the visitor pass below), so mutate
3863 // an aggregate PHI, select or freeze to match. The original type is tracked
3864 // in AggrConstTypes (used to assign the SPIR-V type) and its extractvalue
3865 // users are lowered to spv_extractv.
3866 Type *I32Ty = B.getInt32Ty();
3867 for (Instruction &I : instructions(F&: Func)) {
3868 if (!isAggregateValueIdInstr(I))
3869 continue;
3870 // Give multi-register arms a value-id first, before the result is mutated.
3871 insertCompositeAggregateArms(I: &I, B);
3872 AggrConstTypes[&I] = I.getType();
3873 I.mutateType(Ty: I32Ty);
3874 }
3875
3876 preprocessBoolVectorBitcasts(F&: Func);
3877 SmallVector<Instruction *> Worklist(
3878 llvm::make_pointer_range(Range: instructions(F&: Func)));
3879
3880 applyDemangledPtrArgTypes(B);
3881
3882 // Pass forward: use operand to deduce instructions result.
3883 for (auto &I : Worklist) {
3884 // Don't emit intrinsincs for convergence intrinsics.
3885 if (isConvergenceIntrinsic(I))
3886 continue;
3887
3888 bool Postpone = insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: false);
3889 // if Postpone is true, we can't decide on pointee type yet
3890 insertAssignTypeIntrs(I, B);
3891 insertPtrCastOrAssignTypeInstr(I, B);
3892 insertSpirvDecorations(I, B);
3893 // if instruction requires a pointee type set, let's check if we know it
3894 // already, and force it to be i8 if not
3895 if (Postpone && !GR->findAssignPtrTypeInstr(Val: I))
3896 insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: true);
3897
3898 if (auto *FPI = dyn_cast<ConstrainedFPIntrinsic>(Val: I))
3899 useRoundingMode(FPI, B);
3900 }
3901
3902 // Pass backward: use instructions results to specify/update/cast operands
3903 // where needed.
3904 SmallPtrSet<Instruction *, 4> IncompleteRets;
3905 for (auto &I : llvm::reverse(C: instructions(F&: Func)))
3906 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets);
3907
3908 // Pass forward for PHIs only, their operands are not preceed the
3909 // instruction in meaning of `instructions(Func)`.
3910 for (BasicBlock &BB : Func)
3911 for (PHINode &Phi : BB.phis())
3912 if (isPointerTy(T: Phi.getType()))
3913 deduceOperandElementType(I: &Phi, IncompleteRets: nullptr);
3914
3915 for (auto *I : Worklist) {
3916 TrackConstants = true;
3917 if (!I->getType()->isVoidTy() || isa<StoreInst>(Val: I))
3918 setInsertPointAfterDef(B, I);
3919 // Visitors return either the original/newly created instruction for
3920 // further processing, nullptr otherwise.
3921 I = visit(I&: *I);
3922 if (!I)
3923 continue;
3924
3925 // Don't emit intrinsics for convergence operations.
3926 if (isConvergenceIntrinsic(I))
3927 continue;
3928
3929 addSaturatedDecorationToIntrinsic(I, B);
3930 processInstrAfterVisit(I, B);
3931 }
3932
3933 emitUnstructuredLoopControls(F&: Func, B);
3934
3935 return true;
3936}
3937
3938// Try to deduce a better type for pointers to untyped ptr.
3939bool SPIRVEmitIntrinsicsImpl::postprocessTypes(Module &M) {
3940 if (!GR || TodoTypeSz == 0)
3941 return false;
3942
3943 unsigned SzTodo = TodoTypeSz;
3944 DenseMap<Value *, SmallPtrSet<Value *, 4>> ToProcess;
3945 for (auto [Op, Enabled] : TodoType) {
3946 // TODO: add isa<CallInst>(Op) to continue
3947 if (!Enabled || isaGEP(V: Op))
3948 continue;
3949 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Op);
3950 Type *KnownTy = GR->findDeducedElementType(Val: Op);
3951 if (!KnownTy || !AssignCI)
3952 continue;
3953 assert(Op == AssignCI->getArgOperand(0));
3954 // Try to improve the type deduced after all Functions are processed.
3955 if (auto *CI = dyn_cast<Instruction>(Val: Op)) {
3956 CurrF = CI->getParent()->getParent();
3957 SmallPtrSet<Value *, 0> Visited;
3958 if (Type *ElemTy = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8: false, IgnoreKnownType: true)) {
3959 if (ElemTy != KnownTy) {
3960 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3961 propagateElemType(Op: CI, ElemTy, VisitedSubst);
3962 eraseTodoType(Op);
3963 continue;
3964 }
3965 }
3966 }
3967
3968 if (Op->hasUseList()) {
3969 for (User *U : Op->users()) {
3970 Instruction *Inst = dyn_cast<Instruction>(Val: U);
3971 if (Inst && !isa<IntrinsicInst>(Val: Inst))
3972 ToProcess[Inst].insert(Ptr: Op);
3973 }
3974 }
3975 }
3976 if (TodoTypeSz == 0)
3977 return true;
3978
3979 for (auto &F : M) {
3980 CurrF = &F;
3981 SmallPtrSet<Instruction *, 4> IncompleteRets;
3982 for (auto &I : llvm::reverse(C: instructions(F))) {
3983 auto It = ToProcess.find(Val: &I);
3984 if (It == ToProcess.end())
3985 continue;
3986 It->second.remove_if(P: [this](Value *V) { return !isTodoType(Op: V); });
3987 if (It->second.size() == 0)
3988 continue;
3989 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets, AskOps: &It->second, IsPostprocessing: true);
3990 if (TodoTypeSz == 0)
3991 return true;
3992 }
3993 }
3994
3995 return SzTodo > TodoTypeSz;
3996}
3997
3998// Parse unambiguous scalar/vector pointees, e.g. "float vector[4] const*".
3999static Type *parseDeclPtrElemType(StringRef TypeStr, LLVMContext &Ctx) {
4000 if (!TypeStr.consume_back(Suffix: "*") && !TypeStr.consume_back(Suffix: "&"))
4001 return nullptr;
4002 StringRef ScalarName = TypeStr;
4003 Type *ElemTy = parseBasicTypeName(TypeName&: TypeStr, Ctx);
4004 if (!ElemTy || ElemTy->isVoidTy() || ElemTy->isIntegerTy(BitWidth: 1))
4005 return nullptr;
4006 ScalarName = ScalarName.drop_back(N: TypeStr.size());
4007 // OpenCL aliases such as uint can also name C++ classes.
4008 if (!StringSwitch<bool>(ScalarName)
4009 .Cases(CaseStrings: {"char", "signed char", "unsigned char", "short",
4010 "unsigned short", "int", "unsigned int", "_Float16", "float",
4011 "double"},
4012 Value: true)
4013 .Default(Value: false) ||
4014 (!TypeStr.empty() && !TypeStr.starts_with(Prefix: " ")))
4015 return nullptr;
4016 if (TypeStr.consume_front(Prefix: " vector[")) {
4017 auto [Count, Rest] = TypeStr.split(Separator: ']');
4018 unsigned NumElts = 0;
4019 if (Count.getAsInteger(Radix: 10, Result&: NumElts) || NumElts == 0)
4020 return nullptr;
4021 ElemTy = FixedVectorType::get(ElementType: ElemTy, NumElts);
4022 TypeStr = Rest;
4023 }
4024 // Only cv and address space qualifiers may follow.
4025 SmallVector<StringRef, 4> Quals;
4026 TypeStr.split(A&: Quals, Separator: ' ', MaxSplit: -1, /*KeepEmpty=*/false);
4027 for (StringRef Qual : Quals) {
4028 unsigned AS;
4029 if (Qual != "const" && Qual != "volatile" && Qual != "restrict" &&
4030 !(Qual.consume_front(Prefix: "AS") && !Qual.getAsInteger(Radix: 10, Result&: AS)))
4031 return nullptr;
4032 }
4033 return ElemTy;
4034}
4035
4036// Parse and store argument types of function declarations where needed.
4037void SPIRVEmitIntrinsicsImpl::parseFunDeclarations(Module &M) {
4038 for (auto &F : M) {
4039 if (!F.isDeclaration() || F.isIntrinsic())
4040 continue;
4041 std::string DemangledName = getOclOrSpirvBuiltinDemangledName(Name: F.getName());
4042 if (DemangledName.empty())
4043 continue;
4044 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F);
4045 auto Set = ST.getPreferredInstructionSet();
4046 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(DemangledCall: DemangledName, Set);
4047 // Some builtins have no opcode mapping.
4048 const bool IsBuiltin = Grp >= 0 || SPIRV::isBuiltin(DemangledCall: DemangledName, Set);
4049 if (IsBuiltin && Opcode != SPIRV::OpGroupAsyncCopy)
4050 continue;
4051 SmallVector<unsigned> Idxs;
4052 for (Argument &Arg : F.args())
4053 if (isPointerTy(T: Arg.getType()) && !hasPointeeTypeAttr(Arg: &Arg))
4054 Idxs.push_back(Elt: Arg.getArgNo());
4055 if (Idxs.empty())
4056 continue;
4057 LLVMContext &Ctx = F.getContext();
4058 SmallVector<StringRef, 10> TypeStrs;
4059 SPIRV::parseBuiltinTypeStr(BuiltinArgsTypeStrs&: TypeStrs, DemangledCall: DemangledName, Ctx);
4060 if (TypeStrs.empty())
4061 continue;
4062 // Avoid nested signatures and hidden ABI arguments such as this and sret.
4063 if (!IsBuiltin && (!F.getName().starts_with(Prefix: "_Z") || F.hasStructRetAttr() ||
4064 StringRef(DemangledName).count(C: '(') != 1 ||
4065 StringRef(DemangledName).count(C: ')') != 1 ||
4066 StringRef(DemangledName).contains(C: '<') ||
4067 TypeStrs.size() != F.arg_size()))
4068 continue;
4069 bool AllowLongVectors =
4070 ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector);
4071 for (unsigned Idx : Idxs) {
4072 if (Idx >= TypeStrs.size())
4073 continue;
4074 StringRef TypeStr = TypeStrs[Idx].trim();
4075 Type *ElemTy = IsBuiltin
4076 ? SPIRV::parseBuiltinCallArgumentType(TypeStr, Ctx)
4077 : parseDeclPtrElemType(TypeStr, Ctx);
4078 if (!ElemTy || !TypedPointerType::isValidElementType(ElemTy) ||
4079 ElemTy->isTargetExtTy())
4080 continue;
4081 if (!IsBuiltin) {
4082 ElemTy = normalizeType(Ty: ElemTy, CanUseAnyVectorRank: AllowLongVectors);
4083 if (!AllowLongVectors && isLongVectorEXT(Ty: ElemTy))
4084 continue;
4085 }
4086 FDeclPtrTys[&F].emplace_back(Args&: Idx, Args&: ElemTy);
4087 if (!IsBuiltin)
4088 ST.getSPIRVGlobalRegistry()->addDeducedElementType(Val: F.getArg(i: Idx),
4089 Ty: ElemTy);
4090 }
4091 }
4092}
4093
4094bool SPIRVEmitIntrinsicsImpl::processMaskedMemIntrinsic(IntrinsicInst &I) {
4095 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
4096
4097 if (I.getIntrinsicID() == Intrinsic::masked_gather) {
4098 if (!ST.canUseExtension(
4099 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
4100 I.getContext().emitError(
4101 I: &I, ErrorStr: "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter "
4102 "extension");
4103 // Replace with poison to allow compilation to continue and report error.
4104 I.replaceAllUsesWith(V: PoisonValue::get(T: I.getType()));
4105 I.eraseFromParent();
4106 return true;
4107 }
4108
4109 IRBuilder<> B(&I);
4110
4111 Value *Ptrs = I.getArgOperand(i: 0);
4112 Value *Mask = I.getArgOperand(i: 1);
4113 Value *Passthru = I.getArgOperand(i: 2);
4114
4115 // Alignment is stored as a parameter attribute, not as a regular parameter.
4116 uint32_t Alignment = I.getParamAlign(ArgNo: 0).valueOrOne().value();
4117
4118 SmallVector<Value *, 4> Args = {Ptrs, B.getInt32(C: Alignment), Mask,
4119 Passthru};
4120 SmallVector<Type *, 4> Types = {I.getType(), Ptrs->getType(),
4121 Mask->getType(), Passthru->getType()};
4122
4123 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_masked_gather, OverloadTypes: Types, Args);
4124 I.replaceAllUsesWith(V: NewI);
4125 I.eraseFromParent();
4126 return true;
4127 }
4128
4129 if (I.getIntrinsicID() == Intrinsic::masked_scatter) {
4130 if (!ST.canUseExtension(
4131 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
4132 I.getContext().emitError(
4133 I: &I, ErrorStr: "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter "
4134 "extension");
4135 // Erase the intrinsic to allow compilation to continue and report error.
4136 I.eraseFromParent();
4137 return true;
4138 }
4139
4140 IRBuilder<> B(&I);
4141
4142 Value *Values = I.getArgOperand(i: 0);
4143 Value *Ptrs = I.getArgOperand(i: 1);
4144 Value *Mask = I.getArgOperand(i: 2);
4145
4146 // Alignment is stored as a parameter attribute on the ptrs parameter (arg
4147 // 1).
4148 uint32_t Alignment = I.getParamAlign(ArgNo: 1).valueOrOne().value();
4149
4150 SmallVector<Value *, 4> Args = {Values, Ptrs, B.getInt32(C: Alignment), Mask};
4151 SmallVector<Type *, 3> Types = {Values->getType(), Ptrs->getType(),
4152 Mask->getType()};
4153
4154 B.CreateIntrinsic(ID: Intrinsic::spv_masked_scatter, OverloadTypes: Types, Args);
4155 I.eraseFromParent();
4156 return true;
4157 }
4158
4159 return false;
4160}
4161
4162// SPIR-V doesn't support bitcasts involving vector boolean type. Decompose such
4163// bitcasts into element-wise operations before building instructions
4164// worklist, so new instructions are properly visited and converted to
4165// SPIR-V intrinsics.
4166void SPIRVEmitIntrinsicsImpl::preprocessBoolVectorBitcasts(Function &F) {
4167 struct BoolVecBitcast {
4168 BitCastInst *BC;
4169 FixedVectorType *BoolVecTy;
4170 bool SrcIsBoolVec;
4171 };
4172
4173 auto getAsBoolVec = [](Type *Ty) -> FixedVectorType * {
4174 auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
4175 return (VTy && VTy->getElementType()->isIntegerTy(BitWidth: 1)) ? VTy : nullptr;
4176 };
4177
4178 SmallVector<BoolVecBitcast, 4> ToReplace;
4179 for (auto &I : instructions(F)) {
4180 auto *BC = dyn_cast<BitCastInst>(Val: &I);
4181 if (!BC)
4182 continue;
4183 if (auto *BVTy = getAsBoolVec(BC->getSrcTy()))
4184 ToReplace.push_back(Elt: {.BC: BC, .BoolVecTy: BVTy, .SrcIsBoolVec: true});
4185 else if (auto *BVTy = getAsBoolVec(BC->getDestTy()))
4186 ToReplace.push_back(Elt: {.BC: BC, .BoolVecTy: BVTy, .SrcIsBoolVec: false});
4187 }
4188
4189 for (auto &[BC, BoolVecTy, SrcIsBoolVec] : ToReplace) {
4190 IRBuilder<> B(BC);
4191 Value *Src = BC->getOperand(i_nocapture: 0);
4192 unsigned BoolVecN = BoolVecTy->getNumElements();
4193 // Convert per element of the other side, as iN can exceed the maximum
4194 // SPIR-V integer width.
4195 Type *OtherTy = SrcIsBoolVec ? BC->getDestTy() : Src->getType();
4196 unsigned NumElts = 1;
4197 if (auto *VTy = dyn_cast<FixedVectorType>(Val: OtherTy))
4198 NumElts = VTy->getNumElements();
4199 unsigned EltBits = OtherTy->getScalarSizeInBits();
4200 Type *EltTy = B.getIntNTy(N: EltBits);
4201 Type *IntTy = NumElts > 1 ? FixedVectorType::get(ElementType: EltTy, NumElts) : EltTy;
4202
4203 Value *Result;
4204 if (!SrcIsBoolVec) {
4205 Value *IntVal = B.CreateBitCast(V: Src, DestTy: IntTy);
4206 // Lane I is bit I % EltBits of element I / EltBits.
4207 auto GetLane = [&](unsigned I) {
4208 Value *Elt = IntVal;
4209 if (NumElts > 1)
4210 Elt = B.CreateExtractElement(Vec: IntVal, Idx: B.getInt32(C: I / EltBits));
4211 Value *Mask =
4212 ConstantInt::get(Ty: EltTy, V: APInt::getOneBitSet(numBits: EltBits, BitNo: I % EltBits));
4213 Value *And = B.CreateAnd(LHS: Elt, RHS: Mask);
4214 return B.CreateICmpNE(LHS: And, RHS: ConstantInt::get(Ty: EltTy, V: 0));
4215 };
4216
4217 // Compute constant-index extracts directly, as no later pass removes
4218 // the unused lanes.
4219 for (User *U : make_early_inc_range(Range: BC->users())) {
4220 auto *EE = dyn_cast<ExtractElementInst>(Val: U);
4221 auto *Idx = EE ? dyn_cast<ConstantInt>(Val: EE->getIndexOperand()) : nullptr;
4222 if (!Idx || Idx->getValue().uge(RHS: BoolVecN))
4223 continue;
4224 B.SetInsertPoint(EE);
4225 EE->replaceAllUsesWith(V: GetLane(Idx->getZExtValue()));
4226 EE->eraseFromParent();
4227 }
4228 if (BC->use_empty()) {
4229 BC->eraseFromParent();
4230 continue;
4231 }
4232
4233 B.SetInsertPoint(BC);
4234 Result = PoisonValue::get(T: BoolVecTy);
4235 for (unsigned I = 0; I < BoolVecN; ++I)
4236 Result = B.CreateInsertElement(Vec: Result, NewElt: GetLane(I), Idx: B.getInt32(C: I));
4237 } else {
4238 // Build each element from its lanes.
4239 Value *IntVal = PoisonValue::get(T: IntTy);
4240 for (unsigned J = 0; J < NumElts; ++J) {
4241 Value *Elt = ConstantInt::get(Ty: EltTy, V: 0);
4242 for (unsigned Bit = 0; Bit < EltBits; ++Bit) {
4243 Value *Lane =
4244 B.CreateExtractElement(Vec: Src, Idx: B.getInt32(C: J * EltBits + Bit));
4245 Value *Ext = B.CreateZExt(V: Lane, DestTy: EltTy);
4246 if (Bit > 0)
4247 Ext = B.CreateShl(LHS: Ext, RHS: ConstantInt::get(Ty: EltTy, V: Bit));
4248 Elt = B.CreateOr(LHS: Elt, RHS: Ext);
4249 }
4250 IntVal = NumElts > 1 ? B.CreateInsertElement(Vec: IntVal, NewElt: Elt, Idx: B.getInt32(C: J))
4251 : Elt;
4252 }
4253 Result = B.CreateBitCast(V: IntVal, DestTy: BC->getDestTy());
4254 }
4255
4256 BC->replaceAllUsesWith(V: Result);
4257 BC->eraseFromParent();
4258 }
4259}
4260
4261bool SPIRVEmitIntrinsicsImpl::convertMaskedMemIntrinsics(Module &M) {
4262 bool Changed = false;
4263
4264 for (Function &F : make_early_inc_range(Range&: M)) {
4265 if (!F.isIntrinsic())
4266 continue;
4267 Intrinsic::ID IID = F.getIntrinsicID();
4268 if (IID != Intrinsic::masked_gather && IID != Intrinsic::masked_scatter)
4269 continue;
4270
4271 for (User *U : make_early_inc_range(Range: F.users())) {
4272 if (auto *II = dyn_cast<IntrinsicInst>(Val: U))
4273 Changed |= processMaskedMemIntrinsic(I&: *II);
4274 }
4275
4276 if (F.use_empty())
4277 F.eraseFromParent();
4278 }
4279
4280 return Changed;
4281}
4282
4283bool SPIRVEmitIntrinsicsImpl::runOnModule(Module &M) {
4284 bool Changed = false;
4285
4286 Changed |= convertMaskedMemIntrinsics(M);
4287
4288 parseFunDeclarations(M);
4289 insertConstantsForFPFastMathDefault(M);
4290 GVUsers.init(M);
4291
4292 TodoType.clear();
4293 for (auto &F : M)
4294 Changed |= runOnFunction(Func&: F);
4295
4296 // Specify function parameters after all functions were processed.
4297 for (auto &F : M) {
4298 // check if function parameter types are set
4299 CurrF = &F;
4300 if (!F.isDeclaration() && !F.isIntrinsic()) {
4301 IRBuilder<> B(M);
4302 processParamTypes(F: &F, B);
4303 }
4304 }
4305
4306 CanTodoType = false;
4307 Changed |= postprocessTypes(M);
4308
4309 if (HaveFunPtrs)
4310 Changed |= processFunctionPointers(M);
4311
4312 return Changed;
4313}
4314
4315PreservedAnalyses
4316llvm::SPIRVEmitIntrinsicsPass::run(Module &M, ModuleAnalysisManager &AM) {
4317 if (SPIRVEmitIntrinsicsImpl(TM).runOnModule(M))
4318 return PreservedAnalyses::none();
4319 return PreservedAnalyses::all();
4320}
4321
4322ModulePass *llvm::createSPIRVEmitIntrinsicsPass(const SPIRVTargetMachine &TM) {
4323 return new SPIRVEmitIntrinsicsLegacy(TM);
4324}
4325