1//===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass emits SPIRV intrinsics keeping essential high-level information for
10// the translation of LLVM IR to SPIR-V.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRV.h"
15#include "SPIRVBuiltins.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVTargetMachine.h"
18#include "SPIRVUtils.h"
19#include "llvm/ADT/DenseMap.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/StringSwitch.h"
24#include "llvm/Analysis/LoopInfo.h"
25#include "llvm/IR/IRBuilder.h"
26#include "llvm/IR/InstIterator.h"
27#include "llvm/IR/InstVisitor.h"
28#include "llvm/IR/IntrinsicsSPIRV.h"
29#include "llvm/IR/LLVMContext.h"
30#include "llvm/IR/PatternMatch.h"
31#include "llvm/IR/TypedPointerType.h"
32#include "llvm/IR/Value.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/raw_ostream.h"
36#include "llvm/Transforms/Utils/Local.h"
37
38#include <cassert>
39#include <optional>
40#include <queue>
41
42// This pass performs the following transformation on LLVM IR level required
43// for the following translation to SPIR-V:
44// - replaces direct usages of aggregate constants with target-specific
45// intrinsics;
46// - replaces aggregates-related instructions (extract/insert, ld/st, etc)
47// with a target-specific intrinsics;
48// - emits intrinsics for the global variable initializers since IRTranslator
49// doesn't handle them and it's not very convenient to translate them
50// ourselves;
51// - emits intrinsics to keep track of the string names assigned to the values;
52// - emits intrinsics to keep track of constants (this is necessary to have an
53// LLVM IR constant after the IRTranslation is completed) for their further
54// deduplication;
55// - emits intrinsics to keep track of original LLVM types of the values
56// to be able to emit proper SPIR-V types eventually.
57//
58// TODO: consider removing spv.track.constant in favor of spv.assign.type.
59
60using namespace llvm;
61using namespace llvm::PatternMatch;
62
63#define DEBUG_TYPE "spirv-emit-intrinsics"
64
65static cl::opt<bool>
66 SpirvEmitOpNames("spirv-emit-op-names",
67 cl::desc("Emit OpName for all instructions"),
68 cl::init(Val: false));
69
70namespace llvm::SPIRV {
71#define GET_BuiltinGroup_DECL
72#include "SPIRVGenTables.inc"
73} // namespace llvm::SPIRV
74
75namespace {
76// This class keeps track of which functions reference which global variables.
77class GlobalVariableUsers {
78 template <typename T1, typename T2>
79 using OneToManyMapTy = DenseMap<T1, SmallPtrSet<T2, 4>>;
80
81 OneToManyMapTy<const GlobalVariable *, const Function *> GlobalIsUsedByFun;
82
83 void collectGlobalUsers(
84 const GlobalVariable *GV,
85 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
86 &GlobalIsUsedByGlobal) {
87 SmallVector<const Value *> Stack = {GV->user_begin(), GV->user_end()};
88 while (!Stack.empty()) {
89 const Value *V = Stack.pop_back_val();
90
91 if (const Instruction *I = dyn_cast<Instruction>(Val: V)) {
92 GlobalIsUsedByFun[GV].insert(Ptr: I->getFunction());
93 continue;
94 }
95
96 if (const GlobalVariable *UserGV = dyn_cast<GlobalVariable>(Val: V)) {
97 GlobalIsUsedByGlobal[GV].insert(Ptr: UserGV);
98 continue;
99 }
100
101 if (const Constant *C = dyn_cast<Constant>(Val: V))
102 Stack.append(in_start: C->user_begin(), in_end: C->user_end());
103 }
104 }
105
106 bool propagateGlobalToGlobalUsers(
107 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
108 &GlobalIsUsedByGlobal) {
109 SmallVector<const GlobalVariable *> OldUsersGlobals;
110 bool Changed = false;
111 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
112 OldUsersGlobals.assign(in_start: UserGlobals.begin(), in_end: UserGlobals.end());
113 for (const GlobalVariable *UserGV : OldUsersGlobals) {
114 auto It = GlobalIsUsedByGlobal.find(Val: UserGV);
115 if (It == GlobalIsUsedByGlobal.end())
116 continue;
117 Changed |= set_union(S1&: UserGlobals, S2: It->second);
118 }
119 }
120 return Changed;
121 }
122
123 void propagateGlobalToFunctionReferences(
124 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
125 &GlobalIsUsedByGlobal) {
126 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
127 auto &UserFunctions = GlobalIsUsedByFun[GV];
128 for (const GlobalVariable *UserGV : UserGlobals) {
129 auto It = GlobalIsUsedByFun.find(Val: UserGV);
130 if (It == GlobalIsUsedByFun.end())
131 continue;
132 set_union(S1&: UserFunctions, S2: It->second);
133 }
134 }
135 }
136
137public:
138 void init(Module &M) {
139 // Collect which global variables are referenced by which global variables
140 // and which functions reference each global variables.
141 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
142 GlobalIsUsedByGlobal;
143 GlobalIsUsedByFun.clear();
144 for (GlobalVariable &GV : M.globals())
145 collectGlobalUsers(GV: &GV, GlobalIsUsedByGlobal);
146
147 // Compute indirect references by iterating until a fixed point is reached.
148 while (propagateGlobalToGlobalUsers(GlobalIsUsedByGlobal))
149 (void)0;
150
151 propagateGlobalToFunctionReferences(GlobalIsUsedByGlobal);
152 }
153
154 using FunctionSetType = typename decltype(GlobalIsUsedByFun)::mapped_type;
155 const FunctionSetType &
156 getTransitiveUserFunctions(const GlobalVariable &GV) const {
157 auto It = GlobalIsUsedByFun.find(Val: &GV);
158 if (It != GlobalIsUsedByFun.end())
159 return It->second;
160
161 static const FunctionSetType Empty{};
162 return Empty;
163 }
164};
165
166static bool isaGEP(const Value *V) {
167 return isa<StructuredGEPInst>(Val: V) || isa<GetElementPtrInst>(Val: V);
168}
169
170// If Ty is a byte-addressing type, return the multiplier for the offset.
171// Otherwise return std::nullopt.
172static std::optional<uint64_t> getByteAddressingMultiplier(Type *Ty) {
173 if (Ty == IntegerType::getInt8Ty(C&: Ty->getContext())) {
174 return 1;
175 }
176 if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
177 if (AT->getElementType() == IntegerType::getInt8Ty(C&: Ty->getContext())) {
178 return AT->getNumElements();
179 }
180 }
181 return std::nullopt;
182}
183
184class SPIRVEmitIntrinsicsImpl
185 : public InstVisitor<SPIRVEmitIntrinsicsImpl, Instruction *> {
186 const SPIRVTargetMachine &TM;
187 SPIRVGlobalRegistry *GR = nullptr;
188 Function *CurrF = nullptr;
189 bool TrackConstants = true;
190 bool HaveFunPtrs = false;
191 bool CanUseAnyVectorRank = false;
192 DenseMap<Instruction *, Constant *> AggrConsts;
193 DenseMap<Instruction *, Type *> AggrConstTypes;
194 SmallPtrSet<Instruction *, 0> AggrStores;
195 GlobalVariableUsers GVUsers;
196 SmallPtrSet<Value *, 0> Named;
197
198 // map of function declarations to <pointer arg index => element type>
199 DenseMap<Function *, SmallVector<std::pair<unsigned, Type *>>> FDeclPtrTys;
200
201 // a register of Instructions that don't have a complete type definition
202 bool CanTodoType = true;
203 unsigned TodoTypeSz = 0;
204 DenseMap<Value *, bool> TodoType;
205 void insertTodoType(Value *Op) {
206 // TODO: add isa<CallInst>(Op) to no-insert
207 if (CanTodoType && !isaGEP(V: Op)) {
208 auto It = TodoType.try_emplace(Key: Op, Args: true);
209 if (It.second)
210 ++TodoTypeSz;
211 }
212 }
213 void eraseTodoType(Value *Op) {
214 auto It = TodoType.find(Val: Op);
215 if (It != TodoType.end() && It->second) {
216 It->second = false;
217 --TodoTypeSz;
218 }
219 }
220 bool isTodoType(Value *Op) {
221 if (isaGEP(V: Op))
222 return false;
223 auto It = TodoType.find(Val: Op);
224 return It != TodoType.end() && It->second;
225 }
226 // a register of Instructions that were visited by deduceOperandElementType()
227 // to validate operand types with an instruction
228 SmallPtrSet<Instruction *, 0> TypeValidated;
229
230 // well known result types of builtins
231 enum WellKnownTypes { Event };
232
233 // deduce element type of untyped pointers
234 Type *deduceElementType(Value *I, bool UnknownElemTypeI8);
235 Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8);
236 Type *deduceElementTypeHelper(Value *I, SmallPtrSetImpl<Value *> &Visited,
237 bool UnknownElemTypeI8,
238 bool IgnoreKnownType = false);
239 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
240 bool UnknownElemTypeI8);
241 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
242 SmallPtrSetImpl<Value *> &Visited,
243 bool UnknownElemTypeI8);
244 Type *deduceElementTypeByUsersDeep(Value *Op,
245 SmallPtrSetImpl<Value *> &Visited,
246 bool UnknownElemTypeI8);
247 void maybeAssignPtrType(Type *&Ty, Value *I, Type *RefTy,
248 bool UnknownElemTypeI8);
249
250 // deduce nested types of composites
251 Type *deduceNestedTypeHelper(User *U, bool UnknownElemTypeI8);
252 Type *deduceNestedTypeHelper(User *U, Type *Ty,
253 SmallPtrSetImpl<Value *> &Visited,
254 bool UnknownElemTypeI8);
255
256 // deduce Types of operands of the Instruction if possible
257 void
258 deduceOperandElementType(Instruction *I,
259 SmallPtrSetImpl<Instruction *> *IncompleteRets,
260 const SmallPtrSetImpl<Value *> *AskOps = nullptr,
261 bool IsPostprocessing = false);
262
263 void preprocessCompositeConstants(IRBuilder<> &B);
264 Value *lowerUndefOrPoison(Value *Op, IRBuilder<> &B, bool HasPoisonExt);
265 void preprocessUndefsAndPoisons(IRBuilder<> &B);
266 void insertCompositeAggregateArms(Instruction *I, IRBuilder<> &B);
267 void simplifyNullAddrSpaceCasts();
268
269 Type *reconstructType(Value *Op, bool UnknownElemTypeI8,
270 bool IsPostprocessing);
271
272 void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
273 void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
274 bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
275 bool UnknownElemTypeI8);
276 void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B);
277 void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V,
278 IRBuilder<> &B);
279 void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer,
280 Type *ExpectedElementType,
281 unsigned OperandToReplace,
282 IRBuilder<> &B);
283 void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
284 bool shouldTryToAddMemAliasingDecoration(Instruction *Inst);
285 void insertSpirvDecorations(Instruction *I, IRBuilder<> &B);
286 void insertConstantsForFPFastMathDefault(Module &M);
287 Value *buildSpvUndefComposite(Type *AggrTy, IRBuilder<> &B);
288 void reconstructAggregateReturns(Function &Func, IRBuilder<> &B);
289 void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
290 void processParamTypes(Function *F, IRBuilder<> &B);
291 void processParamTypesByFunHeader(Function *F, IRBuilder<> &B);
292 Type *deduceFunParamElementType(Function *F, unsigned OpIdx);
293 Type *deduceFunParamElementType(Function *F, unsigned OpIdx,
294 SmallPtrSetImpl<Function *> &FVisited);
295
296 bool deduceOperandElementTypeCalledFunction(
297 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
298 Type *&KnownElemTy, bool &Incomplete);
299 void deduceOperandElementTypeFunctionPointer(
300 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
301 Type *&KnownElemTy, bool IsPostprocessing);
302 bool deduceOperandElementTypeFunctionRet(
303 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
304 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing,
305 Type *&KnownElemTy, Value *Op, Function *F);
306
307 CallInst *buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy);
308 void replaceUsesOfWithSpvPtrcast(Value *Op, Type *ElemTy, Instruction *I,
309 DenseMap<Function *, CallInst *> Ptrcasts);
310 void propagateElemType(Value *Op, Type *ElemTy,
311 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
312 void
313 propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
314 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
315 void propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
316 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
317 SmallPtrSetImpl<Value *> &Visited,
318 DenseMap<Function *, CallInst *> Ptrcasts);
319
320 void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
321 void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src,
322 Instruction *Dest, bool DeleteOld = true);
323
324 void applyDemangledPtrArgTypes(IRBuilder<> &B);
325
326 GetElementPtrInst *simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP);
327
328 bool runOnFunction(Function &F);
329 bool postprocessTypes(Module &M);
330 bool processFunctionPointers(Module &M);
331 void parseFunDeclarations(Module &M);
332 void useRoundingMode(ConstrainedFPIntrinsic *FPI, IRBuilder<> &B);
333 bool processMaskedMemIntrinsic(IntrinsicInst &I);
334 bool convertMaskedMemIntrinsics(Module &M);
335 void preprocessBoolVectorBitcasts(Function &F);
336
337 void emitUnstructuredLoopControls(Function &F, IRBuilder<> &B);
338
339 // Tries to walk the type accessed by the given GEP instruction.
340 // For each nested type access, one of the 2 callbacks is called:
341 // - OnLiteralIndexing when the index is a known constant value.
342 // Parameters:
343 // PointedType: the pointed type resulting of this indexing.
344 // If the parent type is an array, this is the index in the array.
345 // If the parent type is a struct, this is the field index.
346 // Index: index of the element in the parent type.
347 // - OnDynamnicIndexing when the index is a non-constant value.
348 // This callback is only called when indexing into an array.
349 // Parameters:
350 // ElementType: the type of the elements stored in the parent array.
351 // Offset: the Value* containing the byte offset into the array.
352 // Multiplier: a scaling factor for the offset.
353 // Return true if an error occurred during the walk, false otherwise.
354 bool walkLogicalAccessChain(
355 GetElementPtrInst &GEP,
356 const std::function<void(Type *PointedType, uint64_t Index)>
357 &OnLiteralIndexing,
358 const std::function<void(Type *ElementType, Value *Offset,
359 uint64_t Multiplier)> &OnDynamicIndexing);
360
361 bool walkLogicalAccessChainDynamic(
362 Type *CurType, Value *Operand, uint64_t Multiplier,
363 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
364 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing);
365
366 bool walkLogicalAccessChainConstant(
367 Type *CurType, uint64_t Offset,
368 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing);
369
370 // Returns the type accessed using the given GEP instruction by relying
371 // on the GEP type.
372 // FIXME: GEP types are not supposed to be used to retrieve the pointed
373 // type. This must be fixed.
374 Type *getGEPType(GetElementPtrInst *GEP);
375
376 // Returns the type accessed using the given GEP instruction by walking
377 // the source type using the GEP indices.
378 // FIXME: without help from the frontend, this method cannot reliably retrieve
379 // the stored type, nor can robustly determine the depth of the type
380 // we are accessing.
381 Type *getGEPTypeLogical(GetElementPtrInst *GEP);
382
383 Instruction *buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP);
384
385public:
386 SPIRVEmitIntrinsicsImpl(const SPIRVTargetMachine &TM) : TM(TM) {}
387 Instruction *visitInstruction(Instruction &I) { return &I; }
388 Instruction *visitSwitchInst(SwitchInst &I);
389 Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
390 Instruction *visitIntrinsicInst(IntrinsicInst &I);
391 Instruction *visitBitCastInst(BitCastInst &I);
392 Instruction *visitInsertElementInst(InsertElementInst &I);
393 Instruction *visitExtractElementInst(ExtractElementInst &I);
394 Instruction *visitInsertValueInst(InsertValueInst &I);
395 Instruction *visitExtractValueInst(ExtractValueInst &I);
396 Instruction *visitLoadInst(LoadInst &I);
397 Instruction *visitStoreInst(StoreInst &I);
398 Instruction *visitAllocaInst(AllocaInst &I);
399 Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
400 Instruction *visitAtomicRMWInst(AtomicRMWInst &I);
401 Instruction *visitUnreachableInst(UnreachableInst &I);
402 Instruction *visitCallInst(CallInst &I);
403
404 bool runOnModule(Module &M);
405};
406
407class SPIRVEmitIntrinsicsLegacy : public ModulePass {
408 const SPIRVTargetMachine &TM;
409
410public:
411 static char ID;
412 SPIRVEmitIntrinsicsLegacy(const SPIRVTargetMachine &TM)
413 : ModulePass(ID), TM(TM) {}
414
415 StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
416
417 bool runOnModule(Module &M) override {
418 return SPIRVEmitIntrinsicsImpl(TM).runOnModule(M);
419 }
420};
421
422bool isConvergenceIntrinsic(const Instruction *I) {
423 return match(V: I, P: m_AnyIntrinsic<Intrinsic::experimental_convergence_entry,
424 Intrinsic::experimental_convergence_loop,
425 Intrinsic::experimental_convergence_anchor>());
426}
427
428bool expectIgnoredInIRTranslation(const Instruction *I) {
429 return match(V: I, P: m_AnyIntrinsic<Intrinsic::invariant_start,
430 Intrinsic::spv_resource_handlefrombinding,
431 Intrinsic::spv_resource_getbasepointer,
432 Intrinsic::spv_resource_getpointer>());
433}
434
435// Returns the source pointer from `I` ignoring intermediate ptrcast.
436Value *getPointerRoot(Value *I) {
437 Value *V;
438 if (match(V: I, P: m_Intrinsic<Intrinsic::spv_ptrcast>(Ops: m_Value(V))))
439 return getPointerRoot(I: V);
440 return I;
441}
442
443} // namespace
444
445char SPIRVEmitIntrinsicsLegacy::ID = 0;
446
447INITIALIZE_PASS(SPIRVEmitIntrinsicsLegacy, "spirv-emit-intrinsics",
448 "SPIRV emit intrinsics", false, false)
449
450static inline bool isAssignTypeInstr(const Instruction *I) {
451 return match(V: I, P: m_Intrinsic<Intrinsic::spv_assign_type>());
452}
453
454static bool isMemInstrToReplace(Instruction *I) {
455 return isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I) || isa<InsertValueInst>(Val: I) ||
456 isa<ExtractValueInst>(Val: I) || isa<AtomicCmpXchgInst>(Val: I);
457}
458
459static bool isAggrConstForceInt32(const Value *V) {
460 bool IsAggrZero =
461 isa<ConstantAggregateZero>(Val: V) && !V->getType()->isVectorTy();
462 bool IsUndefAggregate = isa<UndefValue>(Val: V) && V->getType()->isAggregateType();
463 return isa<ConstantArray>(Val: V) || isa<ConstantStruct>(Val: V) ||
464 isa<ConstantDataArray>(Val: V) || IsAggrZero || IsUndefAggregate;
465}
466
467static bool isSpvAggrPlaceholder(const Value *V) {
468 return match(
469 V,
470 P: m_AnyIntrinsic<Intrinsic::spv_undef, Intrinsic::spv_const_composite>());
471}
472
473static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
474 if (isa<PHINode>(Val: I))
475 B.SetInsertPoint(I->getParent()->getFirstNonPHIOrDbgOrAlloca());
476 else
477 B.SetInsertPoint(I);
478}
479
480static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I) {
481 B.SetCurrentDebugLocation(I->getDebugLoc());
482 if (I->getType()->isVoidTy())
483 B.SetInsertPoint(I->getNextNode());
484 else
485 B.SetInsertPoint(*I->getInsertionPointAfterDef());
486}
487
488static bool requireAssignType(Instruction *I) {
489 return !match(
490 V: I,
491 P: m_AnyIntrinsic<Intrinsic::invariant_start, Intrinsic::invariant_end>());
492}
493
494static inline void reportFatalOnTokenType(const Instruction *I) {
495 if (I->getType()->isTokenTy())
496 report_fatal_error(reason: "A token is encountered but SPIR-V without extensions "
497 "does not support token type",
498 gen_crash_diag: false);
499}
500
501static void emitAssignName(Instruction *I, IRBuilder<> &B) {
502 if (!I->hasName() || I->getType()->isAggregateType() ||
503 expectIgnoredInIRTranslation(I))
504 return;
505
506 // We want to be conservative when adding the names because they can interfere
507 // with later optimizations.
508 bool KeepName = SpirvEmitOpNames;
509 if (!KeepName) {
510 if (isa<AllocaInst>(Val: I)) {
511 KeepName = true;
512 } else if (auto *CI = dyn_cast<CallBase>(Val: I)) {
513 Function *F = CI->getCalledFunction();
514 if (F && F->getName().starts_with(Prefix: "llvm.spv.alloca"))
515 KeepName = true;
516 }
517 }
518
519 if (!KeepName)
520 return;
521
522 reportFatalOnTokenType(I);
523 setInsertPointAfterDef(B, I);
524 LLVMContext &Ctx = I->getContext();
525 std::vector<Value *> Args = {
526 I, MetadataAsValue::get(
527 Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: I->getName())))};
528 B.CreateIntrinsic(ID: Intrinsic::spv_assign_name, OverloadTypes: {I->getType()}, Args);
529}
530
531void SPIRVEmitIntrinsicsImpl::replaceAllUsesWith(Value *Src, Value *Dest,
532 bool DeleteOld) {
533 GR->replaceAllUsesWith(Old: Src, New: Dest, DeleteOld);
534 // Update uncomplete type records if any
535 if (isTodoType(Op: Src)) {
536 if (DeleteOld)
537 eraseTodoType(Op: Src);
538 insertTodoType(Op: Dest);
539 }
540}
541
542void SPIRVEmitIntrinsicsImpl::replaceAllUsesWithAndErase(IRBuilder<> &B,
543 Instruction *Src,
544 Instruction *Dest,
545 bool DeleteOld) {
546 replaceAllUsesWith(Src, Dest, DeleteOld);
547 std::string Name = Src->hasName() ? Src->getName().str() : "";
548 Src->eraseFromParent();
549 if (!Name.empty()) {
550 Dest->setName(Name);
551 if (Named.insert(Ptr: Dest).second)
552 emitAssignName(I: Dest, B);
553 }
554}
555
556static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
557 return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
558 isPointerTy(T: SI->getValueOperand()->getType()) &&
559 isa<Argument>(Val: SI->getValueOperand());
560}
561
562// A pointer-typed local holds a pointer, so its deduced pointee must stay a
563// pointer.
564static bool tracesToPointerAlloca(Value *V) {
565 using namespace PatternMatch;
566 V = V->stripPointerCasts();
567 if (auto *AI = dyn_cast<AllocaInst>(Val: V))
568 return isUntypedPointerTy(T: AI->getAllocatedType());
569 return match(
570 V, P: m_AnyIntrinsic<Intrinsic::spv_alloca, Intrinsic::spv_alloca_array>());
571}
572
573// Maybe restore original function return type.
574static inline Type *restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I,
575 Type *Ty) {
576 CallInst *CI = dyn_cast<CallInst>(Val: I);
577 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
578 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
579 return Ty;
580 if (Type *OriginalTy = GR->findMutated(Val: CI->getCalledFunction()))
581 return OriginalTy;
582 return Ty;
583}
584
585// Reconstruct type with nested element types according to deduced type info.
586// Return nullptr if no detailed type info is available.
587Type *SPIRVEmitIntrinsicsImpl::reconstructType(Value *Op,
588 bool UnknownElemTypeI8,
589 bool IsPostprocessing) {
590 Type *Ty = Op->getType();
591 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
592 Ty = restoreMutatedType(GR, I: OpI, Ty);
593 if (auto It = AggrConstTypes.find(Val: OpI); It != AggrConstTypes.end())
594 Ty = It->second;
595 }
596 if (!isUntypedPointerTy(T: Ty))
597 return Ty;
598 // try to find the pointee type
599 if (Type *NestedTy = GR->findDeducedElementType(Val: Op))
600 return getTypedPointerWrapper(ElemTy: NestedTy, AS: getPointerAddressSpace(T: Ty));
601 // not a pointer according to the type info (e.g., Event object)
602 CallInst *CI = GR->findAssignPtrTypeInstr(Val: Op);
603 if (CI) {
604 MetadataAsValue *MD = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
605 return cast<ConstantAsMetadata>(Val: MD->getMetadata())->getType();
606 }
607 if (UnknownElemTypeI8) {
608 if (!IsPostprocessing)
609 insertTodoType(Op);
610 return getTypedPointerWrapper(ElemTy: IntegerType::getInt8Ty(C&: Op->getContext()),
611 AS: getPointerAddressSpace(T: Ty));
612 }
613 return nullptr;
614}
615
616CallInst *SPIRVEmitIntrinsicsImpl::buildSpvPtrcast(Function *F, Value *Op,
617 Type *ElemTy) {
618 IRBuilder<> B(*F->getParent());
619 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
620 // spv_ptrcast's argument Op denotes an instruction that generates
621 // a value, and we may use getInsertionPointAfterDef()
622 setInsertPointAfterDef(B, I: OpI);
623 } else if (auto *OpA = dyn_cast<Argument>(Val: Op)) {
624 B.SetInsertPointPastAllocas(OpA->getParent());
625 B.SetCurrentDebugLocation(DebugLoc());
626 } else {
627 B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca());
628 }
629 Type *OpTy = Op->getType();
630 SmallVector<Type *, 2> Types = {OpTy, OpTy};
631 SmallVector<Value *, 2> Args = {
632 Op, buildMD(Arg: getNormalizedPoisonValue(Ty: ElemTy, CanUseAnyVectorRank)),
633 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
634 CallInst *PtrCasted =
635 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
636 GR->buildAssignPtr(B, ElemTy, Arg: PtrCasted);
637 return PtrCasted;
638}
639
640void SPIRVEmitIntrinsicsImpl::replaceUsesOfWithSpvPtrcast(
641 Value *Op, Type *ElemTy, Instruction *I,
642 DenseMap<Function *, CallInst *> Ptrcasts) {
643 Function *F = I->getParent()->getParent();
644 CallInst *PtrCastedI = nullptr;
645 auto It = Ptrcasts.find(Val: F);
646 if (It == Ptrcasts.end()) {
647 PtrCastedI = buildSpvPtrcast(F, Op, ElemTy);
648 Ptrcasts[F] = PtrCastedI;
649 } else {
650 PtrCastedI = It->second;
651 }
652 I->replaceUsesOfWith(From: Op, To: PtrCastedI);
653}
654
655void SPIRVEmitIntrinsicsImpl::propagateElemType(
656 Value *Op, Type *ElemTy,
657 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
658 DenseMap<Function *, CallInst *> Ptrcasts;
659 SmallVector<User *> Users(Op->users());
660 for (auto *U : Users) {
661 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
662 continue;
663 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
664 continue;
665 Instruction *UI = dyn_cast<Instruction>(Val: U);
666 // If the instruction was validated already, we need to keep it valid by
667 // keeping current Op type.
668 if (isaGEP(V: UI) || TypeValidated.find(Ptr: UI) != TypeValidated.end())
669 replaceUsesOfWithSpvPtrcast(Op, ElemTy, I: UI, Ptrcasts);
670 }
671}
672
673void SPIRVEmitIntrinsicsImpl::propagateElemTypeRec(
674 Value *Op, Type *PtrElemTy, Type *CastElemTy,
675 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
676 SmallPtrSet<Value *, 0> Visited;
677 DenseMap<Function *, CallInst *> Ptrcasts;
678 propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited,
679 Ptrcasts: std::move(Ptrcasts));
680}
681
682void SPIRVEmitIntrinsicsImpl::propagateElemTypeRec(
683 Value *Op, Type *PtrElemTy, Type *CastElemTy,
684 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
685 SmallPtrSetImpl<Value *> &Visited,
686 DenseMap<Function *, CallInst *> Ptrcasts) {
687 if (!Visited.insert(Ptr: Op).second)
688 return;
689 SmallVector<User *> Users(Op->users());
690 for (auto *U : Users) {
691 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
692 continue;
693 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
694 continue;
695 Instruction *UI = dyn_cast<Instruction>(Val: U);
696 // If the instruction was validated already, we need to keep it valid by
697 // keeping current Op type.
698 if (isaGEP(V: UI) || TypeValidated.find(Ptr: UI) != TypeValidated.end())
699 replaceUsesOfWithSpvPtrcast(Op, ElemTy: CastElemTy, I: UI, Ptrcasts);
700 }
701}
702
703// Set element pointer type to the given value of ValueTy and tries to
704// specify this type further (recursively) by Operand value, if needed.
705
706Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByValueDeep(
707 Type *ValueTy, Value *Operand, bool UnknownElemTypeI8) {
708 SmallPtrSet<Value *, 0> Visited;
709 return deduceElementTypeByValueDeep(ValueTy, Operand, Visited,
710 UnknownElemTypeI8);
711}
712
713Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByValueDeep(
714 Type *ValueTy, Value *Operand, SmallPtrSetImpl<Value *> &Visited,
715 bool UnknownElemTypeI8) {
716 Type *Ty = ValueTy;
717 if (Operand) {
718 if (auto *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
719 if (Type *NestedTy =
720 deduceElementTypeHelper(I: Operand, Visited, UnknownElemTypeI8))
721 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
722 } else {
723 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Operand), Ty, Visited,
724 UnknownElemTypeI8);
725 }
726 }
727 return Ty;
728}
729
730// Traverse User instructions to deduce an element pointer type of the operand.
731Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByUsersDeep(
732 Value *Op, SmallPtrSetImpl<Value *> &Visited, bool UnknownElemTypeI8) {
733 if (!Op || !isPointerTy(T: Op->getType()) || isa<ConstantPointerNull>(Val: Op) ||
734 isa<UndefValue>(Val: Op))
735 return nullptr;
736
737 if (auto ElemTy = getPointeeType(Ty: Op->getType()))
738 return ElemTy;
739
740 // maybe we already know operand's element type
741 if (Type *KnownTy = GR->findDeducedElementType(Val: Op))
742 return KnownTy;
743
744 for (User *OpU : Op->users()) {
745 if (Instruction *Inst = dyn_cast<Instruction>(Val: OpU)) {
746 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8))
747 return Ty;
748 }
749 }
750 return nullptr;
751}
752
753// Implements what we know in advance about intrinsics and builtin calls
754// TODO: consider feasibility of this particular case to be generalized by
755// encoding knowledge about intrinsics and builtin calls by corresponding
756// specification rules
757static Type *getPointeeTypeByCallInst(StringRef DemangledName,
758 Function *CalledF, unsigned OpIdx) {
759 // OpenCL.std printf takes its format string as a pointer to i8. Match the
760 // bare builtin name, as lowering does, to also cover unmangled `printf`.
761 if (OpIdx == 0 && SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName) == "printf")
762 return IntegerType::getInt8Ty(C&: CalledF->getContext());
763 return nullptr;
764}
765
766// Deduce and return a successfully deduced Type of the Instruction,
767// or nullptr otherwise.
768Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeHelper(Value *I,
769 bool UnknownElemTypeI8) {
770 SmallPtrSet<Value *, 0> Visited;
771 return deduceElementTypeHelper(I, Visited, UnknownElemTypeI8);
772}
773
774void SPIRVEmitIntrinsicsImpl::maybeAssignPtrType(Type *&Ty, Value *Op,
775 Type *RefTy,
776 bool UnknownElemTypeI8) {
777 if (isUntypedPointerTy(T: RefTy)) {
778 if (!UnknownElemTypeI8)
779 return;
780 insertTodoType(Op);
781 if (isa<IntToPtrInst>(Val: Op))
782 return;
783 }
784 Ty = RefTy;
785}
786
787bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChainDynamic(
788 Type *CurType, Value *Operand, uint64_t Multiplier,
789 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
790 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing) {
791 // Dynamic indexing into a struct is not possible.
792 // We know that we must be accessing the first element
793 // of the struct if the current type is a struct.
794 // Try to find the first array type that is at offset 0 in the struct.
795 while (auto *ST = dyn_cast<StructType>(Val: CurType)) {
796 if (ST->getNumElements() == 0)
797 break;
798 CurType = ST->getElementType(N: 0);
799 OnLiteralIndexing(CurType, 0);
800 }
801
802 assert(CurType);
803 ArrayType *AT = dyn_cast<ArrayType>(Val: CurType);
804 // Operand is not constant. Either we have an array and accept it, or we
805 // give up.
806 if (AT)
807 OnDynamicIndexing(AT->getElementType(), Operand, Multiplier);
808 return AT == nullptr;
809}
810
811bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChainConstant(
812 Type *CurType, uint64_t Offset,
813 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing) {
814 auto &DL = CurrF->getDataLayout();
815
816 do {
817 if (ArrayType *AT = dyn_cast<ArrayType>(Val: CurType)) {
818 uint64_t EltTypeSize = DL.getTypeAllocSize(Ty: AT->getElementType());
819 assert(Offset < AT->getNumElements() * EltTypeSize);
820 uint64_t Index = Offset / EltTypeSize;
821 Offset = Offset - (Index * EltTypeSize);
822 CurType = AT->getElementType();
823 OnLiteralIndexing(CurType, Index);
824 } else if (StructType *ST = dyn_cast<StructType>(Val: CurType)) {
825 uint32_t StructSize = DL.getTypeSizeInBits(Ty: ST) / 8;
826 assert(Offset < StructSize);
827 (void)StructSize;
828 const auto &STL = DL.getStructLayout(Ty: ST);
829 unsigned Element = STL->getElementContainingOffset(FixedOffset: Offset);
830 Offset -= STL->getElementOffset(Idx: Element);
831 CurType = ST->getElementType(N: Element);
832 OnLiteralIndexing(CurType, Element);
833 } else if (auto *VT = dyn_cast<FixedVectorType>(Val: CurType)) {
834 Type *EltTy = VT->getElementType();
835 TypeSize EltSizeBits = DL.getTypeSizeInBits(Ty: EltTy);
836 assert(EltSizeBits % 8 == 0 &&
837 "Element type size in bits must be a multiple of 8.");
838 uint32_t EltTypeSize = EltSizeBits / 8;
839 assert(Offset < VT->getNumElements() * EltTypeSize);
840 uint64_t Index = Offset / EltTypeSize;
841 Offset -= Index * EltTypeSize;
842 CurType = EltTy;
843 OnLiteralIndexing(CurType, Index);
844 } else {
845 // Unknown composite kind; give up.
846 return true;
847 }
848 } while (Offset > 0);
849
850 return false;
851}
852
853bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChain(
854 GetElementPtrInst &GEP,
855 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
856 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing) {
857 // We only rewrite byte-addressing GEP. Other should be left as-is.
858 // Valid byte-addressing GEP must always have a single index.
859 std::optional<uint64_t> MultiplierOpt =
860 getByteAddressingMultiplier(Ty: GEP.getSourceElementType());
861 assert(MultiplierOpt && "We only rewrite byte-addressing GEP");
862 uint64_t Multiplier = *MultiplierOpt;
863 assert(GEP.getNumIndices() == 1);
864
865 Value *Src = getPointerRoot(I: GEP.getPointerOperand());
866 Type *CurType = deduceElementType(I: Src, UnknownElemTypeI8: true);
867
868 Value *Operand = *GEP.idx_begin();
869 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Operand))
870 return walkLogicalAccessChainConstant(
871 CurType, Offset: CI->getZExtValue() * Multiplier, OnLiteralIndexing);
872
873 return walkLogicalAccessChainDynamic(CurType, Operand, Multiplier,
874 OnLiteralIndexing, OnDynamicIndexing);
875}
876
877Instruction *SPIRVEmitIntrinsicsImpl::buildLogicalAccessChainFromGEP(
878 GetElementPtrInst &GEP) {
879 auto &DL = CurrF->getDataLayout();
880 IRBuilder<> B(GEP.getParent());
881 B.SetInsertPoint(&GEP);
882
883 std::vector<Value *> Indices;
884 Indices.push_back(x: ConstantInt::get(
885 Ty: IntegerType::getInt32Ty(C&: CurrF->getContext()), V: 0, /* Signed= */ IsSigned: false));
886 walkLogicalAccessChain(
887 GEP,
888 OnLiteralIndexing: [&Indices, &B](Type *EltType, uint64_t Index) {
889 Indices.push_back(
890 x: ConstantInt::get(Ty: B.getInt64Ty(), V: Index, /* Signed= */ IsSigned: false));
891 },
892 OnDynamicIndexing: [&Indices, &B, &DL, this](Type *EltType, Value *Offset,
893 uint64_t Multiplier) {
894 Value *Index = nullptr;
895 uint32_t EltTypeSize = DL.getTypeSizeInBits(Ty: EltType) / 8;
896 assert(Multiplier != 0);
897 if (Multiplier == EltTypeSize) {
898 Index = Offset;
899 } else if (EltTypeSize % Multiplier == 0) {
900 Index =
901 B.CreateUDiv(LHS: Offset, RHS: ConstantInt::get(Ty: Offset->getType(),
902 V: EltTypeSize / Multiplier,
903 /* Signed= */ IsSigned: false));
904 } else {
905 Index = B.CreateMul(LHS: Offset,
906 RHS: ConstantInt::get(Ty: Offset->getType(), V: Multiplier,
907 /* Signed= */ IsSigned: false));
908 insertAssignTypeIntrs(I: cast<Instruction>(Val: Index), B);
909 Index = B.CreateUDiv(LHS: Index,
910 RHS: ConstantInt::get(Ty: Offset->getType(), V: EltTypeSize,
911 /* Signed= */ IsSigned: false));
912 }
913 insertAssignTypeIntrs(I: cast<Instruction>(Val: Index), B);
914 Indices.push_back(x: Index);
915 });
916
917 SmallVector<Type *, 2> Types = {GEP.getType(), GEP.getOperand(i_nocapture: 0)->getType()};
918 SmallVector<Value *, 4> Args;
919 Args.push_back(Elt: B.getInt1(V: GEP.isInBounds()));
920 Args.push_back(Elt: GEP.getOperand(i_nocapture: 0));
921 llvm::append_range(C&: Args, R&: Indices);
922 Instruction *NewI =
923 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
924 replaceAllUsesWithAndErase(B, Src: &GEP, Dest: NewI);
925 return NewI;
926}
927
928Type *SPIRVEmitIntrinsicsImpl::getGEPTypeLogical(GetElementPtrInst *GEP) {
929
930 Type *CurType = GEP->getResultElementType();
931
932 bool Interrupted = walkLogicalAccessChain(
933 GEP&: *GEP, OnLiteralIndexing: [&CurType](Type *EltType, uint64_t Index) { CurType = EltType; },
934 OnDynamicIndexing: [&CurType](Type *EltType, Value *Index, uint64_t) { CurType = EltType; });
935
936 return Interrupted ? GEP->getResultElementType() : CurType;
937}
938
939Type *SPIRVEmitIntrinsicsImpl::getGEPType(GetElementPtrInst *Ref) {
940 if (getByteAddressingMultiplier(Ty: Ref->getSourceElementType()) &&
941 TM.getSubtargetImpl()->isLogicalSPIRV()) {
942 return getGEPTypeLogical(GEP: Ref);
943 }
944
945 Type *Ty = nullptr;
946 // TODO: not sure if GetElementPtrInst::getTypeAtIndex() does anything
947 // useful here
948 if (isNestedPointer(Ty: Ref->getSourceElementType())) {
949 Ty = Ref->getSourceElementType();
950 for (Use &U : drop_begin(RangeOrContainer: Ref->indices()))
951 Ty = GetElementPtrInst::getTypeAtIndex(Ty, Idx: U.get());
952 } else {
953 Ty = Ref->getResultElementType();
954 }
955 return Ty;
956}
957
958Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeHelper(
959 Value *I, SmallPtrSetImpl<Value *> &Visited, bool UnknownElemTypeI8,
960 bool IgnoreKnownType) {
961 // allow to pass nullptr as an argument
962 if (!I)
963 return nullptr;
964
965 // maybe already known
966 if (!IgnoreKnownType)
967 if (Type *KnownTy = GR->findDeducedElementType(Val: I))
968 return KnownTy;
969
970 // maybe a cycle
971 if (!Visited.insert(Ptr: I).second)
972 return nullptr;
973
974 // fallback value in case when we fail to deduce a type
975 Type *Ty = nullptr;
976 // look for known basic patterns of type inference
977 if (auto *Ref = dyn_cast<AllocaInst>(Val: I)) {
978 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getAllocatedType(), UnknownElemTypeI8);
979 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
980 Ty = getGEPType(Ref);
981 } else if (auto *SGEP = dyn_cast<StructuredGEPInst>(Val: I)) {
982 Ty = SGEP->getResultElementType();
983 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
984 Value *Op = Ref->getPointerOperand();
985 Type *KnownTy = GR->findDeducedElementType(Val: Op);
986 if (!KnownTy)
987 KnownTy = Op->getType();
988 if (Type *ElemTy = getPointeeType(Ty: KnownTy))
989 maybeAssignPtrType(Ty, Op: I, RefTy: ElemTy, UnknownElemTypeI8);
990 } else if (auto *Ref = dyn_cast<GlobalValue>(Val: I)) {
991 if (auto *Fn = dyn_cast<Function>(Val: Ref)) {
992 Ty = SPIRV::getOriginalFunctionType(F: *Fn);
993 GR->addDeducedElementType(Val: I, Ty);
994 } else {
995 Ty = deduceElementTypeByValueDeep(
996 ValueTy: Ref->getValueType(),
997 Operand: Ref->getNumOperands() > 0 ? Ref->getOperand(i: 0) : nullptr, Visited,
998 UnknownElemTypeI8);
999 }
1000 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1001 Type *RefTy = deduceElementTypeHelper(I: Ref->getPointerOperand(), Visited,
1002 UnknownElemTypeI8);
1003 maybeAssignPtrType(Ty, Op: I, RefTy, UnknownElemTypeI8);
1004 } else if (auto *Ref = dyn_cast<IntToPtrInst>(Val: I)) {
1005 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getDestTy(), UnknownElemTypeI8);
1006 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1007 if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
1008 isPointerTy(T: Src) && isPointerTy(T: Dest))
1009 Ty = deduceElementTypeHelper(I: Ref->getOperand(i_nocapture: 0), Visited,
1010 UnknownElemTypeI8);
1011 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1012 Value *Op = Ref->getNewValOperand();
1013 if (isPointerTy(T: Op->getType()))
1014 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
1015 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1016 Value *Op = Ref->getValOperand();
1017 if (isPointerTy(T: Op->getType()))
1018 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
1019 } else if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1020 Type *BestTy = nullptr;
1021 unsigned MaxN = 1;
1022 DenseMap<Type *, unsigned> PhiTys;
1023 for (int i = Ref->getNumIncomingValues() - 1; i >= 0; --i) {
1024 Ty = deduceElementTypeByUsersDeep(Op: Ref->getIncomingValue(i), Visited,
1025 UnknownElemTypeI8);
1026 if (!Ty)
1027 continue;
1028 auto It = PhiTys.try_emplace(Key: Ty, Args: 1);
1029 if (!It.second) {
1030 ++It.first->second;
1031 if (It.first->second > MaxN) {
1032 MaxN = It.first->second;
1033 BestTy = Ty;
1034 }
1035 }
1036 }
1037 if (BestTy)
1038 Ty = BestTy;
1039 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1040 for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) {
1041 // A function pointer operand carries its function type directly. Other
1042 // operands are deduced from their uses.
1043 Ty = isa<Function>(Val: Op)
1044 ? deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8)
1045 : deduceElementTypeByUsersDeep(Op, Visited, UnknownElemTypeI8);
1046 if (Ty)
1047 break;
1048 }
1049 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
1050 static StringMap<unsigned> ResTypeByArg = {
1051 {"to_global", 0},
1052 {"to_local", 0},
1053 {"to_private", 0},
1054 {"__to_global", 0},
1055 {"__to_local", 0},
1056 {"__to_private", 0},
1057 {"__spirv_GenericCastToPtr_ToGlobal", 0},
1058 {"__spirv_GenericCastToPtr_ToLocal", 0},
1059 {"__spirv_GenericCastToPtr_ToPrivate", 0},
1060 {"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
1061 {"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
1062 {"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
1063 // TODO: maybe improve performance by caching demangled names
1064
1065 auto *II = dyn_cast<IntrinsicInst>(Val: I);
1066 if (II && (II->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer ||
1067 II->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) {
1068 auto *HandleType = cast<TargetExtType>(Val: II->getOperand(i_nocapture: 0)->getType());
1069 if (HandleType->getTargetExtName() == "spirv.Image" ||
1070 HandleType->getTargetExtName() == "spirv.SignedImage") {
1071 for (User *U : II->users()) {
1072 Ty = cast<Instruction>(Val: U)->getAccessType();
1073 if (Ty)
1074 break;
1075 }
1076 } else if (HandleType->getTargetExtName() == "spirv.VulkanBuffer") {
1077 // This call is supposed to index into an array
1078 Ty = HandleType->getTypeParameter(i: 0);
1079 if (II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1080 if (Ty->isArrayTy())
1081 Ty = Ty->getArrayElementType();
1082 else {
1083 assert(Ty && Ty->isStructTy());
1084 uint32_t Index =
1085 cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1))->getZExtValue();
1086 Ty = cast<StructType>(Val: Ty)->getElementType(N: Index);
1087 }
1088 }
1089 Ty = reconstitutePeeledArrayType(Ty);
1090 } else {
1091 llvm_unreachable("Unknown handle type for spv_resource_getpointer.");
1092 }
1093 } else if (II && II->getIntrinsicID() ==
1094 Intrinsic::spv_generic_cast_to_ptr_explicit) {
1095 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: 0), Visited,
1096 UnknownElemTypeI8);
1097 } else if (Function *CalledF = CI->getCalledFunction()) {
1098 std::string DemangledName =
1099 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1100 if (DemangledName.length() > 0)
1101 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName);
1102 auto AsArgIt = ResTypeByArg.find(Key: DemangledName);
1103 if (AsArgIt != ResTypeByArg.end())
1104 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: AsArgIt->second),
1105 Visited, UnknownElemTypeI8);
1106 else if (Type *KnownRetTy = GR->findDeducedElementType(Val: CalledF))
1107 Ty = KnownRetTy;
1108 }
1109 }
1110
1111 // remember the found relationship
1112 if (Ty && !IgnoreKnownType) {
1113 // specify nested types if needed, otherwise return unchanged
1114 GR->addDeducedElementType(Val: I, Ty: normalizeType(Ty, CanUseAnyVectorRank));
1115 }
1116
1117 return Ty;
1118}
1119
1120// Re-create a type of the value if it has untyped pointer fields, also nested.
1121// Return the original value type if no corrections of untyped pointer
1122// information is found or needed.
1123Type *SPIRVEmitIntrinsicsImpl::deduceNestedTypeHelper(User *U,
1124 bool UnknownElemTypeI8) {
1125 SmallPtrSet<Value *, 0> Visited;
1126 return deduceNestedTypeHelper(U, Ty: U->getType(), Visited, UnknownElemTypeI8);
1127}
1128
1129Type *SPIRVEmitIntrinsicsImpl::deduceNestedTypeHelper(
1130 User *U, Type *OrigTy, SmallPtrSetImpl<Value *> &Visited,
1131 bool UnknownElemTypeI8) {
1132 if (!U)
1133 return OrigTy;
1134
1135 // maybe already known
1136 if (Type *KnownTy = GR->findDeducedCompositeType(Val: U))
1137 return KnownTy;
1138
1139 // maybe a cycle
1140 if (!Visited.insert(Ptr: U).second)
1141 return OrigTy;
1142
1143 if (auto *OrigStructTy = dyn_cast<StructType>(Val: OrigTy)) {
1144 SmallVector<Type *> Tys;
1145 bool Change = false;
1146 for (unsigned i = 0; i < U->getNumOperands(); ++i) {
1147 Value *Op = U->getOperand(i);
1148 assert(Op && "Operands should not be null.");
1149 Type *OpTy = Op->getType();
1150 Type *Ty = OpTy;
1151 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1152 if (Type *NestedTy =
1153 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1154 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1155 } else {
1156 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1157 UnknownElemTypeI8);
1158 }
1159 Tys.push_back(Elt: Ty);
1160 Change |= Ty != OpTy;
1161 }
1162 if (Change) {
1163 Type *NewTy = StructType::create(
1164 Elements: Tys, Name: OrigStructTy->isLiteral() ? "" : OrigStructTy->getName(),
1165 isPacked: OrigStructTy->isPacked());
1166 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1167 return NewTy;
1168 }
1169 } else if (auto *ArrTy = dyn_cast<ArrayType>(Val: OrigTy)) {
1170 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1171 Type *OpTy = ArrTy->getElementType();
1172 Type *Ty = OpTy;
1173 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1174 if (Type *NestedTy =
1175 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1176 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1177 } else {
1178 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1179 UnknownElemTypeI8);
1180 }
1181 if (Ty != OpTy) {
1182 Type *NewTy = ArrayType::get(ElementType: Ty, NumElements: ArrTy->getNumElements());
1183 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1184 return NewTy;
1185 }
1186 }
1187 } else if (auto *VecTy = dyn_cast<VectorType>(Val: OrigTy)) {
1188 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1189 Type *OpTy = VecTy->getElementType();
1190 Type *Ty = OpTy;
1191 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1192 if (Type *NestedTy =
1193 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1194 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1195 } else {
1196 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1197 UnknownElemTypeI8);
1198 }
1199 if (Ty != OpTy) {
1200 Type *NewTy = VectorType::get(ElementType: Ty, EC: VecTy->getElementCount());
1201 GR->addDeducedCompositeType(Val: U,
1202 Ty: normalizeType(Ty: NewTy, CanUseAnyVectorRank));
1203 return NewTy;
1204 }
1205 }
1206 }
1207
1208 return OrigTy;
1209}
1210
1211Type *SPIRVEmitIntrinsicsImpl::deduceElementType(Value *I,
1212 bool UnknownElemTypeI8) {
1213 if (Type *Ty = deduceElementTypeHelper(I, UnknownElemTypeI8))
1214 return Ty;
1215 if (!UnknownElemTypeI8)
1216 return nullptr;
1217 insertTodoType(Op: I);
1218 return IntegerType::getInt8Ty(C&: I->getContext());
1219}
1220
1221static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I,
1222 Value *PointerOperand) {
1223 Type *PointeeTy = GR->findDeducedElementType(Val: PointerOperand);
1224 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1225 return nullptr;
1226 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType());
1227 if (!PtrTy)
1228 return I->getType();
1229 if (Type *NestedTy = GR->findDeducedElementType(Val: I))
1230 return getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1231 return nullptr;
1232}
1233
1234// Try to deduce element type for a call base. Returns false if this is an
1235// indirect function invocation, and true otherwise.
1236bool SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeCalledFunction(
1237 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1238 Type *&KnownElemTy, bool &Incomplete) {
1239 Function *CalledF = CI->getCalledFunction();
1240 if (!CalledF)
1241 return false;
1242 std::string DemangledName =
1243 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1244 if (DemangledName.length() > 0 &&
1245 !StringRef(DemangledName).starts_with(Prefix: "llvm.")) {
1246 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *CalledF);
1247 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
1248 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
1249 if (Opcode == SPIRV::OpGroupAsyncCopy) {
1250 for (unsigned i = 0, PtrCnt = 0; i < CI->arg_size() && PtrCnt < 2; ++i) {
1251 Value *Op = CI->getArgOperand(i);
1252 if (!isPointerTy(T: Op->getType()))
1253 continue;
1254 ++PtrCnt;
1255 if (Type *ElemTy = GR->findDeducedElementType(Val: Op))
1256 KnownElemTy = ElemTy; // src will rewrite dest if both are defined
1257 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1258 }
1259 } else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) {
1260 if (CI->arg_size() == 0)
1261 return true;
1262 Value *Op = CI->getArgOperand(i: 0);
1263 if (!isPointerTy(T: Op->getType()))
1264 return true;
1265 switch (Opcode) {
1266 case SPIRV::OpAtomicFAddEXT:
1267 case SPIRV::OpAtomicFMinEXT:
1268 case SPIRV::OpAtomicFMaxEXT:
1269 case SPIRV::OpAtomicLoad:
1270 case SPIRV::OpAtomicCompareExchangeWeak:
1271 case SPIRV::OpAtomicCompareExchange:
1272 case SPIRV::OpAtomicExchange:
1273 case SPIRV::OpAtomicIAdd:
1274 case SPIRV::OpAtomicISub:
1275 case SPIRV::OpAtomicOr:
1276 case SPIRV::OpAtomicXor:
1277 case SPIRV::OpAtomicAnd:
1278 case SPIRV::OpAtomicUMin:
1279 case SPIRV::OpAtomicUMax:
1280 case SPIRV::OpAtomicSMin:
1281 case SPIRV::OpAtomicSMax: {
1282 KnownElemTy = isPointerTy(T: CI->getType()) ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1283 : CI->getType();
1284 if (!KnownElemTy)
1285 return true;
1286 Incomplete = isTodoType(Op);
1287 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1288 } break;
1289 case SPIRV::OpAtomicStore: {
1290 if (CI->arg_size() < 4)
1291 return true;
1292 Value *ValOp = CI->getArgOperand(i: 3);
1293 KnownElemTy = isPointerTy(T: ValOp->getType())
1294 ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1295 : ValOp->getType();
1296 if (!KnownElemTy)
1297 return true;
1298 Incomplete = isTodoType(Op);
1299 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1300 } break;
1301 }
1302 }
1303 }
1304 return true;
1305}
1306
1307// Try to deduce element type for a function pointer.
1308void SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeFunctionPointer(
1309 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1310 Type *&KnownElemTy, bool IsPostprocessing) {
1311 Value *Op = CI->getCalledOperand();
1312 if (!Op || !isPointerTy(T: Op->getType()))
1313 return;
1314 Ops.push_back(Elt: std::make_pair(x&: Op, y: std::numeric_limits<unsigned>::max()));
1315 FunctionType *FTy = SPIRV::getOriginalFunctionType(CB: *CI);
1316 bool IsNewFTy = false, IsIncomplete = false;
1317 SmallVector<Type *, 4> ArgTys;
1318 for (auto &&[ParmIdx, Arg] : llvm::enumerate(First: CI->args())) {
1319 Type *ArgTy = Arg->getType();
1320 if (ArgTy->isPointerTy()) {
1321 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg)) {
1322 IsNewFTy = true;
1323 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
1324 if (isTodoType(Op: Arg))
1325 IsIncomplete = true;
1326 } else {
1327 IsIncomplete = true;
1328 }
1329 } else {
1330 ArgTy = FTy->getFunctionParamType(i: ParmIdx);
1331 }
1332 ArgTys.push_back(Elt: ArgTy);
1333 }
1334 Type *RetTy = FTy->getReturnType();
1335 if (CI->getType()->isPointerTy()) {
1336 if (Type *ElemTy = GR->findDeducedElementType(Val: CI)) {
1337 IsNewFTy = true;
1338 RetTy =
1339 getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: CI->getType()));
1340 if (isTodoType(Op: CI))
1341 IsIncomplete = true;
1342 } else {
1343 IsIncomplete = true;
1344 }
1345 }
1346 if (!IsPostprocessing && IsIncomplete)
1347 insertTodoType(Op);
1348 KnownElemTy =
1349 IsNewFTy ? FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: FTy->isVarArg()) : FTy;
1350}
1351
1352bool SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeFunctionRet(
1353 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
1354 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing,
1355 Type *&KnownElemTy, Value *Op, Function *F) {
1356 KnownElemTy = GR->findDeducedElementType(Val: F);
1357 if (KnownElemTy)
1358 return false;
1359 if (Type *OpElemTy = GR->findDeducedElementType(Val: Op)) {
1360 OpElemTy = normalizeType(Ty: OpElemTy, CanUseAnyVectorRank);
1361 GR->addDeducedElementType(Val: F, Ty: OpElemTy);
1362 GR->addReturnType(
1363 ArgF: F, DerivedTy: TypedPointerType::get(ElementType: OpElemTy,
1364 AddressSpace: getPointerAddressSpace(T: F->getReturnType())));
1365 // non-recursive update of types in function uses
1366 DenseSet<std::pair<Value *, Value *>> VisitedSubst{std::make_pair(x&: I, y&: Op)};
1367 for (User *U : F->users()) {
1368 CallInst *CI = dyn_cast<CallInst>(Val: U);
1369 if (!CI || CI->getCalledFunction() != F)
1370 continue;
1371 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: CI)) {
1372 if (Type *PrevElemTy = GR->findDeducedElementType(Val: CI)) {
1373 GR->updateAssignType(
1374 AssignCI, Arg: CI,
1375 OfType: getNormalizedPoisonValue(Ty: OpElemTy, CanUseAnyVectorRank));
1376 propagateElemType(Op: CI, ElemTy: PrevElemTy, VisitedSubst);
1377 }
1378 }
1379 }
1380 // Non-recursive update of types in the function uncomplete returns.
1381 // This may happen just once per a function, the latch is a pair of
1382 // findDeducedElementType(F) / addDeducedElementType(F, ...).
1383 // With or without the latch it is a non-recursive call due to
1384 // IncompleteRets set to nullptr in this call.
1385 if (IncompleteRets)
1386 for (Instruction *IncompleteRetI : *IncompleteRets)
1387 deduceOperandElementType(I: IncompleteRetI, IncompleteRets: nullptr, AskOps,
1388 IsPostprocessing);
1389 } else if (IncompleteRets) {
1390 IncompleteRets->insert(Ptr: I);
1391 }
1392 TypeValidated.insert(Ptr: I);
1393 return true;
1394}
1395
1396// If the Instruction has Pointer operands with unresolved types, this function
1397// tries to deduce them. If the Instruction has Pointer operands with known
1398// types which differ from expected, this function tries to insert a bitcast to
1399// resolve the issue.
1400void SPIRVEmitIntrinsicsImpl::deduceOperandElementType(
1401 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
1402 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing) {
1403 SmallVector<std::pair<Value *, unsigned>> Ops;
1404 Type *KnownElemTy = nullptr;
1405 bool Incomplete = false;
1406 // look for known basic patterns of type inference
1407 if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1408 if (!isPointerTy(T: I->getType()) ||
1409 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1410 return;
1411 Incomplete = isTodoType(Op: I);
1412 for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
1413 Value *Op = Ref->getIncomingValue(i);
1414 if (isPointerTy(T: Op->getType()))
1415 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1416 }
1417 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1418 KnownElemTy = GR->findDeducedElementType(Val: I);
1419 if (!KnownElemTy)
1420 return;
1421 Incomplete = isTodoType(Op: I);
1422 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(), y: 0));
1423 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1424 if (!isPointerTy(T: I->getType()))
1425 return;
1426 KnownElemTy = GR->findDeducedElementType(Val: I);
1427 if (!KnownElemTy)
1428 return;
1429 Incomplete = isTodoType(Op: I);
1430 Ops.push_back(Elt: std::make_pair(x: Ref->getOperand(i_nocapture: 0), y: 0));
1431 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
1432 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1433 return;
1434 KnownElemTy = Ref->getSourceElementType();
1435 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1436 y: GetElementPtrInst::getPointerOperandIndex()));
1437 } else if (auto *Ref = dyn_cast<StructuredGEPInst>(Val: I)) {
1438 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1439 return;
1440 KnownElemTy = Ref->getBaseType();
1441 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1442 y: StructuredGEPInst::getPointerOperandIndex()));
1443 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
1444 KnownElemTy = I->getType();
1445 if (isUntypedPointerTy(T: KnownElemTy)) {
1446 // A T** loaded back from its alloca comes out opaque, dropping type info.
1447 // When the load is a pointer-to-pointer, type the alloca as that pointer.
1448 Type *LoadedElemTy = GR->findDeducedElementType(Val: I);
1449 if (!LoadedElemTy || !isPointerTyOrWrapper(Ty: LoadedElemTy))
1450 return;
1451 Value *Root = Ref->getPointerOperand()->stripPointerCasts();
1452 if (!isa<AllocaInst>(Val: Root))
1453 return;
1454 KnownElemTy = getTypedPointerWrapper(ElemTy: LoadedElemTy,
1455 AS: getPointerAddressSpace(T: KnownElemTy));
1456 }
1457 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1458 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1459 return;
1460 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1461 y: LoadInst::getPointerOperandIndex()));
1462 } else if (auto *Ref = dyn_cast<StoreInst>(Val: I)) {
1463 if (!(KnownElemTy =
1464 reconstructType(Op: Ref->getValueOperand(), UnknownElemTypeI8: false, IsPostprocessing)))
1465 return;
1466 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1467 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1468 return;
1469 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1470 y: StoreInst::getPointerOperandIndex()));
1471 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1472 KnownElemTy = isPointerTy(T: I->getType())
1473 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1474 : I->getType();
1475 if (!KnownElemTy)
1476 return;
1477 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1478 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1479 y: AtomicCmpXchgInst::getPointerOperandIndex()));
1480 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1481 KnownElemTy = isPointerTy(T: I->getType())
1482 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1483 : I->getType();
1484 if (!KnownElemTy)
1485 return;
1486 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1487 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1488 y: AtomicRMWInst::getPointerOperandIndex()));
1489 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1490 if (!isPointerTy(T: I->getType()) ||
1491 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1492 return;
1493 Incomplete = isTodoType(Op: I);
1494 for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
1495 Value *Op = Ref->getOperand(i_nocapture: i);
1496 if (isPointerTy(T: Op->getType()))
1497 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1498 }
1499 } else if (auto *Ref = dyn_cast<ReturnInst>(Val: I)) {
1500 if (!isPointerTy(T: CurrF->getReturnType()))
1501 return;
1502 Value *Op = Ref->getReturnValue();
1503 if (!Op)
1504 return;
1505 if (deduceOperandElementTypeFunctionRet(I, IncompleteRets, AskOps,
1506 IsPostprocessing, KnownElemTy, Op,
1507 F: CurrF))
1508 return;
1509 Incomplete = isTodoType(Op: CurrF);
1510 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1511 } else if (auto *Ref = dyn_cast<ICmpInst>(Val: I)) {
1512 if (!isPointerTy(T: Ref->getOperand(i_nocapture: 0)->getType()))
1513 return;
1514 Value *Op0 = Ref->getOperand(i_nocapture: 0);
1515 Value *Op1 = Ref->getOperand(i_nocapture: 1);
1516 bool Incomplete0 = isTodoType(Op: Op0);
1517 bool Incomplete1 = isTodoType(Op: Op1);
1518 Type *ElemTy1 = GR->findDeducedElementType(Val: Op1);
1519 Type *ElemTy0 = (Incomplete0 && !Incomplete1 && ElemTy1)
1520 ? nullptr
1521 : GR->findDeducedElementType(Val: Op0);
1522 if (ElemTy0) {
1523 KnownElemTy = ElemTy0;
1524 Incomplete = Incomplete0;
1525 Ops.push_back(Elt: std::make_pair(x&: Op1, y: 1));
1526 } else if (ElemTy1) {
1527 KnownElemTy = ElemTy1;
1528 Incomplete = Incomplete1;
1529 Ops.push_back(Elt: std::make_pair(x&: Op0, y: 0));
1530 }
1531 } else if (CallInst *CI = dyn_cast<CallInst>(Val: I)) {
1532 if (!CI->isIndirectCall())
1533 deduceOperandElementTypeCalledFunction(CI, Ops, KnownElemTy, Incomplete);
1534 else if (HaveFunPtrs)
1535 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy,
1536 IsPostprocessing);
1537 }
1538
1539 // There is no enough info to deduce types or all is valid.
1540 if (!KnownElemTy || Ops.size() == 0)
1541 return;
1542
1543 IRBuilder<> B(*CurrF->getParent());
1544 for (auto &OpIt : Ops) {
1545 Value *Op = OpIt.first;
1546 if (AskOps && !AskOps->contains(Ptr: Op))
1547 continue;
1548 Type *AskTy = nullptr;
1549 CallInst *AskCI = nullptr;
1550 if (IsPostprocessing && AskOps) {
1551 AskTy = GR->findDeducedElementType(Val: Op);
1552 AskCI = GR->findAssignPtrTypeInstr(Val: Op);
1553 assert(AskTy && AskCI);
1554 }
1555 Type *Ty = AskTy ? AskTy : GR->findDeducedElementType(Val: Op);
1556 if (Ty == KnownElemTy)
1557 continue;
1558 Value *OpTyVal = getNormalizedPoisonValue(Ty: KnownElemTy, CanUseAnyVectorRank);
1559 Type *OpTy = Op->getType();
1560 // Do not let a non-pointer element type clobber an already-deduced pointer
1561 // element type for the same operand.
1562 bool WouldClobberPtrWithNonPtr = Ty && isPointerTyOrWrapper(Ty) &&
1563 !isPointerTyOrWrapper(Ty: KnownElemTy) &&
1564 tracesToPointerAlloca(V: Op);
1565 if (Op->hasUseList() && !WouldClobberPtrWithNonPtr &&
1566 (!Ty || AskTy || isUntypedPointerTy(T: Ty) || isTodoType(Op))) {
1567 Type *PrevElemTy = GR->findDeducedElementType(Val: Op);
1568 GR->addDeducedElementType(
1569 Val: Op, Ty: normalizeType(Ty: KnownElemTy, CanUseAnyVectorRank));
1570 // check if KnownElemTy is complete
1571 if (!Incomplete)
1572 eraseTodoType(Op);
1573 else if (!IsPostprocessing)
1574 insertTodoType(Op);
1575 // check if there is existing Intrinsic::spv_assign_ptr_type instruction
1576 CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Val: Op);
1577 if (AssignCI == nullptr) {
1578 Instruction *User = dyn_cast<Instruction>(Val: Op->use_begin()->get());
1579 setInsertPointSkippingPhis(B, I: User ? User->getNextNode() : I);
1580 CallInst *CI =
1581 buildIntrWithMD(IntrID: Intrinsic::spv_assign_ptr_type, Types: {OpTy}, Arg: OpTyVal, Arg2: Op,
1582 Imms: {B.getInt32(C: getPointerAddressSpace(T: OpTy))}, B);
1583 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: CI);
1584 } else {
1585 GR->updateAssignType(AssignCI, Arg: Op, OfType: OpTyVal);
1586 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1587 std::make_pair(x&: I, y&: Op)};
1588 propagateElemTypeRec(Op, PtrElemTy: KnownElemTy, CastElemTy: PrevElemTy, VisitedSubst);
1589 }
1590 } else {
1591 eraseTodoType(Op);
1592 CallInst *PtrCastI =
1593 buildSpvPtrcast(F: I->getParent()->getParent(), Op, ElemTy: KnownElemTy);
1594 if (OpIt.second == std::numeric_limits<unsigned>::max())
1595 dyn_cast<CallInst>(Val: I)->setCalledOperand(PtrCastI);
1596 else
1597 I->setOperand(i: OpIt.second, Val: PtrCastI);
1598 }
1599 }
1600 TypeValidated.insert(Ptr: I);
1601}
1602
1603void SPIRVEmitIntrinsicsImpl::replaceMemInstrUses(Instruction *Old,
1604 Instruction *New,
1605 IRBuilder<> &B) {
1606 while (!Old->user_empty()) {
1607 auto *U = Old->user_back();
1608 if (isAssignTypeInstr(I: U)) {
1609 B.SetInsertPoint(U);
1610 SmallVector<Value *, 2> Args = {New, U->getOperand(i: 1)};
1611 CallInst *AssignCI = B.CreateIntrinsicWithoutFolding(
1612 ID: Intrinsic::spv_assign_type, OverloadTypes: {New->getType()}, Args);
1613 GR->addAssignPtrTypeInstr(Val: New, AssignPtrTyCI: AssignCI);
1614 U->eraseFromParent();
1615 } else if (isMemInstrToReplace(I: U) || isa<ReturnInst>(Val: U) ||
1616 isa<CallInst>(Val: U)) {
1617 U->replaceUsesOfWith(From: Old, To: New);
1618 // For a `llvm.spv.abort` call whose composite message argument was
1619 // rewritten to a value-id (i32), also retarget the call to a matching
1620 // intrinsic declaration so the IR verifier is satisfied. The SPIR-V
1621 // type of the value is tracked via the GlobalRegistry, so the selector
1622 // still emits OpAbortKHR with the original composite type.
1623 if (auto *CI = dyn_cast<CallInst>(Val: U);
1624 CI && CI->getIntrinsicID() == Intrinsic::spv_abort) {
1625 Type *NewArgTy = New->getType();
1626 Type *ExpectedArgTy = CI->getFunctionType()->getParamType(i: 0);
1627 if (NewArgTy != ExpectedArgTy) {
1628 Module *M = CI->getModule();
1629 Function *NewF = Intrinsic::getOrInsertDeclaration(
1630 M, id: Intrinsic::spv_abort, OverloadTys: {NewArgTy});
1631 CI->setCalledFunction(NewF);
1632 }
1633 }
1634 } else if (isa<PHINode>(Val: U) || isa<SelectInst>(Val: U) || isa<FreezeInst>(Val: U)) {
1635 // Aggregate-typed PHIs, selects and freezes have already been mutated to
1636 // the i32 value-id type up front in runOnFunction, so only the operand
1637 // needs replacing here; their extractvalue users are lowered to
1638 // spv_extractv by visitExtractValueInst.
1639 assert(U->getType() == New->getType() &&
1640 "aggregate PHI/select/freeze should have been mutated to value-id "
1641 "type");
1642 U->replaceUsesOfWith(From: Old, To: New);
1643 } else {
1644 llvm_unreachable("illegal aggregate intrinsic user");
1645 }
1646 }
1647 New->copyMetadata(SrcInst: *Old);
1648 Old->eraseFromParent();
1649}
1650
1651// Lower a poison or undef Op to its placeholder intrinsic.
1652Value *SPIRVEmitIntrinsicsImpl::lowerUndefOrPoison(Value *Op, IRBuilder<> &B,
1653 bool HasPoisonExt) {
1654 auto *UV = dyn_cast<UndefValue>(Val: Op);
1655 if (!UV)
1656 return nullptr;
1657
1658 bool AsPoison = HasPoisonExt && isa<PoisonValue>(Val: UV);
1659 if (isa<PoisonValue>(Val: UV) && !HasPoisonExt)
1660 LLVM_DEBUG(dbgs() << "SPV_KHR_poison_freeze is not enabled. Poison is "
1661 "lowered as undef\n");
1662
1663 Intrinsic::ID IID = AsPoison ? Intrinsic::spv_poison : Intrinsic::spv_undef;
1664 Type *Ty = UV->getType();
1665
1666 // Aggregates use an i32-result placeholder with the real type kept in
1667 // AggrConstTypes and scalar poison uses a type-overloaded one.
1668 if (Ty->isAggregateType()) {
1669 auto *Call =
1670 AsPoison ? B.CreateIntrinsicWithoutFolding(ID: IID, OverloadTypes: {B.getInt32Ty()}, Args: {})
1671 : B.CreateIntrinsicWithoutFolding(ID: IID, Args: {});
1672 AggrConsts[Call] = UV;
1673 AggrConstTypes[Call] = Ty;
1674 return Call;
1675 }
1676
1677 if (AsPoison)
1678 return B.CreateIntrinsic(ID: IID, OverloadTypes: {Ty}, Args: {});
1679 return nullptr;
1680}
1681
1682// Replace aggregate undef or poison operands and extension-enabled scalar
1683// poison operands with placeholder intrinsics. Scalar undef is left as is. See
1684// lowerUndefOrPoison.
1685void SPIRVEmitIntrinsicsImpl::preprocessUndefsAndPoisons(IRBuilder<> &B) {
1686 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*CurrF);
1687 bool HasPoisonExt =
1688 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
1689
1690 SmallVector<Instruction *, 16> Insts;
1691 for (auto &I : instructions(F: CurrF))
1692 Insts.push_back(Elt: &I);
1693
1694 for (Instruction *I : Insts) {
1695 bool BPrepared = false;
1696 auto *Phi = dyn_cast<PHINode>(Val: I);
1697 for (unsigned Idx = 0; Idx < I->getNumOperands(); ++Idx) {
1698 Value *Op = I->getOperand(i: Idx);
1699 if (!isa<UndefValue>(Val: Op) || Op->getType()->isMetadataTy())
1700 continue;
1701 bool IsScalar = !Op->getType()->isAggregateType();
1702 bool AsPoison = HasPoisonExt && isa<PoisonValue>(Val: Op);
1703 // Scalar undef or extensionless scalar poison is directly translatable.
1704 if (IsScalar && !AsPoison)
1705 continue;
1706 // Scalar poison in a phi materializes in the incoming block. Everything
1707 // else materializes right before I.
1708 if (IsScalar && Phi)
1709 B.SetInsertPoint(Phi->getIncomingBlock(i: Idx)->getTerminator());
1710 else if (!BPrepared) {
1711 setInsertPointSkippingPhis(B, I);
1712 BPrepared = true;
1713 }
1714 if (Value *Repl = lowerUndefOrPoison(Op, B, HasPoisonExt))
1715 I->setOperand(i: Idx, Val: Repl);
1716 }
1717 }
1718}
1719
1720// Simplify addrspacecast(null) instructions to ConstantPointerNull of the
1721// target type. Casting null always yields null, and this avoids SPIR-V
1722// lowering issues where the null gets typed as an integer instead of a
1723// pointer.
1724void SPIRVEmitIntrinsicsImpl::simplifyNullAddrSpaceCasts() {
1725 for (Instruction &I : make_early_inc_range(Range: instructions(F: CurrF)))
1726 if (auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: &I))
1727 if (isa<ConstantPointerNull>(Val: ASC->getPointerOperand())) {
1728 ASC->replaceAllUsesWith(
1729 V: ConstantPointerNull::get(T: cast<PointerType>(Val: ASC->getType())));
1730 ASC->eraseFromParent();
1731 }
1732}
1733
1734// True for an aggregate value the legalizer splits into a multi-result op
1735// (with.overflow -> G_UADDO, frexp/sincos/modf -> G_FFREXP/...). These keep a
1736// genuine multi-register result; all other aggregates become a single value-id.
1737static bool isMultiRegisterAggregate(Value *V) {
1738 if (!V->getType()->isAggregateType())
1739 return false;
1740 return isa<IntrinsicInst>(Val: V) && !isSpvIntrinsic(Arg: V);
1741}
1742
1743// True for an aggregate PHI/select/freeze, which is lowered to a single
1744// value-id.
1745static bool isAggregateValueIdInstr(const Instruction &I) {
1746 return (isa<PHINode>(Val: I) || isa<SelectInst>(Val: I) || isa<FreezeInst>(Val: I)) &&
1747 I.getType()->isAggregateType();
1748}
1749
1750// Give each multi-register aggregate arm of an aggregate PHI/select/freeze a
1751// single value-id by reassembling it with extractvalue + insertvalue, so the
1752// arm matches the result once it is mutated to a value-id.
1753void SPIRVEmitIntrinsicsImpl::insertCompositeAggregateArms(Instruction *I,
1754 IRBuilder<> &B) {
1755 auto *Phi = dyn_cast<PHINode>(Val: I);
1756 for (Use &U : I->operands()) {
1757 Value *Op = U.get();
1758 if (!isMultiRegisterAggregate(V: Op))
1759 continue;
1760 // A PHI arm materializes in its incoming block, everything else after the
1761 // producer.
1762 if (Phi)
1763 B.SetInsertPoint(Phi->getIncomingBlock(U)->getTerminator());
1764 else
1765 setInsertPointAfterDef(B, I: cast<Instruction>(Val: Op));
1766 auto *AggrTy = cast<StructType>(Val: Op->getType());
1767 Value *Composite = PoisonValue::get(T: AggrTy);
1768 for (unsigned Idx = 0, E = AggrTy->getNumElements(); Idx != E; ++Idx) {
1769 Value *Field = B.CreateExtractValue(Agg: Op, Idxs: Idx);
1770 Composite = B.CreateInsertValue(Agg: Composite, Val: Field, Idxs: Idx);
1771 }
1772 U.set(Composite);
1773 }
1774}
1775
1776void SPIRVEmitIntrinsicsImpl::preprocessCompositeConstants(IRBuilder<> &B) {
1777 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*CurrF);
1778 bool HasPoisonExt =
1779 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
1780 std::queue<Instruction *> Worklist;
1781 for (auto &I : instructions(F: CurrF))
1782 Worklist.push(x: &I);
1783
1784 while (!Worklist.empty()) {
1785 auto *I = Worklist.front();
1786 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
1787 assert(I);
1788 bool KeepInst = false;
1789 for (const auto &Op : I->operands()) {
1790 Constant *AggrConst = nullptr;
1791 Type *ResTy = nullptr;
1792 if (auto *COp = dyn_cast<ConstantVector>(Val: Op)) {
1793 AggrConst = COp;
1794 ResTy = COp->getType();
1795 } else if (auto *COp = dyn_cast<ConstantArray>(Val: Op)) {
1796 AggrConst = COp;
1797 ResTy = B.getInt32Ty();
1798 } else if (auto *COp = dyn_cast<ConstantStruct>(Val: Op)) {
1799 AggrConst = COp;
1800 ResTy = B.getInt32Ty();
1801 } else if (auto *COp = dyn_cast<ConstantDataArray>(Val: Op)) {
1802 AggrConst = COp;
1803 ResTy = B.getInt32Ty();
1804 } else if (auto *COp = dyn_cast<ConstantAggregateZero>(Val: Op)) {
1805 AggrConst = COp;
1806 ResTy = Op->getType()->isVectorTy() ? COp->getType() : B.getInt32Ty();
1807 }
1808 if (AggrConst) {
1809 auto PrepareInsert = [&]() {
1810 if (BPrepared)
1811 return;
1812 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
1813 : B.SetInsertPoint(I);
1814 BPrepared = true;
1815 };
1816 SmallVector<Value *> Args;
1817 if (auto *COp = dyn_cast<ConstantDataSequential>(Val: Op))
1818 for (unsigned i = 0; i < COp->getNumElements(); ++i)
1819 Args.push_back(Elt: COp->getElementAsConstant(i));
1820 else
1821 for (Value *Op : AggrConst->operands()) {
1822 // Simplify addrspacecast(null) to null in the target address space
1823 // so that null pointers get the correct pointer type when lowered.
1824 if (auto *CE = dyn_cast<ConstantExpr>(Val: Op);
1825 CE && CE->getOpcode() == Instruction::AddrSpaceCast &&
1826 isa<ConstantPointerNull>(Val: CE->getOperand(i_nocapture: 0)))
1827 Op = ConstantPointerNull::get(T: cast<PointerType>(Val: CE->getType()));
1828 // Undef or poison nested in a constant aggregate is not a direct
1829 // instruction operand, so preprocessUndefsAndPoisons() misses it.
1830 // An unlowered aggregate one would reach IRTranslator as an
1831 // untranslatable spv_const_composite operand.
1832 if (isa<UndefValue>(Val: Op)) {
1833 PrepareInsert();
1834 if (Value *Repl = lowerUndefOrPoison(Op, B, HasPoisonExt))
1835 Op = Repl;
1836 }
1837 Args.push_back(Elt: Op);
1838 }
1839 PrepareInsert();
1840 auto *CI = B.CreateIntrinsicWithoutFolding(
1841 ID: Intrinsic::spv_const_composite, OverloadTypes: {ResTy}, Args: {Args});
1842 Worklist.push(x: CI);
1843 I->replaceUsesOfWith(From: Op, To: CI);
1844 KeepInst = true;
1845 AggrConsts[CI] = AggrConst;
1846 AggrConstTypes[CI] = deduceNestedTypeHelper(U: AggrConst, UnknownElemTypeI8: false);
1847 }
1848 }
1849 if (!KeepInst)
1850 Worklist.pop();
1851 }
1852}
1853
1854static void createDecorationIntrinsic(Instruction *I, MDNode *Node,
1855 IRBuilder<> &B) {
1856 LLVMContext &Ctx = I->getContext();
1857 setInsertPointAfterDef(B, I);
1858 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
1859 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: {Node}))});
1860}
1861
1862static void createRoundingModeDecoration(Instruction *I,
1863 unsigned RoundingModeDeco,
1864 IRBuilder<> &B) {
1865 LLVMContext &Ctx = I->getContext();
1866 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1867 MDNode *RoundingModeNode = MDNode::get(
1868 Context&: Ctx,
1869 MDs: {ConstantAsMetadata::get(
1870 C: ConstantInt::get(Ty: Int32Ty, V: SPIRV::Decoration::FPRoundingMode)),
1871 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: RoundingModeDeco))});
1872 createDecorationIntrinsic(I, Node: RoundingModeNode, B);
1873}
1874
1875static void createSaturatedConversionDecoration(Instruction *I,
1876 IRBuilder<> &B) {
1877 LLVMContext &Ctx = I->getContext();
1878 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1879 MDNode *SaturatedConversionNode =
1880 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1881 Ty: Int32Ty, V: SPIRV::Decoration::SaturatedConversion))});
1882 createDecorationIntrinsic(I, Node: SaturatedConversionNode, B);
1883}
1884
1885static void addSaturatedDecorationToIntrinsic(Instruction *I, IRBuilder<> &B) {
1886 if (match(V: I, P: m_AnyIntrinsic<Intrinsic::fptosi_sat, Intrinsic::fptoui_sat>()))
1887 createSaturatedConversionDecoration(I, B);
1888}
1889
1890Instruction *SPIRVEmitIntrinsicsImpl::visitCallInst(CallInst &Call) {
1891 if (!Call.isInlineAsm())
1892 return &Call;
1893
1894 LLVMContext &Ctx = CurrF->getContext();
1895 // TODO: this does not retain elementtype info for memory constraints, which
1896 // in turn means that we lower them into pointers to i8, rather than
1897 // pointers to elementtype; this can be fixed during reverse translation
1898 // but we should correct it here, possibly by tweaking the function
1899 // type to take TypedPointerType args.
1900 Constant *TyC = UndefValue::get(T: SPIRV::getOriginalFunctionType(CB: Call));
1901 MDString *ConstraintString =
1902 MDString::get(Context&: Ctx, Str: SPIRV::getOriginalAsmConstraints(CB: Call));
1903 SmallVector<Value *> Args = {
1904 buildMD(Arg: TyC),
1905 MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: ConstraintString))};
1906 for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++)
1907 Args.push_back(Elt: Call.getArgOperand(i: OpIdx));
1908
1909 IRBuilder<> B(Call.getParent());
1910 B.SetInsertPoint(&Call);
1911 B.CreateIntrinsic(ID: Intrinsic::spv_inline_asm, Args: {Args});
1912 return &Call;
1913}
1914
1915// Use a tip about rounding mode to create a decoration.
1916void SPIRVEmitIntrinsicsImpl::useRoundingMode(ConstrainedFPIntrinsic *FPI,
1917 IRBuilder<> &B) {
1918 std::optional<RoundingMode> RM = FPI->getRoundingMode();
1919 if (!RM.has_value())
1920 return;
1921 unsigned RoundingModeDeco = std::numeric_limits<unsigned>::max();
1922 switch (RM.value()) {
1923 default:
1924 // ignore unknown rounding modes
1925 break;
1926 case RoundingMode::NearestTiesToEven:
1927 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTE;
1928 break;
1929 case RoundingMode::TowardNegative:
1930 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTN;
1931 break;
1932 case RoundingMode::TowardPositive:
1933 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTP;
1934 break;
1935 case RoundingMode::TowardZero:
1936 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTZ;
1937 break;
1938 case RoundingMode::Dynamic:
1939 case RoundingMode::NearestTiesToAway:
1940 // TODO: check if supported
1941 break;
1942 }
1943 if (RoundingModeDeco == std::numeric_limits<unsigned>::max())
1944 return;
1945 // Convert the tip about rounding mode into a decoration record.
1946 createRoundingModeDecoration(I: FPI, RoundingModeDeco, B);
1947}
1948
1949Instruction *SPIRVEmitIntrinsicsImpl::visitSwitchInst(SwitchInst &I) {
1950 BasicBlock *ParentBB = I.getParent();
1951 Function *F = ParentBB->getParent();
1952 IRBuilder<> B(ParentBB);
1953 B.SetInsertPoint(&I);
1954 SmallVector<Value *, 4> Args;
1955 SmallVector<BasicBlock *> BBCases;
1956 Args.push_back(Elt: I.getCondition());
1957 BBCases.push_back(Elt: I.getDefaultDest());
1958 Args.push_back(Elt: BlockAddress::get(F, BB: I.getDefaultDest()));
1959 for (auto &Case : I.cases()) {
1960 Args.push_back(Elt: Case.getCaseValue());
1961 BBCases.push_back(Elt: Case.getCaseSuccessor());
1962 Args.push_back(Elt: BlockAddress::get(F, BB: Case.getCaseSuccessor()));
1963 }
1964 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
1965 ID: Intrinsic::spv_switch, OverloadTypes: {I.getOperand(i_nocapture: 0)->getType()}, Args: {Args});
1966 // remove switch to avoid its unneeded and undesirable unwrap into branches
1967 // and conditions
1968 replaceAllUsesWith(Src: &I, Dest: NewI);
1969 I.eraseFromParent();
1970 // insert artificial and temporary instruction to preserve valid CFG,
1971 // it will be removed after IR translation pass
1972 B.SetInsertPoint(ParentBB);
1973 IndirectBrInst *BrI = B.CreateIndirectBr(
1974 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: ParentBB->getContext())),
1975 NumDests: BBCases.size());
1976 for (BasicBlock *BBCase : BBCases)
1977 BrI->addDestination(Dest: BBCase);
1978 return BrI;
1979}
1980
1981static bool isFirstIndexZero(const GetElementPtrInst *GEP) {
1982 return GEP->getNumIndices() > 0 && match(V: GEP->getOperand(i_nocapture: 1), P: m_Zero());
1983}
1984
1985Instruction *SPIRVEmitIntrinsicsImpl::visitIntrinsicInst(IntrinsicInst &I) {
1986 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
1987 if (!SGEP)
1988 return &I;
1989
1990 IRBuilder<> B(I.getParent());
1991 B.SetInsertPoint(&I);
1992 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
1993 SmallVector<Value *, 4> Args;
1994 Args.push_back(/* inBounds= */ Elt: B.getInt1(V: true));
1995 Args.push_back(Elt: I.getOperand(i_nocapture: 0));
1996 Args.push_back(/* zero index */ Elt: B.getInt32(C: 0));
1997 for (unsigned J = 0; J < SGEP->getNumIndices(); ++J)
1998 Args.push_back(Elt: SGEP->getIndexOperand(Index: J));
1999
2000 Instruction *NewI =
2001 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: Types, Args);
2002 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2003 return NewI;
2004}
2005
2006Instruction *
2007SPIRVEmitIntrinsicsImpl::visitGetElementPtrInst(GetElementPtrInst &I) {
2008 IRBuilder<> B(I.getParent());
2009 B.SetInsertPoint(&I);
2010
2011 // OpPtrAccessChain requires a scalar pointer result; scalarize per-lane
2012 // GEPs that return <N x ptr> and rebuild the vector via insertelement.
2013 if (auto *RetVTy = dyn_cast<FixedVectorType>(Val: I.getType())) {
2014 unsigned N = RetVTy->getNumElements();
2015 Value *PtrOp = I.getPointerOperand();
2016 bool PtrIsVec = isa<VectorType>(Val: PtrOp->getType());
2017 Type *ResultPtrTy = RetVTy->getElementType();
2018 Type *ScalarPtrTy = PtrOp->getType()->getScalarType();
2019 SmallVector<Type *, 2> GepTypes = {ResultPtrTy, ScalarPtrTy};
2020 Value *InBounds = B.getInt1(V: I.isInBounds());
2021 Type *LanePointeeTy = getGEPType(Ref: &I);
2022 Type *SrcElemTy = I.getSourceElementType();
2023
2024 // Pin the lane pointee type on the vector operand and on each extracted
2025 // lane so the prelegalizer wraps them as OpTypeVector/OpTypePointer of
2026 // the right element type instead of defaulting to i8.
2027 if (PtrIsVec)
2028 GR->buildAssignPtr(B, ElemTy: SrcElemTy, Arg: PtrOp);
2029
2030 Value *VecResult = PoisonValue::get(T: RetVTy);
2031 for (unsigned Lane = 0; Lane < N; ++Lane) {
2032 Value *LaneIdx = B.getInt32(C: Lane);
2033 Value *ScalarPtr = PtrOp;
2034 if (PtrIsVec) {
2035 SmallVector<Type *, 3> ExtractTypes = {ScalarPtrTy, PtrOp->getType(),
2036 LaneIdx->getType()};
2037 ScalarPtr = B.CreateIntrinsic(ID: Intrinsic::spv_extractelt, OverloadTypes: {ExtractTypes},
2038 Args: {PtrOp, LaneIdx});
2039 GR->buildAssignPtr(B, ElemTy: SrcElemTy, Arg: ScalarPtr);
2040 }
2041 SmallVector<Value *, 4> Args;
2042 Args.push_back(Elt: InBounds);
2043 Args.push_back(Elt: ScalarPtr);
2044 for (Value *Idx : I.indices()) {
2045 if (isa<VectorType>(Val: Idx->getType())) {
2046 // We cannot use the builder here as for splat-ed / constant vectors
2047 // it will fold to the scalar, and then it becomes impossible to
2048 // retrieve / retain the vectorness.
2049 auto *EI =
2050 ExtractElementInst::Create(Vec: Idx, Idx: LaneIdx, NameStr: "", InsertBefore: B.GetInsertPoint());
2051 if (isVector1(Ty: Idx->getType())) // IRTranslator clobbers <1 x T>.
2052 Args.push_back(Elt: visitExtractElementInst(I&: *EI));
2053 else
2054 Args.push_back(Elt: EI);
2055 } else {
2056 Args.push_back(Elt: Idx);
2057 }
2058 }
2059 Value *ScalarGep = B.CreateIntrinsic(ID: Intrinsic::spv_gep, OverloadTypes: GepTypes, Args);
2060 GR->buildAssignPtr(B, ElemTy: LanePointeeTy, Arg: ScalarGep);
2061 VecResult = B.CreateInsertElement(Vec: VecResult, NewElt: ScalarGep, Idx: LaneIdx);
2062 }
2063
2064 auto *NewI = cast<Instruction>(Val: VecResult);
2065 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2066
2067 if (CallInst *Old = GR->findAssignPtrTypeInstr(Val: NewI)) {
2068 Old->eraseFromParent();
2069 GR->addAssignPtrTypeInstr(Val: NewI, AssignPtrTyCI: nullptr);
2070 }
2071 setInsertPointAfterDef(B, I: NewI);
2072 GR->buildAssignPtr(B, ElemTy: LanePointeeTy, Arg: NewI);
2073
2074 return NewI;
2075 }
2076
2077 if (TM.getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: &I)) {
2078 // Logical SPIR-V cannot use the OpPtrAccessChain instruction. If the first
2079 // index of the GEP is not 0, then we need to try to adjust it.
2080 //
2081 // If the GEP is doing byte addressing, try to rebuild the full access chain
2082 // from the type of the pointer.
2083 if (getByteAddressingMultiplier(Ty: I.getSourceElementType())) {
2084 return buildLogicalAccessChainFromGEP(GEP&: I);
2085 }
2086
2087 // Look for the array-to-pointer decay. If this is the pattern
2088 // we can adjust the types, and prepend a 0 to the indices.
2089 Value *PtrOp = I.getPointerOperand();
2090 Type *SrcElemTy = I.getSourceElementType();
2091 Type *DeducedPointeeTy = deduceElementType(I: PtrOp, UnknownElemTypeI8: true);
2092
2093 if (auto *ArrTy = dyn_cast<ArrayType>(Val: DeducedPointeeTy)) {
2094 if (ArrTy->getElementType() == SrcElemTy) {
2095 SmallVector<Value *> NewIndices;
2096 Type *FirstIdxType = I.getOperand(i_nocapture: 1)->getType();
2097 NewIndices.push_back(Elt: ConstantInt::get(Ty: FirstIdxType, V: 0));
2098 for (Value *Idx : I.indices())
2099 NewIndices.push_back(Elt: Idx);
2100
2101 SmallVector<Type *, 2> Types = {I.getType(), I.getPointerOperandType()};
2102 SmallVector<Value *, 4> Args;
2103 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
2104 Args.push_back(Elt: I.getPointerOperand());
2105 Args.append(in_start: NewIndices.begin(), in_end: NewIndices.end());
2106
2107 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep,
2108 OverloadTypes: {Types}, Args: {Args});
2109 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2110 return NewI;
2111 }
2112 }
2113 }
2114
2115 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
2116 SmallVector<Value *, 4> Args;
2117 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
2118 llvm::append_range(C&: Args, R: I.operands());
2119 Instruction *NewI =
2120 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
2121 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2122 return NewI;
2123}
2124
2125Instruction *SPIRVEmitIntrinsicsImpl::visitBitCastInst(BitCastInst &I) {
2126 IRBuilder<> B(I.getParent());
2127 B.SetInsertPoint(&I);
2128 Value *Source = I.getOperand(i_nocapture: 0);
2129
2130 // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of
2131 // varying element types. In case of IR coming from older versions of LLVM
2132 // such bitcasts do not provide sufficient information, should be just skipped
2133 // here, and handled in insertPtrCastOrAssignTypeInstr.
2134 if (isPointerTy(T: I.getType())) {
2135 replaceAllUsesWith(Src: &I, Dest: Source);
2136 I.eraseFromParent();
2137 return nullptr;
2138 }
2139
2140 SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
2141 SmallVector<Value *> Args(I.op_begin(), I.op_end());
2142 Instruction *NewI =
2143 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_bitcast, OverloadTypes: {Types}, Args: {Args});
2144 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2145 return NewI;
2146}
2147
2148void SPIRVEmitIntrinsicsImpl::insertAssignPtrTypeTargetExt(
2149 TargetExtType *AssignedType, Value *V, IRBuilder<> &B) {
2150 Type *VTy = V->getType();
2151
2152 // A couple of sanity checks.
2153 assert((isPointerTy(VTy)) && "Expect a pointer type!");
2154 if (Type *ElemTy = getPointeeType(Ty: VTy))
2155 if (ElemTy != AssignedType)
2156 report_fatal_error(reason: "Unexpected pointer element type!");
2157
2158 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: V);
2159 if (!AssignCI) {
2160 GR->buildAssignType(B, Ty: AssignedType, Arg: V, CanUseAnyVectorRank);
2161 return;
2162 }
2163
2164 Type *CurrentType =
2165 dyn_cast<ConstantAsMetadata>(
2166 Val: cast<MetadataAsValue>(Val: AssignCI->getOperand(i_nocapture: 1))->getMetadata())
2167 ->getType();
2168 if (CurrentType == AssignedType)
2169 return;
2170
2171 // Builtin types cannot be redeclared or casted.
2172 if (CurrentType->isTargetExtTy())
2173 report_fatal_error(reason: "Type mismatch " + CurrentType->getTargetExtName() +
2174 "/" + AssignedType->getTargetExtName() +
2175 " for value " + V->getName(),
2176 gen_crash_diag: false);
2177
2178 // Our previous guess about the type seems to be wrong, let's update
2179 // inferred type according to a new, more precise type information.
2180 GR->updateAssignType(
2181 AssignCI, Arg: V, OfType: getNormalizedPoisonValue(Ty: AssignedType, CanUseAnyVectorRank));
2182}
2183
2184void SPIRVEmitIntrinsicsImpl::replacePointerOperandWithPtrCast(
2185 Instruction *I, Value *Pointer, Type *ExpectedElementType,
2186 unsigned OperandToReplace, IRBuilder<> &B) {
2187 TypeValidated.insert(Ptr: I);
2188
2189 // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType
2190 Type *PointerElemTy = deduceElementTypeHelper(I: Pointer, UnknownElemTypeI8: false);
2191 if (PointerElemTy == ExpectedElementType ||
2192 isEquivalentTypes(Ty1: PointerElemTy, Ty2: ExpectedElementType))
2193 return;
2194
2195 setInsertPointSkippingPhis(B, I);
2196 Value *ExpectedElementVal =
2197 getNormalizedPoisonValue(Ty: ExpectedElementType, CanUseAnyVectorRank);
2198 MetadataAsValue *VMD = buildMD(Arg: ExpectedElementVal);
2199 unsigned AddressSpace = getPointerAddressSpace(T: Pointer->getType());
2200 bool FirstPtrCastOrAssignPtrType = true;
2201
2202 // Do not emit new spv_ptrcast if equivalent one already exists or when
2203 // spv_assign_ptr_type already targets this pointer with the same element
2204 // type.
2205 if (Pointer->hasUseList()) {
2206 for (auto User : Pointer->users()) {
2207 auto *II = dyn_cast<IntrinsicInst>(Val: User);
2208 if (!II ||
2209 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
2210 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
2211 II->getOperand(i_nocapture: 0) != Pointer)
2212 continue;
2213
2214 // There is some spv_ptrcast/spv_assign_ptr_type already targeting this
2215 // pointer.
2216 FirstPtrCastOrAssignPtrType = false;
2217 if (II->getOperand(i_nocapture: 1) != VMD ||
2218 dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 2))->getSExtValue() !=
2219 AddressSpace)
2220 continue;
2221
2222 // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the
2223 // same element type and address space.
2224 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
2225 return;
2226
2227 // This must be a spv_ptrcast, do not emit new if this one has the same BB
2228 // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type.
2229 if (II->getParent() != I->getParent())
2230 continue;
2231
2232 I->setOperand(i: OperandToReplace, Val: II);
2233 return;
2234 }
2235 }
2236
2237 // Never replace an already-deduced pointer element type with a non-pointer
2238 // one. The conflicting use comes from a mis-deduced expected type. Leave the
2239 // operand untouched rather than emitting a ptrcast that re-introduces the
2240 // collapsed type at the use site.
2241 if (PointerElemTy && isPointerTyOrWrapper(Ty: PointerElemTy) &&
2242 !isPointerTyOrWrapper(Ty: ExpectedElementType) &&
2243 tracesToPointerAlloca(V: Pointer))
2244 return;
2245
2246 if (isa<Instruction>(Val: Pointer) || isa<Argument>(Val: Pointer)) {
2247 if (FirstPtrCastOrAssignPtrType) {
2248 // If this would be the first spv_ptrcast, do not emit spv_ptrcast and
2249 // emit spv_assign_ptr_type instead.
2250 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
2251 return;
2252 } else if (isTodoType(Op: Pointer)) {
2253 eraseTodoType(Op: Pointer);
2254 if (!isa<CallInst>(Val: Pointer) && !isaGEP(V: Pointer) &&
2255 !isa<AllocaInst>(Val: Pointer)) {
2256 // If this wouldn't be the first spv_ptrcast but existing type info is
2257 // uncomplete, update spv_assign_ptr_type arguments.
2258 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Pointer)) {
2259 Type *PrevElemTy = GR->findDeducedElementType(Val: Pointer);
2260 assert(PrevElemTy);
2261 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
2262 std::make_pair(x&: I, y&: Pointer)};
2263 GR->updateAssignType(AssignCI, Arg: Pointer, OfType: ExpectedElementVal);
2264 propagateElemType(Op: Pointer, ElemTy: PrevElemTy, VisitedSubst);
2265 } else {
2266 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
2267 }
2268 return;
2269 }
2270 }
2271 }
2272
2273 // Emit spv_ptrcast
2274 SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()};
2275 SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(C: AddressSpace)};
2276 auto *PtrCastI =
2277 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
2278 I->setOperand(i: OperandToReplace, Val: PtrCastI);
2279 // We need to set up a pointee type for the newly created spv_ptrcast.
2280 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: PtrCastI);
2281}
2282
2283void SPIRVEmitIntrinsicsImpl::insertPtrCastOrAssignTypeInstr(Instruction *I,
2284 IRBuilder<> &B) {
2285 // Handle basic instructions:
2286 StoreInst *SI = dyn_cast<StoreInst>(Val: I);
2287 if (IsKernelArgInt8(F: CurrF, SI)) {
2288 replacePointerOperandWithPtrCast(
2289 I, Pointer: SI->getValueOperand(), ExpectedElementType: IntegerType::getInt8Ty(C&: CurrF->getContext()),
2290 OperandToReplace: 0, B);
2291 }
2292 if (SI) {
2293 Value *Op = SI->getValueOperand();
2294 Value *Pointer = SI->getPointerOperand();
2295 Type *OpTy = Op->getType();
2296 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
2297 OpTy = restoreMutatedType(GR, I: OpI, Ty: OpTy);
2298 if (auto It = AggrConstTypes.find(Val: OpI); It != AggrConstTypes.end())
2299 OpTy = It->second;
2300 }
2301 if (OpTy == Op->getType())
2302 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: Op, UnknownElemTypeI8: false);
2303 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 1, B);
2304 return;
2305 }
2306 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
2307 Value *Pointer = LI->getPointerOperand();
2308 Type *OpTy = LI->getType();
2309 // If the loaded from pointer carries byref/byval, the dominant type is the
2310 // pointee type specified therein, and we should retain it.
2311 if (Argument *Arg;
2312 (Arg = dyn_cast<Argument>(Val: Pointer)) && hasPointeeTypeAttr(Arg))
2313 OpTy = getPointeeTypeByAttr(Arg);
2314
2315 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
2316 if (Type *ElemTy = GR->findDeducedElementType(Val: LI)) {
2317 OpTy = getTypedPointerWrapper(ElemTy, AS: PtrTy->getAddressSpace());
2318 } else {
2319 Type *NewOpTy = OpTy;
2320 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: LI, UnknownElemTypeI8: false);
2321 if (OpTy == NewOpTy)
2322 insertTodoType(Op: Pointer);
2323 }
2324 }
2325 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
2326 return;
2327 }
2328 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(Val: I)) {
2329 Value *Pointer = GEPI->getPointerOperand();
2330 Type *OpTy = nullptr;
2331
2332 // Logical SPIR-V is not allowed to use Op*PtrAccessChain instructions. If
2333 // the first index is 0, then we can trivially lower to OpAccessChain. If
2334 // not we need to try to rewrite the GEP. We avoid adding a pointer cast at
2335 // this time, and will rewrite the GEP when visiting it.
2336 if (TM.getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: GEPI)) {
2337 return;
2338 }
2339
2340 // In all cases, fall back to the GEP type if type scavenging failed.
2341 if (!OpTy)
2342 OpTy = GEPI->getSourceElementType();
2343
2344 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
2345 if (isNestedPointer(Ty: OpTy))
2346 insertTodoType(Op: Pointer);
2347 return;
2348 }
2349
2350 // TODO: review and merge with existing logics:
2351 // Handle calls to builtins (non-intrinsics):
2352 CallInst *CI = dyn_cast<CallInst>(Val: I);
2353 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
2354 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
2355 return;
2356
2357 // collect information about formal parameter types
2358 std::string DemangledName =
2359 getOclOrSpirvBuiltinDemangledName(Name: CI->getCalledFunction()->getName());
2360 Function *CalledF = CI->getCalledFunction();
2361 SmallVector<Type *, 4> CalledArgTys;
2362 bool HaveTypes = false;
2363 for (unsigned OpIdx = 0; OpIdx < CalledF->arg_size(); ++OpIdx) {
2364 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2365 Type *ArgType = CalledArg->getType();
2366 if (!isPointerTy(T: ArgType)) {
2367 CalledArgTys.push_back(Elt: nullptr);
2368 } else if (Type *ArgTypeElem = getPointeeType(Ty: ArgType)) {
2369 CalledArgTys.push_back(Elt: ArgTypeElem);
2370 HaveTypes = true;
2371 } else {
2372 Type *ElemTy = GR->findDeducedElementType(Val: CalledArg);
2373 if (!ElemTy && hasPointeeTypeAttr(Arg: CalledArg))
2374 ElemTy = getPointeeTypeByAttr(Arg: CalledArg);
2375 if (!ElemTy) {
2376 ElemTy = getPointeeTypeByCallInst(DemangledName, CalledF, OpIdx);
2377 if (ElemTy) {
2378 GR->addDeducedElementType(Val: CalledArg,
2379 Ty: normalizeType(Ty: ElemTy, CanUseAnyVectorRank));
2380 } else {
2381 for (User *U : CalledArg->users()) {
2382 if (Instruction *Inst = dyn_cast<Instruction>(Val: U)) {
2383 if ((ElemTy = deduceElementTypeHelper(I: Inst, UnknownElemTypeI8: false)) != nullptr)
2384 break;
2385 }
2386 }
2387 }
2388 }
2389 HaveTypes |= ElemTy != nullptr;
2390 CalledArgTys.push_back(Elt: ElemTy);
2391 }
2392 }
2393
2394 if (DemangledName.empty() && !HaveTypes)
2395 return;
2396
2397 auto Set =
2398 TM.getSubtarget<SPIRVSubtarget>(F: *CalledF).getPreferredInstructionSet();
2399 // Do not reparse rejected declaration types with the builtin parser.
2400 bool ParseBuiltinTypes =
2401 !DemangledName.empty() &&
2402 (!CalledF->isDeclaration() || SPIRV::isBuiltin(DemangledCall: DemangledName, Set));
2403 for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) {
2404 Value *ArgOperand = CI->getArgOperand(i: OpIdx);
2405 if (!isPointerTy(T: ArgOperand->getType()))
2406 continue;
2407
2408 // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs()
2409 if (!isa<Instruction>(Val: ArgOperand) && !isa<Argument>(Val: ArgOperand)) {
2410 // However, we may have assumptions about the formal argument's type and
2411 // may have a need to insert a ptr cast for the actual parameter of this
2412 // call.
2413 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2414 if (!GR->findDeducedElementType(Val: CalledArg))
2415 continue;
2416 }
2417
2418 Type *ExpectedType =
2419 OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr;
2420 if (!ExpectedType && ParseBuiltinTypes)
2421 ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType(
2422 DemangledCall: DemangledName, ArgIdx: OpIdx, Ctx&: I->getContext());
2423 if (!ExpectedType || ExpectedType->isVoidTy())
2424 continue;
2425
2426 if (ExpectedType->isTargetExtTy() &&
2427 !isTypedPointerWrapper(ExtTy: cast<TargetExtType>(Val: ExpectedType)))
2428 insertAssignPtrTypeTargetExt(AssignedType: cast<TargetExtType>(Val: ExpectedType),
2429 V: ArgOperand, B);
2430 else
2431 replacePointerOperandWithPtrCast(I: CI, Pointer: ArgOperand, ExpectedElementType: ExpectedType, OperandToReplace: OpIdx, B);
2432 }
2433}
2434
2435Instruction *
2436SPIRVEmitIntrinsicsImpl::visitInsertElementInst(InsertElementInst &I) {
2437 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2438 // type in LLT and IRTranslator will replace it by the scalar.
2439 if (isVector1(Ty: I.getType()) && !CanUseAnyVectorRank)
2440 return &I;
2441
2442 SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType(),
2443 I.getOperand(i_nocapture: 1)->getType(),
2444 I.getOperand(i_nocapture: 2)->getType()};
2445 IRBuilder<> B(I.getParent());
2446 B.SetInsertPoint(&I);
2447 SmallVector<Value *> Args(I.op_begin(), I.op_end());
2448 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_insertelt,
2449 OverloadTypes: {Types}, Args: {Args});
2450 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2451 return NewI;
2452}
2453
2454Instruction *
2455SPIRVEmitIntrinsicsImpl::visitExtractElementInst(ExtractElementInst &I) {
2456 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2457 // type in LLT and IRTranslator will replace it by the scalar.
2458 if (isVector1(Ty: I.getVectorOperandType()) && !CanUseAnyVectorRank)
2459 return &I;
2460
2461 IRBuilder<> B(I.getParent());
2462 B.SetInsertPoint(&I);
2463 SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
2464 I.getIndexOperand()->getType()};
2465 SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
2466 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_extractelt,
2467 OverloadTypes: {Types}, Args: {Args});
2468 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2469 return NewI;
2470}
2471
2472Instruction *SPIRVEmitIntrinsicsImpl::visitInsertValueInst(InsertValueInst &I) {
2473 IRBuilder<> B(I.getParent());
2474 B.SetInsertPoint(&I);
2475 SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
2476 SmallVector<Value *> Args;
2477 Value *AggregateOp = I.getAggregateOperand();
2478 if (isa<UndefValue>(Val: AggregateOp))
2479 Args.push_back(Elt: UndefValue::get(T: B.getInt32Ty()));
2480 else
2481 Args.push_back(Elt: AggregateOp);
2482 Args.push_back(Elt: I.getInsertedValueOperand());
2483 for (auto &Op : I.indices())
2484 Args.push_back(Elt: B.getInt32(C: Op));
2485 Instruction *NewI =
2486 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_insertv, OverloadTypes: {Types}, Args: {Args});
2487 replaceMemInstrUses(Old: &I, New: NewI, B);
2488 return NewI;
2489}
2490
2491Instruction *
2492SPIRVEmitIntrinsicsImpl::visitExtractValueInst(ExtractValueInst &I) {
2493 IRBuilder<> B(I.getParent());
2494 B.SetInsertPoint(&I);
2495 if (I.getAggregateOperand()->getType()->isAggregateType()) {
2496 // Mutate an aggregate-returning spv_extractv producer to i32 so
2497 // IRTranslator does not see a multi-register value.
2498 CallBase *CB = dyn_cast<CallBase>(Val: I.getAggregateOperand());
2499 if (!CB || CB->getIntrinsicID() != Intrinsic::spv_extractv)
2500 return &I;
2501 CB->mutateType(Ty: B.getInt32Ty());
2502 }
2503 SmallVector<Value *> Args(I.operands());
2504 for (auto &Op : I.indices())
2505 Args.push_back(Elt: B.getInt32(C: Op));
2506 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_extractv,
2507 OverloadTypes: {I.getType()}, Args: {Args});
2508 // If this aggregate extract feeds another insertvalue, the extracted
2509 // composite is used as a SPIR-V value-id by llvm.spv.insertv. Keep the real
2510 // aggregate type in metadata, but expose the value itself as i32 so the
2511 // intrinsic signature remains valid.
2512 if (NewI->getType()->isAggregateType() &&
2513 any_of(Range: I.users(), P: [](User *U) { return isa<InsertValueInst>(Val: U); })) {
2514 AggrConstTypes[NewI] = I.getType();
2515 NewI->mutateType(Ty: B.getInt32Ty());
2516 replaceMemInstrUses(Old: &I, New: NewI, B);
2517 return NewI;
2518 }
2519 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2520 // If the aggregate result feeds a return or callsite whose type was rewritten
2521 // to an i32 value-id by SPIRVPrepareFunctions, mutate it to match.
2522 if (NewI->getType()->isAggregateType()) {
2523 for (const Use &U : NewI->uses()) {
2524 User *Usr = U.getUser();
2525 if (auto *RI = dyn_cast<ReturnInst>(Val: Usr)) {
2526 if (RI->getFunction()->getReturnType() != NewI->getType()) {
2527 NewI->mutateType(Ty: B.getInt32Ty());
2528 break;
2529 }
2530 continue;
2531 }
2532 auto *CB = dyn_cast<CallBase>(Val: Usr);
2533 if (!CB || !CB->isArgOperand(U: &U))
2534 continue;
2535 unsigned ArgNo = CB->getArgOperandNo(U: &U);
2536 FunctionType *FT = CB->getFunctionType();
2537 if (ArgNo < FT->getNumParams() &&
2538 !FT->getParamType(i: ArgNo)->isAggregateType()) {
2539 NewI->mutateType(Ty: B.getInt32Ty());
2540 break;
2541 }
2542 }
2543 }
2544 return NewI;
2545}
2546
2547Instruction *SPIRVEmitIntrinsicsImpl::visitLoadInst(LoadInst &I) {
2548 if (!I.getType()->isAggregateType())
2549 return &I;
2550 IRBuilder<> B(I.getParent());
2551 B.SetInsertPoint(&I);
2552 TrackConstants = false;
2553 const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
2554 MachineMemOperand::Flags Flags =
2555 TLI->getLoadMemOperandFlags(LI: I, DL: CurrF->getDataLayout());
2556
2557 unsigned IntrinsicId;
2558 SmallVector<Value *, 4> Args = {I.getPointerOperand(), B.getInt16(C: Flags)};
2559 if (!I.isAtomic()) {
2560 IntrinsicId = Intrinsic::spv_load;
2561 Args.push_back(Elt: B.getInt32(C: I.getAlign().value()));
2562 } else {
2563 IntrinsicId = Intrinsic::spv_atomic_load;
2564 Args.push_back(Elt: B.getInt8(C: static_cast<uint8_t>(I.getOrdering())));
2565 }
2566 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
2567 ID: IntrinsicId, OverloadTypes: {I.getOperand(i_nocapture: 0)->getType()}, Args);
2568
2569 replaceMemInstrUses(Old: &I, New: NewI, B);
2570 return NewI;
2571}
2572
2573Instruction *SPIRVEmitIntrinsicsImpl::visitStoreInst(StoreInst &I) {
2574 if (!AggrStores.contains(Ptr: &I))
2575 return &I;
2576 IRBuilder<> B(I.getParent());
2577 B.SetInsertPoint(&I);
2578 TrackConstants = false;
2579 const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
2580 MachineMemOperand::Flags Flags =
2581 TLI->getStoreMemOperandFlags(SI: I, DL: CurrF->getDataLayout());
2582 auto *PtrOp = I.getPointerOperand();
2583
2584 if (I.getValueOperand()->getType()->isAggregateType()) {
2585 // It is possible that what used to be an ExtractValueInst has been replaced
2586 // with a call to the spv_extractv intrinsic, and that said call hasn't
2587 // had its return type replaced with i32 during the dedicated pass (because
2588 // it was emitted later); we have to handle this here, because IRTranslator
2589 // cannot deal with multi-register types at the moment.
2590 CallBase *CB = dyn_cast<CallBase>(Val: I.getValueOperand());
2591 assert(CB && CB->getIntrinsicID() == Intrinsic::spv_extractv &&
2592 "Unexpected argument of aggregate type, should be spv_extractv!");
2593 CB->mutateType(Ty: B.getInt32Ty());
2594 }
2595
2596 unsigned IntrinsicId;
2597 SmallVector<Value *, 4> Args = {I.getValueOperand(), PtrOp,
2598 B.getInt16(C: Flags)};
2599 if (!I.isAtomic()) {
2600 IntrinsicId = Intrinsic::spv_store;
2601 Args.push_back(Elt: B.getInt32(C: I.getAlign().value()));
2602 } else {
2603 IntrinsicId = Intrinsic::spv_atomic_store;
2604 Args.push_back(Elt: B.getInt8(C: static_cast<uint8_t>(I.getOrdering())));
2605 }
2606 Instruction *NewI = B.CreateIntrinsicWithoutFolding(
2607 ID: IntrinsicId, OverloadTypes: {I.getValueOperand()->getType(), PtrOp->getType()}, Args);
2608 NewI->copyMetadata(SrcInst: I);
2609 I.eraseFromParent();
2610 return NewI;
2611}
2612
2613Instruction *SPIRVEmitIntrinsicsImpl::visitAllocaInst(AllocaInst &I) {
2614 Value *ArraySize = nullptr;
2615 if (I.isArrayAllocation()) {
2616 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I.getFunction());
2617 if (!STI->canUseExtension(
2618 E: SPIRV::Extension::SPV_INTEL_variable_length_array))
2619 report_fatal_error(
2620 reason: "array allocation: this instruction requires the following "
2621 "SPIR-V extension: SPV_INTEL_variable_length_array",
2622 gen_crash_diag: false);
2623 ArraySize = I.getArraySize();
2624 }
2625 IRBuilder<> B(I.getParent());
2626 B.SetInsertPoint(&I);
2627 TrackConstants = false;
2628 Type *PtrTy = I.getType();
2629 Instruction *NewI =
2630 ArraySize
2631 ? B.CreateIntrinsicWithoutFolding(
2632 ID: Intrinsic::spv_alloca_array, OverloadTypes: {PtrTy, ArraySize->getType()},
2633 Args: {ArraySize, B.getInt32(C: I.getAlign().value())})
2634 : B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_alloca, OverloadTypes: {PtrTy},
2635 Args: {B.getInt32(C: I.getAlign().value())});
2636 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2637 return NewI;
2638}
2639
2640Instruction *
2641SPIRVEmitIntrinsicsImpl::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2642 assert(I.getType()->isAggregateType() && "Aggregate result is expected");
2643 IRBuilder<> B(I.getParent());
2644 B.SetInsertPoint(&I);
2645 SmallVector<Value *> Args(I.operands());
2646 const Triple &TT = TM.getTargetTriple();
2647 Args.push_back(Elt: B.getInt32(C: static_cast<uint32_t>(
2648 getMemScope(TT, Ctx&: I.getContext(), Id: I.getSyncScopeID()))));
2649 // Per SPIR-V spec atomic ops must combine the ordering bits with the
2650 // storage-class bit.
2651 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2652 unsigned AS = I.getPointerOperand()->getType()->getPointerAddressSpace();
2653 uint32_t ScSem = static_cast<uint32_t>(
2654 getMemSemanticsForStorageClass(SC: addressSpaceToStorageClass(AddrSpace: AS, STI: ST)));
2655 Args.push_back(Elt: B.getInt32(C: getMemSemanticsWithStorageClass(
2656 TT, OrderSem: static_cast<uint32_t>(getMemSemantics(Ord: I.getSuccessOrdering())),
2657 StorageClassSem: ScSem)));
2658 Args.push_back(Elt: B.getInt32(C: getMemSemanticsWithStorageClass(
2659 TT, OrderSem: static_cast<uint32_t>(getMemSemantics(Ord: I.getFailureOrdering())),
2660 StorageClassSem: ScSem)));
2661 Instruction *NewI = B.CreateIntrinsicWithoutFolding(
2662 ID: Intrinsic::spv_cmpxchg, OverloadTypes: {I.getPointerOperand()->getType()}, Args: {Args});
2663 replaceMemInstrUses(Old: &I, New: NewI, B);
2664 return NewI;
2665}
2666
2667Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
2668 auto Op = I.getOperation();
2669 if (Op != AtomicRMWInst::UIncWrap && Op != AtomicRMWInst::UDecWrap)
2670 return &I;
2671
2672 // No SPIR-V opcode exists for these, so lowering to an imported helper is an
2673 // AMD extension. Other targets keep the generic expansion.
2674 if (TM.getTargetTriple().getVendor() != Triple::AMD)
2675 return &I;
2676
2677 Module *M = I.getModule();
2678 IRBuilder<> B(I.getParent());
2679 B.SetInsertPoint(&I);
2680
2681 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2682 unsigned AS = I.getPointerOperand()->getType()->getPointerAddressSpace();
2683
2684 uint32_t Scope = static_cast<uint32_t>(
2685 getMemScope(TT: M->getTargetTriple(), Ctx&: I.getContext(), Id: I.getSyncScopeID()));
2686 uint32_t ScSem = static_cast<uint32_t>(
2687 getMemSemanticsForStorageClass(SC: addressSpaceToStorageClass(AddrSpace: AS, STI: ST)));
2688 uint32_t MemSem = getMemSemanticsWithStorageClass(
2689 TT: M->getTargetTriple(),
2690 OrderSem: static_cast<uint32_t>(getMemSemantics(Ord: I.getOrdering())), StorageClassSem: ScSem);
2691
2692 SmallString<64> FuncName(Op == AtomicRMWInst::UIncWrap
2693 ? "__translate_spirv_atomic_uinc_wrap"
2694 : "__translate_spirv_atomic_udec_wrap");
2695
2696 Type *ValTy = I.getValOperand()->getType();
2697 Type *PtrTy = I.getPointerOperand()->getType();
2698 // Encode the address space and value type in the name for overload.
2699 raw_svector_ostream OS(FuncName);
2700 OS << "_p" << AS << "_";
2701 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: ValTy))
2702 OS << "v" << VecTy->getNumElements();
2703 OS << "i" << ValTy->getScalarSizeInBits();
2704
2705 Type *Int32Ty = B.getInt32Ty();
2706 Type *BoolTy = B.getInt1Ty();
2707 SmallVector<Type *, 6> ArgTys = {PtrTy, Int32Ty, Int32Ty,
2708 ValTy, BoolTy, BoolTy};
2709 FunctionType *FT = FunctionType::get(Result: ValTy, Params: ArgTys, isVarArg: false);
2710 FunctionCallee FC = M->getOrInsertFunction(Name: FuncName, T: FT);
2711 cast<Function>(Val: FC.getCallee())->setCallingConv(CallingConv::SPIR_FUNC);
2712
2713 SmallVector<Value *, 6> Args = {
2714 I.getPointerOperand(), B.getInt32(C: Scope),
2715 B.getInt32(C: MemSem), I.getValOperand(),
2716 B.getInt1(V: I.isVolatile()), B.getInt1(V: I.isElementwise())};
2717 CallInst *CI = B.CreateCall(Callee: FC, Args);
2718 CI->setCallingConv(CallingConv::SPIR_FUNC);
2719 // SPIRVCallLowering reads alias.scope/noalias off the call to build the
2720 // aliasing decorations and runs after this pass, so preserve the metadata.
2721 CI->copyMetadata(SrcInst: I);
2722
2723 replaceAllUsesWithAndErase(B, Src: &I, Dest: CI);
2724 return CI;
2725}
2726
2727static bool isAbortCall(const Instruction &I, const SPIRVSubtarget &ST) {
2728 auto *CI = dyn_cast<CallInst>(Val: &I);
2729 if (!CI)
2730 return false;
2731 switch (CI->getIntrinsicID()) {
2732 case Intrinsic::spv_abort:
2733 return true;
2734 case Intrinsic::trap:
2735 case Intrinsic::ubsantrap:
2736 // When the extension is enabled, selection lowers these to OpAbortKHR.
2737 return ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_abort);
2738 default:
2739 return false;
2740 }
2741}
2742
2743// The OpAbortKHR instruction itself is a block terminator, so we don't need to
2744// emit an extra OpUnreachable instruction.
2745static bool precededByAbortIntrinsic(const UnreachableInst &I,
2746 const SPIRVSubtarget &ST) {
2747 // Find a previous non-debug instruction.
2748 const Instruction *Prev = I.getPrevNode();
2749 while (Prev && Prev->isDebugOrPseudoInst())
2750 Prev = Prev->getPrevNode();
2751
2752 if (Prev && isAbortCall(I: *Prev, ST))
2753 return true;
2754
2755 assert(llvm::none_of(
2756 *I.getParent(),
2757 [&ST](const Instruction &II) { return isAbortCall(II, ST); }) &&
2758 "abort-like call must be the last non-debug instruction before its "
2759 "block's terminator");
2760 return false;
2761}
2762
2763Instruction *SPIRVEmitIntrinsicsImpl::visitUnreachableInst(UnreachableInst &I) {
2764 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2765 if (precededByAbortIntrinsic(I, ST))
2766 return &I;
2767 IRBuilder<> B(&I);
2768 B.CreateIntrinsic(ID: Intrinsic::spv_unreachable, Args: {});
2769 return &I;
2770}
2771
2772// llvm.compiler.used and llvm.used hold use-list entries that protect their
2773// referenced globals from DCE without participating in code generation.
2774static bool isUseListGlobal(StringRef Name) {
2775 return Name == "llvm.compiler.used" || Name == "llvm.used";
2776}
2777
2778// Returns true for module-level globals that should not have SPIR-V intrinsics
2779// emitted (use-list globals plus llvm.global.annotations).
2780static bool isArtificialGlobal(StringRef Name) {
2781 return isUseListGlobal(Name) || Name == "llvm.global.annotations";
2782}
2783
2784// Returns true if every use of GV traces back to llvm.compiler.used or
2785// llvm.used.
2786static bool hasOnlyArtificialUses(const GlobalVariable &GV) {
2787 SmallPtrSet<const Value *, 8> Visited;
2788 SmallVector<const Value *> Stack(GV.users());
2789 while (!Stack.empty()) {
2790 const Value *V = Stack.pop_back_val();
2791 if (!Visited.insert(Ptr: V).second)
2792 continue;
2793 if (const auto *GVUser = dyn_cast<GlobalVariable>(Val: V)) {
2794 if (!isUseListGlobal(Name: GVUser->getName()))
2795 return false;
2796 continue;
2797 }
2798 if (const auto *C = dyn_cast<Constant>(Val: V)) {
2799 Stack.append(in_start: C->user_begin(), in_end: C->user_end());
2800 continue;
2801 }
2802 return false;
2803 }
2804 return true;
2805}
2806
2807static bool
2808shouldEmitIntrinsicsForGlobalValue(const GlobalVariableUsers &GVUsers,
2809 const GlobalVariable &GV,
2810 const Function *F) {
2811 // Skip special artificial variables.
2812 if (isArtificialGlobal(Name: GV.getName()))
2813 return false;
2814
2815 auto &UserFunctions = GVUsers.getTransitiveUserFunctions(GV);
2816 if (UserFunctions.contains(Ptr: F))
2817 return true;
2818
2819 // Do not emit the intrinsics in this function, it's going to be emitted on
2820 // the functions that reference it.
2821 if (!UserFunctions.empty())
2822 return false;
2823
2824 // Emit definitions for globals that are not referenced by any function on the
2825 // first function definition.
2826 const Module &M = *F->getParent();
2827 const Function &FirstDefinition = *M.getFunctionDefs().begin();
2828 return F == &FirstDefinition;
2829}
2830
2831Value *SPIRVEmitIntrinsicsImpl::buildSpvUndefComposite(Type *AggrTy,
2832 IRBuilder<> &B) {
2833 auto MakeLeaf = [&](Type *ElemTy) -> Instruction * {
2834 CallInst *Leaf = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_undef, Args: {});
2835 AggrConsts[Leaf] = PoisonValue::get(T: ElemTy);
2836 AggrConstTypes[Leaf] = ElemTy;
2837 return Leaf;
2838 };
2839 SmallVector<Value *, 4> Elems;
2840 if (auto *ArrTy = dyn_cast<ArrayType>(Val: AggrTy)) {
2841 Elems.assign(NumElts: ArrTy->getNumElements(), Elt: MakeLeaf(ArrTy->getElementType()));
2842 } else {
2843 auto *StructTy = cast<StructType>(Val: AggrTy);
2844 DenseMap<Type *, Instruction *> LeafByType;
2845 for (unsigned I = 0; I < StructTy->getNumElements(); ++I) {
2846 Type *ElemTy = StructTy->getContainedType(i: I);
2847 auto &Entry = LeafByType[ElemTy];
2848 if (!Entry)
2849 Entry = MakeLeaf(ElemTy);
2850 Elems.push_back(Elt: Entry);
2851 }
2852 }
2853 CallInst *Composite = B.CreateIntrinsicWithoutFolding(
2854 ID: Intrinsic::spv_const_composite, OverloadTypes: {B.getInt32Ty()}, Args: Elems);
2855 AggrConsts[Composite] = PoisonValue::get(T: AggrTy);
2856 AggrConstTypes[Composite] = AggrTy;
2857 return Composite;
2858}
2859
2860// If a function directly returns an aggregate-typed call result,
2861// the ReturnInst carries an aggregate while the function signature
2862// was rewritten to i32 by SPIRVPrepareFunctions. Rebuild the return value
2863// via extractvalue/insertvalue so the regular spv_extractv/spv_insertv
2864// lowering produces a valid OpReturnValue.
2865void SPIRVEmitIntrinsicsImpl::reconstructAggregateReturns(Function &Func,
2866 IRBuilder<> &B) {
2867 Type *OrigRetTy = GR->findMutated(Val: &Func);
2868 if (!OrigRetTy || !OrigRetTy->isAggregateType())
2869 return;
2870 for (BasicBlock &BB : Func) {
2871 auto *RI = dyn_cast<ReturnInst>(Val: BB.getTerminator());
2872 if (!RI)
2873 continue;
2874 Value *RetVal = RI->getReturnValue();
2875 if (!RetVal || RetVal->getType() != OrigRetTy || !isa<CallBase>(Val: RetVal))
2876 continue;
2877 Type *AggrTy = RetVal->getType();
2878 uint64_t NumElts = isa<StructType>(Val: AggrTy)
2879 ? cast<StructType>(Val: AggrTy)->getNumElements()
2880 : cast<ArrayType>(Val: AggrTy)->getNumElements();
2881 B.SetInsertPoint(RI);
2882 Value *Rebuilt = PoisonValue::get(T: AggrTy);
2883 for (uint64_t I = 0; I < NumElts; ++I) {
2884 Value *Elt = B.CreateExtractValue(Agg: RetVal, Idxs: I);
2885 Rebuilt = B.CreateInsertValue(Agg: Rebuilt, Val: Elt, Idxs: I);
2886 }
2887 RI->setOperand(i_nocapture: 0, Val_nocapture: Rebuilt);
2888 }
2889}
2890
2891void SPIRVEmitIntrinsicsImpl::processGlobalValue(GlobalVariable &GV,
2892 IRBuilder<> &B) {
2893
2894 if (!shouldEmitIntrinsicsForGlobalValue(GVUsers, GV, F: CurrF))
2895 return;
2896
2897 // Record the pointee type for every global, not only initialized ones, so an
2898 // undef non-constant aggregate global is not later collapsed to its element
2899 // type. Result is ignored, because TypedPointerType is not supported
2900 // by llvm IR general logic.
2901 deduceElementTypeHelper(I: &GV, UnknownElemTypeI8: false);
2902
2903 Constant *Init = nullptr;
2904 if (hasInitializer(GV: &GV)) {
2905 Init = GV.getInitializer();
2906 Value *InitOp = Init;
2907 if (isa<UndefValue>(Val: Init) && Init->getType()->isAggregateType()) {
2908 const SPIRVSubtarget *STI = TM.getSubtargetImpl();
2909 bool UsePoison =
2910 isa<PoisonValue>(Val: Init) &&
2911 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
2912 if (UsePoison) {
2913 CallInst *Call = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_poison,
2914 OverloadTypes: {B.getInt32Ty()}, Args: {});
2915 AggrConsts[Call] = cast<PoisonValue>(Val: Init);
2916 AggrConstTypes[Call] = Init->getType();
2917 InitOp = Call;
2918 } else {
2919 InitOp = buildSpvUndefComposite(AggrTy: Init->getType(), B);
2920 }
2921 }
2922 Type *Ty = isAggrConstForceInt32(V: Init) ? B.getInt32Ty() : Init->getType();
2923 Constant *Const = isAggrConstForceInt32(V: Init) ? B.getInt32(C: 1) : Init;
2924 CallInst *InitInst = B.CreateIntrinsicWithoutFolding(
2925 ID: Intrinsic::spv_init_global, OverloadTypes: {GV.getType(), Ty}, Args: {&GV, Const});
2926 InitInst->setArgOperand(i: 1, v: InitOp);
2927 }
2928 // Globals with only use-list references have no real function uses. Emit
2929 // spv_unref_global so buildGlobalVariable is called for them.
2930 if (!Init && hasOnlyArtificialUses(GV))
2931 B.CreateIntrinsic(ID: Intrinsic::spv_unref_global, OverloadTypes: GV.getType(), Args: &GV);
2932}
2933
2934// Return true, if we can't decide what is the pointee type now and will get
2935// back to the question later. Return false is spv_assign_ptr_type is not needed
2936// or can be inserted immediately.
2937bool SPIRVEmitIntrinsicsImpl::insertAssignPtrTypeIntrs(Instruction *I,
2938 IRBuilder<> &B,
2939 bool UnknownElemTypeI8) {
2940 reportFatalOnTokenType(I);
2941 if (!isPointerTy(T: I->getType()) || !requireAssignType(I))
2942 return false;
2943
2944 setInsertPointAfterDef(B, I);
2945 if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) {
2946 GR->buildAssignPtr(B, ElemTy, Arg: I);
2947 return false;
2948 }
2949 return true;
2950}
2951
2952void SPIRVEmitIntrinsicsImpl::insertAssignTypeIntrs(Instruction *I,
2953 IRBuilder<> &B) {
2954 // TODO: extend the list of functions with known result types
2955 static StringMap<unsigned> ResTypeWellKnown = {
2956 {"async_work_group_copy", WellKnownTypes::Event},
2957 {"async_work_group_strided_copy", WellKnownTypes::Event},
2958 {"__spirv_GroupAsyncCopy", WellKnownTypes::Event}};
2959
2960 reportFatalOnTokenType(I);
2961
2962 bool IsKnown = false;
2963 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
2964 if (!CI->isIndirectCall() && !CI->isInlineAsm() &&
2965 CI->getCalledFunction() && !CI->getCalledFunction()->isIntrinsic()) {
2966 Function *CalledF = CI->getCalledFunction();
2967 std::string DemangledName =
2968 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
2969 FPDecorationId DecorationId = FPDecorationId::NONE;
2970 if (DemangledName.length() > 0)
2971 DemangledName =
2972 SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName, DecorationId: &DecorationId);
2973 auto ResIt = ResTypeWellKnown.find(Key: DemangledName);
2974 if (ResIt != ResTypeWellKnown.end()) {
2975 IsKnown = true;
2976 setInsertPointAfterDef(B, I);
2977 switch (ResIt->second) {
2978 case WellKnownTypes::Event:
2979 GR->buildAssignType(
2980 B, Ty: TargetExtType::get(Context&: I->getContext(), Name: "spirv.Event"), Arg: I,
2981 CanUseAnyVectorRank);
2982 break;
2983 }
2984 }
2985 // check if a floating rounding mode or saturation info is present
2986 switch (DecorationId) {
2987 default:
2988 break;
2989 case FPDecorationId::SAT:
2990 createSaturatedConversionDecoration(I: CI, B);
2991 break;
2992 case FPDecorationId::RTE:
2993 createRoundingModeDecoration(
2994 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTE, B);
2995 break;
2996 case FPDecorationId::RTZ:
2997 createRoundingModeDecoration(
2998 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTZ, B);
2999 break;
3000 case FPDecorationId::RTP:
3001 createRoundingModeDecoration(
3002 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTP, B);
3003 break;
3004 case FPDecorationId::RTN:
3005 createRoundingModeDecoration(
3006 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTN, B);
3007 break;
3008 }
3009 }
3010 }
3011
3012 Type *Ty = I->getType();
3013 if (!IsKnown && !Ty->isVoidTy() && !isPointerTy(T: Ty) && requireAssignType(I)) {
3014 setInsertPointAfterDef(B, I);
3015 Type *TypeToAssign = Ty;
3016 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
3017 if (isSpvAggrPlaceholder(V: II)) {
3018 auto It = AggrConstTypes.find(Val: II);
3019 if (It == AggrConstTypes.end())
3020 report_fatal_error(reason: "Unknown composite intrinsic type");
3021 TypeToAssign = It->second;
3022 } else if (II->getIntrinsicID() == Intrinsic::spv_poison) {
3023 if (auto It = AggrConstTypes.find(Val: II); It != AggrConstTypes.end())
3024 TypeToAssign = It->second;
3025 }
3026 } else if (auto It = AggrConstTypes.find(Val: I); It != AggrConstTypes.end())
3027 TypeToAssign = It->second;
3028 TypeToAssign = restoreMutatedType(GR, I, Ty: TypeToAssign);
3029 GR->buildAssignType(B, Ty: TypeToAssign, Arg: I, CanUseAnyVectorRank);
3030 }
3031 for (const auto &Op : I->operands()) {
3032 if (isa<ConstantPointerNull>(Val: Op) || isa<UndefValue>(Val: Op) ||
3033 isVector1(Ty: Op->getType()) || // <1 x T> gets clobbered ty IRTranslator.
3034 // Check GetElementPtrConstantExpr case.
3035 (isa<ConstantExpr>(Val: Op) &&
3036 (isa<GEPOperator>(Val: Op) ||
3037 (cast<ConstantExpr>(Val: Op)->getOpcode() == CastInst::IntToPtr)))) {
3038 setInsertPointSkippingPhis(B, I);
3039 Type *OpTy = Op->getType();
3040 if (isa<UndefValue>(Val: Op) && OpTy->isAggregateType()) {
3041 CallInst *AssignCI =
3042 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {B.getInt32Ty()}, Arg: Op,
3043 Arg2: UndefValue::get(T: B.getInt32Ty()), Imms: {}, B);
3044 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: AssignCI);
3045 } else if (!isa<Instruction>(Val: Op)) {
3046 Type *OpTy = Op->getType();
3047 Type *OpTyElem = getPointeeType(Ty: OpTy);
3048 if (OpTyElem) {
3049 GR->buildAssignPtr(B, ElemTy: OpTyElem, Arg: Op);
3050 } else if (isPointerTy(T: OpTy)) {
3051 Type *ElemTy = GR->findDeducedElementType(Val: Op);
3052 GR->buildAssignPtr(B, ElemTy: ElemTy ? ElemTy : deduceElementType(I: Op, UnknownElemTypeI8: true),
3053 Arg: Op);
3054 } else {
3055 Value *OpTyVal = Op;
3056 if (OpTy->isTargetExtTy()) {
3057 // We need to do this in order to be consistent with how target ext
3058 // types are handled in `processInstrAfterVisit`
3059 OpTyVal = getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank);
3060 }
3061 CallInst *AssignCI = buildIntrWithMD(
3062 IntrID: Intrinsic::spv_assign_type, Types: {OpTy},
3063 Arg: getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank), Arg2: OpTyVal, Imms: {},
3064 B);
3065 GR->addAssignPtrTypeInstr(Val: OpTyVal, AssignPtrTyCI: AssignCI);
3066 }
3067 }
3068 }
3069 }
3070}
3071
3072bool SPIRVEmitIntrinsicsImpl::shouldTryToAddMemAliasingDecoration(
3073 Instruction *Inst) {
3074 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*Inst->getFunction());
3075 if (!STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing))
3076 return false;
3077 // Add aliasing decorations to internal load and store intrinsics.
3078 // Do not attach them to store atomic or load atomic intrinsics / instructions
3079 // since the extension is inconsistent at the moment (we cannot add the
3080 // decoration to atomic stores because they do not have an id).
3081 return match(V: Inst,
3082 P: m_AnyIntrinsic<Intrinsic::spv_load, Intrinsic::spv_store>());
3083}
3084
3085void SPIRVEmitIntrinsicsImpl::insertSpirvDecorations(Instruction *I,
3086 IRBuilder<> &B) {
3087 if (MDNode *MD = I->getMetadata(Kind: "spirv.Decorations")) {
3088 setInsertPointAfterDef(B, I);
3089 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
3090 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
3091 }
3092 // Lower alias.scope/noalias metadata
3093 {
3094 auto processMemAliasingDecoration = [&](unsigned Kind) {
3095 if (MDNode *AliasListMD = I->getMetadata(KindID: Kind)) {
3096 if (shouldTryToAddMemAliasingDecoration(Inst: I)) {
3097 uint32_t Dec = Kind == LLVMContext::MD_alias_scope
3098 ? SPIRV::Decoration::AliasScopeINTEL
3099 : SPIRV::Decoration::NoAliasINTEL;
3100 SmallVector<Value *, 3> Args = {
3101 I, ConstantInt::get(Ty: B.getInt32Ty(), V: Dec),
3102 MetadataAsValue::get(Context&: I->getContext(), MD: AliasListMD)};
3103 setInsertPointAfterDef(B, I);
3104 B.CreateIntrinsic(ID: Intrinsic::spv_assign_aliasing_decoration,
3105 OverloadTypes: {I->getType()}, Args: {Args});
3106 }
3107 }
3108 };
3109 processMemAliasingDecoration(LLVMContext::MD_alias_scope);
3110 processMemAliasingDecoration(LLVMContext::MD_noalias);
3111 }
3112 // MD_fpmath
3113 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_fpmath)) {
3114 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I->getFunction());
3115 bool AllowFPMaxError =
3116 STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_fp_max_error);
3117 if (!AllowFPMaxError)
3118 return;
3119
3120 setInsertPointAfterDef(B, I);
3121 B.CreateIntrinsic(ID: Intrinsic::spv_assign_fpmaxerror_decoration,
3122 OverloadTypes: {I->getType()},
3123 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
3124 }
3125 if (I->getModule()->getTargetTriple().getVendor() == Triple::AMD &&
3126 isa<AtomicRMWInst>(Val: I)) {
3127 // If present, we encode AMDGPU atomic metadata as UserSemantic string
3128 // decorations, which will be parsed during reverse translation.
3129 auto &Ctx = B.getContext();
3130 auto *US = ConstantAsMetadata::get(
3131 C: ConstantInt::get(Ty: B.getInt32Ty(), V: SPIRV::Decoration::UserSemantic));
3132
3133 SmallVector<Metadata *> MDs;
3134 if (I->hasMetadata(Kind: "amdgpu.no.fine.grained.memory"))
3135 MDs.push_back(Elt: MDNode::get(
3136 Context&: Ctx, MDs: {US, MDString::get(Context&: Ctx, Str: "amdgpu.no.fine.grained.memory")}));
3137 if (I->hasMetadata(Kind: "amdgpu.no.remote.memory"))
3138 MDs.push_back(Elt: MDNode::get(
3139 Context&: Ctx, MDs: {US, MDString::get(Context&: Ctx, Str: "amdgpu.no.remote.memory")}));
3140 if (I->hasMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode))
3141 MDs.push_back(Elt: MDNode::get(
3142 Context&: Ctx, MDs: {US, MDString::get(Context&: Ctx, Str: "atomic.ignore.denormal.mode")}));
3143 if (!MDs.empty())
3144 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
3145 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs))});
3146 }
3147}
3148
3149static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
3150 const Module &M,
3151 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
3152 &FPFastMathDefaultInfoMap,
3153 Function *F) {
3154 auto it = FPFastMathDefaultInfoMap.find(Val: F);
3155 if (it != FPFastMathDefaultInfoMap.end())
3156 return it->second;
3157
3158 // If the map does not contain the entry, create a new one. Initialize it to
3159 // contain all 3 elements sorted by bit width of target type: {half, float,
3160 // double}.
3161 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
3162 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
3163 Args: SPIRV::FPFastMathMode::None);
3164 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
3165 Args: SPIRV::FPFastMathMode::None);
3166 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
3167 Args: SPIRV::FPFastMathMode::None);
3168 return FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
3169}
3170
3171static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
3172 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
3173 const Type *Ty) {
3174 size_t BitWidth = Ty->getScalarSizeInBits();
3175 int Index =
3176 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
3177 BitWidth);
3178 assert(Index >= 0 && Index < 3 &&
3179 "Expected FPFastMathDefaultInfo for half, float, or double");
3180 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3181 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3182 return FPFastMathDefaultInfoVec[Index];
3183}
3184
3185void SPIRVEmitIntrinsicsImpl::insertConstantsForFPFastMathDefault(Module &M) {
3186 const SPIRVSubtarget *ST = TM.getSubtargetImpl();
3187 if (!ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
3188 return;
3189
3190 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
3191 // We need the entry point (function) as the key, and the target
3192 // type and flags as the value.
3193 // We also need to check ContractionOff and SignedZeroInfNanPreserve
3194 // execution modes, as they are now deprecated and must be replaced
3195 // with FPFastMathDefaultInfo.
3196 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
3197 if (!Node) {
3198 if (!M.getNamedMetadata(Name: "opencl.enable.FP_CONTRACT")) {
3199 // This requires emitting ContractionOff. However, because
3200 // ContractionOff is now deprecated, we need to replace it with
3201 // FPFastMathDefaultInfo with FP Fast Math Mode bitmask set to all 0.
3202 // We need to create the constant for that.
3203
3204 // Create constant instruction with the bitmask flags.
3205 Constant *InitValue =
3206 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: 0);
3207 // TODO: Reuse constant if there is one already with the required
3208 // value.
3209 [[maybe_unused]] GlobalVariable *GV =
3210 new GlobalVariable(M, // Module
3211 Type::getInt32Ty(C&: M.getContext()), // Type
3212 true, // isConstant
3213 GlobalValue::InternalLinkage, // Linkage
3214 InitValue // Initializer
3215 );
3216 }
3217 return;
3218 }
3219
3220 // The table maps function pointers to their default FP fast math info. It
3221 // can be assumed that the SmallVector is sorted by the bit width of the
3222 // type. The first element is the smallest bit width, and the last element
3223 // is the largest bit width, therefore, we will have {half, float, double}
3224 // in the order of their bit widths.
3225 DenseMap<Function *, SPIRV::FPFastMathDefaultInfoVector>
3226 FPFastMathDefaultInfoMap;
3227
3228 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
3229 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
3230 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
3231 Function *F = cast<Function>(
3232 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
3233 const auto EM =
3234 cast<ConstantInt>(
3235 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
3236 ->getZExtValue();
3237 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
3238 assert(MDN->getNumOperands() == 4 &&
3239 "Expected 4 operands for FPFastMathDefault");
3240 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
3241 unsigned Flags =
3242 cast<ConstantInt>(
3243 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
3244 ->getZExtValue();
3245 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3246 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3247 SPIRV::FPFastMathDefaultInfo &Info =
3248 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
3249 Info.FastMathFlags = Flags;
3250 Info.FPFastMathDefault = true;
3251 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
3252 assert(MDN->getNumOperands() == 2 &&
3253 "Expected no operands for ContractionOff");
3254
3255 // We need to save this info for every possible FP type, i.e. {half,
3256 // float, double, fp128}.
3257 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3258 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3259 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3260 Info.ContractionOff = true;
3261 }
3262 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
3263 assert(MDN->getNumOperands() == 3 &&
3264 "Expected 1 operand for SignedZeroInfNanPreserve");
3265 unsigned TargetWidth =
3266 cast<ConstantInt>(
3267 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
3268 ->getZExtValue();
3269 // We need to save this info only for the FP type with TargetWidth.
3270 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3271 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3272 int Index = SPIRV::FPFastMathDefaultInfoVector::
3273 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
3274 assert(Index >= 0 && Index < 3 &&
3275 "Expected FPFastMathDefaultInfo for half, float, or double");
3276 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3277 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3278 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
3279 }
3280 }
3281
3282 DenseMap<unsigned, GlobalVariable *> GlobalVars;
3283 for (auto &[Func, FPFastMathDefaultInfoVec] : FPFastMathDefaultInfoMap) {
3284 if (FPFastMathDefaultInfoVec.empty())
3285 continue;
3286
3287 for (const SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3288 assert(Info.Ty && "Expected target type for FPFastMathDefaultInfo");
3289 // Skip if none of the execution modes was used.
3290 unsigned Flags = Info.FastMathFlags;
3291 if (Flags == SPIRV::FPFastMathMode::None && !Info.ContractionOff &&
3292 !Info.SignedZeroInfNanPreserve && !Info.FPFastMathDefault)
3293 continue;
3294
3295 // Check if flags are compatible.
3296 if (Info.ContractionOff && (Flags & SPIRV::FPFastMathMode::AllowContract))
3297 report_fatal_error(reason: "Conflicting FPFastMathFlags: ContractionOff "
3298 "and AllowContract");
3299
3300 if (Info.SignedZeroInfNanPreserve &&
3301 !(Flags &
3302 (SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
3303 SPIRV::FPFastMathMode::NSZ))) {
3304 if (Info.FPFastMathDefault)
3305 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
3306 "SignedZeroInfNanPreserve but at least one of "
3307 "NotNaN/NotInf/NSZ is enabled.");
3308 }
3309
3310 if ((Flags & SPIRV::FPFastMathMode::AllowTransform) &&
3311 !((Flags & SPIRV::FPFastMathMode::AllowReassoc) &&
3312 (Flags & SPIRV::FPFastMathMode::AllowContract))) {
3313 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
3314 "AllowTransform requires AllowReassoc and "
3315 "AllowContract to be set.");
3316 }
3317
3318 auto it = GlobalVars.find(Val: Flags);
3319 GlobalVariable *GV = nullptr;
3320 if (it != GlobalVars.end()) {
3321 // Reuse existing global variable.
3322 GV = it->second;
3323 } else {
3324 // Create constant instruction with the bitmask flags.
3325 Constant *InitValue =
3326 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: Flags);
3327 // TODO: Reuse constant if there is one already with the required
3328 // value.
3329 GV = new GlobalVariable(M, // Module
3330 Type::getInt32Ty(C&: M.getContext()), // Type
3331 true, // isConstant
3332 GlobalValue::InternalLinkage, // Linkage
3333 InitValue // Initializer
3334 );
3335 GlobalVars[Flags] = GV;
3336 }
3337 }
3338 }
3339}
3340
3341void SPIRVEmitIntrinsicsImpl::processInstrAfterVisit(Instruction *I,
3342 IRBuilder<> &B) {
3343 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3344 bool IsConstComposite =
3345 II && II->getIntrinsicID() == Intrinsic::spv_const_composite;
3346 if (IsConstComposite && TrackConstants) {
3347 setInsertPointAfterDef(B, I);
3348 auto t = AggrConsts.find(Val: I);
3349 assert(t != AggrConsts.end());
3350 auto *NewOp =
3351 buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
3352 Types: {II->getType(), II->getType()}, Arg: t->second, Arg2: I, Imms: {}, B);
3353 replaceAllUsesWith(Src: I, Dest: NewOp, DeleteOld: false);
3354 NewOp->setArgOperand(i: 0, v: I);
3355 }
3356 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
3357 for (const auto &Op : I->operands()) {
3358 if (isa<PHINode>(Val: I) || isa<SwitchInst>(Val: I) ||
3359 !(isa<ConstantData>(Val: Op) || isa<ConstantExpr>(Val: Op)))
3360 continue;
3361 unsigned OpNo = Op.getOperandNo();
3362 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
3363 (!II->isBundleOperand(Idx: OpNo) &&
3364 II->paramHasAttr(ArgNo: OpNo, Kind: Attribute::ImmArg))))
3365 continue;
3366
3367 if (!BPrepared) {
3368 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
3369 : B.SetInsertPoint(I);
3370 BPrepared = true;
3371 }
3372 Type *OpTy = Op->getType();
3373 Type *OpElemTy = GR->findDeducedElementType(Val: Op);
3374 Value *NewOp = Op;
3375 if (OpTy->isTargetExtTy()) {
3376 // Since this value is replaced by poison, we need to do the same in
3377 // `insertAssignTypeIntrs`.
3378 Value *OpTyVal = getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank);
3379 NewOp = buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
3380 Types: {OpTy, OpTyVal->getType()}, Arg: Op, Arg2: OpTyVal, Imms: {}, B);
3381 }
3382 if (!IsConstComposite && isPointerTy(T: OpTy) && OpElemTy != nullptr &&
3383 OpElemTy != IntegerType::getInt8Ty(C&: I->getContext())) {
3384 SmallVector<Type *, 2> Types = {OpTy, OpTy};
3385 SmallVector<Value *, 2> Args = {
3386 NewOp,
3387 buildMD(Arg: getNormalizedPoisonValue(Ty: OpElemTy, CanUseAnyVectorRank)),
3388 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
3389 CallInst *PtrCasted = B.CreateIntrinsicWithoutFolding(
3390 ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
3391 GR->buildAssignPtr(B, ElemTy: OpElemTy, Arg: PtrCasted);
3392 NewOp = PtrCasted;
3393 }
3394 if (NewOp != Op)
3395 I->setOperand(i: OpNo, Val: NewOp);
3396 }
3397 if (Named.insert(Ptr: I).second)
3398 emitAssignName(I, B);
3399}
3400
3401Type *SPIRVEmitIntrinsicsImpl::deduceFunParamElementType(Function *F,
3402 unsigned OpIdx) {
3403 SmallPtrSet<Function *, 0> FVisited;
3404 return deduceFunParamElementType(F, OpIdx, FVisited);
3405}
3406
3407Type *SPIRVEmitIntrinsicsImpl::deduceFunParamElementType(
3408 Function *F, unsigned OpIdx, SmallPtrSetImpl<Function *> &FVisited) {
3409 // maybe a cycle
3410 if (!FVisited.insert(Ptr: F).second)
3411 return nullptr;
3412
3413 SmallPtrSet<Value *, 0> Visited;
3414 SmallVector<std::pair<Function *, unsigned>> Lookup;
3415 // search in function's call sites
3416 for (User *U : F->users()) {
3417 CallInst *CI = dyn_cast<CallInst>(Val: U);
3418 if (!CI || OpIdx >= CI->arg_size())
3419 continue;
3420 Value *OpArg = CI->getArgOperand(i: OpIdx);
3421 if (!isPointerTy(T: OpArg->getType()))
3422 continue;
3423 // maybe we already know operand's element type
3424 if (Type *KnownTy = GR->findDeducedElementType(Val: OpArg))
3425 return KnownTy;
3426 // try to deduce from the operand itself
3427 Visited.clear();
3428 if (Type *Ty = deduceElementTypeHelper(I: OpArg, Visited, UnknownElemTypeI8: false))
3429 return Ty;
3430 // search in actual parameter's users
3431 for (User *OpU : OpArg->users()) {
3432 Instruction *Inst = dyn_cast<Instruction>(Val: OpU);
3433 if (!Inst || Inst == CI)
3434 continue;
3435 Visited.clear();
3436 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8: false))
3437 return Ty;
3438 }
3439 // check if it's a formal parameter of the outer function
3440 if (!CI->getParent() || !CI->getParent()->getParent())
3441 continue;
3442 Function *OuterF = CI->getParent()->getParent();
3443 if (FVisited.find(Ptr: OuterF) != FVisited.end())
3444 continue;
3445 for (unsigned i = 0; i < OuterF->arg_size(); ++i) {
3446 if (OuterF->getArg(i) == OpArg) {
3447 Lookup.push_back(Elt: std::make_pair(x&: OuterF, y&: i));
3448 break;
3449 }
3450 }
3451 }
3452
3453 // search in function parameters
3454 for (auto &Pair : Lookup) {
3455 if (Type *Ty = deduceFunParamElementType(F: Pair.first, OpIdx: Pair.second, FVisited))
3456 return Ty;
3457 }
3458
3459 return nullptr;
3460}
3461
3462void SPIRVEmitIntrinsicsImpl::processParamTypesByFunHeader(Function *F,
3463 IRBuilder<> &B) {
3464 B.SetInsertPointPastAllocas(F);
3465 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
3466 Argument *Arg = F->getArg(i: OpIdx);
3467 // Vector-of-pointers arg: deduce pointee from a GEP user so the function
3468 // type isn't emitted with the default i8 pointee.
3469 if (isUntypedPointerVectorTy(T: Arg->getType()) &&
3470 !GR->findDeducedElementType(Val: Arg)) {
3471 for (User *U : Arg->users()) {
3472 auto *GEP = dyn_cast<GetElementPtrInst>(Val: U);
3473 if (GEP && GEP->getPointerOperand() == Arg) {
3474 GR->buildAssignPtr(B, ElemTy: GEP->getSourceElementType(), Arg);
3475 break;
3476 }
3477 }
3478 continue;
3479 }
3480 if (!isUntypedPointerTy(T: Arg->getType()))
3481 continue;
3482 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
3483 if (ElemTy)
3484 continue;
3485 if (hasPointeeTypeAttr(Arg) &&
3486 (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) {
3487 GR->buildAssignPtr(B, ElemTy, Arg);
3488 continue;
3489 }
3490 // search in function's call sites
3491 for (User *U : F->users()) {
3492 CallInst *CI = dyn_cast<CallInst>(Val: U);
3493 if (!CI || OpIdx >= CI->arg_size())
3494 continue;
3495 Value *OpArg = CI->getArgOperand(i: OpIdx);
3496 if (!isPointerTy(T: OpArg->getType()))
3497 continue;
3498 // maybe we already know operand's element type
3499 if ((ElemTy = GR->findDeducedElementType(Val: OpArg)) != nullptr)
3500 break;
3501 }
3502 if (ElemTy) {
3503 GR->buildAssignPtr(B, ElemTy, Arg);
3504 continue;
3505 }
3506 if (HaveFunPtrs) {
3507 for (User *U : Arg->users()) {
3508 CallInst *CI = dyn_cast<CallInst>(Val: U);
3509 if (CI && !isa<IntrinsicInst>(Val: CI) && CI->isIndirectCall() &&
3510 CI->getCalledOperand() == Arg &&
3511 CI->getParent()->getParent() == CurrF) {
3512 SmallVector<std::pair<Value *, unsigned>> Ops;
3513 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy&: ElemTy, IsPostprocessing: false);
3514 if (ElemTy) {
3515 GR->buildAssignPtr(B, ElemTy, Arg);
3516 break;
3517 }
3518 }
3519 }
3520 }
3521 }
3522}
3523
3524void SPIRVEmitIntrinsicsImpl::processParamTypes(Function *F, IRBuilder<> &B) {
3525 B.SetInsertPointPastAllocas(F);
3526 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
3527 Argument *Arg = F->getArg(i: OpIdx);
3528 if (!isUntypedPointerTy(T: Arg->getType()))
3529 continue;
3530 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
3531 if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) {
3532 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Arg)) {
3533 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3534 GR->updateAssignType(
3535 AssignCI, Arg,
3536 OfType: getNormalizedPoisonValue(Ty: ElemTy, CanUseAnyVectorRank));
3537 propagateElemType(Op: Arg, ElemTy: IntegerType::getInt8Ty(C&: F->getContext()),
3538 VisitedSubst);
3539 } else {
3540 GR->buildAssignPtr(B, ElemTy, Arg);
3541 }
3542 }
3543 }
3544}
3545
3546static FunctionType *getFunctionPointerElemType(Function *F,
3547 SPIRVGlobalRegistry *GR) {
3548 FunctionType *FTy = F->getFunctionType();
3549 bool IsNewFTy = false;
3550 SmallVector<Type *, 4> ArgTys;
3551 for (Argument &Arg : F->args()) {
3552 Type *ArgTy = Arg.getType();
3553 if (ArgTy->isPointerTy())
3554 if (Type *ElemTy = GR->findDeducedElementType(Val: &Arg)) {
3555 IsNewFTy = true;
3556 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
3557 }
3558 ArgTys.push_back(Elt: ArgTy);
3559 }
3560 return IsNewFTy
3561 ? FunctionType::get(Result: FTy->getReturnType(), Params: ArgTys, isVarArg: FTy->isVarArg())
3562 : FTy;
3563}
3564
3565bool SPIRVEmitIntrinsicsImpl::processFunctionPointers(Module &M) {
3566 SmallVector<Function *> Worklist;
3567 for (auto &F : M) {
3568 if (F.isIntrinsic())
3569 continue;
3570 if (F.isDeclaration()) {
3571 for (User *U : F.users()) {
3572 CallInst *CI = dyn_cast<CallInst>(Val: U);
3573 if (!CI || CI->getCalledFunction() != &F) {
3574 Worklist.push_back(Elt: &F);
3575 break;
3576 }
3577 }
3578 } else {
3579 if (F.user_empty())
3580 continue;
3581 Type *FPElemTy = GR->findDeducedElementType(Val: &F);
3582 if (!FPElemTy)
3583 FPElemTy = getFunctionPointerElemType(F: &F, GR);
3584 for (User *U : F.users()) {
3585 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
3586 if (!II || II->arg_size() != 3 || II->getOperand(i_nocapture: 0) != &F)
3587 continue;
3588 if (II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type ||
3589 II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
3590 GR->updateAssignType(
3591 AssignCI: II, Arg: &F, OfType: getNormalizedPoisonValue(Ty: FPElemTy, CanUseAnyVectorRank));
3592 break;
3593 }
3594 }
3595 }
3596 }
3597 if (Worklist.empty())
3598 return false;
3599
3600 LLVMContext &Ctx = M.getContext();
3601 Function *SF = getOrCreateBackendServiceFunction(M);
3602 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: SF);
3603 IRBuilder<> IRB(BB);
3604
3605 for (Function *F : Worklist) {
3606 SmallVector<Value *> Args;
3607 for (const auto &Arg : F->args())
3608 Args.push_back(
3609 Elt: getNormalizedPoisonValue(Ty: Arg.getType(), CanUseAnyVectorRank));
3610 IRB.CreateCall(Callee: F, Args);
3611 }
3612 IRB.CreateRetVoid();
3613
3614 return true;
3615}
3616
3617// Apply types parsed from demangled function declarations.
3618void SPIRVEmitIntrinsicsImpl::applyDemangledPtrArgTypes(IRBuilder<> &B) {
3619 DenseMap<Function *, CallInst *> Ptrcasts;
3620 for (auto It : FDeclPtrTys) {
3621 Function *F = It.first;
3622 for (auto *U : F->users()) {
3623 CallInst *CI = dyn_cast<CallInst>(Val: U);
3624 if (!CI || CI->getCalledFunction() != F)
3625 continue;
3626 unsigned Sz = CI->arg_size();
3627 for (auto [Idx, ElemTy] : It.second) {
3628 if (Idx >= Sz)
3629 continue;
3630 Value *Param = CI->getArgOperand(i: Idx);
3631 if (GR->findDeducedElementType(Val: Param) || isa<GlobalValue>(Val: Param))
3632 continue;
3633 if (Argument *Arg = dyn_cast<Argument>(Val: Param)) {
3634 if (!hasPointeeTypeAttr(Arg)) {
3635 B.SetInsertPointPastAllocas(Arg->getParent());
3636 B.SetCurrentDebugLocation(DebugLoc());
3637 GR->buildAssignPtr(B, ElemTy, Arg);
3638 }
3639 } else if (isaGEP(V: Param)) {
3640 replaceUsesOfWithSpvPtrcast(
3641 Op: Param, ElemTy: normalizeType(Ty: ElemTy, CanUseAnyVectorRank), I: CI, Ptrcasts);
3642 } else if (isa<Instruction>(Val: Param)) {
3643 GR->addDeducedElementType(Val: Param,
3644 Ty: normalizeType(Ty: ElemTy, CanUseAnyVectorRank));
3645 // insertAssignTypeIntrs() will complete buildAssignPtr()
3646 } else {
3647 B.SetInsertPoint(CI->getParent()
3648 ->getParent()
3649 ->getEntryBlock()
3650 .getFirstNonPHIOrDbgOrAlloca());
3651 GR->buildAssignPtr(B, ElemTy, Arg: Param);
3652 }
3653 CallInst *Ref = dyn_cast<CallInst>(Val: Param);
3654 if (!Ref)
3655 continue;
3656 Function *RefF = Ref->getCalledFunction();
3657 if (!RefF || !isPointerTy(T: RefF->getReturnType()) ||
3658 GR->findDeducedElementType(Val: RefF))
3659 continue;
3660 ElemTy = normalizeType(Ty: ElemTy, CanUseAnyVectorRank);
3661 GR->addDeducedElementType(Val: RefF, Ty: ElemTy);
3662 GR->addReturnType(
3663 ArgF: RefF, DerivedTy: TypedPointerType::get(
3664 ElementType: ElemTy, AddressSpace: getPointerAddressSpace(T: RefF->getReturnType())));
3665 }
3666 }
3667 }
3668 FDeclPtrTys.clear();
3669}
3670
3671GetElementPtrInst *SPIRVEmitIntrinsicsImpl::simplifyZeroLengthArrayGepInst(
3672 GetElementPtrInst *GEP) {
3673 // getelementptr [0 x T], P, 0 (zero), I -> getelementptr T, P, I.
3674 // If type is 0-length array and first index is 0 (zero), drop both the
3675 // 0-length array type and the first index. This is a common pattern in
3676 // the IR, e.g. when using a zero-length array as a placeholder for a
3677 // flexible array such as unbound arrays.
3678 assert(GEP && "GEP is null");
3679 Type *SrcTy = GEP->getSourceElementType();
3680 SmallVector<Value *, 8> Indices(GEP->indices());
3681 ArrayType *ArrTy = dyn_cast<ArrayType>(Val: SrcTy);
3682 if (ArrTy && ArrTy->getNumElements() == 0 && match(V: Indices[0], P: m_Zero())) {
3683 Indices.erase(CI: Indices.begin());
3684 SrcTy = ArrTy->getElementType();
3685 return GetElementPtrInst::Create(PointeeType: SrcTy, Ptr: GEP->getPointerOperand(), IdxList: Indices,
3686 NW: GEP->getNoWrapFlags(), NameStr: "",
3687 InsertBefore: GEP->getIterator());
3688 }
3689 return nullptr;
3690}
3691
3692void SPIRVEmitIntrinsicsImpl::emitUnstructuredLoopControls(Function &F,
3693 IRBuilder<> &B) {
3694 const SPIRVSubtarget *ST = TM.getSubtargetImpl(F);
3695 // Shaders use SPIRVStructurizer which emits OpLoopMerge via spv_loop_merge.
3696 if (ST->isShader())
3697 return;
3698
3699 if (ST->canUseExtension(
3700 E: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls)) {
3701 for (BasicBlock &BB : F) {
3702 Instruction *Term = BB.getTerminator();
3703 MDNode *LoopMD = Term->getMetadata(KindID: LLVMContext::MD_loop);
3704 if (!LoopMD)
3705 continue;
3706
3707 SmallVector<unsigned, 1> Ops =
3708 getSpirvLoopControlOperandsFromLoopMetadata(LoopMD);
3709 unsigned LC = Ops[0];
3710 if (LC == SPIRV::LoopControl::None)
3711 continue;
3712
3713 // Emit intrinsic: loop control mask + optional parameters.
3714 B.SetInsertPoint(Term);
3715 SmallVector<Value *, 4> IntrArgs;
3716 for (unsigned Op : Ops)
3717 IntrArgs.push_back(Elt: B.getInt32(C: Op));
3718 B.CreateIntrinsic(ID: Intrinsic::spv_loop_control_intel, Args: IntrArgs);
3719 }
3720 return;
3721 }
3722
3723 // For non-shader targets without the Intel extension, emit OpLoopMerge
3724 // using spv_loop_merge intrinsics, mirroring the structurizer approach.
3725 LoopInfo LI;
3726 LI.analyze(F: &F);
3727 if (LI.empty())
3728 return;
3729
3730 SmallPtrSet<BasicBlock *, 8> UsedMergeBlocks;
3731 for (Loop *L : LI.getLoopsInPreorder()) {
3732 BasicBlock *Latch = L->getLoopLatch();
3733 if (!Latch)
3734 continue;
3735
3736 // Check for loop unroll metadata on the latch terminator.
3737 SmallVector<unsigned, 1> LoopControlOps =
3738 getSpirvLoopControlOperandsFromLoopMetadata(L);
3739 if (LoopControlOps[0] == SPIRV::LoopControl::None)
3740 continue;
3741
3742 BasicBlock *Header = L->getHeader();
3743 // OpLoopMerge must immediately precede an OpBranch or OpBranchConditional.
3744 // Switches are already lowered to spv_switch + indirectbr at this point.
3745 if (!isa<UncondBrInst, CondBrInst>(Val: Header->getTerminator()))
3746 continue;
3747
3748 BasicBlock *MergeBlock = L->getUniqueExitBlock();
3749 // LoopSimplify does not guarantee a unique exit block. Try the normal exit
3750 // of a rotated loop (from the latch), then an unrotated loop (from the
3751 // header). A merge block cannot be shared by multiple loop headers.
3752 for (BasicBlock *BB : {Latch, Header}) {
3753 if (MergeBlock || !isa<CondBrInst>(Val: BB->getTerminator()))
3754 continue;
3755 for (BasicBlock *Succ : successors(BB))
3756 if (!L->contains(BB: Succ) && !UsedMergeBlocks.contains(Ptr: Succ))
3757 MergeBlock = Succ;
3758 }
3759 if (!MergeBlock || !UsedMergeBlocks.insert(Ptr: MergeBlock).second)
3760 continue;
3761
3762 B.SetInsertPoint(Header->getTerminator());
3763 auto *MergeAddress = BlockAddress::get(F: &F, BB: MergeBlock);
3764 auto *ContinueAddress = BlockAddress::get(F: &F, BB: Latch);
3765 SmallVector<Value *, 4> Args = {MergeAddress, ContinueAddress};
3766 for (unsigned Imm : LoopControlOps)
3767 Args.emplace_back(Args: B.getInt32(C: Imm));
3768 B.CreateIntrinsic(ID: Intrinsic::spv_loop_merge, Args: {Args});
3769 }
3770}
3771
3772bool SPIRVEmitIntrinsicsImpl::runOnFunction(Function &Func) {
3773 if (Func.isDeclaration())
3774 return false;
3775
3776 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: Func);
3777 // LoopSimplify runs after SPIRVPrepareFunctions sorted the blocks, and the
3778 // preheaders/dedicated exits it creates can end up before their dominator,
3779 // which SPIR-V forbids.
3780 if (!ST.isShader())
3781 sortBlocks(F&: Func);
3782 GR = ST.getSPIRVGlobalRegistry();
3783
3784 if (!CurrF)
3785 HaveFunPtrs =
3786 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers);
3787
3788 CanUseAnyVectorRank =
3789 ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector);
3790 CurrF = &Func;
3791 IRBuilder<> B(*Func.getParent());
3792 AggrConsts.clear();
3793 AggrConstTypes.clear();
3794 AggrStores.clear();
3795
3796 processParamTypesByFunHeader(F: CurrF, B);
3797
3798 // Fix GEP result types ahead of inference, and simplify if possible.
3799 // Data structure for dead instructions that were simplified and replaced.
3800 SmallPtrSet<Instruction *, 4> DeadInsts;
3801 for (auto &I : instructions(F&: Func)) {
3802 if (StoreInst *SI = dyn_cast<StoreInst>(Val: &I)) {
3803 Type *ElTy = SI->getValueOperand()->getType();
3804 if (ElTy->isAggregateType() || ElTy->isVectorTy())
3805 AggrStores.insert(Ptr: &I);
3806 continue;
3807 }
3808
3809 auto *GEP = dyn_cast<GetElementPtrInst>(Val: &I);
3810 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
3811
3812 if ((!GEP && !SGEP) || GR->findDeducedElementType(Val: &I))
3813 continue;
3814
3815 if (SGEP) {
3816 GR->addDeducedElementType(
3817 Val: SGEP,
3818 Ty: normalizeType(Ty: SGEP->getResultElementType(), CanUseAnyVectorRank));
3819 continue;
3820 }
3821
3822 GetElementPtrInst *NewGEP = simplifyZeroLengthArrayGepInst(GEP);
3823 if (NewGEP) {
3824 GEP->replaceAllUsesWith(V: NewGEP);
3825 DeadInsts.insert(Ptr: GEP);
3826 GEP = NewGEP;
3827 }
3828 if (Type *GepTy = getGEPType(Ref: GEP))
3829 GR->addDeducedElementType(Val: GEP, Ty: normalizeType(Ty: GepTy, CanUseAnyVectorRank));
3830 }
3831 // Remove dead instructions that were simplified and replaced.
3832 for (auto *I : DeadInsts) {
3833 assert(I->use_empty() && "Dead instruction should not have any uses left");
3834 I->eraseFromParent();
3835 }
3836
3837 B.SetInsertPoint(Func.getEntryBlock().begin());
3838 for (auto &GV : Func.getParent()->globals())
3839 processGlobalValue(GV, B);
3840
3841 reconstructAggregateReturns(Func, B);
3842 preprocessUndefsAndPoisons(B);
3843 simplifyNullAddrSpaceCasts();
3844 preprocessCompositeConstants(B);
3845
3846 // A PHINode, SelectInst or FreezeInst takes its result type from its
3847 // operands. Aggregate arms are lowered to i32 value-ids (composite constants
3848 // here, loads and other producers during the visitor pass below), so mutate
3849 // an aggregate PHI, select or freeze to match. The original type is tracked
3850 // in AggrConstTypes (used to assign the SPIR-V type) and its extractvalue
3851 // users are lowered to spv_extractv.
3852 Type *I32Ty = B.getInt32Ty();
3853 for (Instruction &I : instructions(F&: Func)) {
3854 if (!isAggregateValueIdInstr(I))
3855 continue;
3856 // Give multi-register arms a value-id first, before the result is mutated.
3857 insertCompositeAggregateArms(I: &I, B);
3858 AggrConstTypes[&I] = I.getType();
3859 I.mutateType(Ty: I32Ty);
3860 }
3861
3862 preprocessBoolVectorBitcasts(F&: Func);
3863 SmallVector<Instruction *> Worklist(
3864 llvm::make_pointer_range(Range: instructions(F&: Func)));
3865
3866 applyDemangledPtrArgTypes(B);
3867
3868 // Pass forward: use operand to deduce instructions result.
3869 for (auto &I : Worklist) {
3870 // Don't emit intrinsincs for convergence intrinsics.
3871 if (isConvergenceIntrinsic(I))
3872 continue;
3873
3874 bool Postpone = insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: false);
3875 // if Postpone is true, we can't decide on pointee type yet
3876 insertAssignTypeIntrs(I, B);
3877 insertPtrCastOrAssignTypeInstr(I, B);
3878 insertSpirvDecorations(I, B);
3879 // if instruction requires a pointee type set, let's check if we know it
3880 // already, and force it to be i8 if not
3881 if (Postpone && !GR->findAssignPtrTypeInstr(Val: I))
3882 insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: true);
3883
3884 if (auto *FPI = dyn_cast<ConstrainedFPIntrinsic>(Val: I))
3885 useRoundingMode(FPI, B);
3886 }
3887
3888 // Pass backward: use instructions results to specify/update/cast operands
3889 // where needed.
3890 SmallPtrSet<Instruction *, 4> IncompleteRets;
3891 for (auto &I : llvm::reverse(C: instructions(F&: Func)))
3892 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets);
3893
3894 // Pass forward for PHIs only, their operands are not preceed the
3895 // instruction in meaning of `instructions(Func)`.
3896 for (BasicBlock &BB : Func)
3897 for (PHINode &Phi : BB.phis())
3898 if (isPointerTy(T: Phi.getType()))
3899 deduceOperandElementType(I: &Phi, IncompleteRets: nullptr);
3900
3901 for (auto *I : Worklist) {
3902 TrackConstants = true;
3903 if (!I->getType()->isVoidTy() || isa<StoreInst>(Val: I))
3904 setInsertPointAfterDef(B, I);
3905 // Visitors return either the original/newly created instruction for
3906 // further processing, nullptr otherwise.
3907 I = visit(I&: *I);
3908 if (!I)
3909 continue;
3910
3911 // Don't emit intrinsics for convergence operations.
3912 if (isConvergenceIntrinsic(I))
3913 continue;
3914
3915 addSaturatedDecorationToIntrinsic(I, B);
3916 processInstrAfterVisit(I, B);
3917 }
3918
3919 emitUnstructuredLoopControls(F&: Func, B);
3920
3921 return true;
3922}
3923
3924// Try to deduce a better type for pointers to untyped ptr.
3925bool SPIRVEmitIntrinsicsImpl::postprocessTypes(Module &M) {
3926 if (!GR || TodoTypeSz == 0)
3927 return false;
3928
3929 unsigned SzTodo = TodoTypeSz;
3930 DenseMap<Value *, SmallPtrSet<Value *, 4>> ToProcess;
3931 for (auto [Op, Enabled] : TodoType) {
3932 // TODO: add isa<CallInst>(Op) to continue
3933 if (!Enabled || isaGEP(V: Op))
3934 continue;
3935 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Op);
3936 Type *KnownTy = GR->findDeducedElementType(Val: Op);
3937 if (!KnownTy || !AssignCI)
3938 continue;
3939 assert(Op == AssignCI->getArgOperand(0));
3940 // Try to improve the type deduced after all Functions are processed.
3941 if (auto *CI = dyn_cast<Instruction>(Val: Op)) {
3942 CurrF = CI->getParent()->getParent();
3943 SmallPtrSet<Value *, 0> Visited;
3944 if (Type *ElemTy = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8: false, IgnoreKnownType: true)) {
3945 if (ElemTy != KnownTy) {
3946 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3947 propagateElemType(Op: CI, ElemTy, VisitedSubst);
3948 eraseTodoType(Op);
3949 continue;
3950 }
3951 }
3952 }
3953
3954 if (Op->hasUseList()) {
3955 for (User *U : Op->users()) {
3956 Instruction *Inst = dyn_cast<Instruction>(Val: U);
3957 if (Inst && !isa<IntrinsicInst>(Val: Inst))
3958 ToProcess[Inst].insert(Ptr: Op);
3959 }
3960 }
3961 }
3962 if (TodoTypeSz == 0)
3963 return true;
3964
3965 for (auto &F : M) {
3966 CurrF = &F;
3967 SmallPtrSet<Instruction *, 4> IncompleteRets;
3968 for (auto &I : llvm::reverse(C: instructions(F))) {
3969 auto It = ToProcess.find(Val: &I);
3970 if (It == ToProcess.end())
3971 continue;
3972 It->second.remove_if(P: [this](Value *V) { return !isTodoType(Op: V); });
3973 if (It->second.size() == 0)
3974 continue;
3975 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets, AskOps: &It->second, IsPostprocessing: true);
3976 if (TodoTypeSz == 0)
3977 return true;
3978 }
3979 }
3980
3981 return SzTodo > TodoTypeSz;
3982}
3983
3984// Parse unambiguous scalar/vector pointees, e.g. "float vector[4] const*".
3985static Type *parseDeclPtrElemType(StringRef TypeStr, LLVMContext &Ctx) {
3986 if (!TypeStr.consume_back(Suffix: "*") && !TypeStr.consume_back(Suffix: "&"))
3987 return nullptr;
3988 StringRef ScalarName = TypeStr;
3989 Type *ElemTy = parseBasicTypeName(TypeName&: TypeStr, Ctx);
3990 if (!ElemTy || ElemTy->isVoidTy() || ElemTy->isIntegerTy(BitWidth: 1))
3991 return nullptr;
3992 ScalarName = ScalarName.drop_back(N: TypeStr.size());
3993 // OpenCL aliases such as uint can also name C++ classes.
3994 if (!StringSwitch<bool>(ScalarName)
3995 .Cases(CaseStrings: {"char", "signed char", "unsigned char", "short",
3996 "unsigned short", "int", "unsigned int", "_Float16", "float",
3997 "double"},
3998 Value: true)
3999 .Default(Value: false) ||
4000 (!TypeStr.empty() && !TypeStr.starts_with(Prefix: " ")))
4001 return nullptr;
4002 if (TypeStr.consume_front(Prefix: " vector[")) {
4003 auto [Count, Rest] = TypeStr.split(Separator: ']');
4004 unsigned NumElts = 0;
4005 if (Count.getAsInteger(Radix: 10, Result&: NumElts) || NumElts == 0)
4006 return nullptr;
4007 ElemTy = FixedVectorType::get(ElementType: ElemTy, NumElts);
4008 TypeStr = Rest;
4009 }
4010 // Only cv and address space qualifiers may follow.
4011 SmallVector<StringRef, 4> Quals;
4012 TypeStr.split(A&: Quals, Separator: ' ', MaxSplit: -1, /*KeepEmpty=*/false);
4013 for (StringRef Qual : Quals) {
4014 unsigned AS;
4015 if (Qual != "const" && Qual != "volatile" && Qual != "restrict" &&
4016 !(Qual.consume_front(Prefix: "AS") && !Qual.getAsInteger(Radix: 10, Result&: AS)))
4017 return nullptr;
4018 }
4019 return ElemTy;
4020}
4021
4022// Parse and store argument types of function declarations where needed.
4023void SPIRVEmitIntrinsicsImpl::parseFunDeclarations(Module &M) {
4024 for (auto &F : M) {
4025 if (!F.isDeclaration() || F.isIntrinsic())
4026 continue;
4027 std::string DemangledName = getOclOrSpirvBuiltinDemangledName(Name: F.getName());
4028 if (DemangledName.empty())
4029 continue;
4030 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F);
4031 auto Set = ST.getPreferredInstructionSet();
4032 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(DemangledCall: DemangledName, Set);
4033 // Some builtins have no opcode mapping.
4034 const bool IsBuiltin = Grp >= 0 || SPIRV::isBuiltin(DemangledCall: DemangledName, Set);
4035 if (IsBuiltin && Opcode != SPIRV::OpGroupAsyncCopy)
4036 continue;
4037 SmallVector<unsigned> Idxs;
4038 for (Argument &Arg : F.args())
4039 if (isPointerTy(T: Arg.getType()) && !hasPointeeTypeAttr(Arg: &Arg))
4040 Idxs.push_back(Elt: Arg.getArgNo());
4041 if (Idxs.empty())
4042 continue;
4043 LLVMContext &Ctx = F.getContext();
4044 SmallVector<StringRef, 10> TypeStrs;
4045 SPIRV::parseBuiltinTypeStr(BuiltinArgsTypeStrs&: TypeStrs, DemangledCall: DemangledName, Ctx);
4046 if (TypeStrs.empty())
4047 continue;
4048 // Avoid nested signatures and hidden ABI arguments such as this and sret.
4049 if (!IsBuiltin && (!F.getName().starts_with(Prefix: "_Z") || F.hasStructRetAttr() ||
4050 StringRef(DemangledName).count(C: '(') != 1 ||
4051 StringRef(DemangledName).count(C: ')') != 1 ||
4052 StringRef(DemangledName).contains(C: '<') ||
4053 TypeStrs.size() != F.arg_size()))
4054 continue;
4055 bool AllowLongVectors =
4056 ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector);
4057 for (unsigned Idx : Idxs) {
4058 if (Idx >= TypeStrs.size())
4059 continue;
4060 StringRef TypeStr = TypeStrs[Idx].trim();
4061 Type *ElemTy = IsBuiltin
4062 ? SPIRV::parseBuiltinCallArgumentType(TypeStr, Ctx)
4063 : parseDeclPtrElemType(TypeStr, Ctx);
4064 if (!ElemTy || !TypedPointerType::isValidElementType(ElemTy) ||
4065 ElemTy->isTargetExtTy())
4066 continue;
4067 if (!IsBuiltin) {
4068 ElemTy = normalizeType(Ty: ElemTy, CanUseAnyVectorRank: AllowLongVectors);
4069 if (!AllowLongVectors && isLongVectorEXT(Ty: ElemTy))
4070 continue;
4071 }
4072 FDeclPtrTys[&F].emplace_back(Args&: Idx, Args&: ElemTy);
4073 if (!IsBuiltin)
4074 ST.getSPIRVGlobalRegistry()->addDeducedElementType(Val: F.getArg(i: Idx),
4075 Ty: ElemTy);
4076 }
4077 }
4078}
4079
4080bool SPIRVEmitIntrinsicsImpl::processMaskedMemIntrinsic(IntrinsicInst &I) {
4081 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
4082
4083 if (I.getIntrinsicID() == Intrinsic::masked_gather) {
4084 if (!ST.canUseExtension(
4085 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
4086 I.getContext().emitError(
4087 I: &I, ErrorStr: "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter "
4088 "extension");
4089 // Replace with poison to allow compilation to continue and report error.
4090 I.replaceAllUsesWith(V: PoisonValue::get(T: I.getType()));
4091 I.eraseFromParent();
4092 return true;
4093 }
4094
4095 IRBuilder<> B(&I);
4096
4097 Value *Ptrs = I.getArgOperand(i: 0);
4098 Value *Mask = I.getArgOperand(i: 1);
4099 Value *Passthru = I.getArgOperand(i: 2);
4100
4101 // Alignment is stored as a parameter attribute, not as a regular parameter.
4102 uint32_t Alignment = I.getParamAlign(ArgNo: 0).valueOrOne().value();
4103
4104 SmallVector<Value *, 4> Args = {Ptrs, B.getInt32(C: Alignment), Mask,
4105 Passthru};
4106 SmallVector<Type *, 4> Types = {I.getType(), Ptrs->getType(),
4107 Mask->getType(), Passthru->getType()};
4108
4109 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_masked_gather, OverloadTypes: Types, Args);
4110 I.replaceAllUsesWith(V: NewI);
4111 I.eraseFromParent();
4112 return true;
4113 }
4114
4115 if (I.getIntrinsicID() == Intrinsic::masked_scatter) {
4116 if (!ST.canUseExtension(
4117 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
4118 I.getContext().emitError(
4119 I: &I, ErrorStr: "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter "
4120 "extension");
4121 // Erase the intrinsic to allow compilation to continue and report error.
4122 I.eraseFromParent();
4123 return true;
4124 }
4125
4126 IRBuilder<> B(&I);
4127
4128 Value *Values = I.getArgOperand(i: 0);
4129 Value *Ptrs = I.getArgOperand(i: 1);
4130 Value *Mask = I.getArgOperand(i: 2);
4131
4132 // Alignment is stored as a parameter attribute on the ptrs parameter (arg
4133 // 1).
4134 uint32_t Alignment = I.getParamAlign(ArgNo: 1).valueOrOne().value();
4135
4136 SmallVector<Value *, 4> Args = {Values, Ptrs, B.getInt32(C: Alignment), Mask};
4137 SmallVector<Type *, 3> Types = {Values->getType(), Ptrs->getType(),
4138 Mask->getType()};
4139
4140 B.CreateIntrinsic(ID: Intrinsic::spv_masked_scatter, OverloadTypes: Types, Args);
4141 I.eraseFromParent();
4142 return true;
4143 }
4144
4145 return false;
4146}
4147
4148// SPIR-V doesn't support bitcasts involving vector boolean type. Decompose such
4149// bitcasts into element-wise operations before building instructions
4150// worklist, so new instructions are properly visited and converted to
4151// SPIR-V intrinsics.
4152void SPIRVEmitIntrinsicsImpl::preprocessBoolVectorBitcasts(Function &F) {
4153 struct BoolVecBitcast {
4154 BitCastInst *BC;
4155 FixedVectorType *BoolVecTy;
4156 bool SrcIsBoolVec;
4157 };
4158
4159 auto getAsBoolVec = [](Type *Ty) -> FixedVectorType * {
4160 auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
4161 return (VTy && VTy->getElementType()->isIntegerTy(BitWidth: 1)) ? VTy : nullptr;
4162 };
4163
4164 SmallVector<BoolVecBitcast, 4> ToReplace;
4165 for (auto &I : instructions(F)) {
4166 auto *BC = dyn_cast<BitCastInst>(Val: &I);
4167 if (!BC)
4168 continue;
4169 if (auto *BVTy = getAsBoolVec(BC->getSrcTy()))
4170 ToReplace.push_back(Elt: {.BC: BC, .BoolVecTy: BVTy, .SrcIsBoolVec: true});
4171 else if (auto *BVTy = getAsBoolVec(BC->getDestTy()))
4172 ToReplace.push_back(Elt: {.BC: BC, .BoolVecTy: BVTy, .SrcIsBoolVec: false});
4173 }
4174
4175 for (auto &[BC, BoolVecTy, SrcIsBoolVec] : ToReplace) {
4176 IRBuilder<> B(BC);
4177 Value *Src = BC->getOperand(i_nocapture: 0);
4178 unsigned BoolVecN = BoolVecTy->getNumElements();
4179 // Use iN as the scalar intermediate type for the bool vector side.
4180 Type *IntTy = B.getIntNTy(N: BoolVecN);
4181
4182 // Convert source to scalar integer.
4183 Value *IntVal;
4184 if (SrcIsBoolVec) {
4185 // Extract each bool, zext, shift, and OR.
4186 IntVal = ConstantInt::get(Ty: IntTy, V: 0);
4187 for (unsigned I = 0; I < BoolVecN; ++I) {
4188 Value *Elem = B.CreateExtractElement(Vec: Src, Idx: B.getInt32(C: I));
4189 Value *Ext = B.CreateZExt(V: Elem, DestTy: IntTy);
4190 if (I > 0)
4191 Ext = B.CreateShl(LHS: Ext, RHS: ConstantInt::get(Ty: IntTy, V: I));
4192 IntVal = B.CreateOr(LHS: IntVal, RHS: Ext);
4193 }
4194 } else {
4195 // Source is a non-bool type. If it's already a scalar integer, use it
4196 // directly, otherwise bitcast to iN first.
4197 IntVal = Src;
4198 if (!Src->getType()->isIntegerTy())
4199 IntVal = B.CreateBitCast(V: Src, DestTy: IntTy);
4200 }
4201
4202 // Convert scalar integer to destination type.
4203 Value *Result;
4204 if (!SrcIsBoolVec) {
4205 // Test each bit with AND + icmp.
4206 Result = PoisonValue::get(T: BoolVecTy);
4207 for (unsigned I = 0; I < BoolVecN; ++I) {
4208 Value *Mask = ConstantInt::get(Ty: IntTy, V: APInt::getOneBitSet(numBits: BoolVecN, BitNo: I));
4209 Value *And = B.CreateAnd(LHS: IntVal, RHS: Mask);
4210 Value *Cmp = B.CreateICmpNE(LHS: And, RHS: ConstantInt::get(Ty: IntTy, V: 0));
4211 Result = B.CreateInsertElement(Vec: Result, NewElt: Cmp, Idx: B.getInt32(C: I));
4212 }
4213 } else {
4214 // Destination is a non-bool type. If it's a scalar integer, use IntVal
4215 // directly, otherwise bitcast from iN.
4216 Result = IntVal;
4217 if (!BC->getDestTy()->isIntegerTy())
4218 Result = B.CreateBitCast(V: IntVal, DestTy: BC->getDestTy());
4219 }
4220
4221 BC->replaceAllUsesWith(V: Result);
4222 BC->eraseFromParent();
4223 }
4224}
4225
4226bool SPIRVEmitIntrinsicsImpl::convertMaskedMemIntrinsics(Module &M) {
4227 bool Changed = false;
4228
4229 for (Function &F : make_early_inc_range(Range&: M)) {
4230 if (!F.isIntrinsic())
4231 continue;
4232 Intrinsic::ID IID = F.getIntrinsicID();
4233 if (IID != Intrinsic::masked_gather && IID != Intrinsic::masked_scatter)
4234 continue;
4235
4236 for (User *U : make_early_inc_range(Range: F.users())) {
4237 if (auto *II = dyn_cast<IntrinsicInst>(Val: U))
4238 Changed |= processMaskedMemIntrinsic(I&: *II);
4239 }
4240
4241 if (F.use_empty())
4242 F.eraseFromParent();
4243 }
4244
4245 return Changed;
4246}
4247
4248bool SPIRVEmitIntrinsicsImpl::runOnModule(Module &M) {
4249 bool Changed = false;
4250
4251 Changed |= convertMaskedMemIntrinsics(M);
4252
4253 parseFunDeclarations(M);
4254 insertConstantsForFPFastMathDefault(M);
4255 GVUsers.init(M);
4256
4257 TodoType.clear();
4258 for (auto &F : M)
4259 Changed |= runOnFunction(Func&: F);
4260
4261 // Specify function parameters after all functions were processed.
4262 for (auto &F : M) {
4263 // check if function parameter types are set
4264 CurrF = &F;
4265 if (!F.isDeclaration() && !F.isIntrinsic()) {
4266 IRBuilder<> B(M);
4267 processParamTypes(F: &F, B);
4268 }
4269 }
4270
4271 CanTodoType = false;
4272 Changed |= postprocessTypes(M);
4273
4274 if (HaveFunPtrs)
4275 Changed |= processFunctionPointers(M);
4276
4277 return Changed;
4278}
4279
4280PreservedAnalyses
4281llvm::SPIRVEmitIntrinsicsPass::run(Module &M, ModuleAnalysisManager &AM) {
4282 if (SPIRVEmitIntrinsicsImpl(TM).runOnModule(M))
4283 return PreservedAnalyses::none();
4284 return PreservedAnalyses::all();
4285}
4286
4287ModulePass *llvm::createSPIRVEmitIntrinsicsPass(const SPIRVTargetMachine &TM) {
4288 return new SPIRVEmitIntrinsicsLegacy(TM);
4289}
4290