1//===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass emits SPIRV intrinsics keeping essential high-level information for
10// the translation of LLVM IR to SPIR-V.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRV.h"
15#include "SPIRVAuxDataHandler.h"
16#include "SPIRVBuiltins.h"
17#include "SPIRVSubtarget.h"
18#include "SPIRVTargetMachine.h"
19#include "SPIRVUtils.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/DenseSet.h"
22#include "llvm/ADT/MapVector.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallString.h"
25#include "llvm/ADT/StringSwitch.h"
26#include "llvm/Analysis/LoopInfo.h"
27#include "llvm/Analysis/VectorUtils.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/InstIterator.h"
30#include "llvm/IR/InstVisitor.h"
31#include "llvm/IR/IntrinsicsSPIRV.h"
32#include "llvm/IR/LLVMContext.h"
33#include "llvm/IR/PatternMatch.h"
34#include "llvm/IR/ReplaceConstant.h"
35#include "llvm/IR/TypedPointerType.h"
36#include "llvm/IR/Value.h"
37#include "llvm/Support/Debug.h"
38#include "llvm/Support/Error.h"
39#include "llvm/Support/raw_ostream.h"
40#include "llvm/Transforms/Utils/Local.h"
41
42#include <cassert>
43#include <optional>
44#include <queue>
45
46// This pass performs the following transformation on LLVM IR level required
47// for the following translation to SPIR-V:
48// - replaces direct usages of aggregate constants with target-specific
49// intrinsics;
50// - replaces aggregates-related instructions (extract/insert, ld/st, etc)
51// with a target-specific intrinsics;
52// - emits intrinsics for the global variable initializers since IRTranslator
53// doesn't handle them and it's not very convenient to translate them
54// ourselves;
55// - emits intrinsics to keep track of the string names assigned to the values;
56// - emits intrinsics to keep track of constants (this is necessary to have an
57// LLVM IR constant after the IRTranslation is completed) for their further
58// deduplication;
59// - emits intrinsics to keep track of original LLVM types of the values
60// to be able to emit proper SPIR-V types eventually.
61//
62// TODO: consider removing spv.track.constant in favor of spv.assign.type.
63
64using namespace llvm;
65using namespace llvm::PatternMatch;
66
67#define DEBUG_TYPE "spirv-emit-intrinsics"
68
69namespace llvm::SPIRV {
70#define GET_BuiltinGroup_DECL
71#include "SPIRVGenTables.inc"
72} // namespace llvm::SPIRV
73
74namespace {
75// This class keeps track of which functions reference which global variables.
76class GlobalVariableUsers {
77 template <typename T1, typename T2>
78 using OneToManyMapTy = DenseMap<T1, SmallPtrSet<T2, 4>>;
79
80 OneToManyMapTy<const GlobalVariable *, const Function *> GlobalIsUsedByFun;
81
82 void collectGlobalUsers(
83 const GlobalVariable *GV,
84 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
85 &GlobalIsUsedByGlobal) {
86 SmallVector<const Value *> Stack = {GV->user_begin(), GV->user_end()};
87 while (!Stack.empty()) {
88 const Value *V = Stack.pop_back_val();
89
90 if (const Instruction *I = dyn_cast<Instruction>(Val: V)) {
91 GlobalIsUsedByFun[GV].insert(Ptr: I->getFunction());
92 continue;
93 }
94
95 if (const GlobalVariable *UserGV = dyn_cast<GlobalVariable>(Val: V)) {
96 GlobalIsUsedByGlobal[GV].insert(Ptr: UserGV);
97 continue;
98 }
99
100 if (const Constant *C = dyn_cast<Constant>(Val: V))
101 Stack.append(in_start: C->user_begin(), in_end: C->user_end());
102 }
103 }
104
105 bool propagateGlobalToGlobalUsers(
106 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
107 &GlobalIsUsedByGlobal) {
108 SmallVector<const GlobalVariable *> OldUsersGlobals;
109 bool Changed = false;
110 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
111 OldUsersGlobals.assign(in_start: UserGlobals.begin(), in_end: UserGlobals.end());
112 for (const GlobalVariable *UserGV : OldUsersGlobals) {
113 auto It = GlobalIsUsedByGlobal.find(Val: UserGV);
114 if (It == GlobalIsUsedByGlobal.end())
115 continue;
116 Changed |= set_union(S1&: UserGlobals, S2: It->second);
117 }
118 }
119 return Changed;
120 }
121
122 void propagateGlobalToFunctionReferences(
123 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
124 &GlobalIsUsedByGlobal) {
125 for (auto &[GV, UserGlobals] : GlobalIsUsedByGlobal) {
126 auto &UserFunctions = GlobalIsUsedByFun[GV];
127 for (const GlobalVariable *UserGV : UserGlobals) {
128 auto It = GlobalIsUsedByFun.find(Val: UserGV);
129 if (It == GlobalIsUsedByFun.end())
130 continue;
131 set_union(S1&: UserFunctions, S2: It->second);
132 }
133 }
134 }
135
136public:
137 void init(Module &M) {
138 // Collect which global variables are referenced by which global variables
139 // and which functions reference each global variables.
140 OneToManyMapTy<const GlobalVariable *, const GlobalVariable *>
141 GlobalIsUsedByGlobal;
142 GlobalIsUsedByFun.clear();
143 for (GlobalVariable &GV : M.globals())
144 collectGlobalUsers(GV: &GV, GlobalIsUsedByGlobal);
145
146 // Compute indirect references by iterating until a fixed point is reached.
147 while (propagateGlobalToGlobalUsers(GlobalIsUsedByGlobal))
148 (void)0;
149
150 propagateGlobalToFunctionReferences(GlobalIsUsedByGlobal);
151 }
152
153 using FunctionSetType = decltype(GlobalIsUsedByFun)::mapped_type;
154 const FunctionSetType &
155 getTransitiveUserFunctions(const GlobalVariable &GV) const {
156 auto It = GlobalIsUsedByFun.find(Val: &GV);
157 if (It != GlobalIsUsedByFun.end())
158 return It->second;
159
160 static const FunctionSetType Empty{};
161 return Empty;
162 }
163};
164
165static bool isaGEP(const Value *V) {
166 return isa<StructuredGEPInst>(Val: V) || isa<GetElementPtrInst>(Val: V);
167}
168
169// If Ty is a byte-addressing type, return the multiplier for the offset.
170// Otherwise return std::nullopt.
171static std::optional<uint64_t> getByteAddressingMultiplier(Type *Ty) {
172 if (Ty == IntegerType::getInt8Ty(C&: Ty->getContext())) {
173 return 1;
174 }
175 if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
176 if (AT->getElementType() == IntegerType::getInt8Ty(C&: Ty->getContext())) {
177 return AT->getNumElements();
178 }
179 }
180 return std::nullopt;
181}
182
183class SPIRVEmitIntrinsicsImpl
184 : public InstVisitor<SPIRVEmitIntrinsicsImpl, Instruction *> {
185 const SPIRVTargetMachine &TM;
186 SPIRVGlobalRegistry *GR = nullptr;
187 Function *CurrF = nullptr;
188 bool TrackConstants = true;
189 bool HaveFunPtrs = false;
190 bool CanUseAnyVectorRank = false;
191 DenseMap<Instruction *, Constant *> AggrConsts;
192 DenseMap<Instruction *, Type *> AggrConstTypes;
193 SmallPtrSet<Instruction *, 0> AggrStores;
194 GlobalVariableUsers GVUsers;
195 SmallPtrSet<Value *, 0> Named;
196 Error AccessChainErrors = Error::success();
197 SmallPtrSet<GetElementPtrInst *, 4> InvalidGEPs;
198
199 // map of function declarations to <pointer arg index => element type>
200 MapVector<Function *, SmallVector<std::pair<unsigned, Type *>>> FDeclPtrTys;
201
202 // a register of Instructions that don't have a complete type definition
203 bool CanTodoType = true;
204 unsigned TodoTypeSz = 0;
205 DenseMap<Value *, bool> TodoType;
206 void insertTodoType(Value *Op) {
207 // TODO: add isa<CallInst>(Op) to no-insert
208 if (CanTodoType && !isaGEP(V: Op)) {
209 auto It = TodoType.try_emplace(Key: Op, Args: true);
210 if (It.second)
211 ++TodoTypeSz;
212 }
213 }
214 void eraseTodoType(Value *Op) {
215 auto It = TodoType.find(Val: Op);
216 if (It != TodoType.end() && It->second) {
217 It->second = false;
218 --TodoTypeSz;
219 }
220 }
221 bool isTodoType(Value *Op) {
222 if (isaGEP(V: Op))
223 return false;
224 auto It = TodoType.find(Val: Op);
225 return It != TodoType.end() && It->second;
226 }
227 // a register of Instructions that were visited by deduceOperandElementType()
228 // to validate operand types with an instruction
229 SmallPtrSet<Instruction *, 0> TypeValidated;
230 // Type-validated PHIs and selects whose element type changed since, to be
231 // revalidated by the outermost deduceOperandElementType() call.
232 SmallVector<Instruction *> RetypedToRevalidate;
233 bool RevalidatingRetyped = false;
234
235 // well known result types of builtins
236 enum WellKnownTypes { Event };
237
238 // deduce element type of untyped pointers
239 Type *deduceElementType(Value *I, bool UnknownElemTypeI8);
240 Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8);
241 Type *deduceElementTypeHelper(Value *I, SmallPtrSetImpl<Value *> &Visited,
242 bool UnknownElemTypeI8,
243 bool IgnoreKnownType = false);
244 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
245 bool UnknownElemTypeI8);
246 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand,
247 SmallPtrSetImpl<Value *> &Visited,
248 bool UnknownElemTypeI8,
249 std::optional<unsigned> AddrSpace = {});
250 Type *deduceElementTypeByUsersDeep(Value *Op,
251 SmallPtrSetImpl<Value *> &Visited,
252 bool UnknownElemTypeI8);
253 void maybeAssignPtrType(Type *&Ty, Value *I, Type *RefTy,
254 bool UnknownElemTypeI8);
255
256 // deduce nested types of composites
257 Type *deduceNestedTypeHelper(User *U, bool UnknownElemTypeI8);
258 Type *deduceNestedTypeHelper(User *U, Type *Ty,
259 SmallPtrSetImpl<Value *> &Visited,
260 bool UnknownElemTypeI8);
261
262 // deduce Types of operands of the Instruction if possible
263 void
264 deduceOperandElementType(Instruction *I,
265 SmallPtrSetImpl<Instruction *> *IncompleteRets,
266 const SmallPtrSetImpl<Value *> *AskOps = nullptr,
267 bool IsPostprocessing = false);
268
269 void preprocessCompositeConstants(IRBuilder<> &B);
270 Value *lowerUndefOrPoison(Value *Op, IRBuilder<> &B, bool HasPoisonExt);
271 void preprocessUndefsAndPoisons(IRBuilder<> &B);
272 void insertCompositeAggregateArms(Instruction *I, IRBuilder<> &B);
273 void simplifyNullAddrSpaceCasts();
274
275 Type *reconstructType(Value *Op, bool UnknownElemTypeI8,
276 bool IsPostprocessing);
277
278 void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B);
279 void processInstrAfterVisit(Instruction *I, IRBuilder<> &B);
280 bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B,
281 bool UnknownElemTypeI8);
282 void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B);
283 void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V,
284 IRBuilder<> &B);
285 void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer,
286 Type *ExpectedElementType,
287 unsigned OperandToReplace,
288 IRBuilder<> &B);
289 void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B);
290 bool shouldTryToAddMemAliasingDecoration(Instruction *Inst);
291 void insertSpirvDecorations(Instruction *I, IRBuilder<> &B);
292 void insertConstantsForFPFastMathDefault(Module &M);
293 Value *buildSpvUndefComposite(Type *AggrTy, IRBuilder<> &B);
294 void reconstructAggregateReturns(Function &Func, IRBuilder<> &B);
295 void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B);
296 void processParamTypes(Function *F, IRBuilder<> &B);
297 void processParamTypesByFunHeader(Function *F, IRBuilder<> &B);
298 Type *deduceFunParamElementType(Function *F, unsigned OpIdx);
299 Type *deduceFunParamElementType(Function *F, unsigned OpIdx,
300 SmallPtrSetImpl<Function *> &FVisited);
301
302 bool deduceOperandElementTypeCalledFunction(
303 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
304 Type *&KnownElemTy, bool &Incomplete);
305 void deduceOperandElementTypeFunctionPointer(
306 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
307 Type *&KnownElemTy, bool IsPostprocessing);
308 bool deduceOperandElementTypeFunctionRet(
309 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
310 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing,
311 Type *&KnownElemTy, Value *Op, Function *F);
312
313 CallInst *buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy);
314 void replaceUsesOfWithSpvPtrcast(Value *Op, Type *ElemTy, Instruction *I,
315 DenseMap<Function *, CallInst *> Ptrcasts);
316 void propagateElemType(Value *Op, Type *ElemTy,
317 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
318 void
319 propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
320 DenseSet<std::pair<Value *, Value *>> &VisitedSubst);
321 void propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy,
322 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
323 SmallPtrSetImpl<Value *> &Visited,
324 DenseMap<Function *, CallInst *> Ptrcasts);
325
326 void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true);
327 void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src,
328 Instruction *Dest, bool DeleteOld = true);
329
330 void applyDemangledPtrArgTypes(IRBuilder<> &B);
331
332 GetElementPtrInst *simplifyZeroLengthArrayGepInst(GetElementPtrInst *GEP);
333
334 bool runOnFunction(Function &F);
335 bool postprocessTypes(Module &M);
336 bool processFunctionPointers(Module &M);
337 void parseFunDeclarations(Module &M);
338 void useRoundingMode(ConstrainedFPIntrinsic *FPI, IRBuilder<> &B);
339 bool processMaskedMemIntrinsic(IntrinsicInst &I);
340 bool convertMaskedMemIntrinsics(Module &M);
341 bool preprocessModule(Module &M);
342 void preprocessBoolVectorBitcasts(Function &F);
343 void scalarizeVectorPtrCasts(Function &F);
344
345 void emitUnstructuredLoopControls(Function &F, IRBuilder<> &B);
346
347 // Tries to walk the type accessed by the given GEP instruction.
348 // For each nested type access, one of the 2 callbacks is called:
349 // - OnLiteralIndexing when the index is a known constant value.
350 // Parameters:
351 // PointedType: the pointed type resulting of this indexing.
352 // If the parent type is an array, this is the index in the array.
353 // If the parent type is a struct, this is the field index.
354 // Index: index of the element in the parent type.
355 // - OnDynamnicIndexing when the index is a non-constant value.
356 // This callback is only called when indexing into an array.
357 // Parameters:
358 // ElementType: the type of the elements stored in the parent array.
359 // Offset: the Value* containing the byte offset into the array.
360 // Multiplier: a scaling factor for the offset.
361 // Return true for a fallback, false for a complete walk, or an error.
362 Expected<bool> walkLogicalAccessChain(
363 GetElementPtrInst &GEP,
364 const std::function<void(Type *PointedType, uint64_t Index)>
365 &OnLiteralIndexing,
366 const std::function<void(Type *ElementType, Value *Offset,
367 uint64_t Multiplier)> &OnDynamicIndexing);
368
369 bool walkLogicalAccessChainDynamic(
370 Type *CurType, Value *Operand, uint64_t Multiplier,
371 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
372 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing);
373
374 Expected<bool> walkLogicalAccessChainConstant(
375 Type *CurType, uint64_t Offset,
376 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing);
377
378 void recordAccessChainError(GetElementPtrInst &GEP, Error Err);
379
380 // Returns the type accessed using the given GEP instruction by relying
381 // on the GEP type.
382 // FIXME: GEP types are not supposed to be used to retrieve the pointed
383 // type. This must be fixed.
384 Type *getGEPType(GetElementPtrInst *GEP);
385
386 // Returns the type accessed using the given GEP instruction by walking
387 // the source type using the GEP indices.
388 // FIXME: without help from the frontend, this method cannot reliably retrieve
389 // the stored type, nor can robustly determine the depth of the type
390 // we are accessing.
391 Type *getGEPTypeLogical(GetElementPtrInst *GEP);
392
393 Instruction *buildLogicalAccessChainFromGEP(GetElementPtrInst &GEP);
394
395public:
396 SPIRVEmitIntrinsicsImpl(const SPIRVTargetMachine &TM) : TM(TM) {}
397 Instruction *visitInstruction(Instruction &I) { return &I; }
398 Instruction *visitSwitchInst(SwitchInst &I);
399 Instruction *visitGetElementPtrInst(GetElementPtrInst &I);
400 Instruction *visitIntrinsicInst(IntrinsicInst &I);
401 Instruction *visitBitCastInst(BitCastInst &I);
402 Instruction *visitInsertElementInst(InsertElementInst &I);
403 Instruction *visitExtractElementInst(ExtractElementInst &I);
404 Instruction *visitInsertValueInst(InsertValueInst &I);
405 Instruction *visitExtractValueInst(ExtractValueInst &I);
406 Instruction *visitLoadInst(LoadInst &I);
407 Instruction *visitStoreInst(StoreInst &I);
408 Instruction *visitAllocaInst(AllocaInst &I);
409 Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
410 Instruction *visitAtomicRMWInst(AtomicRMWInst &I);
411 Instruction *visitUnreachableInst(UnreachableInst &I);
412 Instruction *visitCallInst(CallInst &I);
413
414 bool runOnModule(Module &M);
415};
416
417class SPIRVEmitIntrinsicsLegacy : public ModulePass {
418 const SPIRVTargetMachine &TM;
419
420public:
421 static char ID;
422 SPIRVEmitIntrinsicsLegacy(const SPIRVTargetMachine &TM)
423 : ModulePass(ID), TM(TM) {}
424
425 StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
426
427 bool runOnModule(Module &M) override {
428 return SPIRVEmitIntrinsicsImpl(TM).runOnModule(M);
429 }
430};
431
432bool isConvergenceIntrinsic(const Instruction *I) {
433 return match(V: I, P: m_AnyIntrinsic<Intrinsic::experimental_convergence_entry,
434 Intrinsic::experimental_convergence_loop,
435 Intrinsic::experimental_convergence_anchor>());
436}
437
438bool expectIgnoredInIRTranslation(const Instruction *I) {
439 return match(V: I, P: m_AnyIntrinsic<Intrinsic::invariant_start,
440 Intrinsic::spv_resource_handlefrombinding,
441 Intrinsic::spv_resource_getbasepointer,
442 Intrinsic::spv_resource_getpointer>());
443}
444
445// Returns the source pointer from `I` ignoring intermediate ptrcast.
446Value *getPointerRoot(Value *I) {
447 Value *V;
448 if (match(V: I, P: m_Intrinsic<Intrinsic::spv_ptrcast>(Ops: m_Value(V))))
449 return getPointerRoot(I: V);
450 return I;
451}
452
453} // namespace
454
455char SPIRVEmitIntrinsicsLegacy::ID = 0;
456
457INITIALIZE_PASS(SPIRVEmitIntrinsicsLegacy, "spirv-emit-intrinsics",
458 "SPIRV emit intrinsics", false, false)
459
460static inline bool isAssignTypeInstr(const Instruction *I) {
461 return match(V: I, P: m_Intrinsic<Intrinsic::spv_assign_type>());
462}
463
464static bool isMemInstrToReplace(Instruction *I) {
465 return isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I) || isa<InsertValueInst>(Val: I) ||
466 isa<ExtractValueInst>(Val: I) || isa<AtomicCmpXchgInst>(Val: I);
467}
468
469static bool isAggrConstForceInt32(const Value *V) {
470 bool IsAggrZero =
471 isa<ConstantAggregateZero>(Val: V) && !V->getType()->isVectorTy();
472 bool IsUndefAggregate = isa<UndefValue>(Val: V) && V->getType()->isAggregateType();
473 return isa<ConstantArray>(Val: V) || isa<ConstantStruct>(Val: V) ||
474 isa<ConstantDataArray>(Val: V) || IsAggrZero || IsUndefAggregate;
475}
476
477static bool isSpvAggrPlaceholder(const Value *V) {
478 return match(
479 V,
480 P: m_AnyIntrinsic<Intrinsic::spv_undef, Intrinsic::spv_const_composite>());
481}
482
483static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) {
484 if (isa<PHINode>(Val: I))
485 B.SetInsertPoint(I->getParent()->getFirstNonPHIOrAlloca());
486 else
487 B.SetInsertPoint(I);
488}
489
490static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I) {
491 B.SetCurrentDebugLocation(I->getDebugLoc());
492 if (I->getType()->isVoidTy())
493 B.SetInsertPoint(I->getNextNode());
494 else
495 B.SetInsertPoint(*I->getInsertionPointAfterDef());
496}
497
498static bool requireAssignType(Instruction *I) {
499 return !match(
500 V: I,
501 P: m_AnyIntrinsic<Intrinsic::invariant_start, Intrinsic::invariant_end>());
502}
503
504static inline void reportFatalOnTokenType(const Instruction *I) {
505 if (I->getType()->isTokenTy())
506 report_fatal_error(reason: "A token is encountered but SPIR-V without extensions "
507 "does not support token type",
508 gen_crash_diag: false);
509}
510
511static void emitAssignName(const SPIRVOptions &CLOpts, Instruction *I,
512 IRBuilder<> &B) {
513 if (!I->hasName() || I->getType()->isAggregateType() ||
514 expectIgnoredInIRTranslation(I))
515 return;
516
517 // We want to be conservative when adding the names because they can interfere
518 // with later optimizations.
519 bool KeepName = CLOpts.emit_op_names;
520 if (!KeepName) {
521 if (isa<AllocaInst>(Val: I)) {
522 KeepName = true;
523 } else if (auto *CI = dyn_cast<CallBase>(Val: I)) {
524 Function *F = CI->getCalledFunction();
525 if (F && F->getName().starts_with(Prefix: "llvm.spv.alloca"))
526 KeepName = true;
527 }
528 }
529
530 if (!KeepName)
531 return;
532
533 reportFatalOnTokenType(I);
534 setInsertPointAfterDef(B, I);
535 LLVMContext &Ctx = I->getContext();
536 std::vector<Value *> Args = {
537 I, MetadataAsValue::get(
538 Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: I->getName())))};
539 B.CreateIntrinsic(ID: Intrinsic::spv_assign_name, OverloadTypes: {I->getType()}, Args);
540}
541
542void SPIRVEmitIntrinsicsImpl::replaceAllUsesWith(Value *Src, Value *Dest,
543 bool DeleteOld) {
544 GR->replaceAllUsesWith(Old: Src, New: Dest, DeleteOld);
545 // Update uncomplete type records if any
546 if (isTodoType(Op: Src)) {
547 if (DeleteOld)
548 eraseTodoType(Op: Src);
549 insertTodoType(Op: Dest);
550 }
551}
552
553void SPIRVEmitIntrinsicsImpl::replaceAllUsesWithAndErase(IRBuilder<> &B,
554 Instruction *Src,
555 Instruction *Dest,
556 bool DeleteOld) {
557 replaceAllUsesWith(Src, Dest, DeleteOld);
558 std::string Name = Src->hasName() ? Src->getName().str() : "";
559 Src->eraseFromParent();
560 if (!Name.empty()) {
561 Dest->setName(Name);
562 if (Named.insert(Ptr: Dest).second)
563 emitAssignName(CLOpts: TM.getCLOpts(), I: Dest, B);
564 }
565}
566
567static bool IsKernelArgInt8(Function *F, StoreInst *SI) {
568 return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL &&
569 isPointerTy(T: SI->getValueOperand()->getType()) &&
570 isa<Argument>(Val: SI->getValueOperand());
571}
572
573// A pointer-typed local holds a pointer, so its deduced pointee must stay a
574// pointer.
575static bool tracesToPointerAlloca(Value *V) {
576 using namespace PatternMatch;
577 V = V->stripPointerCasts();
578 if (auto *AI = dyn_cast<AllocaInst>(Val: V))
579 return isUntypedPointerTy(T: AI->getAllocatedType());
580 return match(
581 V, P: m_AnyIntrinsic<Intrinsic::spv_alloca, Intrinsic::spv_alloca_array>());
582}
583
584// Maybe restore original function return type.
585static inline Type *restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I,
586 Type *Ty) {
587 CallInst *CI = dyn_cast<CallInst>(Val: I);
588 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
589 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
590 return Ty;
591 if (Type *OriginalTy = GR->findMutated(Val: CI->getCalledFunction()))
592 return OriginalTy;
593 return Ty;
594}
595
596// Return the pointee type of the pointer member of AggrTy selected by Indices.
597// Untyped pointer members are lowered as pointers to i8.
598static Type *getMemberPointeeType(Type *AggrTy, ArrayRef<unsigned> Indices) {
599 Type *MemberTy = ExtractValueInst::getIndexedType(Agg: AggrTy, Idxs: Indices);
600 if (Type *ElemTy = getPointeeType(Ty: MemberTy))
601 return ElemTy;
602 return IntegerType::getInt8Ty(C&: AggrTy->getContext());
603}
604
605// Reconstruct type with nested element types according to deduced type info.
606// Return nullptr if no detailed type info is available.
607Type *SPIRVEmitIntrinsicsImpl::reconstructType(Value *Op,
608 bool UnknownElemTypeI8,
609 bool IsPostprocessing) {
610 Type *Ty = Op->getType();
611 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
612 Ty = restoreMutatedType(GR, I: OpI, Ty);
613 if (auto It = AggrConstTypes.find(Val: OpI); It != AggrConstTypes.end())
614 Ty = It->second;
615 }
616 if (!isUntypedPointerTy(T: Ty))
617 return Ty;
618 // try to find the pointee type
619 if (Type *NestedTy = GR->findDeducedElementType(Val: Op))
620 return getTypedPointerWrapper(ElemTy: NestedTy, AS: getPointerAddressSpace(T: Ty));
621 // not a pointer according to the type info (e.g., Event object)
622 CallInst *CI = GR->findAssignPtrTypeInstr(Val: Op);
623 if (CI) {
624 MetadataAsValue *MD = cast<MetadataAsValue>(Val: CI->getArgOperand(i: 1));
625 return cast<ConstantAsMetadata>(Val: MD->getMetadata())->getType();
626 }
627 if (UnknownElemTypeI8) {
628 if (!IsPostprocessing)
629 insertTodoType(Op);
630 return getTypedPointerWrapper(ElemTy: IntegerType::getInt8Ty(C&: Op->getContext()),
631 AS: getPointerAddressSpace(T: Ty));
632 }
633 return nullptr;
634}
635
636CallInst *SPIRVEmitIntrinsicsImpl::buildSpvPtrcast(Function *F, Value *Op,
637 Type *ElemTy) {
638 IRBuilder<> B(*F->getParent());
639 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
640 // spv_ptrcast's argument Op denotes an instruction that generates
641 // a value, and we may use getInsertionPointAfterDef()
642 setInsertPointAfterDef(B, I: OpI);
643 } else if (auto *OpA = dyn_cast<Argument>(Val: Op)) {
644 B.SetInsertPointPastAllocas(OpA->getParent());
645 B.SetCurrentDebugLocation(DebugLoc());
646 } else {
647 B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrAlloca());
648 }
649 Type *OpTy = Op->getType();
650 SmallVector<Type *, 2> Types = {OpTy, OpTy};
651 SmallVector<Value *, 2> Args = {
652 Op, buildMD(Arg: getNormalizedPoisonValue(Ty: ElemTy, CanUseAnyVectorRank)),
653 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
654 CallInst *PtrCasted =
655 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
656 GR->buildAssignPtr(B, ElemTy, Arg: PtrCasted);
657 return PtrCasted;
658}
659
660void SPIRVEmitIntrinsicsImpl::replaceUsesOfWithSpvPtrcast(
661 Value *Op, Type *ElemTy, Instruction *I,
662 DenseMap<Function *, CallInst *> Ptrcasts) {
663 Function *F = I->getParent()->getParent();
664 CallInst *PtrCastedI = nullptr;
665 auto It = Ptrcasts.find(Val: F);
666 if (It == Ptrcasts.end()) {
667 PtrCastedI = buildSpvPtrcast(F, Op, ElemTy);
668 Ptrcasts[F] = PtrCastedI;
669 } else {
670 PtrCastedI = It->second;
671 }
672 I->replaceUsesOfWith(From: Op, To: PtrCastedI);
673}
674
675void SPIRVEmitIntrinsicsImpl::propagateElemType(
676 Value *Op, Type *ElemTy,
677 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
678 DenseMap<Function *, CallInst *> Ptrcasts;
679 SmallVector<User *> Users(Op->users());
680 for (auto *U : Users) {
681 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
682 continue;
683 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
684 continue;
685 Instruction *UI = dyn_cast<Instruction>(Val: U);
686 // If the instruction was validated already, we need to keep it valid by
687 // keeping current Op type.
688 if (isaGEP(V: UI) || TypeValidated.find(Ptr: UI) != TypeValidated.end())
689 replaceUsesOfWithSpvPtrcast(Op, ElemTy, I: UI, Ptrcasts);
690 }
691}
692
693void SPIRVEmitIntrinsicsImpl::propagateElemTypeRec(
694 Value *Op, Type *PtrElemTy, Type *CastElemTy,
695 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) {
696 SmallPtrSet<Value *, 0> Visited;
697 DenseMap<Function *, CallInst *> Ptrcasts;
698 propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited,
699 Ptrcasts: std::move(Ptrcasts));
700}
701
702void SPIRVEmitIntrinsicsImpl::propagateElemTypeRec(
703 Value *Op, Type *PtrElemTy, Type *CastElemTy,
704 DenseSet<std::pair<Value *, Value *>> &VisitedSubst,
705 SmallPtrSetImpl<Value *> &Visited,
706 DenseMap<Function *, CallInst *> Ptrcasts) {
707 if (!Visited.insert(Ptr: Op).second)
708 return;
709 SmallVector<User *> Users(Op->users());
710 for (auto *U : Users) {
711 if (!isa<Instruction>(Val: U) || isSpvIntrinsic(Arg: U))
712 continue;
713 if (!VisitedSubst.insert(V: std::make_pair(x&: U, y&: Op)).second)
714 continue;
715 Instruction *UI = dyn_cast<Instruction>(Val: U);
716 // If the instruction was validated already, we need to keep it valid by
717 // keeping current Op type.
718 if (isaGEP(V: UI) || TypeValidated.find(Ptr: UI) != TypeValidated.end())
719 replaceUsesOfWithSpvPtrcast(Op, ElemTy: CastElemTy, I: UI, Ptrcasts);
720 }
721}
722
723// Set element pointer type to the given value of ValueTy and tries to
724// specify this type further (recursively) by Operand value, if needed.
725
726Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByValueDeep(
727 Type *ValueTy, Value *Operand, bool UnknownElemTypeI8) {
728 SmallPtrSet<Value *, 0> Visited;
729 return deduceElementTypeByValueDeep(ValueTy, Operand, Visited,
730 UnknownElemTypeI8);
731}
732
733Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByValueDeep(
734 Type *ValueTy, Value *Operand, SmallPtrSetImpl<Value *> &Visited,
735 bool UnknownElemTypeI8, std::optional<unsigned> AddrSpace) {
736 Type *Ty = ValueTy;
737 if (Operand) {
738 if (auto *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
739 if (Type *NestedTy =
740 deduceElementTypeHelper(I: Operand, Visited, UnknownElemTypeI8))
741 Ty = getTypedPointerWrapper(
742 ElemTy: NestedTy, AS: AddrSpace.value_or(u: PtrTy->getAddressSpace()));
743 } else {
744 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Operand), Ty, Visited,
745 UnknownElemTypeI8);
746 }
747 }
748 return Ty;
749}
750
751// Traverse User instructions to deduce an element pointer type of the operand.
752Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeByUsersDeep(
753 Value *Op, SmallPtrSetImpl<Value *> &Visited, bool UnknownElemTypeI8) {
754 if (!Op || !isPointerTy(T: Op->getType()) || isa<ConstantPointerNull>(Val: Op) ||
755 isa<UndefValue>(Val: Op))
756 return nullptr;
757
758 if (auto ElemTy = getPointeeType(Ty: Op->getType()))
759 return ElemTy;
760
761 // maybe we already know operand's element type
762 if (Type *KnownTy = GR->findDeducedElementType(Val: Op))
763 return KnownTy;
764
765 for (User *OpU : Op->users()) {
766 if (Instruction *Inst = dyn_cast<Instruction>(Val: OpU)) {
767 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8))
768 return Ty;
769 }
770 }
771 return nullptr;
772}
773
774// Implements what we know in advance about intrinsics and builtin calls
775// TODO: consider feasibility of this particular case to be generalized by
776// encoding knowledge about intrinsics and builtin calls by corresponding
777// specification rules
778static Type *getPointeeTypeByCallInst(StringRef DemangledName,
779 Function *CalledF, unsigned OpIdx) {
780 // OpenCL.std printf takes its format string as a pointer to i8. Match the
781 // bare builtin name, as lowering does, to also cover unmangled `printf`.
782 std::string Name = SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName);
783 // OpenCL get_fence only has void* overloads.
784 if (OpIdx == 0 && (Name == "printf" || Name == "get_fence"))
785 return IntegerType::getInt8Ty(C&: CalledF->getContext());
786 // OpCaptureEventProfilingInfo requires Value to point to an 8-bit integer.
787 if (OpIdx == 2 && (Name == "capture_event_profiling_info" ||
788 Name == "__spirv_CaptureEventProfilingInfo"))
789 return IntegerType::getInt8Ty(C&: CalledF->getContext());
790 return nullptr;
791}
792
793// Deduce and return a successfully deduced Type of the Instruction,
794// or nullptr otherwise.
795Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeHelper(Value *I,
796 bool UnknownElemTypeI8) {
797 SmallPtrSet<Value *, 0> Visited;
798 return deduceElementTypeHelper(I, Visited, UnknownElemTypeI8);
799}
800
801void SPIRVEmitIntrinsicsImpl::maybeAssignPtrType(Type *&Ty, Value *Op,
802 Type *RefTy,
803 bool UnknownElemTypeI8) {
804 if (isUntypedPointerTy(T: RefTy)) {
805 if (!UnknownElemTypeI8)
806 return;
807 insertTodoType(Op);
808 if (isa<IntToPtrInst>(Val: Op))
809 return;
810 }
811 Ty = RefTy;
812}
813
814bool SPIRVEmitIntrinsicsImpl::walkLogicalAccessChainDynamic(
815 Type *CurType, Value *Operand, uint64_t Multiplier,
816 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
817 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing) {
818 // Dynamic indexing into a struct is not possible.
819 // We know that we must be accessing the first element
820 // of the struct if the current type is a struct.
821 // Try to find the first array type that is at offset 0 in the struct.
822 while (auto *ST = dyn_cast<StructType>(Val: CurType)) {
823 if (ST->getNumElements() == 0)
824 break;
825 CurType = ST->getElementType(N: 0);
826 OnLiteralIndexing(CurType, 0);
827 }
828
829 assert(CurType);
830 ArrayType *AT = dyn_cast<ArrayType>(Val: CurType);
831 // Operand is not constant. Either we have an array and accept it, or we
832 // give up.
833 if (AT)
834 OnDynamicIndexing(AT->getElementType(), Operand, Multiplier);
835 return AT == nullptr;
836}
837
838Expected<bool> SPIRVEmitIntrinsicsImpl::walkLogicalAccessChainConstant(
839 Type *CurType, uint64_t Offset,
840 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing) {
841 auto &DL = CurrF->getDataLayout();
842 Type *BaseType = CurType;
843
844 do {
845 if (ArrayType *AT = dyn_cast<ArrayType>(Val: CurType)) {
846 uint64_t EltTypeSize = DL.getTypeAllocSize(Ty: AT->getElementType());
847 assert(Offset < AT->getNumElements() * EltTypeSize);
848 uint64_t Index = Offset / EltTypeSize;
849 Offset = Offset - (Index * EltTypeSize);
850 CurType = AT->getElementType();
851 OnLiteralIndexing(CurType, Index);
852 } else if (StructType *ST = dyn_cast<StructType>(Val: CurType)) {
853 uint32_t StructSize = DL.getTypeSizeInBits(Ty: ST) / 8;
854 assert(Offset < StructSize);
855 (void)StructSize;
856 const auto &STL = DL.getStructLayout(Ty: ST);
857 unsigned Element = STL->getElementContainingOffset(FixedOffset: Offset);
858 Offset -= STL->getElementOffset(Idx: Element);
859 CurType = ST->getElementType(N: Element);
860 OnLiteralIndexing(CurType, Element);
861 } else if (auto *VT = dyn_cast<FixedVectorType>(Val: CurType)) {
862 Type *EltTy = VT->getElementType();
863 TypeSize EltSizeBits = DL.getTypeSizeInBits(Ty: EltTy);
864 assert(EltSizeBits % 8 == 0 &&
865 "Element type size in bits must be a multiple of 8.");
866 uint32_t EltTypeSize = EltSizeBits / 8;
867 if (Offset >= VT->getNumElements() * EltTypeSize)
868 return createStringError(
869 Fmt: "Cannot lower an access to vector padding for Logical SPIR-V");
870 uint64_t Index = Offset / EltTypeSize;
871 Offset -= Index * EltTypeSize;
872 CurType = EltTy;
873 OnLiteralIndexing(CurType, Index);
874 } else {
875 if (CurType == BaseType && Offset >= DL.getTypeAllocSize(Ty: CurType))
876 return true;
877 return createStringError(
878 Fmt: "Cannot lower a byte offset within a scalar or padding for "
879 "Logical SPIR-V");
880 }
881 } while (Offset > 0);
882
883 return false;
884}
885
886Expected<bool> SPIRVEmitIntrinsicsImpl::walkLogicalAccessChain(
887 GetElementPtrInst &GEP,
888 const std::function<void(Type *, uint64_t)> &OnLiteralIndexing,
889 const std::function<void(Type *, Value *, uint64_t)> &OnDynamicIndexing) {
890 // We only rewrite byte-addressing GEP. Other should be left as-is.
891 // Valid byte-addressing GEP must always have a single index.
892 std::optional<uint64_t> MultiplierOpt =
893 getByteAddressingMultiplier(Ty: GEP.getSourceElementType());
894 assert(MultiplierOpt && "We only rewrite byte-addressing GEP");
895 uint64_t Multiplier = *MultiplierOpt;
896 assert(GEP.getNumIndices() == 1);
897
898 Value *Src = getPointerRoot(I: GEP.getPointerOperand());
899 Type *CurType = deduceElementType(I: Src, UnknownElemTypeI8: true);
900 if (!CurType->isSized())
901 return createStringError(
902 Fmt: "Cannot lower byte addressing with an unsized type for Logical SPIR-V");
903
904 Value *Operand = *GEP.idx_begin();
905 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: Operand))
906 return walkLogicalAccessChainConstant(
907 CurType, Offset: CI->getZExtValue() * Multiplier, OnLiteralIndexing);
908
909 return walkLogicalAccessChainDynamic(CurType, Operand, Multiplier,
910 OnLiteralIndexing, OnDynamicIndexing);
911}
912
913void SPIRVEmitIntrinsicsImpl::recordAccessChainError(GetElementPtrInst &GEP,
914 Error Err) {
915 InvalidGEPs.insert(Ptr: &GEP);
916 AccessChainErrors = joinErrors(
917 E1: std::move(AccessChainErrors),
918 E2: createStringError(S: Twine(toString(E: std::move(Err))) + " (in function '" +
919 GEP.getFunction()->getName() + "')"));
920}
921
922Instruction *SPIRVEmitIntrinsicsImpl::buildLogicalAccessChainFromGEP(
923 GetElementPtrInst &GEP) {
924 if (InvalidGEPs.contains(Ptr: &GEP))
925 return nullptr;
926
927 auto &DL = CurrF->getDataLayout();
928 IRBuilder<> B(GEP.getParent());
929 B.SetInsertPoint(&GEP);
930
931 std::vector<Value *> Indices;
932 Indices.push_back(x: ConstantInt::get(
933 Ty: IntegerType::getInt32Ty(C&: CurrF->getContext()), V: 0, /* Signed= */ IsSigned: false));
934 auto Result = walkLogicalAccessChain(
935 GEP,
936 OnLiteralIndexing: [&Indices, &B](Type *EltType, uint64_t Index) {
937 Indices.push_back(
938 x: ConstantInt::get(Ty: B.getInt64Ty(), V: Index, /* Signed= */ IsSigned: false));
939 },
940 OnDynamicIndexing: [&Indices, &B, &DL, this](Type *EltType, Value *Offset,
941 uint64_t Multiplier) {
942 Value *Index = nullptr;
943 uint32_t EltTypeSize = DL.getTypeSizeInBits(Ty: EltType) / 8;
944 assert(Multiplier != 0);
945 if (Multiplier == EltTypeSize) {
946 Index = Offset;
947 } else if (EltTypeSize % Multiplier == 0) {
948 Index =
949 B.CreateUDiv(LHS: Offset, RHS: ConstantInt::get(Ty: Offset->getType(),
950 V: EltTypeSize / Multiplier,
951 /* Signed= */ IsSigned: false));
952 } else {
953 Index = B.CreateMul(LHS: Offset,
954 RHS: ConstantInt::get(Ty: Offset->getType(), V: Multiplier,
955 /* Signed= */ IsSigned: false));
956 insertAssignTypeIntrs(I: cast<Instruction>(Val: Index), B);
957 Index = B.CreateUDiv(LHS: Index,
958 RHS: ConstantInt::get(Ty: Offset->getType(), V: EltTypeSize,
959 /* Signed= */ IsSigned: false));
960 }
961 insertAssignTypeIntrs(I: cast<Instruction>(Val: Index), B);
962 Indices.push_back(x: Index);
963 });
964 if (!Result) {
965 recordAccessChainError(GEP, Err: Result.takeError());
966 return nullptr;
967 }
968
969 B.SetInsertPoint(&GEP);
970 SmallVector<Type *, 2> Types = {GEP.getType(), GEP.getOperand(i_nocapture: 0)->getType()};
971 SmallVector<Value *, 4> Args;
972 Args.push_back(Elt: B.getInt1(V: GEP.isInBounds()));
973 Args.push_back(Elt: GEP.getOperand(i_nocapture: 0));
974 llvm::append_range(C&: Args, R&: Indices);
975 Instruction *NewI =
976 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
977 replaceAllUsesWithAndErase(B, Src: &GEP, Dest: NewI);
978 return NewI;
979}
980
981Type *SPIRVEmitIntrinsicsImpl::getGEPTypeLogical(GetElementPtrInst *GEP) {
982 if (InvalidGEPs.contains(Ptr: GEP))
983 return nullptr;
984
985 Type *CurType = GEP->getResultElementType();
986
987 auto Result = walkLogicalAccessChain(
988 GEP&: *GEP, OnLiteralIndexing: [&CurType](Type *EltType, uint64_t Index) { CurType = EltType; },
989 OnDynamicIndexing: [&CurType](Type *EltType, Value *Index, uint64_t) { CurType = EltType; });
990 if (!Result) {
991 recordAccessChainError(GEP&: *GEP, Err: Result.takeError());
992 return nullptr;
993 }
994
995 return *Result ? GEP->getResultElementType() : CurType;
996}
997
998Type *SPIRVEmitIntrinsicsImpl::getGEPType(GetElementPtrInst *Ref) {
999 if (getByteAddressingMultiplier(Ty: Ref->getSourceElementType()) &&
1000 TM.getSubtargetImpl()->isLogicalSPIRV()) {
1001 return getGEPTypeLogical(GEP: Ref);
1002 }
1003
1004 Type *Ty = nullptr;
1005 // TODO: not sure if GetElementPtrInst::getTypeAtIndex() does anything
1006 // useful here
1007 if (isNestedPointer(Ty: Ref->getSourceElementType())) {
1008 Ty = Ref->getSourceElementType();
1009 for (Use &U : drop_begin(RangeOrContainer: Ref->indices()))
1010 Ty = GetElementPtrInst::getTypeAtIndex(Ty, Idx: U.get());
1011 } else {
1012 Ty = Ref->getResultElementType();
1013 }
1014 return Ty;
1015}
1016
1017Type *SPIRVEmitIntrinsicsImpl::deduceElementTypeHelper(
1018 Value *I, SmallPtrSetImpl<Value *> &Visited, bool UnknownElemTypeI8,
1019 bool IgnoreKnownType) {
1020 // allow to pass nullptr as an argument
1021 if (!I)
1022 return nullptr;
1023
1024 // maybe already known
1025 if (!IgnoreKnownType)
1026 if (Type *KnownTy = GR->findDeducedElementType(Val: I))
1027 return KnownTy;
1028
1029 // maybe a cycle
1030 if (!Visited.insert(Ptr: I).second)
1031 return nullptr;
1032
1033 // fallback value in case when we fail to deduce a type
1034 Type *Ty = nullptr;
1035 // look for known basic patterns of type inference
1036 if (auto *Ref = dyn_cast<AllocaInst>(Val: I)) {
1037 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getAllocatedType(), UnknownElemTypeI8);
1038 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
1039 Ty = getGEPType(Ref);
1040 } else if (auto *SGEP = dyn_cast<StructuredGEPInst>(Val: I)) {
1041 Ty = SGEP->getResultElementType();
1042 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
1043 Value *Op = Ref->getPointerOperand();
1044 Type *KnownTy = GR->findDeducedElementType(Val: Op);
1045 if (!KnownTy)
1046 KnownTy = Op->getType();
1047 if (Type *ElemTy = getPointeeType(Ty: KnownTy))
1048 maybeAssignPtrType(Ty, Op: I, RefTy: ElemTy, UnknownElemTypeI8);
1049 } else if (auto *Ref = dyn_cast<GlobalValue>(Val: I)) {
1050 if (auto *Fn = dyn_cast<Function>(Val: Ref)) {
1051 Ty = SPIRV::getOriginalFunctionType(F: *Fn);
1052 GR->addDeducedElementType(Val: I, Ty);
1053 } else {
1054 Value *Op = Ref->getNumOperands() > 0 ? Ref->getOperand(i: 0) : nullptr;
1055 // Code lives in the program address space, not in the address space of
1056 // the global. For known SPIR-V targets, program address space 0 maps
1057 // to Function/Private, which cannot hold code, so treat it as unset.
1058 std::optional<unsigned> ProgramAS;
1059 if (isa_and_nonnull<Function>(Val: Op)) {
1060 if (unsigned AS = CurrF->getDataLayout().getProgramAddressSpace())
1061 ProgramAS = AS;
1062 else if (TM.getSubtargetImpl()->canUseExtension(
1063 E: SPIRV::Extension::SPV_INTEL_function_pointers))
1064 ProgramAS =
1065 storageClassToAddressSpace(SC: SPIRV::StorageClass::CodeSectionINTEL);
1066 }
1067 Ty = deduceElementTypeByValueDeep(ValueTy: Ref->getValueType(), Operand: Op, Visited,
1068 UnknownElemTypeI8, AddrSpace: ProgramAS);
1069 }
1070 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1071 Type *RefTy = deduceElementTypeHelper(I: Ref->getPointerOperand(), Visited,
1072 UnknownElemTypeI8);
1073 maybeAssignPtrType(Ty, Op: I, RefTy, UnknownElemTypeI8);
1074 } else if (auto *Ref = dyn_cast<IntToPtrInst>(Val: I)) {
1075 maybeAssignPtrType(Ty, Op: I, RefTy: Ref->getDestTy(), UnknownElemTypeI8);
1076 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1077 if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy();
1078 isPointerTy(T: Src) && isPointerTy(T: Dest))
1079 Ty = deduceElementTypeHelper(I: Ref->getOperand(i_nocapture: 0), Visited,
1080 UnknownElemTypeI8);
1081 } else if (auto *Ref = dyn_cast<ExtractValueInst>(Val: I)) {
1082 // An extracted pointer has the type of its aggregate member.
1083 if (isPointerTy(T: I->getType()))
1084 Ty = getMemberPointeeType(
1085 AggrTy: reconstructType(Op: Ref->getAggregateOperand(), UnknownElemTypeI8: false, IsPostprocessing: false),
1086 Indices: Ref->getIndices());
1087 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1088 Value *Op = Ref->getNewValOperand();
1089 if (isPointerTy(T: Op->getType()))
1090 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
1091 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1092 Value *Op = Ref->getValOperand();
1093 if (isPointerTy(T: Op->getType()))
1094 Ty = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8);
1095 } else if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1096 Type *BestTy = nullptr;
1097 unsigned MaxN = 1;
1098 DenseMap<Type *, unsigned> PhiTys;
1099 for (int i = Ref->getNumIncomingValues() - 1; i >= 0; --i) {
1100 Ty = deduceElementTypeByUsersDeep(Op: Ref->getIncomingValue(i), Visited,
1101 UnknownElemTypeI8);
1102 if (!Ty)
1103 continue;
1104 auto It = PhiTys.try_emplace(Key: Ty, Args: 1);
1105 if (!It.second) {
1106 ++It.first->second;
1107 if (It.first->second > MaxN) {
1108 MaxN = It.first->second;
1109 BestTy = Ty;
1110 }
1111 }
1112 }
1113 if (BestTy)
1114 Ty = BestTy;
1115 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1116 for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) {
1117 // A function pointer operand carries its function type directly. Other
1118 // operands are deduced from their uses.
1119 Ty = isa<Function>(Val: Op)
1120 ? deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8)
1121 : deduceElementTypeByUsersDeep(Op, Visited, UnknownElemTypeI8);
1122 if (Ty)
1123 break;
1124 }
1125 } else if (auto *CI = dyn_cast<CallInst>(Val: I)) {
1126 static StringMap<unsigned> ResTypeByArg = {
1127 {"to_global", 0},
1128 {"to_local", 0},
1129 {"to_private", 0},
1130 {"__to_global", 0},
1131 {"__to_local", 0},
1132 {"__to_private", 0},
1133 {"__spirv_GenericCastToPtr_ToGlobal", 0},
1134 {"__spirv_GenericCastToPtr_ToLocal", 0},
1135 {"__spirv_GenericCastToPtr_ToPrivate", 0},
1136 {"__spirv_GenericCastToPtrExplicit_ToGlobal", 0},
1137 {"__spirv_GenericCastToPtrExplicit_ToLocal", 0},
1138 {"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}};
1139 // TODO: maybe improve performance by caching demangled names
1140
1141 auto *II = dyn_cast<IntrinsicInst>(Val: I);
1142 if (II && (II->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer ||
1143 II->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) {
1144 auto *HandleType = cast<TargetExtType>(Val: II->getOperand(i_nocapture: 0)->getType());
1145 if (HandleType->getTargetExtName() == "spirv.Image" ||
1146 HandleType->getTargetExtName() == "spirv.SignedImage") {
1147 for (User *U : II->users()) {
1148 Ty = cast<Instruction>(Val: U)->getAccessType();
1149 if (Ty)
1150 break;
1151 }
1152 } else if (HandleType->getTargetExtName() == "spirv.VulkanBuffer") {
1153 // This call is supposed to index into an array
1154 Ty = HandleType->getTypeParameter(i: 0);
1155 if (II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
1156 if (Ty->isArrayTy())
1157 Ty = Ty->getArrayElementType();
1158 else {
1159 assert(Ty && Ty->isStructTy());
1160 uint32_t Index =
1161 cast<ConstantInt>(Val: II->getOperand(i_nocapture: 1))->getZExtValue();
1162 Ty = cast<StructType>(Val: Ty)->getElementType(N: Index);
1163 }
1164 }
1165 Ty = reconstitutePeeledArrayType(Ty);
1166 } else {
1167 llvm_unreachable("Unknown handle type for spv_resource_getpointer.");
1168 }
1169 } else if (II && II->getIntrinsicID() ==
1170 Intrinsic::spv_generic_cast_to_ptr_explicit) {
1171 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: 0), Visited,
1172 UnknownElemTypeI8);
1173 } else if (Function *CalledF = CI->getCalledFunction()) {
1174 std::string DemangledName =
1175 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1176 if (DemangledName.length() > 0)
1177 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName);
1178 auto AsArgIt = ResTypeByArg.find(Key: DemangledName);
1179 if (AsArgIt != ResTypeByArg.end())
1180 Ty = deduceElementTypeHelper(I: CI->getArgOperand(i: AsArgIt->second),
1181 Visited, UnknownElemTypeI8);
1182 else if (Type *KnownRetTy = GR->findDeducedElementType(Val: CalledF))
1183 Ty = KnownRetTy;
1184 }
1185 }
1186
1187 // remember the found relationship
1188 if (Ty && !IgnoreKnownType) {
1189 // specify nested types if needed, otherwise return unchanged
1190 GR->addDeducedElementType(Val: I, Ty: normalizeType(Ty, CanUseAnyVectorRank));
1191 }
1192
1193 return Ty;
1194}
1195
1196// Re-create a type of the value if it has untyped pointer fields, also nested.
1197// Return the original value type if no corrections of untyped pointer
1198// information is found or needed.
1199Type *SPIRVEmitIntrinsicsImpl::deduceNestedTypeHelper(User *U,
1200 bool UnknownElemTypeI8) {
1201 SmallPtrSet<Value *, 0> Visited;
1202 return deduceNestedTypeHelper(U, Ty: U->getType(), Visited, UnknownElemTypeI8);
1203}
1204
1205Type *SPIRVEmitIntrinsicsImpl::deduceNestedTypeHelper(
1206 User *U, Type *OrigTy, SmallPtrSetImpl<Value *> &Visited,
1207 bool UnknownElemTypeI8) {
1208 // Only constant aggregate operands map to aggregate elements.
1209 if (!isa_and_nonnull<ConstantAggregate>(Val: U))
1210 return OrigTy;
1211
1212 // maybe already known
1213 if (Type *KnownTy = GR->findDeducedCompositeType(Val: U))
1214 return KnownTy;
1215
1216 // maybe a cycle
1217 if (!Visited.insert(Ptr: U).second)
1218 return OrigTy;
1219
1220 if (auto *OrigStructTy = dyn_cast<StructType>(Val: OrigTy)) {
1221 SmallVector<Type *> Tys;
1222 bool Change = false;
1223 for (unsigned i = 0; i < U->getNumOperands(); ++i) {
1224 Value *Op = U->getOperand(i);
1225 assert(Op && "Operands should not be null.");
1226 Type *OpTy = Op->getType();
1227 Type *Ty = OpTy;
1228 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1229 if (Type *NestedTy =
1230 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1231 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1232 } else {
1233 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1234 UnknownElemTypeI8);
1235 }
1236 Tys.push_back(Elt: Ty);
1237 Change |= Ty != OpTy;
1238 }
1239 if (Change) {
1240 Type *NewTy = StructType::create(
1241 Elements: Tys, Name: OrigStructTy->isLiteral() ? "" : OrigStructTy->getName(),
1242 isPacked: OrigStructTy->isPacked());
1243 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1244 return NewTy;
1245 }
1246 } else if (auto *ArrTy = dyn_cast<ArrayType>(Val: OrigTy)) {
1247 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1248 Type *OpTy = ArrTy->getElementType();
1249 Type *Ty = OpTy;
1250 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1251 if (Type *NestedTy =
1252 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1253 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1254 } else {
1255 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1256 UnknownElemTypeI8);
1257 }
1258 if (Ty != OpTy) {
1259 Type *NewTy = ArrayType::get(ElementType: Ty, NumElements: ArrTy->getNumElements());
1260 GR->addDeducedCompositeType(Val: U, Ty: NewTy);
1261 return NewTy;
1262 }
1263 }
1264 } else if (auto *VecTy = dyn_cast<VectorType>(Val: OrigTy)) {
1265 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(i: 0) : nullptr) {
1266 Type *OpTy = VecTy->getElementType();
1267 Type *Ty = OpTy;
1268 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
1269 if (Type *NestedTy =
1270 deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8))
1271 Ty = getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1272 } else {
1273 Ty = deduceNestedTypeHelper(U: dyn_cast<User>(Val: Op), OrigTy: OpTy, Visited,
1274 UnknownElemTypeI8);
1275 }
1276 if (Ty != OpTy) {
1277 Type *NewTy = VectorType::get(ElementType: Ty, EC: VecTy->getElementCount());
1278 GR->addDeducedCompositeType(Val: U,
1279 Ty: normalizeType(Ty: NewTy, CanUseAnyVectorRank));
1280 return NewTy;
1281 }
1282 }
1283 }
1284
1285 return OrigTy;
1286}
1287
1288Type *SPIRVEmitIntrinsicsImpl::deduceElementType(Value *I,
1289 bool UnknownElemTypeI8) {
1290 if (Type *Ty = deduceElementTypeHelper(I, UnknownElemTypeI8))
1291 return Ty;
1292 if (!UnknownElemTypeI8)
1293 return nullptr;
1294 insertTodoType(Op: I);
1295 return IntegerType::getInt8Ty(C&: I->getContext());
1296}
1297
1298static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I,
1299 Value *PointerOperand) {
1300 Type *PointeeTy = GR->findDeducedElementType(Val: PointerOperand);
1301 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1302 return nullptr;
1303 auto *PtrTy = dyn_cast<PointerType>(Val: I->getType());
1304 if (!PtrTy)
1305 return I->getType();
1306 if (Type *NestedTy = GR->findDeducedElementType(Val: I))
1307 return getTypedPointerWrapper(ElemTy: NestedTy, AS: PtrTy->getAddressSpace());
1308 return nullptr;
1309}
1310
1311// Try to deduce element type for a call base. Returns false if this is an
1312// indirect function invocation, and true otherwise.
1313bool SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeCalledFunction(
1314 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1315 Type *&KnownElemTy, bool &Incomplete) {
1316 Function *CalledF = CI->getCalledFunction();
1317 if (!CalledF)
1318 return false;
1319 std::string DemangledName =
1320 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
1321 if (DemangledName.length() > 0 &&
1322 !StringRef(DemangledName).starts_with(Prefix: "llvm.")) {
1323 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *CalledF);
1324 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(
1325 DemangledCall: DemangledName, Set: ST.getPreferredInstructionSet());
1326 if (Opcode == SPIRV::OpGroupAsyncCopy) {
1327 for (unsigned i = 0, PtrCnt = 0; i < CI->arg_size() && PtrCnt < 2; ++i) {
1328 Value *Op = CI->getArgOperand(i);
1329 if (!isPointerTy(T: Op->getType()))
1330 continue;
1331 ++PtrCnt;
1332 if (Type *ElemTy = GR->findDeducedElementType(Val: Op))
1333 KnownElemTy = ElemTy; // src will rewrite dest if both are defined
1334 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1335 }
1336 } else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) {
1337 if (CI->arg_size() == 0)
1338 return true;
1339 Value *Op = CI->getArgOperand(i: 0);
1340 if (!isPointerTy(T: Op->getType()))
1341 return true;
1342 switch (Opcode) {
1343 case SPIRV::OpAtomicFAddEXT:
1344 case SPIRV::OpAtomicFMinEXT:
1345 case SPIRV::OpAtomicFMaxEXT:
1346 case SPIRV::OpAtomicLoad:
1347 case SPIRV::OpAtomicCompareExchangeWeak:
1348 case SPIRV::OpAtomicCompareExchange:
1349 case SPIRV::OpAtomicExchange:
1350 case SPIRV::OpAtomicIAdd:
1351 case SPIRV::OpAtomicISub:
1352 case SPIRV::OpAtomicOr:
1353 case SPIRV::OpAtomicXor:
1354 case SPIRV::OpAtomicAnd:
1355 case SPIRV::OpAtomicUMin:
1356 case SPIRV::OpAtomicUMax:
1357 case SPIRV::OpAtomicSMin:
1358 case SPIRV::OpAtomicSMax: {
1359 KnownElemTy = isPointerTy(T: CI->getType()) ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1360 : CI->getType();
1361 if (!KnownElemTy)
1362 return true;
1363 Incomplete = isTodoType(Op);
1364 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1365 } break;
1366 case SPIRV::OpAtomicStore: {
1367 if (CI->arg_size() < 4)
1368 return true;
1369 Value *ValOp = CI->getArgOperand(i: 3);
1370 KnownElemTy = isPointerTy(T: ValOp->getType())
1371 ? getAtomicElemTy(GR, I: CI, PointerOperand: Op)
1372 : ValOp->getType();
1373 if (!KnownElemTy)
1374 return true;
1375 Incomplete = isTodoType(Op);
1376 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1377 } break;
1378 }
1379 }
1380 }
1381 return true;
1382}
1383
1384// Try to deduce element type for a function pointer.
1385void SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeFunctionPointer(
1386 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops,
1387 Type *&KnownElemTy, bool IsPostprocessing) {
1388 Value *Op = CI->getCalledOperand();
1389 if (!Op || !isPointerTy(T: Op->getType()))
1390 return;
1391 Ops.push_back(Elt: std::make_pair(x&: Op, y: std::numeric_limits<unsigned>::max()));
1392 FunctionType *FTy = SPIRV::getOriginalFunctionType(CB: *CI);
1393 bool IsNewFTy = false, IsIncomplete = false;
1394 SmallVector<Type *, 4> ArgTys;
1395 for (auto &&[ParmIdx, Arg] : llvm::enumerate(First: CI->args())) {
1396 Type *ArgTy = Arg->getType();
1397 if (ArgTy->isPointerTy()) {
1398 if (Type *ElemTy = GR->findDeducedElementType(Val: Arg)) {
1399 IsNewFTy = true;
1400 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
1401 if (isTodoType(Op: Arg))
1402 IsIncomplete = true;
1403 } else {
1404 IsIncomplete = true;
1405 }
1406 } else {
1407 ArgTy = FTy->getFunctionParamType(i: ParmIdx);
1408 }
1409 ArgTys.push_back(Elt: ArgTy);
1410 }
1411 Type *RetTy = FTy->getReturnType();
1412 if (CI->getType()->isPointerTy()) {
1413 if (Type *ElemTy = GR->findDeducedElementType(Val: CI)) {
1414 IsNewFTy = true;
1415 RetTy =
1416 getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: CI->getType()));
1417 if (isTodoType(Op: CI))
1418 IsIncomplete = true;
1419 } else {
1420 IsIncomplete = true;
1421 }
1422 }
1423 if (!IsPostprocessing && IsIncomplete)
1424 insertTodoType(Op);
1425 KnownElemTy =
1426 IsNewFTy ? FunctionType::get(Result: RetTy, Params: ArgTys, isVarArg: FTy->isVarArg()) : FTy;
1427}
1428
1429bool SPIRVEmitIntrinsicsImpl::deduceOperandElementTypeFunctionRet(
1430 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
1431 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing,
1432 Type *&KnownElemTy, Value *Op, Function *F) {
1433 KnownElemTy = GR->findDeducedElementType(Val: F);
1434 if (KnownElemTy)
1435 return false;
1436 if (Type *OpElemTy = GR->findDeducedElementType(Val: Op)) {
1437 OpElemTy = normalizeType(Ty: OpElemTy, CanUseAnyVectorRank);
1438 GR->addDeducedElementType(Val: F, Ty: OpElemTy);
1439 GR->addReturnType(
1440 ArgF: F, DerivedTy: TypedPointerType::get(ElementType: OpElemTy,
1441 AddressSpace: getPointerAddressSpace(T: F->getReturnType())));
1442 // non-recursive update of types in function uses
1443 DenseSet<std::pair<Value *, Value *>> VisitedSubst{std::make_pair(x&: I, y&: Op)};
1444 for (User *U : F->users()) {
1445 CallInst *CI = dyn_cast<CallInst>(Val: U);
1446 if (!CI || CI->getCalledFunction() != F)
1447 continue;
1448 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: CI)) {
1449 if (Type *PrevElemTy = GR->findDeducedElementType(Val: CI)) {
1450 GR->updateAssignType(
1451 AssignCI, Arg: CI,
1452 OfType: getNormalizedPoisonValue(Ty: OpElemTy, CanUseAnyVectorRank));
1453 propagateElemType(Op: CI, ElemTy: PrevElemTy, VisitedSubst);
1454 }
1455 }
1456 }
1457 // Non-recursive update of types in the function uncomplete returns.
1458 // This may happen just once per a function, the latch is a pair of
1459 // findDeducedElementType(F) / addDeducedElementType(F, ...).
1460 // With or without the latch it is a non-recursive call due to
1461 // IncompleteRets set to nullptr in this call.
1462 if (IncompleteRets)
1463 for (Instruction *IncompleteRetI : *IncompleteRets)
1464 deduceOperandElementType(I: IncompleteRetI, IncompleteRets: nullptr, AskOps,
1465 IsPostprocessing);
1466 } else if (IncompleteRets) {
1467 IncompleteRets->insert(Ptr: I);
1468 }
1469 TypeValidated.insert(Ptr: I);
1470 return true;
1471}
1472
1473// If the Instruction has Pointer operands with unresolved types, this function
1474// tries to deduce them. If the Instruction has Pointer operands with known
1475// types which differ from expected, this function tries to insert a bitcast to
1476// resolve the issue.
1477void SPIRVEmitIntrinsicsImpl::deduceOperandElementType(
1478 Instruction *I, SmallPtrSetImpl<Instruction *> *IncompleteRets,
1479 const SmallPtrSetImpl<Value *> *AskOps, bool IsPostprocessing) {
1480 SmallVector<std::pair<Value *, unsigned>> Ops;
1481 Type *KnownElemTy = nullptr;
1482 bool Incomplete = false;
1483 // look for known basic patterns of type inference
1484 if (auto *Ref = dyn_cast<PHINode>(Val: I)) {
1485 if (!isPointerTy(T: I->getType()) ||
1486 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1487 return;
1488 Incomplete = isTodoType(Op: I);
1489 for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
1490 Value *Op = Ref->getIncomingValue(i);
1491 if (isPointerTy(T: Op->getType()))
1492 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1493 }
1494 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(Val: I)) {
1495 KnownElemTy = GR->findDeducedElementType(Val: I);
1496 if (!KnownElemTy)
1497 return;
1498 Incomplete = isTodoType(Op: I);
1499 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(), y: 0));
1500 } else if (auto *Ref = dyn_cast<BitCastInst>(Val: I)) {
1501 if (!isPointerTy(T: I->getType()))
1502 return;
1503 KnownElemTy = GR->findDeducedElementType(Val: I);
1504 if (!KnownElemTy)
1505 return;
1506 Incomplete = isTodoType(Op: I);
1507 Ops.push_back(Elt: std::make_pair(x: Ref->getOperand(i_nocapture: 0), y: 0));
1508 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(Val: I)) {
1509 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1510 return;
1511 KnownElemTy = Ref->getSourceElementType();
1512 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1513 y: GetElementPtrInst::getPointerOperandIndex()));
1514 } else if (auto *Ref = dyn_cast<StructuredGEPInst>(Val: I)) {
1515 if (GR->findDeducedElementType(Val: Ref->getPointerOperand()))
1516 return;
1517 KnownElemTy = Ref->getBaseType();
1518 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1519 y: StructuredGEPInst::getPointerOperandIndex()));
1520 } else if (auto *Ref = dyn_cast<LoadInst>(Val: I)) {
1521 KnownElemTy = I->getType();
1522 if (isUntypedPointerTy(T: KnownElemTy)) {
1523 // A T** loaded back from its alloca comes out opaque, dropping type info.
1524 // When the load is a pointer-to-pointer, type the alloca as that pointer.
1525 Type *LoadedElemTy = GR->findDeducedElementType(Val: I);
1526 if (!LoadedElemTy || !isPointerTyOrWrapper(Ty: LoadedElemTy))
1527 return;
1528 Value *Root = Ref->getPointerOperand()->stripPointerCasts();
1529 if (!isa<AllocaInst>(Val: Root))
1530 return;
1531 KnownElemTy = getTypedPointerWrapper(ElemTy: LoadedElemTy,
1532 AS: getPointerAddressSpace(T: KnownElemTy));
1533 }
1534 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1535 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1536 return;
1537 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1538 y: LoadInst::getPointerOperandIndex()));
1539 } else if (auto *Ref = dyn_cast<StoreInst>(Val: I)) {
1540 if (!(KnownElemTy =
1541 reconstructType(Op: Ref->getValueOperand(), UnknownElemTypeI8: false, IsPostprocessing)))
1542 return;
1543 Type *PointeeTy = GR->findDeducedElementType(Val: Ref->getPointerOperand());
1544 if (PointeeTy && !isUntypedPointerTy(T: PointeeTy))
1545 return;
1546 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1547 y: StoreInst::getPointerOperandIndex()));
1548 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
1549 KnownElemTy = isPointerTy(T: I->getType())
1550 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1551 : I->getType();
1552 if (!KnownElemTy)
1553 return;
1554 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1555 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1556 y: AtomicCmpXchgInst::getPointerOperandIndex()));
1557 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(Val: I)) {
1558 KnownElemTy = isPointerTy(T: I->getType())
1559 ? getAtomicElemTy(GR, I, PointerOperand: Ref->getPointerOperand())
1560 : I->getType();
1561 if (!KnownElemTy)
1562 return;
1563 Incomplete = isTodoType(Op: Ref->getPointerOperand());
1564 Ops.push_back(Elt: std::make_pair(x: Ref->getPointerOperand(),
1565 y: AtomicRMWInst::getPointerOperandIndex()));
1566 } else if (auto *Ref = dyn_cast<SelectInst>(Val: I)) {
1567 if (!isPointerTy(T: I->getType()) ||
1568 !(KnownElemTy = GR->findDeducedElementType(Val: I)))
1569 return;
1570 Incomplete = isTodoType(Op: I);
1571 for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
1572 Value *Op = Ref->getOperand(i_nocapture: i);
1573 if (isPointerTy(T: Op->getType()))
1574 Ops.push_back(Elt: std::make_pair(x&: Op, y&: i));
1575 }
1576 } else if (auto *Ref = dyn_cast<ReturnInst>(Val: I)) {
1577 if (!isPointerTy(T: CurrF->getReturnType()))
1578 return;
1579 Value *Op = Ref->getReturnValue();
1580 if (!Op)
1581 return;
1582 if (deduceOperandElementTypeFunctionRet(I, IncompleteRets, AskOps,
1583 IsPostprocessing, KnownElemTy, Op,
1584 F: CurrF))
1585 return;
1586 Incomplete = isTodoType(Op: CurrF);
1587 Ops.push_back(Elt: std::make_pair(x&: Op, y: 0));
1588 } else if (auto *Ref = dyn_cast<ICmpInst>(Val: I)) {
1589 if (!isPointerTy(T: Ref->getOperand(i_nocapture: 0)->getType()))
1590 return;
1591 Value *Op0 = Ref->getOperand(i_nocapture: 0);
1592 Value *Op1 = Ref->getOperand(i_nocapture: 1);
1593 bool Incomplete0 = isTodoType(Op: Op0);
1594 bool Incomplete1 = isTodoType(Op: Op1);
1595 Type *ElemTy1 = GR->findDeducedElementType(Val: Op1);
1596 Type *ElemTy0 = (Incomplete0 && !Incomplete1 && ElemTy1)
1597 ? nullptr
1598 : GR->findDeducedElementType(Val: Op0);
1599 if (ElemTy0) {
1600 KnownElemTy = ElemTy0;
1601 Incomplete = Incomplete0;
1602 Ops.push_back(Elt: std::make_pair(x&: Op1, y: 1));
1603 } else if (ElemTy1) {
1604 KnownElemTy = ElemTy1;
1605 Incomplete = Incomplete1;
1606 Ops.push_back(Elt: std::make_pair(x&: Op0, y: 0));
1607 }
1608 } else if (CallInst *CI = dyn_cast<CallInst>(Val: I)) {
1609 if (!CI->isIndirectCall())
1610 deduceOperandElementTypeCalledFunction(CI, Ops, KnownElemTy, Incomplete);
1611 else if (HaveFunPtrs)
1612 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy,
1613 IsPostprocessing);
1614 }
1615
1616 // There is no enough info to deduce types or all is valid.
1617 if (!KnownElemTy || Ops.size() == 0)
1618 return;
1619
1620 IRBuilder<> B(*CurrF->getParent());
1621 for (auto &OpIt : Ops) {
1622 Value *Op = OpIt.first;
1623 if (AskOps && !AskOps->contains(Ptr: Op))
1624 continue;
1625 Type *AskTy = nullptr;
1626 CallInst *AskCI = nullptr;
1627 if (IsPostprocessing && AskOps) {
1628 AskTy = GR->findDeducedElementType(Val: Op);
1629 AskCI = GR->findAssignPtrTypeInstr(Val: Op);
1630 assert(AskTy && AskCI);
1631 }
1632 Type *Ty = AskTy ? AskTy : GR->findDeducedElementType(Val: Op);
1633 if (Ty == KnownElemTy)
1634 continue;
1635 Value *OpTyVal = getNormalizedPoisonValue(Ty: KnownElemTy, CanUseAnyVectorRank);
1636 Type *OpTy = Op->getType();
1637 // Do not let a non-pointer element type clobber an already-deduced pointer
1638 // element type for the same operand.
1639 bool WouldClobberPtrWithNonPtr = Ty && isPointerTyOrWrapper(Ty) &&
1640 !isPointerTyOrWrapper(Ty: KnownElemTy) &&
1641 tracesToPointerAlloca(V: Op);
1642 if (Op->hasUseList() && !WouldClobberPtrWithNonPtr &&
1643 (!Ty || AskTy || isUntypedPointerTy(T: Ty) || isTodoType(Op))) {
1644 Type *PrevElemTy = GR->findDeducedElementType(Val: Op);
1645 GR->addDeducedElementType(
1646 Val: Op, Ty: normalizeType(Ty: KnownElemTy, CanUseAnyVectorRank));
1647 // check if KnownElemTy is complete
1648 if (!Incomplete)
1649 eraseTodoType(Op);
1650 else if (!IsPostprocessing)
1651 insertTodoType(Op);
1652 // check if there is existing Intrinsic::spv_assign_ptr_type instruction
1653 CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Val: Op);
1654 if (AssignCI == nullptr) {
1655 Instruction *User = dyn_cast<Instruction>(Val: Op->use_begin()->get());
1656 setInsertPointSkippingPhis(B, I: User ? User->getNextNode() : I);
1657 CallInst *CI =
1658 buildIntrWithMD(IntrID: Intrinsic::spv_assign_ptr_type, Types: {OpTy}, Arg: OpTyVal, Arg2: Op,
1659 Imms: {B.getInt32(C: getPointerAddressSpace(T: OpTy))}, B);
1660 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: CI);
1661 } else {
1662 GR->updateAssignType(AssignCI, Arg: Op, OfType: OpTyVal);
1663 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
1664 std::make_pair(x&: I, y&: Op)};
1665 propagateElemTypeRec(Op, PtrElemTy: KnownElemTy, CastElemTy: PrevElemTy, VisitedSubst);
1666 // A PHI or select validated its operands against its previous element
1667 // type. Null constants have no use list and are not revisited by
1668 // postprocessing, so validate the operands against the new type.
1669 if (auto *OpI = dyn_cast<Instruction>(Val: Op);
1670 OpI && (isa<PHINode>(Val: OpI) || isa<SelectInst>(Val: OpI)) &&
1671 TypeValidated.contains(Ptr: OpI))
1672 RetypedToRevalidate.push_back(Elt: OpI);
1673 }
1674 } else {
1675 eraseTodoType(Op);
1676 CallInst *PtrCastI =
1677 buildSpvPtrcast(F: I->getParent()->getParent(), Op, ElemTy: KnownElemTy);
1678 if (OpIt.second == std::numeric_limits<unsigned>::max())
1679 dyn_cast<CallInst>(Val: I)->setCalledOperand(PtrCastI);
1680 else
1681 I->setOperand(i: OpIt.second, Val: PtrCastI);
1682 }
1683 }
1684 TypeValidated.insert(Ptr: I);
1685
1686 // Revalidating one PHI can retype the next one in a chain. Use a worklist
1687 // rather than recursion, as such chains can be long.
1688 if (RevalidatingRetyped)
1689 return;
1690 RevalidatingRetyped = true;
1691 while (!RetypedToRevalidate.empty())
1692 deduceOperandElementType(I: RetypedToRevalidate.pop_back_val(), IncompleteRets: nullptr,
1693 AskOps: nullptr, IsPostprocessing);
1694 RevalidatingRetyped = false;
1695}
1696
1697void SPIRVEmitIntrinsicsImpl::replaceMemInstrUses(Instruction *Old,
1698 Instruction *New,
1699 IRBuilder<> &B) {
1700 while (!Old->user_empty()) {
1701 auto *U = Old->user_back();
1702 if (isAssignTypeInstr(I: U)) {
1703 B.SetInsertPoint(U);
1704 SmallVector<Value *, 2> Args = {New, U->getOperand(i: 1)};
1705 CallInst *AssignCI = B.CreateIntrinsicWithoutFolding(
1706 ID: Intrinsic::spv_assign_type, OverloadTypes: {New->getType()}, Args);
1707 GR->addAssignPtrTypeInstr(Val: New, AssignPtrTyCI: AssignCI);
1708 U->eraseFromParent();
1709 } else if (isMemInstrToReplace(I: U) || isa<ReturnInst>(Val: U) ||
1710 isa<CallInst>(Val: U)) {
1711 U->replaceUsesOfWith(From: Old, To: New);
1712 // For a `llvm.spv.abort` call whose composite message argument was
1713 // rewritten to a value-id (i32), also retarget the call to a matching
1714 // intrinsic declaration so the IR verifier is satisfied. The SPIR-V
1715 // type of the value is tracked via the GlobalRegistry, so the selector
1716 // still emits OpAbortKHR with the original composite type.
1717 if (auto *CI = dyn_cast<CallInst>(Val: U);
1718 CI && CI->getIntrinsicID() == Intrinsic::spv_abort) {
1719 Type *NewArgTy = New->getType();
1720 Type *ExpectedArgTy = CI->getFunctionType()->getParamType(i: 0);
1721 if (NewArgTy != ExpectedArgTy) {
1722 Module *M = CI->getModule();
1723 Function *NewF = Intrinsic::getOrInsertDeclaration(
1724 M, id: Intrinsic::spv_abort, OverloadTys: {NewArgTy});
1725 CI->setCalledFunction(NewF);
1726 }
1727 }
1728 } else if (isa<PHINode>(Val: U) || isa<SelectInst>(Val: U) || isa<FreezeInst>(Val: U)) {
1729 // Aggregate-typed PHIs, selects and freezes have already been mutated to
1730 // the i32 value-id type up front in runOnFunction, so only the operand
1731 // needs replacing here; their extractvalue users are lowered to
1732 // spv_extractv by visitExtractValueInst.
1733 assert(U->getType() == New->getType() &&
1734 "aggregate PHI/select/freeze should have been mutated to value-id "
1735 "type");
1736 U->replaceUsesOfWith(From: Old, To: New);
1737 } else {
1738 llvm_unreachable("illegal aggregate intrinsic user");
1739 }
1740 }
1741 New->copyMetadata(SrcInst: *Old);
1742 GR->removeAssignPtrTypeInstr(OldVal: Old); // Old's address could be re-used.
1743 TypeValidated.erase(Ptr: Old);
1744 Old->eraseFromParent();
1745}
1746
1747// Lower a poison or undef Op to its placeholder intrinsic.
1748Value *SPIRVEmitIntrinsicsImpl::lowerUndefOrPoison(Value *Op, IRBuilder<> &B,
1749 bool HasPoisonExt) {
1750 auto *UV = dyn_cast<UndefValue>(Val: Op);
1751 if (!UV)
1752 return nullptr;
1753
1754 bool AsPoison = HasPoisonExt && isa<PoisonValue>(Val: UV);
1755 if (isa<PoisonValue>(Val: UV) && !HasPoisonExt)
1756 LLVM_DEBUG(dbgs() << "SPV_KHR_poison_freeze is not enabled. Poison is "
1757 "lowered as undef\n");
1758
1759 Intrinsic::ID IID = AsPoison ? Intrinsic::spv_poison : Intrinsic::spv_undef;
1760 Type *Ty = UV->getType();
1761
1762 // Aggregates use an i32-result placeholder with the real type kept in
1763 // AggrConstTypes and scalar poison uses a type-overloaded one.
1764 if (Ty->isAggregateType()) {
1765 auto *Call =
1766 AsPoison ? B.CreateIntrinsicWithoutFolding(ID: IID, OverloadTypes: {B.getInt32Ty()}, Args: {})
1767 : B.CreateIntrinsicWithoutFolding(ID: IID, Args: {});
1768 AggrConsts[Call] = UV;
1769 AggrConstTypes[Call] = Ty;
1770 return Call;
1771 }
1772
1773 if (AsPoison)
1774 return B.CreateIntrinsic(ID: IID, OverloadTypes: {Ty}, Args: {});
1775 return nullptr;
1776}
1777
1778// Replace aggregate undef or poison operands and extension-enabled scalar
1779// poison operands with placeholder intrinsics. Scalar undef is left as is. See
1780// lowerUndefOrPoison.
1781void SPIRVEmitIntrinsicsImpl::preprocessUndefsAndPoisons(IRBuilder<> &B) {
1782 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*CurrF);
1783 bool HasPoisonExt =
1784 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
1785
1786 SmallVector<Instruction *, 16> Insts;
1787 for (auto &I : instructions(F: CurrF))
1788 Insts.push_back(Elt: &I);
1789
1790 for (Instruction *I : Insts) {
1791 bool BPrepared = false;
1792 auto *Phi = dyn_cast<PHINode>(Val: I);
1793 for (unsigned Idx = 0; Idx < I->getNumOperands(); ++Idx) {
1794 Value *Op = I->getOperand(i: Idx);
1795 if (!isa<UndefValue>(Val: Op) || Op->getType()->isMetadataTy())
1796 continue;
1797 bool IsScalar = !Op->getType()->isAggregateType();
1798 bool AsPoison = HasPoisonExt && isa<PoisonValue>(Val: Op);
1799 // Scalar undef or extensionless scalar poison is directly translatable.
1800 if (IsScalar && !AsPoison)
1801 continue;
1802 // Scalar poison in a phi materializes in the incoming block. Everything
1803 // else materializes right before I.
1804 if (IsScalar && Phi)
1805 B.SetInsertPoint(Phi->getIncomingBlock(i: Idx)->getTerminator());
1806 else if (!BPrepared) {
1807 setInsertPointSkippingPhis(B, I);
1808 BPrepared = true;
1809 }
1810 if (Value *Repl = lowerUndefOrPoison(Op, B, HasPoisonExt))
1811 I->setOperand(i: Idx, Val: Repl);
1812 }
1813 }
1814}
1815
1816// Simplify addrspacecast(null) instructions to ConstantPointerNull of the
1817// target type. Casting null always yields null, and this avoids SPIR-V
1818// lowering issues where the null gets typed as an integer instead of a
1819// pointer.
1820void SPIRVEmitIntrinsicsImpl::simplifyNullAddrSpaceCasts() {
1821 for (Instruction &I : make_early_inc_range(Range: instructions(F: CurrF)))
1822 if (auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: &I))
1823 if (isa<ConstantPointerNull>(Val: ASC->getPointerOperand())) {
1824 ASC->replaceAllUsesWith(
1825 V: ConstantPointerNull::get(T: cast<PointerType>(Val: ASC->getType())));
1826 ASC->eraseFromParent();
1827 }
1828}
1829
1830// True for an aggregate value the legalizer splits into a multi-result op
1831// (with.overflow -> G_UADDO, frexp/sincos/modf -> G_FFREXP/...). These keep a
1832// genuine multi-register result; all other aggregates become a single value-id.
1833static bool isMultiRegisterAggregate(Value *V) {
1834 if (!V->getType()->isAggregateType())
1835 return false;
1836 return isa<IntrinsicInst>(Val: V) && !isSpvIntrinsic(Arg: V);
1837}
1838
1839// True for an aggregate PHI/select/freeze, which is lowered to a single
1840// value-id.
1841static bool isAggregateValueIdInstr(const Instruction &I) {
1842 return (isa<PHINode>(Val: I) || isa<SelectInst>(Val: I) || isa<FreezeInst>(Val: I)) &&
1843 I.getType()->isAggregateType();
1844}
1845
1846// Give each multi-register aggregate arm of an aggregate PHI/select/freeze a
1847// single value-id by reassembling it with extractvalue + insertvalue, so the
1848// arm matches the result once it is mutated to a value-id.
1849void SPIRVEmitIntrinsicsImpl::insertCompositeAggregateArms(Instruction *I,
1850 IRBuilder<> &B) {
1851 auto *Phi = dyn_cast<PHINode>(Val: I);
1852 for (Use &U : I->operands()) {
1853 Value *Op = U.get();
1854 if (!isMultiRegisterAggregate(V: Op))
1855 continue;
1856 // A PHI arm materializes in its incoming block, everything else after the
1857 // producer.
1858 if (Phi)
1859 B.SetInsertPoint(Phi->getIncomingBlock(U)->getTerminator());
1860 else
1861 setInsertPointAfterDef(B, I: cast<Instruction>(Val: Op));
1862 auto *AggrTy = cast<StructType>(Val: Op->getType());
1863 Value *Composite = PoisonValue::get(T: AggrTy);
1864 for (unsigned Idx = 0, E = AggrTy->getNumElements(); Idx != E; ++Idx) {
1865 Value *Field = B.CreateExtractValue(Agg: Op, Idxs: Idx);
1866 Composite = B.CreateInsertValue(Agg: Composite, Val: Field, Idxs: Idx);
1867 }
1868 U.set(Composite);
1869 }
1870}
1871
1872void SPIRVEmitIntrinsicsImpl::preprocessCompositeConstants(IRBuilder<> &B) {
1873 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*CurrF);
1874 bool HasPoisonExt =
1875 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
1876 std::queue<Instruction *> Worklist;
1877 for (auto &I : instructions(F: CurrF))
1878 Worklist.push(x: &I);
1879
1880 while (!Worklist.empty()) {
1881 auto *I = Worklist.front();
1882 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
1883 assert(I);
1884 bool KeepInst = false;
1885 for (const auto &Op : I->operands()) {
1886 Constant *AggrConst = nullptr;
1887 Type *ResTy = nullptr;
1888 if (auto *COp = dyn_cast<ConstantVector>(Val: Op)) {
1889 AggrConst = COp;
1890 ResTy = COp->getType();
1891 } else if (auto *COp = dyn_cast<ConstantArray>(Val: Op)) {
1892 AggrConst = COp;
1893 ResTy = B.getInt32Ty();
1894 } else if (auto *COp = dyn_cast<ConstantStruct>(Val: Op)) {
1895 AggrConst = COp;
1896 ResTy = B.getInt32Ty();
1897 } else if (auto *COp = dyn_cast<ConstantDataArray>(Val: Op)) {
1898 AggrConst = COp;
1899 ResTy = B.getInt32Ty();
1900 } else if (auto *COp = dyn_cast<ConstantAggregateZero>(Val: Op)) {
1901 AggrConst = COp;
1902 ResTy = Op->getType()->isVectorTy() ? COp->getType() : B.getInt32Ty();
1903 }
1904 if (AggrConst) {
1905 auto PrepareInsert = [&]() {
1906 if (BPrepared)
1907 return;
1908 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
1909 : B.SetInsertPoint(I);
1910 BPrepared = true;
1911 };
1912 SmallVector<Value *> Args;
1913 if (auto *COp = dyn_cast<ConstantDataSequential>(Val: Op))
1914 for (unsigned i = 0; i < COp->getNumElements(); ++i)
1915 Args.push_back(Elt: COp->getElementAsConstant(i));
1916 else
1917 for (Value *Op : AggrConst->operands()) {
1918 // Simplify addrspacecast(null) to null in the target address space
1919 // so that null pointers get the correct pointer type when lowered.
1920 if (auto *CE = dyn_cast<ConstantExpr>(Val: Op);
1921 CE && CE->getOpcode() == Instruction::AddrSpaceCast &&
1922 isa<ConstantPointerNull>(Val: CE->getOperand(i_nocapture: 0)))
1923 Op = ConstantPointerNull::get(T: cast<PointerType>(Val: CE->getType()));
1924 // Undef or poison nested in a constant aggregate is not a direct
1925 // instruction operand, so preprocessUndefsAndPoisons() misses it.
1926 // An unlowered aggregate one would reach IRTranslator as an
1927 // untranslatable spv_const_composite operand.
1928 if (isa<UndefValue>(Val: Op)) {
1929 PrepareInsert();
1930 if (Value *Repl = lowerUndefOrPoison(Op, B, HasPoisonExt))
1931 Op = Repl;
1932 }
1933 Args.push_back(Elt: Op);
1934 }
1935 PrepareInsert();
1936 auto *CI = B.CreateIntrinsicWithoutFolding(
1937 ID: Intrinsic::spv_const_composite, OverloadTypes: {ResTy}, Args: {Args});
1938 Worklist.push(x: CI);
1939 I->replaceUsesOfWith(From: Op, To: CI);
1940 KeepInst = true;
1941 AggrConsts[CI] = AggrConst;
1942 AggrConstTypes[CI] = deduceNestedTypeHelper(U: AggrConst, UnknownElemTypeI8: false);
1943 }
1944 }
1945 if (!KeepInst)
1946 Worklist.pop();
1947 }
1948}
1949
1950static void createDecorationIntrinsic(Instruction *I, MDNode *Node,
1951 IRBuilder<> &B) {
1952 LLVMContext &Ctx = I->getContext();
1953 setInsertPointAfterDef(B, I);
1954 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
1955 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: {Node}))});
1956}
1957
1958static void createRoundingModeDecoration(Instruction *I,
1959 unsigned RoundingModeDeco,
1960 IRBuilder<> &B) {
1961 LLVMContext &Ctx = I->getContext();
1962 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1963 MDNode *RoundingModeNode = MDNode::get(
1964 Context&: Ctx,
1965 MDs: {ConstantAsMetadata::get(
1966 C: ConstantInt::get(Ty: Int32Ty, V: SPIRV::Decoration::FPRoundingMode)),
1967 ConstantAsMetadata::get(C: ConstantInt::get(Ty: Int32Ty, V: RoundingModeDeco))});
1968 createDecorationIntrinsic(I, Node: RoundingModeNode, B);
1969}
1970
1971static void createSaturatedConversionDecoration(Instruction *I,
1972 IRBuilder<> &B) {
1973 LLVMContext &Ctx = I->getContext();
1974 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
1975 MDNode *SaturatedConversionNode =
1976 MDNode::get(Context&: Ctx, MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1977 Ty: Int32Ty, V: SPIRV::Decoration::SaturatedConversion))});
1978 createDecorationIntrinsic(I, Node: SaturatedConversionNode, B);
1979}
1980
1981static void addSaturatedDecorationToIntrinsic(Instruction *I, IRBuilder<> &B) {
1982 if (match(V: I, P: m_AnyIntrinsic<Intrinsic::fptosi_sat, Intrinsic::fptoui_sat>()))
1983 createSaturatedConversionDecoration(I, B);
1984}
1985
1986Instruction *SPIRVEmitIntrinsicsImpl::visitCallInst(CallInst &Call) {
1987 if (!Call.isInlineAsm())
1988 return &Call;
1989
1990 LLVMContext &Ctx = CurrF->getContext();
1991 // TODO: this does not retain elementtype info for memory constraints, which
1992 // in turn means that we lower them into pointers to i8, rather than
1993 // pointers to elementtype; this can be fixed during reverse translation
1994 // but we should correct it here, possibly by tweaking the function
1995 // type to take TypedPointerType args.
1996 Constant *TyC = UndefValue::get(T: SPIRV::getOriginalFunctionType(CB: Call));
1997 MDString *ConstraintString =
1998 MDString::get(Context&: Ctx, Str: SPIRV::getOriginalAsmConstraints(CB: Call));
1999 SmallVector<Value *> Args = {
2000 buildMD(Arg: TyC),
2001 MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs: ConstraintString))};
2002 for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++)
2003 Args.push_back(Elt: Call.getArgOperand(i: OpIdx));
2004
2005 IRBuilder<> B(Call.getParent());
2006 B.SetInsertPoint(&Call);
2007 B.CreateIntrinsic(ID: Intrinsic::spv_inline_asm, Args: {Args});
2008 return &Call;
2009}
2010
2011// Use a tip about rounding mode to create a decoration.
2012void SPIRVEmitIntrinsicsImpl::useRoundingMode(ConstrainedFPIntrinsic *FPI,
2013 IRBuilder<> &B) {
2014 std::optional<RoundingMode> RM = FPI->getRoundingMode();
2015 if (!RM.has_value())
2016 return;
2017 unsigned RoundingModeDeco = std::numeric_limits<unsigned>::max();
2018 switch (RM.value()) {
2019 default:
2020 // ignore unknown rounding modes
2021 break;
2022 case RoundingMode::NearestTiesToEven:
2023 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTE;
2024 break;
2025 case RoundingMode::TowardNegative:
2026 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTN;
2027 break;
2028 case RoundingMode::TowardPositive:
2029 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTP;
2030 break;
2031 case RoundingMode::TowardZero:
2032 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTZ;
2033 break;
2034 case RoundingMode::Dynamic:
2035 case RoundingMode::NearestTiesToAway:
2036 // TODO: check if supported
2037 break;
2038 }
2039 if (RoundingModeDeco == std::numeric_limits<unsigned>::max())
2040 return;
2041 // Convert the tip about rounding mode into a decoration record.
2042 createRoundingModeDecoration(I: FPI, RoundingModeDeco, B);
2043}
2044
2045Instruction *SPIRVEmitIntrinsicsImpl::visitSwitchInst(SwitchInst &I) {
2046 BasicBlock *ParentBB = I.getParent();
2047 Function *F = ParentBB->getParent();
2048 IRBuilder<> B(ParentBB);
2049 B.SetInsertPoint(&I);
2050 SmallVector<Value *, 4> Args;
2051 SmallVector<BasicBlock *> BBCases;
2052 Args.push_back(Elt: I.getCondition());
2053 BBCases.push_back(Elt: I.getDefaultDest());
2054 Args.push_back(Elt: BlockAddress::get(F, BB: I.getDefaultDest()));
2055 for (auto &Case : I.cases()) {
2056 Args.push_back(Elt: Case.getCaseValue());
2057 BBCases.push_back(Elt: Case.getCaseSuccessor());
2058 Args.push_back(Elt: BlockAddress::get(F, BB: Case.getCaseSuccessor()));
2059 }
2060 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
2061 ID: Intrinsic::spv_switch, OverloadTypes: {I.getOperand(i_nocapture: 0)->getType()}, Args: {Args});
2062 // remove switch to avoid its unneeded and undesirable unwrap into branches
2063 // and conditions
2064 replaceAllUsesWith(Src: &I, Dest: NewI);
2065 I.eraseFromParent();
2066 // insert artificial and temporary instruction to preserve valid CFG,
2067 // it will be removed after IR translation pass
2068 B.SetInsertPoint(ParentBB);
2069 IndirectBrInst *BrI = B.CreateIndirectBr(
2070 Addr: Constant::getNullValue(Ty: PointerType::getUnqual(C&: ParentBB->getContext())),
2071 NumDests: BBCases.size());
2072 for (BasicBlock *BBCase : BBCases)
2073 BrI->addDestination(Dest: BBCase);
2074 return BrI;
2075}
2076
2077static bool isFirstIndexZero(const GetElementPtrInst *GEP) {
2078 return GEP->getNumIndices() > 0 && match(V: GEP->getOperand(i_nocapture: 1), P: m_Zero());
2079}
2080
2081Instruction *SPIRVEmitIntrinsicsImpl::visitIntrinsicInst(IntrinsicInst &I) {
2082 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
2083 if (!SGEP)
2084 return &I;
2085
2086 IRBuilder<> B(I.getParent());
2087 B.SetInsertPoint(&I);
2088 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
2089 SmallVector<Value *, 4> Args;
2090 Args.push_back(/* inBounds= */ Elt: B.getInt1(V: true));
2091 Args.push_back(Elt: I.getOperand(i_nocapture: 0));
2092 Args.push_back(/* zero index */ Elt: B.getInt32(C: 0));
2093 for (unsigned J = 0; J < SGEP->getNumIndices(); ++J)
2094 Args.push_back(Elt: SGEP->getIndexOperand(Index: J));
2095
2096 Instruction *NewI =
2097 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: Types, Args);
2098 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2099 return NewI;
2100}
2101
2102Instruction *
2103SPIRVEmitIntrinsicsImpl::visitGetElementPtrInst(GetElementPtrInst &I) {
2104 IRBuilder<> B(I.getParent());
2105 B.SetInsertPoint(&I);
2106
2107 // OpPtrAccessChain requires a scalar pointer result; scalarize per-lane
2108 // GEPs that return <N x ptr> and rebuild the vector via insertelement.
2109 if (auto *RetVTy = dyn_cast<FixedVectorType>(Val: I.getType())) {
2110 unsigned N = RetVTy->getNumElements();
2111 Value *PtrOp = I.getPointerOperand();
2112 bool PtrIsVec = isa<VectorType>(Val: PtrOp->getType());
2113 Type *ResultPtrTy = RetVTy->getElementType();
2114 Type *ScalarPtrTy = PtrOp->getType()->getScalarType();
2115 SmallVector<Type *, 2> GepTypes = {ResultPtrTy, ScalarPtrTy};
2116 Value *InBounds = B.getInt1(V: I.isInBounds());
2117 Type *LanePointeeTy = getGEPType(Ref: &I);
2118 if (!LanePointeeTy)
2119 return nullptr;
2120 Type *SrcElemTy = I.getSourceElementType();
2121
2122 // Pin the lane pointee type on the vector operand and on each extracted
2123 // lane so the prelegalizer wraps them as OpTypeVector/OpTypePointer of
2124 // the right element type instead of defaulting to i8.
2125 if (PtrIsVec)
2126 GR->buildAssignPtr(B, ElemTy: SrcElemTy, Arg: PtrOp);
2127
2128 Value *VecResult = PoisonValue::get(T: RetVTy);
2129 for (unsigned Lane = 0; Lane < N; ++Lane) {
2130 Value *LaneIdx = B.getInt32(C: Lane);
2131 Value *ScalarPtr = PtrOp;
2132 if (PtrIsVec) {
2133 SmallVector<Type *, 3> ExtractTypes = {ScalarPtrTy, PtrOp->getType(),
2134 LaneIdx->getType()};
2135 ScalarPtr = B.CreateIntrinsic(ID: Intrinsic::spv_extractelt, OverloadTypes: {ExtractTypes},
2136 Args: {PtrOp, LaneIdx});
2137 GR->buildAssignPtr(B, ElemTy: SrcElemTy, Arg: ScalarPtr);
2138 }
2139 SmallVector<Value *, 4> Args;
2140 Args.push_back(Elt: InBounds);
2141 Args.push_back(Elt: ScalarPtr);
2142 for (Value *Idx : I.indices()) {
2143 if (isa<VectorType>(Val: Idx->getType())) {
2144 // We cannot use the builder here as for splat-ed / constant vectors
2145 // it will fold to the scalar, and then it becomes impossible to
2146 // retrieve / retain the vectorness.
2147 auto *EI =
2148 ExtractElementInst::Create(Vec: Idx, Idx: LaneIdx, NameStr: "", InsertBefore: B.GetInsertPoint());
2149 if (isVector1(Ty: Idx->getType())) // IRTranslator clobbers <1 x T>.
2150 Args.push_back(Elt: visitExtractElementInst(I&: *EI));
2151 else
2152 Args.push_back(Elt: EI);
2153 } else {
2154 Args.push_back(Elt: Idx);
2155 }
2156 }
2157 Value *ScalarGep = B.CreateIntrinsic(ID: Intrinsic::spv_gep, OverloadTypes: GepTypes, Args);
2158 GR->buildAssignPtr(B, ElemTy: LanePointeeTy, Arg: ScalarGep);
2159 VecResult = B.CreateInsertElement(Vec: VecResult, NewElt: ScalarGep, Idx: LaneIdx);
2160 }
2161
2162 auto *NewI = cast<Instruction>(Val: VecResult);
2163 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2164
2165 if (CallInst *Old = GR->findAssignPtrTypeInstr(Val: NewI)) {
2166 Old->eraseFromParent();
2167 GR->addAssignPtrTypeInstr(Val: NewI, AssignPtrTyCI: nullptr);
2168 }
2169 setInsertPointAfterDef(B, I: NewI);
2170 GR->buildAssignPtr(B, ElemTy: LanePointeeTy, Arg: NewI);
2171
2172 return NewI;
2173 }
2174
2175 if (TM.getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: &I)) {
2176 // Logical SPIR-V cannot use the OpPtrAccessChain instruction. If the first
2177 // index of the GEP is not 0, then we need to try to adjust it.
2178 //
2179 // If the GEP is doing byte addressing, try to rebuild the full access chain
2180 // from the type of the pointer.
2181 if (getByteAddressingMultiplier(Ty: I.getSourceElementType())) {
2182 return buildLogicalAccessChainFromGEP(GEP&: I);
2183 }
2184
2185 // Look for the array-to-pointer decay. If this is the pattern
2186 // we can adjust the types, and prepend a 0 to the indices.
2187 Value *PtrOp = I.getPointerOperand();
2188 Type *SrcElemTy = I.getSourceElementType();
2189 Type *DeducedPointeeTy = deduceElementType(I: PtrOp, UnknownElemTypeI8: true);
2190
2191 if (auto *ArrTy = dyn_cast<ArrayType>(Val: DeducedPointeeTy)) {
2192 if (ArrTy->getElementType() == SrcElemTy) {
2193 SmallVector<Value *> NewIndices;
2194 Type *FirstIdxType = I.getOperand(i_nocapture: 1)->getType();
2195 NewIndices.push_back(Elt: ConstantInt::get(Ty: FirstIdxType, V: 0));
2196 for (Value *Idx : I.indices())
2197 NewIndices.push_back(Elt: Idx);
2198
2199 SmallVector<Type *, 2> Types = {I.getType(), I.getPointerOperandType()};
2200 SmallVector<Value *, 4> Args;
2201 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
2202 Args.push_back(Elt: I.getPointerOperand());
2203 Args.append(in_start: NewIndices.begin(), in_end: NewIndices.end());
2204
2205 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep,
2206 OverloadTypes: {Types}, Args: {Args});
2207 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2208 return NewI;
2209 }
2210 }
2211 }
2212
2213 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType()};
2214 SmallVector<Value *, 4> Args;
2215 Args.push_back(Elt: B.getInt1(V: I.isInBounds()));
2216 llvm::append_range(C&: Args, R: I.operands());
2217 Instruction *NewI =
2218 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_gep, OverloadTypes: {Types}, Args: {Args});
2219 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2220 return NewI;
2221}
2222
2223Instruction *SPIRVEmitIntrinsicsImpl::visitBitCastInst(BitCastInst &I) {
2224 IRBuilder<> B(I.getParent());
2225 B.SetInsertPoint(&I);
2226 Value *Source = I.getOperand(i_nocapture: 0);
2227
2228 // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of
2229 // varying element types. In case of IR coming from older versions of LLVM
2230 // such bitcasts do not provide sufficient information, should be just skipped
2231 // here, and handled in insertPtrCastOrAssignTypeInstr.
2232 if (isPointerTy(T: I.getType())) {
2233 replaceAllUsesWith(Src: &I, Dest: Source);
2234 I.eraseFromParent();
2235 return nullptr;
2236 }
2237
2238 SmallVector<Type *, 2> Types = {I.getType(), Source->getType()};
2239 SmallVector<Value *> Args(I.op_begin(), I.op_end());
2240 Instruction *NewI =
2241 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_bitcast, OverloadTypes: {Types}, Args: {Args});
2242 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2243 return NewI;
2244}
2245
2246void SPIRVEmitIntrinsicsImpl::insertAssignPtrTypeTargetExt(
2247 TargetExtType *AssignedType, Value *V, IRBuilder<> &B) {
2248 Type *VTy = V->getType();
2249
2250 // A couple of sanity checks.
2251 assert((isPointerTy(VTy)) && "Expect a pointer type!");
2252 if (Type *ElemTy = getPointeeType(Ty: VTy))
2253 if (ElemTy != AssignedType)
2254 report_fatal_error(reason: "Unexpected pointer element type!");
2255
2256 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: V);
2257 if (!AssignCI) {
2258 GR->buildAssignType(B, Ty: AssignedType, Arg: V, CanUseAnyVectorRank);
2259 return;
2260 }
2261
2262 Type *CurrentType =
2263 dyn_cast<ConstantAsMetadata>(
2264 Val: cast<MetadataAsValue>(Val: AssignCI->getOperand(i_nocapture: 1))->getMetadata())
2265 ->getType();
2266 if (CurrentType == AssignedType)
2267 return;
2268
2269 // Builtin types cannot be redeclared or casted.
2270 if (CurrentType->isTargetExtTy())
2271 report_fatal_error(reason: "Type mismatch " + CurrentType->getTargetExtName() +
2272 "/" + AssignedType->getTargetExtName() +
2273 " for value " + V->getName(),
2274 gen_crash_diag: false);
2275
2276 // Our previous guess about the type seems to be wrong, let's update
2277 // inferred type according to a new, more precise type information.
2278 GR->updateAssignType(
2279 AssignCI, Arg: V, OfType: getNormalizedPoisonValue(Ty: AssignedType, CanUseAnyVectorRank));
2280}
2281
2282void SPIRVEmitIntrinsicsImpl::replacePointerOperandWithPtrCast(
2283 Instruction *I, Value *Pointer, Type *ExpectedElementType,
2284 unsigned OperandToReplace, IRBuilder<> &B) {
2285 TypeValidated.insert(Ptr: I);
2286
2287 // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType
2288 Type *PointerElemTy = deduceElementTypeHelper(I: Pointer, UnknownElemTypeI8: false);
2289 if (PointerElemTy == ExpectedElementType ||
2290 isEquivalentTypes(Ty1: PointerElemTy, Ty2: ExpectedElementType))
2291 return;
2292
2293 setInsertPointSkippingPhis(B, I);
2294 Value *ExpectedElementVal =
2295 getNormalizedPoisonValue(Ty: ExpectedElementType, CanUseAnyVectorRank);
2296 MetadataAsValue *VMD = buildMD(Arg: ExpectedElementVal);
2297 unsigned AddressSpace = getPointerAddressSpace(T: Pointer->getType());
2298 bool FirstPtrCastOrAssignPtrType = true;
2299
2300 // Do not emit new spv_ptrcast if equivalent one already exists or when
2301 // spv_assign_ptr_type already targets this pointer with the same element
2302 // type.
2303 if (Pointer->hasUseList()) {
2304 for (auto User : Pointer->users()) {
2305 auto *II = dyn_cast<IntrinsicInst>(Val: User);
2306 if (!II ||
2307 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type &&
2308 II->getIntrinsicID() != Intrinsic::spv_ptrcast) ||
2309 II->getOperand(i_nocapture: 0) != Pointer)
2310 continue;
2311
2312 // There is some spv_ptrcast/spv_assign_ptr_type already targeting this
2313 // pointer.
2314 FirstPtrCastOrAssignPtrType = false;
2315 if (II->getOperand(i_nocapture: 1) != VMD ||
2316 dyn_cast<ConstantInt>(Val: II->getOperand(i_nocapture: 2))->getSExtValue() !=
2317 AddressSpace)
2318 continue;
2319
2320 // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the
2321 // same element type and address space.
2322 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast)
2323 return;
2324
2325 // This must be a spv_ptrcast, do not emit new if this one has the same BB
2326 // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type.
2327 if (II->getParent() != I->getParent())
2328 continue;
2329
2330 I->setOperand(i: OperandToReplace, Val: II);
2331 return;
2332 }
2333 }
2334
2335 // Never replace an already-deduced pointer element type with a non-pointer
2336 // one. The conflicting use comes from a mis-deduced expected type. Leave the
2337 // operand untouched rather than emitting a ptrcast that re-introduces the
2338 // collapsed type at the use site.
2339 if (PointerElemTy && isPointerTyOrWrapper(Ty: PointerElemTy) &&
2340 !isPointerTyOrWrapper(Ty: ExpectedElementType) &&
2341 tracesToPointerAlloca(V: Pointer))
2342 return;
2343
2344 if (isa<Instruction>(Val: Pointer) || isa<Argument>(Val: Pointer)) {
2345 if (FirstPtrCastOrAssignPtrType) {
2346 // If this would be the first spv_ptrcast, do not emit spv_ptrcast and
2347 // emit spv_assign_ptr_type instead.
2348 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
2349 return;
2350 } else if (isTodoType(Op: Pointer)) {
2351 eraseTodoType(Op: Pointer);
2352 if (!isa<CallInst>(Val: Pointer) && !isaGEP(V: Pointer) &&
2353 !isa<AllocaInst>(Val: Pointer)) {
2354 // If this wouldn't be the first spv_ptrcast but existing type info is
2355 // uncomplete, update spv_assign_ptr_type arguments.
2356 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Pointer)) {
2357 Type *PrevElemTy = GR->findDeducedElementType(Val: Pointer);
2358 assert(PrevElemTy);
2359 DenseSet<std::pair<Value *, Value *>> VisitedSubst{
2360 std::make_pair(x&: I, y&: Pointer)};
2361 GR->updateAssignType(AssignCI, Arg: Pointer, OfType: ExpectedElementVal);
2362 propagateElemType(Op: Pointer, ElemTy: PrevElemTy, VisitedSubst);
2363 } else {
2364 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: Pointer);
2365 }
2366 return;
2367 }
2368 }
2369 }
2370
2371 // Emit spv_ptrcast
2372 SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()};
2373 SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(C: AddressSpace)};
2374 auto *PtrCastI =
2375 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
2376 I->setOperand(i: OperandToReplace, Val: PtrCastI);
2377 // We need to set up a pointee type for the newly created spv_ptrcast.
2378 GR->buildAssignPtr(B, ElemTy: ExpectedElementType, Arg: PtrCastI);
2379}
2380
2381void SPIRVEmitIntrinsicsImpl::insertPtrCastOrAssignTypeInstr(Instruction *I,
2382 IRBuilder<> &B) {
2383 // Handle basic instructions:
2384 StoreInst *SI = dyn_cast<StoreInst>(Val: I);
2385 if (IsKernelArgInt8(F: CurrF, SI)) {
2386 replacePointerOperandWithPtrCast(
2387 I, Pointer: SI->getValueOperand(), ExpectedElementType: IntegerType::getInt8Ty(C&: CurrF->getContext()),
2388 OperandToReplace: 0, B);
2389 }
2390 if (SI) {
2391 Value *Op = SI->getValueOperand();
2392 Value *Pointer = SI->getPointerOperand();
2393 Type *OpTy = Op->getType();
2394 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
2395 OpTy = restoreMutatedType(GR, I: OpI, Ty: OpTy);
2396 if (auto It = AggrConstTypes.find(Val: OpI); It != AggrConstTypes.end())
2397 OpTy = It->second;
2398 }
2399 if (OpTy == Op->getType())
2400 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: Op, UnknownElemTypeI8: false);
2401 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 1, B);
2402 return;
2403 }
2404 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
2405 Value *Pointer = LI->getPointerOperand();
2406 Type *OpTy = LI->getType();
2407 // If the loaded from pointer carries byref/byval, the dominant type is the
2408 // pointee type specified therein, and we should retain it.
2409 if (Argument *Arg;
2410 (Arg = dyn_cast<Argument>(Val: Pointer)) && hasPointeeTypeAttr(Arg))
2411 OpTy = getPointeeTypeByAttr(Arg);
2412
2413 if (auto *PtrTy = dyn_cast<PointerType>(Val: OpTy)) {
2414 if (Type *ElemTy = GR->findDeducedElementType(Val: LI)) {
2415 OpTy = getTypedPointerWrapper(ElemTy, AS: PtrTy->getAddressSpace());
2416 } else {
2417 Type *NewOpTy = OpTy;
2418 OpTy = deduceElementTypeByValueDeep(ValueTy: OpTy, Operand: LI, UnknownElemTypeI8: false);
2419 if (OpTy == NewOpTy)
2420 insertTodoType(Op: Pointer);
2421 }
2422 }
2423 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
2424 return;
2425 }
2426 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(Val: I)) {
2427 Value *Pointer = GEPI->getPointerOperand();
2428 Type *OpTy = nullptr;
2429
2430 // Logical SPIR-V is not allowed to use Op*PtrAccessChain instructions. If
2431 // the first index is 0, then we can trivially lower to OpAccessChain. If
2432 // not we need to try to rewrite the GEP. We avoid adding a pointer cast at
2433 // this time, and will rewrite the GEP when visiting it.
2434 if (TM.getSubtargetImpl()->isLogicalSPIRV() && !isFirstIndexZero(GEP: GEPI)) {
2435 return;
2436 }
2437
2438 // In all cases, fall back to the GEP type if type scavenging failed.
2439 if (!OpTy)
2440 OpTy = GEPI->getSourceElementType();
2441
2442 replacePointerOperandWithPtrCast(I, Pointer, ExpectedElementType: OpTy, OperandToReplace: 0, B);
2443 if (isNestedPointer(Ty: OpTy))
2444 insertTodoType(Op: Pointer);
2445 return;
2446 }
2447
2448 // An inserted pointer must have the type of its aggregate member.
2449 if (auto *IVI = dyn_cast<InsertValueInst>(Val: I)) {
2450 Value *Op = IVI->getInsertedValueOperand();
2451 if (isPointerTy(T: Op->getType()))
2452 replacePointerOperandWithPtrCast(
2453 I, Pointer: Op, ExpectedElementType: getMemberPointeeType(AggrTy: IVI->getType(), Indices: IVI->getIndices()),
2454 OperandToReplace: InsertValueInst::getInsertedValueOperandIndex(), B);
2455 return;
2456 }
2457
2458 // TODO: review and merge with existing logics:
2459 // Handle calls to builtins (non-intrinsics):
2460 CallInst *CI = dyn_cast<CallInst>(Val: I);
2461 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() ||
2462 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic())
2463 return;
2464
2465 // collect information about formal parameter types
2466 std::string DemangledName =
2467 getOclOrSpirvBuiltinDemangledName(Name: CI->getCalledFunction()->getName());
2468 Function *CalledF = CI->getCalledFunction();
2469 SmallVector<Type *, 4> CalledArgTys;
2470 bool HaveTypes = false;
2471 for (unsigned OpIdx = 0; OpIdx < CalledF->arg_size(); ++OpIdx) {
2472 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2473 Type *ArgType = CalledArg->getType();
2474 if (!isPointerTy(T: ArgType)) {
2475 CalledArgTys.push_back(Elt: nullptr);
2476 } else if (Type *ArgTypeElem = getPointeeType(Ty: ArgType)) {
2477 CalledArgTys.push_back(Elt: ArgTypeElem);
2478 HaveTypes = true;
2479 } else {
2480 Type *ElemTy = GR->findDeducedElementType(Val: CalledArg);
2481 if (!ElemTy && hasPointeeTypeAttr(Arg: CalledArg))
2482 ElemTy = getPointeeTypeByAttr(Arg: CalledArg);
2483 if (!ElemTy) {
2484 ElemTy = getPointeeTypeByCallInst(DemangledName, CalledF, OpIdx);
2485 if (ElemTy) {
2486 GR->addDeducedElementType(Val: CalledArg,
2487 Ty: normalizeType(Ty: ElemTy, CanUseAnyVectorRank));
2488 } else {
2489 for (User *U : CalledArg->users()) {
2490 if (Instruction *Inst = dyn_cast<Instruction>(Val: U)) {
2491 if ((ElemTy = deduceElementTypeHelper(I: Inst, UnknownElemTypeI8: false)) != nullptr)
2492 break;
2493 }
2494 }
2495 }
2496 }
2497 HaveTypes |= ElemTy != nullptr;
2498 CalledArgTys.push_back(Elt: ElemTy);
2499 }
2500 }
2501
2502 if (DemangledName.empty() && !HaveTypes)
2503 return;
2504
2505 auto Set =
2506 TM.getSubtarget<SPIRVSubtarget>(F: *CalledF).getPreferredInstructionSet();
2507 // Do not reparse rejected declaration types with the builtin parser.
2508 bool ParseBuiltinTypes =
2509 !DemangledName.empty() &&
2510 (!CalledF->isDeclaration() || SPIRV::isBuiltin(DemangledCall: DemangledName, Set));
2511 for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) {
2512 Value *ArgOperand = CI->getArgOperand(i: OpIdx);
2513 if (!isPointerTy(T: ArgOperand->getType()))
2514 continue;
2515
2516 // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs()
2517 if (!isa<Instruction>(Val: ArgOperand) && !isa<Argument>(Val: ArgOperand)) {
2518 // However, we may have assumptions about the formal argument's type and
2519 // may have a need to insert a ptr cast for the actual parameter of this
2520 // call.
2521 Argument *CalledArg = CalledF->getArg(i: OpIdx);
2522 if (!GR->findDeducedElementType(Val: CalledArg))
2523 continue;
2524 }
2525
2526 Type *ExpectedType =
2527 OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr;
2528 if (!ExpectedType && ParseBuiltinTypes)
2529 ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType(
2530 DemangledCall: DemangledName, ArgIdx: OpIdx, Ctx&: I->getContext());
2531 if (!ExpectedType || ExpectedType->isVoidTy())
2532 continue;
2533
2534 if (ExpectedType->isTargetExtTy() &&
2535 !isTypedPointerWrapper(ExtTy: cast<TargetExtType>(Val: ExpectedType)))
2536 insertAssignPtrTypeTargetExt(AssignedType: cast<TargetExtType>(Val: ExpectedType),
2537 V: ArgOperand, B);
2538 else
2539 replacePointerOperandWithPtrCast(I: CI, Pointer: ArgOperand, ExpectedElementType: ExpectedType, OperandToReplace: OpIdx, B);
2540 }
2541}
2542
2543Instruction *
2544SPIRVEmitIntrinsicsImpl::visitInsertElementInst(InsertElementInst &I) {
2545 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2546 // type in LLT and IRTranslator will replace it by the scalar.
2547 if (isVector1(Ty: I.getType()) && !CanUseAnyVectorRank)
2548 return &I;
2549
2550 SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(i_nocapture: 0)->getType(),
2551 I.getOperand(i_nocapture: 1)->getType(),
2552 I.getOperand(i_nocapture: 2)->getType()};
2553 IRBuilder<> B(I.getParent());
2554 B.SetInsertPoint(&I);
2555 SmallVector<Value *> Args(I.op_begin(), I.op_end());
2556 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_insertelt,
2557 OverloadTypes: {Types}, Args: {Args});
2558 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2559 return NewI;
2560}
2561
2562Instruction *
2563SPIRVEmitIntrinsicsImpl::visitExtractElementInst(ExtractElementInst &I) {
2564 // If it's a <1 x Type> vector type, don't modify it. It's not a legal vector
2565 // type in LLT and IRTranslator will replace it by the scalar.
2566 if (isVector1(Ty: I.getVectorOperandType()) && !CanUseAnyVectorRank)
2567 return &I;
2568
2569 IRBuilder<> B(I.getParent());
2570 B.SetInsertPoint(&I);
2571 SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(),
2572 I.getIndexOperand()->getType()};
2573 SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()};
2574 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_extractelt,
2575 OverloadTypes: {Types}, Args: {Args});
2576 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2577 return NewI;
2578}
2579
2580// Scalarize ptrtoint and inttoptr instructions operating on vectors of
2581// pointers, since SPIR-V does not support vectors of pointers without
2582// extensions. Run before preprocessCompositeConstants so that
2583// findScalarElement still sees constant vectors.
2584void SPIRVEmitIntrinsicsImpl::scalarizeVectorPtrCasts(Function &F) {
2585 // If the extension for vector-of-pointers is available, no scalarization
2586 // is needed - the backend can handle them natively.
2587 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F);
2588 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
2589 return;
2590
2591 SmallVector<Instruction *, 8> ToErase;
2592
2593 for (Instruction &I : make_early_inc_range(Range: instructions(F))) {
2594 if (auto *PTI = dyn_cast<PtrToIntInst>(Val: &I)) {
2595 auto *VecTy = dyn_cast<FixedVectorType>(Val: PTI->getOperand(i_nocapture: 0)->getType());
2596 if (!VecTy || !VecTy->getElementType()->isPointerTy())
2597 continue;
2598
2599 IRBuilder<> B(PTI);
2600 unsigned NumElems = VecTy->getNumElements();
2601 Type *ElemIntTy = cast<VectorType>(Val: PTI->getType())->getElementType();
2602 Value *Result = PoisonValue::get(T: PTI->getType());
2603
2604 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {
2605 Value *ElemPtr = findScalarElement(V: PTI->getOperand(i_nocapture: 0), EltNo: Idx);
2606 if (!ElemPtr)
2607 ElemPtr = B.CreateExtractElement(Vec: PTI->getOperand(i_nocapture: 0), Idx: B.getInt32(C: Idx));
2608 Value *Conv = B.CreatePtrToInt(V: ElemPtr, DestTy: ElemIntTy);
2609 Result = B.CreateInsertElement(Vec: Result, NewElt: Conv, Idx: B.getInt32(C: Idx));
2610 }
2611
2612 PTI->replaceAllUsesWith(V: Result);
2613 ToErase.push_back(Elt: PTI);
2614 } else if (auto *ITP = dyn_cast<IntToPtrInst>(Val: &I)) {
2615 auto *VecTy = dyn_cast<FixedVectorType>(Val: ITP->getType());
2616 if (!VecTy || !VecTy->getElementType()->isPointerTy())
2617 continue;
2618
2619 // Only scalarize if all users are extractelement with constant index.
2620 // Other uses (e.g. masked.gather) need the vector-of-pointers and
2621 // should be handled by extension-specific lowering.
2622 bool AllUsersExtract = llvm::all_of(Range: ITP->users(), P: [&](User *U) {
2623 auto *EE = dyn_cast<ExtractElementInst>(Val: U);
2624 return EE && isa<ConstantInt>(Val: EE->getIndexOperand());
2625 });
2626 if (!AllUsersExtract)
2627 continue;
2628
2629 IRBuilder<> B(ITP);
2630 Type *ElemPtrTy = VecTy->getElementType();
2631
2632 // Replace extractelement users with scalar inttoptr.
2633 for (User *U : make_early_inc_range(Range: ITP->users())) {
2634 auto *EE = cast<ExtractElementInst>(Val: U);
2635 unsigned Idx = cast<ConstantInt>(Val: EE->getIndexOperand())->getZExtValue();
2636 B.SetInsertPoint(EE);
2637 Value *Elem =
2638 B.CreateExtractElement(Vec: ITP->getOperand(i_nocapture: 0), Idx: B.getInt32(C: Idx));
2639 Value *Conv = B.CreateIntToPtr(V: Elem, DestTy: ElemPtrTy);
2640 EE->replaceAllUsesWith(V: Conv);
2641 ToErase.push_back(Elt: EE);
2642 }
2643
2644 ToErase.push_back(Elt: ITP);
2645 }
2646 }
2647
2648 if (ToErase.empty())
2649 return;
2650 SmallVector<WeakTrackingVH, 8> MaybeDead;
2651 for (Instruction *I : ToErase) {
2652 for (Value *Op : I->operand_values())
2653 MaybeDead.push_back(Elt: Op);
2654 I->eraseFromParent();
2655 }
2656 RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts&: MaybeDead);
2657}
2658
2659Instruction *SPIRVEmitIntrinsicsImpl::visitInsertValueInst(InsertValueInst &I) {
2660 IRBuilder<> B(I.getParent());
2661 B.SetInsertPoint(&I);
2662 SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()};
2663 SmallVector<Value *> Args;
2664 Value *AggregateOp = I.getAggregateOperand();
2665 if (isa<UndefValue>(Val: AggregateOp))
2666 Args.push_back(Elt: UndefValue::get(T: B.getInt32Ty()));
2667 else
2668 Args.push_back(Elt: AggregateOp);
2669 Args.push_back(Elt: I.getInsertedValueOperand());
2670 for (auto &Op : I.indices())
2671 Args.push_back(Elt: B.getInt32(C: Op));
2672 Instruction *NewI =
2673 B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_insertv, OverloadTypes: {Types}, Args: {Args});
2674 replaceMemInstrUses(Old: &I, New: NewI, B);
2675 return NewI;
2676}
2677
2678Instruction *
2679SPIRVEmitIntrinsicsImpl::visitExtractValueInst(ExtractValueInst &I) {
2680 IRBuilder<> B(I.getParent());
2681 B.SetInsertPoint(&I);
2682 if (I.getAggregateOperand()->getType()->isAggregateType()) {
2683 // Mutate an aggregate-returning spv_extractv producer to i32 so
2684 // IRTranslator does not see a multi-register value.
2685 CallBase *CB = dyn_cast<CallBase>(Val: I.getAggregateOperand());
2686 if (!CB || CB->getIntrinsicID() != Intrinsic::spv_extractv)
2687 return &I;
2688 CB->mutateType(Ty: B.getInt32Ty());
2689 }
2690 SmallVector<Value *> Args(I.operands());
2691 for (auto &Op : I.indices())
2692 Args.push_back(Elt: B.getInt32(C: Op));
2693 Instruction *NewI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_extractv,
2694 OverloadTypes: {I.getType()}, Args: {Args});
2695 // If this aggregate extract feeds another insertvalue, the extracted
2696 // composite is used as a SPIR-V value-id by llvm.spv.insertv. Keep the real
2697 // aggregate type in metadata, but expose the value itself as i32 so the
2698 // intrinsic signature remains valid.
2699 if (NewI->getType()->isAggregateType() &&
2700 any_of(Range: I.users(), P: [](User *U) { return isa<InsertValueInst>(Val: U); })) {
2701 AggrConstTypes[NewI] = I.getType();
2702 NewI->mutateType(Ty: B.getInt32Ty());
2703 replaceMemInstrUses(Old: &I, New: NewI, B);
2704 return NewI;
2705 }
2706 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2707 // If the aggregate result feeds a return or callsite whose type was rewritten
2708 // to an i32 value-id by SPIRVPrepareFunctions, mutate it to match.
2709 if (NewI->getType()->isAggregateType()) {
2710 for (const Use &U : NewI->uses()) {
2711 User *Usr = U.getUser();
2712 if (auto *RI = dyn_cast<ReturnInst>(Val: Usr)) {
2713 if (RI->getFunction()->getReturnType() != NewI->getType()) {
2714 NewI->mutateType(Ty: B.getInt32Ty());
2715 break;
2716 }
2717 continue;
2718 }
2719 auto *CB = dyn_cast<CallBase>(Val: Usr);
2720 if (!CB || !CB->isArgOperand(U: &U))
2721 continue;
2722 unsigned ArgNo = CB->getArgOperandNo(U: &U);
2723 FunctionType *FT = CB->getFunctionType();
2724 if (ArgNo < FT->getNumParams() &&
2725 !FT->getParamType(i: ArgNo)->isAggregateType()) {
2726 NewI->mutateType(Ty: B.getInt32Ty());
2727 break;
2728 }
2729 }
2730 }
2731 return NewI;
2732}
2733
2734Instruction *SPIRVEmitIntrinsicsImpl::visitLoadInst(LoadInst &I) {
2735 if (!I.getType()->isAggregateType())
2736 return &I;
2737 IRBuilder<> B(I.getParent());
2738 B.SetInsertPoint(&I);
2739 TrackConstants = false;
2740 const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
2741 MachineMemOperand::Flags Flags =
2742 TLI->getLoadMemOperandFlags(LI: I, DL: CurrF->getDataLayout());
2743
2744 unsigned IntrinsicId;
2745 SmallVector<Value *, 4> Args = {I.getPointerOperand(), B.getInt16(C: Flags)};
2746 if (!I.isAtomic()) {
2747 IntrinsicId = Intrinsic::spv_load;
2748 Args.push_back(Elt: B.getInt32(C: I.getAlign().value()));
2749 } else {
2750 IntrinsicId = Intrinsic::spv_atomic_load;
2751 Args.push_back(Elt: B.getInt8(C: static_cast<uint8_t>(I.getOrdering())));
2752 }
2753 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
2754 ID: IntrinsicId, OverloadTypes: {I.getOperand(i_nocapture: 0)->getType()}, Args);
2755
2756 replaceMemInstrUses(Old: &I, New: NewI, B);
2757 return NewI;
2758}
2759
2760Instruction *SPIRVEmitIntrinsicsImpl::visitStoreInst(StoreInst &I) {
2761 if (!AggrStores.contains(Ptr: &I))
2762 return &I;
2763 IRBuilder<> B(I.getParent());
2764 B.SetInsertPoint(&I);
2765 TrackConstants = false;
2766 const auto *TLI = TM.getSubtargetImpl()->getTargetLowering();
2767 MachineMemOperand::Flags Flags =
2768 TLI->getStoreMemOperandFlags(SI: I, DL: CurrF->getDataLayout());
2769 auto *PtrOp = I.getPointerOperand();
2770
2771 if (I.getValueOperand()->getType()->isAggregateType()) {
2772 // It is possible that what used to be an ExtractValueInst has been replaced
2773 // with a call to the spv_extractv intrinsic, and that said call hasn't
2774 // had its return type replaced with i32 during the dedicated pass (because
2775 // it was emitted later); we have to handle this here, because IRTranslator
2776 // cannot deal with multi-register types at the moment.
2777 CallBase *CB = dyn_cast<CallBase>(Val: I.getValueOperand());
2778 assert(CB && CB->getIntrinsicID() == Intrinsic::spv_extractv &&
2779 "Unexpected argument of aggregate type, should be spv_extractv!");
2780 CB->mutateType(Ty: B.getInt32Ty());
2781 }
2782
2783 unsigned IntrinsicId;
2784 SmallVector<Value *, 4> Args = {I.getValueOperand(), PtrOp,
2785 B.getInt16(C: Flags)};
2786 if (!I.isAtomic()) {
2787 IntrinsicId = Intrinsic::spv_store;
2788 Args.push_back(Elt: B.getInt32(C: I.getAlign().value()));
2789 } else {
2790 IntrinsicId = Intrinsic::spv_atomic_store;
2791 Args.push_back(Elt: B.getInt8(C: static_cast<uint8_t>(I.getOrdering())));
2792 }
2793 Instruction *NewI = B.CreateIntrinsicWithoutFolding(
2794 ID: IntrinsicId, OverloadTypes: {I.getValueOperand()->getType(), PtrOp->getType()}, Args);
2795 NewI->copyMetadata(SrcInst: I);
2796 I.eraseFromParent();
2797 return NewI;
2798}
2799
2800Instruction *SPIRVEmitIntrinsicsImpl::visitAllocaInst(AllocaInst &I) {
2801 Value *ArraySize = nullptr;
2802 if (I.isArrayAllocation()) {
2803 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I.getFunction());
2804 if (!STI->canUseExtension(
2805 E: SPIRV::Extension::SPV_INTEL_variable_length_array))
2806 report_fatal_error(
2807 reason: "array allocation: this instruction requires the following "
2808 "SPIR-V extension: SPV_INTEL_variable_length_array",
2809 gen_crash_diag: false);
2810 ArraySize = I.getArraySize();
2811 }
2812 IRBuilder<> B(I.getParent());
2813 B.SetInsertPoint(&I);
2814 TrackConstants = false;
2815 Type *PtrTy = I.getType();
2816 Instruction *NewI =
2817 ArraySize
2818 ? B.CreateIntrinsicWithoutFolding(
2819 ID: Intrinsic::spv_alloca_array, OverloadTypes: {PtrTy, ArraySize->getType()},
2820 Args: {ArraySize, B.getInt32(C: I.getAlign().value())})
2821 : B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_alloca, OverloadTypes: {PtrTy},
2822 Args: {B.getInt32(C: I.getAlign().value())});
2823 replaceAllUsesWithAndErase(B, Src: &I, Dest: NewI);
2824 return NewI;
2825}
2826
2827Instruction *
2828SPIRVEmitIntrinsicsImpl::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2829 assert(I.getType()->isAggregateType() && "Aggregate result is expected");
2830 IRBuilder<> B(I.getParent());
2831 B.SetInsertPoint(&I);
2832 SmallVector<Value *> Args(I.operands());
2833 const Triple &TT = TM.getTargetTriple();
2834 Args.push_back(Elt: B.getInt32(C: static_cast<uint32_t>(
2835 getMemScope(TT, Ctx&: I.getContext(), Id: I.getSyncScopeID()))));
2836 // Per SPIR-V spec atomic ops must combine the ordering bits with the
2837 // storage-class bit.
2838 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2839 unsigned AS = I.getPointerOperand()->getType()->getPointerAddressSpace();
2840 uint32_t ScSem = static_cast<uint32_t>(
2841 getMemSemanticsForStorageClass(SC: addressSpaceToStorageClass(AddrSpace: AS, STI: ST)));
2842 Args.push_back(Elt: B.getInt32(C: getMemSemanticsWithStorageClass(
2843 TT, OrderSem: static_cast<uint32_t>(getMemSemantics(Ord: I.getSuccessOrdering())),
2844 StorageClassSem: ScSem)));
2845 Args.push_back(Elt: B.getInt32(C: getMemSemanticsWithStorageClass(
2846 TT, OrderSem: static_cast<uint32_t>(getMemSemantics(Ord: I.getFailureOrdering())),
2847 StorageClassSem: ScSem)));
2848 Instruction *NewI = B.CreateIntrinsicWithoutFolding(
2849 ID: Intrinsic::spv_cmpxchg, OverloadTypes: {I.getPointerOperand()->getType()}, Args: {Args});
2850 replaceMemInstrUses(Old: &I, New: NewI, B);
2851 return NewI;
2852}
2853
2854Instruction *SPIRVEmitIntrinsicsImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
2855 auto Op = I.getOperation();
2856 if (Op != AtomicRMWInst::UIncWrap && Op != AtomicRMWInst::UDecWrap)
2857 return &I;
2858
2859 // No SPIR-V opcode exists for these, so lowering to an imported helper is an
2860 // AMD extension. Other targets keep the generic expansion.
2861 if (TM.getTargetTriple().getVendor() != Triple::AMD)
2862 return &I;
2863
2864 Module *M = I.getModule();
2865 IRBuilder<> B(I.getParent());
2866 B.SetInsertPoint(&I);
2867
2868 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2869 unsigned AS = I.getPointerOperand()->getType()->getPointerAddressSpace();
2870
2871 uint32_t Scope = static_cast<uint32_t>(
2872 getMemScope(TT: M->getTargetTriple(), Ctx&: I.getContext(), Id: I.getSyncScopeID()));
2873 uint32_t ScSem = static_cast<uint32_t>(
2874 getMemSemanticsForStorageClass(SC: addressSpaceToStorageClass(AddrSpace: AS, STI: ST)));
2875 uint32_t MemSem = getMemSemanticsWithStorageClass(
2876 TT: M->getTargetTriple(),
2877 OrderSem: static_cast<uint32_t>(getMemSemantics(Ord: I.getOrdering())), StorageClassSem: ScSem);
2878
2879 SmallString<64> FuncName(Op == AtomicRMWInst::UIncWrap
2880 ? "__translate_spirv_atomic_uinc_wrap"
2881 : "__translate_spirv_atomic_udec_wrap");
2882
2883 Type *ValTy = I.getValOperand()->getType();
2884 Type *PtrTy = I.getPointerOperand()->getType();
2885 // Encode the address space and value type in the name for overload.
2886 raw_svector_ostream OS(FuncName);
2887 OS << "_p" << AS << "_";
2888 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: ValTy))
2889 OS << "v" << VecTy->getNumElements();
2890 OS << "i" << ValTy->getScalarSizeInBits();
2891
2892 Type *Int32Ty = B.getInt32Ty();
2893 Type *BoolTy = B.getInt1Ty();
2894 SmallVector<Type *, 6> ArgTys = {PtrTy, Int32Ty, Int32Ty,
2895 ValTy, BoolTy, BoolTy};
2896 FunctionType *FT = FunctionType::get(Result: ValTy, Params: ArgTys, isVarArg: false);
2897 FunctionCallee FC = M->getOrInsertFunction(Name: FuncName, T: FT);
2898 cast<Function>(Val: FC.getCallee())->setCallingConv(CallingConv::SPIR_FUNC);
2899
2900 SmallVector<Value *, 6> Args = {
2901 I.getPointerOperand(), B.getInt32(C: Scope),
2902 B.getInt32(C: MemSem), I.getValOperand(),
2903 B.getInt1(V: I.isVolatile()), B.getInt1(V: I.isElementwise())};
2904 CallInst *CI = B.CreateCall(Callee: FC, Args);
2905 CI->setCallingConv(CallingConv::SPIR_FUNC);
2906 // SPIRVCallLowering reads alias.scope/noalias off the call to build the
2907 // aliasing decorations and runs after this pass, so preserve the metadata.
2908 CI->copyMetadata(SrcInst: I);
2909
2910 replaceAllUsesWithAndErase(B, Src: &I, Dest: CI);
2911 return CI;
2912}
2913
2914static bool isAbortCall(const Instruction &I, const SPIRVSubtarget &ST) {
2915 auto *CI = dyn_cast<CallInst>(Val: &I);
2916 if (!CI)
2917 return false;
2918 switch (CI->getIntrinsicID()) {
2919 case Intrinsic::spv_abort:
2920 return true;
2921 case Intrinsic::trap:
2922 case Intrinsic::ubsantrap:
2923 // When the extension is enabled, selection lowers these to OpAbortKHR.
2924 return ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_abort);
2925 default:
2926 return false;
2927 }
2928}
2929
2930// The OpAbortKHR instruction itself is a block terminator, so we don't need to
2931// emit an extra OpUnreachable instruction.
2932static bool precededByAbortIntrinsic(const UnreachableInst &I,
2933 const SPIRVSubtarget &ST) {
2934 const Instruction *Prev = I.getPrevNode();
2935 if (Prev && isAbortCall(I: *Prev, ST))
2936 return true;
2937
2938 assert(llvm::none_of(
2939 *I.getParent(),
2940 [&ST](const Instruction &II) { return isAbortCall(II, ST); }) &&
2941 "abort-like call must be the last non-debug instruction before its "
2942 "block's terminator");
2943 return false;
2944}
2945
2946Instruction *SPIRVEmitIntrinsicsImpl::visitUnreachableInst(UnreachableInst &I) {
2947 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
2948 if (precededByAbortIntrinsic(I, ST))
2949 return &I;
2950 IRBuilder<> B(&I);
2951 B.CreateIntrinsic(ID: Intrinsic::spv_unreachable, Args: {});
2952 return &I;
2953}
2954
2955// llvm.compiler.used and llvm.used hold use-list entries that protect their
2956// referenced globals from DCE without participating in code generation.
2957static bool isUseListGlobal(StringRef Name) {
2958 return Name == "llvm.compiler.used" || Name == "llvm.used";
2959}
2960
2961// Returns true for module-level globals that should not have SPIR-V intrinsics
2962// emitted (use-list globals plus llvm.global.annotations).
2963static bool isArtificialGlobal(StringRef Name) {
2964 return isUseListGlobal(Name) || Name == "llvm.global.annotations";
2965}
2966
2967// Returns true if every use of GV traces back to llvm.compiler.used or
2968// llvm.used.
2969static bool hasOnlyArtificialUses(const GlobalVariable &GV) {
2970 SmallPtrSet<const Value *, 8> Visited;
2971 SmallVector<const Value *> Stack(GV.users());
2972 while (!Stack.empty()) {
2973 const Value *V = Stack.pop_back_val();
2974 if (!Visited.insert(Ptr: V).second)
2975 continue;
2976 if (const auto *GVUser = dyn_cast<GlobalVariable>(Val: V)) {
2977 if (!isUseListGlobal(Name: GVUser->getName()))
2978 return false;
2979 continue;
2980 }
2981 if (const auto *C = dyn_cast<Constant>(Val: V)) {
2982 Stack.append(in_start: C->user_begin(), in_end: C->user_end());
2983 continue;
2984 }
2985 return false;
2986 }
2987 return true;
2988}
2989
2990static bool
2991shouldEmitIntrinsicsForGlobalValue(const GlobalVariableUsers &GVUsers,
2992 const GlobalVariable &GV,
2993 const Function *F) {
2994 // Skip special artificial variables.
2995 if (isArtificialGlobal(Name: GV.getName()))
2996 return false;
2997
2998 auto &UserFunctions = GVUsers.getTransitiveUserFunctions(GV);
2999 if (UserFunctions.contains(Ptr: F))
3000 return true;
3001
3002 // Do not emit the intrinsics in this function, it's going to be emitted on
3003 // the functions that reference it.
3004 if (!UserFunctions.empty())
3005 return false;
3006
3007 // Emit definitions for globals that are not referenced by any function on the
3008 // first function definition.
3009 const Module &M = *F->getParent();
3010 const Function &FirstDefinition = *M.getFunctionDefs().begin();
3011 return F == &FirstDefinition;
3012}
3013
3014Value *SPIRVEmitIntrinsicsImpl::buildSpvUndefComposite(Type *AggrTy,
3015 IRBuilder<> &B) {
3016 auto MakeLeaf = [&](Type *ElemTy) -> Instruction * {
3017 CallInst *Leaf = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_undef, Args: {});
3018 AggrConsts[Leaf] = PoisonValue::get(T: ElemTy);
3019 AggrConstTypes[Leaf] = ElemTy;
3020 return Leaf;
3021 };
3022 SmallVector<Value *, 4> Elems;
3023 if (auto *ArrTy = dyn_cast<ArrayType>(Val: AggrTy)) {
3024 Elems.assign(NumElts: ArrTy->getNumElements(), Elt: MakeLeaf(ArrTy->getElementType()));
3025 } else {
3026 auto *StructTy = cast<StructType>(Val: AggrTy);
3027 DenseMap<Type *, Instruction *> LeafByType;
3028 for (unsigned I = 0; I < StructTy->getNumElements(); ++I) {
3029 Type *ElemTy = StructTy->getContainedType(i: I);
3030 auto &Entry = LeafByType[ElemTy];
3031 if (!Entry)
3032 Entry = MakeLeaf(ElemTy);
3033 Elems.push_back(Elt: Entry);
3034 }
3035 }
3036 CallInst *Composite = B.CreateIntrinsicWithoutFolding(
3037 ID: Intrinsic::spv_const_composite, OverloadTypes: {B.getInt32Ty()}, Args: Elems);
3038 AggrConsts[Composite] = PoisonValue::get(T: AggrTy);
3039 AggrConstTypes[Composite] = AggrTy;
3040 return Composite;
3041}
3042
3043// If a function directly returns an aggregate-typed call result,
3044// the ReturnInst carries an aggregate while the function signature
3045// was rewritten to i32 by SPIRVPrepareFunctions. Rebuild the return value
3046// via extractvalue/insertvalue so the regular spv_extractv/spv_insertv
3047// lowering produces a valid OpReturnValue.
3048void SPIRVEmitIntrinsicsImpl::reconstructAggregateReturns(Function &Func,
3049 IRBuilder<> &B) {
3050 Type *OrigRetTy = GR->findMutated(Val: &Func);
3051 if (!OrigRetTy || !OrigRetTy->isAggregateType())
3052 return;
3053 for (BasicBlock &BB : Func) {
3054 auto *RI = dyn_cast<ReturnInst>(Val: BB.getTerminator());
3055 if (!RI)
3056 continue;
3057 Value *RetVal = RI->getReturnValue();
3058 if (!RetVal || RetVal->getType() != OrigRetTy || !isa<CallBase>(Val: RetVal))
3059 continue;
3060 Type *AggrTy = RetVal->getType();
3061 uint64_t NumElts = isa<StructType>(Val: AggrTy)
3062 ? cast<StructType>(Val: AggrTy)->getNumElements()
3063 : cast<ArrayType>(Val: AggrTy)->getNumElements();
3064 B.SetInsertPoint(RI);
3065 Value *Rebuilt = PoisonValue::get(T: AggrTy);
3066 for (uint64_t I = 0; I < NumElts; ++I) {
3067 Value *Elt = B.CreateExtractValue(Agg: RetVal, Idxs: I);
3068 Rebuilt = B.CreateInsertValue(Agg: Rebuilt, Val: Elt, Idxs: I);
3069 }
3070 RI->setOperand(i_nocapture: 0, Val_nocapture: Rebuilt);
3071 }
3072}
3073
3074void SPIRVEmitIntrinsicsImpl::processGlobalValue(GlobalVariable &GV,
3075 IRBuilder<> &B) {
3076
3077 if (!shouldEmitIntrinsicsForGlobalValue(GVUsers, GV, F: CurrF))
3078 return;
3079
3080 // Record the pointee type for every global, not only initialized ones, so an
3081 // undef non-constant aggregate global is not later collapsed to its element
3082 // type. Result is ignored, because TypedPointerType is not supported
3083 // by llvm IR general logic.
3084 deduceElementTypeHelper(I: &GV, UnknownElemTypeI8: false);
3085
3086 Constant *Init = nullptr;
3087 if (hasInitializer(GV: &GV)) {
3088 Init = GV.getInitializer();
3089 Value *InitOp = Init;
3090 if (isa<UndefValue>(Val: Init) && Init->getType()->isAggregateType()) {
3091 const SPIRVSubtarget *STI = TM.getSubtargetImpl();
3092 bool UsePoison =
3093 isa<PoisonValue>(Val: Init) &&
3094 STI->canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze);
3095 if (UsePoison) {
3096 CallInst *Call = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::spv_poison,
3097 OverloadTypes: {B.getInt32Ty()}, Args: {});
3098 AggrConsts[Call] = cast<PoisonValue>(Val: Init);
3099 AggrConstTypes[Call] = Init->getType();
3100 InitOp = Call;
3101 } else {
3102 InitOp = buildSpvUndefComposite(AggrTy: Init->getType(), B);
3103 }
3104 }
3105 Type *Ty = isAggrConstForceInt32(V: Init) ? B.getInt32Ty() : Init->getType();
3106 Constant *Const = isAggrConstForceInt32(V: Init) ? B.getInt32(C: 1) : Init;
3107 CallInst *InitInst = B.CreateIntrinsicWithoutFolding(
3108 ID: Intrinsic::spv_init_global, OverloadTypes: {GV.getType(), Ty}, Args: {&GV, Const});
3109 InitInst->setArgOperand(i: 1, v: InitOp);
3110 }
3111 // Globals with only use-list references have no real function uses. Emit
3112 // spv_unref_global so buildGlobalVariable is called for them.
3113 if (!Init && hasOnlyArtificialUses(GV))
3114 B.CreateIntrinsic(ID: Intrinsic::spv_unref_global, OverloadTypes: GV.getType(), Args: &GV);
3115}
3116
3117// Return true, if we can't decide what is the pointee type now and will get
3118// back to the question later. Return false is spv_assign_ptr_type is not needed
3119// or can be inserted immediately.
3120bool SPIRVEmitIntrinsicsImpl::insertAssignPtrTypeIntrs(Instruction *I,
3121 IRBuilder<> &B,
3122 bool UnknownElemTypeI8) {
3123 reportFatalOnTokenType(I);
3124 if (!isPointerTy(T: I->getType()) || !requireAssignType(I))
3125 return false;
3126
3127 setInsertPointAfterDef(B, I);
3128 if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) {
3129 GR->buildAssignPtr(B, ElemTy, Arg: I);
3130 return false;
3131 }
3132 return true;
3133}
3134
3135void SPIRVEmitIntrinsicsImpl::insertAssignTypeIntrs(Instruction *I,
3136 IRBuilder<> &B) {
3137 // TODO: extend the list of functions with known result types
3138 static StringMap<unsigned> ResTypeWellKnown = {
3139 {"async_work_group_copy", WellKnownTypes::Event},
3140 {"async_work_group_strided_copy", WellKnownTypes::Event},
3141 {"__spirv_GroupAsyncCopy", WellKnownTypes::Event}};
3142
3143 reportFatalOnTokenType(I);
3144
3145 bool IsKnown = false;
3146 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
3147 if (!CI->isIndirectCall() && !CI->isInlineAsm() &&
3148 CI->getCalledFunction() && !CI->getCalledFunction()->isIntrinsic()) {
3149 Function *CalledF = CI->getCalledFunction();
3150 std::string DemangledName =
3151 getOclOrSpirvBuiltinDemangledName(Name: CalledF->getName());
3152 FPDecorationId DecorationId = FPDecorationId::NONE;
3153 if (DemangledName.length() > 0)
3154 DemangledName =
3155 SPIRV::lookupBuiltinNameHelper(DemangledCall: DemangledName, DecorationId: &DecorationId);
3156 auto ResIt = ResTypeWellKnown.find(Key: DemangledName);
3157 if (ResIt != ResTypeWellKnown.end()) {
3158 IsKnown = true;
3159 setInsertPointAfterDef(B, I);
3160 switch (ResIt->second) {
3161 case WellKnownTypes::Event:
3162 GR->buildAssignType(
3163 B, Ty: TargetExtType::get(Context&: I->getContext(), Name: "spirv.Event"), Arg: I,
3164 CanUseAnyVectorRank);
3165 break;
3166 }
3167 }
3168 // check if a floating rounding mode or saturation info is present
3169 switch (DecorationId) {
3170 default:
3171 break;
3172 case FPDecorationId::SAT:
3173 createSaturatedConversionDecoration(I: CI, B);
3174 break;
3175 case FPDecorationId::RTE:
3176 createRoundingModeDecoration(
3177 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTE, B);
3178 break;
3179 case FPDecorationId::RTZ:
3180 createRoundingModeDecoration(
3181 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTZ, B);
3182 break;
3183 case FPDecorationId::RTP:
3184 createRoundingModeDecoration(
3185 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTP, B);
3186 break;
3187 case FPDecorationId::RTN:
3188 createRoundingModeDecoration(
3189 I: CI, RoundingModeDeco: SPIRV::FPRoundingMode::FPRoundingMode::RTN, B);
3190 break;
3191 }
3192 }
3193 }
3194
3195 Type *Ty = I->getType();
3196 if (!IsKnown && !Ty->isVoidTy() && !isPointerTy(T: Ty) && requireAssignType(I)) {
3197 setInsertPointAfterDef(B, I);
3198 Type *TypeToAssign = Ty;
3199 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
3200 if (isSpvAggrPlaceholder(V: II)) {
3201 auto It = AggrConstTypes.find(Val: II);
3202 if (It == AggrConstTypes.end())
3203 report_fatal_error(reason: "Unknown composite intrinsic type");
3204 TypeToAssign = It->second;
3205 } else if (II->getIntrinsicID() == Intrinsic::spv_poison) {
3206 if (auto It = AggrConstTypes.find(Val: II); It != AggrConstTypes.end())
3207 TypeToAssign = It->second;
3208 }
3209 } else if (auto It = AggrConstTypes.find(Val: I); It != AggrConstTypes.end())
3210 TypeToAssign = It->second;
3211 TypeToAssign = restoreMutatedType(GR, I, Ty: TypeToAssign);
3212 GR->buildAssignType(B, Ty: TypeToAssign, Arg: I, CanUseAnyVectorRank);
3213 }
3214 for (const auto &Op : I->operands()) {
3215 if (isa<ConstantPointerNull>(Val: Op) || isa<UndefValue>(Val: Op) ||
3216 isVector1(Ty: Op->getType()) || // <1 x T> gets clobbered ty IRTranslator.
3217 // Check GetElementPtrConstantExpr case.
3218 (isa<ConstantExpr>(Val: Op) &&
3219 (isa<GEPOperator>(Val: Op) ||
3220 (cast<ConstantExpr>(Val: Op)->getOpcode() == CastInst::IntToPtr)))) {
3221 setInsertPointSkippingPhis(B, I);
3222 Type *OpTy = Op->getType();
3223 if (isa<UndefValue>(Val: Op) && OpTy->isAggregateType()) {
3224 CallInst *AssignCI =
3225 buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {B.getInt32Ty()}, Arg: Op,
3226 Arg2: UndefValue::get(T: B.getInt32Ty()), Imms: {}, B);
3227 GR->addAssignPtrTypeInstr(Val: Op, AssignPtrTyCI: AssignCI);
3228 } else if (!isa<Instruction>(Val: Op)) {
3229 Type *OpTy = Op->getType();
3230 Type *OpTyElem = getPointeeType(Ty: OpTy);
3231 if (OpTyElem) {
3232 GR->buildAssignPtr(B, ElemTy: OpTyElem, Arg: Op);
3233 } else if (isPointerTy(T: OpTy)) {
3234 Type *ElemTy = GR->findDeducedElementType(Val: Op);
3235 GR->buildAssignPtr(B, ElemTy: ElemTy ? ElemTy : deduceElementType(I: Op, UnknownElemTypeI8: true),
3236 Arg: Op);
3237 } else {
3238 Value *OpTyVal = Op;
3239 if (OpTy->isTargetExtTy()) {
3240 // We need to do this in order to be consistent with how target ext
3241 // types are handled in `processInstrAfterVisit`
3242 OpTyVal = getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank);
3243 }
3244 CallInst *AssignCI = buildIntrWithMD(
3245 IntrID: Intrinsic::spv_assign_type, Types: {OpTy},
3246 Arg: getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank), Arg2: OpTyVal, Imms: {},
3247 B);
3248 GR->addAssignPtrTypeInstr(Val: OpTyVal, AssignPtrTyCI: AssignCI);
3249 }
3250 }
3251 }
3252 }
3253}
3254
3255bool SPIRVEmitIntrinsicsImpl::shouldTryToAddMemAliasingDecoration(
3256 Instruction *Inst) {
3257 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*Inst->getFunction());
3258 if (!STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_memory_access_aliasing))
3259 return false;
3260 // Add aliasing decorations to internal load and store intrinsics.
3261 // Do not attach them to store atomic or load atomic intrinsics / instructions
3262 // since the extension is inconsistent at the moment (we cannot add the
3263 // decoration to atomic stores because they do not have an id).
3264 return match(V: Inst,
3265 P: m_AnyIntrinsic<Intrinsic::spv_load, Intrinsic::spv_store>());
3266}
3267
3268void SPIRVEmitIntrinsicsImpl::insertSpirvDecorations(Instruction *I,
3269 IRBuilder<> &B) {
3270 if (MDNode *MD = I->getMetadata(Kind: "spirv.Decorations")) {
3271 setInsertPointAfterDef(B, I);
3272 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
3273 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
3274 }
3275 // Lower alias.scope/noalias metadata
3276 {
3277 auto processMemAliasingDecoration = [&](unsigned Kind) {
3278 if (MDNode *AliasListMD = I->getMetadata(KindID: Kind)) {
3279 if (shouldTryToAddMemAliasingDecoration(Inst: I)) {
3280 uint32_t Dec = Kind == LLVMContext::MD_alias_scope
3281 ? SPIRV::Decoration::AliasScopeINTEL
3282 : SPIRV::Decoration::NoAliasINTEL;
3283 SmallVector<Value *, 3> Args = {
3284 I, ConstantInt::get(Ty: B.getInt32Ty(), V: Dec),
3285 MetadataAsValue::get(Context&: I->getContext(), MD: AliasListMD)};
3286 setInsertPointAfterDef(B, I);
3287 B.CreateIntrinsic(ID: Intrinsic::spv_assign_aliasing_decoration,
3288 OverloadTypes: {I->getType()}, Args: {Args});
3289 }
3290 }
3291 };
3292 processMemAliasingDecoration(LLVMContext::MD_alias_scope);
3293 processMemAliasingDecoration(LLVMContext::MD_noalias);
3294 }
3295 // MD_fpmath
3296 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_fpmath)) {
3297 const SPIRVSubtarget *STI = TM.getSubtargetImpl(*I->getFunction());
3298 bool AllowFPMaxError =
3299 STI->canUseExtension(E: SPIRV::Extension::SPV_INTEL_fp_max_error);
3300 if (!AllowFPMaxError)
3301 return;
3302
3303 setInsertPointAfterDef(B, I);
3304 B.CreateIntrinsic(ID: Intrinsic::spv_assign_fpmaxerror_decoration,
3305 OverloadTypes: {I->getType()},
3306 Args: {I, MetadataAsValue::get(Context&: I->getContext(), MD)});
3307 }
3308 if (spirvPreserveAuxData(CLOpts: TM.getCLOpts(), TT: I->getModule()->getTargetTriple()) &&
3309 isa<AtomicRMWInst>(Val: I)) {
3310 LLVMContext &Ctx = B.getContext();
3311 auto *AuxMD = ConstantAsMetadata::get(C: ConstantInt::get(
3312 Ty: B.getInt32Ty(), V: SPIRV::Decoration::AuxDataInstructionMetadata));
3313
3314 SmallVector<Metadata *> MDs;
3315 if (I->hasMetadata(Kind: "amdgpu.no.fine.grained.memory"))
3316 MDs.push_back(Elt: MDNode::get(
3317 Context&: Ctx, MDs: {AuxMD, MDString::get(Context&: Ctx, Str: "amdgpu.no.fine.grained.memory")}));
3318 if (I->hasMetadata(Kind: "amdgpu.no.remote.memory"))
3319 MDs.push_back(Elt: MDNode::get(
3320 Context&: Ctx, MDs: {AuxMD, MDString::get(Context&: Ctx, Str: "amdgpu.no.remote.memory")}));
3321 if (I->hasMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode))
3322 MDs.push_back(Elt: MDNode::get(
3323 Context&: Ctx, MDs: {AuxMD, MDString::get(Context&: Ctx, Str: "atomic.ignore.denormal.mode")}));
3324 if (!MDs.empty()) {
3325 setInsertPointAfterDef(B, I);
3326 B.CreateIntrinsic(ID: Intrinsic::spv_assign_decoration, OverloadTypes: {I->getType()},
3327 Args: {I, MetadataAsValue::get(Context&: Ctx, MD: MDNode::get(Context&: Ctx, MDs))});
3328 }
3329 }
3330}
3331
3332static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
3333 const Module &M,
3334 MapVector<Function *, SPIRV::FPFastMathDefaultInfoVector>
3335 &FPFastMathDefaultInfoMap,
3336 Function *F) {
3337 auto it = FPFastMathDefaultInfoMap.find(Key: F);
3338 if (it != FPFastMathDefaultInfoMap.end())
3339 return it->second;
3340
3341 // If the map does not contain the entry, create a new one. Initialize it to
3342 // contain all 3 elements sorted by bit width of target type: {half, float,
3343 // double}.
3344 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
3345 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
3346 Args: SPIRV::FPFastMathMode::None);
3347 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
3348 Args: SPIRV::FPFastMathMode::None);
3349 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
3350 Args: SPIRV::FPFastMathMode::None);
3351 return FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
3352}
3353
3354static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
3355 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
3356 const Type *Ty) {
3357 size_t BitWidth = Ty->getScalarSizeInBits();
3358 int Index =
3359 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
3360 BitWidth);
3361 assert(Index >= 0 && Index < 3 &&
3362 "Expected FPFastMathDefaultInfo for half, float, or double");
3363 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3364 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3365 return FPFastMathDefaultInfoVec[Index];
3366}
3367
3368void SPIRVEmitIntrinsicsImpl::insertConstantsForFPFastMathDefault(Module &M) {
3369 const SPIRVSubtarget *ST = TM.getSubtargetImpl();
3370 if (!ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
3371 return;
3372
3373 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
3374 // We need the entry point (function) as the key, and the target
3375 // type and flags as the value.
3376 // We also need to check ContractionOff and SignedZeroInfNanPreserve
3377 // execution modes, as they are now deprecated and must be replaced
3378 // with FPFastMathDefaultInfo.
3379 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
3380 if (!Node) {
3381 if (!M.getNamedMetadata(Name: "opencl.enable.FP_CONTRACT")) {
3382 // This requires emitting ContractionOff. However, because
3383 // ContractionOff is now deprecated, we need to replace it with
3384 // FPFastMathDefaultInfo with FP Fast Math Mode bitmask set to all 0.
3385 // We need to create the constant for that.
3386
3387 // Create constant instruction with the bitmask flags.
3388 Constant *InitValue =
3389 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: 0);
3390 // TODO: Reuse constant if there is one already with the required
3391 // value.
3392 [[maybe_unused]] GlobalVariable *GV =
3393 new GlobalVariable(M, // Module
3394 Type::getInt32Ty(C&: M.getContext()), // Type
3395 true, // isConstant
3396 GlobalValue::InternalLinkage, // Linkage
3397 InitValue // Initializer
3398 );
3399 }
3400 return;
3401 }
3402
3403 // The table maps function pointers to their default FP fast math info. It
3404 // can be assumed that the SmallVector is sorted by the bit width of the
3405 // type. The first element is the smallest bit width, and the last element
3406 // is the largest bit width, therefore, we will have {half, float, double}
3407 // in the order of their bit widths.
3408 MapVector<Function *, SPIRV::FPFastMathDefaultInfoVector>
3409 FPFastMathDefaultInfoMap;
3410
3411 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
3412 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
3413 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
3414 Function *F = cast<Function>(
3415 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
3416 const auto EM =
3417 cast<ConstantInt>(
3418 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
3419 ->getZExtValue();
3420 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
3421 assert(MDN->getNumOperands() == 4 &&
3422 "Expected 4 operands for FPFastMathDefault");
3423 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
3424 unsigned Flags =
3425 cast<ConstantInt>(
3426 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
3427 ->getZExtValue();
3428 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3429 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3430 SPIRV::FPFastMathDefaultInfo &Info =
3431 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
3432 Info.FastMathFlags = Flags;
3433 Info.FPFastMathDefault = true;
3434 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
3435 assert(MDN->getNumOperands() == 2 &&
3436 "Expected no operands for ContractionOff");
3437
3438 // We need to save this info for every possible FP type, i.e. {half,
3439 // float, double, fp128}.
3440 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3441 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3442 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3443 Info.ContractionOff = true;
3444 }
3445 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
3446 assert(MDN->getNumOperands() == 3 &&
3447 "Expected 1 operand for SignedZeroInfNanPreserve");
3448 unsigned TargetWidth =
3449 cast<ConstantInt>(
3450 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
3451 ->getZExtValue();
3452 // We need to save this info only for the FP type with TargetWidth.
3453 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3454 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3455 int Index = SPIRV::FPFastMathDefaultInfoVector::
3456 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
3457 assert(Index >= 0 && Index < 3 &&
3458 "Expected FPFastMathDefaultInfo for half, float, or double");
3459 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3460 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3461 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
3462 }
3463 }
3464
3465 DenseMap<unsigned, GlobalVariable *> GlobalVars;
3466 for (auto &[Func, FPFastMathDefaultInfoVec] : FPFastMathDefaultInfoMap) {
3467 if (FPFastMathDefaultInfoVec.empty())
3468 continue;
3469
3470 for (const SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3471 assert(Info.Ty && "Expected target type for FPFastMathDefaultInfo");
3472 // Skip if none of the execution modes was used.
3473 unsigned Flags = Info.FastMathFlags;
3474 if (Flags == SPIRV::FPFastMathMode::None && !Info.ContractionOff &&
3475 !Info.SignedZeroInfNanPreserve && !Info.FPFastMathDefault)
3476 continue;
3477
3478 // Check if flags are compatible.
3479 if (Info.ContractionOff && (Flags & SPIRV::FPFastMathMode::AllowContract))
3480 report_fatal_error(reason: "Conflicting FPFastMathFlags: ContractionOff "
3481 "and AllowContract");
3482
3483 if (Info.SignedZeroInfNanPreserve &&
3484 !(Flags &
3485 (SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
3486 SPIRV::FPFastMathMode::NSZ))) {
3487 if (Info.FPFastMathDefault)
3488 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
3489 "SignedZeroInfNanPreserve but at least one of "
3490 "NotNaN/NotInf/NSZ is enabled.");
3491 }
3492
3493 if ((Flags & SPIRV::FPFastMathMode::AllowTransform) &&
3494 !((Flags & SPIRV::FPFastMathMode::AllowReassoc) &&
3495 (Flags & SPIRV::FPFastMathMode::AllowContract))) {
3496 report_fatal_error(reason: "Conflicting FPFastMathFlags: "
3497 "AllowTransform requires AllowReassoc and "
3498 "AllowContract to be set.");
3499 }
3500
3501 auto it = GlobalVars.find(Val: Flags);
3502 GlobalVariable *GV = nullptr;
3503 if (it != GlobalVars.end()) {
3504 // Reuse existing global variable.
3505 GV = it->second;
3506 } else {
3507 // Create constant instruction with the bitmask flags.
3508 Constant *InitValue =
3509 ConstantInt::get(Ty: Type::getInt32Ty(C&: M.getContext()), V: Flags);
3510 // TODO: Reuse constant if there is one already with the required
3511 // value.
3512 GV = new GlobalVariable(M, // Module
3513 Type::getInt32Ty(C&: M.getContext()), // Type
3514 true, // isConstant
3515 GlobalValue::InternalLinkage, // Linkage
3516 InitValue // Initializer
3517 );
3518 GlobalVars[Flags] = GV;
3519 }
3520 }
3521 }
3522}
3523
3524void SPIRVEmitIntrinsicsImpl::processInstrAfterVisit(Instruction *I,
3525 IRBuilder<> &B) {
3526 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3527 bool IsConstComposite =
3528 II && II->getIntrinsicID() == Intrinsic::spv_const_composite;
3529 if (IsConstComposite && TrackConstants) {
3530 setInsertPointAfterDef(B, I);
3531 auto t = AggrConsts.find(Val: I);
3532 assert(t != AggrConsts.end());
3533 auto *NewOp =
3534 buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
3535 Types: {II->getType(), II->getType()}, Arg: t->second, Arg2: I, Imms: {}, B);
3536 replaceAllUsesWith(Src: I, Dest: NewOp, DeleteOld: false);
3537 NewOp->setArgOperand(i: 0, v: I);
3538 }
3539 bool IsPhi = isa<PHINode>(Val: I), BPrepared = false;
3540 for (const auto &Op : I->operands()) {
3541 if (isa<PHINode>(Val: I) || isa<SwitchInst>(Val: I) ||
3542 !(isa<ConstantData>(Val: Op) || isa<ConstantExpr>(Val: Op)))
3543 continue;
3544 unsigned OpNo = Op.getOperandNo();
3545 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) ||
3546 (!II->isBundleOperand(Idx: OpNo) &&
3547 II->paramHasAttr(ArgNo: OpNo, Kind: Attribute::ImmArg))))
3548 continue;
3549
3550 if (!BPrepared) {
3551 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent())
3552 : B.SetInsertPoint(I);
3553 BPrepared = true;
3554 }
3555 Type *OpTy = Op->getType();
3556 Type *OpElemTy = GR->findDeducedElementType(Val: Op);
3557 Value *NewOp = Op;
3558 if (OpTy->isTargetExtTy()) {
3559 // Since this value is replaced by poison, we need to do the same in
3560 // `insertAssignTypeIntrs`.
3561 Value *OpTyVal = getNormalizedPoisonValue(Ty: OpTy, CanUseAnyVectorRank);
3562 NewOp = buildIntrWithMD(IntrID: Intrinsic::spv_track_constant,
3563 Types: {OpTy, OpTyVal->getType()}, Arg: Op, Arg2: OpTyVal, Imms: {}, B);
3564 }
3565 if (!IsConstComposite && isPointerTy(T: OpTy) && OpElemTy != nullptr &&
3566 OpElemTy != IntegerType::getInt8Ty(C&: I->getContext())) {
3567 SmallVector<Type *, 2> Types = {OpTy, OpTy};
3568 SmallVector<Value *, 2> Args = {
3569 NewOp,
3570 buildMD(Arg: getNormalizedPoisonValue(Ty: OpElemTy, CanUseAnyVectorRank)),
3571 B.getInt32(C: getPointerAddressSpace(T: OpTy))};
3572 CallInst *PtrCasted = B.CreateIntrinsicWithoutFolding(
3573 ID: Intrinsic::spv_ptrcast, OverloadTypes: {Types}, Args);
3574 GR->buildAssignPtr(B, ElemTy: OpElemTy, Arg: PtrCasted);
3575 NewOp = PtrCasted;
3576 }
3577 if (NewOp != Op)
3578 I->setOperand(i: OpNo, Val: NewOp);
3579 }
3580 if (Named.insert(Ptr: I).second)
3581 emitAssignName(CLOpts: TM.getCLOpts(), I, B);
3582}
3583
3584Type *SPIRVEmitIntrinsicsImpl::deduceFunParamElementType(Function *F,
3585 unsigned OpIdx) {
3586 SmallPtrSet<Function *, 0> FVisited;
3587 return deduceFunParamElementType(F, OpIdx, FVisited);
3588}
3589
3590Type *SPIRVEmitIntrinsicsImpl::deduceFunParamElementType(
3591 Function *F, unsigned OpIdx, SmallPtrSetImpl<Function *> &FVisited) {
3592 // maybe a cycle
3593 if (!FVisited.insert(Ptr: F).second)
3594 return nullptr;
3595
3596 SmallPtrSet<Value *, 0> Visited;
3597 SmallVector<std::pair<Function *, unsigned>> Lookup;
3598 // search in function's call sites
3599 for (User *U : F->users()) {
3600 CallInst *CI = dyn_cast<CallInst>(Val: U);
3601 if (!CI || OpIdx >= CI->arg_size())
3602 continue;
3603 Value *OpArg = CI->getArgOperand(i: OpIdx);
3604 if (!isPointerTy(T: OpArg->getType()))
3605 continue;
3606 // maybe we already know operand's element type
3607 if (Type *KnownTy = GR->findDeducedElementType(Val: OpArg))
3608 return KnownTy;
3609 // try to deduce from the operand itself
3610 Visited.clear();
3611 if (Type *Ty = deduceElementTypeHelper(I: OpArg, Visited, UnknownElemTypeI8: false))
3612 return Ty;
3613 // search in actual parameter's users
3614 for (User *OpU : OpArg->users()) {
3615 Instruction *Inst = dyn_cast<Instruction>(Val: OpU);
3616 if (!Inst || Inst == CI)
3617 continue;
3618 Visited.clear();
3619 if (Type *Ty = deduceElementTypeHelper(I: Inst, Visited, UnknownElemTypeI8: false))
3620 return Ty;
3621 }
3622 // check if it's a formal parameter of the outer function
3623 if (!CI->getParent() || !CI->getParent()->getParent())
3624 continue;
3625 Function *OuterF = CI->getParent()->getParent();
3626 if (FVisited.find(Ptr: OuterF) != FVisited.end())
3627 continue;
3628 for (unsigned i = 0; i < OuterF->arg_size(); ++i) {
3629 if (OuterF->getArg(i) == OpArg) {
3630 Lookup.push_back(Elt: std::make_pair(x&: OuterF, y&: i));
3631 break;
3632 }
3633 }
3634 }
3635
3636 // search in function parameters
3637 for (auto &Pair : Lookup) {
3638 if (Type *Ty = deduceFunParamElementType(F: Pair.first, OpIdx: Pair.second, FVisited))
3639 return Ty;
3640 }
3641
3642 return nullptr;
3643}
3644
3645void SPIRVEmitIntrinsicsImpl::processParamTypesByFunHeader(Function *F,
3646 IRBuilder<> &B) {
3647 B.SetInsertPointPastAllocas(F);
3648 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
3649 Argument *Arg = F->getArg(i: OpIdx);
3650 // Vector-of-pointers arg: deduce pointee from a GEP user so the function
3651 // type isn't emitted with the default i8 pointee.
3652 if (isUntypedPointerVectorTy(T: Arg->getType()) &&
3653 !GR->findDeducedElementType(Val: Arg)) {
3654 for (User *U : Arg->users()) {
3655 auto *GEP = dyn_cast<GetElementPtrInst>(Val: U);
3656 if (GEP && GEP->getPointerOperand() == Arg) {
3657 GR->buildAssignPtr(B, ElemTy: GEP->getSourceElementType(), Arg);
3658 break;
3659 }
3660 }
3661 continue;
3662 }
3663 if (!isUntypedPointerTy(T: Arg->getType()))
3664 continue;
3665 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
3666 if (ElemTy)
3667 continue;
3668 if (hasPointeeTypeAttr(Arg) &&
3669 (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) {
3670 GR->buildAssignPtr(B, ElemTy, Arg);
3671 continue;
3672 }
3673 // search in function's call sites
3674 for (User *U : F->users()) {
3675 CallInst *CI = dyn_cast<CallInst>(Val: U);
3676 if (!CI || OpIdx >= CI->arg_size())
3677 continue;
3678 Value *OpArg = CI->getArgOperand(i: OpIdx);
3679 if (!isPointerTy(T: OpArg->getType()))
3680 continue;
3681 // maybe we already know operand's element type
3682 if ((ElemTy = GR->findDeducedElementType(Val: OpArg)) != nullptr)
3683 break;
3684 }
3685 if (ElemTy) {
3686 GR->buildAssignPtr(B, ElemTy, Arg);
3687 continue;
3688 }
3689 if (HaveFunPtrs) {
3690 for (User *U : Arg->users()) {
3691 CallInst *CI = dyn_cast<CallInst>(Val: U);
3692 if (CI && !isa<IntrinsicInst>(Val: CI) && CI->isIndirectCall() &&
3693 CI->getCalledOperand() == Arg &&
3694 CI->getParent()->getParent() == CurrF) {
3695 SmallVector<std::pair<Value *, unsigned>> Ops;
3696 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy&: ElemTy, IsPostprocessing: false);
3697 if (ElemTy) {
3698 GR->buildAssignPtr(B, ElemTy, Arg);
3699 break;
3700 }
3701 }
3702 }
3703 }
3704 }
3705}
3706
3707void SPIRVEmitIntrinsicsImpl::processParamTypes(Function *F, IRBuilder<> &B) {
3708 B.SetInsertPointPastAllocas(F);
3709 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) {
3710 Argument *Arg = F->getArg(i: OpIdx);
3711 if (!isUntypedPointerTy(T: Arg->getType()))
3712 continue;
3713 Type *ElemTy = GR->findDeducedElementType(Val: Arg);
3714 if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) {
3715 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Arg)) {
3716 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
3717 GR->updateAssignType(
3718 AssignCI, Arg,
3719 OfType: getNormalizedPoisonValue(Ty: ElemTy, CanUseAnyVectorRank));
3720 propagateElemType(Op: Arg, ElemTy: IntegerType::getInt8Ty(C&: F->getContext()),
3721 VisitedSubst);
3722 } else {
3723 GR->buildAssignPtr(B, ElemTy, Arg);
3724 }
3725 }
3726 }
3727}
3728
3729static FunctionType *getFunctionPointerElemType(Function *F,
3730 SPIRVGlobalRegistry *GR) {
3731 FunctionType *FTy = F->getFunctionType();
3732 bool IsNewFTy = false;
3733 SmallVector<Type *, 4> ArgTys;
3734 for (Argument &Arg : F->args()) {
3735 Type *ArgTy = Arg.getType();
3736 if (ArgTy->isPointerTy())
3737 if (Type *ElemTy = GR->findDeducedElementType(Val: &Arg)) {
3738 IsNewFTy = true;
3739 ArgTy = getTypedPointerWrapper(ElemTy, AS: getPointerAddressSpace(T: ArgTy));
3740 }
3741 ArgTys.push_back(Elt: ArgTy);
3742 }
3743 return IsNewFTy
3744 ? FunctionType::get(Result: FTy->getReturnType(), Params: ArgTys, isVarArg: FTy->isVarArg())
3745 : FTy;
3746}
3747
3748bool SPIRVEmitIntrinsicsImpl::processFunctionPointers(Module &M) {
3749 SmallVector<Function *> Worklist;
3750 for (auto &F : M) {
3751 if (F.isIntrinsic())
3752 continue;
3753 if (F.isDeclaration()) {
3754 for (User *U : F.users()) {
3755 // SPIR-V has no personality, so it is not a function pointer use.
3756 if (auto *UF = dyn_cast<Function>(Val: U);
3757 UF && UF->hasPersonalityFn() && UF->getPersonalityFn() == &F)
3758 continue;
3759 CallInst *CI = dyn_cast<CallInst>(Val: U);
3760 if (!CI || CI->getCalledFunction() != &F) {
3761 Worklist.push_back(Elt: &F);
3762 break;
3763 }
3764 }
3765 } else {
3766 if (F.user_empty())
3767 continue;
3768 Type *FPElemTy = GR->findDeducedElementType(Val: &F);
3769 if (!FPElemTy)
3770 FPElemTy = getFunctionPointerElemType(F: &F, GR);
3771 for (User *U : F.users()) {
3772 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U);
3773 if (!II || II->arg_size() != 3 || II->getOperand(i_nocapture: 0) != &F)
3774 continue;
3775 if (II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type ||
3776 II->getIntrinsicID() == Intrinsic::spv_ptrcast) {
3777 GR->updateAssignType(
3778 AssignCI: II, Arg: &F, OfType: getNormalizedPoisonValue(Ty: FPElemTy, CanUseAnyVectorRank));
3779 break;
3780 }
3781 }
3782 }
3783 }
3784 if (Worklist.empty())
3785 return false;
3786
3787 LLVMContext &Ctx = M.getContext();
3788 Function *SF = getOrCreateBackendServiceFunction(M);
3789 BasicBlock *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: SF);
3790 IRBuilder<> IRB(BB);
3791
3792 for (Function *F : Worklist) {
3793 SmallVector<Value *> Args;
3794 for (const auto &Arg : F->args())
3795 Args.push_back(
3796 Elt: getNormalizedPoisonValue(Ty: Arg.getType(), CanUseAnyVectorRank));
3797 IRB.CreateCall(Callee: F, Args)->setTailCall(true);
3798 }
3799 IRB.CreateRetVoid();
3800
3801 return true;
3802}
3803
3804// Apply types parsed from demangled function declarations.
3805void SPIRVEmitIntrinsicsImpl::applyDemangledPtrArgTypes(IRBuilder<> &B) {
3806 DenseMap<Function *, CallInst *> Ptrcasts;
3807 for (auto It : FDeclPtrTys) {
3808 Function *F = It.first;
3809 for (auto *U : F->users()) {
3810 CallInst *CI = dyn_cast<CallInst>(Val: U);
3811 if (!CI || CI->getCalledFunction() != F)
3812 continue;
3813 unsigned Sz = CI->arg_size();
3814 for (auto [Idx, ElemTy] : It.second) {
3815 if (Idx >= Sz)
3816 continue;
3817 Value *Param = CI->getArgOperand(i: Idx);
3818 if (GR->findDeducedElementType(Val: Param) || isa<GlobalValue>(Val: Param))
3819 continue;
3820 if (Argument *Arg = dyn_cast<Argument>(Val: Param)) {
3821 if (!hasPointeeTypeAttr(Arg)) {
3822 B.SetInsertPointPastAllocas(Arg->getParent());
3823 B.SetCurrentDebugLocation(DebugLoc());
3824 GR->buildAssignPtr(B, ElemTy, Arg);
3825 }
3826 } else if (isaGEP(V: Param) || isa<AllocaInst>(Val: Param)) {
3827 // The allocated type defines the size of an alloca, so cast at the
3828 // call instead of overriding its pointee type.
3829 replaceUsesOfWithSpvPtrcast(
3830 Op: Param, ElemTy: normalizeType(Ty: ElemTy, CanUseAnyVectorRank), I: CI, Ptrcasts);
3831 } else if (isa<Instruction>(Val: Param)) {
3832 GR->addDeducedElementType(Val: Param,
3833 Ty: normalizeType(Ty: ElemTy, CanUseAnyVectorRank));
3834 // insertAssignTypeIntrs() will complete buildAssignPtr()
3835 } else {
3836 B.SetInsertPoint(CI->getParent()
3837 ->getParent()
3838 ->getEntryBlock()
3839 .getFirstNonPHIOrAlloca());
3840 GR->buildAssignPtr(B, ElemTy, Arg: Param);
3841 }
3842 CallInst *Ref = dyn_cast<CallInst>(Val: Param);
3843 if (!Ref)
3844 continue;
3845 Function *RefF = Ref->getCalledFunction();
3846 if (!RefF || !isPointerTy(T: RefF->getReturnType()) ||
3847 GR->findDeducedElementType(Val: RefF))
3848 continue;
3849 ElemTy = normalizeType(Ty: ElemTy, CanUseAnyVectorRank);
3850 GR->addDeducedElementType(Val: RefF, Ty: ElemTy);
3851 GR->addReturnType(
3852 ArgF: RefF, DerivedTy: TypedPointerType::get(
3853 ElementType: ElemTy, AddressSpace: getPointerAddressSpace(T: RefF->getReturnType())));
3854 }
3855 }
3856 }
3857 FDeclPtrTys.clear();
3858}
3859
3860GetElementPtrInst *SPIRVEmitIntrinsicsImpl::simplifyZeroLengthArrayGepInst(
3861 GetElementPtrInst *GEP) {
3862 // getelementptr [0 x T], P, 0 (zero), I -> getelementptr T, P, I.
3863 // If type is 0-length array and first index is 0 (zero), drop both the
3864 // 0-length array type and the first index. This is a common pattern in
3865 // the IR, e.g. when using a zero-length array as a placeholder for a
3866 // flexible array such as unbound arrays.
3867 assert(GEP && "GEP is null");
3868 Type *SrcTy = GEP->getSourceElementType();
3869 SmallVector<Value *, 8> Indices(GEP->indices());
3870 ArrayType *ArrTy = dyn_cast<ArrayType>(Val: SrcTy);
3871 if (ArrTy && ArrTy->getNumElements() == 0 && match(V: Indices[0], P: m_Zero())) {
3872 Indices.erase(CI: Indices.begin());
3873 SrcTy = ArrTy->getElementType();
3874 return GetElementPtrInst::Create(PointeeType: SrcTy, Ptr: GEP->getPointerOperand(), IdxList: Indices,
3875 NW: GEP->getNoWrapFlags(), NameStr: "",
3876 InsertBefore: GEP->getIterator());
3877 }
3878 return nullptr;
3879}
3880
3881void SPIRVEmitIntrinsicsImpl::emitUnstructuredLoopControls(Function &F,
3882 IRBuilder<> &B) {
3883 const SPIRVSubtarget *ST = TM.getSubtargetImpl(F);
3884 // Shaders use SPIRVStructurizer which emits OpLoopMerge via spv_loop_merge.
3885 if (ST->isShader())
3886 return;
3887
3888 if (ST->canUseExtension(
3889 E: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls)) {
3890 for (BasicBlock &BB : F) {
3891 Instruction *Term = BB.getTerminator();
3892 MDNode *LoopMD = Term->getMetadata(KindID: LLVMContext::MD_loop);
3893 if (!LoopMD)
3894 continue;
3895
3896 SmallVector<unsigned, 1> Ops =
3897 getSpirvLoopControlOperandsFromLoopMetadata(LoopMD);
3898 unsigned LC = Ops[0];
3899 if (LC == SPIRV::LoopControl::None)
3900 continue;
3901
3902 // Emit intrinsic: loop control mask + optional parameters.
3903 B.SetInsertPoint(Term);
3904 SmallVector<Value *, 4> IntrArgs;
3905 for (unsigned Op : Ops)
3906 IntrArgs.push_back(Elt: B.getInt32(C: Op));
3907 B.CreateIntrinsic(ID: Intrinsic::spv_loop_control_intel, Args: IntrArgs);
3908 }
3909 return;
3910 }
3911
3912 // For non-shader targets without the Intel extension, emit OpLoopMerge
3913 // using spv_loop_merge intrinsics, mirroring the structurizer approach.
3914 LoopInfo LI;
3915 LI.analyze(F: &F);
3916 if (LI.empty())
3917 return;
3918
3919 SmallPtrSet<BasicBlock *, 8> UsedMergeBlocks;
3920 for (Loop *L : LI.getLoopsInPreorder()) {
3921 BasicBlock *Latch = L->getLoopLatch();
3922 if (!Latch)
3923 continue;
3924
3925 // Check for loop unroll metadata on the latch terminator.
3926 SmallVector<unsigned, 1> LoopControlOps =
3927 getSpirvLoopControlOperandsFromLoopMetadata(L);
3928 if (LoopControlOps[0] == SPIRV::LoopControl::None)
3929 continue;
3930
3931 BasicBlock *Header = L->getHeader();
3932 // OpLoopMerge must immediately precede an OpBranch or OpBranchConditional.
3933 // Switches are already lowered to spv_switch + indirectbr at this point.
3934 if (!isa<UncondBrInst, CondBrInst>(Val: Header->getTerminator()))
3935 continue;
3936
3937 BasicBlock *MergeBlock = L->getUniqueExitBlock();
3938 // LoopSimplify does not guarantee a unique exit block. Try the normal exit
3939 // of a rotated loop (from the latch), then an unrotated loop (from the
3940 // header). A merge block cannot be shared by multiple loop headers.
3941 for (BasicBlock *BB : {Latch, Header}) {
3942 if (MergeBlock || !isa<CondBrInst>(Val: BB->getTerminator()))
3943 continue;
3944 for (BasicBlock *Succ : successors(BB))
3945 if (!L->contains(BB: Succ) && !UsedMergeBlocks.contains(Ptr: Succ))
3946 MergeBlock = Succ;
3947 }
3948 if (!MergeBlock || !UsedMergeBlocks.insert(Ptr: MergeBlock).second)
3949 continue;
3950
3951 B.SetInsertPoint(Header->getTerminator());
3952 auto *MergeAddress = BlockAddress::get(F: &F, BB: MergeBlock);
3953 auto *ContinueAddress = BlockAddress::get(F: &F, BB: Latch);
3954 SmallVector<Value *, 4> Args = {MergeAddress, ContinueAddress};
3955 for (unsigned Imm : LoopControlOps)
3956 Args.emplace_back(Args: B.getInt32(C: Imm));
3957 B.CreateIntrinsic(ID: Intrinsic::spv_loop_merge, Args: {Args});
3958 }
3959}
3960
3961bool SPIRVEmitIntrinsicsImpl::runOnFunction(Function &Func) {
3962 if (Func.isDeclaration())
3963 return false;
3964
3965 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: Func);
3966 // LoopSimplify runs after SPIRVPrepareFunctions sorted the blocks, and the
3967 // preheaders/dedicated exits it creates can end up before their dominator,
3968 // which SPIR-V forbids.
3969 if (!ST.isShader())
3970 sortBlocks(F&: Func);
3971 GR = ST.getSPIRVGlobalRegistry();
3972
3973 if (!CurrF)
3974 HaveFunPtrs =
3975 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers);
3976
3977 CanUseAnyVectorRank =
3978 ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector);
3979 CurrF = &Func;
3980 IRBuilder<> B(*Func.getParent());
3981 AggrConsts.clear();
3982 AggrConstTypes.clear();
3983 AggrStores.clear();
3984
3985 scalarizeVectorPtrCasts(F&: Func);
3986 processParamTypesByFunHeader(F: CurrF, B);
3987
3988 // Fix GEP result types ahead of inference, and simplify if possible.
3989 // Data structure for dead instructions that were simplified and replaced.
3990 SmallPtrSet<Instruction *, 4> DeadInsts;
3991 for (auto &I : instructions(F&: Func)) {
3992 if (StoreInst *SI = dyn_cast<StoreInst>(Val: &I)) {
3993 Type *ElTy = SI->getValueOperand()->getType();
3994 if (ElTy->isAggregateType() || ElTy->isVectorTy())
3995 AggrStores.insert(Ptr: &I);
3996 continue;
3997 }
3998
3999 auto *GEP = dyn_cast<GetElementPtrInst>(Val: &I);
4000 auto *SGEP = dyn_cast<StructuredGEPInst>(Val: &I);
4001
4002 if ((!GEP && !SGEP) || GR->findDeducedElementType(Val: &I))
4003 continue;
4004
4005 if (SGEP) {
4006 GR->addDeducedElementType(
4007 Val: SGEP,
4008 Ty: normalizeType(Ty: SGEP->getResultElementType(), CanUseAnyVectorRank));
4009 continue;
4010 }
4011
4012 GetElementPtrInst *NewGEP = simplifyZeroLengthArrayGepInst(GEP);
4013 if (NewGEP) {
4014 GEP->replaceAllUsesWith(V: NewGEP);
4015 DeadInsts.insert(Ptr: GEP);
4016 GEP = NewGEP;
4017 }
4018 if (Type *GepTy = getGEPType(Ref: GEP))
4019 GR->addDeducedElementType(Val: GEP, Ty: normalizeType(Ty: GepTy, CanUseAnyVectorRank));
4020 }
4021 // Remove dead instructions that were simplified and replaced.
4022 for (auto *I : DeadInsts) {
4023 assert(I->use_empty() && "Dead instruction should not have any uses left");
4024 I->eraseFromParent();
4025 }
4026 if (AccessChainErrors)
4027 return true;
4028
4029 B.SetInsertPoint(Func.getEntryBlock().begin());
4030 for (auto &GV : Func.getParent()->globals())
4031 processGlobalValue(GV, B);
4032
4033 reconstructAggregateReturns(Func, B);
4034 preprocessUndefsAndPoisons(B);
4035 simplifyNullAddrSpaceCasts();
4036 preprocessCompositeConstants(B);
4037
4038 // A PHINode, SelectInst or FreezeInst takes its result type from its
4039 // operands. Aggregate arms are lowered to i32 value-ids (composite constants
4040 // here, loads and other producers during the visitor pass below), so mutate
4041 // an aggregate PHI, select or freeze to match. The original type is tracked
4042 // in AggrConstTypes (used to assign the SPIR-V type) and its extractvalue
4043 // users are lowered to spv_extractv.
4044 Type *I32Ty = B.getInt32Ty();
4045 for (Instruction &I : instructions(F&: Func)) {
4046 if (!isAggregateValueIdInstr(I))
4047 continue;
4048 // Give multi-register arms a value-id first, before the result is mutated.
4049 insertCompositeAggregateArms(I: &I, B);
4050 AggrConstTypes[&I] = I.getType();
4051 I.mutateType(Ty: I32Ty);
4052 }
4053
4054 preprocessBoolVectorBitcasts(F&: Func);
4055 SmallVector<Instruction *> Worklist(
4056 llvm::make_pointer_range(Range: instructions(F&: Func)));
4057
4058 applyDemangledPtrArgTypes(B);
4059
4060 // Pass forward: use operand to deduce instructions result.
4061 for (auto &I : Worklist) {
4062 // Don't emit intrinsincs for convergence intrinsics.
4063 if (isConvergenceIntrinsic(I))
4064 continue;
4065
4066 bool Postpone = insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: false);
4067 // if Postpone is true, we can't decide on pointee type yet
4068 insertAssignTypeIntrs(I, B);
4069 insertPtrCastOrAssignTypeInstr(I, B);
4070 insertSpirvDecorations(I, B);
4071 // if instruction requires a pointee type set, let's check if we know it
4072 // already, and force it to be i8 if not
4073 if (Postpone && !GR->findAssignPtrTypeInstr(Val: I))
4074 insertAssignPtrTypeIntrs(I, B, UnknownElemTypeI8: true);
4075
4076 if (auto *FPI = dyn_cast<ConstrainedFPIntrinsic>(Val: I))
4077 useRoundingMode(FPI, B);
4078 }
4079
4080 // Pass backward: use instructions results to specify/update/cast operands
4081 // where needed.
4082 SmallPtrSet<Instruction *, 4> IncompleteRets;
4083 for (auto &I : llvm::reverse(C: instructions(F&: Func)))
4084 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets);
4085
4086 // Pass forward for PHIs only, their operands are not preceed the
4087 // instruction in meaning of `instructions(Func)`.
4088 for (BasicBlock &BB : Func)
4089 for (PHINode &Phi : BB.phis())
4090 if (isPointerTy(T: Phi.getType()))
4091 deduceOperandElementType(I: &Phi, IncompleteRets: nullptr);
4092
4093 for (auto *I : Worklist) {
4094 TrackConstants = true;
4095 if (!I->getType()->isVoidTy() || isa<StoreInst>(Val: I))
4096 setInsertPointAfterDef(B, I);
4097 // Visitors return either the original/newly created instruction for
4098 // further processing, nullptr otherwise.
4099 I = visit(I&: *I);
4100 if (AccessChainErrors)
4101 return true;
4102 if (!I)
4103 continue;
4104
4105 // Don't emit intrinsics for convergence operations.
4106 if (isConvergenceIntrinsic(I))
4107 continue;
4108
4109 addSaturatedDecorationToIntrinsic(I, B);
4110 processInstrAfterVisit(I, B);
4111 }
4112
4113 emitUnstructuredLoopControls(F&: Func, B);
4114
4115 return true;
4116}
4117
4118// Try to deduce a better type for pointers to untyped ptr.
4119bool SPIRVEmitIntrinsicsImpl::postprocessTypes(Module &M) {
4120 if (!GR || TodoTypeSz == 0)
4121 return false;
4122
4123 unsigned SzTodo = TodoTypeSz;
4124 DenseMap<Value *, SmallPtrSet<Value *, 4>> ToProcess;
4125 for (auto [Op, Enabled] : TodoType) {
4126 // TODO: add isa<CallInst>(Op) to continue
4127 if (!Enabled || isaGEP(V: Op))
4128 continue;
4129 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Val: Op);
4130 Type *KnownTy = GR->findDeducedElementType(Val: Op);
4131 if (!KnownTy || !AssignCI)
4132 continue;
4133 assert(Op == AssignCI->getArgOperand(0));
4134 // Try to improve the type deduced after all Functions are processed.
4135 if (auto *CI = dyn_cast<Instruction>(Val: Op)) {
4136 CurrF = CI->getParent()->getParent();
4137 SmallPtrSet<Value *, 0> Visited;
4138 if (Type *ElemTy = deduceElementTypeHelper(I: Op, Visited, UnknownElemTypeI8: false, IgnoreKnownType: true)) {
4139 if (ElemTy != KnownTy) {
4140 DenseSet<std::pair<Value *, Value *>> VisitedSubst;
4141 propagateElemType(Op: CI, ElemTy, VisitedSubst);
4142 eraseTodoType(Op);
4143 continue;
4144 }
4145 }
4146 }
4147
4148 if (Op->hasUseList()) {
4149 for (User *U : Op->users()) {
4150 Instruction *Inst = dyn_cast<Instruction>(Val: U);
4151 if (Inst && !isa<IntrinsicInst>(Val: Inst))
4152 ToProcess[Inst].insert(Ptr: Op);
4153 }
4154 }
4155 }
4156 if (TodoTypeSz == 0)
4157 return true;
4158
4159 for (auto &F : M) {
4160 CurrF = &F;
4161 SmallPtrSet<Instruction *, 4> IncompleteRets;
4162 for (auto &I : llvm::reverse(C: instructions(F))) {
4163 auto It = ToProcess.find(Val: &I);
4164 if (It == ToProcess.end())
4165 continue;
4166 It->second.remove_if(P: [this](Value *V) { return !isTodoType(Op: V); });
4167 if (It->second.size() == 0)
4168 continue;
4169 deduceOperandElementType(I: &I, IncompleteRets: &IncompleteRets, AskOps: &It->second, IsPostprocessing: true);
4170 if (TodoTypeSz == 0)
4171 return true;
4172 }
4173 }
4174
4175 return SzTodo > TodoTypeSz;
4176}
4177
4178// Parse unambiguous scalar/vector pointees, e.g. "float vector[4] const*".
4179static Type *parseDeclPtrElemType(StringRef TypeStr, LLVMContext &Ctx) {
4180 if (!TypeStr.consume_back(Suffix: "*") && !TypeStr.consume_back(Suffix: "&"))
4181 return nullptr;
4182 StringRef ScalarName = TypeStr;
4183 Type *ElemTy = parseBasicTypeName(TypeName&: TypeStr, Ctx);
4184 if (!ElemTy || ElemTy->isVoidTy() || ElemTy->isIntegerTy(BitWidth: 1))
4185 return nullptr;
4186 ScalarName = ScalarName.drop_back(N: TypeStr.size());
4187 // OpenCL aliases such as uint can also name C++ classes.
4188 if (!StringSwitch<bool>(ScalarName)
4189 .Cases(CaseStrings: {"char", "signed char", "unsigned char", "short",
4190 "unsigned short", "int", "unsigned int", "_Float16", "float",
4191 "double"},
4192 Value: true)
4193 .Default(Value: false) ||
4194 (!TypeStr.empty() && !TypeStr.starts_with(Prefix: " ")))
4195 return nullptr;
4196 if (TypeStr.consume_front(Prefix: " vector[")) {
4197 auto [Count, Rest] = TypeStr.split(Separator: ']');
4198 unsigned NumElts = 0;
4199 if (Count.getAsInteger(Radix: 10, Result&: NumElts) || NumElts == 0)
4200 return nullptr;
4201 ElemTy = FixedVectorType::get(ElementType: ElemTy, NumElts);
4202 TypeStr = Rest;
4203 }
4204 // Only cv and address space qualifiers may follow.
4205 SmallVector<StringRef, 4> Quals;
4206 TypeStr.split(A&: Quals, Separator: ' ', MaxSplit: -1, /*KeepEmpty=*/false);
4207 for (StringRef Qual : Quals) {
4208 unsigned AS;
4209 if (Qual != "const" && Qual != "volatile" && Qual != "restrict" &&
4210 !(Qual.consume_front(Prefix: "AS") && !Qual.getAsInteger(Radix: 10, Result&: AS)))
4211 return nullptr;
4212 }
4213 return ElemTy;
4214}
4215
4216// Parse and store argument types of function declarations where needed.
4217void SPIRVEmitIntrinsicsImpl::parseFunDeclarations(Module &M) {
4218 for (auto &F : M) {
4219 if (!F.isDeclaration() || F.isIntrinsic())
4220 continue;
4221 std::string DemangledName = getOclOrSpirvBuiltinDemangledName(Name: F.getName());
4222 if (DemangledName.empty())
4223 continue;
4224 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F);
4225 auto Set = ST.getPreferredInstructionSet();
4226 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode(DemangledCall: DemangledName, Set);
4227 // Some builtins have no opcode mapping.
4228 const bool IsBuiltin = Grp >= 0 || SPIRV::isBuiltin(DemangledCall: DemangledName, Set);
4229 if (IsBuiltin && Opcode != SPIRV::OpGroupAsyncCopy)
4230 continue;
4231 SmallVector<unsigned> Idxs;
4232 for (Argument &Arg : F.args())
4233 if (isPointerTy(T: Arg.getType()) && !hasPointeeTypeAttr(Arg: &Arg))
4234 Idxs.push_back(Elt: Arg.getArgNo());
4235 if (Idxs.empty())
4236 continue;
4237 LLVMContext &Ctx = F.getContext();
4238 SmallVector<StringRef, 10> TypeStrs;
4239 SPIRV::parseBuiltinTypeStr(BuiltinArgsTypeStrs&: TypeStrs, DemangledCall: DemangledName, Ctx);
4240 if (TypeStrs.empty())
4241 continue;
4242 // Avoid nested signatures and hidden ABI arguments such as this and sret.
4243 if (!IsBuiltin && (!F.getName().starts_with(Prefix: "_Z") || F.hasStructRetAttr() ||
4244 StringRef(DemangledName).count(C: '(') != 1 ||
4245 StringRef(DemangledName).count(C: ')') != 1 ||
4246 StringRef(DemangledName).contains(C: '<') ||
4247 TypeStrs.size() != F.arg_size()))
4248 continue;
4249 bool AllowLongVectors =
4250 ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector);
4251 for (unsigned Idx : Idxs) {
4252 if (Idx >= TypeStrs.size())
4253 continue;
4254 StringRef TypeStr = TypeStrs[Idx].trim();
4255 Type *ElemTy = IsBuiltin
4256 ? SPIRV::parseBuiltinCallArgumentType(TypeStr, Ctx)
4257 : parseDeclPtrElemType(TypeStr, Ctx);
4258 if (!ElemTy || !TypedPointerType::isValidElementType(ElemTy) ||
4259 ElemTy->isTargetExtTy())
4260 continue;
4261 if (!IsBuiltin) {
4262 ElemTy = normalizeType(Ty: ElemTy, CanUseAnyVectorRank: AllowLongVectors);
4263 if (!AllowLongVectors && isLongVectorEXT(Ty: ElemTy))
4264 continue;
4265 }
4266 FDeclPtrTys[&F].emplace_back(Args&: Idx, Args&: ElemTy);
4267 if (!IsBuiltin)
4268 ST.getSPIRVGlobalRegistry()->addDeducedElementType(Val: F.getArg(i: Idx),
4269 Ty: ElemTy);
4270 }
4271 }
4272}
4273
4274bool SPIRVEmitIntrinsicsImpl::processMaskedMemIntrinsic(IntrinsicInst &I) {
4275 const SPIRVSubtarget &ST = TM.getSubtarget<SPIRVSubtarget>(F: *I.getFunction());
4276
4277 if (I.getIntrinsicID() == Intrinsic::masked_gather) {
4278 if (!ST.canUseExtension(
4279 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
4280 I.getContext().emitError(
4281 I: &I, ErrorStr: "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter "
4282 "extension");
4283 // Replace with poison to allow compilation to continue and report error.
4284 I.replaceAllUsesWith(V: PoisonValue::get(T: I.getType()));
4285 I.eraseFromParent();
4286 return true;
4287 }
4288
4289 IRBuilder<> B(&I);
4290
4291 Value *Ptrs = I.getArgOperand(i: 0);
4292 Value *Mask = I.getArgOperand(i: 1);
4293 Value *Passthru = I.getArgOperand(i: 2);
4294
4295 // Alignment is stored as a parameter attribute, not as a regular parameter.
4296 uint32_t Alignment = I.getParamAlign(ArgNo: 0).valueOrOne().value();
4297
4298 SmallVector<Value *, 4> Args = {Ptrs, B.getInt32(C: Alignment), Mask,
4299 Passthru};
4300 SmallVector<Type *, 4> Types = {I.getType(), Ptrs->getType(),
4301 Mask->getType(), Passthru->getType()};
4302
4303 auto *NewI = B.CreateIntrinsic(ID: Intrinsic::spv_masked_gather, OverloadTypes: Types, Args);
4304 I.replaceAllUsesWith(V: NewI);
4305 I.eraseFromParent();
4306 return true;
4307 }
4308
4309 if (I.getIntrinsicID() == Intrinsic::masked_scatter) {
4310 if (!ST.canUseExtension(
4311 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
4312 I.getContext().emitError(
4313 I: &I, ErrorStr: "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter "
4314 "extension");
4315 // Erase the intrinsic to allow compilation to continue and report error.
4316 I.eraseFromParent();
4317 return true;
4318 }
4319
4320 IRBuilder<> B(&I);
4321
4322 Value *Values = I.getArgOperand(i: 0);
4323 Value *Ptrs = I.getArgOperand(i: 1);
4324 Value *Mask = I.getArgOperand(i: 2);
4325
4326 // Alignment is stored as a parameter attribute on the ptrs parameter (arg
4327 // 1).
4328 uint32_t Alignment = I.getParamAlign(ArgNo: 1).valueOrOne().value();
4329
4330 SmallVector<Value *, 4> Args = {Values, Ptrs, B.getInt32(C: Alignment), Mask};
4331 SmallVector<Type *, 3> Types = {Values->getType(), Ptrs->getType(),
4332 Mask->getType()};
4333
4334 B.CreateIntrinsic(ID: Intrinsic::spv_masked_scatter, OverloadTypes: Types, Args);
4335 I.eraseFromParent();
4336 return true;
4337 }
4338
4339 return false;
4340}
4341
4342// SPIR-V doesn't support bitcasts involving vector boolean type. Decompose such
4343// bitcasts into element-wise operations before building instructions
4344// worklist, so new instructions are properly visited and converted to
4345// SPIR-V intrinsics.
4346void SPIRVEmitIntrinsicsImpl::preprocessBoolVectorBitcasts(Function &F) {
4347 struct BoolVecBitcast {
4348 BitCastInst *BC;
4349 FixedVectorType *BoolVecTy;
4350 bool SrcIsBoolVec;
4351 };
4352
4353 auto getAsBoolVec = [](Type *Ty) -> FixedVectorType * {
4354 auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
4355 return (VTy && VTy->getElementType()->isIntegerTy(BitWidth: 1)) ? VTy : nullptr;
4356 };
4357
4358 SmallVector<BoolVecBitcast, 4> ToReplace;
4359 for (auto &I : instructions(F)) {
4360 auto *BC = dyn_cast<BitCastInst>(Val: &I);
4361 if (!BC)
4362 continue;
4363 if (auto *BVTy = getAsBoolVec(BC->getSrcTy()))
4364 ToReplace.push_back(Elt: {.BC: BC, .BoolVecTy: BVTy, .SrcIsBoolVec: true});
4365 else if (auto *BVTy = getAsBoolVec(BC->getDestTy()))
4366 ToReplace.push_back(Elt: {.BC: BC, .BoolVecTy: BVTy, .SrcIsBoolVec: false});
4367 }
4368
4369 for (auto &[BC, BoolVecTy, SrcIsBoolVec] : ToReplace) {
4370 IRBuilder<> B(BC);
4371 Value *Src = BC->getOperand(i_nocapture: 0);
4372 unsigned BoolVecN = BoolVecTy->getNumElements();
4373 // Convert per element of the other side, as iN can exceed the maximum
4374 // SPIR-V integer width.
4375 Type *OtherTy = SrcIsBoolVec ? BC->getDestTy() : Src->getType();
4376 unsigned NumElts = 1;
4377 if (auto *VTy = dyn_cast<FixedVectorType>(Val: OtherTy))
4378 NumElts = VTy->getNumElements();
4379 unsigned EltBits = OtherTy->getScalarSizeInBits();
4380 Type *EltTy = B.getIntNTy(N: EltBits);
4381 Type *IntTy = NumElts > 1 ? FixedVectorType::get(ElementType: EltTy, NumElts) : EltTy;
4382
4383 Value *Result;
4384 if (!SrcIsBoolVec) {
4385 Value *IntVal = B.CreateBitCast(V: Src, DestTy: IntTy);
4386 // Lane I is bit I % EltBits of element I / EltBits.
4387 auto GetLane = [&](unsigned I) {
4388 Value *Elt = IntVal;
4389 if (NumElts > 1)
4390 Elt = B.CreateExtractElement(Vec: IntVal, Idx: B.getInt32(C: I / EltBits));
4391 Value *Mask =
4392 ConstantInt::get(Ty: EltTy, V: APInt::getOneBitSet(numBits: EltBits, BitNo: I % EltBits));
4393 Value *And = B.CreateAnd(LHS: Elt, RHS: Mask);
4394 return B.CreateICmpNE(LHS: And, RHS: ConstantInt::get(Ty: EltTy, V: 0));
4395 };
4396
4397 // Compute constant-index extracts directly, as no later pass removes
4398 // the unused lanes.
4399 for (User *U : make_early_inc_range(Range: BC->users())) {
4400 auto *EE = dyn_cast<ExtractElementInst>(Val: U);
4401 auto *Idx = EE ? dyn_cast<ConstantInt>(Val: EE->getIndexOperand()) : nullptr;
4402 if (!Idx || Idx->getValue().uge(RHS: BoolVecN))
4403 continue;
4404 B.SetInsertPoint(EE);
4405 EE->replaceAllUsesWith(V: GetLane(Idx->getZExtValue()));
4406 EE->eraseFromParent();
4407 }
4408 if (BC->use_empty()) {
4409 BC->eraseFromParent();
4410 continue;
4411 }
4412
4413 B.SetInsertPoint(BC);
4414 Result = PoisonValue::get(T: BoolVecTy);
4415 for (unsigned I = 0; I < BoolVecN; ++I)
4416 Result = B.CreateInsertElement(Vec: Result, NewElt: GetLane(I), Idx: B.getInt32(C: I));
4417 } else {
4418 // Build each element from its lanes.
4419 Value *IntVal = PoisonValue::get(T: IntTy);
4420 for (unsigned J = 0; J < NumElts; ++J) {
4421 Value *Elt = ConstantInt::get(Ty: EltTy, V: 0);
4422 for (unsigned Bit = 0; Bit < EltBits; ++Bit) {
4423 Value *Lane =
4424 B.CreateExtractElement(Vec: Src, Idx: B.getInt32(C: J * EltBits + Bit));
4425 Value *Ext = B.CreateZExt(V: Lane, DestTy: EltTy);
4426 if (Bit > 0)
4427 Ext = B.CreateShl(LHS: Ext, RHS: ConstantInt::get(Ty: EltTy, V: Bit));
4428 Elt = B.CreateOr(LHS: Elt, RHS: Ext);
4429 }
4430 IntVal = NumElts > 1 ? B.CreateInsertElement(Vec: IntVal, NewElt: Elt, Idx: B.getInt32(C: J))
4431 : Elt;
4432 }
4433 Result = B.CreateBitCast(V: IntVal, DestTy: BC->getDestTy());
4434 }
4435
4436 BC->replaceAllUsesWith(V: Result);
4437 BC->eraseFromParent();
4438 }
4439}
4440
4441bool SPIRVEmitIntrinsicsImpl::convertMaskedMemIntrinsics(Module &M) {
4442 bool Changed = false;
4443
4444 for (Function &F : make_early_inc_range(Range&: M)) {
4445 if (!F.isIntrinsic())
4446 continue;
4447 Intrinsic::ID IID = F.getIntrinsicID();
4448 if (IID != Intrinsic::masked_gather && IID != Intrinsic::masked_scatter)
4449 continue;
4450
4451 for (User *U : make_early_inc_range(Range: F.users())) {
4452 if (auto *II = dyn_cast<IntrinsicInst>(Val: U))
4453 Changed |= processMaskedMemIntrinsic(I&: *II);
4454 }
4455
4456 if (F.use_empty())
4457 F.eraseFromParent();
4458 }
4459
4460 return Changed;
4461}
4462
4463bool SPIRVEmitIntrinsicsImpl::preprocessModule(Module &M) {
4464 bool Changed = false;
4465 bool IsLogical = TM.getSubtargetImpl()->isLogicalSPIRV();
4466 SmallVector<Constant *> GEPs;
4467 Type *I32Ty = Type::getInt32Ty(C&: M.getContext());
4468 for (Function &F : M) {
4469 if (F.isDeclaration())
4470 continue;
4471 for (Instruction &I : instructions(F)) {
4472 if (IsLogical)
4473 for (Value *Op : I.operands())
4474 if (auto *CE = dyn_cast<ConstantExpr>(Val: Op);
4475 CE && CE->getOpcode() == Instruction::GetElementPtr)
4476 GEPs.push_back(Elt: CE);
4477
4478 // Only entry-block allocations execute once per call, so a single array
4479 // object has the same meaning as the counted allocation. Allocations in
4480 // other blocks keep their lowering.
4481 auto *AI = dyn_cast<AllocaInst>(Val: &I);
4482 if (!AI || !AI->isStaticAlloca() || !AI->isArrayAllocation() ||
4483 AI->getAddressSpace() !=
4484 storageClassToAddressSpace(SC: SPIRV::StorageClass::Function))
4485 continue;
4486 auto *Count = cast<ConstantInt>(Val: AI->getArraySize());
4487 // Zero sizes are legalized earlier; array lengths must fit in 32 bits.
4488 if (Count->isZero() || !Count->getValue().isIntN(N: 32))
4489 continue;
4490 AI->setAllocatedType(
4491 ArrayType::get(ElementType: AI->getAllocatedType(), NumElements: Count->getZExtValue()));
4492 AI->setOperand(i_nocapture: 0, Val_nocapture: ConstantInt::get(Ty: I32Ty, V: 1));
4493 Changed = true;
4494 }
4495 }
4496 Changed |= convertMaskedMemIntrinsics(M);
4497
4498 // Materialize GEPs before pointer type inference so logical targets use
4499 // the typed access-chain lowering. Physical targets keep the existing
4500 // constant-expression handling.
4501 if (IsLogical) {
4502 // The list can contain both a constant and its users.
4503 Changed |= convertUsersOfConstantsToInstructions(
4504 Consts: GEPs, RestrictToFunc: nullptr, /*RemoveDeadConstants=*/false, /*IncludeSelf=*/true);
4505 }
4506 return Changed;
4507}
4508
4509bool SPIRVEmitIntrinsicsImpl::runOnModule(Module &M) {
4510 bool Changed = preprocessModule(M);
4511
4512 parseFunDeclarations(M);
4513 insertConstantsForFPFastMathDefault(M);
4514 GVUsers.init(M);
4515
4516 TodoType.clear();
4517 for (auto &F : M)
4518 Changed |= runOnFunction(Func&: F);
4519 if (AccessChainErrors)
4520 report_fatal_error(reason: Twine(toString(E: std::move(AccessChainErrors))), gen_crash_diag: false);
4521
4522 // Specify function parameters after all functions were processed.
4523 for (auto &F : M) {
4524 // check if function parameter types are set
4525 CurrF = &F;
4526 if (!F.isDeclaration() && !F.isIntrinsic()) {
4527 IRBuilder<> B(M);
4528 processParamTypes(F: &F, B);
4529 }
4530 }
4531
4532 CanTodoType = false;
4533 Changed |= postprocessTypes(M);
4534
4535 if (HaveFunPtrs)
4536 Changed |= processFunctionPointers(M);
4537
4538 return Changed;
4539}
4540
4541PreservedAnalyses
4542llvm::SPIRVEmitIntrinsicsPass::run(Module &M, ModuleAnalysisManager &AM) {
4543 if (SPIRVEmitIntrinsicsImpl(TM).runOnModule(M))
4544 return PreservedAnalyses::none();
4545 return PreservedAnalyses::all();
4546}
4547
4548ModulePass *llvm::createSPIRVEmitIntrinsicsPass(const SPIRVTargetMachine &TM) {
4549 return new SPIRVEmitIntrinsicsLegacy(TM);
4550}
4551