1//===-- SPIRVGlobalRegistry.cpp - SPIR-V Global Registry --------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the SPIRVGlobalRegistry class,
10// which is used to maintain rich type information required for SPIR-V even
11// after lowering from LLVM IR to GMIR. It can convert an llvm::Type into
12// an OpTypeXXX instruction, and map it to a virtual register. Also it builds
13// and supports consistency of constants and global variables.
14//
15//===----------------------------------------------------------------------===//
16
17#include "SPIRVGlobalRegistry.h"
18#include "SPIRV.h"
19#include "SPIRVBuiltins.h"
20#include "SPIRVSubtarget.h"
21#include "SPIRVUtils.h"
22#include "llvm/ADT/APInt.h"
23#include "llvm/IR/Constants.h"
24#include "llvm/IR/DiagnosticInfo.h"
25#include "llvm/IR/Function.h"
26#include "llvm/IR/IntrinsicInst.h"
27#include "llvm/IR/Intrinsics.h"
28#include "llvm/IR/IntrinsicsSPIRV.h"
29#include "llvm/IR/Type.h"
30#include "llvm/Support/Casting.h"
31#include "llvm/Support/MathExtras.h"
32#include <cassert>
33#include <functional>
34
35using namespace llvm;
36
37static bool allowEmitFakeUse(const Value *Arg) {
38 if (isSpvIntrinsic(Arg))
39 return false;
40 if (isa<AtomicCmpXchgInst, InsertValueInst, UndefValue>(Val: Arg))
41 return false;
42 if (const auto *LI = dyn_cast<LoadInst>(Val: Arg))
43 if (LI->getType()->isAggregateType())
44 return false;
45 return true;
46}
47
48static unsigned typeToAddressSpace(const Type *Ty) {
49 if (auto PType = dyn_cast<TypedPointerType>(Val: Ty))
50 return PType->getAddressSpace();
51 if (auto PType = dyn_cast<PointerType>(Val: Ty))
52 return PType->getAddressSpace();
53 if (auto *ExtTy = dyn_cast<TargetExtType>(Val: Ty);
54 ExtTy && isTypedPointerWrapper(ExtTy))
55 return ExtTy->getIntParameter(i: 0);
56 reportFatalInternalError(reason: "Unable to convert LLVM type to SPIRVType");
57}
58
59static bool
60storageClassRequiresExplictLayout(SPIRV::StorageClass::StorageClass SC) {
61 switch (SC) {
62 case SPIRV::StorageClass::Uniform:
63 case SPIRV::StorageClass::PushConstant:
64 case SPIRV::StorageClass::StorageBuffer:
65 case SPIRV::StorageClass::PhysicalStorageBufferEXT:
66 return true;
67 case SPIRV::StorageClass::UniformConstant:
68 case SPIRV::StorageClass::Input:
69 case SPIRV::StorageClass::Output:
70 case SPIRV::StorageClass::Workgroup:
71 case SPIRV::StorageClass::CrossWorkgroup:
72 case SPIRV::StorageClass::Private:
73 case SPIRV::StorageClass::Function:
74 case SPIRV::StorageClass::Generic:
75 case SPIRV::StorageClass::AtomicCounter:
76 case SPIRV::StorageClass::Image:
77 case SPIRV::StorageClass::CallableDataNV:
78 case SPIRV::StorageClass::IncomingCallableDataNV:
79 case SPIRV::StorageClass::RayPayloadNV:
80 case SPIRV::StorageClass::HitAttributeNV:
81 case SPIRV::StorageClass::IncomingRayPayloadNV:
82 case SPIRV::StorageClass::ShaderRecordBufferNV:
83 case SPIRV::StorageClass::CodeSectionINTEL:
84 case SPIRV::StorageClass::DeviceOnlyINTEL:
85 case SPIRV::StorageClass::HostOnlyINTEL:
86 return false;
87 }
88 llvm_unreachable("Unknown SPIRV::StorageClass enum");
89}
90
91SPIRVGlobalRegistry::SPIRVGlobalRegistry(DataLayout DL)
92 : DL(DL), Bound(0), CurMF(nullptr) {}
93
94void SPIRVGlobalRegistry::constrainSelectedInstRegOperands(
95 MachineInstrBuilder &MIB) const {
96 const auto &ST = CurMF->getSubtarget();
97 MIB.constrainAllUses(TII: *ST.getInstrInfo(), TRI: *ST.getRegisterInfo(),
98 RBI: *ST.getRegBankInfo());
99}
100
101SPIRVTypeInst
102SPIRVGlobalRegistry::assignIntTypeToVReg(unsigned BitWidth, Register VReg,
103 MachineInstr &I,
104 const SPIRVInstrInfo &TII) {
105 SPIRVTypeInst SpirvType = getOrCreateSPIRVIntegerType(BitWidth, I, TII);
106 assignSPIRVTypeToVReg(Type: SpirvType, VReg, MF: *CurMF);
107 return SpirvType;
108}
109
110SPIRVTypeInst SPIRVGlobalRegistry::assignTypeToVReg(
111 const Type *Type, Register VReg, MachineIRBuilder &MIRBuilder,
112 SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) {
113 SPIRVTypeInst SpirvType =
114 getOrCreateSPIRVType(Type, MIRBuilder, AQ: AccessQual, EmitIR);
115 assignSPIRVTypeToVReg(Type: SpirvType, VReg, MF: MIRBuilder.getMF());
116 return SpirvType;
117}
118
119void SPIRVGlobalRegistry::assignSPIRVTypeToVReg(SPIRVTypeInst SpirvType,
120 Register VReg,
121 const MachineFunction &MF) {
122 VRegToTypeMap[&MF][VReg] = SpirvType;
123}
124
125static Register createTypeVReg(MachineRegisterInfo &MRI) {
126 auto Res = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
127 MRI.setRegClass(Reg: Res, RC: &SPIRV::TYPERegClass);
128 return Res;
129}
130
131inline Register createTypeVReg(MachineIRBuilder &MIRBuilder) {
132 return createTypeVReg(MRI&: MIRBuilder.getMF().getRegInfo());
133}
134
135SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeBool(MachineIRBuilder &MIRBuilder) {
136 return createConstOrTypeAtFunctionEntry(
137 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
138 return MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeBool)
139 .addDef(RegNo: createTypeVReg(MIRBuilder));
140 });
141}
142
143unsigned SPIRVGlobalRegistry::adjustOpTypeIntWidth(unsigned Width) const {
144 const SPIRVSubtarget &ST = cast<SPIRVSubtarget>(Val: CurMF->getSubtarget());
145 if (ST.canUseExtension(
146 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers) ||
147 (Width == 4 && ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4)))
148 return Width;
149 if (Width <= 8)
150 return 8;
151 else if (Width <= 16)
152 return 16;
153 else if (Width <= 32)
154 return 32;
155 else if (Width <= 64)
156 return 64;
157 else if (Width <= 128)
158 return 128;
159 reportFatalUsageError(reason: "Unsupported Integer width!");
160}
161
162SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeInt(unsigned Width,
163 MachineIRBuilder &MIRBuilder,
164 bool IsSigned) {
165 Width = adjustOpTypeIntWidth(Width);
166 const SPIRVSubtarget &ST =
167 cast<SPIRVSubtarget>(Val: MIRBuilder.getMF().getSubtarget());
168 return createConstOrTypeAtFunctionEntry(MIRBuilder, Op: [&](MachineIRBuilder
169 &MIRBuilder) {
170 if (Width == 4 && ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4)) {
171 MIRBuilder.buildInstr(Opcode: SPIRV::OpExtension)
172 .addImm(Val: SPIRV::Extension::SPV_INTEL_int4);
173 MIRBuilder.buildInstr(Opcode: SPIRV::OpCapability)
174 .addImm(Val: SPIRV::Capability::Int4TypeINTEL);
175 } else if ((!isPowerOf2_32(Value: Width) || Width < 8) &&
176 ST.canUseExtension(
177 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers)) {
178 MIRBuilder.buildInstr(Opcode: SPIRV::OpExtension)
179 .addImm(Val: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
180 MIRBuilder.buildInstr(Opcode: SPIRV::OpCapability)
181 .addImm(Val: SPIRV::Capability::ArbitraryPrecisionIntegersALTERA);
182 }
183 return MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeInt)
184 .addDef(RegNo: createTypeVReg(MIRBuilder))
185 .addImm(Val: Width)
186 .addImm(Val: IsSigned ? 1 : 0);
187 });
188}
189
190SPIRVTypeInst
191SPIRVGlobalRegistry::getOpTypeFloat(uint32_t Width,
192 MachineIRBuilder &MIRBuilder) {
193 return createConstOrTypeAtFunctionEntry(MIRBuilder, Op: [&](MachineIRBuilder
194 &MIRBuilder) {
195 return MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeFloat)
196 .addDef(RegNo: createTypeVReg(MIRBuilder))
197 .addImm(Val: Width);
198 });
199}
200
201SPIRVTypeInst
202SPIRVGlobalRegistry::getOpTypeFloat(uint32_t Width,
203 MachineIRBuilder &MIRBuilder,
204 SPIRV::FPEncoding::FPEncoding FPEncode) {
205 return createConstOrTypeAtFunctionEntry(MIRBuilder, Op: [&](MachineIRBuilder
206 &MIRBuilder) {
207 return MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeFloat)
208 .addDef(RegNo: createTypeVReg(MIRBuilder))
209 .addImm(Val: Width)
210 .addImm(Val: FPEncode);
211 });
212}
213
214SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeVoid(MachineIRBuilder &MIRBuilder) {
215 return createConstOrTypeAtFunctionEntry(
216 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
217 return MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeVoid)
218 .addDef(RegNo: createTypeVReg(MIRBuilder));
219 });
220}
221
222void SPIRVGlobalRegistry::invalidateMachineInstr(MachineInstr *MI) {
223 // Other maps that may hold MachineInstr*:
224 // - VRegToTypeMap: We cannot remove the definitions of `MI` from
225 // VRegToTypeMap because some calls to invalidateMachineInstr are replacing MI
226 // with another instruction defining the same register. We expect that if MI
227 // is a type instruction, and it is still referenced in VRegToTypeMap, then
228 // those registers are dead or the VRegToTypeMap is out-of-date. We do not
229 // expect passes to ask for the SPIR-V type of a dead register. If the
230 // VRegToTypeMap is out-of-date already, then there was an error before. We
231 // cannot add an assert to verify this because the VRegToTypeMap can be
232 // out-of-date.
233 // - FunctionToInstr & FunctionToInstrRev: At this point, we should not be
234 // deleting functions. No need to update.
235 // - AliasInstMDMap: Would require a linear search, and the Intel Alias
236 // instruction are not instructions instruction selection will be able to
237 // remove.
238
239 const SPIRVSubtarget &ST = MI->getMF()->getSubtarget<SPIRVSubtarget>();
240 [[maybe_unused]] const SPIRVInstrInfo *TII = ST.getInstrInfo();
241 assert(!TII->isAliasingInstr(*MI) &&
242 "Cannot invalidate aliasing instructions.");
243 assert(MI->getOpcode() != SPIRV::OpFunction &&
244 "Cannot invalidate OpFunction.");
245
246 if (MI->getOpcode() == SPIRV::OpFunctionCall) {
247 if (const auto *F = dyn_cast<Function>(Val: MI->getOperand(i: 2).getGlobal())) {
248 auto It = ForwardCalls.find(Val: F);
249 if (It != ForwardCalls.end()) {
250 It->second.erase(Ptr: MI);
251 if (It->second.empty())
252 ForwardCalls.erase(I: It);
253 }
254 }
255 }
256
257 const MachineFunction *MF = MI->getMF();
258 auto It = LastInsertedTypeMap.find(Val: MF);
259 if (It != LastInsertedTypeMap.end() && It->second == MI)
260 LastInsertedTypeMap.erase(Val: MF);
261 // remove from the duplicate tracker to avoid incorrect reuse
262 erase(MI);
263}
264
265const MachineInstr *SPIRVGlobalRegistry::createConstOrTypeAtFunctionEntry(
266 MachineIRBuilder &MIRBuilder,
267 std::function<MachineInstr *(MachineIRBuilder &)> Op) {
268 auto oldInsertPoint = MIRBuilder.getInsertPt();
269 assert(oldInsertPoint.isValid() && "Builder must have insertion point set!");
270 MachineBasicBlock *OldMBB = &MIRBuilder.getMBB();
271 MachineBasicBlock *NewMBB = &*MIRBuilder.getMF().begin();
272
273 auto LastInsertedType = LastInsertedTypeMap.find(Val: CurMF);
274 if (LastInsertedType != LastInsertedTypeMap.end()) {
275 auto It = LastInsertedType->second->getIterator();
276 // It might happen that this instruction was removed from the first MBB,
277 // hence the Parent's check.
278 MachineBasicBlock::iterator InsertAt;
279 if (It->getParent() != NewMBB)
280 InsertAt = oldInsertPoint->getParent() == NewMBB
281 ? oldInsertPoint
282 : getInsertPtValidEnd(MBB: NewMBB);
283 else if (It->getNextNode())
284 InsertAt = It->getNextNode()->getIterator();
285 else
286 InsertAt = getInsertPtValidEnd(MBB: NewMBB);
287 MIRBuilder.setInsertPt(MBB&: *NewMBB, II: InsertAt);
288 } else {
289 MIRBuilder.setInsertPt(MBB&: *NewMBB, II: NewMBB->begin());
290 auto Result = LastInsertedTypeMap.try_emplace(Key: CurMF, Args: nullptr);
291 assert(Result.second);
292 LastInsertedType = Result.first;
293 }
294
295 MachineInstr *ConstOrType = Op(MIRBuilder);
296 // We expect all users of this function to insert definitions at the insertion
297 // point set above that is always the first MBB.
298 assert(ConstOrType->getParent() == NewMBB);
299 LastInsertedType->second = ConstOrType;
300 // Advance past any continued instructions so that the next type/constant
301 // is inserted after the full group, preserving required adjacency.
302 while (auto *Next = LastInsertedType->second->getNextNode()) {
303 unsigned Opc = Next->getOpcode();
304 if (Opc == SPIRV::OpTypeStructContinuedINTEL ||
305 Opc == SPIRV::OpConstantCompositeContinuedINTEL ||
306 Opc == SPIRV::OpSpecConstantCompositeContinuedINTEL ||
307 Opc == SPIRV::OpCompositeConstructContinuedINTEL)
308 LastInsertedType->second = Next;
309 else
310 break;
311 }
312
313 MIRBuilder.setInsertPt(MBB&: *OldMBB, II: oldInsertPoint);
314 return ConstOrType;
315}
316
317SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeVectorImpl(
318 uint32_t NumElems, SPIRVTypeInst ElemType, MachineIRBuilder &MIRBuilder,
319 bool IsLongVectorEXT) {
320 if (ElemType.isPointer()) {
321 if (!cast<SPIRVSubtarget>(Val: MIRBuilder.getMF().getSubtarget())
322 .canUseExtension(
323 E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
324 const Function &F = MIRBuilder.getMF().getFunction();
325 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
326 F,
327 "Vector of pointers requires SPV_INTEL_masked_gather_scatter "
328 "extension",
329 DebugLoc(), DS_Error));
330 }
331 } else {
332 [[maybe_unused]] auto EleOpc = ElemType->getOpcode();
333 assert((EleOpc == SPIRV::OpTypeInt || EleOpc == SPIRV::OpTypeFloat ||
334 EleOpc == SPIRV::OpTypeBool) &&
335 "Invalid vector element type");
336 }
337
338 return createConstOrTypeAtFunctionEntry(
339 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
340 unsigned Op =
341 IsLongVectorEXT ? SPIRV::OpTypeVectorIdEXT : SPIRV::OpTypeVector;
342 Register VTy = createTypeVReg(MIRBuilder);
343 Register Ty = getSPIRVTypeID(SpirvType: ElemType);
344 auto MIB = MIRBuilder.buildInstr(Opcode: Op).addDef(RegNo: VTy).addUse(RegNo: Ty);
345 if (!IsLongVectorEXT)
346 return MIB.addImm(Val: NumElems);
347
348 const auto &ST = MIRBuilder.getMF().getSubtarget<SPIRVSubtarget>();
349 SPIRVTypeInst Int32Ty = getOrCreateSPIRVIntegerType(BitWidth: 32, MIRBuilder);
350 return MIB.addUse(RegNo: getOrCreateConstInt(Val: NumElems, I&: *MIB.getInstr(),
351 SpvType: Int32Ty, TII: *ST.getInstrInfo()));
352 });
353}
354
355SPIRVTypeInst
356SPIRVGlobalRegistry::getOpTypeVector(uint32_t NumElems, SPIRVTypeInst ElemType,
357 MachineIRBuilder &MIRBuilder) {
358 assert(NumElems >= 2 && "SPIR-V OpTypeVector requires at least 2 components");
359 return getOpTypeVectorImpl(NumElems, ElemType, MIRBuilder);
360}
361
362SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeVectorIdEXT(
363 uint32_t NumElems, SPIRVTypeInst ElemType, MachineIRBuilder &MIRBuilder) {
364 assert(NumElems != 2 && NumElems != 3 && NumElems != 4 &&
365 (!MIRBuilder.getMF().getSubtarget<SPIRVSubtarget>().isKernel() ||
366 (NumElems != 8 && NumElems != 16)) &&
367 "SPIR-V OpTypeVectorIdExt should only be used for extended vectors");
368 return getOpTypeVectorImpl(NumElems, ElemType, MIRBuilder, IsLongVectorEXT: true);
369}
370
371Register SPIRVGlobalRegistry::getOrCreateConstFP(APFloat Val, MachineInstr &I,
372 SPIRVTypeInst SpvType,
373 const SPIRVInstrInfo &TII,
374 bool ZeroAsNull) {
375 LLVMContext &Ctx = CurMF->getFunction().getContext();
376 auto *const CF = ConstantFP::get(Context&: Ctx, V: Val);
377 const MachineInstr *MI = findMI(Obj: CF, MF: CurMF);
378 if (MI && (MI->getOpcode() == SPIRV::OpConstantNull ||
379 MI->getOpcode() == SPIRV::OpConstantF))
380 return MI->getOperand(i: 0).getReg();
381 return createConstFP(CF, I, SpvType, TII, ZeroAsNull);
382}
383
384Register SPIRVGlobalRegistry::createConstFP(const ConstantFP *CF,
385 MachineInstr &I,
386 SPIRVTypeInst SpvType,
387 const SPIRVInstrInfo &TII,
388 bool ZeroAsNull) {
389 unsigned BitWidth = getScalarOrVectorBitWidth(Type: SpvType);
390 LLT LLTy = LLT::scalar(SizeInBits: BitWidth);
391 Register Res = CurMF->getRegInfo().createGenericVirtualRegister(Ty: LLTy);
392 CurMF->getRegInfo().setRegClass(Reg: Res, RC: &SPIRV::fIDRegClass);
393 assignSPIRVTypeToVReg(SpirvType: SpvType, VReg: Res, MF: *CurMF);
394
395 MachineInstr *DepMI =
396 const_cast<MachineInstr *>(static_cast<const MachineInstr *>(SpvType));
397 MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());
398 const MachineInstr *Const = createConstOrTypeAtFunctionEntry(
399 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
400 MachineInstrBuilder MIB;
401 // In OpenCL OpConstantNull - Scalar floating point: +0.0 (all bits 0)
402 if (CF->getValue().isPosZero() && ZeroAsNull) {
403 MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpConstantNull)
404 .addDef(RegNo: Res)
405 .addUse(RegNo: getSPIRVTypeID(SpirvType: SpvType));
406 } else {
407 MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpConstantF)
408 .addDef(RegNo: Res)
409 .addUse(RegNo: getSPIRVTypeID(SpirvType: SpvType));
410 addNumImm(Imm: APInt(BitWidth,
411 CF->getValueAPF().bitcastToAPInt().getZExtValue()),
412 MIB);
413 }
414 constrainSelectedInstRegOperands(MIB);
415 return MIB;
416 });
417 add(V: CF, MI: Const);
418 return Res;
419}
420
421Register SPIRVGlobalRegistry::getOrCreateConstInt(uint64_t Val, MachineInstr &I,
422 SPIRVTypeInst SpvType,
423 const SPIRVInstrInfo &TII,
424 bool ZeroAsNull) {
425 return getOrCreateConstInt(Val: APInt(getScalarOrVectorBitWidth(Type: SpvType), Val), I,
426 SpvType, TII, ZeroAsNull);
427}
428
429Register SPIRVGlobalRegistry::getOrCreateConstInt(const APInt &Val,
430 MachineInstr &I,
431 SPIRVTypeInst SpvType,
432 const SPIRVInstrInfo &TII,
433 bool ZeroAsNull) {
434 auto *const CI = ConstantInt::get(
435 Context&: cast<IntegerType>(Val: getTypeForSPIRVType(Ty: SpvType))->getContext(), V: Val);
436 const MachineInstr *MI = findMI(Obj: CI, MF: CurMF);
437 if (MI && (MI->getOpcode() == SPIRV::OpConstantNull ||
438 MI->getOpcode() == SPIRV::OpConstantI))
439 return MI->getOperand(i: 0).getReg();
440 return createConstInt(CI, I, SpvType, TII, ZeroAsNull);
441}
442
443Register SPIRVGlobalRegistry::createConstInt(const ConstantInt *CI,
444 MachineInstr &I,
445 SPIRVTypeInst SpvType,
446 const SPIRVInstrInfo &TII,
447 bool ZeroAsNull) {
448 unsigned BitWidth = getScalarOrVectorBitWidth(Type: SpvType);
449 LLT LLTy = LLT::scalar(SizeInBits: BitWidth);
450 Register Res = CurMF->getRegInfo().createGenericVirtualRegister(Ty: LLTy);
451 CurMF->getRegInfo().setRegClass(Reg: Res, RC: &SPIRV::iIDRegClass);
452 assignIntTypeToVReg(BitWidth, VReg: Res, I, TII);
453
454 MachineInstr *DepMI =
455 const_cast<MachineInstr *>(static_cast<const MachineInstr *>(SpvType));
456 MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());
457 const MachineInstr *Const = createConstOrTypeAtFunctionEntry(
458 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
459 MachineInstrBuilder MIB;
460 if (BitWidth == 1) {
461 MIB = MIRBuilder
462 .buildInstr(Opcode: CI->isZero() ? SPIRV::OpConstantFalse
463 : SPIRV::OpConstantTrue)
464 .addDef(RegNo: Res)
465 .addUse(RegNo: getSPIRVTypeID(SpirvType: SpvType));
466 } else if (!CI->isZero() || !ZeroAsNull) {
467 MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpConstantI)
468 .addDef(RegNo: Res)
469 .addUse(RegNo: getSPIRVTypeID(SpirvType: SpvType));
470 addNumImm(Imm: CI->getValue(), MIB);
471 } else {
472 MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpConstantNull)
473 .addDef(RegNo: Res)
474 .addUse(RegNo: getSPIRVTypeID(SpirvType: SpvType));
475 }
476 constrainSelectedInstRegOperands(MIB);
477 return MIB;
478 });
479 add(V: CI, MI: Const);
480 return Res;
481}
482
483Register SPIRVGlobalRegistry::buildConstantInt(uint64_t Val,
484 MachineIRBuilder &MIRBuilder,
485 SPIRVTypeInst SpvType,
486 bool EmitIR, bool ZeroAsNull) {
487 assert(SpvType);
488 auto &MF = MIRBuilder.getMF();
489 const IntegerType *Ty = cast<IntegerType>(Val: getTypeForSPIRVType(Ty: SpvType));
490 // TODO: Avoid implicit trunc?
491 // See https://github.com/llvm/llvm-project/issues/112510.
492 auto *const CI = ConstantInt::get(Ty: const_cast<IntegerType *>(Ty), V: Val,
493 /*IsSigned=*/false, /*ImplicitTrunc=*/true);
494 Register Res = find(V: CI, MF: &MF);
495 if (Res.isValid())
496 return Res;
497
498 unsigned BitWidth = getScalarOrVectorBitWidth(Type: SpvType);
499 LLT LLTy = LLT::scalar(SizeInBits: BitWidth);
500 MachineRegisterInfo &MRI = MF.getRegInfo();
501 Res = MRI.createGenericVirtualRegister(Ty: LLTy);
502 MRI.setRegClass(Reg: Res, RC: &SPIRV::iIDRegClass);
503 assignTypeToVReg(Type: Ty, VReg: Res, MIRBuilder, AccessQual: SPIRV::AccessQualifier::ReadWrite,
504 EmitIR);
505
506 const MachineInstr *Const = createConstOrTypeAtFunctionEntry(
507 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
508 if (EmitIR)
509 return MIRBuilder.buildConstant(Res, Val: *CI);
510 Register SpvTypeReg = getSPIRVTypeID(SpirvType: SpvType);
511 MachineInstrBuilder MIB;
512 if (Val || !ZeroAsNull) {
513 MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpConstantI)
514 .addDef(RegNo: Res)
515 .addUse(RegNo: SpvTypeReg);
516 addNumImm(Imm: APInt(BitWidth, Val), MIB);
517 } else {
518 MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpConstantNull)
519 .addDef(RegNo: Res)
520 .addUse(RegNo: SpvTypeReg);
521 }
522 constrainSelectedInstRegOperands(MIB);
523 return MIB;
524 });
525 add(V: CI, MI: Const);
526 return Res;
527}
528
529Register SPIRVGlobalRegistry::buildConstantFP(APFloat Val,
530 MachineIRBuilder &MIRBuilder,
531 SPIRVTypeInst SpvType) {
532 auto &MF = MIRBuilder.getMF();
533 LLVMContext &Ctx = MF.getFunction().getContext();
534 if (!SpvType)
535 SpvType = getOrCreateSPIRVType(Type: Type::getFloatTy(C&: Ctx), MIRBuilder,
536 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
537 auto *const CF = ConstantFP::get(Context&: Ctx, V: Val);
538 Register Res = find(V: CF, MF: &MF);
539 if (Res.isValid())
540 return Res;
541
542 LLT LLTy = LLT::scalar(SizeInBits: getScalarOrVectorBitWidth(Type: SpvType));
543 Res = MF.getRegInfo().createGenericVirtualRegister(Ty: LLTy);
544 MF.getRegInfo().setRegClass(Reg: Res, RC: &SPIRV::fIDRegClass);
545 assignSPIRVTypeToVReg(SpirvType: SpvType, VReg: Res, MF);
546
547 const MachineInstr *Const = createConstOrTypeAtFunctionEntry(
548 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
549 MachineInstrBuilder MIB;
550 MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpConstantF)
551 .addDef(RegNo: Res)
552 .addUse(RegNo: getSPIRVTypeID(SpirvType: SpvType));
553 addNumImm(Imm: CF->getValueAPF().bitcastToAPInt(), MIB);
554 return MIB;
555 });
556 add(V: CF, MI: Const);
557 return Res;
558}
559
560Register SPIRVGlobalRegistry::getOrCreateBaseRegister(
561 Constant *Val, MachineInstr &I, SPIRVTypeInst SpvType,
562 const SPIRVInstrInfo &TII, unsigned BitWidth, bool ZeroAsNull) {
563 SPIRVTypeInst Type = SpvType;
564 if (isVectorType(SPVTy: SpvType) || SpvType->getOpcode() == SPIRV::OpTypeArray) {
565 auto EleTypeReg = SpvType->getOperand(i: 1).getReg();
566 Type = getSPIRVTypeForVReg(VReg: EleTypeReg);
567 }
568 if (Type->getOpcode() == SPIRV::OpTypeFloat) {
569 SPIRVTypeInst SpvBaseType = getOrCreateSPIRVFloatType(BitWidth, I, TII);
570 return getOrCreateConstFP(Val: cast<ConstantFP>(Val)->getValue(), I, SpvType: SpvBaseType,
571 TII, ZeroAsNull);
572 }
573 assert(Type->getOpcode() == SPIRV::OpTypeInt);
574 SPIRVTypeInst SpvBaseType = getOrCreateSPIRVIntegerType(BitWidth, I, TII);
575 return getOrCreateConstInt(Val: Val->getUniqueInteger(), I, SpvType: SpvBaseType, TII,
576 ZeroAsNull);
577}
578
579Register SPIRVGlobalRegistry::getOrCreateCompositeOrNull(
580 Constant *Val, MachineInstr &I, SPIRVTypeInst SpvType,
581 const SPIRVInstrInfo &TII, Constant *CA, unsigned BitWidth,
582 unsigned ElemCnt, bool ZeroAsNull) {
583 if (Register R = find(V: CA, MF: CurMF); R.isValid())
584 return R;
585
586 bool IsNull = Val->isNullValue() && ZeroAsNull;
587 Register ElemReg;
588 if (!IsNull)
589 ElemReg =
590 getOrCreateBaseRegister(Val, I, SpvType, TII, BitWidth, ZeroAsNull);
591
592 LLT LLTy = LLT::scalar(SizeInBits: 64);
593 Register Res = CurMF->getRegInfo().createGenericVirtualRegister(Ty: LLTy);
594 CurMF->getRegInfo().setRegClass(Reg: Res, RC: getRegClass(SpvType));
595 assignSPIRVTypeToVReg(SpirvType: SpvType, VReg: Res, MF: *CurMF);
596
597 MachineInstr *DepMI =
598 const_cast<MachineInstr *>(static_cast<const MachineInstr *>(SpvType));
599 MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());
600 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
601 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
602 MachineInstrBuilder MIB;
603 if (!IsNull) {
604 MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpConstantComposite)
605 .addDef(RegNo: Res)
606 .addUse(RegNo: getSPIRVTypeID(SpirvType: SpvType));
607 for (unsigned i = 0; i < ElemCnt; ++i)
608 MIB.addUse(RegNo: ElemReg);
609 } else {
610 MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpConstantNull)
611 .addDef(RegNo: Res)
612 .addUse(RegNo: getSPIRVTypeID(SpirvType: SpvType));
613 }
614 constrainSelectedInstRegOperands(MIB);
615 return MIB;
616 });
617 add(V: CA, MI: NewMI);
618 return Res;
619}
620
621Register SPIRVGlobalRegistry::getOrCreateConstVector(uint64_t Val,
622 MachineInstr &I,
623 SPIRVTypeInst SpvType,
624 const SPIRVInstrInfo &TII,
625 bool ZeroAsNull) {
626 return getOrCreateConstVector(Val: APInt(getScalarOrVectorBitWidth(Type: SpvType), Val),
627 I, SpvType, TII, ZeroAsNull);
628}
629
630Register SPIRVGlobalRegistry::getOrCreateConstVector(const APInt &Val,
631 MachineInstr &I,
632 SPIRVTypeInst SpvType,
633 const SPIRVInstrInfo &TII,
634 bool ZeroAsNull) {
635 const Type *LLVMTy = getTypeForSPIRVType(Ty: SpvType);
636 assert(LLVMTy->isVectorTy() &&
637 "Expected vector type for constant vector creation");
638 const FixedVectorType *LLVMVecTy = cast<FixedVectorType>(Val: LLVMTy);
639 Type *LLVMBaseTy = LLVMVecTy->getElementType();
640 [[maybe_unused]] const auto &ST = I.getMF()->getSubtarget<SPIRVSubtarget>();
641 assert((LLVMBaseTy->isIntegerTy() ||
642 (LLVMBaseTy->isPointerTy() &&
643 ST.canUseExtension(
644 SPIRV::Extension::SPV_INTEL_masked_gather_scatter))) &&
645 "Expected either integer element type for APInt constant vector or "
646 "pointer type if the SPV_INTEL_masked_gather_scatter extension is "
647 "enabled!");
648 Constant *ConstVal = nullptr;
649 if (LLVMBaseTy->isIntegerTy()) {
650 ConstVal = ConstantInt::get(Ty: LLVMBaseTy, V: Val);
651 } else {
652 if (Val.isZero())
653 ConstVal = ConstantPointerNull::get(T: LLVMBaseTy);
654 else
655 llvm_unreachable("Vectors of non-null constant pointers unimplemented!");
656 }
657 auto *ConstVec =
658 ConstantVector::getSplat(EC: LLVMVecTy->getElementCount(), Elt: ConstVal);
659 unsigned BW = getScalarOrVectorBitWidth(Type: SpvType);
660 return getOrCreateCompositeOrNull(Val: ConstVal, I, SpvType, TII, CA: ConstVec, BitWidth: BW,
661 ElemCnt: getScalarOrVectorComponentCount(Type: SpvType),
662 ZeroAsNull);
663}
664
665Register SPIRVGlobalRegistry::getOrCreateConstVector(APFloat Val,
666 MachineInstr &I,
667 SPIRVTypeInst SpvType,
668 const SPIRVInstrInfo &TII,
669 bool ZeroAsNull) {
670 const Type *LLVMTy = getTypeForSPIRVType(Ty: SpvType);
671 assert(LLVMTy->isVectorTy());
672 const FixedVectorType *LLVMVecTy = cast<FixedVectorType>(Val: LLVMTy);
673 Type *LLVMBaseTy = LLVMVecTy->getElementType();
674 assert(LLVMBaseTy->isFloatingPointTy());
675 auto *ConstVal = ConstantFP::get(Ty: LLVMBaseTy, V: Val);
676 auto *ConstVec =
677 ConstantVector::getSplat(EC: LLVMVecTy->getElementCount(), Elt: ConstVal);
678 unsigned BW = getScalarOrVectorBitWidth(Type: SpvType);
679 return getOrCreateCompositeOrNull(Val: ConstVal, I, SpvType, TII, CA: ConstVec, BitWidth: BW,
680 ElemCnt: getScalarOrVectorComponentCount(Type: SpvType),
681 ZeroAsNull);
682}
683
684Register SPIRVGlobalRegistry::getOrCreateConstIntArray(
685 uint64_t Val, size_t Num, MachineInstr &I, SPIRVTypeInst SpvType,
686 const SPIRVInstrInfo &TII) {
687 const Type *LLVMTy = getTypeForSPIRVType(Ty: SpvType);
688 assert(LLVMTy->isArrayTy());
689 const ArrayType *LLVMArrTy = cast<ArrayType>(Val: LLVMTy);
690 Type *LLVMBaseTy = LLVMArrTy->getElementType();
691 Constant *CI = ConstantInt::get(Ty: LLVMBaseTy, V: Val);
692 SPIRVTypeInst SpvBaseTy =
693 getSPIRVTypeForVReg(VReg: SpvType->getOperand(i: 1).getReg());
694 unsigned BW = getScalarOrVectorBitWidth(Type: SpvBaseTy);
695 // The following is reasonably unique key that is better that [Val]. The naive
696 // alternative would be something along the lines of:
697 // SmallVector<Constant *> NumCI(Num, CI);
698 // Constant *UniqueKey =
699 // ConstantArray::get(const_cast<ArrayType*>(LLVMArrTy), NumCI);
700 // that would be a truly unique but dangerous key, because it could lead to
701 // the creation of constants of arbitrary length (that is, the parameter of
702 // memset) which were missing in the original module.
703 Type *I64Ty = Type::getInt64Ty(C&: LLVMBaseTy->getContext());
704 Constant *UniqueKey = ConstantStruct::getAnon(
705 V: {PoisonValue::get(T: const_cast<ArrayType *>(LLVMArrTy)),
706 ConstantInt::get(Ty: LLVMBaseTy, V: Val), ConstantInt::get(Ty: I64Ty, V: Num)});
707 return getOrCreateCompositeOrNull(Val: CI, I, SpvType, TII, CA: UniqueKey, BitWidth: BW,
708 ElemCnt: LLVMArrTy->getNumElements());
709}
710
711Register SPIRVGlobalRegistry::getOrCreateIntCompositeOrNull(
712 uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType,
713 bool EmitIR, Constant *CA, unsigned BitWidth, unsigned ElemCnt) {
714 if (Register R = find(V: CA, MF: CurMF); R.isValid())
715 return R;
716
717 Register ElemReg;
718 if (Val || EmitIR) {
719 SPIRVTypeInst SpvBaseType =
720 getOrCreateSPIRVIntegerType(BitWidth, MIRBuilder);
721 ElemReg = buildConstantInt(Val, MIRBuilder, SpvType: SpvBaseType, EmitIR);
722 }
723 LLT LLTy = EmitIR ? LLT::fixed_vector(NumElements: ElemCnt, ScalarSizeInBits: BitWidth) : LLT::scalar(SizeInBits: 64);
724 Register Res = CurMF->getRegInfo().createGenericVirtualRegister(Ty: LLTy);
725 CurMF->getRegInfo().setRegClass(Reg: Res, RC: &SPIRV::iIDRegClass);
726 assignSPIRVTypeToVReg(SpirvType: SpvType, VReg: Res, MF: *CurMF);
727
728 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
729 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
730 if (EmitIR)
731 return MIRBuilder.buildSplatBuildVector(Res, Src: ElemReg);
732
733 if (Val) {
734 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpConstantComposite)
735 .addDef(RegNo: Res)
736 .addUse(RegNo: getSPIRVTypeID(SpirvType: SpvType));
737 for (unsigned i = 0; i < ElemCnt; ++i)
738 MIB.addUse(RegNo: ElemReg);
739 return MIB;
740 }
741
742 return MIRBuilder.buildInstr(Opcode: SPIRV::OpConstantNull)
743 .addDef(RegNo: Res)
744 .addUse(RegNo: getSPIRVTypeID(SpirvType: SpvType));
745 });
746 add(V: CA, MI: NewMI);
747 return Res;
748}
749
750Register SPIRVGlobalRegistry::getOrCreateConsIntVector(
751 uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType,
752 bool EmitIR) {
753 const Type *LLVMTy = getTypeForSPIRVType(Ty: SpvType);
754 assert(LLVMTy->isVectorTy());
755 const FixedVectorType *LLVMVecTy = cast<FixedVectorType>(Val: LLVMTy);
756 Type *LLVMBaseTy = LLVMVecTy->getElementType();
757 const auto ConstInt = ConstantInt::get(Ty: LLVMBaseTy, V: Val);
758 auto ConstVec =
759 ConstantVector::getSplat(EC: LLVMVecTy->getElementCount(), Elt: ConstInt);
760 unsigned BW = getScalarOrVectorBitWidth(Type: SpvType);
761 return getOrCreateIntCompositeOrNull(
762 Val, MIRBuilder, SpvType, EmitIR, CA: ConstVec, BitWidth: BW,
763 ElemCnt: getScalarOrVectorComponentCount(Type: SpvType));
764}
765
766Register
767SPIRVGlobalRegistry::getOrCreateConstNullPtr(MachineIRBuilder &MIRBuilder,
768 SPIRVTypeInst SpvType) {
769 const Type *Ty = getTypeForSPIRVType(Ty: SpvType);
770 unsigned AddressSpace = typeToAddressSpace(Ty);
771 Type *ElemTy = ::getPointeeType(Ty);
772 assert(ElemTy);
773 const Constant *CP = ConstantTargetNone::get(
774 T: dyn_cast<TargetExtType>(Val: getTypedPointerWrapper(ElemTy, AS: AddressSpace)));
775 Register Res = find(V: CP, MF: CurMF);
776 if (Res.isValid())
777 return Res;
778
779 LLT LLTy = LLT::pointer(AddressSpace, SizeInBits: getPointerSize());
780 Res = CurMF->getRegInfo().createGenericVirtualRegister(Ty: LLTy);
781 CurMF->getRegInfo().setRegClass(Reg: Res, RC: &SPIRV::pIDRegClass);
782 assignSPIRVTypeToVReg(SpirvType: SpvType, VReg: Res, MF: *CurMF);
783
784 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
785 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
786 return MIRBuilder.buildInstr(Opcode: SPIRV::OpConstantNull)
787 .addDef(RegNo: Res)
788 .addUse(RegNo: getSPIRVTypeID(SpirvType: SpvType));
789 });
790 add(V: CP, MI: NewMI);
791 return Res;
792}
793
794Register
795SPIRVGlobalRegistry::buildConstantSampler(Register ResReg, unsigned AddrMode,
796 unsigned Param, unsigned FilerMode,
797 MachineIRBuilder &MIRBuilder) {
798 auto Sampler =
799 ResReg.isValid()
800 ? ResReg
801 : MIRBuilder.getMRI()->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
802 SPIRVTypeInst TypeSampler = getOrCreateOpTypeSampler(MIRBuilder);
803 Register TypeSamplerReg = getSPIRVTypeID(SpirvType: TypeSampler);
804 // We cannot use createOpType() logic here, because of the
805 // GlobalISel/IRTranslator.cpp check for a tail call that expects that
806 // MIRBuilder.getInsertPt() has a previous instruction. If this constant is
807 // inserted as a result of "__translate_sampler_initializer()" this would
808 // break this IRTranslator assumption.
809 MIRBuilder.buildInstr(Opcode: SPIRV::OpConstantSampler)
810 .addDef(RegNo: Sampler)
811 .addUse(RegNo: TypeSamplerReg)
812 .addImm(Val: AddrMode)
813 .addImm(Val: Param)
814 .addImm(Val: FilerMode);
815 return Sampler;
816}
817
818Register SPIRVGlobalRegistry::buildGlobalVariable(
819 Register ResVReg, SPIRVTypeInst BaseType, StringRef Name,
820 const GlobalValue *GV, SPIRV::StorageClass::StorageClass Storage,
821 const MachineInstr *Init, bool IsConst,
822 const std::optional<SPIRV::LinkageType::LinkageType> &LinkageType,
823 MachineIRBuilder &MIRBuilder, bool IsInstSelector) {
824 const GlobalVariable *GVar = nullptr;
825 if (GV) {
826 GVar = cast<const GlobalVariable>(Val: GV);
827 } else {
828 // If GV is not passed explicitly, use the name to find or construct
829 // the global variable.
830 Module *M = MIRBuilder.getMF().getFunction().getParent();
831 GVar = M->getGlobalVariable(Name);
832 if (GVar == nullptr) {
833 const Type *Ty = getTypeForSPIRVType(Ty: BaseType); // TODO: check type.
834 if (auto *TPTy = dyn_cast<TypedPointerType>(Val: Ty))
835 Ty = PointerType::get(C&: M->getContext(), AddressSpace: TPTy->getAddressSpace());
836 // Module takes ownership of the global var.
837 GVar = new GlobalVariable(*M, const_cast<Type *>(Ty), false,
838 GlobalValue::ExternalLinkage, nullptr,
839 Twine(Name));
840 }
841 GV = GVar;
842 }
843
844 const MachineFunction *MF = &MIRBuilder.getMF();
845 Register Reg = find(V: GVar, MF);
846 if (Reg.isValid()) {
847 if (Reg != ResVReg)
848 MIRBuilder.buildCopy(Res: ResVReg, Op: Reg);
849 return ResVReg;
850 }
851
852 // Emit the OpVariable into the entry block to ensure the def dominates
853 // all uses across all MBBs.
854 MachineBasicBlock &EntryBB = MIRBuilder.getMF().front();
855 MachineIRBuilder GVBuilder(MIRBuilder.getState());
856 if (&GVBuilder.getMBB() != &EntryBB)
857 GVBuilder.setInsertPt(MBB&: EntryBB, II: EntryBB.getFirstTerminator());
858
859 // Pointers to opaque types stay typed even with the extension on, so emit the
860 // untyped variant only when the result is actually an untyped pointer.
861 const bool UseUntypedPointers =
862 BaseType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
863 const unsigned VariableOpcode =
864 UseUntypedPointers ? SPIRV::OpUntypedVariableKHR : SPIRV::OpVariable;
865
866 auto MIB = GVBuilder.buildInstr(Opcode: VariableOpcode)
867 .addDef(RegNo: ResVReg)
868 .addUse(RegNo: getSPIRVTypeID(SpirvType: BaseType))
869 .addImm(Val: static_cast<uint32_t>(Storage));
870
871 // OpUntypedVariableKHR takes an extra Data Type operand right after the
872 // storage class, holding the global's value type.
873 if (UseUntypedPointers) {
874 SPIRVTypeInst DataType = getPointeeType(PtrType: BaseType);
875 if (!DataType)
876 DataType = getOrCreateSPIRVType(Type: GV->getValueType(), MIRBuilder&: GVBuilder,
877 AQ: SPIRV::AccessQualifier::ReadWrite,
878 /*EmitIR=*/false);
879 if (!DataType) {
880 const Function &F = MIRBuilder.getMF().getFunction();
881 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
882 F,
883 "Could not deduce the data type of untyped global variable '" +
884 GVar->getName() + "'",
885 DebugLoc(), DS_Error));
886 // Recover with i8 so that codegen can finish and report the error.
887 DataType =
888 getOrCreateSPIRVType(Type: Type::getInt8Ty(C&: F.getContext()), MIRBuilder&: GVBuilder,
889 AQ: SPIRV::AccessQualifier::ReadWrite,
890 /*EmitIR=*/false);
891 }
892 MIB.addUse(RegNo: getSPIRVTypeID(SpirvType: DataType));
893 }
894
895 if (Init)
896 MIB.addUse(RegNo: Init->getOperand(i: 0).getReg());
897 // ISel may introduce a new register on this step, so we need to add it to
898 // DT and correct its type avoiding fails on the next stage.
899 if (IsInstSelector) {
900 constrainSelectedInstRegOperands(MIB);
901 }
902 add(V: GVar, MI: MIB);
903
904 Reg = MIB->getOperand(i: 0).getReg();
905 addGlobalObject(V: GVar, MF, R: Reg);
906
907 // Set to Reg the same type as ResVReg has.
908 auto MRI = MIRBuilder.getMRI();
909 if (Reg != ResVReg) {
910 LLT RegLLTy =
911 LLT::pointer(AddressSpace: MRI->getType(Reg: ResVReg).getAddressSpace(), SizeInBits: getPointerSize());
912 MRI->setType(VReg: Reg, Ty: RegLLTy);
913 assignSPIRVTypeToVReg(SpirvType: BaseType, VReg: Reg, MF: MIRBuilder.getMF());
914 } else {
915 // Our knowledge about the type may be updated.
916 // If that's the case, we need to update a type
917 // associated with the register.
918 SPIRVTypeInst DefType = getSPIRVTypeForVReg(VReg: ResVReg);
919 if (!DefType || DefType != SPIRVTypeInst(BaseType))
920 assignSPIRVTypeToVReg(SpirvType: BaseType, VReg: Reg, MF: MIRBuilder.getMF());
921 }
922
923 // If it's a global variable with name, output OpName for it.
924 if (GVar && GVar->hasName())
925 buildOpName(Target: Reg, Name: GVar->getName(), MIRBuilder);
926
927 // Output decorations for the GV.
928 // TODO: maybe move to GenerateDecorations pass.
929 const SPIRVSubtarget &ST =
930 cast<SPIRVSubtarget>(Val: MIRBuilder.getMF().getSubtarget());
931 if (IsConst && !ST.isShader())
932 buildOpDecorate(Reg, MIRBuilder, Dec: SPIRV::Decoration::Constant, DecArgs: {});
933
934 if (GVar && GVar->getAlign().valueOrOne().value() != 1 && !ST.isShader()) {
935 unsigned Alignment = (unsigned)GVar->getAlign().valueOrOne().value();
936 buildOpDecorate(Reg, MIRBuilder, Dec: SPIRV::Decoration::Alignment, DecArgs: {Alignment});
937 }
938
939 if (LinkageType)
940 buildOpDecorate(Reg, MIRBuilder, Dec: SPIRV::Decoration::LinkageAttributes,
941 DecArgs: {static_cast<uint32_t>(*LinkageType)}, StrImm: Name);
942
943 SPIRV::BuiltIn::BuiltIn BuiltInId;
944 if (getSpirvBuiltInIdByName(Name, BI&: BuiltInId))
945 buildOpDecorate(Reg, MIRBuilder, Dec: SPIRV::Decoration::BuiltIn,
946 DecArgs: {static_cast<uint32_t>(BuiltInId)});
947
948 // If it's a global variable with "spirv.Decorations" metadata node
949 // recognize it as a SPIR-V friendly LLVM IR and parse "spirv.Decorations"
950 // arguments.
951 MDNode *GVarMD = nullptr;
952 if (GVar && (GVarMD = GVar->getMetadata(Kind: "spirv.Decorations")) != nullptr)
953 buildOpSpirvDecorations(Reg, MIRBuilder, GVarMD, ST);
954
955 return Reg;
956}
957
958// Returns a name based on the Type. Notes that this does not look at
959// decorations, and will return the same string for two types that are the same
960// except for decorations.
961Register SPIRVGlobalRegistry::getOrCreateGlobalVariableWithBinding(
962 SPIRVTypeInst VarType, uint32_t Set, uint32_t Binding, StringRef Name,
963 MachineIRBuilder &MIRBuilder) {
964 Register VarReg =
965 MIRBuilder.getMRI()->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
966
967 buildGlobalVariable(ResVReg: VarReg, BaseType: VarType, Name, GV: nullptr,
968 Storage: getPointerStorageClass(Type: VarType), Init: nullptr, IsConst: false,
969 LinkageType: std::nullopt, MIRBuilder, IsInstSelector: false);
970
971 buildOpDecorate(Reg: VarReg, MIRBuilder, Dec: SPIRV::Decoration::DescriptorSet, DecArgs: {Set});
972 buildOpDecorate(Reg: VarReg, MIRBuilder, Dec: SPIRV::Decoration::Binding, DecArgs: {Binding});
973 return VarReg;
974}
975
976// TODO: Double check the calls to getOpTypeArray to make sure that `ElemType`
977// is explicitly laid out when required.
978SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeArray(uint32_t NumElems,
979 SPIRVTypeInst ElemType,
980 MachineIRBuilder &MIRBuilder,
981 bool ExplicitLayoutRequired,
982 bool EmitIR) {
983 assert((ElemType->getOpcode() != SPIRV::OpTypeVoid) &&
984 "Invalid array element type");
985 SPIRVTypeInst SpvTypeInt32 = getOrCreateSPIRVIntegerType(BitWidth: 32, MIRBuilder);
986 SPIRVTypeInst ArrayType = nullptr;
987 const SPIRVSubtarget &ST =
988 cast<SPIRVSubtarget>(Val: MIRBuilder.getMF().getSubtarget());
989 if (NumElems != 0) {
990 Register NumElementsVReg =
991 buildConstantInt(Val: NumElems, MIRBuilder, SpvType: SpvTypeInt32, EmitIR);
992 ArrayType = createConstOrTypeAtFunctionEntry(
993 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
994 return MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeArray)
995 .addDef(RegNo: createTypeVReg(MIRBuilder))
996 .addUse(RegNo: getSPIRVTypeID(SpirvType: ElemType))
997 .addUse(RegNo: NumElementsVReg);
998 });
999 } else if (ST.getTargetTriple().getVendor() == Triple::VendorType::AMD) {
1000 // We set the array size to the token UINT64_MAX value, which is generally
1001 // illegal (the maximum legal size is 61-bits) for the foreseeable future.
1002 SPIRVTypeInst SpvTypeInt64 = getOrCreateSPIRVIntegerType(BitWidth: 64, MIRBuilder);
1003 Register NumElementsVReg =
1004 buildConstantInt(UINT64_MAX, MIRBuilder, SpvType: SpvTypeInt64, EmitIR);
1005 ArrayType = createConstOrTypeAtFunctionEntry(
1006 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
1007 return MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeArray)
1008 .addDef(RegNo: createTypeVReg(MIRBuilder))
1009 .addUse(RegNo: getSPIRVTypeID(SpirvType: ElemType))
1010 .addUse(RegNo: NumElementsVReg);
1011 });
1012 } else {
1013 if (!ST.isShader()) {
1014 llvm::reportFatalUsageError(
1015 reason: "Runtime arrays are not allowed in non-shader "
1016 "SPIR-V modules");
1017 return nullptr;
1018 }
1019 ArrayType = createConstOrTypeAtFunctionEntry(
1020 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
1021 return MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeRuntimeArray)
1022 .addDef(RegNo: createTypeVReg(MIRBuilder))
1023 .addUse(RegNo: getSPIRVTypeID(SpirvType: ElemType));
1024 });
1025 }
1026
1027 if (ExplicitLayoutRequired && !isResourceType(Type: ElemType)) {
1028 Type *ET = const_cast<Type *>(getTypeForSPIRVType(Ty: ElemType));
1029 addArrayStrideDecorations(Reg: ArrayType->defs().begin()->getReg(), ElementType: ET,
1030 MIRBuilder);
1031 }
1032
1033 return ArrayType;
1034}
1035
1036SPIRVTypeInst
1037SPIRVGlobalRegistry::getOpTypeOpaque(const StructType *Ty,
1038 MachineIRBuilder &MIRBuilder) {
1039 assert(Ty->hasName());
1040 StringRef Name = Ty->hasName() ? Ty->getName() : "";
1041 Register ResVReg = createTypeVReg(MIRBuilder);
1042 return createConstOrTypeAtFunctionEntry(
1043 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
1044 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeOpaque).addDef(RegNo: ResVReg);
1045 addStringImm(Str: Name, MIB);
1046 buildOpName(Target: ResVReg, Name, MIRBuilder);
1047 return MIB;
1048 });
1049}
1050
1051SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeStruct(
1052 const StructType *Ty, MachineIRBuilder &MIRBuilder,
1053 SPIRV::AccessQualifier::AccessQualifier AccQual,
1054 StructOffsetDecorator Decorator, bool EmitIR) {
1055 Type *OriginalElementType = nullptr;
1056 uint64_t TotalSize = 0;
1057 if (matchPeeledArrayPattern(Ty, OriginalElementType, TotalSize)) {
1058 SPIRVTypeInst ElementSPIRVType = findSPIRVType(
1059 Ty: OriginalElementType, MIRBuilder, accessQual: AccQual,
1060 /* ExplicitLayoutRequired= */ Decorator != nullptr, EmitIR);
1061 return getOpTypeArray(NumElems: TotalSize, ElemType: ElementSPIRVType, MIRBuilder,
1062 /*ExplicitLayoutRequired=*/Decorator != nullptr,
1063 EmitIR);
1064 }
1065
1066 const SPIRVSubtarget &ST =
1067 cast<SPIRVSubtarget>(Val: MIRBuilder.getMF().getSubtarget());
1068 SmallVector<Register, 4> FieldTypes;
1069 constexpr unsigned MaxWordCount = UINT16_MAX;
1070 const size_t NumElements = Ty->getNumElements();
1071
1072 size_t MaxNumElements = MaxWordCount - 2;
1073 size_t SPIRVStructNumElements = NumElements;
1074 if (NumElements > MaxNumElements) {
1075 // Do adjustments for continued instructions.
1076 SPIRVStructNumElements = MaxNumElements;
1077 MaxNumElements = MaxWordCount - 1;
1078 }
1079
1080 for (const auto &Elem : Ty->elements()) {
1081 SPIRVTypeInst ElemTy = findSPIRVType(
1082 Ty: toTypedPointer(Ty: Elem), MIRBuilder, accessQual: AccQual,
1083 /* ExplicitLayoutRequired= */ Decorator != nullptr, EmitIR);
1084 assert(ElemTy && ElemTy->getOpcode() != SPIRV::OpTypeVoid &&
1085 "Invalid struct element type");
1086 FieldTypes.push_back(Elt: getSPIRVTypeID(SpirvType: ElemTy));
1087 }
1088 Register ResVReg = createTypeVReg(MIRBuilder);
1089 if (Ty->hasName())
1090 buildOpName(Target: ResVReg, Name: Ty->getName(), MIRBuilder);
1091 if (Ty->isPacked() && !ST.isShader())
1092 buildOpDecorate(Reg: ResVReg, MIRBuilder, Dec: SPIRV::Decoration::CPacked, DecArgs: {});
1093
1094 SPIRVTypeInst SPVType = createConstOrTypeAtFunctionEntry(
1095 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
1096 auto MIBStruct =
1097 MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeStruct).addDef(RegNo: ResVReg);
1098 for (size_t I = 0; I < SPIRVStructNumElements; ++I)
1099 MIBStruct.addUse(RegNo: FieldTypes[I]);
1100 for (size_t I = SPIRVStructNumElements; I < NumElements;
1101 I += MaxNumElements) {
1102 auto MIBCont =
1103 MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeStructContinuedINTEL);
1104 for (size_t J = I; J < std::min(a: I + MaxNumElements, b: NumElements); ++J)
1105 MIBCont.addUse(RegNo: FieldTypes[J]);
1106 }
1107 return MIBStruct;
1108 });
1109
1110 if (Decorator)
1111 Decorator(SPVType->defs().begin()->getReg());
1112
1113 return SPVType;
1114}
1115
1116SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSpecialType(
1117 const Type *Ty, MachineIRBuilder &MIRBuilder,
1118 SPIRV::AccessQualifier::AccessQualifier AccQual) {
1119 assert(isSpecialOpaqueType(Ty) && "Not a special opaque builtin type");
1120 return SPIRV::lowerBuiltinType(Type: Ty, AccessQual: AccQual, MIRBuilder, GR: this);
1121}
1122
1123SPIRVTypeInst SPIRVGlobalRegistry::getOpTypePointer(
1124 SPIRV::StorageClass::StorageClass SC, SPIRVTypeInst ElemType,
1125 MachineIRBuilder &MIRBuilder, Register Reg) {
1126 // Check if we should use untyped pointers.
1127 const SPIRVSubtarget &ST =
1128 cast<SPIRVSubtarget>(Val: MIRBuilder.getMF().getSubtarget());
1129 if (shouldUseUntypedPointer(ElemType, ST))
1130 return getOrCreateSPIRVUntypedPointerType(SC, MIRBuilder);
1131
1132 if (!Reg.isValid())
1133 Reg = createTypeVReg(MIRBuilder);
1134
1135 return createConstOrTypeAtFunctionEntry(MIRBuilder, Op: [&](MachineIRBuilder
1136 &MIRBuilder) {
1137 return MIRBuilder.buildInstr(Opcode: SPIRV::OpTypePointer)
1138 .addDef(RegNo: Reg)
1139 .addImm(Val: static_cast<uint32_t>(SC))
1140 .addUse(RegNo: getSPIRVTypeID(SpirvType: ElemType));
1141 });
1142}
1143
1144SPIRVTypeInst SPIRVGlobalRegistry::getOpTypeFunction(
1145 const FunctionType *Ty, SPIRVTypeInst RetType,
1146 const SmallVectorImpl<SPIRVTypeInst> &ArgTypes,
1147 MachineIRBuilder &MIRBuilder) {
1148 const SPIRVSubtarget *ST =
1149 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
1150 if (Ty->isVarArg() && ST->isShader()) {
1151 Function &Fn = MIRBuilder.getMF().getFunction();
1152 Ty->getContext().diagnose(DI: DiagnosticInfoUnsupported(
1153 Fn, "SPIR-V shaders do not support variadic functions",
1154 MIRBuilder.getDebugLoc()));
1155 }
1156 return createConstOrTypeAtFunctionEntry(MIRBuilder, Op: [&](MachineIRBuilder
1157 &MIRBuilder) {
1158 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeFunction)
1159 .addDef(RegNo: createTypeVReg(MIRBuilder))
1160 .addUse(RegNo: getSPIRVTypeID(SpirvType: RetType));
1161 for (auto &ArgType : ArgTypes)
1162 MIB.addUse(RegNo: getSPIRVTypeID(SpirvType: ArgType));
1163 return MIB;
1164 });
1165}
1166
1167SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateOpTypeFunctionWithArgs(
1168 const Type *Ty, SPIRVTypeInst RetType,
1169 const SmallVectorImpl<SPIRVTypeInst> &ArgTypes,
1170 MachineIRBuilder &MIRBuilder) {
1171 if (const MachineInstr *MI = findMI(T: Ty, RequiresExplicitLayout: false, MF: &MIRBuilder.getMF()))
1172 return MI;
1173 const MachineInstr *NewMI =
1174 getOpTypeFunction(Ty: cast<FunctionType>(Val: Ty), RetType, ArgTypes, MIRBuilder);
1175 add(T: Ty, RequiresExplicitLayout: false, MI: NewMI);
1176 return finishCreatingSPIRVType(LLVMTy: Ty, SpirvType: NewMI);
1177}
1178
1179SPIRVTypeInst SPIRVGlobalRegistry::findSPIRVType(
1180 const Type *Ty, MachineIRBuilder &MIRBuilder,
1181 SPIRV::AccessQualifier::AccessQualifier AccQual,
1182 bool ExplicitLayoutRequired, bool EmitIR) {
1183 const auto &STI = MIRBuilder.getMF().getSubtarget<SPIRVSubtarget>();
1184 // Treat <1 x T> as T if the SPV_EXT_long_vector extension is not available.
1185 if (auto *FVT = dyn_cast<FixedVectorType>(Val: Ty);
1186 FVT && FVT->getNumElements() == 1 &&
1187 !STI.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector))
1188 return findSPIRVType(Ty: FVT->getElementType(), MIRBuilder, AccQual,
1189 ExplicitLayoutRequired, EmitIR);
1190 Ty = adjustIntTypeByWidth(Ty);
1191 // TODO: findMI needs to know if a layout is required.
1192 if (const MachineInstr *MI =
1193 findMI(T: Ty, RequiresExplicitLayout: ExplicitLayoutRequired, MF: &MIRBuilder.getMF()))
1194 return MI;
1195 if (auto It = ForwardPointerTypes.find(Val: Ty); It != ForwardPointerTypes.end())
1196 return It->second;
1197 return restOfCreateSPIRVType(Type: Ty, MIRBuilder, AccessQual: AccQual, ExplicitLayoutRequired,
1198 EmitIR);
1199}
1200
1201Register SPIRVGlobalRegistry::getSPIRVTypeID(SPIRVTypeInst SpirvType) const {
1202 assert(SpirvType && "Attempting to get type id for nullptr type.");
1203 if (SpirvType->getOpcode() == SPIRV::OpTypeForwardPointer ||
1204 SpirvType->getOpcode() == SPIRV::OpTypeStructContinuedINTEL)
1205 return SpirvType->uses().begin()->getReg();
1206 return SpirvType->defs().begin()->getReg();
1207}
1208
1209// We need to use a new LLVM integer type if there is a mismatch between
1210// number of bits in LLVM and SPIRV integer types to let DuplicateTracker
1211// ensure uniqueness of a SPIRV type by the corresponding LLVM type. Without
1212// such an adjustment SPIRVGlobalRegistry::getOpTypeInt() could create the
1213// same "OpTypeInt 8" type for a series of LLVM integer types with number of
1214// bits less than 8. This would lead to duplicate type definitions
1215// eventually due to the method that DuplicateTracker utilizes to reason
1216// about uniqueness of type records.
1217const Type *SPIRVGlobalRegistry::adjustIntTypeByWidth(const Type *Ty) const {
1218 if (auto IType = dyn_cast<IntegerType>(Val: Ty)) {
1219 unsigned SrcBitWidth = IType->getBitWidth();
1220 if (SrcBitWidth > 1) {
1221 unsigned BitWidth = adjustOpTypeIntWidth(Width: SrcBitWidth);
1222 // Maybe change source LLVM type to keep DuplicateTracker consistent.
1223 if (SrcBitWidth != BitWidth)
1224 Ty = IntegerType::get(C&: Ty->getContext(), NumBits: BitWidth);
1225 }
1226 }
1227 return Ty;
1228}
1229
1230SPIRVTypeInst SPIRVGlobalRegistry::createSPIRVType(
1231 const Type *Ty, MachineIRBuilder &MIRBuilder,
1232 SPIRV::AccessQualifier::AccessQualifier AccQual,
1233 bool ExplicitLayoutRequired, bool EmitIR) {
1234 if (isSpecialOpaqueType(Ty))
1235 return getOrCreateSpecialType(Ty, MIRBuilder, AccQual);
1236
1237 if (const MachineInstr *MI =
1238 findMI(T: Ty, RequiresExplicitLayout: ExplicitLayoutRequired, MF: &MIRBuilder.getMF()))
1239 return MI;
1240
1241 if (auto IType = dyn_cast<IntegerType>(Val: Ty)) {
1242 const unsigned Width = IType->getBitWidth();
1243 return Width == 1 ? getOpTypeBool(MIRBuilder)
1244 : getOpTypeInt(Width, MIRBuilder, IsSigned: false);
1245 }
1246 if (Ty->isFloatingPointTy()) {
1247 if (Ty->isFP128Ty() || Ty->isPPC_FP128Ty())
1248 llvm::reportFatalUsageError(reason: "fp128 is not supported in SPIR-V");
1249 if (Ty->isBFloatTy()) {
1250 return getOpTypeFloat(Width: Ty->getPrimitiveSizeInBits(), MIRBuilder,
1251 FPEncode: SPIRV::FPEncoding::BFloat16KHR);
1252 } else {
1253 return getOpTypeFloat(Width: Ty->getPrimitiveSizeInBits(), MIRBuilder);
1254 }
1255 }
1256 if (Ty->isVoidTy())
1257 return getOpTypeVoid(MIRBuilder);
1258 if (Ty->isVectorTy()) {
1259 SPIRVTypeInst El =
1260 findSPIRVType(Ty: cast<FixedVectorType>(Val: Ty)->getElementType(), MIRBuilder,
1261 AccQual, ExplicitLayoutRequired, EmitIR);
1262 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
1263 const auto &STI = MIRBuilder.getMF().getSubtarget<SPIRVSubtarget>();
1264 if (isLongVectorEXT(Ty, CanUseVector16: STI.isKernel()) &&
1265 STI.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector))
1266 return getOpTypeVectorIdEXT(NumElems: NumElts, ElemType: El, MIRBuilder);
1267 return getOpTypeVector(NumElems: NumElts, ElemType: El, MIRBuilder);
1268 }
1269 if (Ty->isArrayTy()) {
1270 SPIRVTypeInst El = findSPIRVType(Ty: Ty->getArrayElementType(), MIRBuilder,
1271 AccQual, ExplicitLayoutRequired, EmitIR);
1272 return getOpTypeArray(NumElems: Ty->getArrayNumElements(), ElemType: El, MIRBuilder,
1273 ExplicitLayoutRequired, EmitIR);
1274 }
1275 if (auto SType = dyn_cast<StructType>(Val: Ty)) {
1276 if (SType->isOpaque())
1277 return getOpTypeOpaque(Ty: SType, MIRBuilder);
1278
1279 StructOffsetDecorator Decorator = nullptr;
1280 if (ExplicitLayoutRequired) {
1281 Decorator = [&MIRBuilder, SType, this](Register Reg) {
1282 addStructOffsetDecorations(Reg, Ty: const_cast<StructType *>(SType),
1283 MIRBuilder);
1284 };
1285 }
1286 return getOpTypeStruct(Ty: SType, MIRBuilder, AccQual, Decorator: std::move(Decorator),
1287 EmitIR);
1288 }
1289 if (auto FType = dyn_cast<FunctionType>(Val: Ty)) {
1290 SPIRVTypeInst RetTy =
1291 findSPIRVType(Ty: FType->getReturnType(), MIRBuilder, AccQual,
1292 ExplicitLayoutRequired, EmitIR);
1293 SmallVector<SPIRVTypeInst, 4> ParamTypes;
1294 for (const auto &ParamTy : FType->params())
1295 ParamTypes.push_back(Elt: findSPIRVType(Ty: ParamTy, MIRBuilder, AccQual,
1296 ExplicitLayoutRequired, EmitIR));
1297 return getOpTypeFunction(Ty: FType, RetType: RetTy, ArgTypes: ParamTypes, MIRBuilder);
1298 }
1299
1300 unsigned AddrSpace = typeToAddressSpace(Ty);
1301
1302 // Get access to information about available extensions
1303 const SPIRVSubtarget *ST =
1304 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
1305 auto SC = addressSpaceToStorageClass(AddrSpace, STI: *ST);
1306
1307 SPIRVTypeInst SpvElementType = nullptr;
1308 Type *ElemTy = ::getPointeeType(Ty);
1309 if (ElemTy && isa<FunctionType>(Val: ElemTy) &&
1310 !ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers))
1311 ElemTy = nullptr;
1312 if (ElemTy)
1313 SpvElementType = getOrCreateSPIRVType(Type: ElemTy, MIRBuilder, AQ: AccQual, EmitIR);
1314 else
1315 SpvElementType = getOrCreateSPIRVIntegerType(BitWidth: 8, MIRBuilder);
1316
1317 if (!ElemTy) {
1318 ElemTy = Type::getInt8Ty(C&: MIRBuilder.getContext());
1319 }
1320
1321 // If we have forward pointer associated with this type, use its register
1322 // operand to create OpTypePointer.
1323 if (auto It = ForwardPointerTypes.find(Val: Ty); It != ForwardPointerTypes.end()) {
1324 Register Reg = getSPIRVTypeID(SpirvType: It->second);
1325 // TODO: what does getOpTypePointer do?
1326 return getOpTypePointer(SC, ElemType: SpvElementType, MIRBuilder, Reg);
1327 }
1328
1329 return getOrCreateSPIRVPointerType(BaseType: ElemTy, MIRBuilder, SC);
1330}
1331
1332SPIRVTypeInst SPIRVGlobalRegistry::restOfCreateSPIRVType(
1333 const Type *Ty, MachineIRBuilder &MIRBuilder,
1334 SPIRV::AccessQualifier::AccessQualifier AccessQual,
1335 bool ExplicitLayoutRequired, bool EmitIR) {
1336 // TODO: Could this create a problem if one requires an explicit layout, and
1337 // the next time it does not?
1338 if (TypesInProcessing.count(Ptr: Ty) && !isPointerTyOrWrapper(Ty))
1339 return nullptr;
1340 TypesInProcessing.insert(Ptr: Ty);
1341 SPIRVTypeInst SpirvType = createSPIRVType(Ty, MIRBuilder, AccQual: AccessQual,
1342 ExplicitLayoutRequired, EmitIR);
1343 TypesInProcessing.erase(Ptr: Ty);
1344 VRegToTypeMap[&MIRBuilder.getMF()][getSPIRVTypeID(SpirvType)] = SpirvType;
1345
1346 // TODO: We could end up with two SPIR-V types pointing to the same llvm type.
1347 // Is that a problem?
1348 SPIRVToLLVMType[SpirvType] = unifyPtrType(Ty);
1349
1350 if (SpirvType->getOpcode() == SPIRV::OpTypeForwardPointer ||
1351 SpirvType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR ||
1352 findMI(T: Ty, RequiresExplicitLayout: false, MF: &MIRBuilder.getMF()) || isSpecialOpaqueType(Ty))
1353 return SpirvType;
1354
1355 if (auto *ExtTy = dyn_cast<TargetExtType>(Val: Ty);
1356 ExtTy && isTypedPointerWrapper(ExtTy))
1357 add(PointeeTy: ExtTy->getTypeParameter(i: 0), AddressSpace: ExtTy->getIntParameter(i: 0), MI: SpirvType);
1358 else if (!isPointerTy(T: Ty))
1359 add(T: Ty, RequiresExplicitLayout: ExplicitLayoutRequired, MI: SpirvType);
1360 else if (isTypedPointerTy(T: Ty))
1361 add(PointeeTy: cast<TypedPointerType>(Val: Ty)->getElementType(),
1362 AddressSpace: getPointerAddressSpace(T: Ty), MI: SpirvType);
1363 else
1364 add(PointeeTy: Type::getInt8Ty(C&: MIRBuilder.getMF().getFunction().getContext()),
1365 AddressSpace: getPointerAddressSpace(T: Ty), MI: SpirvType);
1366 return SpirvType;
1367}
1368
1369SPIRVTypeInst
1370SPIRVGlobalRegistry::getSPIRVTypeForVReg(Register VReg,
1371 const MachineFunction *MF) const {
1372 auto t = VRegToTypeMap.find(Val: MF ? MF : CurMF);
1373 if (t != VRegToTypeMap.end()) {
1374 auto tt = t->second.find(Val: VReg);
1375 if (tt != t->second.end())
1376 return tt->second;
1377 }
1378 return nullptr;
1379}
1380
1381SPIRVTypeInst SPIRVGlobalRegistry::getResultType(Register VReg,
1382 MachineFunction *MF) {
1383 if (!MF)
1384 MF = CurMF;
1385 MachineInstr *Instr = getVRegDef(MRI&: MF->getRegInfo(), Reg: VReg);
1386 return getSPIRVTypeForVReg(VReg: Instr->getOperand(i: 1).getReg(), MF);
1387}
1388
1389SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSPIRVType(
1390 const Type *Ty, MachineIRBuilder &MIRBuilder,
1391 SPIRV::AccessQualifier::AccessQualifier AccessQual,
1392 bool ExplicitLayoutRequired, bool EmitIR) {
1393 // SPIR-V doesn't support single-element vectors. Treat <1 x T> as T if the
1394 // SPV_EXT_long_vector extension is not available.
1395 const auto &STI = MIRBuilder.getMF().getSubtarget<SPIRVSubtarget>();
1396 if (auto *FVT = dyn_cast<FixedVectorType>(Val: Ty);
1397 FVT && FVT->getNumElements() == 1 &&
1398 !STI.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector))
1399 return getOrCreateSPIRVType(Ty: FVT->getElementType(), MIRBuilder, AccessQual,
1400 ExplicitLayoutRequired, EmitIR);
1401 const MachineFunction *MF = &MIRBuilder.getMF();
1402 Register Reg;
1403 if (auto *ExtTy = dyn_cast<TargetExtType>(Val: Ty);
1404 ExtTy && isTypedPointerWrapper(ExtTy))
1405 Reg = find(PointeeTy: ExtTy->getTypeParameter(i: 0), AddressSpace: ExtTy->getIntParameter(i: 0), MF);
1406 else if (!isPointerTy(T: Ty))
1407 Reg = find(T: Ty = adjustIntTypeByWidth(Ty), RequiresExplicitLayout: ExplicitLayoutRequired, MF);
1408 else if (isTypedPointerTy(T: Ty))
1409 Reg = find(PointeeTy: cast<TypedPointerType>(Val: Ty)->getElementType(),
1410 AddressSpace: getPointerAddressSpace(T: Ty), MF);
1411 else
1412 Reg = find(PointeeTy: Type::getInt8Ty(C&: MIRBuilder.getMF().getFunction().getContext()),
1413 AddressSpace: getPointerAddressSpace(T: Ty), MF);
1414 if (Reg.isValid() && !isSpecialOpaqueType(Ty))
1415 return getSPIRVTypeForVReg(VReg: Reg);
1416
1417 TypesInProcessing.clear();
1418 SPIRVTypeInst STy = restOfCreateSPIRVType(Ty, MIRBuilder, AccessQual,
1419 ExplicitLayoutRequired, EmitIR);
1420 // Create normal pointer types for the corresponding OpTypeForwardPointers.
1421 for (auto &CU : ForwardPointerTypes) {
1422 // Pointer type themselves do not require an explicit layout. The types
1423 // they pointer to might, but that is taken care of when creating the type.
1424 bool PtrNeedsLayout = false;
1425 const Type *Ty2 = CU.first;
1426 SPIRVTypeInst STy2 = CU.second;
1427 if ((Reg = find(T: Ty2, RequiresExplicitLayout: PtrNeedsLayout, MF)).isValid())
1428 STy2 = getSPIRVTypeForVReg(VReg: Reg);
1429 else
1430 STy2 = restOfCreateSPIRVType(Ty: Ty2, MIRBuilder, AccessQual, ExplicitLayoutRequired: PtrNeedsLayout,
1431 EmitIR);
1432 if (Ty == Ty2)
1433 STy = STy2;
1434 }
1435 ForwardPointerTypes.clear();
1436 return STy;
1437}
1438
1439bool SPIRVGlobalRegistry::isScalarOfType(Register VReg,
1440 unsigned TypeOpcode) const {
1441 SPIRVTypeInst Type = getSPIRVTypeForVReg(VReg);
1442 assert(Type && "isScalarOfType VReg has no type assigned");
1443 return Type->getOpcode() == TypeOpcode;
1444}
1445
1446bool SPIRVGlobalRegistry::isScalarOrVectorOfType(Register VReg,
1447 unsigned TypeOpcode) const {
1448 SPIRVTypeInst Type = getSPIRVTypeForVReg(VReg);
1449 assert(Type && "isScalarOrVectorOfType VReg has no type assigned");
1450 if (Type->getOpcode() == TypeOpcode)
1451 return true;
1452 if (!isVectorType(SPVTy: Type))
1453 return false;
1454 Register ScalarTypeVReg = Type->getOperand(i: 1).getReg();
1455 SPIRVTypeInst ScalarType = getSPIRVTypeForVReg(VReg: ScalarTypeVReg);
1456 return ScalarType->getOpcode() == TypeOpcode;
1457}
1458
1459bool SPIRVGlobalRegistry::isResourceType(SPIRVTypeInst Type) const {
1460 switch (Type->getOpcode()) {
1461 case SPIRV::OpTypeImage:
1462 case SPIRV::OpTypeSampler:
1463 case SPIRV::OpTypeSampledImage:
1464 return true;
1465 case SPIRV::OpTypeStruct:
1466 return BlockDecoratedTypes.contains(V: Type);
1467 default:
1468 return false;
1469 }
1470 return false;
1471}
1472unsigned
1473SPIRVGlobalRegistry::getScalarOrVectorComponentCount(Register VReg) const {
1474 return getScalarOrVectorComponentCount(Type: getSPIRVTypeForVReg(VReg));
1475}
1476
1477unsigned
1478SPIRVGlobalRegistry::getScalarOrVectorComponentCount(SPIRVTypeInst Type) const {
1479 if (!Type)
1480 return 0;
1481 if (isVectorType(SPVTy: Type))
1482 return (Type->getOpcode() == SPIRV::OpTypeVector)
1483 ? static_cast<unsigned>(Type->getOperand(i: 2).getImm())
1484 : foldImm(MO: Type->getOperand(i: 2), MRI: &CurMF->getRegInfo());
1485 return 1;
1486}
1487
1488SPIRVTypeInst
1489SPIRVGlobalRegistry::getScalarOrVectorComponentType(SPIRVTypeInst Type) const {
1490 if (!Type)
1491 return nullptr;
1492 Register ScalarReg = (isVectorType(SPVTy: Type)) ? Type->getOperand(i: 1).getReg()
1493 : Type->getOperand(i: 0).getReg();
1494 SPIRVTypeInst ScalarType = getSPIRVTypeForVReg(VReg: ScalarReg);
1495 assert(isScalarOrVectorOfType(Type->getOperand(0).getReg(),
1496 ScalarType->getOpcode()));
1497 return ScalarType;
1498}
1499
1500unsigned
1501SPIRVGlobalRegistry::getScalarOrVectorBitWidth(SPIRVTypeInst Type) const {
1502 assert(Type && "Invalid Type pointer");
1503 SPIRVTypeInst ScalarType = getScalarOrVectorComponentType(Type);
1504 if (ScalarType->getOpcode() == SPIRV::OpTypeInt ||
1505 ScalarType->getOpcode() == SPIRV::OpTypeFloat)
1506 return ScalarType->getOperand(i: 1).getImm();
1507 if (ScalarType->getOpcode() == SPIRV::OpTypeBool)
1508 return 1;
1509 if (ScalarType->getOpcode() == SPIRV::OpTypePointer)
1510 return getPointerSize(); // TODO: does not work for different per AS sizes.
1511 llvm_unreachable(
1512 "Attempting to get bit width of non-integer/float/pointer type.");
1513}
1514
1515unsigned SPIRVGlobalRegistry::getNumScalarOrVectorTotalBitWidth(
1516 SPIRVTypeInst Type) const {
1517 assert(Type && "Invalid Type pointer");
1518 unsigned NumElements = getScalarOrVectorComponentCount(Type);
1519 SPIRVTypeInst ScalarType = getScalarOrVectorComponentType(Type);
1520 return ScalarType->getOpcode() == SPIRV::OpTypeInt ||
1521 ScalarType->getOpcode() == SPIRV::OpTypeFloat
1522 ? NumElements * ScalarType->getOperand(i: 1).getImm()
1523 : 0;
1524}
1525
1526bool SPIRVGlobalRegistry::shouldKeepTypedPtrType(SPIRVTypeInst ElemType) const {
1527 // A function pointer has to keep its function type, which an untyped pointer
1528 // cannot express.
1529 if (ElemType && ElemType->getOpcode() == SPIRV::OpTypeFunction)
1530 return true;
1531 auto It = SPIRVToLLVMType.find(Val: ElemType);
1532 return It != SPIRVToLLVMType.end() && It->second &&
1533 isSpecialOpaqueType(Ty: It->second);
1534}
1535
1536bool SPIRVGlobalRegistry::shouldUseUntypedPointer(
1537 SPIRVTypeInst ElemType, const SPIRVSubtarget &ST) const {
1538 // Shaders keep typed pointers, as this implementation targets compute.
1539 return ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_untyped_pointers) &&
1540 !ST.isShader() && !shouldKeepTypedPtrType(ElemType);
1541}
1542
1543SPIRVTypeInst
1544SPIRVGlobalRegistry::retrieveScalarOrVectorIntType(SPIRVTypeInst Type) const {
1545 SPIRVTypeInst ScalarType = getScalarOrVectorComponentType(Type);
1546 return ScalarType && ScalarType->getOpcode() == SPIRV::OpTypeInt ? ScalarType
1547 : nullptr;
1548}
1549
1550bool SPIRVGlobalRegistry::isScalarOrVectorSigned(SPIRVTypeInst Type) const {
1551 SPIRVTypeInst IntType = retrieveScalarOrVectorIntType(Type);
1552 return IntType && IntType->getOperand(i: 2).getImm() != 0;
1553}
1554
1555SPIRVTypeInst SPIRVGlobalRegistry::getPointeeType(SPIRVTypeInst PtrType) {
1556 return PtrType && PtrType->getOpcode() == SPIRV::OpTypePointer
1557 ? getSPIRVTypeForVReg(VReg: PtrType->getOperand(i: 2).getReg())
1558 : nullptr;
1559}
1560
1561bool SPIRVGlobalRegistry::isBitcastCompatible(SPIRVTypeInst Type1,
1562 SPIRVTypeInst Type2) const {
1563 if (!Type1 || !Type2)
1564 return false;
1565 // Ignore difference between <1.5 and >=1.5 protocol versions:
1566 // it's valid if either Result Type or Operand is a pointer, and the other
1567 // is a pointer, an integer scalar, or an integer vector.
1568 if (Type1.isPointer() &&
1569 (Type2.isPointer() || retrieveScalarOrVectorIntType(Type: Type2)))
1570 return true;
1571 if (Type2.isPointer() &&
1572 (Type1.isPointer() || retrieveScalarOrVectorIntType(Type: Type1)))
1573 return true;
1574 unsigned Bits1 = getNumScalarOrVectorTotalBitWidth(Type: Type1),
1575 Bits2 = getNumScalarOrVectorTotalBitWidth(Type: Type2);
1576 return Bits1 > 0 && Bits1 == Bits2;
1577}
1578
1579SPIRV::StorageClass::StorageClass
1580SPIRVGlobalRegistry::getPointerStorageClass(Register VReg) const {
1581 SPIRVTypeInst Type = getSPIRVTypeForVReg(VReg);
1582 assert(Type && Type.isPointer() && Type->getOperand(1).isImm() &&
1583 "Pointer type is expected");
1584 return getPointerStorageClass(Type);
1585}
1586
1587SPIRV::StorageClass::StorageClass
1588SPIRVGlobalRegistry::getPointerStorageClass(SPIRVTypeInst Type) const {
1589 return static_cast<SPIRV::StorageClass::StorageClass>(
1590 Type->getOperand(i: 1).getImm());
1591}
1592
1593SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateVulkanBufferType(
1594 MachineIRBuilder &MIRBuilder, Type *ElemType,
1595 SPIRV::StorageClass::StorageClass SC, bool IsWritable, bool EmitIr) {
1596 auto Key = SPIRV::irhandle_vkbuffer(ElementType: ElemType, SC, IsWriteable: IsWritable);
1597 if (const MachineInstr *MI = findMI(Handle: Key, MF: &MIRBuilder.getMF()))
1598 return MI;
1599
1600 bool ExplicitLayoutRequired = storageClassRequiresExplictLayout(SC);
1601 // We need to get the SPIR-V type for the element here, so we can add the
1602 // decoration to it.
1603 auto *T = StructType::create(Elements: ElemType);
1604 SPIRVTypeInst BlockType =
1605 getOrCreateSPIRVType(Ty: T, MIRBuilder, AccessQual: SPIRV::AccessQualifier::None,
1606 ExplicitLayoutRequired, EmitIR: EmitIr);
1607
1608 buildOpDecorate(Reg: BlockType->defs().begin()->getReg(), MIRBuilder,
1609 Dec: SPIRV::Decoration::Block, DecArgs: {});
1610 BlockDecoratedTypes.insert(V: BlockType);
1611
1612 if (!IsWritable) {
1613 buildOpMemberDecorate(Reg: BlockType->defs().begin()->getReg(), MIRBuilder,
1614 Dec: SPIRV::Decoration::NonWritable, Member: 0, DecArgs: {});
1615 }
1616
1617 SPIRVTypeInst R =
1618 getOrCreateSPIRVPointerTypeInternal(BaseType: BlockType, MIRBuilder, SC);
1619 add(Handle: Key, MI: R);
1620 return R;
1621}
1622
1623SPIRVTypeInst
1624SPIRVGlobalRegistry::getOrCreatePaddingType(MachineIRBuilder &MIRBuilder) {
1625 auto Key = SPIRV::irhandle_padding();
1626 if (const MachineInstr *MI = findMI(Handle: Key, MF: &MIRBuilder.getMF()))
1627 return MI;
1628 auto *T = Type::getInt8Ty(C&: MIRBuilder.getContext());
1629 SPIRVTypeInst R = getOrCreateSPIRVIntegerType(BitWidth: 8, MIRBuilder);
1630 finishCreatingSPIRVType(LLVMTy: T, SpirvType: R);
1631 add(Handle: Key, MI: R);
1632 return R;
1633}
1634
1635SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateVulkanPushConstantType(
1636 MachineIRBuilder &MIRBuilder, Type *T) {
1637 const auto SC = SPIRV::StorageClass::PushConstant;
1638
1639 auto Key = SPIRV::irhandle_vkbuffer(ElementType: T, SC, /* IsWritable= */ IsWriteable: false);
1640 if (const MachineInstr *MI = findMI(Handle: Key, MF: &MIRBuilder.getMF()))
1641 return MI;
1642
1643 // We need to get the SPIR-V type for the element here, so we can add the
1644 // decoration to it.
1645 SPIRVTypeInst BlockType = getOrCreateSPIRVType(
1646 Ty: T, MIRBuilder, AccessQual: SPIRV::AccessQualifier::None,
1647 /* ExplicitLayoutRequired= */ true, /* EmitIr= */ EmitIR: false);
1648
1649 buildOpDecorate(Reg: BlockType->defs().begin()->getReg(), MIRBuilder,
1650 Dec: SPIRV::Decoration::Block, DecArgs: {});
1651 BlockDecoratedTypes.insert(V: BlockType);
1652 SPIRVTypeInst R = BlockType;
1653 add(Handle: Key, MI: R);
1654 return R;
1655}
1656
1657SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateLayoutType(
1658 MachineIRBuilder &MIRBuilder, const TargetExtType *T, bool EmitIr) {
1659 auto Key = SPIRV::handle(Ty: T);
1660 if (const MachineInstr *MI = findMI(Handle: Key, MF: &MIRBuilder.getMF()))
1661 return MI;
1662
1663 StructType *ST = cast<StructType>(Val: T->getTypeParameter(i: 0));
1664 ArrayRef<uint32_t> Offsets = T->int_params().slice(N: 1);
1665 assert(ST->getNumElements() == Offsets.size());
1666
1667 StructOffsetDecorator Decorator = [&MIRBuilder, &Offsets](Register Reg) {
1668 for (uint32_t I = 0; I < Offsets.size(); ++I) {
1669 buildOpMemberDecorate(Reg, MIRBuilder, Dec: SPIRV::Decoration::Offset, Member: I,
1670 DecArgs: {Offsets[I]});
1671 }
1672 };
1673
1674 // We need a new OpTypeStruct instruction because decorations will be
1675 // different from a struct with an explicit layout created from a different
1676 // entry point.
1677 SPIRVTypeInst SPIRVStructType =
1678 getOpTypeStruct(Ty: ST, MIRBuilder, AccQual: SPIRV::AccessQualifier::None,
1679 Decorator: std::move(Decorator), EmitIR: EmitIr);
1680 add(Handle: Key, MI: SPIRVStructType);
1681 return SPIRVStructType;
1682}
1683
1684SPIRVTypeInst SPIRVGlobalRegistry::getImageType(
1685 const TargetExtType *ExtensionType,
1686 const SPIRV::AccessQualifier::AccessQualifier Qualifier,
1687 MachineIRBuilder &MIRBuilder) {
1688 assert(ExtensionType->getNumTypeParameters() == 1 &&
1689 "SPIR-V image builtin type must have sampled type parameter!");
1690 const SPIRVTypeInst SampledType =
1691 getOrCreateSPIRVType(Type: ExtensionType->getTypeParameter(i: 0), MIRBuilder,
1692 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
1693 assert((ExtensionType->getNumIntParameters() == 7 ||
1694 ExtensionType->getNumIntParameters() == 6) &&
1695 "Invalid number of parameters for SPIR-V image builtin!");
1696
1697 SPIRV::AccessQualifier::AccessQualifier accessQualifier =
1698 SPIRV::AccessQualifier::None;
1699 if (ExtensionType->getNumIntParameters() == 7) {
1700 accessQualifier = Qualifier == SPIRV::AccessQualifier::WriteOnly
1701 ? SPIRV::AccessQualifier::WriteOnly
1702 : SPIRV::AccessQualifier::AccessQualifier(
1703 ExtensionType->getIntParameter(i: 6));
1704 }
1705
1706 // Create or get an existing type from GlobalRegistry.
1707 SPIRVTypeInst R = getOrCreateOpTypeImage(
1708 MIRBuilder, SampledType,
1709 Dim: SPIRV::Dim::Dim(ExtensionType->getIntParameter(i: 0)),
1710 Depth: ExtensionType->getIntParameter(i: 1), Arrayed: ExtensionType->getIntParameter(i: 2),
1711 Multisampled: ExtensionType->getIntParameter(i: 3), Sampled: ExtensionType->getIntParameter(i: 4),
1712 ImageFormat: SPIRV::ImageFormat::ImageFormat(ExtensionType->getIntParameter(i: 5)),
1713 AccQual: accessQualifier);
1714 SPIRVToLLVMType[R] = ExtensionType;
1715 return R;
1716}
1717
1718SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateOpTypeImage(
1719 MachineIRBuilder &MIRBuilder, SPIRVTypeInst SampledType,
1720 SPIRV::Dim::Dim Dim, uint32_t Depth, uint32_t Arrayed,
1721 uint32_t Multisampled, uint32_t Sampled,
1722 SPIRV::ImageFormat::ImageFormat ImageFormat,
1723 SPIRV::AccessQualifier::AccessQualifier AccessQual) {
1724 auto Key = SPIRV::irhandle_image(SampledTy: SPIRVToLLVMType.lookup(Val: SampledType), Dim,
1725 Depth, Arrayed, MS: Multisampled, Sampled,
1726 ImageFormat, AQ: AccessQual);
1727 if (const MachineInstr *MI = findMI(Handle: Key, MF: &MIRBuilder.getMF()))
1728 return MI;
1729 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1730 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
1731 auto MIB =
1732 MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeImage)
1733 .addDef(RegNo: createTypeVReg(MIRBuilder))
1734 .addUse(RegNo: getSPIRVTypeID(SpirvType: SampledType))
1735 .addImm(Val: Dim)
1736 .addImm(Val: Depth) // Depth (whether or not it is a Depth image).
1737 .addImm(Val: Arrayed) // Arrayed.
1738 .addImm(Val: Multisampled) // Multisampled (0 = only single-sample).
1739 .addImm(Val: Sampled) // Sampled (0 = usage known at runtime).
1740 .addImm(Val: ImageFormat);
1741 if (AccessQual != SPIRV::AccessQualifier::None)
1742 MIB.addImm(Val: AccessQual);
1743 return MIB;
1744 });
1745 add(Handle: Key, MI: NewMI);
1746 return NewMI;
1747}
1748
1749SPIRVTypeInst
1750SPIRVGlobalRegistry::getOrCreateOpTypeSampler(MachineIRBuilder &MIRBuilder) {
1751 auto Key = SPIRV::irhandle_sampler();
1752 const MachineFunction *MF = &MIRBuilder.getMF();
1753 if (const MachineInstr *MI = findMI(Handle: Key, MF))
1754 return MI;
1755 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1756 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
1757 return MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeSampler)
1758 .addDef(RegNo: createTypeVReg(MIRBuilder));
1759 });
1760 add(Handle: Key, MI: NewMI);
1761 return NewMI;
1762}
1763
1764SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateOpTypePipe(
1765 MachineIRBuilder &MIRBuilder,
1766 SPIRV::AccessQualifier::AccessQualifier AccessQual) {
1767 auto Key = SPIRV::irhandle_pipe(AQ: AccessQual);
1768 if (const MachineInstr *MI = findMI(Handle: Key, MF: &MIRBuilder.getMF()))
1769 return MI;
1770 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1771 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
1772 return MIRBuilder.buildInstr(Opcode: SPIRV::OpTypePipe)
1773 .addDef(RegNo: createTypeVReg(MIRBuilder))
1774 .addImm(Val: AccessQual);
1775 });
1776 add(Handle: Key, MI: NewMI);
1777 return NewMI;
1778}
1779
1780SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateOpTypeDeviceEvent(
1781 MachineIRBuilder &MIRBuilder) {
1782 auto Key = SPIRV::irhandle_event();
1783 if (const MachineInstr *MI = findMI(Handle: Key, MF: &MIRBuilder.getMF()))
1784 return MI;
1785 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1786 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
1787 return MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeDeviceEvent)
1788 .addDef(RegNo: createTypeVReg(MIRBuilder));
1789 });
1790 add(Handle: Key, MI: NewMI);
1791 return NewMI;
1792}
1793
1794SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateOpTypeSampledImage(
1795 SPIRVTypeInst ImageType, MachineIRBuilder &MIRBuilder) {
1796 auto Key = SPIRV::irhandle_sampled_image(
1797 SampledTy: SPIRVToLLVMType.lookup(Val: MIRBuilder.getMF().getRegInfo().getVRegDef(
1798 Reg: ImageType->getOperand(i: 1).getReg())),
1799 ImageTy: ImageType);
1800 if (const MachineInstr *MI = findMI(Handle: Key, MF: &MIRBuilder.getMF()))
1801 return MI;
1802 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1803 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
1804 return MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeSampledImage)
1805 .addDef(RegNo: createTypeVReg(MIRBuilder))
1806 .addUse(RegNo: getSPIRVTypeID(SpirvType: ImageType));
1807 });
1808 add(Handle: Key, MI: NewMI);
1809 return NewMI;
1810}
1811
1812SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateOpTypeCoopMatr(
1813 MachineIRBuilder &MIRBuilder, const TargetExtType *ExtensionType,
1814 SPIRVTypeInst ElemType, uint32_t Scope, uint32_t Rows, uint32_t Columns,
1815 uint32_t Use, bool EmitIR) {
1816 if (const MachineInstr *MI =
1817 findMI(T: ExtensionType, RequiresExplicitLayout: false, MF: &MIRBuilder.getMF()))
1818 return MI;
1819 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1820 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
1821 SPIRVTypeInst SpvTypeInt32 =
1822 getOrCreateSPIRVIntegerType(BitWidth: 32, MIRBuilder);
1823 const Type *ET = getTypeForSPIRVType(Ty: ElemType);
1824 if (ET->isIntegerTy() && ET->getIntegerBitWidth() == 4 &&
1825 cast<SPIRVSubtarget>(Val: MIRBuilder.getMF().getSubtarget())
1826 .canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4)) {
1827 MIRBuilder.buildInstr(Opcode: SPIRV::OpCapability)
1828 .addImm(Val: SPIRV::Capability::Int4CooperativeMatrixINTEL);
1829 }
1830 return MIRBuilder.buildInstr(Opcode: SPIRV::OpTypeCooperativeMatrixKHR)
1831 .addDef(RegNo: createTypeVReg(MIRBuilder))
1832 .addUse(RegNo: getSPIRVTypeID(SpirvType: ElemType))
1833 .addUse(RegNo: buildConstantInt(Val: Scope, MIRBuilder, SpvType: SpvTypeInt32, EmitIR))
1834 .addUse(RegNo: buildConstantInt(Val: Rows, MIRBuilder, SpvType: SpvTypeInt32, EmitIR))
1835 .addUse(RegNo: buildConstantInt(Val: Columns, MIRBuilder, SpvType: SpvTypeInt32, EmitIR))
1836 .addUse(RegNo: buildConstantInt(Val: Use, MIRBuilder, SpvType: SpvTypeInt32, EmitIR));
1837 });
1838 add(T: ExtensionType, RequiresExplicitLayout: false, MI: NewMI);
1839 return NewMI;
1840}
1841
1842SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateOpTypeByOpcode(
1843 const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode) {
1844 if (const MachineInstr *MI = findMI(T: Ty, RequiresExplicitLayout: false, MF: &MIRBuilder.getMF()))
1845 return MI;
1846 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1847 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
1848 return MIRBuilder.buildInstr(Opcode).addDef(RegNo: createTypeVReg(MIRBuilder));
1849 });
1850 add(T: Ty, RequiresExplicitLayout: false, MI: NewMI);
1851 return NewMI;
1852}
1853
1854SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateUnknownType(
1855 const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode,
1856 const ArrayRef<MachineOperand> Operands) {
1857 if (const MachineInstr *MI = findMI(T: Ty, RequiresExplicitLayout: false, MF: &MIRBuilder.getMF()))
1858 return MI;
1859 Register ResVReg = createTypeVReg(MIRBuilder);
1860 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1861 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
1862 MachineInstrBuilder MIB = MIRBuilder.buildInstr(Opcode: SPIRV::UNKNOWN_type)
1863 .addDef(RegNo: ResVReg)
1864 .addImm(Val: Opcode);
1865 for (const MachineOperand &Operand : Operands) {
1866 assert((Operand.isReg() || Operand.isImm()) &&
1867 "Expected register or immediate operand");
1868 MIB.add(MO: Operand);
1869 }
1870 return MIB;
1871 });
1872 add(T: Ty, RequiresExplicitLayout: false, MI: NewMI);
1873 return NewMI;
1874}
1875
1876// Returns nullptr if unable to recognize SPIRV type name
1877SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSPIRVTypeByName(
1878 StringRef TypeStr, MachineIRBuilder &MIRBuilder, bool EmitIR,
1879 SPIRV::StorageClass::StorageClass SC,
1880 SPIRV::AccessQualifier::AccessQualifier AQ) {
1881 unsigned VecElts = 0;
1882 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
1883
1884 // Parse strings representing either a SPIR-V or OpenCL builtin type.
1885 if (hasBuiltinTypePrefix(Name: TypeStr))
1886 return getOrCreateSPIRVType(Ty: SPIRV::parseBuiltinTypeNameToTargetExtType(
1887 TypeName: TypeStr.str(), Context&: MIRBuilder.getContext()),
1888 MIRBuilder, AccessQual: AQ, ExplicitLayoutRequired: false, EmitIR: true);
1889
1890 // Parse type name in either "typeN" or "type vector[N]" format, where
1891 // N is the number of elements of the vector.
1892 Type *Ty;
1893
1894 Ty = parseBasicTypeName(TypeName&: TypeStr, Ctx);
1895 if (!Ty)
1896 // Unable to recognize SPIRV type name
1897 return nullptr;
1898
1899 SPIRVTypeInst SpirvTy = getOrCreateSPIRVType(Ty, MIRBuilder, AccessQual: AQ, ExplicitLayoutRequired: false, EmitIR: true);
1900
1901 // Handle "type*" or "type* vector[N]".
1902 if (TypeStr.consume_front(Prefix: "*"))
1903 SpirvTy = getOrCreateSPIRVPointerType(BaseType: Ty, MIRBuilder, SC);
1904
1905 // Handle "typeN*" or "type vector[N]*".
1906 bool IsPtrToVec = TypeStr.consume_back(Suffix: "*");
1907
1908 if (TypeStr.consume_front(Prefix: " vector[")) {
1909 TypeStr = TypeStr.substr(Start: 0, N: TypeStr.find(C: ']'));
1910 }
1911 TypeStr.getAsInteger(Radix: 10, Result&: VecElts);
1912 if (VecElts > 0)
1913 SpirvTy = getOrCreateSPIRVVectorType(BaseType: SpirvTy, NumElements: VecElts, MIRBuilder, EmitIR);
1914
1915 if (IsPtrToVec)
1916 SpirvTy = getOrCreateSPIRVPointerType(BaseType: SpirvTy, MIRBuilder, SC);
1917
1918 return SpirvTy;
1919}
1920
1921SPIRVTypeInst
1922SPIRVGlobalRegistry::getOrCreateSPIRVIntegerType(unsigned BitWidth,
1923 MachineIRBuilder &MIRBuilder) {
1924 return getOrCreateSPIRVType(
1925 Ty: IntegerType::get(C&: MIRBuilder.getMF().getFunction().getContext(), NumBits: BitWidth),
1926 MIRBuilder, AccessQual: SPIRV::AccessQualifier::ReadWrite, ExplicitLayoutRequired: false, EmitIR: true);
1927}
1928
1929SPIRVTypeInst
1930SPIRVGlobalRegistry::finishCreatingSPIRVType(const Type *LLVMTy,
1931 SPIRVTypeInst SpirvType) {
1932 assert(CurMF == SpirvType->getMF());
1933 VRegToTypeMap[CurMF][getSPIRVTypeID(SpirvType)] = SpirvType;
1934 SPIRVToLLVMType[SpirvType] = unifyPtrType(Ty: LLVMTy);
1935 return SpirvType;
1936}
1937
1938SPIRVTypeInst
1939SPIRVGlobalRegistry::getOrCreateSPIRVType(unsigned BitWidth, MachineInstr &I,
1940 const SPIRVInstrInfo &TII,
1941 unsigned SPIRVOPcode, Type *Ty) {
1942 if (const MachineInstr *MI = findMI(T: Ty, RequiresExplicitLayout: false, MF: CurMF))
1943 return MI;
1944 MachineBasicBlock &DepMBB = I.getMF()->front();
1945 MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI());
1946 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
1947 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
1948 auto NewTypeMI = BuildMI(BB&: MIRBuilder.getMBB(), I&: *MIRBuilder.getInsertPt(),
1949 MIMD: MIRBuilder.getDL(), MCID: TII.get(Opcode: SPIRVOPcode))
1950 .addDef(RegNo: createTypeVReg(MRI&: CurMF->getRegInfo()))
1951 .addImm(Val: BitWidth);
1952 // Don't add Encoding to FP type
1953 if (!Ty->isFloatTy()) {
1954 return NewTypeMI.addImm(Val: 0);
1955 } else {
1956 return NewTypeMI;
1957 }
1958 });
1959 add(T: Ty, RequiresExplicitLayout: false, MI: NewMI);
1960 return finishCreatingSPIRVType(LLVMTy: Ty, SpirvType: NewMI);
1961}
1962
1963SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSPIRVIntegerType(
1964 unsigned BitWidth, MachineInstr &I, const SPIRVInstrInfo &TII) {
1965 // Maybe adjust bit width to keep DuplicateTracker consistent. Without
1966 // such an adjustment SPIRVGlobalRegistry::getOpTypeInt() could create, for
1967 // example, the same "OpTypeInt 8" type for a series of LLVM integer types
1968 // with number of bits less than 8, causing duplicate type definitions.
1969 if (BitWidth > 1)
1970 BitWidth = adjustOpTypeIntWidth(Width: BitWidth);
1971 Type *LLVMTy = IntegerType::get(C&: CurMF->getFunction().getContext(), NumBits: BitWidth);
1972 return getOrCreateSPIRVType(BitWidth, I, TII, SPIRVOPcode: SPIRV::OpTypeInt, Ty: LLVMTy);
1973}
1974
1975SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSPIRVFloatType(
1976 unsigned BitWidth, MachineInstr &I, const SPIRVInstrInfo &TII) {
1977 LLVMContext &Ctx = CurMF->getFunction().getContext();
1978 Type *LLVMTy;
1979 switch (BitWidth) {
1980 case 16:
1981 LLVMTy = Type::getHalfTy(C&: Ctx);
1982 break;
1983 case 32:
1984 LLVMTy = Type::getFloatTy(C&: Ctx);
1985 break;
1986 case 64:
1987 LLVMTy = Type::getDoubleTy(C&: Ctx);
1988 break;
1989 default:
1990 llvm_unreachable("Bit width is of unexpected size.");
1991 }
1992 return getOrCreateSPIRVType(BitWidth, I, TII, SPIRVOPcode: SPIRV::OpTypeFloat, Ty: LLVMTy);
1993}
1994
1995SPIRVTypeInst
1996SPIRVGlobalRegistry::getOrCreateSPIRVBoolType(MachineIRBuilder &MIRBuilder,
1997 bool EmitIR) {
1998 return getOrCreateSPIRVType(
1999 Ty: IntegerType::get(C&: MIRBuilder.getMF().getFunction().getContext(), NumBits: 1),
2000 MIRBuilder, AccessQual: SPIRV::AccessQualifier::ReadWrite, ExplicitLayoutRequired: false, EmitIR);
2001}
2002
2003SPIRVTypeInst
2004SPIRVGlobalRegistry::getOrCreateSPIRVBoolType(MachineInstr &I,
2005 const SPIRVInstrInfo &TII) {
2006 Type *Ty = IntegerType::get(C&: CurMF->getFunction().getContext(), NumBits: 1);
2007 if (const MachineInstr *MI = findMI(T: Ty, RequiresExplicitLayout: false, MF: CurMF))
2008 return MI;
2009 MachineBasicBlock &DepMBB = I.getMF()->front();
2010 MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI());
2011 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
2012 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
2013 return BuildMI(BB&: MIRBuilder.getMBB(), I&: *MIRBuilder.getInsertPt(),
2014 MIMD: MIRBuilder.getDL(), MCID: TII.get(Opcode: SPIRV::OpTypeBool))
2015 .addDef(RegNo: createTypeVReg(MRI&: CurMF->getRegInfo()));
2016 });
2017 add(T: Ty, RequiresExplicitLayout: false, MI: NewMI);
2018 return finishCreatingSPIRVType(LLVMTy: Ty, SpirvType: NewMI);
2019}
2020
2021SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSPIRVVectorType(
2022 SPIRVTypeInst BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder,
2023 bool EmitIR) {
2024 return getOrCreateSPIRVType(
2025 Ty: FixedVectorType::get(ElementType: const_cast<Type *>(getTypeForSPIRVType(Ty: BaseType)),
2026 NumElts: NumElements),
2027 MIRBuilder, AccessQual: SPIRV::AccessQualifier::ReadWrite, ExplicitLayoutRequired: false, EmitIR);
2028}
2029
2030SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSPIRVVectorType(
2031 SPIRVTypeInst BaseType, unsigned NumElements, MachineInstr &I,
2032 const SPIRVInstrInfo &TII) {
2033 const auto &STI = I.getMF()->getSubtarget<SPIRVSubtarget>();
2034 if (!STI.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector))
2035 // At this point of time all 1-element vectors are resolved. Add assertion
2036 // to fire if anything changes.
2037 assert(NumElements >= 2 &&
2038 "SPIR-V vectors must have at least 2 components");
2039 Type *Ty = FixedVectorType::get(
2040 ElementType: const_cast<Type *>(getTypeForSPIRVType(Ty: BaseType)), NumElts: NumElements);
2041 if (const MachineInstr *MI = findMI(T: Ty, RequiresExplicitLayout: false, MF: CurMF))
2042 return MI;
2043 MachineInstr *DepMI =
2044 const_cast<MachineInstr *>(static_cast<const MachineInstr *>(BaseType));
2045 MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());
2046 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
2047 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
2048 // TODO: consider adding non-const accessors to SPIRVTypeInst, which
2049 // would remove the need for the gash casting here.
2050 return const_cast<MachineInstr *>(
2051 static_cast<const MachineInstr *>(getOpTypeVectorImpl(
2052 NumElems: NumElements, ElemType: BaseType, MIRBuilder, IsLongVectorEXT: isLongVectorEXT(Ty))));
2053 });
2054 add(T: Ty, RequiresExplicitLayout: false, MI: NewMI);
2055 return finishCreatingSPIRVType(LLVMTy: Ty, SpirvType: NewMI);
2056}
2057
2058SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSPIRVPointerType(
2059 const Type *BaseType, MachineInstr &I, SPIRV::StorageClass::StorageClass SC,
2060 bool ForceTyped) {
2061 MachineIRBuilder MIRBuilder(I);
2062 return getOrCreateSPIRVPointerType(BaseType, MIRBuilder, SC, ForceTyped);
2063}
2064
2065SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSPIRVPointerType(
2066 const Type *BaseType, MachineIRBuilder &MIRBuilder,
2067 SPIRV::StorageClass::StorageClass SC, bool ForceTyped) {
2068 if (BaseType->isFunctionTy() &&
2069 !cast<SPIRVSubtarget>(Val: MIRBuilder.getMF().getSubtarget())
2070 .canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers)) {
2071 const Function &F = MIRBuilder.getMF().getFunction();
2072 F.getContext().diagnose(
2073 DI: DiagnosticInfoUnsupported(F,
2074 "Function used as a data pointer requires "
2075 "SPV_INTEL_function_pointers extension",
2076 DebugLoc(), DS_Error));
2077 }
2078 // TODO: Need to check if EmitIr should always be true.
2079 SPIRVTypeInst SpirvBaseType = getOrCreateSPIRVType(
2080 Ty: BaseType, MIRBuilder, AccessQual: SPIRV::AccessQualifier::ReadWrite,
2081 ExplicitLayoutRequired: storageClassRequiresExplictLayout(SC), EmitIR: true);
2082 assert(SpirvBaseType);
2083 return getOrCreateSPIRVPointerTypeInternal(BaseType: SpirvBaseType, MIRBuilder, SC,
2084 ForceTyped);
2085}
2086
2087SPIRVTypeInst SPIRVGlobalRegistry::changePointerStorageClass(
2088 SPIRVTypeInst PtrType, SPIRV::StorageClass::StorageClass SC,
2089 MachineInstr &I) {
2090 [[maybe_unused]] SPIRV::StorageClass::StorageClass OldSC =
2091 getPointerStorageClass(Type: PtrType);
2092 assert(storageClassRequiresExplictLayout(OldSC) ==
2093 storageClassRequiresExplictLayout(SC));
2094
2095 SPIRVTypeInst PointeeType = getPointeeType(PtrType);
2096 MachineIRBuilder MIRBuilder(I);
2097 return getOrCreateSPIRVPointerTypeInternal(BaseType: PointeeType, MIRBuilder, SC);
2098}
2099
2100SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSPIRVPointerType(
2101 SPIRVTypeInst BaseType, MachineIRBuilder &MIRBuilder,
2102 SPIRV::StorageClass::StorageClass SC) {
2103 const Type *LLVMType = getTypeForSPIRVType(Ty: BaseType);
2104 assert(!storageClassRequiresExplictLayout(SC));
2105 SPIRVTypeInst R = getOrCreateSPIRVPointerType(BaseType: LLVMType, MIRBuilder, SC);
2106 assert(
2107 (R->getOpcode() == SPIRV::OpTypeUntypedPointerKHR ||
2108 getPointeeType(R) == BaseType) &&
2109 "The base type was not correctly laid out for the given storage class.");
2110 return R;
2111}
2112
2113SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSPIRVPointerTypeInternal(
2114 SPIRVTypeInst BaseType, MachineIRBuilder &MIRBuilder,
2115 SPIRV::StorageClass::StorageClass SC, bool ForceTyped) {
2116 // Check if we should use untyped pointers.
2117 const SPIRVSubtarget &ST =
2118 cast<SPIRVSubtarget>(Val: MIRBuilder.getMF().getSubtarget());
2119 if (!ForceTyped && shouldUseUntypedPointer(ElemType: BaseType, ST))
2120 return getOrCreateSPIRVUntypedPointerType(SC, MIRBuilder);
2121
2122 const Type *PointerElementType = getTypeForSPIRVType(Ty: BaseType);
2123 unsigned AddressSpace = storageClassToAddressSpace(SC);
2124 if (const MachineInstr *MI = findMI(PointeeTy: PointerElementType, AddressSpace, MF: CurMF))
2125 return MI;
2126 Type *Ty = TypedPointerType::get(ElementType: const_cast<Type *>(PointerElementType),
2127 AddressSpace);
2128 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
2129 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
2130 return BuildMI(BB&: MIRBuilder.getMBB(), I: MIRBuilder.getInsertPt(),
2131 MIMD: MIRBuilder.getDebugLoc(),
2132 MCID: MIRBuilder.getTII().get(Opcode: SPIRV::OpTypePointer))
2133 .addDef(RegNo: createTypeVReg(MRI&: CurMF->getRegInfo()))
2134 .addImm(Val: static_cast<uint32_t>(SC))
2135 .addUse(RegNo: getSPIRVTypeID(SpirvType: BaseType));
2136 });
2137 add(PointeeTy: PointerElementType, AddressSpace, MI: NewMI);
2138 return finishCreatingSPIRVType(LLVMTy: Ty, SpirvType: NewMI);
2139}
2140
2141SPIRVTypeInst SPIRVGlobalRegistry::getOrCreateSPIRVUntypedPointerType(
2142 SPIRV::StorageClass::StorageClass SC, MachineIRBuilder &MIRBuilder) {
2143 [[maybe_unused]] const SPIRVSubtarget &ST =
2144 cast<SPIRVSubtarget>(Val: MIRBuilder.getMF().getSubtarget());
2145 assert(ST.canUseExtension(SPIRV::Extension::SPV_KHR_untyped_pointers) &&
2146 !ST.isShader() && "Untyped pointers are not available");
2147 unsigned AddressSpace = storageClassToAddressSpace(SC);
2148 // Use STK_UntypedPointer handle keyed by address space only.
2149 auto Handle = SPIRV::irhandle_untyped_pointer(AddressSpace);
2150 if (const MachineInstr *MI = findMI(Handle, MF: CurMF))
2151 return MI;
2152
2153 Type *Ty = PointerType::get(C&: MIRBuilder.getMF().getFunction().getContext(),
2154 AddressSpace);
2155 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
2156 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
2157 return BuildMI(BB&: MIRBuilder.getMBB(), I: MIRBuilder.getInsertPt(),
2158 MIMD: MIRBuilder.getDebugLoc(),
2159 MCID: MIRBuilder.getTII().get(Opcode: SPIRV::OpTypeUntypedPointerKHR))
2160 .addDef(RegNo: createTypeVReg(MRI&: CurMF->getRegInfo()))
2161 .addImm(Val: static_cast<uint32_t>(SC));
2162 });
2163 add(Handle, MI: NewMI);
2164 return finishCreatingSPIRVType(LLVMTy: Ty, SpirvType: NewMI);
2165}
2166
2167Register SPIRVGlobalRegistry::getOrCreateUndef(MachineInstr &I,
2168 SPIRVTypeInst SpvType,
2169 const SPIRVInstrInfo &TII) {
2170 UndefValue *UV =
2171 UndefValue::get(T: const_cast<Type *>(getTypeForSPIRVType(Ty: SpvType)));
2172 Register Res = find(V: UV, MF: CurMF);
2173 if (Res.isValid())
2174 return Res;
2175
2176 LLT LLTy = LLT::scalar(SizeInBits: 64);
2177 Res = CurMF->getRegInfo().createGenericVirtualRegister(Ty: LLTy);
2178 CurMF->getRegInfo().setRegClass(Reg: Res, RC: &SPIRV::iIDRegClass);
2179 assignSPIRVTypeToVReg(SpirvType: SpvType, VReg: Res, MF: *CurMF);
2180
2181 MachineInstr *DepMI =
2182 const_cast<MachineInstr *>(static_cast<const MachineInstr *>(SpvType));
2183 MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());
2184 const MachineInstr *NewMI = createConstOrTypeAtFunctionEntry(
2185 MIRBuilder, Op: [&](MachineIRBuilder &MIRBuilder) {
2186 auto MIB = BuildMI(BB&: MIRBuilder.getMBB(), I&: *MIRBuilder.getInsertPt(),
2187 MIMD: MIRBuilder.getDL(), MCID: TII.get(Opcode: SPIRV::OpUndef))
2188 .addDef(RegNo: Res)
2189 .addUse(RegNo: getSPIRVTypeID(SpirvType: SpvType));
2190 constrainSelectedInstRegOperands(MIB);
2191 return MIB;
2192 });
2193 add(V: UV, MI: NewMI);
2194 return Res;
2195}
2196
2197const TargetRegisterClass *
2198SPIRVGlobalRegistry::getRegClass(SPIRVTypeInst SpvType) const {
2199 unsigned Opcode = SpvType->getOpcode();
2200 switch (Opcode) {
2201 case SPIRV::OpTypeFloat:
2202 return &SPIRV::fIDRegClass;
2203 case SPIRV::OpTypePointer:
2204 case SPIRV::OpTypeUntypedPointerKHR:
2205 return &SPIRV::pIDRegClass;
2206 case SPIRV::OpTypeVector:
2207 case SPIRV::OpTypeVectorIdEXT: {
2208 SPIRVTypeInst ElemType = getScalarOrVectorComponentType(Type: SpvType);
2209 unsigned ElemOpcode = ElemType ? ElemType->getOpcode() : 0;
2210 if (ElemOpcode == SPIRV::OpTypeFloat)
2211 return &SPIRV::vfIDRegClass;
2212 if (ElemType && ElemType.isPointer())
2213 return &SPIRV::vpIDRegClass;
2214 return &SPIRV::viIDRegClass;
2215 }
2216 }
2217 return &SPIRV::iIDRegClass;
2218}
2219
2220inline unsigned getAS(SPIRVTypeInst SpvType) {
2221 return storageClassToAddressSpace(
2222 SC: static_cast<SPIRV::StorageClass::StorageClass>(
2223 SpvType->getOperand(i: 1).getImm()));
2224}
2225
2226static LLT getFloatRegType(SPIRVTypeInst FloatType) {
2227 if (FloatType->getNumOperands() == 3 &&
2228 FloatType->getOperand(i: 2).getImm() == SPIRV::FPEncoding::BFloat16KHR)
2229 return LLT::bfloat16();
2230 return LLT::floatIEEE(SizeInBits: FloatType->getOperand(i: 1).getImm());
2231}
2232
2233LLT SPIRVGlobalRegistry::getRegType(SPIRVTypeInst SpvType) const {
2234 unsigned Opcode = SpvType ? SpvType->getOpcode() : 0;
2235 switch (Opcode) {
2236 case SPIRV::OpTypeFloat:
2237 return getFloatRegType(FloatType: SpvType);
2238 case SPIRV::OpTypeInt:
2239 case SPIRV::OpTypeBool:
2240 return LLT::scalar(SizeInBits: getScalarOrVectorBitWidth(Type: SpvType));
2241 case SPIRV::OpTypePointer:
2242 case SPIRV::OpTypeUntypedPointerKHR:
2243 return LLT::pointer(AddressSpace: getAS(SpvType), SizeInBits: getPointerSize());
2244 case SPIRV::OpTypeVector:
2245 case SPIRV::OpTypeVectorIdEXT: {
2246 SPIRVTypeInst ElemType = getScalarOrVectorComponentType(Type: SpvType);
2247 LLT ET;
2248 switch (ElemType ? ElemType->getOpcode() : 0) {
2249 case SPIRV::OpTypePointer:
2250 case SPIRV::OpTypeUntypedPointerKHR:
2251 ET = LLT::pointer(AddressSpace: getAS(SpvType: ElemType), SizeInBits: getPointerSize());
2252 break;
2253 case SPIRV::OpTypeFloat:
2254 ET = getFloatRegType(FloatType: ElemType);
2255 break;
2256 case SPIRV::OpTypeInt:
2257 case SPIRV::OpTypeBool:
2258 ET = LLT::scalar(SizeInBits: getScalarOrVectorBitWidth(Type: ElemType));
2259 break;
2260 default:
2261 ET = LLT::scalar(SizeInBits: 64);
2262 }
2263 auto EC = ElementCount::getFixed(MinVal: getScalarOrVectorComponentCount(Type: SpvType));
2264 return LLT::scalarOrVector(EC, ScalarTy: ET);
2265 }
2266 }
2267 return LLT::scalar(SizeInBits: 64);
2268}
2269
2270// Aliasing list MD contains several scope MD nodes whithin it. Each scope MD
2271// has a selfreference and an extra MD node for aliasing domain and also it
2272// can contain an optional string operand. Domain MD contains a self-reference
2273// with an optional string operand. Here we unfold the list, creating SPIR-V
2274// aliasing instructions.
2275// TODO: add support for an optional string operand.
2276MachineInstr *SPIRVGlobalRegistry::getOrAddMemAliasingINTELInst(
2277 MachineIRBuilder &MIRBuilder, const MDNode *AliasingListMD) {
2278 if (AliasingListMD->getNumOperands() == 0)
2279 return nullptr;
2280 const MachineFunction *MF = &MIRBuilder.getMF();
2281 if (auto L = AliasInstMDMap.find(Val: {MF, AliasingListMD});
2282 L != AliasInstMDMap.end())
2283 return L->second;
2284
2285 SmallVector<MachineInstr *> ScopeList;
2286 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2287 for (const MDOperand &MDListOp : AliasingListMD->operands()) {
2288 if (MDNode *ScopeMD = dyn_cast<MDNode>(Val: MDListOp)) {
2289 if (ScopeMD->getNumOperands() < 2)
2290 return nullptr;
2291 MDNode *DomainMD = dyn_cast<MDNode>(Val: ScopeMD->getOperand(I: 1));
2292 if (!DomainMD)
2293 return nullptr;
2294 auto *Domain = [&] {
2295 auto D = AliasInstMDMap.find(Val: {MF, DomainMD});
2296 if (D != AliasInstMDMap.end())
2297 return D->second;
2298 const Register Ret = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2299 auto MIB =
2300 MIRBuilder.buildInstr(Opcode: SPIRV::OpAliasDomainDeclINTEL).addDef(RegNo: Ret);
2301 return MIB.getInstr();
2302 }();
2303 AliasInstMDMap.insert(
2304 KV: std::make_pair(x: std::make_pair(x&: MF, y&: DomainMD), y&: Domain));
2305 auto *Scope = [&] {
2306 auto S = AliasInstMDMap.find(Val: {MF, ScopeMD});
2307 if (S != AliasInstMDMap.end())
2308 return S->second;
2309 const Register Ret = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2310 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpAliasScopeDeclINTEL)
2311 .addDef(RegNo: Ret)
2312 .addUse(RegNo: Domain->getOperand(i: 0).getReg());
2313 return MIB.getInstr();
2314 }();
2315 AliasInstMDMap.insert(KV: std::make_pair(x: std::make_pair(x&: MF, y&: ScopeMD), y&: Scope));
2316 ScopeList.push_back(Elt: Scope);
2317 }
2318 }
2319
2320 const Register Ret = MRI->createVirtualRegister(RegClass: &SPIRV::IDRegClass);
2321 auto MIB =
2322 MIRBuilder.buildInstr(Opcode: SPIRV::OpAliasScopeListDeclINTEL).addDef(RegNo: Ret);
2323 for (auto *Scope : ScopeList)
2324 MIB.addUse(RegNo: Scope->getOperand(i: 0).getReg());
2325 auto List = MIB.getInstr();
2326 AliasInstMDMap.insert(
2327 KV: std::make_pair(x: std::make_pair(x&: MF, y&: AliasingListMD), y&: List));
2328 return List;
2329}
2330
2331void SPIRVGlobalRegistry::buildMemAliasingOpDecorate(
2332 Register Reg, MachineIRBuilder &MIRBuilder, uint32_t Dec,
2333 const MDNode *AliasingListMD) {
2334 MachineInstr *AliasList =
2335 getOrAddMemAliasingINTELInst(MIRBuilder, AliasingListMD);
2336 if (!AliasList)
2337 return;
2338 MIRBuilder.buildInstr(Opcode: SPIRV::OpDecorateId)
2339 .addUse(RegNo: Reg)
2340 .addImm(Val: Dec)
2341 .addUse(RegNo: AliasList->getOperand(i: 0).getReg());
2342}
2343void SPIRVGlobalRegistry::replaceAllUsesWith(Value *Old, Value *New,
2344 bool DeleteOld) {
2345 Old->replaceAllUsesWith(V: New);
2346 updateIfExistDeducedElementType(OldVal: Old, NewVal: New, DeleteOld);
2347 updateIfExistAssignPtrTypeInstr(OldVal: Old, NewVal: New, DeleteOld);
2348}
2349
2350void SPIRVGlobalRegistry::buildAssignType(IRBuilder<> &B, Type *Ty, Value *Arg,
2351 bool CanUseAnyVectorRank) {
2352 Value *OfType = getNormalizedPoisonValue(Ty, CanUseAnyVectorRank);
2353 CallInst *AssignCI = nullptr;
2354 if (Arg->getType()->isAggregateType() && Ty->isAggregateType() &&
2355 allowEmitFakeUse(Arg)) {
2356 LLVMContext &Ctx = Arg->getContext();
2357 SmallVector<Metadata *, 2> ArgMDs{
2358 MDNode::get(Context&: Ctx, MDs: ValueAsMetadata::getConstant(C: OfType)),
2359 MDString::get(Context&: Ctx, Str: Arg->getName())};
2360 B.CreateIntrinsic(ID: Intrinsic::spv_value_md,
2361 Args: {MetadataAsValue::get(Context&: Ctx, MD: MDTuple::get(Context&: Ctx, MDs: ArgMDs))});
2362 AssignCI = B.CreateIntrinsicWithoutFolding(ID: Intrinsic::fake_use, Args: {Arg});
2363 } else {
2364 AssignCI = buildIntrWithMD(IntrID: Intrinsic::spv_assign_type, Types: {Arg->getType()},
2365 Arg: OfType, Arg2: Arg, Imms: {}, B);
2366 }
2367 addAssignPtrTypeInstr(Val: Arg, AssignPtrTyCI: AssignCI);
2368}
2369
2370void SPIRVGlobalRegistry::buildAssignPtr(IRBuilder<> &B, Type *ElemTy,
2371 Value *Arg) {
2372 Value *OfType = PoisonValue::get(T: ElemTy);
2373 CallInst *AssignPtrTyCI = findAssignPtrTypeInstr(Val: Arg);
2374 Function *CurrF =
2375 B.GetInsertBlock() ? B.GetInsertBlock()->getParent() : nullptr;
2376 if (AssignPtrTyCI == nullptr ||
2377 AssignPtrTyCI->getParent()->getParent() != CurrF) {
2378 AssignPtrTyCI = buildIntrWithMD(
2379 IntrID: Intrinsic::spv_assign_ptr_type, Types: {Arg->getType()}, Arg: OfType, Arg2: Arg,
2380 Imms: {B.getInt32(C: getPointerAddressSpace(T: Arg->getType()))}, B);
2381 addDeducedElementType(Val: AssignPtrTyCI, Ty: ElemTy);
2382 addDeducedElementType(Val: Arg, Ty: ElemTy);
2383 addAssignPtrTypeInstr(Val: Arg, AssignPtrTyCI);
2384 } else {
2385 updateAssignType(AssignCI: AssignPtrTyCI, Arg, OfType);
2386 }
2387}
2388
2389void SPIRVGlobalRegistry::updateAssignType(CallInst *AssignCI, Value *Arg,
2390 Value *OfType) {
2391 AssignCI->setArgOperand(i: 1, v: buildMD(Arg: OfType));
2392 if (cast<IntrinsicInst>(Val: AssignCI)->getIntrinsicID() !=
2393 Intrinsic::spv_assign_ptr_type)
2394 return;
2395
2396 // update association with the pointee type
2397 Type *ElemTy = OfType->getType();
2398 addDeducedElementType(Val: AssignCI, Ty: ElemTy);
2399 addDeducedElementType(Val: Arg, Ty: ElemTy);
2400}
2401
2402void SPIRVGlobalRegistry::addStructOffsetDecorations(
2403 Register Reg, StructType *Ty, MachineIRBuilder &MIRBuilder) {
2404 ArrayRef<TypeSize> Offsets = DL.getStructLayout(Ty)->getMemberOffsets();
2405 for (uint32_t I = 0; I < Ty->getNumElements(); ++I) {
2406 buildOpMemberDecorate(Reg, MIRBuilder, Dec: SPIRV::Decoration::Offset, Member: I,
2407 DecArgs: {static_cast<uint32_t>(Offsets[I])});
2408 }
2409}
2410
2411void SPIRVGlobalRegistry::addArrayStrideDecorations(
2412 Register Reg, Type *ElementType, MachineIRBuilder &MIRBuilder) {
2413 uint32_t SizeInBytes = DL.getTypeAllocSize(Ty: ElementType);
2414 buildOpDecorate(Reg, MIRBuilder, Dec: SPIRV::Decoration::ArrayStride,
2415 DecArgs: {SizeInBytes});
2416}
2417