1//===- SPIRVBuiltins.cpp - SPIR-V Built-in Functions ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements lowering builtin function calls and types using their
10// demangled names and TableGen records.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRVBuiltins.h"
15#include "SPIRV.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVUtils.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/ADT/StringTable.h"
20#include "llvm/Analysis/ValueTracking.h"
21#include "llvm/IR/DiagnosticInfo.h"
22#include "llvm/IR/IntrinsicsSPIRV.h"
23#include <regex>
24#include <string>
25#include <tuple>
26
27#define DEBUG_TYPE "spirv-builtins"
28
29namespace llvm {
30namespace SPIRV {
31#define GET_BuiltinGroup_DECL
32#include "SPIRVGenTables.inc"
33
34struct DemangledBuiltin {
35 StringTable::Offset Name;
36 InstructionSet::InstructionSet Set;
37 BuiltinGroup Group;
38 uint8_t MinNumArgs;
39 uint8_t MaxNumArgs;
40
41 StringRef name() const;
42};
43
44#define GET_DemangledBuiltins_DECL
45#define GET_DemangledBuiltins_IMPL
46
47struct IncomingCall {
48 const std::string BuiltinName;
49 const DemangledBuiltin *Builtin;
50
51 const Register ReturnRegister;
52 const SPIRVTypeInst ReturnType;
53 const SmallVectorImpl<Register> &Arguments;
54
55 IncomingCall(const std::string BuiltinName, const DemangledBuiltin *Builtin,
56 const Register ReturnRegister, SPIRVTypeInst ReturnType,
57 const SmallVectorImpl<Register> &Arguments)
58 : BuiltinName(std::move(BuiltinName)), Builtin(Builtin),
59 ReturnRegister(ReturnRegister), ReturnType(ReturnType),
60 Arguments(Arguments) {}
61
62 bool isSpirvOp() const { return BuiltinName.rfind(s: "__spirv_", pos: 0) == 0; }
63};
64
65struct NativeBuiltin {
66 StringTable::Offset Name;
67 InstructionSet::InstructionSet Set;
68 uint32_t Opcode;
69};
70
71#define GET_NativeBuiltins_DECL
72#define GET_NativeBuiltins_IMPL
73
74struct GroupBuiltin {
75 StringTable::Offset Name;
76 uint32_t Opcode;
77 uint32_t GroupOperation;
78 bool IsElect;
79 bool IsAllOrAny;
80 bool IsAllEqual;
81 bool IsInverseBallot;
82 bool IsBallotBitExtract;
83 bool IsLogical;
84 bool NoGroupOperation;
85 bool HasBoolArg;
86};
87
88#define GET_GroupBuiltins_DECL
89#define GET_GroupBuiltins_IMPL
90
91struct IntelSubgroupsBuiltin {
92 StringTable::Offset Name;
93 uint32_t Opcode;
94 bool IsBlock;
95 bool IsWrite;
96 bool IsMedia;
97};
98
99#define GET_IntelSubgroupsBuiltins_DECL
100#define GET_IntelSubgroupsBuiltins_IMPL
101
102struct AtomicFloatingBuiltin {
103 StringTable::Offset Name;
104 uint32_t Opcode;
105};
106
107#define GET_AtomicFloatingBuiltins_DECL
108#define GET_AtomicFloatingBuiltins_IMPL
109struct GroupUniformBuiltin {
110 StringTable::Offset Name;
111 uint32_t Opcode;
112 bool IsLogical;
113};
114
115#define GET_GroupUniformBuiltins_DECL
116#define GET_GroupUniformBuiltins_IMPL
117
118struct GetBuiltin {
119 StringTable::Offset Name;
120 InstructionSet::InstructionSet Set;
121 BuiltIn::BuiltIn Value;
122};
123
124using namespace BuiltIn;
125#define GET_GetBuiltins_DECL
126#define GET_GetBuiltins_IMPL
127
128struct ImageQueryBuiltin {
129 StringTable::Offset Name;
130 InstructionSet::InstructionSet Set;
131 uint32_t Component;
132};
133
134#define GET_ImageQueryBuiltins_DECL
135#define GET_ImageQueryBuiltins_IMPL
136
137struct IntegerDotProductBuiltin {
138 StringTable::Offset Name;
139 uint32_t Opcode;
140 bool IsSwapReq;
141};
142
143#define GET_IntegerDotProductBuiltins_DECL
144#define GET_IntegerDotProductBuiltins_IMPL
145
146struct ConvertBuiltin {
147 StringTable::Offset Name;
148 InstructionSet::InstructionSet Set;
149 bool IsDestinationSigned;
150 bool IsSaturated;
151 bool IsRounded;
152 bool IsBfloat16;
153 bool IsTF32;
154 FPRoundingMode::FPRoundingMode RoundingMode;
155};
156
157struct VectorLoadStoreBuiltin {
158 StringTable::Offset Name;
159 InstructionSet::InstructionSet Set;
160 uint32_t Number;
161 uint32_t ElementCount;
162 bool IsRounded;
163 FPRoundingMode::FPRoundingMode RoundingMode;
164};
165
166using namespace FPRoundingMode;
167#define GET_ConvertBuiltins_DECL
168#define GET_ConvertBuiltins_IMPL
169
170using namespace InstructionSet;
171#define GET_VectorLoadStoreBuiltins_DECL
172#define GET_VectorLoadStoreBuiltins_IMPL
173
174#define GET_CLMemoryScope_DECL
175#define GET_CLSamplerAddressingMode_DECL
176#define GET_CLMemoryFenceFlags_DECL
177#define GET_ExtendedBuiltins_DECL
178#include "SPIRVGenTables.inc"
179
180// Defined here to reference declarations from tablegen.
181StringRef DemangledBuiltin::name() const {
182 return getDemangledBuiltinStr(Offset: Name);
183}
184} // namespace SPIRV
185
186//===----------------------------------------------------------------------===//
187// Misc functions for looking up builtins and veryfying requirements using
188// TableGen records
189//===----------------------------------------------------------------------===//
190
191namespace SPIRV {
192/// Parses the name part of the demangled builtin call.
193std::string lookupBuiltinNameHelper(StringRef DemangledCall,
194 FPDecorationId *DecorationId) {
195 StringRef PassPrefix = "(anonymous namespace)::";
196 StringRef SpvPrefix = "__spv::";
197 std::string BuiltinName = DemangledCall.str();
198
199 // Check if the extracted name contains type information between angle
200 // brackets. If so, the builtin is an instantiated template - needs to have
201 // the information after angle brackets and return type removed.
202 std::size_t Pos = BuiltinName.find(s: ">(");
203 if (Pos != std::string::npos) {
204 BuiltinName = BuiltinName.substr(pos: 0, n: BuiltinName.rfind(c: '<', pos: Pos));
205 } else {
206 Pos = BuiltinName.find(c: '(');
207 if (Pos != std::string::npos)
208 BuiltinName = BuiltinName.substr(pos: 0, n: Pos);
209 }
210 BuiltinName = BuiltinName.substr(pos: BuiltinName.find_last_of(c: ' ') + 1);
211
212 // Itanium Demangler result may have "(anonymous namespace)::" or "__spv::"
213 // prefix.
214 if (BuiltinName.find(svt: PassPrefix) == 0)
215 BuiltinName = BuiltinName.substr(pos: PassPrefix.size());
216 else if (BuiltinName.find(svt: SpvPrefix) == 0)
217 BuiltinName = BuiltinName.substr(pos: SpvPrefix.size());
218
219 // Account for possible "__spirv_ocl_" prefix in SPIR-V friendly LLVM IR
220 if (BuiltinName.rfind(s: "__spirv_ocl_", pos: 0) == 0)
221 BuiltinName = BuiltinName.substr(pos: 12);
222
223 // Check if the extracted name begins with:
224 // - "__spirv_ImageSampleExplicitLod"
225 // - "__spirv_ImageRead"
226 // - "__spirv_ImageWrite"
227 // - "__spirv_ImageQuerySizeLod"
228 // - "__spirv_UDotKHR"
229 // - "__spirv_SDotKHR"
230 // - "__spirv_SUDotKHR"
231 // - "__spirv_SDotAccSatKHR"
232 // - "__spirv_UDotAccSatKHR"
233 // - "__spirv_SUDotAccSatKHR"
234 // - "__spirv_ReadClockKHR"
235 // - "__spirv_SubgroupBlockReadINTEL"
236 // - "__spirv_SubgroupImageBlockReadINTEL"
237 // - "__spirv_SubgroupImageMediaBlockReadINTEL"
238 // - "__spirv_SubgroupImageMediaBlockWriteINTEL"
239 // - "__spirv_Convert"
240 // - "__spirv_Round"
241 // - "__spirv_UConvert"
242 // - "__spirv_SConvert"
243 // - "__spirv_FConvert"
244 // - "__spirv_SatConvert"
245 // and maybe contains return type information at the end "_R<type>".
246 // If so, extract the plain builtin name without the type information.
247 static const std::regex SpvWithR(
248 "(__spirv_(ImageSampleExplicitLod|ImageRead|ImageWrite|ImageQuerySizeLod|"
249 "UDotKHR|"
250 "SDotKHR|SUDotKHR|SDotAccSatKHR|UDotAccSatKHR|SUDotAccSatKHR|"
251 "ReadClockKHR|SubgroupBlockReadINTEL|SubgroupImageBlockReadINTEL|"
252 "SubgroupImageMediaBlockReadINTEL|SubgroupImageMediaBlockWriteINTEL|"
253 "Convert|Round|"
254 "UConvert|SConvert|FConvert|SatConvert)[^_]*)(_R[^_]*_?(\\w+)?.*)?");
255 std::smatch Match;
256 if (std::regex_match(s: BuiltinName, m&: Match, re: SpvWithR) && Match.size() > 1) {
257 std::ssub_match SubMatch;
258 if (DecorationId && Match.size() > 3) {
259 SubMatch = Match[4];
260 *DecorationId = demangledPostfixToDecorationId(S: SubMatch.str());
261 }
262 SubMatch = Match[1];
263 BuiltinName = SubMatch.str();
264 }
265
266 return BuiltinName;
267}
268} // namespace SPIRV
269
270/// Looks up the demangled builtin call in the SPIRVBuiltins.td records using
271/// the provided \p DemangledCall and specified \p Set.
272///
273/// The lookup follows the following algorithm, returning the first successful
274/// match:
275/// 1. Search with the plain demangled name (expecting a 1:1 match).
276/// 2. Search with the prefix before or suffix after the demangled name
277/// signyfying the type of the first argument.
278///
279/// \returns Wrapper around the demangled call and found builtin definition.
280static std::unique_ptr<const SPIRV::IncomingCall>
281lookupBuiltin(StringRef DemangledCall,
282 SPIRV::InstructionSet::InstructionSet Set,
283 Register ReturnRegister, SPIRVTypeInst ReturnType,
284 const SmallVectorImpl<Register> &Arguments) {
285 std::string BuiltinName = SPIRV::lookupBuiltinNameHelper(DemangledCall);
286
287 SmallVector<StringRef, 10> BuiltinArgumentTypes;
288 StringRef BuiltinArgs =
289 DemangledCall.slice(Start: DemangledCall.find(C: '(') + 1, End: DemangledCall.find(C: ')'));
290 BuiltinArgs.split(A&: BuiltinArgumentTypes, Separator: ',', MaxSplit: -1, KeepEmpty: false);
291
292 // Look up the builtin in the defined set. Start with the plain demangled
293 // name, expecting a 1:1 match in the defined builtin set.
294 const SPIRV::DemangledBuiltin *Builtin;
295 if ((Builtin = SPIRV::lookupBuiltin(Name: BuiltinName, Set)))
296 return std::make_unique<SPIRV::IncomingCall>(
297 args&: BuiltinName, args&: Builtin, args&: ReturnRegister, args&: ReturnType, args: Arguments);
298
299 // If the initial look up was unsuccessful and the demangled call takes at
300 // least 1 argument, add a prefix or suffix signifying the type of the first
301 // argument and repeat the search.
302 if (BuiltinArgumentTypes.size() >= 1) {
303 char FirstArgumentType = BuiltinArgumentTypes[0][0];
304 // Prefix and suffix to be added to the builtin's name for lookup.
305 // For example, OpenCL "abs" taking an unsigned value has a prefix "u_",
306 // and "group_reduce_max" taking an unsigned value has a suffix "u".
307 StringRef Prefix;
308 StringRef Suffix;
309
310 switch (FirstArgumentType) {
311 // Unsigned:
312 case 'u':
313 if (Set == SPIRV::InstructionSet::OpenCL_std)
314 Prefix = "u_";
315 else if (Set == SPIRV::InstructionSet::GLSL_std_450)
316 Prefix = "u";
317 Suffix = "u";
318 break;
319 // Signed:
320 case 'c':
321 case 's':
322 case 'i':
323 case 'l':
324 if (Set == SPIRV::InstructionSet::OpenCL_std)
325 Prefix = "s_";
326 else if (Set == SPIRV::InstructionSet::GLSL_std_450)
327 Prefix = "s";
328 Suffix = "s";
329 break;
330 // Floating-point:
331 case 'f':
332 case 'd':
333 case 'h':
334 if (Set == SPIRV::InstructionSet::OpenCL_std ||
335 Set == SPIRV::InstructionSet::GLSL_std_450)
336 Prefix = "f";
337 Suffix = "f";
338 break;
339 }
340
341 // If argument-type name prefix was added, look up the builtin again.
342 if (!Prefix.empty() &&
343 (Builtin = SPIRV::lookupBuiltin(Name: (Prefix + BuiltinName).str(), Set)))
344 return std::make_unique<SPIRV::IncomingCall>(
345 args&: BuiltinName, args&: Builtin, args&: ReturnRegister, args&: ReturnType, args: Arguments);
346
347 if (!Suffix.empty() &&
348 (Builtin = SPIRV::lookupBuiltin(Name: (BuiltinName + Suffix).str(), Set)))
349 return std::make_unique<SPIRV::IncomingCall>(
350 args&: BuiltinName, args&: Builtin, args&: ReturnRegister, args&: ReturnType, args: Arguments);
351 }
352
353 // No builtin with such name was found in the set.
354 return nullptr;
355}
356
357static MachineInstr *getBlockStructInstr(Register ParamReg,
358 MachineRegisterInfo *MRI) {
359 // We expect ParamReg to be defined by G_ADDRSPACE_CAST with a source from
360 // G_GLOBAL_VALUE or spv_alloca. Returns the source instruction.
361 MachineInstr *MI = MRI->getUniqueVRegDef(Reg: ParamReg);
362 assert(MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST &&
363 MI->getOperand(1).isReg());
364 Register BitcastReg = MI->getOperand(i: 1).getReg();
365 MachineInstr *BitcastMI = MRI->getUniqueVRegDef(Reg: BitcastReg);
366 assert(BitcastMI && "Definition for source reg not found.");
367 if (BitcastMI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
368 isSpvIntrinsic(MI: *BitcastMI, IntrinsicID: Intrinsic::spv_alloca))
369 return BitcastMI;
370 llvm_unreachable("getBlockStructInstr: unexpected instruction pattern");
371}
372
373// Return type of the instruction result from spv_assign_type intrinsic.
374// TODO: maybe unify with prelegalizer pass.
375static const Type *getMachineInstrType(MachineInstr *MI) {
376 MachineInstr *NextMI = MI->getNextNode();
377 if (!NextMI)
378 return nullptr;
379 if (isSpvIntrinsic(MI: *NextMI, IntrinsicID: Intrinsic::spv_assign_name))
380 if ((NextMI = NextMI->getNextNode()) == nullptr)
381 return nullptr;
382 Register ValueReg = MI->getOperand(i: 0).getReg();
383 if ((!isSpvIntrinsic(MI: *NextMI, IntrinsicID: Intrinsic::spv_assign_type) &&
384 !isSpvIntrinsic(MI: *NextMI, IntrinsicID: Intrinsic::spv_assign_ptr_type)) ||
385 NextMI->getOperand(i: 1).getReg() != ValueReg)
386 return nullptr;
387 Type *Ty = getMDOperandAsType(N: NextMI->getOperand(i: 2).getMetadata(), I: 0);
388 assert(Ty && "Type is expected");
389 return Ty;
390}
391
392static const Type *getBlockStructType(Register ParamReg,
393 MachineRegisterInfo *MRI) {
394 // In principle, this information should be passed to us from Clang via
395 // an elementtype attribute. However, said attribute requires that
396 // the function call be an intrinsic, which is not. Instead, we rely on being
397 // able to trace this to the declaration of a variable: OpenCL C specification
398 // section 6.12.5 should guarantee that we can do this.
399 MachineInstr *MI = getBlockStructInstr(ParamReg, MRI);
400 if (MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE)
401 return MI->getOperand(i: 1).getGlobal()->getValueType();
402 assert(isSpvIntrinsic(*MI, Intrinsic::spv_alloca) &&
403 "Blocks in OpenCL C must be traceable to allocation site");
404 return getMachineInstrType(MI);
405}
406
407//===----------------------------------------------------------------------===//
408// Helper functions for building misc instructions
409//===----------------------------------------------------------------------===//
410
411/// Helper function building either a resulting scalar or vector bool register
412/// depending on the expected \p ResultType.
413///
414/// \returns Tuple of the resulting register and its type.
415static std::tuple<Register, SPIRVTypeInst>
416buildBoolRegister(MachineIRBuilder &MIRBuilder, SPIRVTypeInst ResultType,
417 SPIRVGlobalRegistry *GR) {
418 LLT Type;
419 SPIRVTypeInst BoolType = GR->getOrCreateSPIRVBoolType(MIRBuilder, EmitIR: true);
420
421 if (isVectorType(SPVTy: ResultType)) {
422 unsigned VectorElements = GR->getScalarOrVectorComponentCount(Type: ResultType);
423 BoolType = GR->getOrCreateSPIRVVectorType(BaseType: BoolType, NumElements: VectorElements,
424 MIRBuilder, EmitIR: true);
425 const FixedVectorType *LLVMVectorType =
426 cast<FixedVectorType>(Val: GR->getTypeForSPIRVType(Ty: BoolType));
427 Type = LLT::vector(EC: LLVMVectorType->getElementCount(), ScalarSizeInBits: 1);
428 } else {
429 Type = LLT::scalar(SizeInBits: 1);
430 }
431
432 Register ResultRegister =
433 MIRBuilder.getMRI()->createGenericVirtualRegister(Ty: Type);
434 MIRBuilder.getMRI()->setRegClass(Reg: ResultRegister, RC: GR->getRegClass(SpvType: ResultType));
435 GR->assignSPIRVTypeToVReg(Type: BoolType, VReg: ResultRegister, MF: MIRBuilder.getMF());
436 return std::make_tuple(args&: ResultRegister, args&: BoolType);
437}
438
439/// Helper function for building either a vector or scalar select instruction
440/// depending on the expected \p ResultType.
441static bool buildSelectInst(MachineIRBuilder &MIRBuilder,
442 Register ReturnRegister, Register SourceRegister,
443 SPIRVTypeInst ReturnType, SPIRVGlobalRegistry *GR) {
444 Register TrueConst, FalseConst;
445
446 if (isVectorType(SPVTy: ReturnType)) {
447 unsigned Bits = GR->getScalarOrVectorBitWidth(Type: ReturnType);
448 uint64_t AllOnes = APInt::getAllOnes(numBits: Bits).getZExtValue();
449 TrueConst =
450 GR->getOrCreateConsIntVector(Val: AllOnes, MIRBuilder, SpvType: ReturnType, EmitIR: true);
451 FalseConst = GR->getOrCreateConsIntVector(Val: 0, MIRBuilder, SpvType: ReturnType, EmitIR: true);
452 } else {
453 TrueConst = GR->buildConstantInt(Val: 1, MIRBuilder, SpvType: ReturnType, EmitIR: true);
454 FalseConst = GR->buildConstantInt(Val: 0, MIRBuilder, SpvType: ReturnType, EmitIR: true);
455 }
456
457 return MIRBuilder.buildSelect(Res: ReturnRegister, Tst: SourceRegister, Op0: TrueConst,
458 Op1: FalseConst);
459}
460
461/// Helper function for building a load instruction loading into the
462/// \p DestinationReg.
463static Register buildLoadInst(SPIRVTypeInst BaseType, Register PtrRegister,
464 MachineIRBuilder &MIRBuilder,
465 SPIRVGlobalRegistry *GR,
466 Register DestinationReg = Register(0)) {
467 if (!DestinationReg.isValid())
468 DestinationReg = createVirtualRegister(SpvType: BaseType, GR, MIRBuilder);
469 // TODO: consider using correct address space and alignment (p0 is canonical
470 // type for selection though).
471 MachinePointerInfo PtrInfo = MachinePointerInfo();
472 MIRBuilder.buildLoad(Res: DestinationReg, Addr: PtrRegister, PtrInfo, Alignment: Align());
473 return DestinationReg;
474}
475
476/// Helper function for building a load instruction for loading a builtin global
477/// variable of \p BuiltinValue value.
478static Register buildBuiltinVariableLoad(
479 MachineIRBuilder &MIRBuilder, SPIRVTypeInst VariableType,
480 SPIRVGlobalRegistry *GR, SPIRV::BuiltIn::BuiltIn BuiltinValue, LLT LLType,
481 Register Reg = Register(0), bool isConst = true,
482 const std::optional<SPIRV::LinkageType::LinkageType> &LinkageTy = {
483 SPIRV::LinkageType::Import}) {
484 Register NewRegister =
485 MIRBuilder.getMRI()->createVirtualRegister(RegClass: &SPIRV::pIDRegClass);
486 MIRBuilder.getMRI()->setType(
487 VReg: NewRegister,
488 Ty: LLT::pointer(AddressSpace: storageClassToAddressSpace(SC: SPIRV::StorageClass::Function),
489 SizeInBits: GR->getPointerSize()));
490 SPIRVTypeInst PtrType = GR->getOrCreateSPIRVPointerType(
491 BaseType: VariableType, MIRBuilder, SC: SPIRV::StorageClass::Input);
492 GR->assignSPIRVTypeToVReg(Type: PtrType, VReg: NewRegister, MF: MIRBuilder.getMF());
493
494 // Set up the global OpVariable with the necessary builtin decorations.
495 Register Variable = GR->buildGlobalVariable(
496 Reg: NewRegister, BaseType: PtrType, Name: getLinkStringForBuiltIn(BuiltInValue: BuiltinValue), GV: nullptr,
497 Storage: SPIRV::StorageClass::Input, Init: nullptr, /* isConst= */ IsConst: isConst, LinkageType: LinkageTy,
498 MIRBuilder, IsInstSelector: false);
499
500 // Load the value from the global variable.
501 Register LoadedRegister =
502 buildLoadInst(BaseType: VariableType, PtrRegister: Variable, MIRBuilder, GR, DestinationReg: Reg);
503 MIRBuilder.getMRI()->setType(VReg: LoadedRegister, Ty: LLType);
504 return LoadedRegister;
505}
506
507/// Helper external function for assigning a SPIRV type to a register, ensuring
508/// the register class and type are set in MRI. Defined in
509/// SPIRVPreLegalizer.cpp.
510extern void updateRegType(Register Reg, Type *Ty, SPIRVTypeInst SpirvTy,
511 SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,
512 MachineRegisterInfo &MRI);
513
514// TODO: Move to TableGen.
515static SPIRV::MemorySemantics::MemorySemantics
516getSPIRVMemSemantics(std::memory_order MemOrder) {
517 switch (MemOrder) {
518 case std::memory_order_relaxed:
519 return SPIRV::MemorySemantics::None;
520 case std::memory_order_acquire:
521 return SPIRV::MemorySemantics::Acquire;
522 case std::memory_order_release:
523 return SPIRV::MemorySemantics::Release;
524 case std::memory_order_acq_rel:
525 return SPIRV::MemorySemantics::AcquireRelease;
526 case std::memory_order_seq_cst:
527 return SPIRV::MemorySemantics::SequentiallyConsistent;
528 default:
529 report_fatal_error(reason: "Unknown CL memory order");
530 }
531}
532
533static SPIRV::Scope::Scope getSPIRVScope(SPIRV::CLMemoryScope ClScope) {
534 switch (ClScope) {
535 case SPIRV::CLMemoryScope::memory_scope_work_item:
536 return SPIRV::Scope::Invocation;
537 case SPIRV::CLMemoryScope::memory_scope_work_group:
538 return SPIRV::Scope::Workgroup;
539 case SPIRV::CLMemoryScope::memory_scope_device:
540 return SPIRV::Scope::Device;
541 case SPIRV::CLMemoryScope::memory_scope_all_svm_devices:
542 return SPIRV::Scope::CrossDevice;
543 case SPIRV::CLMemoryScope::memory_scope_sub_group:
544 return SPIRV::Scope::Subgroup;
545 }
546 report_fatal_error(reason: "Unknown CL memory scope");
547}
548
549static Register buildConstantIntReg32(uint64_t Val,
550 MachineIRBuilder &MIRBuilder,
551 SPIRVGlobalRegistry *GR) {
552 return GR->buildConstantInt(
553 Val, MIRBuilder, SpvType: GR->getOrCreateSPIRVIntegerType(BitWidth: 32, MIRBuilder), EmitIR: true);
554}
555
556static Register buildScopeReg(Register CLScopeRegister,
557 SPIRV::Scope::Scope Scope,
558 MachineIRBuilder &MIRBuilder,
559 SPIRVGlobalRegistry *GR,
560 MachineRegisterInfo *MRI) {
561 if (CLScopeRegister.isValid()) {
562 auto CLScope =
563 static_cast<SPIRV::CLMemoryScope>(getIConstVal(ConstReg: CLScopeRegister, MRI));
564 Scope = getSPIRVScope(ClScope: CLScope);
565
566 if (CLScope == static_cast<unsigned>(Scope)) {
567 MRI->setRegClass(Reg: CLScopeRegister, RC: &SPIRV::iIDRegClass);
568 return CLScopeRegister;
569 }
570 }
571 return buildConstantIntReg32(Val: Scope, MIRBuilder, GR);
572}
573
574static void setRegClassIfNull(Register Reg, MachineRegisterInfo *MRI,
575 SPIRVGlobalRegistry *GR) {
576 if (MRI->getRegClassOrNull(Reg))
577 return;
578 SPIRVTypeInst SpvType = GR->getSPIRVTypeForVReg(VReg: Reg);
579 MRI->setRegClass(Reg,
580 RC: SpvType ? GR->getRegClass(SpvType) : &SPIRV::iIDRegClass);
581}
582
583/// Translates an OpenCL memory_order argument into the memory ordering part of
584/// the SPIR-V memory semantics.
585static SPIRV::MemorySemantics::MemorySemantics
586getMemOrdering(Register OrderRegister, MachineRegisterInfo *MRI) {
587 return getSPIRVMemSemantics(
588 MemOrder: static_cast<std::memory_order>(getIConstVal(ConstReg: OrderRegister, MRI)));
589}
590
591/// Combines the memory ordering with the storage-class part of the memory
592/// semantics into a constant register.
593static Register
594buildMemSemanticsReg(SPIRV::MemorySemantics::MemorySemantics Ordering,
595 unsigned StorageClassSem, MachineIRBuilder &MIRBuilder,
596 SPIRVGlobalRegistry *GR) {
597 const auto *ST =
598 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
599 return buildConstantIntReg32(
600 Val: getMemSemanticsWithStorageClass(TT: ST->getTargetTriple(), OrderSem: Ordering,
601 StorageClassSem),
602 MIRBuilder, GR);
603}
604
605static bool buildOpFromWrapper(MachineIRBuilder &MIRBuilder, unsigned Opcode,
606 const SPIRV::IncomingCall *Call,
607 Register TypeReg,
608 ArrayRef<uint32_t> ImmArgs = {}) {
609 auto MIB = MIRBuilder.buildInstr(Opcode);
610 if (TypeReg.isValid())
611 MIB.addDef(RegNo: Call->ReturnRegister).addUse(RegNo: TypeReg);
612 unsigned Sz = Call->Arguments.size() - ImmArgs.size();
613 for (unsigned i = 0; i < Sz; ++i)
614 MIB.addUse(RegNo: Call->Arguments[i]);
615 for (uint32_t ImmArg : ImmArgs)
616 MIB.addImm(Val: ImmArg);
617 return true;
618}
619
620/// Helper function for translating atomic init to OpStore.
621static bool buildAtomicInitInst(const SPIRV::IncomingCall *Call,
622 MachineIRBuilder &MIRBuilder) {
623 if (Call->isSpirvOp())
624 return buildOpFromWrapper(MIRBuilder, Opcode: SPIRV::OpStore, Call, TypeReg: Register(0));
625
626 assert(Call->Arguments.size() == 2 &&
627 "Need 2 arguments for atomic init translation");
628 MIRBuilder.buildInstr(Opcode: SPIRV::OpStore)
629 .addUse(RegNo: Call->Arguments[0])
630 .addUse(RegNo: Call->Arguments[1]);
631 return true;
632}
633
634/// Helper function for building an atomic load instruction.
635static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call,
636 MachineIRBuilder &MIRBuilder,
637 SPIRVGlobalRegistry *GR) {
638 Register TypeReg = GR->getSPIRVTypeID(SpirvType: Call->ReturnType);
639 if (Call->isSpirvOp())
640 return buildOpFromWrapper(MIRBuilder, Opcode: SPIRV::OpAtomicLoad, Call, TypeReg);
641
642 // atomic_load_explicit(ptr, memory_order[, memory_scope]).
643 Register PtrRegister = Call->Arguments[0];
644 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
645
646 const SPIRV::MemorySemantics::MemorySemantics Ordering =
647 Call->Arguments.size() >= 2
648 ? getMemOrdering(OrderRegister: Call->Arguments[1], MRI)
649 : SPIRV::MemorySemantics::SequentiallyConsistent;
650 const unsigned StorageClassSem =
651 getMemSemanticsForStorageClass(SC: GR->getPointerStorageClass(VReg: PtrRegister));
652 Register MemSemanticsReg =
653 buildMemSemanticsReg(Ordering, StorageClassSem, MIRBuilder, GR);
654
655 Register ScopeRegister = buildScopeReg(
656 CLScopeRegister: Call->Arguments.size() >= 3 ? Call->Arguments[2] : Register(),
657 Scope: SPIRV::Scope::Device, MIRBuilder, GR, MRI);
658
659 MIRBuilder.buildInstr(Opcode: SPIRV::OpAtomicLoad)
660 .addDef(RegNo: Call->ReturnRegister)
661 .addUse(RegNo: TypeReg)
662 .addUse(RegNo: PtrRegister)
663 .addUse(RegNo: ScopeRegister)
664 .addUse(RegNo: MemSemanticsReg);
665 return true;
666}
667
668/// Helper function for building an atomic store instruction.
669static bool buildAtomicStoreInst(const SPIRV::IncomingCall *Call,
670 MachineIRBuilder &MIRBuilder,
671 SPIRVGlobalRegistry *GR) {
672 if (Call->isSpirvOp())
673 return buildOpFromWrapper(MIRBuilder, Opcode: SPIRV::OpAtomicStore, Call,
674 TypeReg: Register(0));
675
676 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
677 Register PtrRegister = Call->Arguments[0];
678 // atomic_store_explicit(ptr, value, memory_order[, memory_scope]).
679 const SPIRV::MemorySemantics::MemorySemantics Ordering =
680 Call->Arguments.size() >= 3
681 ? getMemOrdering(OrderRegister: Call->Arguments[2], MRI)
682 : SPIRV::MemorySemantics::SequentiallyConsistent;
683 const unsigned StorageClassSem =
684 getMemSemanticsForStorageClass(SC: GR->getPointerStorageClass(VReg: PtrRegister));
685 Register MemSemanticsReg =
686 buildMemSemanticsReg(Ordering, StorageClassSem, MIRBuilder, GR);
687 Register ScopeRegister = buildScopeReg(
688 CLScopeRegister: Call->Arguments.size() >= 4 ? Call->Arguments[3] : Register(),
689 Scope: SPIRV::Scope::Device, MIRBuilder, GR, MRI);
690 MIRBuilder.buildInstr(Opcode: SPIRV::OpAtomicStore)
691 .addUse(RegNo: PtrRegister)
692 .addUse(RegNo: ScopeRegister)
693 .addUse(RegNo: MemSemanticsReg)
694 .addUse(RegNo: Call->Arguments[1]);
695 return true;
696}
697
698/// Helper function for building an atomic compare-exchange instruction.
699static bool buildAtomicCompareExchangeInst(
700 const SPIRV::IncomingCall *Call, const SPIRV::DemangledBuiltin *Builtin,
701 unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) {
702 if (Call->isSpirvOp())
703 return buildOpFromWrapper(MIRBuilder, Opcode, Call,
704 TypeReg: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
705
706 bool IsCmpxchg = Call->Builtin->name().contains(Other: "cmpxchg");
707 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
708
709 Register ObjectPtr = Call->Arguments[0]; // Pointer (volatile A *object.)
710 Register ExpectedArg = Call->Arguments[1]; // Comparator (C* expected).
711 Register Desired = Call->Arguments[2]; // Value (C Desired).
712 SPIRVTypeInst SpvDesiredTy = GR->getSPIRVTypeForVReg(VReg: Desired);
713 LLT DesiredLLT = MRI->getType(Reg: Desired);
714
715 assert(GR->getSPIRVTypeForVReg(ObjectPtr).isPointer());
716 [[maybe_unused]] SPIRVTypeInst ExpectedTy =
717 GR->getSPIRVTypeForVReg(VReg: ExpectedArg);
718 assert(IsCmpxchg ? ExpectedTy->getOpcode() == SPIRV::OpTypeInt
719 : ExpectedTy.isPointer());
720 assert(GR->isScalarOfType(Desired, SPIRV::OpTypeInt));
721
722 SPIRVTypeInst SpvObjectPtrTy = GR->getSPIRVTypeForVReg(VReg: ObjectPtr);
723 assert((SpvObjectPtrTy->getOpcode() == SPIRV::OpTypeUntypedPointerKHR ||
724 SpvObjectPtrTy->getOperand(2).isReg()) &&
725 "SPIRV type is expected");
726 auto StorageClass = static_cast<SPIRV::StorageClass::StorageClass>(
727 SpvObjectPtrTy->getOperand(i: 1).getImm());
728 auto MemSemStorage = getMemSemanticsForStorageClass(SC: StorageClass);
729
730 Register MemSemEqualReg;
731 Register MemSemUnequalReg;
732 uint64_t MemSemEqual =
733 IsCmpxchg
734 ? SPIRV::MemorySemantics::None
735 : SPIRV::MemorySemantics::SequentiallyConsistent | MemSemStorage;
736 uint64_t MemSemUnequal =
737 IsCmpxchg
738 ? SPIRV::MemorySemantics::None
739 : SPIRV::MemorySemantics::SequentiallyConsistent | MemSemStorage;
740 if (Call->Arguments.size() >= 4) {
741 assert(Call->Arguments.size() >= 5 &&
742 "Need 5+ args for explicit atomic cmpxchg");
743 auto MemOrdEq =
744 static_cast<std::memory_order>(getIConstVal(ConstReg: Call->Arguments[3], MRI));
745 auto MemOrdNeq =
746 static_cast<std::memory_order>(getIConstVal(ConstReg: Call->Arguments[4], MRI));
747 MemSemEqual = getSPIRVMemSemantics(MemOrder: MemOrdEq) | MemSemStorage;
748 MemSemUnequal = getSPIRVMemSemantics(MemOrder: MemOrdNeq) | MemSemStorage;
749 if (static_cast<unsigned>(MemOrdEq) == MemSemEqual)
750 MemSemEqualReg = Call->Arguments[3];
751 if (static_cast<unsigned>(MemOrdNeq) == MemSemUnequal)
752 MemSemUnequalReg = Call->Arguments[4];
753 }
754 if (!MemSemEqualReg.isValid())
755 MemSemEqualReg = buildConstantIntReg32(Val: MemSemEqual, MIRBuilder, GR);
756 if (!MemSemUnequalReg.isValid())
757 MemSemUnequalReg = buildConstantIntReg32(Val: MemSemUnequal, MIRBuilder, GR);
758
759 Register ScopeReg;
760 auto Scope = IsCmpxchg ? SPIRV::Scope::Workgroup : SPIRV::Scope::Device;
761 if (Call->Arguments.size() >= 6) {
762 assert(Call->Arguments.size() == 6 &&
763 "Extra args for explicit atomic cmpxchg");
764 auto ClScope = static_cast<SPIRV::CLMemoryScope>(
765 getIConstVal(ConstReg: Call->Arguments[5], MRI));
766 Scope = getSPIRVScope(ClScope);
767 if (ClScope == static_cast<unsigned>(Scope))
768 ScopeReg = Call->Arguments[5];
769 }
770 if (!ScopeReg.isValid())
771 ScopeReg = buildConstantIntReg32(Val: Scope, MIRBuilder, GR);
772
773 Register Expected =
774 IsCmpxchg ? ExpectedArg
775 : buildLoadInst(BaseType: SpvDesiredTy, PtrRegister: ExpectedArg, MIRBuilder, GR);
776 MRI->setType(VReg: Expected, Ty: DesiredLLT);
777 Register Tmp = !IsCmpxchg ? MRI->createGenericVirtualRegister(Ty: DesiredLLT)
778 : Call->ReturnRegister;
779 if (!MRI->getRegClassOrNull(Reg: Tmp))
780 MRI->setRegClass(Reg: Tmp, RC: GR->getRegClass(SpvType: SpvDesiredTy));
781 GR->assignSPIRVTypeToVReg(Type: SpvDesiredTy, VReg: Tmp, MF: MIRBuilder.getMF());
782
783 MIRBuilder.buildInstr(Opcode)
784 .addDef(RegNo: Tmp)
785 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: SpvDesiredTy))
786 .addUse(RegNo: ObjectPtr)
787 .addUse(RegNo: ScopeReg)
788 .addUse(RegNo: MemSemEqualReg)
789 .addUse(RegNo: MemSemUnequalReg)
790 .addUse(RegNo: Desired)
791 .addUse(RegNo: Expected);
792 if (!IsCmpxchg) {
793 MIRBuilder.buildInstr(Opcode: SPIRV::OpStore).addUse(RegNo: ExpectedArg).addUse(RegNo: Tmp);
794 MIRBuilder.buildICmp(Pred: CmpInst::ICMP_EQ, Res: Call->ReturnRegister, Op0: Tmp, Op1: Expected);
795 }
796 return true;
797}
798
799/// Helper function for building atomic instructions.
800static bool buildAtomicRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode,
801 MachineIRBuilder &MIRBuilder,
802 SPIRVGlobalRegistry *GR) {
803 if (Call->isSpirvOp())
804 return buildOpFromWrapper(MIRBuilder, Opcode, Call,
805 TypeReg: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
806
807 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
808 StringRef Name = Call->Builtin->name();
809 // The registry prefixes atomic_fetch_min/max with "s_", "u_" or "f".
810 bool IsOCL20 =
811 Name.contains(Other: "atomic_fetch_") || Name.starts_with(Prefix: "atomic_exchange");
812 SPIRV::Scope::Scope DefaultScope =
813 IsOCL20 ? SPIRV::Scope::Device : SPIRV::Scope::Workgroup;
814 Register ScopeRegister =
815 Call->Arguments.size() >= 4 ? Call->Arguments[3] : Register();
816
817 assert(Call->Arguments.size() <= 4 &&
818 "Too many args for explicit atomic RMW");
819 ScopeRegister =
820 buildScopeReg(CLScopeRegister: ScopeRegister, Scope: DefaultScope, MIRBuilder, GR, MRI);
821
822 Register PtrRegister = Call->Arguments[0];
823 SPIRV::MemorySemantics::MemorySemantics Ordering =
824 IsOCL20 ? SPIRV::MemorySemantics::SequentiallyConsistent
825 : SPIRV::MemorySemantics::None;
826 unsigned StorageClassSem = SPIRV::MemorySemantics::None;
827 if (Call->Arguments.size() >= 3)
828 Ordering = getMemOrdering(OrderRegister: Call->Arguments[2], MRI);
829 if (IsOCL20 || Call->Arguments.size() >= 3)
830 StorageClassSem =
831 getMemSemanticsForStorageClass(SC: GR->getPointerStorageClass(VReg: PtrRegister));
832 Register MemSemanticsReg =
833 buildMemSemanticsReg(Ordering, StorageClassSem, MIRBuilder, GR);
834 Register ValueReg = Call->Arguments[1];
835 Register ValueTypeReg = GR->getSPIRVTypeID(SpirvType: Call->ReturnType);
836 // support cl_ext_float_atomics
837 if (Call->ReturnType->getOpcode() == SPIRV::OpTypeFloat) {
838 if (Opcode == SPIRV::OpAtomicIAdd) {
839 Opcode = SPIRV::OpAtomicFAddEXT;
840 } else if (Opcode == SPIRV::OpAtomicISub) {
841 // Translate OpAtomicISub applied to a floating type argument to
842 // OpAtomicFAddEXT with the negative value operand
843 Opcode = SPIRV::OpAtomicFAddEXT;
844 Register NegValueReg =
845 MRI->createGenericVirtualRegister(Ty: MRI->getType(Reg: ValueReg));
846 MRI->setRegClass(Reg: NegValueReg, RC: GR->getRegClass(SpvType: Call->ReturnType));
847 GR->assignSPIRVTypeToVReg(Type: Call->ReturnType, VReg: NegValueReg,
848 MF: MIRBuilder.getMF());
849 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_FNEG)
850 .addDef(RegNo: NegValueReg)
851 .addUse(RegNo: ValueReg);
852 updateRegType(Reg: NegValueReg, Ty: nullptr, SpirvTy: Call->ReturnType, GR, MIB&: MIRBuilder,
853 MRI&: MIRBuilder.getMF().getRegInfo());
854 ValueReg = NegValueReg;
855 }
856 }
857 MIRBuilder.buildInstr(Opcode)
858 .addDef(RegNo: Call->ReturnRegister)
859 .addUse(RegNo: ValueTypeReg)
860 .addUse(RegNo: PtrRegister)
861 .addUse(RegNo: ScopeRegister)
862 .addUse(RegNo: MemSemanticsReg)
863 .addUse(RegNo: ValueReg);
864 return true;
865}
866
867/// Helper function for building an atomic floating-type instruction.
868static bool buildAtomicFloatingRMWInst(const SPIRV::IncomingCall *Call,
869 unsigned Opcode,
870 MachineIRBuilder &MIRBuilder,
871 SPIRVGlobalRegistry *GR) {
872 assert(Call->Arguments.size() == 4 &&
873 "Wrong number of atomic floating-type builtin");
874 Register PtrReg = Call->Arguments[0];
875 Register ScopeReg = Call->Arguments[1];
876 Register MemSemanticsReg = Call->Arguments[2];
877 Register ValueReg = Call->Arguments[3];
878 MIRBuilder.buildInstr(Opcode)
879 .addDef(RegNo: Call->ReturnRegister)
880 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
881 .addUse(RegNo: PtrReg)
882 .addUse(RegNo: ScopeReg)
883 .addUse(RegNo: MemSemanticsReg)
884 .addUse(RegNo: ValueReg);
885 return true;
886}
887
888/// Helper function for building atomic flag instructions (e.g.
889/// OpAtomicFlagTestAndSet).
890static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call,
891 unsigned Opcode, MachineIRBuilder &MIRBuilder,
892 SPIRVGlobalRegistry *GR) {
893 bool IsSet = Opcode == SPIRV::OpAtomicFlagTestAndSet;
894 Register TypeReg = GR->getSPIRVTypeID(SpirvType: Call->ReturnType);
895 if (Call->isSpirvOp())
896 return buildOpFromWrapper(MIRBuilder, Opcode, Call,
897 TypeReg: IsSet ? TypeReg : Register(0));
898
899 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
900 Register PtrRegister = Call->Arguments[0];
901 SPIRV::MemorySemantics::MemorySemantics Ordering =
902 SPIRV::MemorySemantics::SequentiallyConsistent;
903 unsigned StorageClassSem = SPIRV::MemorySemantics::None;
904 if (Call->Arguments.size() >= 2) {
905 Ordering = getMemOrdering(OrderRegister: Call->Arguments[1], MRI);
906 StorageClassSem =
907 getMemSemanticsForStorageClass(SC: GR->getPointerStorageClass(VReg: PtrRegister));
908 }
909
910 assert((Opcode != SPIRV::OpAtomicFlagClear ||
911 (Ordering != SPIRV::MemorySemantics::Acquire &&
912 Ordering != SPIRV::MemorySemantics::AcquireRelease)) &&
913 "Invalid memory order argument!");
914
915 Register MemSemanticsReg =
916 buildMemSemanticsReg(Ordering, StorageClassSem, MIRBuilder, GR);
917
918 Register ScopeRegister = buildScopeReg(
919 CLScopeRegister: Call->Arguments.size() >= 3 ? Call->Arguments[2] : Register(),
920 Scope: SPIRV::Scope::Device, MIRBuilder, GR, MRI);
921
922 auto MIB = MIRBuilder.buildInstr(Opcode);
923 if (IsSet)
924 MIB.addDef(RegNo: Call->ReturnRegister).addUse(RegNo: TypeReg);
925
926 MIB.addUse(RegNo: PtrRegister).addUse(RegNo: ScopeRegister).addUse(RegNo: MemSemanticsReg);
927 return true;
928}
929
930static void reportUnsupported(MachineIRBuilder &MIRBuilder, const Twine &Msg) {
931 const Function &F = MIRBuilder.getMF().getFunction();
932 F.getContext().diagnose(
933 DI: DiagnosticInfoUnsupported(F, Msg, MIRBuilder.getDebugLoc()));
934}
935
936/// Helper function for building barriers, i.e., memory/control ordering
937/// operations.
938static bool buildBarrierInst(const SPIRV::IncomingCall *Call, unsigned Opcode,
939 MachineIRBuilder &MIRBuilder,
940 SPIRVGlobalRegistry *GR) {
941 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
942 const auto *ST =
943 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
944 if ((Opcode == SPIRV::OpControlBarrierArriveINTEL ||
945 Opcode == SPIRV::OpControlBarrierWaitINTEL) &&
946 !ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_split_barrier)) {
947 std::string DiagMsg = std::string(Builtin->name()) +
948 ": the builtin requires the following SPIR-V "
949 "extension: SPV_INTEL_split_barrier";
950 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
951 }
952
953 if (Call->isSpirvOp())
954 return buildOpFromWrapper(MIRBuilder, Opcode, Call, TypeReg: Register(0));
955
956 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
957 bool IsSubgroupBarrier = Builtin->name() == "sub_group_barrier";
958 if (IsSubgroupBarrier) {
959 // TODO: Support runtime flags and scopes for OpenCL barriers.
960 for (Register Arg : Call->Arguments) {
961 const MachineInstr *MI = getDefInstrMaybeConstant(ConstReg&: Arg, MRI);
962 if (!MI || MI->getOpcode() != TargetOpcode::G_CONSTANT) {
963 reportUnsupported(
964 MIRBuilder,
965 Msg: "sub_group_barrier with non-constant arguments is not supported");
966 return false;
967 }
968 }
969 }
970 unsigned MemFlags = getIConstVal(ConstReg: Call->Arguments[0], MRI);
971 unsigned MemSemantics = SPIRV::MemorySemantics::None;
972
973 if (MemFlags & SPIRV::CLK_LOCAL_MEM_FENCE)
974 MemSemantics |= SPIRV::MemorySemantics::WorkgroupMemory;
975
976 if (MemFlags & SPIRV::CLK_GLOBAL_MEM_FENCE)
977 MemSemantics |= SPIRV::MemorySemantics::CrossWorkgroupMemory;
978
979 if (MemFlags & SPIRV::CLK_IMAGE_MEM_FENCE)
980 MemSemantics |= SPIRV::MemorySemantics::ImageMemory;
981
982 if (Opcode == SPIRV::OpMemoryBarrier)
983 MemSemantics = getSPIRVMemSemantics(MemOrder: static_cast<std::memory_order>(
984 getIConstVal(ConstReg: Call->Arguments[1], MRI))) |
985 MemSemantics;
986 else if (Opcode == SPIRV::OpControlBarrierArriveINTEL)
987 MemSemantics |= SPIRV::MemorySemantics::Release;
988 else if (Opcode == SPIRV::OpControlBarrierWaitINTEL)
989 MemSemantics |= SPIRV::MemorySemantics::Acquire;
990 else
991 MemSemantics |= SPIRV::MemorySemantics::SequentiallyConsistent;
992
993 Register MemSemanticsReg =
994 MemFlags == MemSemantics
995 ? Call->Arguments[0]
996 : buildConstantIntReg32(Val: MemSemantics, MIRBuilder, GR);
997 Register ScopeReg;
998 SPIRV::Scope::Scope Scope =
999 IsSubgroupBarrier ? SPIRV::Scope::Subgroup : SPIRV::Scope::Workgroup;
1000 SPIRV::Scope::Scope MemScope = Scope;
1001 if (Call->Arguments.size() >= 2) {
1002 assert(
1003 ((Opcode != SPIRV::OpMemoryBarrier && Call->Arguments.size() == 2) ||
1004 (Opcode == SPIRV::OpMemoryBarrier && Call->Arguments.size() == 3)) &&
1005 "Extra args for explicitly scoped barrier");
1006 Register ScopeArg = (Opcode == SPIRV::OpMemoryBarrier) ? Call->Arguments[2]
1007 : Call->Arguments[1];
1008 SPIRV::CLMemoryScope CLScope =
1009 static_cast<SPIRV::CLMemoryScope>(getIConstVal(ConstReg: ScopeArg, MRI));
1010 MemScope = getSPIRVScope(ClScope: CLScope);
1011 if (Opcode == SPIRV::OpMemoryBarrier)
1012 Scope = MemScope;
1013 if (CLScope == static_cast<unsigned>(Scope))
1014 ScopeReg = Call->Arguments[1];
1015 }
1016
1017 if (!ScopeReg.isValid())
1018 ScopeReg = buildConstantIntReg32(Val: Scope, MIRBuilder, GR);
1019
1020 auto MIB = MIRBuilder.buildInstr(Opcode).addUse(RegNo: ScopeReg);
1021 if (Opcode != SPIRV::OpMemoryBarrier)
1022 MIB.addUse(RegNo: buildConstantIntReg32(Val: MemScope, MIRBuilder, GR));
1023 MIB.addUse(RegNo: MemSemanticsReg);
1024 return true;
1025}
1026
1027/// Helper function for building extended bit operations.
1028static bool buildExtendedBitOpsInst(const SPIRV::IncomingCall *Call,
1029 unsigned Opcode,
1030 MachineIRBuilder &MIRBuilder,
1031 SPIRVGlobalRegistry *GR) {
1032 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
1033 const auto *ST =
1034 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
1035 if ((Opcode == SPIRV::OpBitFieldInsert ||
1036 Opcode == SPIRV::OpBitFieldSExtract ||
1037 Opcode == SPIRV::OpBitFieldUExtract || Opcode == SPIRV::OpBitReverse) &&
1038 !ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions)) {
1039 std::string DiagMsg = std::string(Builtin->name()) +
1040 ": the builtin requires the following SPIR-V "
1041 "extension: SPV_KHR_bit_instructions";
1042 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
1043 }
1044
1045 // Generate SPIRV instruction accordingly.
1046 if (Call->isSpirvOp())
1047 return buildOpFromWrapper(MIRBuilder, Opcode, Call,
1048 TypeReg: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
1049
1050 auto MIB = MIRBuilder.buildInstr(Opcode)
1051 .addDef(RegNo: Call->ReturnRegister)
1052 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
1053 for (unsigned i = 0; i < Call->Arguments.size(); ++i)
1054 MIB.addUse(RegNo: Call->Arguments[i]);
1055
1056 return true;
1057}
1058
1059/// Helper function for building Intel's bindless image instructions.
1060static bool buildBindlessImageINTELInst(const SPIRV::IncomingCall *Call,
1061 unsigned Opcode,
1062 MachineIRBuilder &MIRBuilder,
1063 SPIRVGlobalRegistry *GR) {
1064 // Generate SPIRV instruction accordingly.
1065 if (Call->isSpirvOp())
1066 return buildOpFromWrapper(MIRBuilder, Opcode, Call,
1067 TypeReg: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
1068
1069 MIRBuilder.buildInstr(Opcode)
1070 .addDef(RegNo: Call->ReturnRegister)
1071 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
1072 .addUse(RegNo: Call->Arguments[0]);
1073
1074 return true;
1075}
1076
1077/// Helper function for building Intel's OpBitwiseFunctionINTEL instruction.
1078static bool buildTernaryBitwiseFunctionINTELInst(
1079 const SPIRV::IncomingCall *Call, unsigned Opcode,
1080 MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) {
1081 // Generate SPIRV instruction accordingly.
1082 if (Call->isSpirvOp())
1083 return buildOpFromWrapper(MIRBuilder, Opcode, Call,
1084 TypeReg: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
1085
1086 auto MIB = MIRBuilder.buildInstr(Opcode)
1087 .addDef(RegNo: Call->ReturnRegister)
1088 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
1089 for (unsigned i = 0; i < Call->Arguments.size(); ++i)
1090 MIB.addUse(RegNo: Call->Arguments[i]);
1091
1092 return true;
1093}
1094
1095static bool buildImageChannelDataTypeInst(const SPIRV::IncomingCall *Call,
1096 unsigned Opcode,
1097 MachineIRBuilder &MIRBuilder,
1098 SPIRVGlobalRegistry *GR) {
1099 if (Call->isSpirvOp())
1100 return buildOpFromWrapper(MIRBuilder, Opcode, Call,
1101 TypeReg: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
1102
1103 auto MIB = MIRBuilder.buildInstr(Opcode)
1104 .addDef(RegNo: Call->ReturnRegister)
1105 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
1106 for (unsigned i = 0; i < Call->Arguments.size(); ++i)
1107 MIB.addUse(RegNo: Call->Arguments[i]);
1108
1109 return true;
1110}
1111
1112/// Helper function for building Intel's 2d block io instructions.
1113static bool build2DBlockIOINTELInst(const SPIRV::IncomingCall *Call,
1114 unsigned Opcode,
1115 MachineIRBuilder &MIRBuilder,
1116 SPIRVGlobalRegistry *GR) {
1117 // Generate SPIRV instruction accordingly.
1118 if (Call->isSpirvOp())
1119 return buildOpFromWrapper(MIRBuilder, Opcode, Call, TypeReg: Register(0));
1120
1121 auto MIB = MIRBuilder.buildInstr(Opcode)
1122 .addDef(RegNo: Call->ReturnRegister)
1123 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
1124 for (unsigned i = 0; i < Call->Arguments.size(); ++i)
1125 MIB.addUse(RegNo: Call->Arguments[i]);
1126
1127 return true;
1128}
1129
1130static bool buildPipeInst(const SPIRV::IncomingCall *Call, unsigned Opcode,
1131 unsigned Scope, MachineIRBuilder &MIRBuilder,
1132 SPIRVGlobalRegistry *GR) {
1133 switch (Opcode) {
1134 case SPIRV::OpCommitReadPipe:
1135 case SPIRV::OpCommitWritePipe:
1136 return buildOpFromWrapper(MIRBuilder, Opcode, Call, TypeReg: Register(0));
1137 case SPIRV::OpGroupCommitReadPipe:
1138 case SPIRV::OpGroupCommitWritePipe:
1139 case SPIRV::OpGroupReserveReadPipePackets:
1140 case SPIRV::OpGroupReserveWritePipePackets: {
1141 Register ScopeConstReg =
1142 MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: Scope).getReg(Idx: 0);
1143 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
1144 MRI->setRegClass(Reg: ScopeConstReg, RC: &SPIRV::iIDRegClass);
1145 MachineInstrBuilder MIB;
1146 MIB = MIRBuilder.buildInstr(Opcode);
1147 // Add Return register and type.
1148 if (Opcode == SPIRV::OpGroupReserveReadPipePackets ||
1149 Opcode == SPIRV::OpGroupReserveWritePipePackets)
1150 MIB.addDef(RegNo: Call->ReturnRegister)
1151 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
1152
1153 MIB.addUse(RegNo: ScopeConstReg);
1154 for (unsigned int i = 0; i < Call->Arguments.size(); ++i)
1155 MIB.addUse(RegNo: Call->Arguments[i]);
1156
1157 return true;
1158 }
1159 default:
1160 return buildOpFromWrapper(MIRBuilder, Opcode, Call,
1161 TypeReg: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
1162 }
1163}
1164
1165static unsigned getNumComponentsForDim(SPIRV::Dim::Dim dim) {
1166 switch (dim) {
1167 case SPIRV::Dim::DIM_1D:
1168 case SPIRV::Dim::DIM_Buffer:
1169 return 1;
1170 case SPIRV::Dim::DIM_2D:
1171 case SPIRV::Dim::DIM_Cube:
1172 case SPIRV::Dim::DIM_Rect:
1173 return 2;
1174 case SPIRV::Dim::DIM_3D:
1175 return 3;
1176 default:
1177 report_fatal_error(reason: "Cannot get num components for given Dim");
1178 }
1179}
1180
1181/// Helper function for obtaining the number of size components.
1182static unsigned getNumSizeComponents(SPIRVTypeInst imgType) {
1183 assert(imgType->getOpcode() == SPIRV::OpTypeImage);
1184 auto dim = static_cast<SPIRV::Dim::Dim>(imgType->getOperand(i: 2).getImm());
1185 unsigned numComps = getNumComponentsForDim(dim);
1186 bool arrayed = imgType->getOperand(i: 4).getImm() == 1;
1187 return arrayed ? numComps + 1 : numComps;
1188}
1189
1190static bool builtinMayNeedPromotionToVec(uint32_t BuiltinNumber) {
1191 switch (BuiltinNumber) {
1192 case SPIRV::OpenCLExtInst::s_min:
1193 case SPIRV::OpenCLExtInst::u_min:
1194 case SPIRV::OpenCLExtInst::s_max:
1195 case SPIRV::OpenCLExtInst::u_max:
1196 case SPIRV::OpenCLExtInst::fmax:
1197 case SPIRV::OpenCLExtInst::fmin:
1198 case SPIRV::OpenCLExtInst::fmax_common:
1199 case SPIRV::OpenCLExtInst::fmin_common:
1200 case SPIRV::OpenCLExtInst::s_clamp:
1201 case SPIRV::OpenCLExtInst::fclamp:
1202 case SPIRV::OpenCLExtInst::u_clamp:
1203 case SPIRV::OpenCLExtInst::mix:
1204 case SPIRV::OpenCLExtInst::step:
1205 case SPIRV::OpenCLExtInst::smoothstep:
1206 case SPIRV::OpenCLExtInst::ldexp:
1207 case SPIRV::OpenCLExtInst::pown:
1208 case SPIRV::OpenCLExtInst::rootn:
1209 return true;
1210 default:
1211 break;
1212 }
1213 return false;
1214}
1215
1216//===----------------------------------------------------------------------===//
1217// Implementation functions for each builtin group
1218//===----------------------------------------------------------------------===//
1219
1220static SmallVector<Register>
1221getBuiltinCallArguments(const SPIRV::IncomingCall *Call, uint32_t BuiltinNumber,
1222 MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) {
1223
1224 Register ReturnTypeId = GR->getSPIRVTypeID(SpirvType: Call->ReturnType);
1225 unsigned ResultElementCount =
1226 GR->getScalarOrVectorComponentCount(VReg: ReturnTypeId);
1227 bool MayNeedPromotionToVec =
1228 builtinMayNeedPromotionToVec(BuiltinNumber) && ResultElementCount > 1;
1229
1230 if (!MayNeedPromotionToVec)
1231 return {Call->Arguments.begin(), Call->Arguments.end()};
1232
1233 SmallVector<Register> Arguments;
1234 for (Register Argument : Call->Arguments) {
1235 Register VecArg = Argument;
1236 SPIRVTypeInst ArgumentType = GR->getSPIRVTypeForVReg(VReg: Argument);
1237 if (GR->getScalarOrVectorComponentCount(Type: ArgumentType) == 1 &&
1238 ArgumentType != Call->ReturnType) {
1239 SPIRVTypeInst VecType = GR->getOrCreateSPIRVVectorType(
1240 BaseType: ArgumentType, NumElements: ResultElementCount, MIRBuilder, /*EmitIR=*/true);
1241 VecArg = createVirtualRegister(SpvType: VecType, GR, MIRBuilder);
1242 Register VecTypeId = GR->getSPIRVTypeID(SpirvType: VecType);
1243 auto VecSplat = MIRBuilder.buildInstr(Opcode: SPIRV::OpCompositeConstruct)
1244 .addDef(RegNo: VecArg)
1245 .addUse(RegNo: VecTypeId);
1246 for (unsigned I = 0; I != ResultElementCount; ++I)
1247 VecSplat.addUse(RegNo: Argument);
1248 }
1249 Arguments.push_back(Elt: VecArg);
1250 }
1251 return Arguments;
1252}
1253
1254static bool generateExtInst(const SPIRV::IncomingCall *Call,
1255 MachineIRBuilder &MIRBuilder,
1256 SPIRVGlobalRegistry *GR, const CallBase &CB) {
1257 // Lookup the extended instruction number in the TableGen records.
1258 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
1259 uint32_t Number =
1260 SPIRV::lookupExtendedBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Number;
1261 // fmin_common and fmax_common are now deprecated, and we should use fmin and
1262 // fmax with NotInf and NotNaN flags instead. Keep original number to add
1263 // later the NoNans and NoInfs flags.
1264 uint32_t OrigNumber = Number;
1265 const SPIRVSubtarget &ST =
1266 cast<SPIRVSubtarget>(Val: MIRBuilder.getMF().getSubtarget());
1267 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2) &&
1268 (Number == SPIRV::OpenCLExtInst::fmin_common ||
1269 Number == SPIRV::OpenCLExtInst::fmax_common)) {
1270 Number = (Number == SPIRV::OpenCLExtInst::fmin_common)
1271 ? SPIRV::OpenCLExtInst::fmin
1272 : SPIRV::OpenCLExtInst::fmax;
1273 }
1274
1275 // ExtInst prefetch cannot take an untyped pointer, so emit
1276 // OpUntypedPrefetchKHR with Num Bytes = num elements * element byte size.
1277 if (Number == SPIRV::OpenCLExtInst::prefetch && Call->Arguments.size() >= 2) {
1278 Register PtrReg = Call->Arguments[0];
1279 SPIRVTypeInst PtrTy = GR->getSPIRVTypeForVReg(VReg: PtrReg);
1280 if (PtrTy && PtrTy->getOpcode() == SPIRV::OpTypeUntypedPointerKHR) {
1281 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
1282 Register NumElems = Call->Arguments[1];
1283 SPIRVTypeInst SizeTy = GR->getSPIRVTypeForVReg(VReg: NumElems);
1284 assert(SizeTy && "Expected a type for the number of elements");
1285 unsigned ElemBytes = GR->getDeducedPointeeByteSize(PtrVal: CB.getArgOperand(i: 0));
1286 Register NumBytes = NumElems;
1287 // A byte sized element already makes the element count a byte count. A
1288 // size of 0 means the element type could not be deduced, which the typed
1289 // lowering resolves to i8, so treat it as a single byte here as well.
1290 if (ElemBytes > 1) {
1291 Register ElemBytesReg = GR->buildConstantInt(Val: ElemBytes, MIRBuilder,
1292 SpvType: SizeTy, /*EmitIR=*/true);
1293 Register Mul =
1294 MRI->createGenericVirtualRegister(Ty: MRI->getType(Reg: NumElems));
1295 MRI->setRegClass(Reg: Mul, RC: GR->getRegClass(SpvType: SizeTy));
1296 GR->assignSPIRVTypeToVReg(Type: SizeTy, VReg: Mul, MF: MIRBuilder.getMF());
1297 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_MUL)
1298 .addDef(RegNo: Mul)
1299 .addUse(RegNo: NumElems)
1300 .addUse(RegNo: ElemBytesReg);
1301 updateRegType(Reg: Mul, /*Ty=*/nullptr, SpirvTy: SizeTy, GR, MIB&: MIRBuilder, MRI&: *MRI);
1302 NumBytes = Mul;
1303 }
1304 MIRBuilder.buildInstr(Opcode: SPIRV::OpUntypedPrefetchKHR)
1305 .addUse(RegNo: PtrReg)
1306 .addUse(RegNo: NumBytes);
1307 return true;
1308 }
1309 }
1310
1311 Register ReturnTypeId = GR->getSPIRVTypeID(SpirvType: Call->ReturnType);
1312 SmallVector<Register> Arguments =
1313 getBuiltinCallArguments(Call, BuiltinNumber: Number, MIRBuilder, GR);
1314
1315 MachineInstrBuilder MIB;
1316 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_fma) &&
1317 Number == SPIRV::OpenCLExtInst::fma) {
1318 // Use the SPIR-V fma instruction instead of the OpenCL extended
1319 // instruction if the extension is available.
1320 MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpFmaKHR)
1321 .addDef(RegNo: Call->ReturnRegister)
1322 .addUse(RegNo: ReturnTypeId);
1323 } else {
1324 // Build extended instruction.
1325 MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpExtInst)
1326 .addDef(RegNo: Call->ReturnRegister)
1327 .addUse(RegNo: ReturnTypeId)
1328 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
1329 .addImm(Val: Number);
1330 }
1331
1332 for (Register Argument : Arguments)
1333 MIB.addUse(RegNo: Argument);
1334
1335 MIB.getInstr()->copyIRFlags(I: CB);
1336 if (OrigNumber == SPIRV::OpenCLExtInst::fmin_common ||
1337 OrigNumber == SPIRV::OpenCLExtInst::fmax_common) {
1338 // Add NoNans and NoInfs flags to fmin/fmax instruction.
1339 MIB.getInstr()->setFlag(MachineInstr::MIFlag::FmNoNans);
1340 MIB.getInstr()->setFlag(MachineInstr::MIFlag::FmNoInfs);
1341 }
1342
1343 // Derive fast-math flags from nofpclass attributes on the called function.
1344 // FPFastMathMode decoration is valid on ExtInst in Kernel environments
1345 // (SPIR-V core) or with SPV_KHR_float_controls2 for any environment.
1346 if (ST.isKernel() ||
1347 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2)) {
1348 if (const Function *F = CB.getCalledFunction()) {
1349 bool AddNoNan = CB.getRetNoFPClass() & fcNan;
1350 bool AddNoInf = CB.getRetNoFPClass() & fcInf;
1351 FunctionType *FTy = F->getFunctionType();
1352 for (unsigned I = 0, E = FTy->getNumParams();
1353 I != E && (AddNoNan || AddNoInf); ++I) {
1354 if (!FTy->getParamType(i: I)->isFloatingPointTy())
1355 continue;
1356 FPClassTest ArgTest = CB.getParamNoFPClass(i: I);
1357 AddNoNan = AddNoNan && ArgTest & fcNan;
1358 AddNoInf = AddNoInf && ArgTest & fcInf;
1359 }
1360 if (AddNoNan)
1361 MIB.getInstr()->setFlag(MachineInstr::MIFlag::FmNoNans);
1362 if (AddNoInf)
1363 MIB.getInstr()->setFlag(MachineInstr::MIFlag::FmNoInfs);
1364 }
1365 }
1366
1367 return true;
1368}
1369
1370static bool generateRelationalInst(const SPIRV::IncomingCall *Call,
1371 MachineIRBuilder &MIRBuilder,
1372 SPIRVGlobalRegistry *GR) {
1373 // Lookup the instruction opcode in the TableGen records.
1374 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
1375 unsigned Opcode =
1376 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
1377
1378 Register CompareRegister;
1379 SPIRVTypeInst RelationType = nullptr;
1380 std::tie(args&: CompareRegister, args&: RelationType) =
1381 buildBoolRegister(MIRBuilder, ResultType: Call->ReturnType, GR);
1382
1383 // OpAny/OpAll require a boolean vector input, but OpenCL any()/all()
1384 // builtins receive integer vectors. Convert via OpINotEqual against zero.
1385 SmallVector<Register> Arguments(Call->Arguments.begin(),
1386 Call->Arguments.end());
1387 if ((Opcode == SPIRV::OpAny || Opcode == SPIRV::OpAll) &&
1388 !GR->isScalarOrVectorOfType(VReg: Arguments[0], TypeOpcode: SPIRV::OpTypeBool)) {
1389 SPIRVTypeInst ArgType = GR->getSPIRVTypeForVReg(VReg: Arguments[0]);
1390 unsigned NumElts = GR->getScalarOrVectorComponentCount(Type: ArgType);
1391 SPIRVTypeInst BoolVecTy = GR->getOrCreateSPIRVVectorType(
1392 BaseType: GR->getOrCreateSPIRVBoolType(MIRBuilder, /*EmitIR=*/true), NumElements: NumElts,
1393 MIRBuilder, /*EmitIR=*/true);
1394 Register ZeroReg =
1395 GR->getOrCreateConsIntVector(Val: uint64_t(0), MIRBuilder, SpvType: ArgType,
1396 /*EmitIR=*/true);
1397 Register BoolVecReg = createVirtualRegister(SpvType: BoolVecTy, GR, MIRBuilder);
1398 MIRBuilder.buildInstr(Opcode: SPIRV::OpINotEqual)
1399 .addDef(RegNo: BoolVecReg)
1400 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: BoolVecTy))
1401 .addUse(RegNo: Arguments[0])
1402 .addUse(RegNo: ZeroReg);
1403 Arguments[0] = BoolVecReg;
1404 }
1405
1406 // Build relational instruction.
1407 auto MIB = MIRBuilder.buildInstr(Opcode)
1408 .addDef(RegNo: CompareRegister)
1409 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RelationType));
1410
1411 for (auto Argument : Arguments)
1412 MIB.addUse(RegNo: Argument);
1413
1414 // Build select instruction.
1415 return buildSelectInst(MIRBuilder, ReturnRegister: Call->ReturnRegister, SourceRegister: CompareRegister,
1416 ReturnType: Call->ReturnType, GR);
1417}
1418
1419static bool generateGroupInst(const SPIRV::IncomingCall *Call,
1420 MachineIRBuilder &MIRBuilder,
1421 SPIRVGlobalRegistry *GR) {
1422 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
1423 const SPIRV::GroupBuiltin *GroupBuiltin =
1424 SPIRV::lookupGroupBuiltin(Name: Builtin->name());
1425
1426 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
1427 if (Call->isSpirvOp()) {
1428 if (GroupBuiltin->NoGroupOperation) {
1429 SmallVector<uint32_t, 1> ImmArgs;
1430 if (GroupBuiltin->Opcode ==
1431 SPIRV::OpSubgroupMatrixMultiplyAccumulateINTEL &&
1432 Call->Arguments.size() > 4)
1433 ImmArgs.push_back(Elt: getIConstVal(ConstReg: Call->Arguments[4], MRI));
1434 return buildOpFromWrapper(MIRBuilder, Opcode: GroupBuiltin->Opcode, Call,
1435 TypeReg: GR->getSPIRVTypeID(SpirvType: Call->ReturnType), ImmArgs);
1436 }
1437
1438 // Group Operation is a literal
1439 Register GroupOpReg = Call->Arguments[1];
1440 const MachineInstr *MI = getDefInstrMaybeConstant(ConstReg&: GroupOpReg, MRI);
1441 if (!MI || MI->getOpcode() != TargetOpcode::G_CONSTANT)
1442 report_fatal_error(
1443 reason: "Group Operation parameter must be an integer constant");
1444 uint64_t GrpOp = MI->getOperand(i: 1).getCImm()->getValue().getZExtValue();
1445 Register ScopeReg = Call->Arguments[0];
1446 auto MIB = MIRBuilder.buildInstr(Opcode: GroupBuiltin->Opcode)
1447 .addDef(RegNo: Call->ReturnRegister)
1448 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
1449 .addUse(RegNo: ScopeReg)
1450 .addImm(Val: GrpOp);
1451 for (unsigned i = 2; i < Call->Arguments.size(); ++i)
1452 MIB.addUse(RegNo: Call->Arguments[i]);
1453 return true;
1454 }
1455
1456 Register Arg0;
1457 if (GroupBuiltin->HasBoolArg) {
1458 SPIRVTypeInst BoolType = GR->getOrCreateSPIRVBoolType(MIRBuilder, EmitIR: true);
1459 Register BoolReg = Call->Arguments[0];
1460 SPIRVTypeInst BoolRegType = GR->getSPIRVTypeForVReg(VReg: BoolReg);
1461 if (!BoolRegType)
1462 report_fatal_error(reason: "Can't find a register's type definition");
1463 MachineInstr *ArgInstruction = getDefInstrMaybeConstant(ConstReg&: BoolReg, MRI);
1464 if (ArgInstruction->getOpcode() == TargetOpcode::G_CONSTANT) {
1465 if (BoolRegType->getOpcode() != SPIRV::OpTypeBool)
1466 Arg0 = GR->buildConstantInt(Val: getIConstVal(ConstReg: BoolReg, MRI) != 0, MIRBuilder,
1467 SpvType: BoolType, EmitIR: true);
1468 } else {
1469 if (BoolRegType->getOpcode() == SPIRV::OpTypeInt) {
1470 Arg0 = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 1));
1471 MRI->setRegClass(Reg: Arg0, RC: &SPIRV::iIDRegClass);
1472 GR->assignSPIRVTypeToVReg(Type: BoolType, VReg: Arg0, MF: MIRBuilder.getMF());
1473 MIRBuilder.buildICmp(
1474 Pred: CmpInst::ICMP_NE, Res: Arg0, Op0: BoolReg,
1475 Op1: GR->buildConstantInt(Val: 0, MIRBuilder, SpvType: BoolRegType, EmitIR: true));
1476 updateRegType(Reg: Arg0, Ty: nullptr, SpirvTy: BoolType, GR, MIB&: MIRBuilder,
1477 MRI&: MIRBuilder.getMF().getRegInfo());
1478 } else if (BoolRegType->getOpcode() != SPIRV::OpTypeBool) {
1479 report_fatal_error(reason: "Expect a boolean argument");
1480 }
1481 // if BoolReg is a boolean register, we don't need to do anything
1482 }
1483 }
1484
1485 Register GroupResultRegister = Call->ReturnRegister;
1486 SPIRVTypeInst GroupResultType = Call->ReturnType;
1487
1488 // TODO: maybe we need to check whether the result type is already boolean
1489 // and in this case do not insert select instruction.
1490 const bool HasBoolReturnTy =
1491 GroupBuiltin->IsElect || GroupBuiltin->IsAllOrAny ||
1492 GroupBuiltin->IsAllEqual || GroupBuiltin->IsLogical ||
1493 GroupBuiltin->IsInverseBallot || GroupBuiltin->IsBallotBitExtract;
1494
1495 if (HasBoolReturnTy)
1496 std::tie(args&: GroupResultRegister, args&: GroupResultType) =
1497 buildBoolRegister(MIRBuilder, ResultType: Call->ReturnType, GR);
1498
1499 auto Scope = Builtin->name().starts_with(Prefix: "sub_group")
1500 ? SPIRV::Scope::Subgroup
1501 : SPIRV::Scope::Workgroup;
1502 Register ScopeRegister = buildConstantIntReg32(Val: Scope, MIRBuilder, GR);
1503
1504 Register VecReg;
1505 if (GroupBuiltin->Opcode == SPIRV::OpGroupBroadcast &&
1506 Call->Arguments.size() > 2) {
1507 // For OpGroupBroadcast "LocalId must be an integer datatype. It must be a
1508 // scalar, a vector with 2 components, or a vector with 3 components.",
1509 // meaning that we must create a vector from the function arguments if
1510 // it's a work_group_broadcast(val, local_id_x, local_id_y) or
1511 // work_group_broadcast(val, local_id_x, local_id_y, local_id_z) call.
1512 Register ElemReg = Call->Arguments[1];
1513 SPIRVTypeInst ElemType = GR->getSPIRVTypeForVReg(VReg: ElemReg);
1514 if (!ElemType || ElemType->getOpcode() != SPIRV::OpTypeInt)
1515 report_fatal_error(reason: "Expect an integer <LocalId> argument");
1516 unsigned VecLen = Call->Arguments.size() - 1;
1517 VecReg = MRI->createGenericVirtualRegister(
1518 Ty: LLT::fixed_vector(NumElements: VecLen, ScalarTy: MRI->getType(Reg: ElemReg)));
1519 MRI->setRegClass(Reg: VecReg, RC: &SPIRV::viIDRegClass);
1520 SPIRVTypeInst VecType =
1521 GR->getOrCreateSPIRVVectorType(BaseType: ElemType, NumElements: VecLen, MIRBuilder, EmitIR: true);
1522 GR->assignSPIRVTypeToVReg(Type: VecType, VReg: VecReg, MF: MIRBuilder.getMF());
1523 auto MIB =
1524 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_BUILD_VECTOR).addDef(RegNo: VecReg);
1525 for (unsigned i = 1; i < Call->Arguments.size(); i++) {
1526 MIB.addUse(RegNo: Call->Arguments[i]);
1527 setRegClassIfNull(Reg: Call->Arguments[i], MRI, GR);
1528 }
1529 updateRegType(Reg: VecReg, Ty: nullptr, SpirvTy: VecType, GR, MIB&: MIRBuilder,
1530 MRI&: MIRBuilder.getMF().getRegInfo());
1531 }
1532
1533 // Build work/sub group instruction.
1534 auto MIB = MIRBuilder.buildInstr(Opcode: GroupBuiltin->Opcode)
1535 .addDef(RegNo: GroupResultRegister)
1536 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: GroupResultType))
1537 .addUse(RegNo: ScopeRegister);
1538
1539 if (!GroupBuiltin->NoGroupOperation)
1540 MIB.addImm(Val: GroupBuiltin->GroupOperation);
1541 if (Call->Arguments.size() > 0) {
1542 MIB.addUse(RegNo: Arg0.isValid() ? Arg0 : Call->Arguments[0]);
1543 setRegClassIfNull(Reg: Call->Arguments[0], MRI, GR);
1544 if (VecReg.isValid())
1545 MIB.addUse(RegNo: VecReg);
1546 else
1547 for (unsigned i = 1; i < Call->Arguments.size(); i++)
1548 MIB.addUse(RegNo: Call->Arguments[i]);
1549 }
1550
1551 // Build select instruction.
1552 if (HasBoolReturnTy)
1553 buildSelectInst(MIRBuilder, ReturnRegister: Call->ReturnRegister, SourceRegister: GroupResultRegister,
1554 ReturnType: Call->ReturnType, GR);
1555 return true;
1556}
1557
1558static bool generateIntelSubgroupsInst(const SPIRV::IncomingCall *Call,
1559 MachineIRBuilder &MIRBuilder,
1560 SPIRVGlobalRegistry *GR) {
1561 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
1562 MachineFunction &MF = MIRBuilder.getMF();
1563 const auto *ST = static_cast<const SPIRVSubtarget *>(&MF.getSubtarget());
1564 const SPIRV::IntelSubgroupsBuiltin *IntelSubgroups =
1565 SPIRV::lookupIntelSubgroupsBuiltin(Name: Builtin->name());
1566
1567 if (IntelSubgroups->IsMedia &&
1568 !ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_media_block_io)) {
1569 std::string DiagMsg = std::string(Builtin->name()) +
1570 ": the builtin requires the following SPIR-V "
1571 "extension: SPV_INTEL_media_block_io";
1572 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
1573 } else if (!IntelSubgroups->IsMedia &&
1574 !ST->canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
1575 std::string DiagMsg = std::string(Builtin->name()) +
1576 ": the builtin requires the following SPIR-V "
1577 "extension: SPV_INTEL_subgroups";
1578 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
1579 }
1580
1581 uint32_t OpCode = IntelSubgroups->Opcode;
1582 if (Call->isSpirvOp()) {
1583 bool IsSet = OpCode != SPIRV::OpSubgroupBlockWriteINTEL &&
1584 OpCode != SPIRV::OpSubgroupImageBlockWriteINTEL &&
1585 OpCode != SPIRV::OpSubgroupImageMediaBlockWriteINTEL;
1586 return buildOpFromWrapper(MIRBuilder, Opcode: OpCode, Call,
1587 TypeReg: IsSet ? GR->getSPIRVTypeID(SpirvType: Call->ReturnType)
1588 : Register(0));
1589 }
1590
1591 if (IntelSubgroups->IsBlock) {
1592 // Minimal number or arguments set in TableGen records is 1
1593 if (SPIRVTypeInst Arg0Type = GR->getSPIRVTypeForVReg(VReg: Call->Arguments[0])) {
1594 if (Arg0Type->getOpcode() == SPIRV::OpTypeImage) {
1595 // TODO: add required validation from the specification:
1596 // "'Image' must be an object whose type is OpTypeImage with a 'Sampled'
1597 // operand of 0 or 2. If the 'Sampled' operand is 2, then some
1598 // dimensions require a capability."
1599 switch (OpCode) {
1600 case SPIRV::OpSubgroupBlockReadINTEL:
1601 OpCode = SPIRV::OpSubgroupImageBlockReadINTEL;
1602 break;
1603 case SPIRV::OpSubgroupBlockWriteINTEL:
1604 OpCode = SPIRV::OpSubgroupImageBlockWriteINTEL;
1605 break;
1606 }
1607 }
1608 }
1609 }
1610
1611 // TODO: opaque pointers types should be eventually resolved in such a way
1612 // that validation of block read is enabled with respect to the following
1613 // specification requirement:
1614 // "'Result Type' may be a scalar or vector type, and its component type must
1615 // be equal to the type pointed to by 'Ptr'."
1616 // For example, function parameter type should not be default i8 pointer, but
1617 // depend on the result type of the instruction where it is used as a pointer
1618 // argument of OpSubgroupBlockReadINTEL
1619
1620 // Build Intel subgroups instruction
1621 MachineInstrBuilder MIB =
1622 IntelSubgroups->IsWrite
1623 ? MIRBuilder.buildInstr(Opcode: OpCode)
1624 : MIRBuilder.buildInstr(Opcode: OpCode)
1625 .addDef(RegNo: Call->ReturnRegister)
1626 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
1627 for (size_t i = 0; i < Call->Arguments.size(); ++i)
1628 MIB.addUse(RegNo: Call->Arguments[i]);
1629 return true;
1630}
1631
1632static bool generateGroupUniformInst(const SPIRV::IncomingCall *Call,
1633 MachineIRBuilder &MIRBuilder,
1634 SPIRVGlobalRegistry *GR) {
1635 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
1636 MachineFunction &MF = MIRBuilder.getMF();
1637 const auto *ST = static_cast<const SPIRVSubtarget *>(&MF.getSubtarget());
1638 if (!ST->canUseExtension(
1639 E: SPIRV::Extension::SPV_KHR_uniform_group_instructions)) {
1640 std::string DiagMsg = std::string(Builtin->name()) +
1641 ": the builtin requires the following SPIR-V "
1642 "extension: SPV_KHR_uniform_group_instructions";
1643 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
1644 }
1645 const SPIRV::GroupUniformBuiltin *GroupUniform =
1646 SPIRV::lookupGroupUniformBuiltin(Name: Builtin->name());
1647 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
1648
1649 Register GroupResultReg = Call->ReturnRegister;
1650 Register ScopeReg = Call->Arguments[0];
1651 Register ValueReg = Call->Arguments[2];
1652
1653 // Group Operation
1654 Register ConstGroupOpReg = Call->Arguments[1];
1655 const MachineInstr *Const = getDefInstrMaybeConstant(ConstReg&: ConstGroupOpReg, MRI);
1656 if (!Const || Const->getOpcode() != TargetOpcode::G_CONSTANT)
1657 report_fatal_error(
1658 reason: "expect a constant group operation for a uniform group instruction",
1659 gen_crash_diag: false);
1660 const MachineOperand &ConstOperand = Const->getOperand(i: 1);
1661 if (!ConstOperand.isCImm())
1662 report_fatal_error(reason: "uniform group instructions: group operation must be an "
1663 "integer constant",
1664 gen_crash_diag: false);
1665
1666 auto MIB = MIRBuilder.buildInstr(Opcode: GroupUniform->Opcode)
1667 .addDef(RegNo: GroupResultReg)
1668 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
1669 .addUse(RegNo: ScopeReg);
1670 addNumImm(Imm: ConstOperand.getCImm()->getValue(), MIB);
1671 MIB.addUse(RegNo: ValueReg);
1672
1673 return true;
1674}
1675
1676static bool generateKernelClockInst(const SPIRV::IncomingCall *Call,
1677 MachineIRBuilder &MIRBuilder,
1678 SPIRVGlobalRegistry *GR) {
1679 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
1680 MachineFunction &MF = MIRBuilder.getMF();
1681 const auto *ST = static_cast<const SPIRVSubtarget *>(&MF.getSubtarget());
1682 if (!ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_shader_clock)) {
1683 std::string DiagMsg = std::string(Builtin->name()) +
1684 ": the builtin requires the following SPIR-V "
1685 "extension: SPV_KHR_shader_clock";
1686 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
1687 }
1688
1689 Register ResultReg = Call->ReturnRegister;
1690
1691 if (Builtin->name() == "__spirv_ReadClockKHR") {
1692 MIRBuilder.buildInstr(Opcode: SPIRV::OpReadClockKHR)
1693 .addDef(RegNo: ResultReg)
1694 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
1695 .addUse(RegNo: Call->Arguments[0]);
1696 } else {
1697 // Deduce the `Scope` operand from the builtin function name.
1698 SPIRV::Scope::Scope ScopeArg =
1699 StringSwitch<SPIRV::Scope::Scope>(Builtin->name())
1700 .EndsWith(S: "device", Value: SPIRV::Scope::Scope::Device)
1701 .EndsWith(S: "work_group", Value: SPIRV::Scope::Scope::Workgroup)
1702 .EndsWith(S: "sub_group", Value: SPIRV::Scope::Scope::Subgroup);
1703 Register ScopeReg = buildConstantIntReg32(Val: ScopeArg, MIRBuilder, GR);
1704
1705 MIRBuilder.buildInstr(Opcode: SPIRV::OpReadClockKHR)
1706 .addDef(RegNo: ResultReg)
1707 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
1708 .addUse(RegNo: ScopeReg);
1709 }
1710
1711 return true;
1712}
1713
1714// These queries ask for a single size_t result for a given dimension index,
1715// e.g. size_t get_global_id(uint dimindex). In SPIR-V, the builtins
1716// corresponding to these values are all vec3 types, so we need to extract the
1717// correct index or return DefaultValue (0 or 1 depending on the query). We also
1718// handle extending or truncating in case size_t does not match the expected
1719// result type's bitwidth.
1720//
1721// For a constant index >= 3 we generate:
1722// %res = OpConstant %SizeT DefaultValue
1723//
1724// For other indices we generate:
1725// %g = OpVariable %ptr_V3_SizeT Input
1726// OpDecorate %g BuiltIn XXX
1727// OpDecorate %g LinkageAttributes "__spirv_BuiltInXXX"
1728// OpDecorate %g Constant
1729// %loadedVec = OpLoad %V3_SizeT %g
1730//
1731// Then, if the index is constant < 3, we generate:
1732// %res = OpCompositeExtract %SizeT %loadedVec idx
1733// If the index is dynamic, we generate:
1734// %tmp = OpVectorExtractDynamic %SizeT %loadedVec %idx
1735// %cmp = OpULessThan %bool %idx %const_3
1736// %res = OpSelect %SizeT %cmp %tmp %const_<DefaultValue>
1737//
1738// If the bitwidth of %res does not match the expected return type, we add an
1739// extend or truncate.
1740static bool genWorkgroupQuery(const SPIRV::IncomingCall *Call,
1741 MachineIRBuilder &MIRBuilder,
1742 SPIRVGlobalRegistry *GR,
1743 SPIRV::BuiltIn::BuiltIn BuiltinValue,
1744 uint64_t DefaultValue) {
1745 Register IndexRegister = Call->Arguments[0];
1746 const unsigned ResultWidth = Call->ReturnType->getOperand(i: 1).getImm();
1747 const unsigned PointerSize = GR->getPointerSize();
1748 const SPIRVTypeInst PointerSizeType =
1749 GR->getOrCreateSPIRVIntegerType(BitWidth: PointerSize, MIRBuilder);
1750 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
1751 auto IndexInstruction = getDefInstrMaybeConstant(ConstReg&: IndexRegister, MRI);
1752
1753 // Set up the final register to do truncation or extension on at the end.
1754 Register ToTruncate = Call->ReturnRegister;
1755
1756 // If the index is constant, we can statically determine if it is in range.
1757 bool IsConstantIndex =
1758 IndexInstruction->getOpcode() == TargetOpcode::G_CONSTANT;
1759
1760 // If it's out of range (max dimension is 3), we can just return the constant
1761 // default value (0 or 1 depending on which query function).
1762 if (IsConstantIndex && getIConstVal(ConstReg: IndexRegister, MRI) >= 3) {
1763 Register DefaultReg = Call->ReturnRegister;
1764 if (PointerSize != ResultWidth) {
1765 DefaultReg = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: PointerSize));
1766 MRI->setRegClass(Reg: DefaultReg, RC: &SPIRV::iIDRegClass);
1767 GR->assignSPIRVTypeToVReg(Type: PointerSizeType, VReg: DefaultReg,
1768 MF: MIRBuilder.getMF());
1769 ToTruncate = DefaultReg;
1770 }
1771 auto NewRegister =
1772 GR->buildConstantInt(Val: DefaultValue, MIRBuilder, SpvType: PointerSizeType, EmitIR: true);
1773 MIRBuilder.buildCopy(Res: DefaultReg, Op: NewRegister);
1774 } else { // If it could be in range, we need to load from the given builtin.
1775 auto Vec3Ty =
1776 GR->getOrCreateSPIRVVectorType(BaseType: PointerSizeType, NumElements: 3, MIRBuilder, EmitIR: true);
1777 Register LoadedVector =
1778 buildBuiltinVariableLoad(MIRBuilder, VariableType: Vec3Ty, GR, BuiltinValue,
1779 LLType: LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: PointerSize));
1780 // Set up the vreg to extract the result to (possibly a new temporary one).
1781 Register Extracted = Call->ReturnRegister;
1782 if (!IsConstantIndex || PointerSize != ResultWidth) {
1783 Extracted = MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: PointerSize));
1784 MRI->setRegClass(Reg: Extracted, RC: &SPIRV::iIDRegClass);
1785 GR->assignSPIRVTypeToVReg(Type: PointerSizeType, VReg: Extracted, MF: MIRBuilder.getMF());
1786 }
1787 // Use Intrinsic::spv_extractelt so dynamic vs static extraction is
1788 // handled later: extr = spv_extractelt LoadedVector, IndexRegister.
1789 MachineInstrBuilder ExtractInst = MIRBuilder.buildIntrinsic(
1790 ID: Intrinsic::spv_extractelt, Res: ArrayRef<Register>{Extracted}, HasSideEffects: true, isConvergent: false);
1791 ExtractInst.addUse(RegNo: LoadedVector).addUse(RegNo: IndexRegister);
1792
1793 // If the index is dynamic, need check if it's < 3, and then use a select.
1794 if (!IsConstantIndex) {
1795 updateRegType(Reg: Extracted, Ty: nullptr, SpirvTy: PointerSizeType, GR, MIB&: MIRBuilder, MRI&: *MRI);
1796
1797 auto IndexType = GR->getSPIRVTypeForVReg(VReg: IndexRegister);
1798 auto BoolType = GR->getOrCreateSPIRVBoolType(MIRBuilder, EmitIR: true);
1799
1800 Register CompareRegister =
1801 MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 1));
1802 MRI->setRegClass(Reg: CompareRegister, RC: &SPIRV::iIDRegClass);
1803 GR->assignSPIRVTypeToVReg(Type: BoolType, VReg: CompareRegister, MF: MIRBuilder.getMF());
1804
1805 // Use G_ICMP to check if idxVReg < 3.
1806 MIRBuilder.buildICmp(
1807 Pred: CmpInst::ICMP_ULT, Res: CompareRegister, Op0: IndexRegister,
1808 Op1: GR->buildConstantInt(Val: 3, MIRBuilder, SpvType: IndexType, EmitIR: true));
1809
1810 // Get constant for the default value (0 or 1 depending on which
1811 // function).
1812 Register DefaultRegister =
1813 GR->buildConstantInt(Val: DefaultValue, MIRBuilder, SpvType: PointerSizeType, EmitIR: true);
1814
1815 // Get a register for the selection result (possibly a new temporary one).
1816 Register SelectionResult = Call->ReturnRegister;
1817 if (PointerSize != ResultWidth) {
1818 SelectionResult =
1819 MRI->createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: PointerSize));
1820 MRI->setRegClass(Reg: SelectionResult, RC: &SPIRV::iIDRegClass);
1821 GR->assignSPIRVTypeToVReg(Type: PointerSizeType, VReg: SelectionResult,
1822 MF: MIRBuilder.getMF());
1823 }
1824 // Create the final G_SELECT to return the extracted value or the default.
1825 MIRBuilder.buildSelect(Res: SelectionResult, Tst: CompareRegister, Op0: Extracted,
1826 Op1: DefaultRegister);
1827 ToTruncate = SelectionResult;
1828 } else {
1829 ToTruncate = Extracted;
1830 }
1831 }
1832 // Alter the result's bitwidth if it does not match the SizeT value extracted.
1833 if (PointerSize != ResultWidth)
1834 MIRBuilder.buildZExtOrTrunc(Res: Call->ReturnRegister, Op: ToTruncate);
1835 return true;
1836}
1837
1838static bool generateBuiltinVar(const SPIRV::IncomingCall *Call,
1839 MachineIRBuilder &MIRBuilder,
1840 SPIRVGlobalRegistry *GR) {
1841 // Lookup the builtin variable record.
1842 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
1843 SPIRV::BuiltIn::BuiltIn Value =
1844 SPIRV::lookupGetBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Value;
1845
1846 if (Value == SPIRV::BuiltIn::GlobalInvocationId)
1847 return genWorkgroupQuery(Call, MIRBuilder, GR, BuiltinValue: Value, DefaultValue: 0);
1848
1849 // Build a load instruction for the builtin variable.
1850 unsigned BitWidth = GR->getScalarOrVectorBitWidth(Type: Call->ReturnType);
1851 LLT LLType;
1852 if (isVectorType(SPVTy: Call->ReturnType))
1853 LLType = LLT::fixed_vector(
1854 NumElements: GR->getScalarOrVectorComponentCount(Type: Call->ReturnType), ScalarSizeInBits: BitWidth);
1855 else
1856 LLType = LLT::scalar(SizeInBits: BitWidth);
1857
1858 return buildBuiltinVariableLoad(MIRBuilder, VariableType: Call->ReturnType, GR, BuiltinValue: Value,
1859 LLType, Reg: Call->ReturnRegister);
1860}
1861
1862static bool generateAtomicInst(const SPIRV::IncomingCall *Call,
1863 MachineIRBuilder &MIRBuilder,
1864 SPIRVGlobalRegistry *GR) {
1865 // Lookup the instruction opcode in the TableGen records.
1866 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
1867 unsigned Opcode =
1868 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
1869
1870 switch (Opcode) {
1871 case SPIRV::OpStore:
1872 return buildAtomicInitInst(Call, MIRBuilder);
1873 case SPIRV::OpAtomicLoad:
1874 return buildAtomicLoadInst(Call, MIRBuilder, GR);
1875 case SPIRV::OpAtomicStore:
1876 return buildAtomicStoreInst(Call, MIRBuilder, GR);
1877 case SPIRV::OpAtomicCompareExchange:
1878 case SPIRV::OpAtomicCompareExchangeWeak:
1879 return buildAtomicCompareExchangeInst(Call, Builtin, Opcode, MIRBuilder,
1880 GR);
1881 case SPIRV::OpAtomicIAdd:
1882 case SPIRV::OpAtomicISub:
1883 case SPIRV::OpAtomicOr:
1884 case SPIRV::OpAtomicXor:
1885 case SPIRV::OpAtomicAnd:
1886 case SPIRV::OpAtomicExchange:
1887 case SPIRV::OpAtomicSMax:
1888 case SPIRV::OpAtomicSMin:
1889 case SPIRV::OpAtomicUMax:
1890 case SPIRV::OpAtomicUMin:
1891 case SPIRV::OpAtomicFMinEXT:
1892 case SPIRV::OpAtomicFMaxEXT:
1893 return buildAtomicRMWInst(Call, Opcode, MIRBuilder, GR);
1894 case SPIRV::OpMemoryBarrier:
1895 return buildBarrierInst(Call, Opcode: SPIRV::OpMemoryBarrier, MIRBuilder, GR);
1896 case SPIRV::OpAtomicFlagTestAndSet:
1897 case SPIRV::OpAtomicFlagClear:
1898 return buildAtomicFlagInst(Call, Opcode, MIRBuilder, GR);
1899 default:
1900 if (Call->isSpirvOp())
1901 return buildOpFromWrapper(MIRBuilder, Opcode, Call,
1902 TypeReg: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
1903 return false;
1904 }
1905}
1906
1907static bool generateAtomicFloatingInst(const SPIRV::IncomingCall *Call,
1908 MachineIRBuilder &MIRBuilder,
1909 SPIRVGlobalRegistry *GR) {
1910 // Lookup the instruction opcode in the TableGen records.
1911 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
1912 unsigned Opcode = SPIRV::lookupAtomicFloatingBuiltin(Name: Builtin->name())->Opcode;
1913
1914 switch (Opcode) {
1915 case SPIRV::OpAtomicFAddEXT:
1916 case SPIRV::OpAtomicFMinEXT:
1917 case SPIRV::OpAtomicFMaxEXT:
1918 return buildAtomicFloatingRMWInst(Call, Opcode, MIRBuilder, GR);
1919 default:
1920 return false;
1921 }
1922}
1923
1924static bool generateBarrierInst(const SPIRV::IncomingCall *Call,
1925 MachineIRBuilder &MIRBuilder,
1926 SPIRVGlobalRegistry *GR) {
1927 // Lookup the instruction opcode in the TableGen records.
1928 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
1929 unsigned Opcode =
1930 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
1931
1932 return buildBarrierInst(Call, Opcode, MIRBuilder, GR);
1933}
1934
1935static bool generateCastToPtrInst(const SPIRV::IncomingCall *Call,
1936 MachineIRBuilder &MIRBuilder,
1937 SPIRVGlobalRegistry *GR) {
1938 // Lookup the instruction opcode in the TableGen records.
1939 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
1940 unsigned Opcode =
1941 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
1942
1943 if (Opcode == SPIRV::OpGenericCastToPtrExplicit) {
1944 SPIRV::StorageClass::StorageClass ResSC =
1945 GR->getPointerStorageClass(VReg: Call->ReturnRegister);
1946 if (!isGenericCastablePtr(SC: ResSC))
1947 return false;
1948
1949 MIRBuilder.buildInstr(Opcode)
1950 .addDef(RegNo: Call->ReturnRegister)
1951 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
1952 .addUse(RegNo: Call->Arguments[0])
1953 .addImm(Val: ResSC);
1954 } else if (Opcode == SPIRV::OpGenericPtrMemSemantics) {
1955 if (GR->getPointerStorageClass(VReg: Call->Arguments[0]) !=
1956 SPIRV::StorageClass::Generic)
1957 return false;
1958
1959 // Shift the WorkgroupMemory/CrossWorkgroupMemory bits down to
1960 // CLK_LOCAL/GLOBAL_MEM_FENCE.
1961 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
1962 SPIRVTypeInst RetTy = Call->ReturnType;
1963 Register SemReg =
1964 MRI->createGenericVirtualRegister(Ty: MRI->getType(Reg: Call->ReturnRegister));
1965 MRI->setRegClass(Reg: SemReg, RC: GR->getRegClass(SpvType: RetTy));
1966 GR->assignSPIRVTypeToVReg(Type: RetTy, VReg: SemReg, MF: MIRBuilder.getMF());
1967 MIRBuilder.buildInstr(Opcode)
1968 .addDef(RegNo: SemReg)
1969 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetTy))
1970 .addUse(RegNo: Call->Arguments[0]);
1971 Register ShiftReg =
1972 GR->buildConstantInt(Val: 8, MIRBuilder, SpvType: RetTy, /*EmitIR=*/true);
1973 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_LSHR)
1974 .addDef(RegNo: Call->ReturnRegister)
1975 .addUse(RegNo: SemReg)
1976 .addUse(RegNo: ShiftReg);
1977 } else {
1978 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_ADDRSPACE_CAST)
1979 .addDef(RegNo: Call->ReturnRegister)
1980 .addUse(RegNo: Call->Arguments[0]);
1981 }
1982 return true;
1983}
1984
1985static bool generateDotOrFMulInst(StringRef DemangledCall,
1986 const SPIRV::IncomingCall *Call,
1987 MachineIRBuilder &MIRBuilder,
1988 SPIRVGlobalRegistry *GR) {
1989 if (Call->isSpirvOp())
1990 return buildOpFromWrapper(MIRBuilder, Opcode: SPIRV::OpDot, Call,
1991 TypeReg: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
1992
1993 // Use OpDot only in case of vector args and OpFMul in case of scalar args.
1994 bool IsVec = isVectorType(SPVTy: GR->getSPIRVTypeForVReg(VReg: Call->Arguments[0]));
1995 uint32_t OC = IsVec ? SPIRV::OpDot : SPIRV::OpFMulS;
1996 bool IsSwapReq = false;
1997
1998 const auto *ST =
1999 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
2000 if (GR->isScalarOrVectorOfType(VReg: Call->ReturnRegister, TypeOpcode: SPIRV::OpTypeInt)) {
2001 if (!ST->canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product) &&
2002 !ST->isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
2003 report_fatal_error(reason: Twine(Call->Builtin->name()) +
2004 ": the builtin requires the following SPIR-V "
2005 "extension: SPV_KHR_integer_dot_product",
2006 gen_crash_diag: false);
2007 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2008 const SPIRV::IntegerDotProductBuiltin *IntDot =
2009 SPIRV::lookupIntegerDotProductBuiltin(Name: Builtin->name());
2010 if (IntDot) {
2011 OC = IntDot->Opcode;
2012 IsSwapReq = IntDot->IsSwapReq;
2013 } else if (IsVec) {
2014 // Handling "dot" and "dot_acc_sat" builtins which use vectors of
2015 // integers.
2016 LLVMContext &Ctx = MIRBuilder.getContext();
2017 SmallVector<StringRef, 10> TypeStrs;
2018 SPIRV::parseBuiltinTypeStr(BuiltinArgsTypeStrs&: TypeStrs, DemangledCall, Ctx);
2019 bool IsFirstSigned = TypeStrs[0].trim()[0] != 'u';
2020 bool IsSecondSigned = TypeStrs[1].trim()[0] != 'u';
2021
2022 if (Call->BuiltinName == "dot") {
2023 if (IsFirstSigned && IsSecondSigned)
2024 OC = SPIRV::OpSDot;
2025 else if (!IsFirstSigned && !IsSecondSigned)
2026 OC = SPIRV::OpUDot;
2027 else {
2028 OC = SPIRV::OpSUDot;
2029 if (!IsFirstSigned)
2030 IsSwapReq = true;
2031 }
2032 } else if (Call->BuiltinName == "dot_acc_sat") {
2033 if (IsFirstSigned && IsSecondSigned)
2034 OC = SPIRV::OpSDotAccSat;
2035 else if (!IsFirstSigned && !IsSecondSigned)
2036 OC = SPIRV::OpUDotAccSat;
2037 else {
2038 OC = SPIRV::OpSUDotAccSat;
2039 if (!IsFirstSigned)
2040 IsSwapReq = true;
2041 }
2042 }
2043 }
2044 }
2045
2046 MachineInstrBuilder MIB = MIRBuilder.buildInstr(Opcode: OC)
2047 .addDef(RegNo: Call->ReturnRegister)
2048 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
2049
2050 if (IsSwapReq) {
2051 MIB.addUse(RegNo: Call->Arguments[1]);
2052 MIB.addUse(RegNo: Call->Arguments[0]);
2053 // needed for dot_acc_sat* builtins
2054 for (size_t i = 2; i < Call->Arguments.size(); ++i)
2055 MIB.addUse(RegNo: Call->Arguments[i]);
2056 } else {
2057 for (size_t i = 0; i < Call->Arguments.size(); ++i)
2058 MIB.addUse(RegNo: Call->Arguments[i]);
2059 }
2060
2061 // Add Packed Vector Format for Integer dot product builtins if arguments are
2062 // scalar
2063 if (!IsVec && OC != SPIRV::OpFMulS)
2064 MIB.addImm(Val: SPIRV::PackedVectorFormat4x8Bit);
2065
2066 return true;
2067}
2068
2069static bool generateWaveInst(const SPIRV::IncomingCall *Call,
2070 MachineIRBuilder &MIRBuilder,
2071 SPIRVGlobalRegistry *GR) {
2072 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2073 SPIRV::BuiltIn::BuiltIn Value =
2074 SPIRV::lookupGetBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Value;
2075
2076 // For now, we only support a single Wave intrinsic with a single return type.
2077 assert(Call->ReturnType->getOpcode() == SPIRV::OpTypeInt);
2078 LLT LLType = LLT::scalar(SizeInBits: GR->getScalarOrVectorBitWidth(Type: Call->ReturnType));
2079
2080 return buildBuiltinVariableLoad(
2081 MIRBuilder, VariableType: Call->ReturnType, GR, BuiltinValue: Value, LLType, Reg: Call->ReturnRegister,
2082 /* isConst= */ false, /* LinkageType= */ LinkageTy: std::nullopt);
2083}
2084
2085// Build a SPIR-V instruction with struct return via sret pointer:
2086// Res = Opcode RetType Op1 Op2
2087// OpStore SRetReg Res
2088static void buildSRetInst(unsigned Opcode, Register SRetReg, Register Op1Reg,
2089 Register Op2Reg, SPIRVTypeInst RetType,
2090 MachineIRBuilder &MIRBuilder,
2091 SPIRVGlobalRegistry *GR) {
2092 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2093 Register ResReg = MRI->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
2094 if (const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(Reg: Op1Reg)) {
2095 MRI->setRegClass(Reg: ResReg, RC: DstRC);
2096 MRI->setType(VReg: ResReg, Ty: MRI->getType(Reg: Op1Reg));
2097 }
2098 GR->assignSPIRVTypeToVReg(Type: RetType, VReg: ResReg, MF: MIRBuilder.getMF());
2099 MIRBuilder.buildInstr(Opcode)
2100 .addDef(RegNo: ResReg)
2101 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetType))
2102 .addUse(RegNo: Op1Reg)
2103 .addUse(RegNo: Op2Reg);
2104 MIRBuilder.buildInstr(Opcode: SPIRV::OpStore).addUse(RegNo: SRetReg).addUse(RegNo: ResReg);
2105}
2106
2107// Find the pointee type of an sret pointer argument. A typed pointer gives us
2108// the type directly. An untyped one does not, so fall back to the element type
2109// we deduced for the matching IR argument, or null if there is nothing to fall
2110// back to.
2111static SPIRVTypeInst deduceSRetPointeeType(Register SRetReg,
2112 const Value *SRetArg,
2113 MachineIRBuilder &MIRBuilder,
2114 SPIRVGlobalRegistry *GR) {
2115 SPIRVTypeInst RetType = GR->getPointeeType(PtrType: GR->getSPIRVTypeForVReg(VReg: SRetReg));
2116 if (!RetType)
2117 if (Type *ElemTy = GR->findDeducedElementType(Val: SRetArg))
2118 RetType = GR->getOrCreateSPIRVType(
2119 Type: ElemTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: false);
2120 return RetType;
2121}
2122
2123// We expect a builtin
2124// Name(ptr sret([RetType]) %result, Type %operand1, Type %operand1)
2125// where %result is a pointer to where the result of the builtin execution
2126// is to be stored, and generate the following instructions:
2127// Res = Opcode RetType Operand1 Operand1
2128// OpStore RetVariable Res
2129static bool generateICarryBorrowInst(const SPIRV::IncomingCall *Call,
2130 MachineIRBuilder &MIRBuilder,
2131 SPIRVGlobalRegistry *GR,
2132 const CallBase &CB) {
2133 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2134 unsigned Opcode =
2135 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2136
2137 Register SRetReg = Call->Arguments[0];
2138 SPIRVTypeInst RetType =
2139 deduceSRetPointeeType(SRetReg, SRetArg: CB.getArgOperand(i: 0), MIRBuilder, GR);
2140 if (!RetType)
2141 report_fatal_error(reason: "The first parameter must be a pointer");
2142 if (RetType->getOpcode() != SPIRV::OpTypeStruct)
2143 report_fatal_error(reason: "Expected struct type result for the arithmetic with "
2144 "overflow builtins");
2145
2146 SPIRVTypeInst OpType1 = GR->getSPIRVTypeForVReg(VReg: Call->Arguments[1]);
2147 SPIRVTypeInst OpType2 = GR->getSPIRVTypeForVReg(VReg: Call->Arguments[2]);
2148 if (!OpType1 || !OpType2 || OpType1 != OpType2)
2149 report_fatal_error(reason: "Operands must have the same type");
2150 if (isVectorType(SPVTy: OpType1))
2151 switch (Opcode) {
2152 case SPIRV::OpIAddCarryS:
2153 Opcode = SPIRV::OpIAddCarryV;
2154 break;
2155 case SPIRV::OpISubBorrowS:
2156 Opcode = SPIRV::OpISubBorrowV;
2157 break;
2158 }
2159
2160 buildSRetInst(Opcode, SRetReg, Op1Reg: Call->Arguments[1], Op2Reg: Call->Arguments[2],
2161 RetType, MIRBuilder, GR);
2162 return true;
2163}
2164
2165// We expect a builtin in one of two forms:
2166//
2167// (1) sret convention (3 arguments):
2168// void Name(ptr sret([RetType]) %result, Type %operand1, Type %operand2)
2169// => Res = Opcode RetType Operand1 Operand2
2170// OpStore %result Res
2171//
2172// (2) direct return convention (2 arguments):
2173// RetType Name(Type %operand1, Type %operand2)
2174// => Res = Opcode RetType Operand1 Operand2
2175//
2176// RetType is a struct with two members of the same type as the operands.
2177static bool generateMulExtendedInst(const SPIRV::IncomingCall *Call,
2178 MachineIRBuilder &MIRBuilder,
2179 SPIRVGlobalRegistry *GR,
2180 const CallBase &CB) {
2181 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2182 unsigned Opcode =
2183 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2184 assert((Opcode == SPIRV::OpUMulExtended || Opcode == SPIRV::OpSMulExtended) &&
2185 "Expected OpUMulExtended or OpSMulExtended");
2186
2187 const bool IsSret =
2188 !Call->ReturnType || Call->ReturnType->getOpcode() == SPIRV::OpTypeVoid;
2189 Register Op1Reg = IsSret ? Call->Arguments[1] : Call->Arguments[0];
2190 Register Op2Reg = IsSret ? Call->Arguments[2] : Call->Arguments[1];
2191
2192 SPIRVTypeInst RetType = nullptr;
2193 if (IsSret) {
2194 Register SRetReg = Call->Arguments[0];
2195 RetType =
2196 deduceSRetPointeeType(SRetReg, SRetArg: CB.getArgOperand(i: 0), MIRBuilder, GR);
2197 if (!RetType)
2198 report_fatal_error(reason: "The first parameter must be a pointer");
2199 } else {
2200 RetType = Call->ReturnType;
2201 }
2202
2203 if (!RetType || RetType->getOpcode() != SPIRV::OpTypeStruct)
2204 report_fatal_error(reason: "Expected struct type result for the extended "
2205 "multiplication builtins");
2206 if (RetType->getNumOperands() != 3)
2207 report_fatal_error(reason: "Expected struct with exactly two members for the "
2208 "extended multiplication builtins");
2209 SPIRVTypeInst Member0Type =
2210 GR->getSPIRVTypeForVReg(VReg: RetType->getOperand(i: 1).getReg());
2211 SPIRVTypeInst Member1Type =
2212 GR->getSPIRVTypeForVReg(VReg: RetType->getOperand(i: 2).getReg());
2213 if (!Member0Type || !Member1Type || Member0Type != Member1Type)
2214 report_fatal_error(reason: "Both struct members must be the same type");
2215
2216 SPIRVTypeInst OpType1 = GR->getSPIRVTypeForVReg(VReg: Op1Reg);
2217 SPIRVTypeInst OpType2 = GR->getSPIRVTypeForVReg(VReg: Op2Reg);
2218 if (!OpType1 || !OpType2 || OpType1 != OpType2)
2219 report_fatal_error(reason: "Operands must have the same type");
2220 if (OpType1 != Member0Type)
2221 report_fatal_error(reason: "Operand type must match the struct member type");
2222
2223 if (IsSret) {
2224 buildSRetInst(Opcode, SRetReg: Call->Arguments[0], Op1Reg, Op2Reg, RetType,
2225 MIRBuilder, GR);
2226 } else {
2227 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2228 Register ResReg = Call->ReturnRegister;
2229 if (const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(Reg: Op1Reg)) {
2230 MRI->setRegClass(Reg: ResReg, RC: DstRC);
2231 }
2232 GR->assignSPIRVTypeToVReg(Type: RetType, VReg: ResReg, MF: MIRBuilder.getMF());
2233 MIRBuilder.buildInstr(Opcode)
2234 .addDef(RegNo: ResReg)
2235 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: RetType))
2236 .addUse(RegNo: Op1Reg)
2237 .addUse(RegNo: Op2Reg);
2238 }
2239 return true;
2240}
2241
2242static bool generateArithmeticInst(const SPIRV::IncomingCall *Call,
2243 MachineIRBuilder &MIRBuilder,
2244 SPIRVGlobalRegistry *GR) {
2245 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2246 unsigned Opcode =
2247 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2248
2249 auto MIB = MIRBuilder.buildInstr(Opcode)
2250 .addDef(RegNo: Call->ReturnRegister)
2251 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
2252 for (Register Arg : Call->Arguments)
2253 MIB.addUse(RegNo: Arg);
2254 return true;
2255}
2256
2257static bool generateGetQueryInst(const SPIRV::IncomingCall *Call,
2258 MachineIRBuilder &MIRBuilder,
2259 SPIRVGlobalRegistry *GR) {
2260 // Lookup the builtin record.
2261 SPIRV::BuiltIn::BuiltIn Value =
2262 SPIRV::lookupGetBuiltin(Name: Call->Builtin->name(), Set: Call->Builtin->Set)->Value;
2263 const bool IsDefaultOne = (Value == SPIRV::BuiltIn::GlobalSize ||
2264 Value == SPIRV::BuiltIn::NumWorkgroups ||
2265 Value == SPIRV::BuiltIn::WorkgroupSize ||
2266 Value == SPIRV::BuiltIn::EnqueuedWorkgroupSize);
2267 return genWorkgroupQuery(Call, MIRBuilder, GR, BuiltinValue: Value, DefaultValue: IsDefaultOne ? 1 : 0);
2268}
2269
2270static bool generateImageSizeQueryInst(const SPIRV::IncomingCall *Call,
2271 MachineIRBuilder &MIRBuilder,
2272 SPIRVGlobalRegistry *GR) {
2273 // Lookup the image size query component number in the TableGen records.
2274 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2275 uint32_t Component =
2276 SPIRV::lookupImageQueryBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Component;
2277 // Query result may either be a vector or a scalar. If return type is not a
2278 // vector, expect only a single size component. Otherwise get the number of
2279 // expected components.
2280 unsigned NumExpectedRetComponents =
2281 GR->getScalarOrVectorComponentCount(Type: Call->ReturnType);
2282 // Get the actual number of query result/size components.
2283 SPIRVTypeInst ImgType = GR->getSPIRVTypeForVReg(VReg: Call->Arguments[0]);
2284 unsigned NumActualRetComponents = getNumSizeComponents(imgType: ImgType);
2285 Register QueryResult = Call->ReturnRegister;
2286 SPIRVTypeInst QueryResultType = Call->ReturnType;
2287 if (NumExpectedRetComponents != NumActualRetComponents) {
2288 unsigned Bitwidth = Call->ReturnType->getOpcode() == SPIRV::OpTypeInt
2289 ? Call->ReturnType->getOperand(i: 1).getImm()
2290 : 32;
2291 QueryResult = MIRBuilder.getMRI()->createGenericVirtualRegister(
2292 Ty: LLT::fixed_vector(NumElements: NumActualRetComponents, ScalarSizeInBits: Bitwidth));
2293 MIRBuilder.getMRI()->setRegClass(Reg: QueryResult, RC: &SPIRV::viIDRegClass);
2294 SPIRVTypeInst IntTy = GR->getOrCreateSPIRVIntegerType(BitWidth: Bitwidth, MIRBuilder);
2295 QueryResultType = GR->getOrCreateSPIRVVectorType(
2296 BaseType: IntTy, NumElements: NumActualRetComponents, MIRBuilder, EmitIR: true);
2297 GR->assignSPIRVTypeToVReg(Type: QueryResultType, VReg: QueryResult, MF: MIRBuilder.getMF());
2298 }
2299 bool IsDimBuf = ImgType->getOperand(i: 2).getImm() == SPIRV::Dim::DIM_Buffer;
2300 bool IsMultisampled = ImgType->getOperand(i: 5).getImm() != 0;
2301 bool UseQuerySize = IsDimBuf || IsMultisampled;
2302 unsigned Opcode =
2303 UseQuerySize ? SPIRV::OpImageQuerySize : SPIRV::OpImageQuerySizeLod;
2304 auto MIB = MIRBuilder.buildInstr(Opcode)
2305 .addDef(RegNo: QueryResult)
2306 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: QueryResultType))
2307 .addUse(RegNo: Call->Arguments[0]);
2308 if (!UseQuerySize)
2309 MIB.addUse(RegNo: buildConstantIntReg32(Val: 0, MIRBuilder, GR)); // Lod id.
2310 if (NumExpectedRetComponents == NumActualRetComponents)
2311 return true;
2312 if (NumExpectedRetComponents == 1) {
2313 // Only 1 component is expected, build OpCompositeExtract instruction.
2314 unsigned ExtractedComposite =
2315 Component == 3 ? NumActualRetComponents - 1 : Component;
2316 assert(ExtractedComposite < NumActualRetComponents &&
2317 "Invalid composite index!");
2318 Register TypeReg = GR->getSPIRVTypeID(SpirvType: Call->ReturnType);
2319 SPIRVTypeInst NewType = nullptr;
2320 if (isVectorType(SPVTy: QueryResultType)) {
2321 NewType = GR->getScalarOrVectorComponentType(Type: QueryResultType);
2322 Register NewTypeReg = GR->getSPIRVTypeID(SpirvType: NewType);
2323 if (TypeReg != NewTypeReg)
2324 TypeReg = NewTypeReg;
2325 else
2326 NewType = nullptr;
2327 }
2328 MIRBuilder.buildInstr(Opcode: SPIRV::OpCompositeExtract)
2329 .addDef(RegNo: Call->ReturnRegister)
2330 .addUse(RegNo: TypeReg)
2331 .addUse(RegNo: QueryResult)
2332 .addImm(Val: ExtractedComposite);
2333 if (NewType)
2334 updateRegType(Reg: Call->ReturnRegister, Ty: nullptr, SpirvTy: NewType, GR, MIB&: MIRBuilder,
2335 MRI&: MIRBuilder.getMF().getRegInfo());
2336 } else {
2337 // More than 1 component is expected, fill a new vector.
2338 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpVectorShuffle)
2339 .addDef(RegNo: Call->ReturnRegister)
2340 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
2341 .addUse(RegNo: QueryResult)
2342 .addUse(RegNo: QueryResult);
2343 for (unsigned i = 0; i < NumExpectedRetComponents; ++i)
2344 MIB.addImm(Val: i < NumActualRetComponents ? i : 0xffffffff);
2345 }
2346 return true;
2347}
2348
2349static bool generateImageMiscQueryInst(const SPIRV::IncomingCall *Call,
2350 MachineIRBuilder &MIRBuilder,
2351 SPIRVGlobalRegistry *GR) {
2352 assert(Call->ReturnType->getOpcode() == SPIRV::OpTypeInt &&
2353 "Image samples query result must be of int type!");
2354
2355 // Lookup the instruction opcode in the TableGen records.
2356 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2357 unsigned Opcode =
2358 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2359
2360 Register Image = Call->Arguments[0];
2361 SPIRV::Dim::Dim ImageDimensionality = static_cast<SPIRV::Dim::Dim>(
2362 GR->getSPIRVTypeForVReg(VReg: Image)->getOperand(i: 2).getImm());
2363 (void)ImageDimensionality;
2364
2365 switch (Opcode) {
2366 case SPIRV::OpImageQuerySamples:
2367 assert(ImageDimensionality == SPIRV::Dim::DIM_2D &&
2368 "Image must be of 2D dimensionality");
2369 break;
2370 case SPIRV::OpImageQueryLevels:
2371 assert((ImageDimensionality == SPIRV::Dim::DIM_1D ||
2372 ImageDimensionality == SPIRV::Dim::DIM_2D ||
2373 ImageDimensionality == SPIRV::Dim::DIM_3D ||
2374 ImageDimensionality == SPIRV::Dim::DIM_Cube) &&
2375 "Image must be of 1D/2D/3D/Cube dimensionality");
2376 break;
2377 }
2378
2379 MIRBuilder.buildInstr(Opcode)
2380 .addDef(RegNo: Call->ReturnRegister)
2381 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
2382 .addUse(RegNo: Image);
2383 return true;
2384}
2385
2386// TODO: Move to TableGen.
2387static SPIRV::SamplerAddressingMode::SamplerAddressingMode
2388getSamplerAddressingModeFromBitmask(unsigned Bitmask) {
2389 switch (Bitmask & SPIRV::CLK_ADDRESS_MODE_MASK) {
2390 case SPIRV::CLK_ADDRESS_CLAMP:
2391 return SPIRV::SamplerAddressingMode::Clamp;
2392 case SPIRV::CLK_ADDRESS_CLAMP_TO_EDGE:
2393 return SPIRV::SamplerAddressingMode::ClampToEdge;
2394 case SPIRV::CLK_ADDRESS_REPEAT:
2395 return SPIRV::SamplerAddressingMode::Repeat;
2396 case SPIRV::CLK_ADDRESS_MIRRORED_REPEAT:
2397 return SPIRV::SamplerAddressingMode::RepeatMirrored;
2398 case SPIRV::CLK_ADDRESS_NONE:
2399 return SPIRV::SamplerAddressingMode::None;
2400 default:
2401 report_fatal_error(reason: "Unknown CL address mode");
2402 }
2403}
2404
2405static unsigned getSamplerParamFromBitmask(unsigned Bitmask) {
2406 return (Bitmask & SPIRV::CLK_NORMALIZED_COORDS_TRUE) ? 1 : 0;
2407}
2408
2409static SPIRV::SamplerFilterMode::SamplerFilterMode
2410getSamplerFilterModeFromBitmask(unsigned Bitmask) {
2411 if (Bitmask & SPIRV::CLK_FILTER_LINEAR)
2412 return SPIRV::SamplerFilterMode::Linear;
2413 if (Bitmask & SPIRV::CLK_FILTER_NEAREST)
2414 return SPIRV::SamplerFilterMode::Nearest;
2415 return SPIRV::SamplerFilterMode::Nearest;
2416}
2417
2418static bool generateReadImageInst(StringRef DemangledCall,
2419 const SPIRV::IncomingCall *Call,
2420 MachineIRBuilder &MIRBuilder,
2421 SPIRVGlobalRegistry *GR) {
2422 if (Call->isSpirvOp())
2423 return buildOpFromWrapper(MIRBuilder, Opcode: SPIRV::OpImageRead, Call,
2424 TypeReg: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
2425 Register Image = Call->Arguments[0];
2426 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2427 bool HasOclSampler = DemangledCall.contains_insensitive(Other: "ocl_sampler");
2428 bool HasMsaa = DemangledCall.contains_insensitive(Other: "msaa");
2429 if (HasOclSampler) {
2430 Register Sampler = Call->Arguments[1];
2431
2432 if (!GR->isScalarOfType(VReg: Sampler, TypeOpcode: SPIRV::OpTypeSampler) &&
2433 getDefInstrMaybeConstant(ConstReg&: Sampler, MRI)->getOperand(i: 1).isCImm()) {
2434 uint64_t SamplerMask = getIConstVal(ConstReg: Sampler, MRI);
2435 Sampler = GR->buildConstantSampler(
2436 Res: Register(), AddrMode: getSamplerAddressingModeFromBitmask(Bitmask: SamplerMask),
2437 Param: getSamplerParamFromBitmask(Bitmask: SamplerMask),
2438 FilerMode: getSamplerFilterModeFromBitmask(Bitmask: SamplerMask), MIRBuilder);
2439 }
2440 SPIRVTypeInst ImageType = GR->getSPIRVTypeForVReg(VReg: Image);
2441 SPIRVTypeInst SampledImageType =
2442 GR->getOrCreateOpTypeSampledImage(ImageType, MIRBuilder);
2443 Register SampledImage = MRI->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
2444
2445 MIRBuilder.buildInstr(Opcode: SPIRV::OpSampledImage)
2446 .addDef(RegNo: SampledImage)
2447 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: SampledImageType))
2448 .addUse(RegNo: Image)
2449 .addUse(RegNo: Sampler);
2450
2451 Register Lod = GR->buildConstantFP(Val: APFloat::getZero(Sem: APFloat::IEEEsingle()),
2452 MIRBuilder);
2453
2454 if (!isVectorType(SPVTy: Call->ReturnType)) {
2455 SPIRVTypeInst TempType =
2456 GR->getOrCreateSPIRVVectorType(BaseType: Call->ReturnType, NumElements: 4, MIRBuilder, EmitIR: true);
2457 Register TempRegister =
2458 MRI->createGenericVirtualRegister(Ty: GR->getRegType(SpvType: TempType));
2459 MRI->setRegClass(Reg: TempRegister, RC: GR->getRegClass(SpvType: TempType));
2460 GR->assignSPIRVTypeToVReg(Type: TempType, VReg: TempRegister, MF: MIRBuilder.getMF());
2461 MIRBuilder.buildInstr(Opcode: SPIRV::OpImageSampleExplicitLod)
2462 .addDef(RegNo: TempRegister)
2463 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: TempType))
2464 .addUse(RegNo: SampledImage)
2465 .addUse(RegNo: Call->Arguments[2]) // Coordinate.
2466 .addImm(Val: SPIRV::ImageOperand::Lod)
2467 .addUse(RegNo: Lod);
2468 MIRBuilder.buildInstr(Opcode: SPIRV::OpCompositeExtract)
2469 .addDef(RegNo: Call->ReturnRegister)
2470 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
2471 .addUse(RegNo: TempRegister)
2472 .addImm(Val: 0);
2473 } else {
2474 MIRBuilder.buildInstr(Opcode: SPIRV::OpImageSampleExplicitLod)
2475 .addDef(RegNo: Call->ReturnRegister)
2476 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
2477 .addUse(RegNo: SampledImage)
2478 .addUse(RegNo: Call->Arguments[2]) // Coordinate.
2479 .addImm(Val: SPIRV::ImageOperand::Lod)
2480 .addUse(RegNo: Lod);
2481 }
2482 } else if (HasMsaa) {
2483 MIRBuilder.buildInstr(Opcode: SPIRV::OpImageRead)
2484 .addDef(RegNo: Call->ReturnRegister)
2485 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
2486 .addUse(RegNo: Image)
2487 .addUse(RegNo: Call->Arguments[1]) // Coordinate.
2488 .addImm(Val: SPIRV::ImageOperand::Sample)
2489 .addUse(RegNo: Call->Arguments[2]);
2490 } else {
2491 MIRBuilder.buildInstr(Opcode: SPIRV::OpImageRead)
2492 .addDef(RegNo: Call->ReturnRegister)
2493 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
2494 .addUse(RegNo: Image)
2495 .addUse(RegNo: Call->Arguments[1]); // Coordinate.
2496 }
2497 return true;
2498}
2499
2500static bool generateWriteImageInst(const SPIRV::IncomingCall *Call,
2501 MachineIRBuilder &MIRBuilder,
2502 SPIRVGlobalRegistry *GR) {
2503 if (Call->isSpirvOp())
2504 return buildOpFromWrapper(MIRBuilder, Opcode: SPIRV::OpImageWrite, Call,
2505 TypeReg: Register(0));
2506 MIRBuilder.buildInstr(Opcode: SPIRV::OpImageWrite)
2507 .addUse(RegNo: Call->Arguments[0]) // Image.
2508 .addUse(RegNo: Call->Arguments[1]) // Coordinate.
2509 .addUse(RegNo: Call->Arguments[2]); // Texel.
2510 return true;
2511}
2512
2513static bool generateSampleImageInst(StringRef DemangledCall,
2514 const SPIRV::IncomingCall *Call,
2515 MachineIRBuilder &MIRBuilder,
2516 SPIRVGlobalRegistry *GR) {
2517 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2518 if (Call->Builtin->name().contains_insensitive(
2519 Other: "__translate_sampler_initializer")) {
2520 // Build sampler literal.
2521 uint64_t Bitmask = getIConstVal(ConstReg: Call->Arguments[0], MRI);
2522 Register Sampler = GR->buildConstantSampler(
2523 Res: Call->ReturnRegister, AddrMode: getSamplerAddressingModeFromBitmask(Bitmask),
2524 Param: getSamplerParamFromBitmask(Bitmask),
2525 FilerMode: getSamplerFilterModeFromBitmask(Bitmask), MIRBuilder);
2526 return Sampler.isValid();
2527 } else if (Call->Builtin->name().contains_insensitive(
2528 Other: "__spirv_SampledImage")) {
2529 // Create OpSampledImage.
2530 Register Image = Call->Arguments[0];
2531 SPIRVTypeInst ImageType = GR->getSPIRVTypeForVReg(VReg: Image);
2532 SPIRVTypeInst SampledImageType =
2533 GR->getOrCreateOpTypeSampledImage(ImageType, MIRBuilder);
2534 Register SampledImage =
2535 Call->ReturnRegister.isValid()
2536 ? Call->ReturnRegister
2537 : MRI->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
2538 MIRBuilder.buildInstr(Opcode: SPIRV::OpSampledImage)
2539 .addDef(RegNo: SampledImage)
2540 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: SampledImageType))
2541 .addUse(RegNo: Image)
2542 .addUse(RegNo: Call->Arguments[1]); // Sampler.
2543 return true;
2544 } else if (Call->Builtin->name().contains_insensitive(
2545 Other: "__spirv_ImageSampleExplicitLod")) {
2546 // Sample an image using an explicit level of detail.
2547 std::string ReturnType = DemangledCall.str();
2548 if (DemangledCall.contains(Other: "_R")) {
2549 ReturnType = ReturnType.substr(pos: ReturnType.find(s: "_R") + 2);
2550 ReturnType = ReturnType.substr(pos: 0, n: ReturnType.find(c: '('));
2551 }
2552 SPIRVTypeInst Type = Call->ReturnType
2553 ? Call->ReturnType
2554 : SPIRVTypeInst(GR->getOrCreateSPIRVTypeByName(
2555 TypeStr: ReturnType, MIRBuilder, EmitIR: true));
2556 if (!Type) {
2557 std::string DiagMsg =
2558 "Unable to recognize SPIRV type name: " + ReturnType;
2559 report_fatal_error(reason: DiagMsg.c_str());
2560 }
2561 MIRBuilder.buildInstr(Opcode: SPIRV::OpImageSampleExplicitLod)
2562 .addDef(RegNo: Call->ReturnRegister)
2563 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Type))
2564 .addUse(RegNo: Call->Arguments[0]) // Image.
2565 .addUse(RegNo: Call->Arguments[1]) // Coordinate.
2566 .addImm(Val: SPIRV::ImageOperand::Lod)
2567 .addUse(RegNo: Call->Arguments[3]);
2568 return true;
2569 }
2570 return false;
2571}
2572
2573static bool generateSelectInst(const SPIRV::IncomingCall *Call,
2574 MachineIRBuilder &MIRBuilder) {
2575 const MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2576 LLT ResTy = MRI->getType(Reg: Call->ReturnRegister);
2577 LLT CondTy = MRI->getType(Reg: Call->Arguments[0]);
2578 if (!ResTy.isVector() && CondTy.isVector())
2579 report_fatal_error(reason: "OpSelect with a scalar result requires a scalar "
2580 "boolean condition");
2581 MIRBuilder.buildSelect(Res: Call->ReturnRegister, Tst: Call->Arguments[0],
2582 Op0: Call->Arguments[1], Op1: Call->Arguments[2]);
2583 return true;
2584}
2585
2586static bool generateConstructInst(const SPIRV::IncomingCall *Call,
2587 MachineIRBuilder &MIRBuilder,
2588 SPIRVGlobalRegistry *GR) {
2589 createContinuedInstructions(MIRBuilder, Opcode: SPIRV::OpCompositeConstruct, MinWC: 3,
2590 ContinuedOpcode: SPIRV::OpCompositeConstructContinuedINTEL,
2591 Args: Call->Arguments, ReturnRegister: Call->ReturnRegister,
2592 TypeID: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
2593 return true;
2594}
2595
2596static bool generateCoopMatrInst(const SPIRV::IncomingCall *Call,
2597 MachineIRBuilder &MIRBuilder,
2598 SPIRVGlobalRegistry *GR) {
2599 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2600 unsigned Opcode =
2601 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2602 bool IsSet = Opcode != SPIRV::OpCooperativeMatrixStoreKHR &&
2603 Opcode != SPIRV::OpCooperativeMatrixStoreCheckedINTEL &&
2604 Opcode != SPIRV::OpCooperativeMatrixPrefetchINTEL;
2605 unsigned ArgSz = Call->Arguments.size();
2606 unsigned LiteralIdx = 0;
2607 switch (Opcode) {
2608 // Memory operand is optional and is literal.
2609 case SPIRV::OpCooperativeMatrixLoadKHR:
2610 LiteralIdx = ArgSz > 3 ? 3 : 0;
2611 break;
2612 case SPIRV::OpCooperativeMatrixStoreKHR:
2613 LiteralIdx = ArgSz > 4 ? 4 : 0;
2614 break;
2615 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2616 LiteralIdx = ArgSz > 7 ? 7 : 0;
2617 break;
2618 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2619 LiteralIdx = ArgSz > 8 ? 8 : 0;
2620 break;
2621 // Cooperative Matrix Operands operand is optional and is literal.
2622 case SPIRV::OpCooperativeMatrixMulAddKHR:
2623 LiteralIdx = ArgSz > 3 ? 3 : 0;
2624 break;
2625 };
2626
2627 SmallVector<uint32_t, 1> ImmArgs;
2628 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2629 if (Opcode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
2630 const uint32_t CacheLevel = getIConstVal(ConstReg: Call->Arguments[3], MRI);
2631 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpCooperativeMatrixPrefetchINTEL)
2632 .addUse(RegNo: Call->Arguments[0]) // pointer
2633 .addUse(RegNo: Call->Arguments[1]) // rows
2634 .addUse(RegNo: Call->Arguments[2]) // columns
2635 .addImm(Val: CacheLevel) // cache level
2636 .addUse(RegNo: Call->Arguments[4]); // memory layout
2637 if (ArgSz > 5)
2638 MIB.addUse(RegNo: Call->Arguments[5]); // stride
2639 if (ArgSz > 6) {
2640 const uint32_t MemOp = getIConstVal(ConstReg: Call->Arguments[6], MRI);
2641 MIB.addImm(Val: MemOp); // memory operand
2642 }
2643 return true;
2644 }
2645 if (LiteralIdx > 0)
2646 ImmArgs.push_back(Elt: getIConstVal(ConstReg: Call->Arguments[LiteralIdx], MRI));
2647 Register TypeReg = GR->getSPIRVTypeID(SpirvType: Call->ReturnType);
2648 if (Opcode == SPIRV::OpCooperativeMatrixLengthKHR) {
2649 SPIRVTypeInst CoopMatrType = GR->getSPIRVTypeForVReg(VReg: Call->Arguments[0]);
2650 if (!CoopMatrType)
2651 report_fatal_error(reason: "Can't find a register's type definition");
2652 MIRBuilder.buildInstr(Opcode)
2653 .addDef(RegNo: Call->ReturnRegister)
2654 .addUse(RegNo: TypeReg)
2655 .addUse(RegNo: CoopMatrType->getOperand(i: 0).getReg());
2656 return true;
2657 }
2658 return buildOpFromWrapper(MIRBuilder, Opcode, Call,
2659 TypeReg: IsSet ? TypeReg : Register(0), ImmArgs);
2660}
2661
2662static bool generateSpecConstantInst(const SPIRV::IncomingCall *Call,
2663 MachineIRBuilder &MIRBuilder,
2664 SPIRVGlobalRegistry *GR) {
2665 // Lookup the instruction opcode in the TableGen records.
2666 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2667 unsigned Opcode =
2668 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2669 const MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2670
2671 switch (Opcode) {
2672 case SPIRV::OpSpecConstant: {
2673 // Determine the constant MI.
2674 Register ConstRegister = Call->Arguments[1];
2675 const MachineInstr *Const = getDefInstrMaybeConstant(ConstReg&: ConstRegister, MRI);
2676 assert(Const &&
2677 (Const->getOpcode() == TargetOpcode::G_CONSTANT ||
2678 Const->getOpcode() == TargetOpcode::G_FCONSTANT) &&
2679 "Argument should be either an int or floating-point constant");
2680 // Determine the opcode and built the OpSpec MI.
2681 const MachineOperand &ConstOperand = Const->getOperand(i: 1);
2682 if (Call->ReturnType->getOpcode() == SPIRV::OpTypeBool) {
2683 assert(ConstOperand.isCImm() && "Int constant operand is expected");
2684 Opcode = ConstOperand.getCImm()->getValue().getZExtValue()
2685 ? SPIRV::OpSpecConstantTrue
2686 : SPIRV::OpSpecConstantFalse;
2687 }
2688 auto MIB = MIRBuilder.buildInstr(Opcode)
2689 .addDef(RegNo: Call->ReturnRegister)
2690 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
2691
2692 if (Call->ReturnType->getOpcode() != SPIRV::OpTypeBool) {
2693 if (Const->getOpcode() == TargetOpcode::G_CONSTANT)
2694 addNumImm(Imm: ConstOperand.getCImm()->getValue(), MIB);
2695 else
2696 addNumImm(Imm: ConstOperand.getFPImm()->getValueAPF().bitcastToAPInt(), MIB);
2697 }
2698 // Build the SpecID decoration.
2699 unsigned SpecId =
2700 static_cast<unsigned>(getIConstVal(ConstReg: Call->Arguments[0], MRI));
2701 buildOpDecorate(Reg: Call->ReturnRegister, MIRBuilder, Dec: SPIRV::Decoration::SpecId,
2702 DecArgs: {SpecId});
2703 return true;
2704 }
2705 case SPIRV::OpSpecConstantComposite: {
2706 createContinuedInstructions(MIRBuilder, Opcode, MinWC: 3,
2707 ContinuedOpcode: SPIRV::OpSpecConstantCompositeContinuedINTEL,
2708 Args: Call->Arguments, ReturnRegister: Call->ReturnRegister,
2709 TypeID: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
2710 return true;
2711 }
2712 default:
2713 return false;
2714 }
2715}
2716
2717static bool generateExtendedBitOpsInst(const SPIRV::IncomingCall *Call,
2718 MachineIRBuilder &MIRBuilder,
2719 SPIRVGlobalRegistry *GR) {
2720 // Lookup the instruction opcode in the TableGen records.
2721 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2722 unsigned Opcode =
2723 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2724
2725 return buildExtendedBitOpsInst(Call, Opcode, MIRBuilder, GR);
2726}
2727
2728static bool generateBindlessImageINTELInst(const SPIRV::IncomingCall *Call,
2729 MachineIRBuilder &MIRBuilder,
2730 SPIRVGlobalRegistry *GR) {
2731 // Lookup the instruction opcode in the TableGen records.
2732 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2733 unsigned Opcode =
2734 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2735
2736 return buildBindlessImageINTELInst(Call, Opcode, MIRBuilder, GR);
2737}
2738
2739static bool generateBlockingPipesInst(const SPIRV::IncomingCall *Call,
2740 MachineIRBuilder &MIRBuilder,
2741 SPIRVGlobalRegistry *GR) {
2742 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2743 unsigned Opcode =
2744 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2745 return buildOpFromWrapper(MIRBuilder, Opcode, Call, TypeReg: Register(0));
2746}
2747
2748static bool buildAPFixedPointInst(const SPIRV::IncomingCall *Call,
2749 unsigned Opcode, MachineIRBuilder &MIRBuilder,
2750 SPIRVGlobalRegistry *GR, const CallBase &CB) {
2751 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2752 SmallVector<uint32_t, 1> ImmArgs;
2753 Register InputReg = Call->Arguments[0];
2754 const Type *RetTy = GR->getTypeForSPIRVType(Ty: Call->ReturnType);
2755 bool IsSRet = RetTy->isVoidTy();
2756
2757 if (IsSRet) {
2758 const LLT ValTy = MRI->getType(Reg: InputReg);
2759 Register ActualRetValReg = MRI->createGenericVirtualRegister(Ty: ValTy);
2760 SPIRVTypeInst InstructionType =
2761 deduceSRetPointeeType(SRetReg: InputReg, SRetArg: CB.getArgOperand(i: 0), MIRBuilder, GR);
2762 InputReg = Call->Arguments[1];
2763 auto InputType = GR->getTypeForSPIRVType(Ty: GR->getSPIRVTypeForVReg(VReg: InputReg));
2764 Register PtrInputReg;
2765 if (InputType->getTypeID() == llvm::Type::TypeID::TypedPointerTyID) {
2766 LLT InputLLT = MRI->getType(Reg: InputReg);
2767 PtrInputReg = MRI->createGenericVirtualRegister(Ty: InputLLT);
2768 SPIRVTypeInst PtrType =
2769 GR->getPointeeType(PtrType: GR->getSPIRVTypeForVReg(VReg: InputReg));
2770 MachineMemOperand *MMO1 = MIRBuilder.getMF().getMachineMemOperand(
2771 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad,
2772 Size: InputLLT.getSizeInBytes(), BaseAlignment: Align(4));
2773 MIRBuilder.buildLoad(Res: PtrInputReg, Addr: InputReg, MMO&: *MMO1);
2774 MRI->setRegClass(Reg: PtrInputReg, RC: &SPIRV::iIDRegClass);
2775 GR->assignSPIRVTypeToVReg(Type: PtrType, VReg: PtrInputReg, MF: MIRBuilder.getMF());
2776 }
2777
2778 for (unsigned index = 2; index < 7; index++) {
2779 ImmArgs.push_back(Elt: getIConstVal(ConstReg: Call->Arguments[index], MRI));
2780 }
2781
2782 // Emit the instruction
2783 auto MIB = MIRBuilder.buildInstr(Opcode)
2784 .addDef(RegNo: ActualRetValReg)
2785 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: InstructionType));
2786 if (PtrInputReg)
2787 MIB.addUse(RegNo: PtrInputReg);
2788 else
2789 MIB.addUse(RegNo: InputReg);
2790
2791 for (uint32_t Imm : ImmArgs)
2792 MIB.addImm(Val: Imm);
2793 unsigned Size = ValTy.getSizeInBytes();
2794 // Store result to the pointer passed in Arg[0]
2795 MachineMemOperand *MMO = MIRBuilder.getMF().getMachineMemOperand(
2796 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOStore, Size, BaseAlignment: Align(4));
2797 MRI->setRegClass(Reg: ActualRetValReg, RC: &SPIRV::pIDRegClass);
2798 MIRBuilder.buildStore(Val: ActualRetValReg, Addr: Call->Arguments[0], MMO&: *MMO);
2799 return true;
2800 } else {
2801 for (unsigned index = 1; index < 6; index++)
2802 ImmArgs.push_back(Elt: getIConstVal(ConstReg: Call->Arguments[index], MRI));
2803
2804 return buildOpFromWrapper(MIRBuilder, Opcode, Call,
2805 TypeReg: GR->getSPIRVTypeID(SpirvType: Call->ReturnType), ImmArgs);
2806 }
2807}
2808
2809static bool generateAPFixedPointInst(const SPIRV::IncomingCall *Call,
2810 MachineIRBuilder &MIRBuilder,
2811 SPIRVGlobalRegistry *GR,
2812 const CallBase &CB) {
2813 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2814 unsigned Opcode =
2815 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2816
2817 return buildAPFixedPointInst(Call, Opcode, MIRBuilder, GR, CB);
2818}
2819
2820static bool
2821generateTernaryBitwiseFunctionINTELInst(const SPIRV::IncomingCall *Call,
2822 MachineIRBuilder &MIRBuilder,
2823 SPIRVGlobalRegistry *GR) {
2824 // Lookup the instruction opcode in the TableGen records.
2825 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2826 unsigned Opcode =
2827 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2828
2829 return buildTernaryBitwiseFunctionINTELInst(Call, Opcode, MIRBuilder, GR);
2830}
2831
2832static bool generateImageChannelDataTypeInst(const SPIRV::IncomingCall *Call,
2833 MachineIRBuilder &MIRBuilder,
2834 SPIRVGlobalRegistry *GR) {
2835 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2836 unsigned Opcode =
2837 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2838
2839 return buildImageChannelDataTypeInst(Call, Opcode, MIRBuilder, GR);
2840}
2841
2842static bool generate2DBlockIOINTELInst(const SPIRV::IncomingCall *Call,
2843 MachineIRBuilder &MIRBuilder,
2844 SPIRVGlobalRegistry *GR) {
2845 // Lookup the instruction opcode in the TableGen records.
2846 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2847 unsigned Opcode =
2848 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2849
2850 return build2DBlockIOINTELInst(Call, Opcode, MIRBuilder, GR);
2851}
2852
2853static bool generatePipeInst(const SPIRV::IncomingCall *Call,
2854 MachineIRBuilder &MIRBuilder,
2855 SPIRVGlobalRegistry *GR) {
2856 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2857 unsigned Opcode =
2858 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2859
2860 unsigned Scope = SPIRV::Scope::Workgroup;
2861 if (Builtin->name().contains(Other: "sub_group"))
2862 Scope = SPIRV::Scope::Subgroup;
2863
2864 return buildPipeInst(Call, Opcode, Scope, MIRBuilder, GR);
2865}
2866
2867static bool generatePredicatedLoadStoreInst(const SPIRV::IncomingCall *Call,
2868 MachineIRBuilder &MIRBuilder,
2869 SPIRVGlobalRegistry *GR) {
2870 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
2871 unsigned Opcode =
2872 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
2873
2874 bool IsSet = Opcode != SPIRV::OpPredicatedStoreINTEL;
2875 unsigned ArgSz = Call->Arguments.size();
2876 SmallVector<uint32_t, 1> ImmArgs;
2877 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2878 // Memory operand is optional and is literal.
2879 if (ArgSz > 3)
2880 ImmArgs.push_back(Elt: getIConstVal(ConstReg: Call->Arguments[/*Literal index*/ 3], MRI));
2881
2882 Register TypeReg = GR->getSPIRVTypeID(SpirvType: Call->ReturnType);
2883 return buildOpFromWrapper(MIRBuilder, Opcode, Call,
2884 TypeReg: IsSet ? TypeReg : Register(0), ImmArgs);
2885}
2886
2887static bool buildNDRange(const SPIRV::IncomingCall *Call,
2888 MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR,
2889 const CallBase &CB) {
2890 // The OpenCL ndrange_*D functions are overloaded and support 1D, 2D, and 3D
2891 // variants, accepting 1 to 3 arguments:
2892 // (global_work_size)
2893 // (global_work_size, local_work_size)
2894 // (global_work_offset, global_work_size, local_work_size)
2895 // Note: When all three arguments are provided, they are reordered compared
2896 // to the one- or two-argument form.
2897 //
2898 // The function may return data through an sret argument at position 0 (with
2899 // a void function return type). When present, all other argument indices are
2900 // adjusted accordingly.
2901 //
2902 // SPIR-V's OpBuildNDRange requires all three arguments (GlobalWorkSize,
2903 // LocalWorkSize, GlobalWorkOffset). For 1D kernels, the values are scalars;
2904 // for 2D/3D kernels, they are arrays of 2 or 3 elements. Missing arguments
2905 // default to zero.
2906 //
2907 // Calculate argument indices based on the number of arguments and presence
2908 // of sret:
2909 const unsigned NumCallArgs = Call->Arguments.size();
2910 const unsigned MaxCallArgs = Call->Builtin->MaxNumArgs;
2911 const unsigned IncorrectArgIdx = MaxCallArgs + 1;
2912
2913 const Type *RetTy = GR->getTypeForSPIRVType(Ty: Call->ReturnType);
2914 bool HasSRetArg = RetTy->isVoidTy();
2915
2916 const unsigned SRetArgIdx = HasSRetArg ? 0 : IncorrectArgIdx;
2917 const unsigned ArgBase = HasSRetArg ? 1 : 0;
2918 const unsigned MaxNDRangeArgs = 3;
2919 const unsigned NumNDRangeArgs = NumCallArgs - ArgBase;
2920
2921 const unsigned GlobalWorkSizeArgIdx =
2922 NumNDRangeArgs < MaxNDRangeArgs ? ArgBase : ArgBase + 1;
2923 const unsigned LocalWorkSizeArgIdx =
2924 (NumNDRangeArgs == 1)
2925 ? IncorrectArgIdx
2926 : (NumNDRangeArgs == MaxNDRangeArgs ? ArgBase + 2 : ArgBase + 1);
2927 const unsigned GlobalWorkOffsetArgIdx =
2928 NumNDRangeArgs == MaxNDRangeArgs ? ArgBase : IncorrectArgIdx;
2929
2930 // Each nd_range field is an array of <Dimension> integers matching the
2931 // address model width (32 or 64 bits).
2932 const unsigned AddressModelBits = GR->getPointerSize();
2933 assert(AddressModelBits == 64 || AddressModelBits == 32);
2934
2935 // The dimension is encoded in the function name as "ndrange_XD" where X is
2936 // 1, 2, or 3.
2937 unsigned Dimension = 0;
2938 Call->Builtin->name().substr(Start: 8, N: 1).getAsInteger(Radix: 10, Result&: Dimension);
2939 assert(Dimension <= 3 && Dimension >= 1);
2940
2941 // Determine the work size type based on the dimension. For missing arguments,
2942 // create a zero constant of the appropriate type.
2943 MachineFunction &MF = MIRBuilder.getMF();
2944 SPIRVTypeInst SpvFieldTy;
2945 Register ConstZero;
2946 if (Dimension == 1) {
2947 SpvFieldTy = GR->getSPIRVTypeForVReg(VReg: Call->Arguments[GlobalWorkSizeArgIdx]);
2948 assert(SpvFieldTy && SpvFieldTy->getOpcode() == SPIRV::OpTypeInt &&
2949 "Expected scalar integer type");
2950
2951 if (NumNDRangeArgs < MaxNDRangeArgs)
2952 ConstZero = GR->buildConstantInt(Val: 0, MIRBuilder, SpvType: SpvFieldTy, EmitIR: true);
2953 } else {
2954 Type *BaseTy =
2955 IntegerType::get(C&: MF.getFunction().getContext(), NumBits: AddressModelBits);
2956 Type *FieldTy = ArrayType::get(ElementType: BaseTy, NumElements: Dimension);
2957 SpvFieldTy = GR->getOrCreateSPIRVType(
2958 Type: FieldTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadOnly, EmitIR: true);
2959
2960 if (NumNDRangeArgs < MaxNDRangeArgs) {
2961 auto InsertIt = MIRBuilder.getInsertPt();
2962 MachineBasicBlock &MBB = MIRBuilder.getMBB();
2963 MachineInstr &InsertMI = (InsertIt != MBB.end()) ? *InsertIt : MBB.back();
2964 const SPIRVSubtarget &ST = cast<SPIRVSubtarget>(Val: MF.getSubtarget());
2965 ConstZero = GR->getOrCreateConstIntArray(Val: 0, Num: Dimension, I&: InsertMI,
2966 SpvType: SpvFieldTy, TII: *ST.getInstrInfo());
2967 }
2968 }
2969
2970 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2971
2972 auto CreateDataRegister = [&](unsigned Idx) -> Register {
2973 Register Reg = (Idx == IncorrectArgIdx) ? ConstZero : Call->Arguments[Idx];
2974
2975 if (GR->getSPIRVTypeForVReg(VReg: Reg) == SpvFieldTy) {
2976 // Already has the correct type.
2977 return Reg;
2978 }
2979
2980 assert(GR->getSPIRVTypeForVReg(Reg).isPointer() &&
2981 "Only pointer types are supported for loading values");
2982
2983 Register Ptr = Reg;
2984
2985 Reg = MRI->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
2986 GR->assignSPIRVTypeToVReg(Type: SpvFieldTy, VReg: Reg, MF);
2987
2988 MIRBuilder.buildInstr(Opcode: SPIRV::OpLoad)
2989 .addDef(RegNo: Reg)
2990 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: SpvFieldTy))
2991 .addUse(RegNo: Ptr);
2992 return Reg;
2993 };
2994
2995 Register GlobalWorkSize = CreateDataRegister(GlobalWorkSizeArgIdx);
2996 Register LocalWorkSize = CreateDataRegister(LocalWorkSizeArgIdx);
2997 Register GlobalWorkOffset = CreateDataRegister(GlobalWorkOffsetArgIdx);
2998
2999 if (!HasSRetArg) {
3000 return MIRBuilder.buildInstr(Opcode: SPIRV::OpBuildNDRange)
3001 .addDef(RegNo: Call->ReturnRegister)
3002 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
3003 .addUse(RegNo: GlobalWorkSize)
3004 .addUse(RegNo: LocalWorkSize)
3005 .addUse(RegNo: GlobalWorkOffset);
3006 }
3007
3008 // When sret is used, store nd_range struct through the pointer in the first
3009 // argument.
3010 Register SRetReg = Call->Arguments[SRetArgIdx];
3011 SPIRVTypeInst SRetType = deduceSRetPointeeType(
3012 SRetReg, SRetArg: CB.getArgOperand(i: SRetArgIdx), MIRBuilder, GR);
3013
3014 Register TmpReg = MRI->createVirtualRegister(RegClass: &SPIRV::iIDRegClass);
3015 GR->assignSPIRVTypeToVReg(Type: SRetType, VReg: TmpReg, MF);
3016
3017 MIRBuilder.buildInstr(Opcode: SPIRV::OpBuildNDRange)
3018 .addDef(RegNo: TmpReg)
3019 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: SRetType))
3020 .addUse(RegNo: GlobalWorkSize)
3021 .addUse(RegNo: LocalWorkSize)
3022 .addUse(RegNo: GlobalWorkOffset);
3023 return MIRBuilder.buildInstr(Opcode: SPIRV::OpStore)
3024 .addUse(RegNo: Call->Arguments[SRetArgIdx])
3025 .addUse(RegNo: TmpReg);
3026}
3027
3028static void buildKernelInvokeOperands(
3029 const SPIRV::IncomingCall *Call, unsigned InvokeIdx, unsigned ParamIdx,
3030 MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR, Register &InvokeReg,
3031 Register &ParamReg, Register &ParamSizeReg, Register &ParamAlignReg) {
3032 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
3033 const DataLayout &DL = MIRBuilder.getDataLayout();
3034
3035 // Bypass the addrspacecast so Invoke references the function's <id>.
3036 MachineInstr *InvokeGlobalMI =
3037 getBlockStructInstr(ParamReg: Call->Arguments[InvokeIdx], MRI);
3038 assert(InvokeGlobalMI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
3039 InvokeReg = InvokeGlobalMI->getOperand(i: 0).getReg();
3040 MRI->setRegClass(Reg: InvokeReg, RC: &SPIRV::pIDRegClass);
3041
3042 Register BlockLiteralReg = Call->Arguments[ParamIdx];
3043 const SPIRVTypeInst Int8Ty = GR->getOrCreateSPIRVIntegerType(BitWidth: 8, MIRBuilder);
3044 const SPIRVTypeInst Int8PtrGen = GR->getOrCreateSPIRVPointerType(
3045 BaseType: Int8Ty, MIRBuilder, SC: SPIRV::StorageClass::Generic);
3046 Type *PType = const_cast<Type *>(getBlockStructType(ParamReg: BlockLiteralReg, MRI));
3047
3048 ParamReg = createVirtualRegister(SpvType: Int8PtrGen, GR, MIRBuilder);
3049 MIRBuilder.buildInstr(Opcode: SPIRV::OpBitcast)
3050 .addDef(RegNo: ParamReg)
3051 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Int8PtrGen))
3052 .addUse(RegNo: BlockLiteralReg);
3053 // TODO: these numbers should be obtained from block literal structure.
3054 ParamSizeReg =
3055 buildConstantIntReg32(Val: DL.getTypeStoreSize(Ty: PType), MIRBuilder, GR);
3056 ParamAlignReg =
3057 buildConstantIntReg32(Val: DL.getPrefTypeAlign(Ty: PType).value(), MIRBuilder, GR);
3058}
3059
3060static bool buildKernelQuery(const SPIRV::IncomingCall *Call, unsigned Opcode,
3061 MachineIRBuilder &MIRBuilder,
3062 SPIRVGlobalRegistry *GR) {
3063 bool HasNDRange = Call->Builtin->name().contains(Other: "_ndrange_impl");
3064 unsigned InvokeIdx = HasNDRange ? 1 : 0;
3065 Register InvokeReg, ParamReg, ParamSizeReg, ParamAlignReg;
3066 buildKernelInvokeOperands(Call, InvokeIdx, ParamIdx: InvokeIdx + 1, MIRBuilder, GR,
3067 InvokeReg, ParamReg, ParamSizeReg, ParamAlignReg);
3068 auto MIB = MIRBuilder.buildInstr(Opcode)
3069 .addDef(RegNo: Call->ReturnRegister)
3070 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
3071 if (HasNDRange)
3072 MIB.addUse(RegNo: Call->Arguments[0]);
3073 MIB.addUse(RegNo: InvokeReg)
3074 .addUse(RegNo: ParamReg)
3075 .addUse(RegNo: ParamSizeReg)
3076 .addUse(RegNo: ParamAlignReg);
3077 return true;
3078}
3079
3080static bool buildEnqueueKernel(const SPIRV::IncomingCall *Call,
3081 MachineIRBuilder &MIRBuilder,
3082 SPIRVGlobalRegistry *GR) {
3083 // In this function there are three stages:
3084 // 1. prepare call indexes in order we expect them.
3085 // 2. process all arguments which requered preparation.
3086 // 3. create a SPIRV operator with arguments.
3087
3088 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
3089 const SPIRVTypeInst Int32Ty = GR->getOrCreateSPIRVIntegerType(BitWidth: 32, MIRBuilder);
3090
3091 // 1. prepare call indexes in order we expect them.
3092 // Based on clang sources, clang/lib/CodeGen/CGBuiltin.cpp, BIenqueue_kernel,
3093 // We expect 4 different layouts of call arguments:
3094 // 1) No events, no vargs: {Queue, Flags, Range, Kernel, Block};
3095 // 2) No events, varargs: {Queue, Flags, Range, Kernel, Block, NumElem,
3096 // ElemPtr};
3097 // 3) events, no varargs: {Queue, Flags, Range, NumEvents,
3098 // EventWaitList, EventRet, Kernel, Block};
3099 // 4) events, varargs: {Queue,
3100 // Flags, Range, NumEvents, EventWaitList, EventRet, Kernel, Block,
3101 // NumElem, ElemPtr};
3102 //
3103 // We also may expect __spirv_EnqueueKernel
3104
3105 bool IsSpirvOp = Call->isSpirvOp();
3106 bool HasEvents = Call->Builtin->name().contains(Other: "_events") || IsSpirvOp;
3107 bool HasVarArgs = Call->Builtin->name().contains(Other: "_varargs") || IsSpirvOp;
3108
3109 const unsigned NumArgs = Call->Arguments.size();
3110 const unsigned BaseArgIdx = 0;
3111 const unsigned IncorrectIdx = NumArgs + 1;
3112
3113 const unsigned QueueIdx = BaseArgIdx;
3114 const unsigned FlagsIdx = BaseArgIdx + 1;
3115 const unsigned NDRangeIdx = BaseArgIdx + 2;
3116 const unsigned NumEventsIdx = HasEvents ? BaseArgIdx + 3 : IncorrectIdx;
3117 const unsigned WaitEventsIdx = HasEvents ? BaseArgIdx + 4 : IncorrectIdx;
3118 const unsigned RetEventIdx = HasEvents ? BaseArgIdx + 5 : IncorrectIdx;
3119 const unsigned InvokeIdx = BaseArgIdx + 3 + (HasEvents ? 3 : 0);
3120 const unsigned ParamIdx = BaseArgIdx + 4 + (HasEvents ? 3 : 0);
3121 const unsigned LocalSizeNumElemIdx =
3122 HasVarArgs ? (BaseArgIdx + 5 + (HasEvents ? 3 : 0)) : IncorrectIdx;
3123 const unsigned LocalSizeElemPtrIdx =
3124 HasVarArgs ? (BaseArgIdx + 6 + (HasEvents ? 3 : 0)) : IncorrectIdx;
3125
3126 [[maybe_unused]] const unsigned LastArgIdx =
3127 (BaseArgIdx + 4 + (HasEvents ? 3 : 0) + (HasVarArgs ? 2 : 0));
3128 assert(LastArgIdx < NumArgs && "Incorrect number arguments");
3129
3130 // 2. Process all arguments which requered preparation.
3131 // 2.1 Events - use Call arguments, or use dummy nulls in case of absence of
3132 // events
3133
3134 auto BuildDeviceEventNullPtr = [&]() {
3135 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
3136 Type *DeviceEventTy = TargetExtType::get(Context&: Ctx, Name: "spirv.DeviceEvent");
3137 SPIRVTypeInst DeviceEventPtrTy = GR->getOrCreateSPIRVPointerType(
3138 BaseType: DeviceEventTy, MIRBuilder, SC: SPIRV::StorageClass::Generic);
3139 return GR->getOrCreateConstNullPtr(MIRBuilder, SpvType: DeviceEventPtrTy);
3140 };
3141
3142 Register NumEventsReg;
3143 Register WaitEventsReg;
3144 Register RetEventReg;
3145 if (HasEvents) {
3146 auto IsNullEvent = [&](Register R) {
3147 MachineInstr *Def = getDefInstrMaybeConstant(ConstReg&: R, MRI);
3148 return Def->getOpcode() == TargetOpcode::G_CONSTANT &&
3149 Def->getOperand(i: 1).getCImm()->isZero();
3150 };
3151
3152 NumEventsReg = Call->Arguments[NumEventsIdx];
3153 WaitEventsReg = Call->Arguments[WaitEventsIdx];
3154 RetEventReg = Call->Arguments[RetEventIdx];
3155 if (IsNullEvent(WaitEventsReg))
3156 WaitEventsReg = BuildDeviceEventNullPtr();
3157 if (IsNullEvent(RetEventReg))
3158 RetEventReg = BuildDeviceEventNullPtr();
3159 } else {
3160 NumEventsReg = buildConstantIntReg32(Val: 0, MIRBuilder, GR);
3161 Register NullPtr = BuildDeviceEventNullPtr();
3162 WaitEventsReg = NullPtr;
3163 RetEventReg = NullPtr;
3164 }
3165
3166 Register InvokeReg, ParamReg, ParamSizeReg, ParamAlignReg;
3167 buildKernelInvokeOperands(Call, InvokeIdx, ParamIdx, MIRBuilder, GR,
3168 InvokeReg, ParamReg, ParamSizeReg, ParamAlignReg);
3169
3170 // 2.3 Local Size Array
3171 SmallVector<Register, 16> LocalSizes;
3172 if (HasVarArgs) {
3173 Register LocalSizeNumElem = Call->Arguments[LocalSizeNumElemIdx];
3174 MachineInstr *LocalSizeNumElemMI = MRI->getUniqueVRegDef(Reg: LocalSizeNumElem);
3175 const MachineOperand &ConstOp = LocalSizeNumElemMI->getOperand(i: 1);
3176 assert(LocalSizeNumElemMI->getOpcode() == TargetOpcode::G_CONSTANT &&
3177 ConstOp.isCImm() && "Expected constant immediate");
3178 uint64_t NumElem = ConstOp.getCImm()->getValue().getZExtValue();
3179
3180 Register LocalSizeArrayReg = Call->Arguments[LocalSizeElemPtrIdx];
3181 SPIRVTypeInst PointeeTy =
3182 GR->getPointeeType(PtrType: GR->getSPIRVTypeForVReg(VReg: LocalSizeArrayReg));
3183 bool IsArrayPtr = PointeeTy && PointeeTy->getOpcode() == SPIRV::OpTypeArray;
3184
3185 for (unsigned i = 0; i < NumElem; ++i) {
3186 Register Reg = MRI->createVirtualRegister(RegClass: &SPIRV::pIDRegClass);
3187 auto GEPInst = MIRBuilder.buildIntrinsic(
3188 ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{Reg}, HasSideEffects: true, isConvergent: false);
3189 GEPInst
3190 .addImm(Val: 0) // In bound.
3191 .addUse(RegNo: LocalSizeArrayReg); // Base pointer.
3192 if (IsArrayPtr)
3193 GEPInst.addUse(RegNo: buildConstantIntReg32(Val: 0, MIRBuilder, GR));
3194 GEPInst.addUse(RegNo: buildConstantIntReg32(Val: i, MIRBuilder, GR));
3195 LocalSizes.push_back(Elt: Reg);
3196 }
3197 }
3198
3199 // 3. create a SPIRV operator with arguments.
3200 auto MIB = MIRBuilder.buildInstr(Opcode: SPIRV::OpEnqueueKernel)
3201 .addDef(RegNo: Call->ReturnRegister)
3202 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Int32Ty))
3203 .addUse(RegNo: Call->Arguments[QueueIdx])
3204 .addUse(RegNo: Call->Arguments[FlagsIdx])
3205 .addUse(RegNo: Call->Arguments[NDRangeIdx])
3206 .addUse(RegNo: NumEventsReg)
3207 .addUse(RegNo: WaitEventsReg)
3208 .addUse(RegNo: RetEventReg)
3209 .addUse(RegNo: InvokeReg)
3210 .addUse(RegNo: ParamReg)
3211 .addUse(RegNo: ParamSizeReg)
3212 .addUse(RegNo: ParamAlignReg);
3213 for (auto &LocalSize : LocalSizes)
3214 MIB.addUse(RegNo: LocalSize);
3215
3216 return true;
3217}
3218
3219static bool generateEnqueueInst(const SPIRV::IncomingCall *Call,
3220 MachineIRBuilder &MIRBuilder,
3221 SPIRVGlobalRegistry *GR, const CallBase &CB) {
3222 // Lookup the instruction opcode in the TableGen records.
3223 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
3224 unsigned Opcode =
3225 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
3226
3227 switch (Opcode) {
3228 case SPIRV::OpRetainEvent:
3229 case SPIRV::OpReleaseEvent:
3230 return MIRBuilder.buildInstr(Opcode).addUse(RegNo: Call->Arguments[0]);
3231 case SPIRV::OpCreateUserEvent:
3232 case SPIRV::OpGetDefaultQueue:
3233 return MIRBuilder.buildInstr(Opcode)
3234 .addDef(RegNo: Call->ReturnRegister)
3235 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
3236 case SPIRV::OpIsValidEvent:
3237 return MIRBuilder.buildInstr(Opcode)
3238 .addDef(RegNo: Call->ReturnRegister)
3239 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
3240 .addUse(RegNo: Call->Arguments[0]);
3241 case SPIRV::OpSetUserEventStatus:
3242 return MIRBuilder.buildInstr(Opcode)
3243 .addUse(RegNo: Call->Arguments[0])
3244 .addUse(RegNo: Call->Arguments[1]);
3245 case SPIRV::OpCaptureEventProfilingInfo:
3246 return MIRBuilder.buildInstr(Opcode)
3247 .addUse(RegNo: Call->Arguments[0])
3248 .addUse(RegNo: Call->Arguments[1])
3249 .addUse(RegNo: Call->Arguments[2]);
3250 case SPIRV::OpBuildNDRange:
3251 return buildNDRange(Call, MIRBuilder, GR, CB);
3252 case SPIRV::OpEnqueueKernel:
3253 return buildEnqueueKernel(Call, MIRBuilder, GR);
3254 case SPIRV::OpGetKernelNDrangeSubGroupCount:
3255 case SPIRV::OpGetKernelNDrangeMaxSubGroupSize:
3256 case SPIRV::OpGetKernelWorkGroupSize:
3257 case SPIRV::OpGetKernelPreferredWorkGroupSizeMultiple:
3258 return buildKernelQuery(Call, Opcode, MIRBuilder, GR);
3259 default:
3260 return false;
3261 }
3262}
3263
3264static bool generateAsyncCopy(const SPIRV::IncomingCall *Call,
3265 MachineIRBuilder &MIRBuilder,
3266 SPIRVGlobalRegistry *GR, const CallBase &CB) {
3267 // Lookup the instruction opcode in the TableGen records.
3268 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
3269 unsigned Opcode =
3270 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
3271
3272 bool IsSet = Opcode == SPIRV::OpGroupAsyncCopy;
3273 Register TypeReg = GR->getSPIRVTypeID(SpirvType: Call->ReturnType);
3274 if (Call->isSpirvOp())
3275 return buildOpFromWrapper(MIRBuilder, Opcode, Call,
3276 TypeReg: IsSet ? TypeReg : Register(0));
3277
3278 auto Scope = buildConstantIntReg32(Val: SPIRV::Scope::Workgroup, MIRBuilder, GR);
3279
3280 switch (Opcode) {
3281 case SPIRV::OpGroupAsyncCopy: {
3282 SPIRVTypeInst NewType =
3283 Call->ReturnType->getOpcode() == SPIRV::OpTypeEvent
3284 ? nullptr
3285 : GR->getOrCreateSPIRVTypeByName(TypeStr: "spirv.Event", MIRBuilder, EmitIR: true);
3286 Register TypeReg = GR->getSPIRVTypeID(SpirvType: NewType ? NewType : Call->ReturnType);
3287 unsigned NumArgs = Call->Arguments.size();
3288 Register EventReg = Call->Arguments[NumArgs - 1];
3289 SPIRVTypeInst EventType = GR->getSPIRVTypeForVReg(VReg: EventReg);
3290 if (!EventType || EventType->getOpcode() != SPIRV::OpTypeEvent) {
3291 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
3292 Register ConstReg = EventReg;
3293 MachineInstr *Def = getDefInstrMaybeConstant(ConstReg, MRI);
3294 SPIRVTypeInst EventPointeeType =
3295 EventType && EventType->getOpcode() == SPIRV::OpTypePointer
3296 ? GR->getPointeeType(PtrType: EventType)
3297 : nullptr;
3298 if (Def->getOpcode() == TargetOpcode::G_CONSTANT &&
3299 Def->getOperand(i: 1).getCImm()->isZero()) {
3300 // Only substitute a null Event for the "ptr null" idiom, not for a
3301 // real event value that just is not typed as OpTypeEvent yet.
3302 SPIRVTypeInst EventTy = NewType ? NewType
3303 : GR->getOrCreateSPIRVTypeByName(
3304 TypeStr: "spirv.Event", MIRBuilder, EmitIR: true);
3305 Register EventTyReg = GR->getSPIRVTypeID(SpirvType: EventTy);
3306 Register NullEventReg = createVirtualRegister(SpvType: EventTy, GR, MIRBuilder);
3307 MIRBuilder.buildInstr(Opcode: SPIRV::OpConstantNull)
3308 .addDef(RegNo: NullEventReg)
3309 .addUse(RegNo: EventTyReg);
3310 EventReg = NullEventReg;
3311 } else if (EventPointeeType &&
3312 EventPointeeType->getOpcode() == SPIRV::OpTypeEvent) {
3313 // Dereference: a real event can end up typed as pointer-to-Event
3314 // after round-tripping through a stack slot under the legacy
3315 // opaque-ptr ocl_event ABI.
3316 Register EventTyReg = GR->getSPIRVTypeID(SpirvType: EventPointeeType);
3317 Register LoadedReg =
3318 createVirtualRegister(SpvType: EventPointeeType, GR, MIRBuilder);
3319 MIRBuilder.buildInstr(Opcode: SPIRV::OpLoad)
3320 .addDef(RegNo: LoadedReg)
3321 .addUse(RegNo: EventTyReg)
3322 .addUse(RegNo: EventReg);
3323 EventReg = LoadedReg;
3324 }
3325 }
3326 Register NumElemReg = Call->Arguments[2];
3327
3328 // Untyped pointers use OpUntypedGroupAsyncCopyKHR, which adds an explicit
3329 // Element Num Bytes operand.
3330 SPIRVTypeInst DestPtrTy = GR->getSPIRVTypeForVReg(VReg: Call->Arguments[0]);
3331 bool IsUntyped =
3332 DestPtrTy && DestPtrTy->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
3333 SPIRVTypeInst SizeTy = GR->getSPIRVTypeForVReg(VReg: NumElemReg);
3334 Register StrideReg =
3335 Call->Arguments.size() > 4
3336 ? Call->Arguments[3]
3337 : (IsUntyped ? GR->buildConstantInt(Val: 1, MIRBuilder, SpvType: SizeTy,
3338 /*EmitIR=*/true)
3339 : buildConstantIntReg32(Val: 1, MIRBuilder, GR));
3340
3341 auto MIB = MIRBuilder
3342 .buildInstr(Opcode: IsUntyped ? SPIRV::OpUntypedGroupAsyncCopyKHR
3343 : SPIRV::OpGroupAsyncCopy)
3344 .addDef(RegNo: Call->ReturnRegister)
3345 .addUse(RegNo: TypeReg)
3346 .addUse(RegNo: Scope)
3347 .addUse(RegNo: Call->Arguments[0])
3348 .addUse(RegNo: Call->Arguments[1]);
3349 if (IsUntyped) {
3350 // Element Num Bytes from the deduced element type of dest (or source).
3351 unsigned ElemBytes = GR->getDeducedPointeeByteSize(PtrVal: CB.getArgOperand(i: 0));
3352 if (!ElemBytes)
3353 ElemBytes = GR->getDeducedPointeeByteSize(PtrVal: CB.getArgOperand(i: 1));
3354 if (!ElemBytes)
3355 report_fatal_error(reason: "Could not deduce the element type of an untyped "
3356 "async copy pointer argument");
3357 MIB.addUse(RegNo: GR->buildConstantInt(Val: ElemBytes, MIRBuilder, SpvType: SizeTy,
3358 /*EmitIR=*/true));
3359 }
3360 MIB.addUse(RegNo: NumElemReg);
3361 MIB.addUse(RegNo: StrideReg);
3362 MIB.addUse(RegNo: EventReg);
3363 if (NewType)
3364 updateRegType(Reg: Call->ReturnRegister, /*Ty=*/nullptr, SpirvTy: NewType, GR,
3365 MIB&: MIRBuilder, MRI&: MIRBuilder.getMF().getRegInfo());
3366 return true;
3367 }
3368 case SPIRV::OpGroupWaitEvents:
3369 return MIRBuilder.buildInstr(Opcode)
3370 .addUse(RegNo: Scope)
3371 .addUse(RegNo: Call->Arguments[0])
3372 .addUse(RegNo: Call->Arguments[1]);
3373 default:
3374 return false;
3375 }
3376}
3377
3378// Same type S/U/FConvert are invalid. OpSatConvert* are valid and must stay.
3379static bool foldNoOpConvert(unsigned Opcode, const SPIRV::IncomingCall *Call,
3380 MachineIRBuilder &MIRBuilder,
3381 SPIRVGlobalRegistry *GR) {
3382 if (Opcode != SPIRV::OpSConvert && Opcode != SPIRV::OpUConvert &&
3383 Opcode != SPIRV::OpFConvert)
3384 return false;
3385 if (Call->Arguments.size() != 1 ||
3386 GR->getSPIRVTypeForVReg(VReg: Call->Arguments[0]) != Call->ReturnType)
3387 return false;
3388 MIRBuilder.buildCopy(Res: Call->ReturnRegister, Op: Call->Arguments[0]);
3389 return true;
3390}
3391
3392static bool generateConvertInst(StringRef DemangledCall,
3393 const SPIRV::IncomingCall *Call,
3394 MachineIRBuilder &MIRBuilder,
3395 SPIRVGlobalRegistry *GR) {
3396 // Lookup the conversion builtin in the TableGen records.
3397 const SPIRV::ConvertBuiltin *Builtin =
3398 SPIRV::lookupConvertBuiltin(Name: Call->Builtin->name(), Set: Call->Builtin->Set);
3399
3400 if (!Builtin && Call->isSpirvOp()) {
3401 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
3402 unsigned Opcode =
3403 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
3404 if (foldNoOpConvert(Opcode, Call, MIRBuilder, GR))
3405 return true;
3406 return buildOpFromWrapper(MIRBuilder, Opcode, Call,
3407 TypeReg: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
3408 }
3409
3410 assert(Builtin && "Conversion builtin not found.");
3411
3412 std::string NeedExtMsg; // no errors if empty
3413 bool IsRightComponentsNumber = true; // check if input/output accepts vectors
3414 unsigned Opcode = SPIRV::OpNop;
3415 if (GR->isScalarOrVectorOfType(VReg: Call->Arguments[0], TypeOpcode: SPIRV::OpTypeInt)) {
3416 // Int -> ...
3417 bool IsSourceSigned =
3418 DemangledCall[DemangledCall.find_first_of(C: '(') + 1] != 'u';
3419 if (GR->isScalarOrVectorOfType(VReg: Call->ReturnRegister, TypeOpcode: SPIRV::OpTypeInt)) {
3420 // Int -> Int
3421 if (Builtin->IsSaturated)
3422 Opcode = Builtin->IsDestinationSigned ? SPIRV::OpSatConvertUToS
3423 : SPIRV::OpSatConvertSToU;
3424 else
3425 Opcode = IsSourceSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
3426 } else if (GR->isScalarOrVectorOfType(VReg: Call->ReturnRegister,
3427 TypeOpcode: SPIRV::OpTypeFloat)) {
3428 // Int -> Float
3429 if (Builtin->IsBfloat16) {
3430 const auto *ST = static_cast<const SPIRVSubtarget *>(
3431 &MIRBuilder.getMF().getSubtarget());
3432 if (!ST->canUseExtension(
3433 E: SPIRV::Extension::SPV_INTEL_bfloat16_conversion))
3434 NeedExtMsg = "SPV_INTEL_bfloat16_conversion";
3435 IsRightComponentsNumber =
3436 GR->getScalarOrVectorComponentCount(VReg: Call->Arguments[0]) ==
3437 GR->getScalarOrVectorComponentCount(VReg: Call->ReturnRegister);
3438 Opcode = SPIRV::OpConvertBF16ToFINTEL;
3439 } else {
3440 Opcode = IsSourceSigned ? SPIRV::OpConvertSToF : SPIRV::OpConvertUToF;
3441 }
3442 }
3443 } else if (GR->isScalarOrVectorOfType(VReg: Call->Arguments[0],
3444 TypeOpcode: SPIRV::OpTypeFloat)) {
3445 // Float -> ...
3446 if (GR->isScalarOrVectorOfType(VReg: Call->ReturnRegister, TypeOpcode: SPIRV::OpTypeInt)) {
3447 // Float -> Int
3448 if (Builtin->IsBfloat16) {
3449 const auto *ST = static_cast<const SPIRVSubtarget *>(
3450 &MIRBuilder.getMF().getSubtarget());
3451 if (!ST->canUseExtension(
3452 E: SPIRV::Extension::SPV_INTEL_bfloat16_conversion))
3453 NeedExtMsg = "SPV_INTEL_bfloat16_conversion";
3454 IsRightComponentsNumber =
3455 GR->getScalarOrVectorComponentCount(VReg: Call->Arguments[0]) ==
3456 GR->getScalarOrVectorComponentCount(VReg: Call->ReturnRegister);
3457 Opcode = SPIRV::OpConvertFToBF16INTEL;
3458 } else {
3459 Opcode = Builtin->IsDestinationSigned ? SPIRV::OpConvertFToS
3460 : SPIRV::OpConvertFToU;
3461 }
3462 } else if (GR->isScalarOrVectorOfType(VReg: Call->ReturnRegister,
3463 TypeOpcode: SPIRV::OpTypeFloat)) {
3464 if (Builtin->IsTF32) {
3465 const auto *ST = static_cast<const SPIRVSubtarget *>(
3466 &MIRBuilder.getMF().getSubtarget());
3467 if (!ST->canUseExtension(
3468 E: SPIRV::Extension::SPV_INTEL_tensor_float32_conversion))
3469 NeedExtMsg = "SPV_INTEL_tensor_float32_conversion";
3470 IsRightComponentsNumber =
3471 GR->getScalarOrVectorComponentCount(VReg: Call->Arguments[0]) ==
3472 GR->getScalarOrVectorComponentCount(VReg: Call->ReturnRegister);
3473 Opcode = SPIRV::OpRoundFToTF32INTEL;
3474 } else {
3475 // Float -> Float
3476 Opcode = SPIRV::OpFConvert;
3477 }
3478 }
3479 }
3480
3481 StringRef BuiltinName = SPIRV::getConvertBuiltinStr(Offset: Builtin->Name);
3482 if (!NeedExtMsg.empty()) {
3483 std::string DiagMsg = std::string(BuiltinName) +
3484 ": the builtin requires the following SPIR-V "
3485 "extension: " +
3486 NeedExtMsg;
3487 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
3488 }
3489 if (!IsRightComponentsNumber) {
3490 std::string DiagMsg =
3491 std::string(BuiltinName) +
3492 ": result and argument must have the same number of components";
3493 report_fatal_error(reason: DiagMsg.c_str(), gen_crash_diag: false);
3494 }
3495 assert(Opcode != SPIRV::OpNop &&
3496 "Conversion between the types not implemented!");
3497
3498 // Must run before the decorations below: a folded conversion has none.
3499 if (foldNoOpConvert(Opcode, Call, MIRBuilder, GR))
3500 return true;
3501
3502 if (Builtin->IsSaturated)
3503 buildOpDecorate(Reg: Call->ReturnRegister, MIRBuilder,
3504 Dec: SPIRV::Decoration::SaturatedConversion, DecArgs: {});
3505
3506 if (Builtin->IsRounded) {
3507 bool AnyTypeIsFloat =
3508 GR->isScalarOrVectorOfType(VReg: Call->ReturnRegister, TypeOpcode: SPIRV::OpTypeFloat) ||
3509 GR->isScalarOrVectorOfType(VReg: Call->Arguments[0], TypeOpcode: SPIRV::OpTypeFloat);
3510
3511 // Rounding mode decorations are only valid for floating point types.
3512 // Conversion builtins from integer to integer are equivalent to their
3513 // non-rounded counterparts.
3514 if (AnyTypeIsFloat) {
3515 buildOpDecorate(Reg: Call->ReturnRegister, MIRBuilder,
3516 Dec: SPIRV::Decoration::FPRoundingMode,
3517 DecArgs: {(unsigned)Builtin->RoundingMode});
3518 }
3519 }
3520
3521 MIRBuilder.buildInstr(Opcode)
3522 .addDef(RegNo: Call->ReturnRegister)
3523 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
3524 .addUse(RegNo: Call->Arguments[0]);
3525 return true;
3526}
3527
3528static bool generateVectorLoadStoreInst(const SPIRV::IncomingCall *Call,
3529 MachineIRBuilder &MIRBuilder,
3530 SPIRVGlobalRegistry *GR) {
3531 // Lookup the vector load/store builtin in the TableGen records.
3532 const SPIRV::VectorLoadStoreBuiltin *Builtin =
3533 SPIRV::lookupVectorLoadStoreBuiltin(Name: Call->Builtin->name(),
3534 Set: Call->Builtin->Set);
3535 // Build extended instruction.
3536 auto MIB =
3537 MIRBuilder.buildInstr(Opcode: SPIRV::OpExtInst)
3538 .addDef(RegNo: Call->ReturnRegister)
3539 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType))
3540 .addImm(Val: static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
3541 .addImm(Val: Builtin->Number);
3542 for (auto Argument : Call->Arguments)
3543 MIB.addUse(RegNo: Argument);
3544 StringRef BuiltinName = SPIRV::getVectorLoadStoreBuiltinStr(Offset: Builtin->Name);
3545 if (BuiltinName.contains(Other: "load") && Builtin->ElementCount > 1)
3546 MIB.addImm(Val: Builtin->ElementCount);
3547
3548 // Rounding mode should be passed as a last argument in the MI for builtins
3549 // like "vstorea_halfn_r".
3550 if (Builtin->IsRounded)
3551 MIB.addImm(Val: static_cast<uint32_t>(Builtin->RoundingMode));
3552 return true;
3553}
3554
3555static bool generateAFPInst(const SPIRV::IncomingCall *Call,
3556 MachineIRBuilder &MIRBuilder,
3557 SPIRVGlobalRegistry *GR) {
3558 const auto *Builtin = Call->Builtin;
3559 auto *MRI = MIRBuilder.getMRI();
3560 unsigned Opcode =
3561 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
3562 const Type *RetTy = GR->getTypeForSPIRVType(Ty: Call->ReturnType);
3563 bool IsVoid = RetTy->isVoidTy();
3564 auto MIB = MIRBuilder.buildInstr(Opcode);
3565 Register DestReg;
3566 if (IsVoid) {
3567 LLT PtrTy = MRI->getType(Reg: Call->Arguments[0]);
3568 DestReg = MRI->createGenericVirtualRegister(Ty: PtrTy);
3569 MRI->setRegClass(Reg: DestReg, RC: &SPIRV::pIDRegClass);
3570 SPIRVTypeInst PointeeTy =
3571 GR->getPointeeType(PtrType: GR->getSPIRVTypeForVReg(VReg: Call->Arguments[0]));
3572 MIB.addDef(RegNo: DestReg);
3573 MIB.addUse(RegNo: GR->getSPIRVTypeID(SpirvType: PointeeTy));
3574 } else {
3575 MIB.addDef(RegNo: Call->ReturnRegister);
3576 MIB.addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
3577 }
3578 for (unsigned i = IsVoid ? 1 : 0; i < Call->Arguments.size(); ++i) {
3579 Register Arg = Call->Arguments[i];
3580 MachineInstr *DefMI = MRI->getUniqueVRegDef(Reg: Arg);
3581 if (DefMI->getOpcode() == TargetOpcode::G_CONSTANT &&
3582 DefMI->getOperand(i: 1).isCImm()) {
3583 MIB.addImm(Val: getIConstVal(ConstReg: Arg, MRI));
3584 } else {
3585 MIB.addUse(RegNo: Arg);
3586 }
3587 }
3588 if (IsVoid) {
3589 LLT PtrTy = MRI->getType(Reg: Call->Arguments[0]);
3590 MachineMemOperand *MMO = MIRBuilder.getMF().getMachineMemOperand(
3591 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOStore,
3592 Size: PtrTy.getSizeInBytes(), BaseAlignment: Align(4));
3593 MIRBuilder.buildStore(Val: DestReg, Addr: Call->Arguments[0], MMO&: *MMO);
3594 }
3595 return true;
3596}
3597
3598static bool generateLoadStoreInst(const SPIRV::IncomingCall *Call,
3599 MachineIRBuilder &MIRBuilder,
3600 SPIRVGlobalRegistry *GR) {
3601 // Lookup the instruction opcode in the TableGen records.
3602 const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
3603 unsigned Opcode =
3604 SPIRV::lookupNativeBuiltin(Name: Builtin->name(), Set: Builtin->Set)->Opcode;
3605 bool IsLoad = Opcode == SPIRV::OpLoad;
3606 // Build the instruction.
3607 auto MIB = MIRBuilder.buildInstr(Opcode);
3608 if (IsLoad) {
3609 MIB.addDef(RegNo: Call->ReturnRegister);
3610 MIB.addUse(RegNo: GR->getSPIRVTypeID(SpirvType: Call->ReturnType));
3611 }
3612 // Add a pointer to the value to load/store.
3613 MIB.addUse(RegNo: Call->Arguments[0]);
3614 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
3615 // Add a value to store.
3616 if (!IsLoad)
3617 MIB.addUse(RegNo: Call->Arguments[1]);
3618 // Add optional memory attributes and an alignment.
3619 unsigned NumArgs = Call->Arguments.size();
3620 if ((IsLoad && NumArgs >= 2) || NumArgs >= 3)
3621 MIB.addImm(Val: getIConstVal(ConstReg: Call->Arguments[IsLoad ? 1 : 2], MRI));
3622 if ((IsLoad && NumArgs >= 3) || NumArgs >= 4)
3623 MIB.addImm(Val: getIConstVal(ConstReg: Call->Arguments[IsLoad ? 2 : 3], MRI));
3624 return true;
3625}
3626
3627namespace SPIRV {
3628// Try to find a builtin function attributes by a demangled function name and
3629// return a tuple <builtin group, op code, ext instruction number>, or a special
3630// tuple value <-1, 0, 0> if the builtin function is not found.
3631// Not all builtin functions are supported, only those with a ready-to-use op
3632// code or instruction number defined in TableGen.
3633// TODO: consider a major rework of mapping demangled calls into a builtin
3634// functions to unify search and decrease number of individual cases.
3635std::tuple<int, unsigned, unsigned>
3636mapBuiltinToOpcode(StringRef DemangledCall,
3637 SPIRV::InstructionSet::InstructionSet Set) {
3638 Register Reg;
3639 SmallVector<Register> Args;
3640 std::unique_ptr<const IncomingCall> Call =
3641 lookupBuiltin(DemangledCall, Set, ReturnRegister: Reg, ReturnType: nullptr, Arguments: Args);
3642 if (!Call)
3643 return std::make_tuple(args: -1, args: 0, args: 0);
3644
3645 switch (Call->Builtin->Group) {
3646 case SPIRV::Relational:
3647 case SPIRV::Atomic:
3648 case SPIRV::Barrier:
3649 case SPIRV::CastToPtr:
3650 case SPIRV::ImageMiscQuery:
3651 case SPIRV::SpecConstant:
3652 case SPIRV::Enqueue:
3653 case SPIRV::AsyncCopy:
3654 case SPIRV::LoadStore:
3655 case SPIRV::CoopMatr:
3656 case SPIRV::Arithmetic:
3657 if (const auto *R = SPIRV::lookupNativeBuiltin(Name: Call->Builtin->name(),
3658 Set: Call->Builtin->Set))
3659 return std::make_tuple(args: Call->Builtin->Group, args: R->Opcode, args: 0);
3660 break;
3661 case SPIRV::Extended:
3662 if (const auto *R = SPIRV::lookupExtendedBuiltin(Name: Call->Builtin->name(),
3663 Set: Call->Builtin->Set))
3664 return std::make_tuple(args: Call->Builtin->Group, args: 0, args: R->Number);
3665 break;
3666 case SPIRV::VectorLoadStore:
3667 if (const auto *R = SPIRV::lookupVectorLoadStoreBuiltin(
3668 Name: Call->Builtin->name(), Set: Call->Builtin->Set))
3669 return std::make_tuple(args: SPIRV::Extended, args: 0, args: R->Number);
3670 break;
3671 case SPIRV::Group:
3672 if (const auto *R = SPIRV::lookupGroupBuiltin(Name: Call->Builtin->name()))
3673 return std::make_tuple(args: Call->Builtin->Group, args: R->Opcode, args: 0);
3674 break;
3675 case SPIRV::AtomicFloating:
3676 if (const auto *R =
3677 SPIRV::lookupAtomicFloatingBuiltin(Name: Call->Builtin->name()))
3678 return std::make_tuple(args: Call->Builtin->Group, args: R->Opcode, args: 0);
3679 break;
3680 case SPIRV::IntelSubgroups:
3681 if (const auto *R =
3682 SPIRV::lookupIntelSubgroupsBuiltin(Name: Call->Builtin->name()))
3683 return std::make_tuple(args: Call->Builtin->Group, args: R->Opcode, args: 0);
3684 break;
3685 case SPIRV::GroupUniform:
3686 if (const auto *R = SPIRV::lookupGroupUniformBuiltin(Name: Call->Builtin->name()))
3687 return std::make_tuple(args: Call->Builtin->Group, args: R->Opcode, args: 0);
3688 break;
3689 case SPIRV::IntegerDot:
3690 if (const auto *R =
3691 SPIRV::lookupIntegerDotProductBuiltin(Name: Call->Builtin->name()))
3692 return std::make_tuple(args: Call->Builtin->Group, args: R->Opcode, args: 0);
3693 break;
3694 case SPIRV::WriteImage:
3695 return std::make_tuple(args: Call->Builtin->Group, args: SPIRV::OpImageWrite, args: 0);
3696 case SPIRV::Select:
3697 return std::make_tuple(args: Call->Builtin->Group, args: TargetOpcode::G_SELECT, args: 0);
3698 case SPIRV::Construct:
3699 return std::make_tuple(args: Call->Builtin->Group, args: SPIRV::OpCompositeConstruct,
3700 args: 0);
3701 case SPIRV::KernelClock:
3702 return std::make_tuple(args: Call->Builtin->Group, args: SPIRV::OpReadClockKHR, args: 0);
3703 default:
3704 return std::make_tuple(args: -1, args: 0, args: 0);
3705 }
3706 return std::make_tuple(args: -1, args: 0, args: 0);
3707}
3708
3709bool isBuiltin(StringRef DemangledCall,
3710 SPIRV::InstructionSet::InstructionSet Set) {
3711 SmallVector<Register> Args;
3712 return lookupBuiltin(DemangledCall, Set, ReturnRegister: Register(), ReturnType: nullptr, Arguments: Args) !=
3713 nullptr;
3714}
3715
3716/// Checks that scalar/vector numeric arguments of \p Call match the types
3717/// implied by their mangling in \p DemangledCall. Pointers and opaque
3718/// builtin types (images, samplers, pipes, etc.) are not validated here, as
3719/// mangling does not enforce their exact spelling.
3720///
3721/// \returns false if a numeric argument's SPIR-V type disagrees with the
3722/// type implied by the mangled name, true otherwise.
3723static bool demangledArgTypesMatchIR(const SPIRV::IncomingCall *Call,
3724 StringRef DemangledCall,
3725 SPIRVGlobalRegistry *GR, LLVMContext &Ctx,
3726 const CallBase &CB) {
3727 if (Call->isSpirvOp())
3728 return true;
3729
3730 SmallVector<StringRef, 10> ArgTypeStrs;
3731 if (!SPIRV::parseBuiltinTypeStr(BuiltinArgsTypeStrs&: ArgTypeStrs, DemangledCall, Ctx))
3732 return true;
3733
3734 unsigned ArgBase = CB.hasStructRetAttr() ? 1 : 0;
3735 if (Call->Arguments.size() < ArgBase)
3736 return true;
3737 unsigned NumMangledArgs = Call->Arguments.size() - ArgBase;
3738 unsigned NumArgsToCheck =
3739 std::min<unsigned>(a: NumMangledArgs, b: ArgTypeStrs.size());
3740 for (unsigned ArgIdx = 0; ArgIdx < NumArgsToCheck; ++ArgIdx) {
3741 StringRef ArgTypeStr = ArgTypeStrs[ArgIdx].trim();
3742 // Opaque/builtin OpenCL and SPIR-V types (images, samplers, pipes,
3743 // reserve_id, etc.) are not validated here, as mangling does not enforce
3744 // their exact spelling, and some builtin type names have no TableGen
3745 // record and would otherwise abort compilation when parsed.
3746 if (hasBuiltinTypePrefix(Name: ArgTypeStr))
3747 continue;
3748
3749 Type *ExpectedType = SPIRV::parseBuiltinCallArgumentType(TypeStr: ArgTypeStr, Ctx);
3750 if (!ExpectedType || ExpectedType->isVoidTy() ||
3751 ExpectedType->isPointerTy() || ExpectedType->isTargetExtTy())
3752 continue;
3753
3754 SPIRVTypeInst ArgType =
3755 GR->getSPIRVTypeForVReg(VReg: Call->Arguments[ArgIdx + ArgBase]);
3756 if (!ArgType)
3757 continue;
3758 unsigned ArgTypeOpcode = ArgType->getOpcode();
3759 if (ArgTypeOpcode != SPIRV::OpTypeInt &&
3760 ArgTypeOpcode != SPIRV::OpTypeFloat &&
3761 ArgTypeOpcode != SPIRV::OpTypeBool &&
3762 ArgTypeOpcode != SPIRV::OpTypeVector)
3763 continue;
3764
3765 auto *ExpectedVecType = dyn_cast<VectorType>(Val: ExpectedType);
3766 Type *ExpectedScalarType =
3767 ExpectedVecType ? ExpectedVecType->getElementType() : ExpectedType;
3768 SPIRVTypeInst ArgScalarType = GR->getScalarOrVectorComponentType(Type: ArgType);
3769 if (!ArgScalarType)
3770 continue;
3771
3772 bool ExpectedIsInt = ExpectedScalarType->isIntegerTy();
3773 unsigned ArgOpcode = ArgScalarType->getOpcode();
3774 bool ArgIsInt =
3775 ArgOpcode == SPIRV::OpTypeInt || ArgOpcode == SPIRV::OpTypeBool;
3776
3777 if (ExpectedIsInt != ArgIsInt)
3778 return false;
3779
3780 unsigned ExpectedElts =
3781 ExpectedVecType ? ExpectedVecType->getElementCount().getFixedValue()
3782 : 1;
3783 if (ExpectedElts != GR->getScalarOrVectorComponentCount(Type: ArgType))
3784 return false;
3785 }
3786 return true;
3787}
3788
3789std::optional<bool> lowerBuiltin(StringRef DemangledCall,
3790 SPIRV::InstructionSet::InstructionSet Set,
3791 MachineIRBuilder &MIRBuilder,
3792 const Register OrigRet, const Type *OrigRetTy,
3793 const SmallVectorImpl<Register> &Args,
3794 SPIRVGlobalRegistry *GR, const CallBase &CB) {
3795 LLVM_DEBUG(dbgs() << "Lowering builtin call: " << DemangledCall << "\n");
3796
3797 // Lookup the builtin in the TableGen records.
3798 SPIRVTypeInst SpvType = GR->getSPIRVTypeForVReg(VReg: OrigRet);
3799 assert(SpvType && "Inconsistent return register: expected valid type info");
3800 std::unique_ptr<const IncomingCall> Call =
3801 lookupBuiltin(DemangledCall, Set, ReturnRegister: OrigRet, ReturnType: SpvType, Arguments: Args);
3802
3803 if (!Call) {
3804 LLVM_DEBUG(dbgs() << "Builtin record was not found!\n");
3805 return std::nullopt;
3806 }
3807
3808 // Check if the provided args meet the builtin requirements. If not, treat
3809 // the call as a regular function call rather than crashing.
3810 if (Args.size() < Call->Builtin->MinNumArgs) {
3811 LLVM_DEBUG(dbgs() << "Too few arguments for builtin " << DemangledCall
3812 << ": expected at least " << Call->Builtin->MinNumArgs
3813 << ", got " << Args.size()
3814 << "; treating as a normal function\n");
3815 return std::nullopt;
3816 }
3817 if (Call->Builtin->MaxNumArgs && Args.size() > Call->Builtin->MaxNumArgs) {
3818 LLVM_DEBUG(dbgs() << "Too many arguments for builtin " << DemangledCall
3819 << ": expected at most " << Call->Builtin->MaxNumArgs
3820 << ", got " << Args.size()
3821 << "; treating as a normal function\n");
3822 return std::nullopt;
3823 }
3824
3825 // Check that argument types match what the mangling implies. If not
3826 // (e.g. broken mangling), treat the call as a regular function call
3827 // rather than crashing.
3828 if (!demangledArgTypesMatchIR(Call: Call.get(), DemangledCall, GR,
3829 Ctx&: MIRBuilder.getContext(), CB)) {
3830 LLVM_DEBUG(dbgs() << "Argument types do not match mangled types for "
3831 << "builtin " << DemangledCall
3832 << "; treating as a normal function\n");
3833 return std::nullopt;
3834 }
3835
3836 // Match the builtin with implementation based on the grouping.
3837 switch (Call->Builtin->Group) {
3838 case SPIRV::Extended:
3839 return generateExtInst(Call: Call.get(), MIRBuilder, GR, CB);
3840 case SPIRV::Relational:
3841 return generateRelationalInst(Call: Call.get(), MIRBuilder, GR);
3842 case SPIRV::Group:
3843 return generateGroupInst(Call: Call.get(), MIRBuilder, GR);
3844 case SPIRV::Variable:
3845 return generateBuiltinVar(Call: Call.get(), MIRBuilder, GR);
3846 case SPIRV::Atomic:
3847 return generateAtomicInst(Call: Call.get(), MIRBuilder, GR);
3848 case SPIRV::AtomicFloating:
3849 return generateAtomicFloatingInst(Call: Call.get(), MIRBuilder, GR);
3850 case SPIRV::Barrier:
3851 return generateBarrierInst(Call: Call.get(), MIRBuilder, GR);
3852 case SPIRV::CastToPtr:
3853 return generateCastToPtrInst(Call: Call.get(), MIRBuilder, GR);
3854 case SPIRV::Dot:
3855 case SPIRV::IntegerDot:
3856 return generateDotOrFMulInst(DemangledCall, Call: Call.get(), MIRBuilder, GR);
3857 case SPIRV::Wave:
3858 return generateWaveInst(Call: Call.get(), MIRBuilder, GR);
3859 case SPIRV::ICarryBorrow:
3860 return generateICarryBorrowInst(Call: Call.get(), MIRBuilder, GR, CB);
3861 case SPIRV::MulExtended:
3862 return generateMulExtendedInst(Call: Call.get(), MIRBuilder, GR, CB);
3863 case SPIRV::Arithmetic:
3864 return generateArithmeticInst(Call: Call.get(), MIRBuilder, GR);
3865 case SPIRV::GetQuery:
3866 return generateGetQueryInst(Call: Call.get(), MIRBuilder, GR);
3867 case SPIRV::ImageSizeQuery:
3868 return generateImageSizeQueryInst(Call: Call.get(), MIRBuilder, GR);
3869 case SPIRV::ImageMiscQuery:
3870 return generateImageMiscQueryInst(Call: Call.get(), MIRBuilder, GR);
3871 case SPIRV::ReadImage:
3872 return generateReadImageInst(DemangledCall, Call: Call.get(), MIRBuilder, GR);
3873 case SPIRV::WriteImage:
3874 return generateWriteImageInst(Call: Call.get(), MIRBuilder, GR);
3875 case SPIRV::SampleImage:
3876 return generateSampleImageInst(DemangledCall, Call: Call.get(), MIRBuilder, GR);
3877 case SPIRV::Select:
3878 return generateSelectInst(Call: Call.get(), MIRBuilder);
3879 case SPIRV::Construct:
3880 return generateConstructInst(Call: Call.get(), MIRBuilder, GR);
3881 case SPIRV::SpecConstant:
3882 return generateSpecConstantInst(Call: Call.get(), MIRBuilder, GR);
3883 case SPIRV::Enqueue:
3884 return generateEnqueueInst(Call: Call.get(), MIRBuilder, GR, CB);
3885 case SPIRV::AsyncCopy:
3886 return generateAsyncCopy(Call: Call.get(), MIRBuilder, GR, CB);
3887 case SPIRV::Convert:
3888 return generateConvertInst(DemangledCall, Call: Call.get(), MIRBuilder, GR);
3889 case SPIRV::VectorLoadStore:
3890 return generateVectorLoadStoreInst(Call: Call.get(), MIRBuilder, GR);
3891 case SPIRV::LoadStore:
3892 return generateLoadStoreInst(Call: Call.get(), MIRBuilder, GR);
3893 case SPIRV::IntelSubgroups:
3894 return generateIntelSubgroupsInst(Call: Call.get(), MIRBuilder, GR);
3895 case SPIRV::GroupUniform:
3896 return generateGroupUniformInst(Call: Call.get(), MIRBuilder, GR);
3897 case SPIRV::KernelClock:
3898 return generateKernelClockInst(Call: Call.get(), MIRBuilder, GR);
3899 case SPIRV::CoopMatr:
3900 return generateCoopMatrInst(Call: Call.get(), MIRBuilder, GR);
3901 case SPIRV::ExtendedBitOps:
3902 return generateExtendedBitOpsInst(Call: Call.get(), MIRBuilder, GR);
3903 case SPIRV::BindlessINTEL:
3904 return generateBindlessImageINTELInst(Call: Call.get(), MIRBuilder, GR);
3905 case SPIRV::TernaryBitwiseINTEL:
3906 return generateTernaryBitwiseFunctionINTELInst(Call: Call.get(), MIRBuilder, GR);
3907 case SPIRV::Block2DLoadStore:
3908 return generate2DBlockIOINTELInst(Call: Call.get(), MIRBuilder, GR);
3909 case SPIRV::Pipe:
3910 return generatePipeInst(Call: Call.get(), MIRBuilder, GR);
3911 case SPIRV::PredicatedLoadStore:
3912 return generatePredicatedLoadStoreInst(Call: Call.get(), MIRBuilder, GR);
3913 case SPIRV::BlockingPipes:
3914 return generateBlockingPipesInst(Call: Call.get(), MIRBuilder, GR);
3915 case SPIRV::ArbitraryPrecisionFixedPoint:
3916 return generateAPFixedPointInst(Call: Call.get(), MIRBuilder, GR, CB);
3917 case SPIRV::ImageChannelDataTypes:
3918 return generateImageChannelDataTypeInst(Call: Call.get(), MIRBuilder, GR);
3919 case SPIRV::ArbitraryFloatingPoint:
3920 return generateAFPInst(Call: Call.get(), MIRBuilder, GR);
3921 }
3922 return false;
3923}
3924
3925Type *parseBuiltinCallArgumentType(StringRef TypeStr, LLVMContext &Ctx) {
3926 // Parse strings representing OpenCL builtin types.
3927 if (hasBuiltinTypePrefix(Name: TypeStr)) {
3928 // OpenCL builtin types in demangled call strings have the following format:
3929 // e.g. ocl_image2d_ro
3930 [[maybe_unused]] bool IsOCLBuiltinType = TypeStr.consume_front(Prefix: "ocl_");
3931 assert(IsOCLBuiltinType && "Invalid OpenCL builtin prefix");
3932
3933 // Check if this is pointer to a builtin type and not just pointer
3934 // representing a builtin type. In case it is a pointer to builtin type,
3935 // this will require additional handling in the method calling
3936 // parseBuiltinCallArgumentBaseType(...) as this function only retrieves the
3937 // base types.
3938 if (TypeStr.ends_with(Suffix: "*"))
3939 TypeStr = TypeStr.slice(Start: 0, End: TypeStr.find_first_of(Chars: " *"));
3940
3941 return parseBuiltinTypeNameToTargetExtType(TypeName: "opencl." + TypeStr.str() + "_t",
3942 Context&: Ctx);
3943 }
3944
3945 // Parse type name in either "typeN" or "type vector[N]" format, where
3946 // N is the number of elements of the vector.
3947 Type *BaseType;
3948 unsigned VecElts = 0;
3949
3950 BaseType = parseBasicTypeName(TypeName&: TypeStr, Ctx);
3951 if (!BaseType)
3952 // Unable to recognize SPIRV type name.
3953 return nullptr;
3954
3955 // Handle "typeN*" or "type vector[N]*".
3956 TypeStr.consume_back(Suffix: "*");
3957
3958 if (TypeStr.consume_front(Prefix: " vector["))
3959 TypeStr = TypeStr.substr(Start: 0, N: TypeStr.find(C: ']'));
3960
3961 TypeStr.getAsInteger(Radix: 10, Result&: VecElts);
3962 if (VecElts > 0)
3963 BaseType = VectorType::get(
3964 ElementType: BaseType->isVoidTy() ? Type::getInt8Ty(C&: Ctx) : BaseType, NumElements: VecElts, Scalable: false);
3965
3966 return BaseType;
3967}
3968
3969bool parseBuiltinTypeStr(SmallVector<StringRef, 10> &BuiltinArgsTypeStrs,
3970 StringRef DemangledCall, LLVMContext &Ctx) {
3971 auto Pos1 = DemangledCall.find(C: '(');
3972 if (Pos1 == StringRef::npos)
3973 return false;
3974 auto Pos2 = DemangledCall.find(C: ')');
3975 if (Pos2 == StringRef::npos || Pos1 > Pos2)
3976 return false;
3977 DemangledCall.slice(Start: Pos1 + 1, End: Pos2)
3978 .split(A&: BuiltinArgsTypeStrs, Separator: ',', MaxSplit: -1, KeepEmpty: false);
3979 return true;
3980}
3981
3982Type *parseBuiltinCallArgumentBaseType(StringRef DemangledCall, unsigned ArgIdx,
3983 LLVMContext &Ctx) {
3984 SmallVector<StringRef, 10> BuiltinArgsTypeStrs;
3985 parseBuiltinTypeStr(BuiltinArgsTypeStrs, DemangledCall, Ctx);
3986 if (ArgIdx >= BuiltinArgsTypeStrs.size())
3987 return nullptr;
3988 StringRef TypeStr = BuiltinArgsTypeStrs[ArgIdx].trim();
3989 return parseBuiltinCallArgumentType(TypeStr, Ctx);
3990}
3991
3992struct BuiltinType {
3993 StringTable::Offset Name;
3994 uint32_t Opcode;
3995};
3996
3997#define GET_BuiltinTypes_DECL
3998#define GET_BuiltinTypes_IMPL
3999
4000struct OpenCLType {
4001 StringTable::Offset Name;
4002 StringTable::Offset SpirvTypeLiteral;
4003};
4004
4005#define GET_OpenCLTypes_DECL
4006#define GET_OpenCLTypes_IMPL
4007
4008#include "SPIRVGenTables.inc"
4009} // namespace SPIRV
4010
4011//===----------------------------------------------------------------------===//
4012// Misc functions for parsing builtin types.
4013//===----------------------------------------------------------------------===//
4014
4015static Type *parseTypeString(StringRef Name, LLVMContext &Context) {
4016 if (Name.starts_with(Prefix: "void"))
4017 return Type::getVoidTy(C&: Context);
4018 else if (Name.starts_with(Prefix: "int") || Name.starts_with(Prefix: "uint"))
4019 return Type::getInt32Ty(C&: Context);
4020 else if (Name.starts_with(Prefix: "bfloat"))
4021 return Type::getBFloatTy(C&: Context);
4022 else if (Name.starts_with(Prefix: "float"))
4023 return Type::getFloatTy(C&: Context);
4024 else if (Name.starts_with(Prefix: "half"))
4025 return Type::getHalfTy(C&: Context);
4026 else if (Name.starts_with(Prefix: "double"))
4027 return Type::getDoubleTy(C&: Context);
4028 report_fatal_error(reason: "Unable to recognize type!");
4029}
4030
4031//===----------------------------------------------------------------------===//
4032// Implementation functions for builtin types.
4033//===----------------------------------------------------------------------===//
4034
4035static SPIRVTypeInst
4036getNonParameterizedType(const TargetExtType *ExtensionType,
4037 const SPIRV::BuiltinType *TypeRecord,
4038 MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) {
4039 unsigned Opcode = TypeRecord->Opcode;
4040 // Create or get an existing type from GlobalRegistry.
4041 return GR->getOrCreateOpTypeByOpcode(Ty: ExtensionType, MIRBuilder, Opcode);
4042}
4043
4044static SPIRVTypeInst getSamplerType(MachineIRBuilder &MIRBuilder,
4045 SPIRVGlobalRegistry *GR) {
4046 // Create or get an existing type from GlobalRegistry.
4047 return GR->getOrCreateOpTypeSampler(MIRBuilder);
4048}
4049
4050static SPIRVTypeInst getPipeType(const TargetExtType *ExtensionType,
4051 MachineIRBuilder &MIRBuilder,
4052 SPIRVGlobalRegistry *GR) {
4053 assert(ExtensionType->getNumIntParameters() == 1 &&
4054 "Invalid number of parameters for SPIR-V pipe builtin!");
4055 // Create or get an existing type from GlobalRegistry.
4056 return GR->getOrCreateOpTypePipe(MIRBuilder,
4057 AccQual: SPIRV::AccessQualifier::AccessQualifier(
4058 ExtensionType->getIntParameter(i: 0)));
4059}
4060
4061static SPIRVTypeInst getCoopMatrType(const TargetExtType *ExtensionType,
4062 MachineIRBuilder &MIRBuilder,
4063 SPIRVGlobalRegistry *GR) {
4064 assert(ExtensionType->getNumIntParameters() == 4 &&
4065 "Invalid number of parameters for SPIR-V coop matrices builtin!");
4066 assert(ExtensionType->getNumTypeParameters() == 1 &&
4067 "SPIR-V coop matrices builtin type must have a type parameter!");
4068 SPIRVTypeInst ElemType =
4069 GR->getOrCreateSPIRVType(Type: ExtensionType->getTypeParameter(i: 0), MIRBuilder,
4070 AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
4071 // Create or get an existing type from GlobalRegistry.
4072 return GR->getOrCreateOpTypeCoopMatr(
4073 MIRBuilder, ExtensionType, ElemType, Scope: ExtensionType->getIntParameter(i: 0),
4074 Rows: ExtensionType->getIntParameter(i: 1), Columns: ExtensionType->getIntParameter(i: 2),
4075 Use: ExtensionType->getIntParameter(i: 3), EmitIR: true);
4076}
4077
4078static SPIRVTypeInst getSampledImageType(const TargetExtType *OpaqueType,
4079 MachineIRBuilder &MIRBuilder,
4080 SPIRVGlobalRegistry *GR) {
4081 SPIRVTypeInst OpaqueImageType = GR->getImageType(
4082 ExtensionType: OpaqueType, Qualifier: SPIRV::AccessQualifier::ReadOnly, MIRBuilder);
4083 // Create or get an existing type from GlobalRegistry.
4084 return GR->getOrCreateOpTypeSampledImage(ImageType: OpaqueImageType, MIRBuilder);
4085}
4086
4087static SPIRVTypeInst getInlineSpirvType(const TargetExtType *ExtensionType,
4088 MachineIRBuilder &MIRBuilder,
4089 SPIRVGlobalRegistry *GR) {
4090 assert(ExtensionType->getNumIntParameters() == 3 &&
4091 "Inline SPIR-V type builtin takes an opcode, size, and alignment "
4092 "parameter");
4093 auto Opcode = ExtensionType->getIntParameter(i: 0);
4094
4095 SmallVector<MachineOperand> Operands;
4096 for (Type *Param : ExtensionType->type_params()) {
4097 if (const TargetExtType *ParamEType = dyn_cast<TargetExtType>(Val: Param)) {
4098 if (ParamEType->getName() == "spirv.IntegralConstant") {
4099 assert(ParamEType->getNumTypeParameters() == 1 &&
4100 "Inline SPIR-V integral constant builtin must have a type "
4101 "parameter");
4102 assert(ParamEType->getNumIntParameters() == 1 &&
4103 "Inline SPIR-V integral constant builtin must have a "
4104 "value parameter");
4105
4106 auto OperandValue = ParamEType->getIntParameter(i: 0);
4107 auto *OperandType = ParamEType->getTypeParameter(i: 0);
4108
4109 SPIRVTypeInst OperandSPIRVType = GR->getOrCreateSPIRVType(
4110 Type: OperandType, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
4111
4112 Operands.push_back(Elt: MachineOperand::CreateReg(
4113 Reg: GR->buildConstantInt(Val: OperandValue, MIRBuilder, SpvType: OperandSPIRVType,
4114 EmitIR: true),
4115 /*isDef=*/false));
4116 continue;
4117 } else if (ParamEType->getName() == "spirv.Literal") {
4118 assert(ParamEType->getNumTypeParameters() == 0 &&
4119 "Inline SPIR-V literal builtin does not take type "
4120 "parameters");
4121 assert(ParamEType->getNumIntParameters() == 1 &&
4122 "Inline SPIR-V literal builtin must have an integer "
4123 "parameter");
4124
4125 auto OperandValue = ParamEType->getIntParameter(i: 0);
4126
4127 Operands.push_back(Elt: MachineOperand::CreateImm(Val: OperandValue));
4128 continue;
4129 }
4130 }
4131 SPIRVTypeInst TypeOperand = GR->getOrCreateSPIRVType(
4132 Type: Param, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
4133 Operands.push_back(
4134 Elt: MachineOperand::CreateReg(Reg: GR->getSPIRVTypeID(SpirvType: TypeOperand),
4135 /*isDef=*/false));
4136 }
4137
4138 return GR->getOrCreateUnknownType(Ty: ExtensionType, MIRBuilder, Opcode,
4139 Operands);
4140}
4141
4142static SPIRVTypeInst getVulkanBufferType(const TargetExtType *ExtensionType,
4143 MachineIRBuilder &MIRBuilder,
4144 SPIRVGlobalRegistry *GR) {
4145 assert(ExtensionType->getNumTypeParameters() == 1 &&
4146 "Vulkan buffers have exactly one type for the type of the buffer.");
4147 assert(ExtensionType->getNumIntParameters() == 2 &&
4148 "Vulkan buffer have 2 integer parameters: storage class and is "
4149 "writable.");
4150
4151 auto *T = ExtensionType->getTypeParameter(i: 0);
4152 auto SC = static_cast<SPIRV::StorageClass::StorageClass>(
4153 ExtensionType->getIntParameter(i: 0));
4154 bool IsWritable = ExtensionType->getIntParameter(i: 1);
4155 return GR->getOrCreateVulkanBufferType(MIRBuilder, ElemType: T, SC, IsWritable);
4156}
4157
4158static SPIRVTypeInst
4159getVulkanPushConstantType(const TargetExtType *ExtensionType,
4160 MachineIRBuilder &MIRBuilder,
4161 SPIRVGlobalRegistry *GR) {
4162 assert(ExtensionType->getNumTypeParameters() == 1 &&
4163 "Vulkan push constants have exactly one type as argument.");
4164 auto *T = ExtensionType->getTypeParameter(i: 0);
4165 return GR->getOrCreateVulkanPushConstantType(MIRBuilder, ElemType: T);
4166}
4167
4168static SPIRVTypeInst getLayoutType(const TargetExtType *ExtensionType,
4169 MachineIRBuilder &MIRBuilder,
4170 SPIRVGlobalRegistry *GR) {
4171 return GR->getOrCreateLayoutType(MIRBuilder, T: ExtensionType);
4172}
4173
4174namespace SPIRV {
4175TargetExtType *parseBuiltinTypeNameToTargetExtType(std::string TypeName,
4176 LLVMContext &Context) {
4177 StringRef NameWithParameters = TypeName;
4178
4179 // Pointers-to-opaque-structs representing OpenCL types are first translated
4180 // to equivalent SPIR-V types. OpenCL builtin type names should have the
4181 // following format: e.g. %opencl.event_t
4182 if (NameWithParameters.starts_with(Prefix: "opencl.")) {
4183 const SPIRV::OpenCLType *OCLTypeRecord =
4184 SPIRV::lookupOpenCLType(Name: NameWithParameters);
4185 if (!OCLTypeRecord)
4186 report_fatal_error(reason: "Missing TableGen record for OpenCL type: " +
4187 NameWithParameters);
4188 NameWithParameters =
4189 SPIRV::getOpenCLTypeStr(Offset: OCLTypeRecord->SpirvTypeLiteral);
4190 // Continue with the SPIR-V builtin type...
4191 }
4192
4193 // Names of the opaque structs representing a SPIR-V builtins without
4194 // parameters should have the following format: e.g. %spirv.Event
4195 assert(NameWithParameters.starts_with("spirv.") &&
4196 "Unknown builtin opaque type!");
4197
4198 // Parameterized SPIR-V builtins names follow this format:
4199 // e.g. %spirv.Image._void_1_0_0_0_0_0_0, %spirv.Pipe._0
4200 if (!NameWithParameters.contains(C: '_'))
4201 return TargetExtType::get(Context, Name: NameWithParameters);
4202
4203 SmallVector<StringRef> Parameters;
4204 unsigned BaseNameLength = NameWithParameters.find(C: '_') - 1;
4205 SplitString(Source: NameWithParameters.substr(Start: BaseNameLength + 1), OutFragments&: Parameters, Delimiters: "_");
4206
4207 SmallVector<Type *, 1> TypeParameters;
4208 bool HasTypeParameter = !isDigit(C: Parameters[0][0]);
4209 if (HasTypeParameter)
4210 TypeParameters.push_back(Elt: parseTypeString(Name: Parameters[0], Context));
4211 SmallVector<unsigned> IntParameters;
4212 for (unsigned i = HasTypeParameter ? 1 : 0; i < Parameters.size(); i++) {
4213 unsigned IntParameter = 0;
4214 bool ValidLiteral = !Parameters[i].getAsInteger(Radix: 10, Result&: IntParameter);
4215 (void)ValidLiteral;
4216 assert(ValidLiteral &&
4217 "Invalid format of SPIR-V builtin parameter literal!");
4218 IntParameters.push_back(Elt: IntParameter);
4219 }
4220 return TargetExtType::get(Context,
4221 Name: NameWithParameters.substr(Start: 0, N: BaseNameLength),
4222 Types: TypeParameters, Ints: IntParameters);
4223}
4224
4225SPIRVTypeInst
4226lowerBuiltinType(const Type *OpaqueType,
4227 SPIRV::AccessQualifier::AccessQualifier AccessQual,
4228 MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) {
4229 // In LLVM IR, SPIR-V and OpenCL builtin types are represented as either
4230 // target(...) target extension types or pointers-to-opaque-structs. The
4231 // approach relying on structs is deprecated and works only in the non-opaque
4232 // pointer mode (-opaque-pointers=0).
4233 // In order to maintain compatibility with LLVM IR generated by older versions
4234 // of Clang and LLVM/SPIR-V Translator, the pointers-to-opaque-structs are
4235 // "translated" to target extension types. This translation is temporary and
4236 // will be removed in the future release of LLVM.
4237 const TargetExtType *BuiltinType = dyn_cast<TargetExtType>(Val: OpaqueType);
4238 if (!BuiltinType)
4239 BuiltinType = parseBuiltinTypeNameToTargetExtType(
4240 TypeName: OpaqueType->getStructName().str(), Context&: MIRBuilder.getContext());
4241
4242 unsigned NumStartingVRegs = MIRBuilder.getMRI()->getNumVirtRegs();
4243
4244 StringRef Name = BuiltinType->getName();
4245 LLVM_DEBUG(dbgs() << "Lowering builtin type: " << Name << "\n");
4246
4247 SPIRVTypeInst TargetType = nullptr;
4248 if (Name == "spirv.Type") {
4249 TargetType = getInlineSpirvType(ExtensionType: BuiltinType, MIRBuilder, GR);
4250 } else if (Name == "spirv.VulkanBuffer") {
4251 TargetType = getVulkanBufferType(ExtensionType: BuiltinType, MIRBuilder, GR);
4252 } else if (Name == "spirv.Padding") {
4253 TargetType = GR->getOrCreatePaddingType(MIRBuilder);
4254 } else if (Name == "spirv.PushConstant") {
4255 TargetType = getVulkanPushConstantType(ExtensionType: BuiltinType, MIRBuilder, GR);
4256 } else if (Name == "spirv.Layout") {
4257 TargetType = getLayoutType(ExtensionType: BuiltinType, MIRBuilder, GR);
4258 } else {
4259 // Lookup the demangled builtin type in the TableGen records.
4260 const SPIRV::BuiltinType *TypeRecord = SPIRV::lookupBuiltinType(Name);
4261 if (!TypeRecord)
4262 report_fatal_error(reason: "Missing TableGen record for builtin type: " + Name);
4263
4264 // "Lower" the BuiltinType into TargetType. The following get<...>Type
4265 // methods use the implementation details from TableGen records or
4266 // TargetExtType parameters to either create a new OpType<...> machine
4267 // instruction or get an existing equivalent SPIRV type from
4268 // GlobalRegistry.
4269
4270 switch (TypeRecord->Opcode) {
4271 case SPIRV::OpTypeImage:
4272 TargetType = GR->getImageType(ExtensionType: BuiltinType, Qualifier: AccessQual, MIRBuilder);
4273 break;
4274 case SPIRV::OpTypePipe:
4275 TargetType = getPipeType(ExtensionType: BuiltinType, MIRBuilder, GR);
4276 break;
4277 case SPIRV::OpTypeDeviceEvent:
4278 TargetType = GR->getOrCreateOpTypeDeviceEvent(MIRBuilder);
4279 break;
4280 case SPIRV::OpTypeSampler:
4281 TargetType = getSamplerType(MIRBuilder, GR);
4282 break;
4283 case SPIRV::OpTypeSampledImage:
4284 TargetType = getSampledImageType(OpaqueType: BuiltinType, MIRBuilder, GR);
4285 break;
4286 case SPIRV::OpTypeCooperativeMatrixKHR:
4287 TargetType = getCoopMatrType(ExtensionType: BuiltinType, MIRBuilder, GR);
4288 break;
4289 default:
4290 TargetType =
4291 getNonParameterizedType(ExtensionType: BuiltinType, TypeRecord, MIRBuilder, GR);
4292 break;
4293 }
4294 }
4295
4296 // Emit OpName instruction if a new OpType<...> instruction was added
4297 // (equivalent type was not found in GlobalRegistry).
4298 if (NumStartingVRegs < MIRBuilder.getMRI()->getNumVirtRegs())
4299 buildOpName(Target: GR->getSPIRVTypeID(SpirvType: TargetType), Name, MIRBuilder);
4300
4301 return TargetType;
4302}
4303
4304bool isPipeOrAddressSpaceCastBuiltin(StringRef Name) {
4305 const DemangledBuiltin *Builtin = lookupBuiltin(Name, Set: OpenCL_std);
4306 if (!Builtin)
4307 return false;
4308 return Builtin->Group == Pipe || Builtin->Group == CastToPtr ||
4309 Builtin->Group == BlockingPipes;
4310}
4311} // namespace SPIRV
4312} // namespace llvm
4313