1//===- SPIRVModuleAnalysis.cpp - analysis of global instrs & regs - C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The analysis collects instructions that should be output at the module level
10// and performs the global register numbering.
11//
12// The results of this analysis are used in AsmPrinter to rename registers
13// globally and to output required instructions at the module level.
14//
15//===----------------------------------------------------------------------===//
16
17// TODO: Per LLVM best practices, the report_fatal_error (deprecated) /
18// ReportFatalUsageError calls in this file should be replaced with the
19// Diagnostic infrastructure (e.g. the reportUnsupported function below).
20
21#include "SPIRVModuleAnalysis.h"
22#include "MCTargetDesc/SPIRVBaseInfo.h"
23#include "MCTargetDesc/SPIRVMCTargetDesc.h"
24#include "SPIRV.h"
25#include "SPIRVSubtarget.h"
26#include "SPIRVTargetMachine.h"
27#include "SPIRVUtils.h"
28#include "llvm/ADT/STLExtras.h"
29#include "llvm/CodeGen/MachineModuleInfo.h"
30#include "llvm/CodeGen/TargetPassConfig.h"
31
32using namespace llvm;
33
34#define DEBUG_TYPE "spirv-module-analysis"
35
36static cl::opt<bool>
37 SPVDumpDeps("spv-dump-deps",
38 cl::desc("Dump MIR with SPIR-V dependencies info"),
39 cl::Optional, cl::init(Val: false));
40
41static cl::list<SPIRV::Capability::Capability>
42 AvoidCapabilities("avoid-spirv-capabilities",
43 cl::desc("SPIR-V capabilities to avoid if there are "
44 "other options enabling a feature"),
45 cl::Hidden,
46 cl::values(clEnumValN(SPIRV::Capability::Shader, "Shader",
47 "SPIR-V Shader capability")));
48// Use sets instead of cl::list to check "if contains" condition
49struct AvoidCapabilitiesSet {
50 SmallSet<SPIRV::Capability::Capability, 4> S;
51 AvoidCapabilitiesSet() { S.insert_range(R&: AvoidCapabilities); }
52};
53
54char llvm::SPIRVModuleAnalysis::ID = 0;
55
56INITIALIZE_PASS(SPIRVModuleAnalysis, DEBUG_TYPE, "SPIRV module analysis", true,
57 true)
58
59static void reportUnsupported(const MachineInstr &MI, const char *Msg) {
60 const Function &Func = MI.getMF()->getFunction();
61 Func.getContext().diagnose(
62 DI: DiagnosticInfoUnsupported(Func, Msg, MI.getDebugLoc()));
63}
64
65// Retrieve an unsigned from an MDNode with a list of them as operands.
66static unsigned getMetadataUInt(MDNode *MdNode, unsigned OpIndex,
67 unsigned DefaultVal = 0) {
68 if (MdNode && OpIndex < MdNode->getNumOperands()) {
69 const auto &Op = MdNode->getOperand(I: OpIndex);
70 return mdconst::extract<ConstantInt>(MD: Op)->getZExtValue();
71 }
72 return DefaultVal;
73}
74
75static SPIRV::Requirements
76getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
77 unsigned i, const SPIRVSubtarget &ST,
78 SPIRV::RequirementHandler &Reqs) {
79 // A set of capabilities to avoid if there is another option.
80 AvoidCapabilitiesSet AvoidCaps;
81 if (!ST.isShader())
82 AvoidCaps.S.insert(V: SPIRV::Capability::Shader);
83 else
84 AvoidCaps.S.insert(V: SPIRV::Capability::Kernel);
85
86 VersionTuple ReqMinVer = getSymbolicOperandMinVersion(Category, Value: i);
87 VersionTuple ReqMaxVer = getSymbolicOperandMaxVersion(Category, Value: i);
88 VersionTuple SPIRVVersion = ST.getSPIRVVersion();
89 bool MinVerOK = SPIRVVersion.empty() || SPIRVVersion >= ReqMinVer;
90 bool MaxVerOK =
91 ReqMaxVer.empty() || SPIRVVersion.empty() || SPIRVVersion <= ReqMaxVer;
92 CapabilityList ReqCaps = getSymbolicOperandCapabilities(Category, Value: i);
93 ExtensionList ReqExts = getSymbolicOperandExtensions(Category, Value: i);
94 if (ReqCaps.empty()) {
95 if (ReqExts.empty()) {
96 if (MinVerOK && MaxVerOK)
97 return {true, {}, {}, ReqMinVer, ReqMaxVer};
98 return {false, {}, {}, VersionTuple(), VersionTuple()};
99 }
100 } else if (MinVerOK && MaxVerOK) {
101 if (ReqCaps.size() == 1) {
102 auto Cap = ReqCaps[0];
103 if (Reqs.isCapabilityAvailable(Cap)) {
104 ReqExts.append(RHS: getSymbolicOperandExtensions(
105 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
106 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
107 }
108 } else {
109 // By SPIR-V specification: "If an instruction, enumerant, or other
110 // feature specifies multiple enabling capabilities, only one such
111 // capability needs to be declared to use the feature." However, one
112 // capability may be preferred over another. We use command line
113 // argument(s) and AvoidCapabilities to avoid selection of certain
114 // capabilities if there are other options.
115 CapabilityList UseCaps;
116 for (auto Cap : ReqCaps)
117 if (Reqs.isCapabilityAvailable(Cap))
118 UseCaps.push_back(Elt: Cap);
119 for (size_t i = 0, Sz = UseCaps.size(); i < Sz; ++i) {
120 auto Cap = UseCaps[i];
121 if (i == Sz - 1 || !AvoidCaps.S.contains(V: Cap)) {
122 ReqExts.append(RHS: getSymbolicOperandExtensions(
123 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
124 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
125 }
126 }
127 }
128 }
129 // If there are no capabilities, or we can't satisfy the version or
130 // capability requirements, use the list of extensions (if the subtarget
131 // can handle them all).
132 if (llvm::all_of(Range&: ReqExts, P: [&ST](const SPIRV::Extension::Extension &Ext) {
133 return ST.canUseExtension(E: Ext);
134 })) {
135 return {true,
136 {},
137 std::move(ReqExts),
138 VersionTuple(),
139 VersionTuple()}; // TODO: add versions to extensions.
140 }
141 return {false, {}, {}, VersionTuple(), VersionTuple()};
142}
143
144void SPIRVModuleAnalysis::setBaseInfo(const Module &M) {
145 MAI.MaxID = 0;
146 for (int i = 0; i < SPIRV::NUM_MODULE_SECTIONS; i++)
147 MAI.MS[i].clear();
148 MAI.RegisterAliasTable.clear();
149 MAI.InstrsToDelete.clear();
150 MAI.GlobalObjMap.clear();
151 MAI.GlobalVarList.clear();
152 MAI.ExtInstSetMap.clear();
153 MAI.Reqs.clear();
154 MAI.Reqs.initAvailableCapabilities(ST: *ST);
155
156 // TODO: determine memory model and source language from the configuratoin.
157 if (auto MemModel = M.getNamedMetadata(Name: "spirv.MemoryModel")) {
158 auto MemMD = MemModel->getOperand(i: 0);
159 MAI.Addr = static_cast<SPIRV::AddressingModel::AddressingModel>(
160 getMetadataUInt(MdNode: MemMD, OpIndex: 0));
161 MAI.Mem =
162 static_cast<SPIRV::MemoryModel::MemoryModel>(getMetadataUInt(MdNode: MemMD, OpIndex: 1));
163 } else {
164 // TODO: Add support for VulkanMemoryModel.
165 MAI.Mem = ST->isShader() ? SPIRV::MemoryModel::GLSL450
166 : SPIRV::MemoryModel::OpenCL;
167 if (MAI.Mem == SPIRV::MemoryModel::OpenCL) {
168 unsigned PtrSize = ST->getPointerSize();
169 MAI.Addr = PtrSize == 32 ? SPIRV::AddressingModel::Physical32
170 : PtrSize == 64 ? SPIRV::AddressingModel::Physical64
171 : SPIRV::AddressingModel::Logical;
172 } else {
173 // TODO: Add support for PhysicalStorageBufferAddress.
174 MAI.Addr = SPIRV::AddressingModel::Logical;
175 }
176 }
177 // Get the OpenCL version number from metadata.
178 // TODO: support other source languages.
179 if (auto VerNode = M.getNamedMetadata(Name: "opencl.ocl.version")) {
180 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_C;
181 // Construct version literal in accordance with SPIRV-LLVM-Translator.
182 // TODO: support multiple OCL version metadata.
183 assert(VerNode->getNumOperands() > 0 && "Invalid SPIR");
184 auto VersionMD = VerNode->getOperand(i: 0);
185 unsigned MajorNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 0, DefaultVal: 2);
186 unsigned MinorNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 1);
187 unsigned RevNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 2);
188 // Prevent Major part of OpenCL version to be 0
189 MAI.SrcLangVersion =
190 (std::max(a: 1U, b: MajorNum) * 100 + MinorNum) * 1000 + RevNum;
191 // When opencl.cxx.version is also present, validate compatibility
192 // and use C++ for OpenCL as source language with the C++ version.
193 if (auto *CxxVerNode = M.getNamedMetadata(Name: "opencl.cxx.version")) {
194 assert(CxxVerNode->getNumOperands() > 0 && "Invalid SPIR");
195 auto *CxxMD = CxxVerNode->getOperand(i: 0);
196 unsigned CxxVer =
197 (getMetadataUInt(MdNode: CxxMD, OpIndex: 0) * 100 + getMetadataUInt(MdNode: CxxMD, OpIndex: 1)) * 1000 +
198 getMetadataUInt(MdNode: CxxMD, OpIndex: 2);
199 if ((MAI.SrcLangVersion == 200000 && CxxVer == 100000) ||
200 (MAI.SrcLangVersion == 300000 && CxxVer == 202100000)) {
201 MAI.SrcLang = SPIRV::SourceLanguage::CPP_for_OpenCL;
202 MAI.SrcLangVersion = CxxVer;
203 } else {
204 report_fatal_error(
205 reason: "opencl cxx version is not compatible with opencl c version!");
206 }
207 }
208 } else {
209 // If there is no information about OpenCL version we are forced to generate
210 // OpenCL 1.0 by default for the OpenCL environment to avoid puzzling
211 // run-times with Unknown/0.0 version output. For a reference, LLVM-SPIRV
212 // Translator avoids potential issues with run-times in a similar manner.
213 if (!ST->isShader()) {
214 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_CPP;
215 MAI.SrcLangVersion = 100000;
216 } else {
217 MAI.SrcLang = SPIRV::SourceLanguage::Unknown;
218 MAI.SrcLangVersion = 0;
219 }
220 }
221
222 if (auto ExtNode = M.getNamedMetadata(Name: "opencl.used.extensions")) {
223 for (unsigned I = 0, E = ExtNode->getNumOperands(); I != E; ++I) {
224 MDNode *MD = ExtNode->getOperand(i: I);
225 if (!MD || MD->getNumOperands() == 0)
226 continue;
227 for (unsigned J = 0, N = MD->getNumOperands(); J != N; ++J)
228 MAI.SrcExt.insert(key: cast<MDString>(Val: MD->getOperand(I: J))->getString());
229 }
230 }
231
232 // Update required capabilities for this memory model, addressing model and
233 // source language.
234 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::MemoryModelOperand,
235 i: MAI.Mem, ST: *ST);
236 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::SourceLanguageOperand,
237 i: MAI.SrcLang, ST: *ST);
238 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::AddressingModelOperand,
239 i: MAI.Addr, ST: *ST);
240
241 if (MAI.Mem == SPIRV::MemoryModel::VulkanKHR)
242 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_vulkan_memory_model);
243
244 if (!ST->isShader()) {
245 // TODO: check if it's required by default.
246 MAI.ExtInstSetMap[static_cast<unsigned>(
247 SPIRV::InstructionSet::OpenCL_std)] = MAI.getNextIDRegister();
248 }
249}
250
251// Appends the signature of the decoration instructions that decorate R to
252// Signature.
253static void appendDecorationsForReg(const MachineRegisterInfo &MRI, Register R,
254 InstrSignature &Signature) {
255 for (MachineInstr &UseMI : MRI.use_instructions(Reg: R)) {
256 // We don't handle OpDecorateId because getting the register alias for the
257 // ID can cause problems, and we do not need it for now.
258 if (UseMI.getOpcode() != SPIRV::OpDecorate &&
259 UseMI.getOpcode() != SPIRV::OpMemberDecorate)
260 continue;
261
262 for (unsigned I = 0; I < UseMI.getNumOperands(); ++I) {
263 const MachineOperand &MO = UseMI.getOperand(i: I);
264 if (MO.isReg())
265 continue;
266 Signature.push_back(Elt: hash_value(MO));
267 }
268 }
269}
270
271// Returns a representation of an instruction as a vector of MachineOperand
272// hash values, see llvm::hash_value(const MachineOperand &MO) for details.
273// This creates a signature of the instruction with the same content
274// that MachineOperand::isIdenticalTo uses for comparison.
275static InstrSignature instrToSignature(const MachineInstr &MI,
276 SPIRV::ModuleAnalysisInfo &MAI,
277 bool UseDefReg) {
278 Register DefReg;
279 InstrSignature Signature{MI.getOpcode()};
280 for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
281 // The only decorations that can be applied more than once to a given <id>
282 // or structure member are FuncParamAttr (38), UserSemantic (5635),
283 // CacheControlLoadINTEL (6442), and CacheControlStoreINTEL (6443). For all
284 // the rest of decorations, we will only add to the signature the Opcode,
285 // the id to which it applies, and the decoration id, disregarding any
286 // decoration flags. This will ensure that any subsequent decoration with
287 // the same id will be deemed as a duplicate. Then, at the call site, we
288 // will be able to handle duplicates in the best way.
289 unsigned Opcode = MI.getOpcode();
290 if ((Opcode == SPIRV::OpDecorate) && i >= 2) {
291 unsigned DecorationID = MI.getOperand(i: 1).getImm();
292 if (DecorationID != SPIRV::Decoration::FuncParamAttr &&
293 DecorationID != SPIRV::Decoration::UserSemantic &&
294 DecorationID != SPIRV::Decoration::CacheControlLoadINTEL &&
295 DecorationID != SPIRV::Decoration::CacheControlStoreINTEL)
296 continue;
297 }
298 const MachineOperand &MO = MI.getOperand(i);
299 size_t h;
300 if (MO.isReg()) {
301 if (!UseDefReg && MO.isDef()) {
302 assert(!DefReg.isValid() && "Multiple def registers.");
303 DefReg = MO.getReg();
304 continue;
305 }
306 Register RegAlias = MAI.getRegisterAlias(MF: MI.getMF(), Reg: MO.getReg());
307 if (!RegAlias.isValid()) {
308 LLVM_DEBUG({
309 dbgs() << "Unexpectedly, no global id found for the operand ";
310 MO.print(dbgs());
311 dbgs() << "\nInstruction: ";
312 MI.print(dbgs());
313 dbgs() << "\n";
314 });
315 report_fatal_error(reason: "All v-regs must have been mapped to global id's");
316 }
317 // mimic llvm::hash_value(const MachineOperand &MO)
318 h = hash_combine(args: MO.getType(), args: (unsigned)RegAlias, args: MO.getSubReg(),
319 args: MO.isDef());
320 } else {
321 h = hash_value(MO);
322 }
323 Signature.push_back(Elt: h);
324 }
325
326 if (DefReg.isValid()) {
327 // Decorations change the semantics of the current instruction. So two
328 // identical instruction with different decorations cannot be merged. That
329 // is why we add the decorations to the signature.
330 appendDecorationsForReg(MRI: MI.getMF()->getRegInfo(), R: DefReg, Signature);
331 }
332 return Signature;
333}
334
335bool SPIRVModuleAnalysis::isDeclSection(const MachineRegisterInfo &MRI,
336 const MachineInstr &MI) {
337 unsigned Opcode = MI.getOpcode();
338 switch (Opcode) {
339 case SPIRV::OpTypeForwardPointer:
340 // omit now, collect later
341 return false;
342 case SPIRV::OpVariable:
343 case SPIRV::OpUntypedVariableKHR:
344 return static_cast<SPIRV::StorageClass::StorageClass>(
345 MI.getOperand(i: 2).getImm()) != SPIRV::StorageClass::Function;
346 case SPIRV::OpFunction:
347 case SPIRV::OpFunctionParameter:
348 return true;
349 }
350 if (GR->hasConstFunPtr() && Opcode == SPIRV::OpUndef) {
351 // The OpUndef may be a placeholder for a function reference recorded by
352 // selectGlobalValue. Skip emitting it if any user consumes it as a
353 // function-pointer-like operand (OpConstantFunctionPointerINTEL operand 2,
354 // or OpEnqueueKernel's Invoke operand at index 8). The rewrite happens
355 // in visitFunPtrUse, which aliases the OpUndef's vreg to the function's
356 // global <id>.
357 Register DefReg = MI.getOperand(i: 0).getReg();
358 if (GR->getFunctionDefinitionByUse(Use: &MI.getOperand(i: 0))) {
359 for (MachineInstr &UseMI : MRI.use_instructions(Reg: DefReg)) {
360 unsigned UseOp = UseMI.getOpcode();
361 if (UseOp == SPIRV::OpConstantFunctionPointerINTEL ||
362 UseOp == SPIRV::OpEnqueueKernel) {
363 MAI.setSkipEmission(&MI);
364 return false;
365 }
366 }
367 }
368 for (MachineInstr &UseMI : MRI.use_instructions(Reg: DefReg)) {
369 if (UseMI.getOpcode() != SPIRV::OpConstantFunctionPointerINTEL)
370 continue;
371 // it's a dummy definition, FP constant refers to a function,
372 // and this is resolved in another way; let's skip this definition
373 assert(UseMI.getOperand(2).isReg() &&
374 UseMI.getOperand(2).getReg() == DefReg);
375 MAI.setSkipEmission(&MI);
376 return false;
377 }
378 }
379 return TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||
380 TII->isInlineAsmDefInstr(MI);
381}
382
383// This is a special case of a function pointer referring to a possibly
384// forward function declaration. The operand is a dummy OpUndef that
385// requires a special treatment.
386// FunPtrOp is the MachineOperand previously recorded via
387// SPIRVGlobalRegistry::recordFunctionPointer, identifying which Function
388// this placeholder refers to.
389void SPIRVModuleAnalysis::visitFunPtrUse(
390 Register OpReg, const MachineOperand *FunPtrOp,
391 InstrGRegsMap &SignatureToGReg,
392 std::map<const Value *, unsigned> &GlobalToGReg,
393 const MachineFunction *MF) {
394 const MachineOperand *OpFunDef = GR->getFunctionDefinitionByUse(Use: FunPtrOp);
395 assert(OpFunDef && OpFunDef->isReg());
396 // find the actual function definition and number it globally in advance
397 const MachineInstr *OpDefMI = OpFunDef->getParent();
398 assert(OpDefMI && OpDefMI->getOpcode() == SPIRV::OpFunction);
399 const MachineFunction *FunDefMF = OpDefMI->getParent()->getParent();
400 const MachineRegisterInfo &FunDefMRI = FunDefMF->getRegInfo();
401 do {
402 visitDecl(MRI: FunDefMRI, SignatureToGReg, GlobalToGReg, MF: FunDefMF, MI: *OpDefMI);
403 OpDefMI = OpDefMI->getNextNode();
404 } while (OpDefMI && (OpDefMI->getOpcode() == SPIRV::OpFunction ||
405 OpDefMI->getOpcode() == SPIRV::OpFunctionParameter));
406 // associate the function pointer with the newly assigned global number
407 MCRegister GlobalFunDefReg =
408 MAI.getRegisterAlias(MF: FunDefMF, Reg: OpFunDef->getReg());
409 assert(GlobalFunDefReg.isValid() &&
410 "Function definition must refer to a global register");
411 MAI.setRegisterAlias(MF, Reg: OpReg, AliasReg: GlobalFunDefReg);
412}
413
414// Depth first recursive traversal of dependencies. Repeated visits are guarded
415// by MAI.hasRegisterAlias().
416void SPIRVModuleAnalysis::visitDecl(
417 const MachineRegisterInfo &MRI, InstrGRegsMap &SignatureToGReg,
418 std::map<const Value *, unsigned> &GlobalToGReg, const MachineFunction *MF,
419 const MachineInstr &MI) {
420 unsigned Opcode = MI.getOpcode();
421
422 // Process each operand of the instruction to resolve dependencies
423 for (const MachineOperand &MO : MI.operands()) {
424 if (!MO.isReg() || MO.isDef())
425 continue;
426 Register OpReg = MO.getReg();
427 // Handle function pointers special case
428 if (Opcode == SPIRV::OpConstantFunctionPointerINTEL &&
429 MRI.getRegClass(Reg: OpReg) == &SPIRV::pIDRegClass) {
430 visitFunPtrUse(OpReg, FunPtrOp: &MI.getOperand(i: 2), SignatureToGReg, GlobalToGReg,
431 MF);
432 continue;
433 }
434 // Skip already processed instructions
435 if (MAI.hasRegisterAlias(MF, Reg: MO.getReg()))
436 continue;
437 // Recursively visit dependencies
438 if (const MachineInstr *OpDefMI = MRI.getUniqueVRegDef(Reg: OpReg)) {
439 if (isDeclSection(MRI, MI: *OpDefMI))
440 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, MI: *OpDefMI);
441 continue;
442 }
443 // Handle the unexpected case of no unique definition for the SPIR-V
444 // instruction
445 LLVM_DEBUG({
446 dbgs() << "Unexpectedly, no unique definition for the operand ";
447 MO.print(dbgs());
448 dbgs() << "\nInstruction: ";
449 MI.print(dbgs());
450 dbgs() << "\n";
451 });
452 report_fatal_error(
453 reason: "No unique definition is found for the virtual register");
454 }
455
456 MCRegister GReg;
457 bool IsFunDef = false;
458 if (TII->isSpecConstantInstr(MI)) {
459 GReg = MAI.getNextIDRegister();
460 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
461 } else if (Opcode == SPIRV::OpFunction ||
462 Opcode == SPIRV::OpFunctionParameter) {
463 GReg = handleFunctionOrParameter(MF, MI, GlobalToGReg, IsFunDef);
464 } else if (Opcode == SPIRV::OpTypeStruct ||
465 Opcode == SPIRV::OpConstantComposite) {
466 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);
467 const MachineInstr *NextInstr = MI.getNextNode();
468 while (NextInstr &&
469 ((Opcode == SPIRV::OpTypeStruct &&
470 NextInstr->getOpcode() == SPIRV::OpTypeStructContinuedINTEL) ||
471 (Opcode == SPIRV::OpConstantComposite &&
472 NextInstr->getOpcode() ==
473 SPIRV::OpConstantCompositeContinuedINTEL))) {
474 MCRegister Tmp = handleTypeDeclOrConstant(MI: *NextInstr, SignatureToGReg);
475 MAI.setRegisterAlias(MF, Reg: NextInstr->getOperand(i: 0).getReg(), AliasReg: Tmp);
476 MAI.setSkipEmission(NextInstr);
477 NextInstr = NextInstr->getNextNode();
478 }
479 } else if (TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||
480 TII->isInlineAsmDefInstr(MI)) {
481 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);
482 } else if (Opcode == SPIRV::OpVariable ||
483 Opcode == SPIRV::OpUntypedVariableKHR) {
484 GReg = handleVariable(MF, MI, GlobalToGReg);
485 } else {
486 LLVM_DEBUG({
487 dbgs() << "\nInstruction: ";
488 MI.print(dbgs());
489 dbgs() << "\n";
490 });
491 llvm_unreachable("Unexpected instruction is visited");
492 }
493 MAI.setRegisterAlias(MF, Reg: MI.getOperand(i: 0).getReg(), AliasReg: GReg);
494 if (!IsFunDef)
495 MAI.setSkipEmission(&MI);
496}
497
498MCRegister SPIRVModuleAnalysis::handleFunctionOrParameter(
499 const MachineFunction *MF, const MachineInstr &MI,
500 std::map<const Value *, unsigned> &GlobalToGReg, bool &IsFunDef) {
501 const Value *GObj = GR->getGlobalObject(MF, R: MI.getOperand(i: 0).getReg());
502 assert(GObj && "Unregistered global definition");
503 const Function *F = dyn_cast<Function>(Val: GObj);
504 if (!F)
505 F = dyn_cast<Argument>(Val: GObj)->getParent();
506 assert(F && "Expected a reference to a function or an argument");
507 IsFunDef = !F->isDeclaration();
508 auto [It, Inserted] = GlobalToGReg.try_emplace(k: GObj);
509 if (!Inserted)
510 return It->second;
511 MCRegister GReg = MAI.getNextIDRegister();
512 It->second = GReg;
513 if (!IsFunDef)
514 MAI.MS[SPIRV::MB_ExtFuncDecls].push_back(Elt: &MI);
515 return GReg;
516}
517
518MCRegister
519SPIRVModuleAnalysis::handleTypeDeclOrConstant(const MachineInstr &MI,
520 InstrGRegsMap &SignatureToGReg) {
521 InstrSignature MISign = instrToSignature(MI, MAI, UseDefReg: false);
522 auto [It, Inserted] = SignatureToGReg.try_emplace(k: MISign);
523 if (!Inserted)
524 return It->second;
525 MCRegister GReg = MAI.getNextIDRegister();
526 It->second = GReg;
527 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
528 return GReg;
529}
530
531MCRegister SPIRVModuleAnalysis::handleVariable(
532 const MachineFunction *MF, const MachineInstr &MI,
533 std::map<const Value *, unsigned> &GlobalToGReg) {
534 MAI.GlobalVarList.push_back(Elt: &MI);
535 const Value *GObj = GR->getGlobalObject(MF, R: MI.getOperand(i: 0).getReg());
536 assert(GObj && "Unregistered global definition");
537 auto [It, Inserted] = GlobalToGReg.try_emplace(k: GObj);
538 if (!Inserted)
539 return It->second;
540 MCRegister GReg = MAI.getNextIDRegister();
541 It->second = GReg;
542 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
543 if (const auto *GV = dyn_cast<GlobalVariable>(Val: GObj))
544 MAI.GlobalObjMap[GV] = GReg;
545 return GReg;
546}
547
548void SPIRVModuleAnalysis::collectDeclarations(const Module &M) {
549 InstrGRegsMap SignatureToGReg;
550 std::map<const Value *, unsigned> GlobalToGReg;
551 for (const Function &F : M) {
552 MachineFunction *MF = MMI->getMachineFunction(F);
553 if (!MF)
554 continue;
555 const MachineRegisterInfo &MRI = MF->getRegInfo();
556 unsigned PastHeader = 0;
557 for (MachineBasicBlock &MBB : *MF) {
558 for (MachineInstr &MI : MBB) {
559 if (MI.getNumOperands() == 0)
560 continue;
561 unsigned Opcode = MI.getOpcode();
562 if (Opcode == SPIRV::OpFunction) {
563 if (PastHeader == 0) {
564 PastHeader = 1;
565 continue;
566 }
567 } else if (Opcode == SPIRV::OpFunctionParameter) {
568 if (PastHeader < 2)
569 continue;
570 } else if (PastHeader > 0) {
571 PastHeader = 2;
572 }
573
574 const MachineOperand &DefMO = MI.getOperand(i: 0);
575 switch (Opcode) {
576 case SPIRV::OpExtension:
577 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::Extension(DefMO.getImm()));
578 MAI.setSkipEmission(&MI);
579 break;
580 case SPIRV::OpCapability:
581 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::Capability(DefMO.getImm()));
582 MAI.setSkipEmission(&MI);
583 if (PastHeader > 0)
584 PastHeader = 2;
585 break;
586 default:
587 if (DefMO.isReg() && isDeclSection(MRI, MI) &&
588 !MAI.hasRegisterAlias(MF, Reg: DefMO.getReg()))
589 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, MI);
590 // OpEnqueueKernel is not a decl, but its Invoke operand may be a
591 // function-pointer placeholder OpUndef recorded by selectGlobalValue.
592 // Resolve it to the OpFunction's global <id> via visitFunPtrUse.
593 if (Opcode == SPIRV::OpEnqueueKernel && MI.getNumOperands() > 8) {
594 const MachineOperand &InvokeMO = MI.getOperand(i: 8);
595 if (InvokeMO.isReg()) {
596 Register InvokeReg = InvokeMO.getReg();
597 if (!MAI.hasRegisterAlias(MF, Reg: InvokeReg)) {
598 if (const MachineInstr *DefMI =
599 MRI.getUniqueVRegDef(Reg: InvokeReg)) {
600 if (DefMI->getOpcode() == SPIRV::OpUndef) {
601 const MachineOperand *FunPtrOp = &DefMI->getOperand(i: 0);
602 if (GR->getFunctionDefinitionByUse(Use: FunPtrOp))
603 visitFunPtrUse(OpReg: InvokeReg, FunPtrOp, SignatureToGReg,
604 GlobalToGReg, MF);
605 }
606 }
607 }
608 }
609 }
610 }
611 }
612 }
613 }
614}
615
616// Look for IDs declared with Import linkage, and map the corresponding function
617// to the register defining that variable (which will usually be the result of
618// an OpFunction). This lets us call externally imported functions using
619// the correct ID registers.
620void SPIRVModuleAnalysis::collectFuncNames(MachineInstr &MI,
621 const Function *F) {
622 if (MI.getOpcode() == SPIRV::OpDecorate) {
623 // If it's got Import linkage.
624 auto Dec = MI.getOperand(i: 1).getImm();
625 if (Dec == SPIRV::Decoration::LinkageAttributes) {
626 auto Lnk = MI.getOperand(i: MI.getNumOperands() - 1).getImm();
627 if (Lnk == SPIRV::LinkageType::Import) {
628 // Map imported function name to function ID register.
629 const Function *ImportedFunc =
630 F->getParent()->getFunction(Name: getStringImm(MI, StartIndex: 2));
631 Register Target = MI.getOperand(i: 0).getReg();
632 MAI.GlobalObjMap[ImportedFunc] =
633 MAI.getRegisterAlias(MF: MI.getMF(), Reg: Target);
634 }
635 }
636 } else if (MI.getOpcode() == SPIRV::OpFunction) {
637 // Record all internal OpFunction declarations.
638 Register Reg = MI.defs().begin()->getReg();
639 MCRegister GlobalReg = MAI.getRegisterAlias(MF: MI.getMF(), Reg);
640 assert(GlobalReg.isValid());
641 MAI.GlobalObjMap[F] = GlobalReg;
642 }
643}
644
645// Collect the given instruction in the specified MS. We assume global register
646// numbering has already occurred by this point. We can directly compare reg
647// arguments when detecting duplicates.
648static void collectOtherInstr(MachineInstr &MI, SPIRV::ModuleAnalysisInfo &MAI,
649 SPIRV::ModuleSectionType MSType, InstrTraces &IS,
650 bool Append = true) {
651 MAI.setSkipEmission(&MI);
652 InstrSignature MISign = instrToSignature(MI, MAI, UseDefReg: true);
653 auto FoundMI = IS.insert(x: std::move(MISign));
654 if (!FoundMI.second) {
655 if (MI.getOpcode() == SPIRV::OpDecorate) {
656 assert(MI.getNumOperands() >= 2 &&
657 "Decoration instructions must have at least 2 operands");
658 assert(MSType == SPIRV::MB_Annotations &&
659 "Only OpDecorate instructions can be duplicates");
660 // For FPFastMathMode decoration, we need to merge the flags of the
661 // duplicate decoration with the original one, so we need to find the
662 // original instruction that has the same signature. For the rest of
663 // instructions, we will simply skip the duplicate.
664 if (MI.getOperand(i: 1).getImm() != SPIRV::Decoration::FPFastMathMode)
665 return; // Skip duplicates of other decorations.
666
667 const SPIRV::InstrList &Decorations = MAI.MS[MSType];
668 for (const MachineInstr *OrigMI : Decorations) {
669 if (instrToSignature(MI: *OrigMI, MAI, UseDefReg: true) == MISign) {
670 assert(OrigMI->getNumOperands() == MI.getNumOperands() &&
671 "Original instruction must have the same number of operands");
672 assert(
673 OrigMI->getNumOperands() == 3 &&
674 "FPFastMathMode decoration must have 3 operands for OpDecorate");
675 unsigned OrigFlags = OrigMI->getOperand(i: 2).getImm();
676 unsigned NewFlags = MI.getOperand(i: 2).getImm();
677 if (OrigFlags == NewFlags)
678 return; // No need to merge, the flags are the same.
679
680 // Emit warning about possible conflict between flags.
681 unsigned FinalFlags = OrigFlags | NewFlags;
682 llvm::errs()
683 << "Warning: Conflicting FPFastMathMode decoration flags "
684 "in instruction: "
685 << *OrigMI << "Original flags: " << OrigFlags
686 << ", new flags: " << NewFlags
687 << ". They will be merged on a best effort basis, but not "
688 "validated. Final flags: "
689 << FinalFlags << "\n";
690 MachineInstr *OrigMINonConst = const_cast<MachineInstr *>(OrigMI);
691 MachineOperand &OrigFlagsOp = OrigMINonConst->getOperand(i: 2);
692 OrigFlagsOp = MachineOperand::CreateImm(Val: FinalFlags);
693 return; // Merge done, so we found a duplicate; don't add it to MAI.MS
694 }
695 }
696 assert(false && "No original instruction found for the duplicate "
697 "OpDecorate, but we found one in IS.");
698 }
699 return; // insert failed, so we found a duplicate; don't add it to MAI.MS
700 }
701 // No duplicates, so add it.
702 if (Append)
703 MAI.MS[MSType].push_back(Elt: &MI);
704 else
705 MAI.MS[MSType].insert(I: MAI.MS[MSType].begin(), Elt: &MI);
706}
707
708// Some global instructions make reference to function-local ID regs, so cannot
709// be correctly collected until these registers are globally numbered.
710void SPIRVModuleAnalysis::processOtherInstrs(const Module &M) {
711 InstrTraces IS;
712 for (const Function &F : M) {
713 if (F.isDeclaration())
714 continue;
715 MachineFunction *MF = MMI->getMachineFunction(F);
716 assert(MF);
717
718 for (MachineBasicBlock &MBB : *MF)
719 for (MachineInstr &MI : MBB) {
720 if (MAI.getSkipEmission(MI: &MI))
721 continue;
722 const unsigned OpCode = MI.getOpcode();
723 if (OpCode == SPIRV::OpString) {
724 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_DebugStrings, IS);
725 } else if (OpCode == SPIRV::OpExtInst && MI.getOperand(i: 2).isImm() &&
726 MI.getOperand(i: 2).getImm() ==
727 SPIRV::InstructionSet::
728 NonSemantic_Shader_DebugInfo_100) {
729 // TODO: This branch is dead. SPIRVNonSemanticDebugHandler emits NSDI
730 // instructions directly as MCInsts at print time; no
731 // MachineInstructions with the NSDI ext set are created anymore.
732 // Remove this block and
733 // MB_NonSemanticGlobalDI once per-function NSDI emission is confirmed
734 // not to need MIR routing.
735 MachineOperand Ins = MI.getOperand(i: 3);
736 namespace NS = SPIRV::NonSemanticExtInst;
737 static constexpr int64_t GlobalNonSemanticDITy[] = {
738 NS::DebugSource, NS::DebugCompilationUnit, NS::DebugInfoNone,
739 NS::DebugTypeBasic, NS::DebugTypePointer};
740 bool IsGlobalDI = false;
741 for (unsigned Idx = 0; Idx < std::size(GlobalNonSemanticDITy); ++Idx)
742 IsGlobalDI |= Ins.getImm() == GlobalNonSemanticDITy[Idx];
743 if (IsGlobalDI)
744 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_NonSemanticGlobalDI, IS);
745 } else if (OpCode == SPIRV::OpName || OpCode == SPIRV::OpMemberName) {
746 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_DebugNames, IS);
747 } else if (OpCode == SPIRV::OpEntryPoint) {
748 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_EntryPoints, IS);
749 } else if (TII->isAliasingInstr(MI)) {
750 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_AliasingInsts, IS);
751 } else if (TII->isDecorationInstr(MI)) {
752 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_Annotations, IS);
753 collectFuncNames(MI, F: &F);
754 } else if (TII->isConstantInstr(MI)) {
755 // Now OpSpecConstant*s are not in DT,
756 // but they need to be collected anyway.
757 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_TypeConstVars, IS);
758 } else if (OpCode == SPIRV::OpFunction) {
759 collectFuncNames(MI, F: &F);
760 } else if (OpCode == SPIRV::OpTypeForwardPointer) {
761 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_TypeConstVars, IS, Append: false);
762 }
763 }
764 }
765 // Selection order can place a scope/list ahead of a domain/scope it
766 // references. The dependency meanwhile is domain -> scope -> list, so sort
767 // the def before its uses.
768 auto AliasingTier = [](const MachineInstr *MI) {
769 switch (MI->getOpcode()) {
770 case SPIRV::OpAliasDomainDeclINTEL:
771 return 0;
772 case SPIRV::OpAliasScopeDeclINTEL:
773 return 1;
774 case SPIRV::OpAliasScopeListDeclINTEL:
775 return 2;
776 default:
777 llvm_unreachable("unexpected aliasing instruction");
778 }
779 };
780 stable_sort(Range&: MAI.MS[SPIRV::MB_AliasingInsts],
781 C: [&](const MachineInstr *LHS, const MachineInstr *RHS) {
782 return AliasingTier(LHS) < AliasingTier(RHS);
783 });
784}
785
786// Number registers in all functions globally from 0 onwards and store
787// the result in global register alias table. Some registers are already
788// numbered.
789void SPIRVModuleAnalysis::numberRegistersGlobally(const Module &M) {
790 for (const Function &F : M) {
791 if (F.isDeclaration())
792 continue;
793 MachineFunction *MF = MMI->getMachineFunction(F);
794 assert(MF);
795 for (MachineBasicBlock &MBB : *MF) {
796 for (MachineInstr &MI : MBB) {
797 for (MachineOperand &Op : MI.operands()) {
798 if (!Op.isReg())
799 continue;
800 Register Reg = Op.getReg();
801 if (MAI.hasRegisterAlias(MF, Reg))
802 continue;
803 MCRegister NewReg = MAI.getNextIDRegister();
804 MAI.setRegisterAlias(MF, Reg, AliasReg: NewReg);
805 }
806 if (MI.getOpcode() != SPIRV::OpExtInst)
807 continue;
808 auto Set = MI.getOperand(i: 2).getImm();
809 auto [It, Inserted] = MAI.ExtInstSetMap.try_emplace(Key: Set);
810 if (Inserted)
811 It->second = MAI.getNextIDRegister();
812 }
813 }
814 }
815}
816
817// RequirementHandler implementations.
818void SPIRV::RequirementHandler::getAndAddRequirements(
819 SPIRV::OperandCategory::OperandCategory Category, uint32_t i,
820 const SPIRVSubtarget &ST) {
821 addRequirements(Req: getSymbolicOperandRequirements(Category, i, ST, Reqs&: *this));
822}
823
824void SPIRV::RequirementHandler::recursiveAddCapabilities(
825 const CapabilityList &ToPrune) {
826 for (const auto &Cap : ToPrune) {
827 AllCaps.insert(V: Cap);
828 CapabilityList ImplicitDecls =
829 getSymbolicOperandCapabilities(Category: OperandCategory::CapabilityOperand, Value: Cap);
830 recursiveAddCapabilities(ToPrune: ImplicitDecls);
831 }
832}
833
834void SPIRV::RequirementHandler::addCapabilities(const CapabilityList &ToAdd) {
835 for (const auto &Cap : ToAdd) {
836 bool IsNewlyInserted = AllCaps.insert(V: Cap).second;
837 if (!IsNewlyInserted) // Don't re-add if it's already been declared.
838 continue;
839 CapabilityList ImplicitDecls =
840 getSymbolicOperandCapabilities(Category: OperandCategory::CapabilityOperand, Value: Cap);
841 recursiveAddCapabilities(ToPrune: ImplicitDecls);
842 MinimalCaps.push_back(Elt: Cap);
843 }
844}
845
846void SPIRV::RequirementHandler::addRequirements(
847 const SPIRV::Requirements &Req) {
848 if (!Req.IsSatisfiable)
849 report_fatal_error(reason: "Adding SPIR-V requirements this target can't satisfy.");
850
851 if (Req.Cap.has_value())
852 addCapabilities(ToAdd: {Req.Cap.value()});
853
854 addExtensions(ToAdd: Req.Exts);
855
856 if (!Req.MinVer.empty()) {
857 if (!MaxVersion.empty() && Req.MinVer > MaxVersion) {
858 LLVM_DEBUG(dbgs() << "Conflicting version requirements: >= " << Req.MinVer
859 << " and <= " << MaxVersion << "\n");
860 report_fatal_error(reason: "Adding SPIR-V requirements that can't be satisfied.");
861 }
862
863 if (MinVersion.empty() || Req.MinVer > MinVersion)
864 MinVersion = Req.MinVer;
865 }
866
867 if (!Req.MaxVer.empty()) {
868 if (!MinVersion.empty() && Req.MaxVer < MinVersion) {
869 LLVM_DEBUG(dbgs() << "Conflicting version requirements: <= " << Req.MaxVer
870 << " and >= " << MinVersion << "\n");
871 report_fatal_error(reason: "Adding SPIR-V requirements that can't be satisfied.");
872 }
873
874 if (MaxVersion.empty() || Req.MaxVer < MaxVersion)
875 MaxVersion = Req.MaxVer;
876 }
877}
878
879void SPIRV::RequirementHandler::checkSatisfiable(
880 const SPIRVSubtarget &ST) const {
881 // Report as many errors as possible before aborting the compilation.
882 bool IsSatisfiable = true;
883 auto TargetVer = ST.getSPIRVVersion();
884
885 if (!MaxVersion.empty() && !TargetVer.empty() && MaxVersion < TargetVer) {
886 LLVM_DEBUG(
887 dbgs() << "Target SPIR-V version too high for required features\n"
888 << "Required max version: " << MaxVersion << " target version "
889 << TargetVer << "\n");
890 IsSatisfiable = false;
891 }
892
893 if (!MinVersion.empty() && !TargetVer.empty() && MinVersion > TargetVer) {
894 LLVM_DEBUG(dbgs() << "Target SPIR-V version too low for required features\n"
895 << "Required min version: " << MinVersion
896 << " target version " << TargetVer << "\n");
897 IsSatisfiable = false;
898 }
899
900 if (!MinVersion.empty() && !MaxVersion.empty() && MinVersion > MaxVersion) {
901 LLVM_DEBUG(
902 dbgs()
903 << "Version is too low for some features and too high for others.\n"
904 << "Required SPIR-V min version: " << MinVersion
905 << " required SPIR-V max version " << MaxVersion << "\n");
906 IsSatisfiable = false;
907 }
908
909 AvoidCapabilitiesSet AvoidCaps;
910 if (!ST.isShader())
911 AvoidCaps.S.insert(V: SPIRV::Capability::Shader);
912 else
913 AvoidCaps.S.insert(V: SPIRV::Capability::Kernel);
914
915 for (auto Cap : MinimalCaps) {
916 if (AvailableCaps.contains(V: Cap) && !AvoidCaps.S.contains(V: Cap))
917 continue;
918 LLVM_DEBUG(dbgs() << "Capability not supported: "
919 << getSymbolicOperandMnemonic(
920 OperandCategory::CapabilityOperand, Cap)
921 << "\n");
922 IsSatisfiable = false;
923 }
924
925 for (auto Ext : AllExtensions) {
926 if (ST.canUseExtension(E: Ext))
927 continue;
928 LLVM_DEBUG(dbgs() << "Extension not supported: "
929 << getSymbolicOperandMnemonic(
930 OperandCategory::ExtensionOperand, Ext)
931 << "\n");
932 IsSatisfiable = false;
933 }
934
935 if (!IsSatisfiable)
936 report_fatal_error(reason: "Unable to meet SPIR-V requirements for this target.");
937}
938
939// Add the given capabilities and all their implicitly defined capabilities too.
940void SPIRV::RequirementHandler::addAvailableCaps(const CapabilityList &ToAdd) {
941 for (const auto Cap : ToAdd)
942 if (AvailableCaps.insert(V: Cap).second)
943 addAvailableCaps(ToAdd: getSymbolicOperandCapabilities(
944 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
945}
946
947void SPIRV::RequirementHandler::removeCapabilityIf(
948 const Capability::Capability ToRemove,
949 const Capability::Capability IfPresent) {
950 if (AllCaps.contains(V: IfPresent)) {
951 AllCaps.erase(V: ToRemove);
952 llvm::erase(C&: MinimalCaps, V: ToRemove);
953 }
954}
955
956namespace llvm {
957namespace SPIRV {
958void RequirementHandler::initAvailableCapabilities(const SPIRVSubtarget &ST) {
959 // Provided by both all supported Vulkan versions and OpenCl.
960 addAvailableCaps(ToAdd: {Capability::Shader, Capability::Linkage, Capability::Int8,
961 Capability::Int16});
962
963 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 3)))
964 addAvailableCaps(ToAdd: {Capability::GroupNonUniform,
965 Capability::GroupNonUniformVote,
966 Capability::GroupNonUniformArithmetic,
967 Capability::GroupNonUniformBallot,
968 Capability::GroupNonUniformClustered,
969 Capability::GroupNonUniformShuffle,
970 Capability::GroupNonUniformShuffleRelative,
971 Capability::GroupNonUniformQuad});
972
973 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
974 addAvailableCaps(ToAdd: {Capability::DotProduct, Capability::DotProductInputAll,
975 Capability::DotProductInput4x8Bit,
976 Capability::DotProductInput4x8BitPacked,
977 Capability::DemoteToHelperInvocation});
978
979 // Add capabilities enabled by extensions.
980 for (auto Extension : ST.getAllAvailableExtensions()) {
981 CapabilityList EnabledCapabilities =
982 getCapabilitiesEnabledByExtension(Extension);
983 addAvailableCaps(ToAdd: EnabledCapabilities);
984 }
985
986 if (!ST.isShader()) {
987 initAvailableCapabilitiesForOpenCL(ST);
988 return;
989 }
990
991 if (ST.isShader()) {
992 initAvailableCapabilitiesForVulkan(ST);
993 return;
994 }
995
996 report_fatal_error(reason: "Unimplemented environment for SPIR-V generation.");
997}
998
999void RequirementHandler::initAvailableCapabilitiesForOpenCL(
1000 const SPIRVSubtarget &ST) {
1001 // Add the min requirements for different OpenCL and SPIR-V versions.
1002 addAvailableCaps(ToAdd: {Capability::Addresses, Capability::Float16Buffer,
1003 Capability::Kernel, Capability::Vector16,
1004 Capability::Groups, Capability::GenericPointer,
1005 Capability::StorageImageWriteWithoutFormat,
1006 Capability::StorageImageReadWithoutFormat});
1007 if (ST.hasOpenCLFullProfile())
1008 addAvailableCaps(ToAdd: {Capability::Int64, Capability::Int64Atomics});
1009 if (ST.hasOpenCLImageSupport()) {
1010 addAvailableCaps(ToAdd: {Capability::ImageBasic, Capability::LiteralSampler,
1011 Capability::Image1D, Capability::SampledBuffer,
1012 Capability::ImageBuffer});
1013 if (ST.isAtLeastOpenCLVer(VerToCompareTo: VersionTuple(2, 0)))
1014 addAvailableCaps(ToAdd: {Capability::ImageReadWrite});
1015 }
1016 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 1)) &&
1017 ST.isAtLeastOpenCLVer(VerToCompareTo: VersionTuple(2, 2)))
1018 addAvailableCaps(ToAdd: {Capability::SubgroupDispatch, Capability::PipeStorage});
1019 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 4)))
1020 addAvailableCaps(ToAdd: {Capability::DenormPreserve, Capability::DenormFlushToZero,
1021 Capability::SignedZeroInfNanPreserve,
1022 Capability::RoundingModeRTE,
1023 Capability::RoundingModeRTZ});
1024 // TODO: verify if this needs some checks.
1025 addAvailableCaps(ToAdd: {Capability::Float16, Capability::Float64});
1026
1027 // TODO: add OpenCL extensions.
1028}
1029
1030void RequirementHandler::initAvailableCapabilitiesForVulkan(
1031 const SPIRVSubtarget &ST) {
1032
1033 // Core in Vulkan 1.1 and earlier.
1034 addAvailableCaps(ToAdd: {Capability::Int64,
1035 Capability::Float16,
1036 Capability::Float64,
1037 Capability::GroupNonUniform,
1038 Capability::Image1D,
1039 Capability::SampledBuffer,
1040 Capability::ImageBuffer,
1041 Capability::UniformBufferArrayDynamicIndexing,
1042 Capability::SampledImageArrayDynamicIndexing,
1043 Capability::StorageBufferArrayDynamicIndexing,
1044 Capability::StorageImageArrayDynamicIndexing,
1045 Capability::DerivativeControl,
1046 Capability::MinLod,
1047 Capability::ImageQuery,
1048 Capability::ImageGatherExtended,
1049 Capability::Addresses,
1050 Capability::VulkanMemoryModelKHR,
1051 Capability::StorageImageExtendedFormats,
1052 Capability::StorageImageMultisample,
1053 Capability::ImageMSArray});
1054
1055 // Became core in Vulkan 1.2
1056 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 5))) {
1057 addAvailableCaps(
1058 ToAdd: {Capability::Int64Atomics, Capability::ShaderNonUniformEXT,
1059 Capability::RuntimeDescriptorArrayEXT,
1060 Capability::InputAttachmentArrayDynamicIndexingEXT,
1061 Capability::UniformTexelBufferArrayDynamicIndexingEXT,
1062 Capability::StorageTexelBufferArrayDynamicIndexingEXT,
1063 Capability::UniformBufferArrayNonUniformIndexingEXT,
1064 Capability::SampledImageArrayNonUniformIndexingEXT,
1065 Capability::StorageBufferArrayNonUniformIndexingEXT,
1066 Capability::StorageImageArrayNonUniformIndexingEXT,
1067 Capability::InputAttachmentArrayNonUniformIndexingEXT,
1068 Capability::UniformTexelBufferArrayNonUniformIndexingEXT,
1069 Capability::StorageTexelBufferArrayNonUniformIndexingEXT});
1070 }
1071
1072 // Became core in Vulkan 1.3
1073 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
1074 addAvailableCaps(ToAdd: {Capability::StorageImageWriteWithoutFormat,
1075 Capability::StorageImageReadWithoutFormat});
1076}
1077
1078} // namespace SPIRV
1079} // namespace llvm
1080
1081// Add the required capabilities from a decoration instruction (including
1082// BuiltIns).
1083static void addOpDecorateReqs(const MachineInstr &MI, unsigned DecIndex,
1084 SPIRV::RequirementHandler &Reqs,
1085 const SPIRVSubtarget &ST) {
1086 int64_t DecOp = MI.getOperand(i: DecIndex).getImm();
1087 auto Dec = static_cast<SPIRV::Decoration::Decoration>(DecOp);
1088 Reqs.addRequirements(Req: getSymbolicOperandRequirements(
1089 Category: SPIRV::OperandCategory::DecorationOperand, i: Dec, ST, Reqs));
1090
1091 if (Dec == SPIRV::Decoration::BuiltIn) {
1092 int64_t BuiltInOp = MI.getOperand(i: DecIndex + 1).getImm();
1093 auto BuiltIn = static_cast<SPIRV::BuiltIn::BuiltIn>(BuiltInOp);
1094 Reqs.addRequirements(Req: getSymbolicOperandRequirements(
1095 Category: SPIRV::OperandCategory::BuiltInOperand, i: BuiltIn, ST, Reqs));
1096 } else if (Dec == SPIRV::Decoration::LinkageAttributes) {
1097 int64_t LinkageOp = MI.getOperand(i: MI.getNumOperands() - 1).getImm();
1098 SPIRV::LinkageType::LinkageType LnkType =
1099 static_cast<SPIRV::LinkageType::LinkageType>(LinkageOp);
1100 if (LnkType == SPIRV::LinkageType::LinkOnceODR)
1101 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_linkonce_odr);
1102 else if (LnkType == SPIRV::LinkageType::WeakAMD) {
1103 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_AMD_weak_linkage);
1104 Reqs.addCapability(ToAdd: SPIRV::Capability::WeakLinkageAMD);
1105 }
1106 } else if (Dec == SPIRV::Decoration::CacheControlLoadINTEL ||
1107 Dec == SPIRV::Decoration::CacheControlStoreINTEL) {
1108 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_cache_controls);
1109 } else if (Dec == SPIRV::Decoration::HostAccessINTEL) {
1110 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_global_variable_host_access);
1111 } else if (Dec == SPIRV::Decoration::InitModeINTEL ||
1112 Dec == SPIRV::Decoration::ImplementInRegisterMapINTEL) {
1113 Reqs.addExtension(
1114 ToAdd: SPIRV::Extension::SPV_INTEL_global_variable_fpga_decorations);
1115 } else if (Dec == SPIRV::Decoration::NonUniformEXT) {
1116 Reqs.addRequirements(Req: SPIRV::Capability::ShaderNonUniformEXT);
1117 } else if (Dec == SPIRV::Decoration::FPMaxErrorDecorationINTEL) {
1118 Reqs.addRequirements(Req: SPIRV::Capability::FPMaxErrorINTEL);
1119 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_fp_max_error);
1120 } else if (Dec == SPIRV::Decoration::FPFastMathMode) {
1121 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2)) {
1122 Reqs.addRequirements(Req: SPIRV::Capability::FloatControls2);
1123 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls2);
1124 }
1125 }
1126}
1127
1128// Add requirements for image handling.
1129static void addOpTypeImageReqs(const MachineInstr &MI,
1130 SPIRV::RequirementHandler &Reqs,
1131 const SPIRVSubtarget &ST) {
1132 assert(MI.getNumOperands() >= 8 && "Insufficient operands for OpTypeImage");
1133 // The operand indices used here are based on the OpTypeImage layout, which
1134 // the MachineInstr follows as well.
1135 int64_t ImgFormatOp = MI.getOperand(i: 7).getImm();
1136 auto ImgFormat = static_cast<SPIRV::ImageFormat::ImageFormat>(ImgFormatOp);
1137 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ImageFormatOperand,
1138 i: ImgFormat, ST);
1139
1140 bool IsArrayed = MI.getOperand(i: 4).getImm() == 1;
1141 bool IsMultisampled = MI.getOperand(i: 5).getImm() == 1;
1142 bool NoSampler = MI.getOperand(i: 6).getImm() == 2;
1143 // Add dimension requirements.
1144 assert(MI.getOperand(2).isImm());
1145 switch (MI.getOperand(i: 2).getImm()) {
1146 case SPIRV::Dim::DIM_1D:
1147 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::Image1D
1148 : SPIRV::Capability::Sampled1D);
1149 break;
1150 case SPIRV::Dim::DIM_2D:
1151 if (IsMultisampled && NoSampler)
1152 Reqs.addRequirements(Req: SPIRV::Capability::StorageImageMultisample);
1153 if (IsMultisampled && IsArrayed)
1154 Reqs.addRequirements(Req: SPIRV::Capability::ImageMSArray);
1155 break;
1156 case SPIRV::Dim::DIM_3D:
1157 break;
1158 case SPIRV::Dim::DIM_Cube:
1159 Reqs.addRequirements(Req: SPIRV::Capability::Shader);
1160 if (IsArrayed)
1161 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageCubeArray
1162 : SPIRV::Capability::SampledCubeArray);
1163 break;
1164 case SPIRV::Dim::DIM_Rect:
1165 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageRect
1166 : SPIRV::Capability::SampledRect);
1167 break;
1168 case SPIRV::Dim::DIM_Buffer:
1169 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageBuffer
1170 : SPIRV::Capability::SampledBuffer);
1171 break;
1172 case SPIRV::Dim::DIM_SubpassData:
1173 Reqs.addRequirements(Req: SPIRV::Capability::InputAttachment);
1174 break;
1175 }
1176
1177 // Has optional access qualifier.
1178 if (!ST.isShader()) {
1179 if (MI.getNumOperands() > 8 &&
1180 MI.getOperand(i: 8).getImm() == SPIRV::AccessQualifier::ReadWrite)
1181 Reqs.addRequirements(Req: SPIRV::Capability::ImageReadWrite);
1182 else
1183 Reqs.addRequirements(Req: SPIRV::Capability::ImageBasic);
1184 }
1185}
1186
1187static bool isBFloat16Type(SPIRVTypeInst TypeDef) {
1188 return TypeDef && TypeDef->getNumOperands() == 3 &&
1189 TypeDef->getOpcode() == SPIRV::OpTypeFloat &&
1190 TypeDef->getOperand(i: 1).getImm() == 16 &&
1191 TypeDef->getOperand(i: 2).getImm() == SPIRV::FPEncoding::BFloat16KHR;
1192}
1193
1194// Add requirements for handling atomic float instructions
1195#define ATOM_FLT_REQ_EXT_MSG(ExtName) \
1196 "The atomic float instruction requires the following SPIR-V " \
1197 "extension: SPV_EXT_shader_atomic_float" ExtName
1198static void AddAtomicVectorFloatRequirements(const MachineInstr &MI,
1199 SPIRV::RequirementHandler &Reqs,
1200 const SPIRVSubtarget &ST) {
1201 SPIRVTypeInst VecTypeDef =
1202 MI.getMF()->getRegInfo().getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1203
1204 const unsigned Rank = VecTypeDef->getOperand(i: 2).getImm();
1205 if (Rank != 2 && Rank != 4)
1206 reportFatalUsageError(reason: "Result type of an atomic vector float instruction "
1207 "must be a 2-component or 4 component vector");
1208
1209 SPIRVTypeInst EltTypeDef =
1210 MI.getMF()->getRegInfo().getVRegDef(Reg: VecTypeDef->getOperand(i: 1).getReg());
1211
1212 if (EltTypeDef->getOpcode() != SPIRV::OpTypeFloat ||
1213 EltTypeDef->getOperand(i: 1).getImm() != 16)
1214 reportFatalUsageError(
1215 reason: "The element type for the result type of an atomic vector float "
1216 "instruction must be a 16-bit floating-point scalar");
1217
1218 // The extension is defined for fp16, but the AMD target lets a bf16 vector
1219 // use the same instruction so it can lower to a packed bf16 atomic.
1220 if (isBFloat16Type(TypeDef: EltTypeDef) &&
1221 ST.getTargetTriple().getVendor() != Triple::AMD)
1222 reportFatalUsageError(
1223 reason: "The element type for the result type of an atomic vector float "
1224 "instruction cannot be a bfloat16 scalar");
1225 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_NV_shader_atomic_fp16_vector))
1226 reportFatalUsageError(
1227 reason: "The atomic float16 vector instruction requires the following SPIR-V "
1228 "extension: SPV_NV_shader_atomic_fp16_vector");
1229
1230 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_NV_shader_atomic_fp16_vector);
1231 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16VectorNV);
1232}
1233
1234static void AddAtomicFloatRequirements(const MachineInstr &MI,
1235 SPIRV::RequirementHandler &Reqs,
1236 const SPIRVSubtarget &ST) {
1237 assert(MI.getOperand(1).isReg() &&
1238 "Expect register operand in atomic float instruction");
1239 Register TypeReg = MI.getOperand(i: 1).getReg();
1240 SPIRVTypeInst TypeDef = MI.getMF()->getRegInfo().getVRegDef(Reg: TypeReg);
1241
1242 if (isVectorType(SPVTy: TypeDef))
1243 return AddAtomicVectorFloatRequirements(MI, Reqs, ST);
1244
1245 if (TypeDef->getOpcode() != SPIRV::OpTypeFloat)
1246 report_fatal_error(reason: "Result type of an atomic float instruction must be a "
1247 "floating-point type scalar");
1248
1249 unsigned BitWidth = TypeDef->getOperand(i: 1).getImm();
1250 unsigned Op = MI.getOpcode();
1251 if (Op == SPIRV::OpAtomicFAddEXT) {
1252 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_shader_atomic_float_add))
1253 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("_add"), gen_crash_diag: false);
1254 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float_add);
1255 switch (BitWidth) {
1256 case 16:
1257 if (isBFloat16Type(TypeDef)) {
1258 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1259 report_fatal_error(
1260 reason: "The atomic bfloat16 instruction requires the following SPIR-V "
1261 "extension: SPV_INTEL_16bit_atomics",
1262 gen_crash_diag: false);
1263 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1264 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicBFloat16AddINTEL);
1265 } else {
1266 if (!ST.canUseExtension(
1267 E: SPIRV::Extension::SPV_EXT_shader_atomic_float16_add))
1268 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("16_add"), gen_crash_diag: false);
1269 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float16_add);
1270 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16AddEXT);
1271 }
1272 break;
1273 case 32:
1274 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat32AddEXT);
1275 break;
1276 case 64:
1277 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat64AddEXT);
1278 break;
1279 default:
1280 report_fatal_error(
1281 reason: "Unexpected floating-point type width in atomic float instruction");
1282 }
1283 } else {
1284 if (!ST.canUseExtension(
1285 E: SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max))
1286 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("_min_max"), gen_crash_diag: false);
1287 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max);
1288 switch (BitWidth) {
1289 case 16:
1290 if (isBFloat16Type(TypeDef)) {
1291 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1292 report_fatal_error(
1293 reason: "The atomic bfloat16 instruction requires the following SPIR-V "
1294 "extension: SPV_INTEL_16bit_atomics",
1295 gen_crash_diag: false);
1296 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1297 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicBFloat16MinMaxINTEL);
1298 } else {
1299 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16MinMaxEXT);
1300 }
1301 break;
1302 case 32:
1303 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat32MinMaxEXT);
1304 break;
1305 case 64:
1306 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat64MinMaxEXT);
1307 break;
1308 default:
1309 report_fatal_error(
1310 reason: "Unexpected floating-point type width in atomic float instruction");
1311 }
1312 }
1313}
1314
1315bool isUniformTexelBuffer(MachineInstr *ImageInst) {
1316 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1317 return false;
1318 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1319 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1320 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 1;
1321}
1322
1323bool isStorageTexelBuffer(MachineInstr *ImageInst) {
1324 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1325 return false;
1326 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1327 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1328 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 2;
1329}
1330
1331bool isSampledImage(MachineInstr *ImageInst) {
1332 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1333 return false;
1334 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1335 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1336 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 1;
1337}
1338
1339bool isInputAttachment(MachineInstr *ImageInst) {
1340 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1341 return false;
1342 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1343 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1344 return Dim == SPIRV::Dim::DIM_SubpassData && Sampled == 2;
1345}
1346
1347bool isStorageImage(MachineInstr *ImageInst) {
1348 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1349 return false;
1350 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1351 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1352 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 2;
1353}
1354
1355bool isCombinedImageSampler(MachineInstr *SampledImageInst) {
1356 if (SampledImageInst->getOpcode() != SPIRV::OpTypeSampledImage)
1357 return false;
1358
1359 const MachineRegisterInfo &MRI = SampledImageInst->getMF()->getRegInfo();
1360 Register ImageReg = SampledImageInst->getOperand(i: 1).getReg();
1361 auto *ImageInst = MRI.getUniqueVRegDef(Reg: ImageReg);
1362 return isSampledImage(ImageInst);
1363}
1364
1365bool hasNonUniformDecoration(Register Reg, const MachineRegisterInfo &MRI) {
1366 for (const auto &MI : MRI.reg_instructions(Reg)) {
1367 if (MI.getOpcode() != SPIRV::OpDecorate)
1368 continue;
1369
1370 uint32_t Dec = MI.getOperand(i: 1).getImm();
1371 if (Dec == SPIRV::Decoration::NonUniformEXT)
1372 return true;
1373 }
1374 return false;
1375}
1376
1377void addOpAccessChainReqs(const MachineInstr &Instr,
1378 SPIRV::RequirementHandler &Handler,
1379 const SPIRVSubtarget &Subtarget) {
1380 const MachineRegisterInfo &MRI = Instr.getMF()->getRegInfo();
1381 // Get the result type. If it is an image type, then the shader uses
1382 // descriptor indexing. The appropriate capabilities will be added based
1383 // on the specifics of the image.
1384 Register ResTypeReg = Instr.getOperand(i: 1).getReg();
1385 MachineInstr *ResTypeInst = MRI.getUniqueVRegDef(Reg: ResTypeReg);
1386
1387 assert(ResTypeInst->getOpcode() == SPIRV::OpTypePointer);
1388 uint32_t StorageClass = ResTypeInst->getOperand(i: 1).getImm();
1389 if (StorageClass != SPIRV::StorageClass::StorageClass::UniformConstant &&
1390 StorageClass != SPIRV::StorageClass::StorageClass::Uniform &&
1391 StorageClass != SPIRV::StorageClass::StorageClass::StorageBuffer) {
1392 return;
1393 }
1394
1395 bool IsNonUniform =
1396 hasNonUniformDecoration(Reg: Instr.getOperand(i: 0).getReg(), MRI);
1397
1398 auto FirstIndexReg = Instr.getOperand(i: 3).getReg();
1399 bool FirstIndexIsConstant =
1400 Subtarget.getInstrInfo()->isConstantInstr(MI: *MRI.getVRegDef(Reg: FirstIndexReg));
1401
1402 if (StorageClass == SPIRV::StorageClass::StorageClass::StorageBuffer) {
1403 if (IsNonUniform)
1404 Handler.addRequirements(
1405 Req: SPIRV::Capability::StorageBufferArrayNonUniformIndexingEXT);
1406 else if (!FirstIndexIsConstant)
1407 Handler.addRequirements(
1408 Req: SPIRV::Capability::StorageBufferArrayDynamicIndexing);
1409 return;
1410 }
1411
1412 Register PointeeTypeReg = ResTypeInst->getOperand(i: 2).getReg();
1413 MachineInstr *PointeeType = MRI.getUniqueVRegDef(Reg: PointeeTypeReg);
1414 if (PointeeType->getOpcode() != SPIRV::OpTypeImage &&
1415 PointeeType->getOpcode() != SPIRV::OpTypeSampledImage &&
1416 PointeeType->getOpcode() != SPIRV::OpTypeSampler) {
1417 return;
1418 }
1419
1420 if (isUniformTexelBuffer(ImageInst: PointeeType)) {
1421 if (IsNonUniform)
1422 Handler.addRequirements(
1423 Req: SPIRV::Capability::UniformTexelBufferArrayNonUniformIndexingEXT);
1424 else if (!FirstIndexIsConstant)
1425 Handler.addRequirements(
1426 Req: SPIRV::Capability::UniformTexelBufferArrayDynamicIndexingEXT);
1427 } else if (isInputAttachment(ImageInst: PointeeType)) {
1428 if (IsNonUniform)
1429 Handler.addRequirements(
1430 Req: SPIRV::Capability::InputAttachmentArrayNonUniformIndexingEXT);
1431 else if (!FirstIndexIsConstant)
1432 Handler.addRequirements(
1433 Req: SPIRV::Capability::InputAttachmentArrayDynamicIndexingEXT);
1434 } else if (isStorageTexelBuffer(ImageInst: PointeeType)) {
1435 if (IsNonUniform)
1436 Handler.addRequirements(
1437 Req: SPIRV::Capability::StorageTexelBufferArrayNonUniformIndexingEXT);
1438 else if (!FirstIndexIsConstant)
1439 Handler.addRequirements(
1440 Req: SPIRV::Capability::StorageTexelBufferArrayDynamicIndexingEXT);
1441 } else if (isSampledImage(ImageInst: PointeeType) ||
1442 isCombinedImageSampler(SampledImageInst: PointeeType) ||
1443 PointeeType->getOpcode() == SPIRV::OpTypeSampler) {
1444 if (IsNonUniform)
1445 Handler.addRequirements(
1446 Req: SPIRV::Capability::SampledImageArrayNonUniformIndexingEXT);
1447 else if (!FirstIndexIsConstant)
1448 Handler.addRequirements(
1449 Req: SPIRV::Capability::SampledImageArrayDynamicIndexing);
1450 } else if (isStorageImage(ImageInst: PointeeType)) {
1451 if (IsNonUniform)
1452 Handler.addRequirements(
1453 Req: SPIRV::Capability::StorageImageArrayNonUniformIndexingEXT);
1454 else if (!FirstIndexIsConstant)
1455 Handler.addRequirements(
1456 Req: SPIRV::Capability::StorageImageArrayDynamicIndexing);
1457 }
1458}
1459
1460static bool isImageTypeWithUnknownFormat(SPIRVTypeInst TypeInst) {
1461 if (TypeInst->getOpcode() != SPIRV::OpTypeImage)
1462 return false;
1463 assert(TypeInst->getOperand(7).isImm() && "The image format must be an imm.");
1464 return TypeInst->getOperand(i: 7).getImm() == 0;
1465}
1466
1467static void AddDotProductRequirements(const MachineInstr &MI,
1468 SPIRV::RequirementHandler &Reqs,
1469 const SPIRVSubtarget &ST) {
1470 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product))
1471 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_integer_dot_product);
1472 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProduct);
1473
1474 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1475 assert(MI.getOperand(2).isReg() && "Unexpected operand in dot");
1476 // We do not consider what the previous instruction is. This is just used
1477 // to get the input register and to check the type.
1478 const MachineInstr *Input = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
1479 assert(Input->getOperand(1).isReg() && "Unexpected operand in dot input");
1480 Register InputReg = Input->getOperand(i: 1).getReg();
1481
1482 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: InputReg);
1483 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {
1484 assert(TypeDef->getOperand(1).getImm() == 32);
1485 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInput4x8BitPacked);
1486 } else if (isVectorType(SPVTy: TypeDef)) {
1487 SPIRVTypeInst ScalarTypeDef =
1488 MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
1489 assert(ScalarTypeDef->getOpcode() == SPIRV::OpTypeInt);
1490 if (ScalarTypeDef->getOperand(i: 1).getImm() == 8) {
1491 assert(TypeDef->getOperand(2).getImm() == 4 &&
1492 "Dot operand of 8-bit integer type requires 4 components");
1493 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInput4x8Bit);
1494 } else {
1495 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInputAll);
1496 }
1497 }
1498}
1499
1500void addPrintfRequirements(const MachineInstr &MI,
1501 SPIRV::RequirementHandler &Reqs,
1502 const SPIRVSubtarget &ST) {
1503 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1504 SPIRVTypeInst PtrType =
1505 GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 4).getReg(), MF: MI.getMF());
1506 if (PtrType) {
1507 MachineOperand ASOp = PtrType->getOperand(i: 1);
1508 if (ASOp.isImm()) {
1509 unsigned AddrSpace = ASOp.getImm();
1510 if (AddrSpace != SPIRV::StorageClass::UniformConstant) {
1511 if (!ST.canUseExtension(
1512 E: SPIRV::Extension::
1513 SPV_EXT_relaxed_printf_string_address_space)) {
1514 report_fatal_error(reason: "SPV_EXT_relaxed_printf_string_address_space is "
1515 "required because printf uses a format string not "
1516 "in constant address space.",
1517 gen_crash_diag: false);
1518 }
1519 Reqs.addExtension(
1520 ToAdd: SPIRV::Extension::SPV_EXT_relaxed_printf_string_address_space);
1521 }
1522 }
1523 }
1524}
1525
1526static void addImageOperandReqs(const MachineInstr &MI,
1527 SPIRV::RequirementHandler &Reqs,
1528 const SPIRVSubtarget &ST, unsigned OpIdx) {
1529 if (MI.getNumOperands() <= OpIdx)
1530 return;
1531 uint32_t Mask = MI.getOperand(i: OpIdx).getImm();
1532 for (uint32_t I = 0; I < 32; ++I)
1533 if (Mask & (1U << I))
1534 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ImageOperandOperand,
1535 i: 1U << I, ST);
1536}
1537
1538static inline void maybeAddScatterGatherReq(const MachineInstr &MI,
1539 SPIRV::RequirementHandler &Reqs,
1540 const SPIRVSubtarget &ST) {
1541 assert(MI.getOperand(1).isReg());
1542 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1543 SPIRVTypeInst ElemTypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1544 if (ElemTypeDef->getOpcode() == SPIRV::OpTypePointer &&
1545 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
1546 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_masked_gather_scatter);
1547 Reqs.addCapability(ToAdd: SPIRV::Capability::MaskedGatherScatterINTEL);
1548 }
1549}
1550
1551void addInstrRequirements(const MachineInstr &MI,
1552 SPIRV::ModuleAnalysisInfo &MAI,
1553 const SPIRVSubtarget &ST) {
1554 SPIRV::RequirementHandler &Reqs = MAI.Reqs;
1555 unsigned Op = MI.getOpcode();
1556 switch (Op) {
1557 case SPIRV::OpMemoryModel: {
1558 int64_t Addr = MI.getOperand(i: 0).getImm();
1559 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::AddressingModelOperand,
1560 i: Addr, ST);
1561 int64_t Mem = MI.getOperand(i: 1).getImm();
1562 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::MemoryModelOperand, i: Mem,
1563 ST);
1564 break;
1565 }
1566 case SPIRV::OpEntryPoint: {
1567 int64_t Exe = MI.getOperand(i: 0).getImm();
1568 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ExecutionModelOperand,
1569 i: Exe, ST);
1570 break;
1571 }
1572 case SPIRV::OpExecutionMode:
1573 case SPIRV::OpExecutionModeId: {
1574 int64_t Exe = MI.getOperand(i: 1).getImm();
1575 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ExecutionModeOperand,
1576 i: Exe, ST);
1577 break;
1578 }
1579 case SPIRV::OpTypeMatrix:
1580 Reqs.addCapability(ToAdd: SPIRV::Capability::Matrix);
1581 break;
1582 case SPIRV::OpTypeInt: {
1583 unsigned BitWidth = MI.getOperand(i: 1).getImm();
1584 if (BitWidth == 64)
1585 Reqs.addCapability(ToAdd: SPIRV::Capability::Int64);
1586 else if (BitWidth == 16)
1587 Reqs.addCapability(ToAdd: SPIRV::Capability::Int16);
1588 else if (BitWidth == 8)
1589 Reqs.addCapability(ToAdd: SPIRV::Capability::Int8);
1590 else if (BitWidth == 4 &&
1591 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4)) {
1592 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_int4);
1593 Reqs.addCapability(ToAdd: SPIRV::Capability::Int4TypeINTEL);
1594 } else if (BitWidth != 32) {
1595 if (!ST.canUseExtension(
1596 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers))
1597 reportFatalUsageError(
1598 reason: "OpTypeInt type with a width other than 8, 16, 32 or 64 bits "
1599 "requires the following SPIR-V extension: "
1600 "SPV_ALTERA_arbitrary_precision_integers");
1601 Reqs.addExtension(
1602 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
1603 Reqs.addCapability(ToAdd: SPIRV::Capability::ArbitraryPrecisionIntegersALTERA);
1604 }
1605 break;
1606 }
1607 case SPIRV::OpDot: {
1608 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1609 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1610 if (isBFloat16Type(TypeDef))
1611 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16DotProductKHR);
1612 break;
1613 }
1614 case SPIRV::OpTypeFloat: {
1615 unsigned BitWidth = MI.getOperand(i: 1).getImm();
1616 if (BitWidth == 64)
1617 Reqs.addCapability(ToAdd: SPIRV::Capability::Float64);
1618 else if (BitWidth == 16) {
1619 if (isBFloat16Type(TypeDef: &MI)) {
1620 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_bfloat16))
1621 report_fatal_error(reason: "OpTypeFloat type with bfloat requires the "
1622 "following SPIR-V extension: SPV_KHR_bfloat16",
1623 gen_crash_diag: false);
1624 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_bfloat16);
1625 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16TypeKHR);
1626 } else {
1627 Reqs.addCapability(ToAdd: SPIRV::Capability::Float16);
1628 }
1629 }
1630 break;
1631 }
1632 case SPIRV::OpTypeVector: {
1633 unsigned NumComponents = MI.getOperand(i: 2).getImm();
1634 if (NumComponents == 8 || NumComponents == 16)
1635 Reqs.addCapability(ToAdd: SPIRV::Capability::Vector16);
1636
1637 maybeAddScatterGatherReq(MI, Reqs, ST);
1638 break;
1639 }
1640 case SPIRV::OpTypeVectorIdEXT: {
1641 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector))
1642 reportFatalUsageError(reason: "OpTypeVectorIdEXT requires the following SPIR-V "
1643 "extension: SPV_EXT_long_vector extension");
1644 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_long_vector);
1645 Reqs.addCapability(ToAdd: SPIRV::Capability::LongVectorEXT);
1646 maybeAddScatterGatherReq(MI, Reqs, ST);
1647 break;
1648 }
1649 case SPIRV::OpTypePointer: {
1650 auto SC = MI.getOperand(i: 1).getImm();
1651 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::StorageClassOperand, i: SC,
1652 ST);
1653 // If it's a type of pointer to float16 targeting OpenCL, add Float16Buffer
1654 // capability.
1655 if (ST.isShader())
1656 break;
1657 assert(MI.getOperand(2).isReg());
1658 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1659 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
1660 if ((TypeDef->getNumOperands() == 2) &&
1661 (TypeDef->getOpcode() == SPIRV::OpTypeFloat) &&
1662 (TypeDef->getOperand(i: 1).getImm() == 16))
1663 Reqs.addCapability(ToAdd: SPIRV::Capability::Float16Buffer);
1664 break;
1665 }
1666 case SPIRV::OpExtInst: {
1667 if (MI.getOperand(i: 2).getImm() ==
1668 static_cast<int64_t>(
1669 SPIRV::InstructionSet::NonSemantic_Shader_DebugInfo_100)) {
1670 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_non_semantic_info);
1671 break;
1672 }
1673 if (MI.getOperand(i: 3).getImm() ==
1674 static_cast<int64_t>(SPIRV::OpenCLExtInst::printf)) {
1675 addPrintfRequirements(MI, Reqs, ST);
1676 break;
1677 }
1678 if (MI.getOperand(i: 2).getImm() ==
1679 static_cast<int64_t>(SPIRV::InstructionSet::OpenCL_std)) {
1680 const MachineFunction *MF = MI.getMF();
1681 const MachineRegisterInfo &MRI = MF->getRegInfo();
1682 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1683
1684 auto IsBFloat16 = [&](SPIRVTypeInst TypeDef) {
1685 if (TypeDef && TypeDef->getOpcode() == SPIRV::OpTypeVector)
1686 TypeDef = MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
1687 return isBFloat16Type(TypeDef);
1688 };
1689
1690 // Result type is operand 1; arguments start at operand 4.
1691 bool UsesBFloat16 = IsBFloat16(MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg()));
1692 for (unsigned I = 4, E = MI.getNumOperands(); I < E && !UsesBFloat16;
1693 ++I) {
1694 const MachineOperand &MO = MI.getOperand(i: I);
1695 if (MO.isReg())
1696 UsesBFloat16 = IsBFloat16(GR->getResultType(
1697 VReg: MO.getReg(), MF: const_cast<MachineFunction *>(MF)));
1698 }
1699
1700 if (UsesBFloat16) {
1701 if (!ST.canUseExtension(
1702 E: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic)) {
1703 reportUnsupported(
1704 MI, Msg: "OpenCL Extended instructions with bfloat16 require the "
1705 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic");
1706 break;
1707 }
1708 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);
1709 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ArithmeticINTEL);
1710 }
1711 }
1712 break;
1713 }
1714 case SPIRV::OpAliasDomainDeclINTEL:
1715 case SPIRV::OpAliasScopeDeclINTEL:
1716 case SPIRV::OpAliasScopeListDeclINTEL: {
1717 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_memory_access_aliasing);
1718 Reqs.addCapability(ToAdd: SPIRV::Capability::MemoryAccessAliasingINTEL);
1719 break;
1720 }
1721 case SPIRV::OpBitReverse:
1722 case SPIRV::OpBitFieldInsert:
1723 case SPIRV::OpBitFieldSExtract:
1724 case SPIRV::OpBitFieldUExtract:
1725 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions)) {
1726 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
1727 break;
1728 }
1729 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_bit_instructions);
1730 Reqs.addCapability(ToAdd: SPIRV::Capability::BitInstructions);
1731 break;
1732 case SPIRV::OpTypeRuntimeArray:
1733 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
1734 break;
1735 case SPIRV::OpTypeOpaque:
1736 case SPIRV::OpTypeEvent:
1737 Reqs.addCapability(ToAdd: SPIRV::Capability::Kernel);
1738 break;
1739 case SPIRV::OpTypePipe:
1740 case SPIRV::OpTypeReserveId:
1741 Reqs.addCapability(ToAdd: SPIRV::Capability::Pipes);
1742 break;
1743 case SPIRV::OpTypeDeviceEvent:
1744 case SPIRV::OpTypeQueue:
1745 case SPIRV::OpBuildNDRange:
1746 case SPIRV::OpEnqueueKernel:
1747 Reqs.addCapability(ToAdd: SPIRV::Capability::DeviceEnqueue);
1748 break;
1749 case SPIRV::OpDecorate:
1750 case SPIRV::OpDecorateId:
1751 case SPIRV::OpDecorateString:
1752 addOpDecorateReqs(MI, DecIndex: 1, Reqs, ST);
1753 break;
1754 case SPIRV::OpMemberDecorate:
1755 case SPIRV::OpMemberDecorateString:
1756 addOpDecorateReqs(MI, DecIndex: 2, Reqs, ST);
1757 break;
1758 case SPIRV::OpInBoundsPtrAccessChain:
1759 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
1760 break;
1761 case SPIRV::OpConstantSampler:
1762 Reqs.addCapability(ToAdd: SPIRV::Capability::LiteralSampler);
1763 break;
1764 case SPIRV::OpInBoundsAccessChain:
1765 case SPIRV::OpAccessChain:
1766 addOpAccessChainReqs(Instr: MI, Handler&: Reqs, Subtarget: ST);
1767 break;
1768 case SPIRV::OpTypeImage:
1769 addOpTypeImageReqs(MI, Reqs, ST);
1770 break;
1771 case SPIRV::OpTypeSampler:
1772 if (!ST.isShader()) {
1773 Reqs.addCapability(ToAdd: SPIRV::Capability::ImageBasic);
1774 }
1775 break;
1776 case SPIRV::OpTypeForwardPointer:
1777 // TODO: check if it's OpenCL's kernel.
1778 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
1779 break;
1780 case SPIRV::OpAtomicFlagTestAndSet:
1781 case SPIRV::OpAtomicLoad:
1782 case SPIRV::OpAtomicStore:
1783 case SPIRV::OpAtomicExchange:
1784 case SPIRV::OpAtomicCompareExchange:
1785 case SPIRV::OpAtomicCompareExchangeWeak:
1786 case SPIRV::OpAtomicIIncrement:
1787 case SPIRV::OpAtomicIDecrement:
1788 case SPIRV::OpAtomicIAdd:
1789 case SPIRV::OpAtomicISub:
1790 case SPIRV::OpAtomicUMin:
1791 case SPIRV::OpAtomicUMax:
1792 case SPIRV::OpAtomicSMin:
1793 case SPIRV::OpAtomicSMax:
1794 case SPIRV::OpAtomicAnd:
1795 case SPIRV::OpAtomicOr:
1796 case SPIRV::OpAtomicXor: {
1797 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1798 const MachineInstr *InstrPtr = &MI;
1799 if (Op == SPIRV::OpAtomicStore) {
1800 assert(MI.getOperand(3).isReg());
1801 InstrPtr = MRI.getVRegDef(Reg: MI.getOperand(i: 3).getReg());
1802 assert(InstrPtr && "Unexpected type instruction for OpAtomicStore");
1803 }
1804 assert(InstrPtr->getOperand(1).isReg() && "Unexpected operand in atomic");
1805 Register TypeReg = InstrPtr->getOperand(i: 1).getReg();
1806 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: TypeReg);
1807
1808 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {
1809 unsigned BitWidth = TypeDef->getOperand(i: 1).getImm();
1810 if (BitWidth == 64)
1811 Reqs.addCapability(ToAdd: SPIRV::Capability::Int64Atomics);
1812 else if (BitWidth == 16) {
1813 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1814 report_fatal_error(
1815 reason: "16-bit integer atomic operations require the following SPIR-V "
1816 "extension: SPV_INTEL_16bit_atomics",
1817 gen_crash_diag: false);
1818 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1819 switch (Op) {
1820 case SPIRV::OpAtomicLoad:
1821 case SPIRV::OpAtomicStore:
1822 case SPIRV::OpAtomicExchange:
1823 case SPIRV::OpAtomicCompareExchange:
1824 case SPIRV::OpAtomicCompareExchangeWeak:
1825 Reqs.addCapability(
1826 ToAdd: SPIRV::Capability::AtomicInt16CompareExchangeINTEL);
1827 break;
1828 default:
1829 Reqs.addCapability(ToAdd: SPIRV::Capability::Int16AtomicsINTEL);
1830 break;
1831 }
1832 }
1833 } else if (isBFloat16Type(TypeDef)) {
1834 if (is_contained(Set: {SPIRV::OpAtomicLoad, SPIRV::OpAtomicStore,
1835 SPIRV::OpAtomicExchange},
1836 Element: Op)) {
1837 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1838 report_fatal_error(
1839 reason: "The atomic bfloat16 instruction requires the following SPIR-V "
1840 "extension: SPV_INTEL_16bit_atomics",
1841 gen_crash_diag: false);
1842 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1843 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicBFloat16LoadStoreINTEL);
1844 }
1845 }
1846 break;
1847 }
1848 case SPIRV::OpGroupNonUniformIAdd:
1849 case SPIRV::OpGroupNonUniformFAdd:
1850 case SPIRV::OpGroupNonUniformIMul:
1851 case SPIRV::OpGroupNonUniformFMul:
1852 case SPIRV::OpGroupNonUniformSMin:
1853 case SPIRV::OpGroupNonUniformUMin:
1854 case SPIRV::OpGroupNonUniformFMin:
1855 case SPIRV::OpGroupNonUniformSMax:
1856 case SPIRV::OpGroupNonUniformUMax:
1857 case SPIRV::OpGroupNonUniformFMax:
1858 case SPIRV::OpGroupNonUniformBitwiseAnd:
1859 case SPIRV::OpGroupNonUniformBitwiseOr:
1860 case SPIRV::OpGroupNonUniformBitwiseXor:
1861 case SPIRV::OpGroupNonUniformLogicalAnd:
1862 case SPIRV::OpGroupNonUniformLogicalOr:
1863 case SPIRV::OpGroupNonUniformLogicalXor: {
1864 assert(MI.getOperand(3).isImm());
1865 int64_t GroupOp = MI.getOperand(i: 3).getImm();
1866 switch (GroupOp) {
1867 case SPIRV::GroupOperation::Reduce:
1868 case SPIRV::GroupOperation::InclusiveScan:
1869 case SPIRV::GroupOperation::ExclusiveScan:
1870 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformArithmetic);
1871 break;
1872 case SPIRV::GroupOperation::ClusteredReduce:
1873 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformClustered);
1874 break;
1875 case SPIRV::GroupOperation::PartitionedReduceNV:
1876 case SPIRV::GroupOperation::PartitionedInclusiveScanNV:
1877 case SPIRV::GroupOperation::PartitionedExclusiveScanNV:
1878 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformPartitionedNV);
1879 break;
1880 }
1881 break;
1882 }
1883 case SPIRV::OpGroupNonUniformQuadSwap:
1884 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformQuad);
1885 break;
1886 case SPIRV::OpImageQueryLod:
1887 Reqs.addCapability(ToAdd: SPIRV::Capability::ImageQuery);
1888 break;
1889 case SPIRV::OpImageQuerySize:
1890 case SPIRV::OpImageQuerySizeLod:
1891 case SPIRV::OpImageQueryLevels:
1892 case SPIRV::OpImageQuerySamples:
1893 if (ST.isShader())
1894 Reqs.addCapability(ToAdd: SPIRV::Capability::ImageQuery);
1895 break;
1896 case SPIRV::OpImageQueryFormat: {
1897 Register ResultReg = MI.getOperand(i: 0).getReg();
1898 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1899 static const unsigned CompareOps[] = {
1900 SPIRV::OpIEqual, SPIRV::OpINotEqual,
1901 SPIRV::OpUGreaterThan, SPIRV::OpUGreaterThanEqual,
1902 SPIRV::OpULessThan, SPIRV::OpULessThanEqual,
1903 SPIRV::OpSGreaterThan, SPIRV::OpSGreaterThanEqual,
1904 SPIRV::OpSLessThan, SPIRV::OpSLessThanEqual};
1905
1906 auto CheckAndAddExtension = [&](int64_t ImmVal) {
1907 if (ImmVal == 4323 || ImmVal == 4324) {
1908 if (ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_image_raw10_raw12))
1909 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_image_raw10_raw12);
1910 else
1911 report_fatal_error(reason: "This requires the "
1912 "SPV_EXT_image_raw10_raw12 extension");
1913 }
1914 };
1915
1916 for (MachineInstr &UseInst : MRI.use_instructions(Reg: ResultReg)) {
1917 unsigned Opc = UseInst.getOpcode();
1918
1919 if (Opc == SPIRV::OpSwitch) {
1920 for (const MachineOperand &Op : UseInst.operands())
1921 if (Op.isImm())
1922 CheckAndAddExtension(Op.getImm());
1923 } else if (llvm::is_contained(Range: CompareOps, Element: Opc)) {
1924 for (unsigned i = 1; i < UseInst.getNumOperands(); ++i) {
1925 Register UseReg = UseInst.getOperand(i).getReg();
1926 MachineInstr *ConstInst = MRI.getVRegDef(Reg: UseReg);
1927 if (ConstInst && ConstInst->getOpcode() == SPIRV::OpConstantI) {
1928 int64_t ImmVal = ConstInst->getOperand(i: 2).getImm();
1929 if (ImmVal)
1930 CheckAndAddExtension(ImmVal);
1931 }
1932 }
1933 }
1934 }
1935 break;
1936 }
1937
1938 case SPIRV::OpGroupNonUniformShuffle:
1939 case SPIRV::OpGroupNonUniformShuffleXor:
1940 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformShuffle);
1941 break;
1942 case SPIRV::OpGroupNonUniformShuffleUp:
1943 case SPIRV::OpGroupNonUniformShuffleDown:
1944 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformShuffleRelative);
1945 break;
1946 case SPIRV::OpGroupAll:
1947 case SPIRV::OpGroupAny:
1948 case SPIRV::OpGroupBroadcast:
1949 case SPIRV::OpGroupIAdd:
1950 case SPIRV::OpGroupFAdd:
1951 case SPIRV::OpGroupFMin:
1952 case SPIRV::OpGroupUMin:
1953 case SPIRV::OpGroupSMin:
1954 case SPIRV::OpGroupFMax:
1955 case SPIRV::OpGroupUMax:
1956 case SPIRV::OpGroupSMax:
1957 Reqs.addCapability(ToAdd: SPIRV::Capability::Groups);
1958 break;
1959 case SPIRV::OpGroupNonUniformElect:
1960 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniform);
1961 break;
1962 case SPIRV::OpGroupNonUniformAll:
1963 case SPIRV::OpGroupNonUniformAny:
1964 case SPIRV::OpGroupNonUniformAllEqual:
1965 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformVote);
1966 break;
1967 case SPIRV::OpGroupNonUniformBroadcast:
1968 case SPIRV::OpGroupNonUniformBroadcastFirst:
1969 case SPIRV::OpGroupNonUniformBallot:
1970 case SPIRV::OpGroupNonUniformInverseBallot:
1971 case SPIRV::OpGroupNonUniformBallotBitExtract:
1972 case SPIRV::OpGroupNonUniformBallotBitCount:
1973 case SPIRV::OpGroupNonUniformBallotFindLSB:
1974 case SPIRV::OpGroupNonUniformBallotFindMSB:
1975 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformBallot);
1976 break;
1977 case SPIRV::OpSubgroupShuffleINTEL:
1978 case SPIRV::OpSubgroupShuffleDownINTEL:
1979 case SPIRV::OpSubgroupShuffleUpINTEL:
1980 case SPIRV::OpSubgroupShuffleXorINTEL:
1981 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
1982 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
1983 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupShuffleINTEL);
1984 }
1985 break;
1986 case SPIRV::OpSubgroupBlockReadINTEL:
1987 case SPIRV::OpSubgroupBlockWriteINTEL:
1988 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
1989 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
1990 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupBufferBlockIOINTEL);
1991 }
1992 break;
1993 case SPIRV::OpSubgroupImageBlockReadINTEL:
1994 case SPIRV::OpSubgroupImageBlockWriteINTEL:
1995 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
1996 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
1997 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupImageBlockIOINTEL);
1998 }
1999 break;
2000 case SPIRV::OpSubgroupImageMediaBlockReadINTEL:
2001 case SPIRV::OpSubgroupImageMediaBlockWriteINTEL:
2002 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_media_block_io)) {
2003 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_media_block_io);
2004 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupImageMediaBlockIOINTEL);
2005 }
2006 break;
2007 case SPIRV::OpAssumeTrueKHR:
2008 case SPIRV::OpExpectKHR:
2009 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume)) {
2010 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_expect_assume);
2011 Reqs.addCapability(ToAdd: SPIRV::Capability::ExpectAssumeKHR);
2012 }
2013 break;
2014 case SPIRV::OpFmaKHR:
2015 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_fma)) {
2016 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_fma);
2017 Reqs.addCapability(ToAdd: SPIRV::Capability::FmaKHR);
2018 }
2019 break;
2020 case SPIRV::OpPtrCastToCrossWorkgroupINTEL:
2021 case SPIRV::OpCrossWorkgroupCastToPtrINTEL:
2022 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)) {
2023 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_usm_storage_classes);
2024 Reqs.addCapability(ToAdd: SPIRV::Capability::USMStorageClassesINTEL);
2025 }
2026 break;
2027 case SPIRV::OpConstantFunctionPointerINTEL:
2028 case SPIRV::OpFunctionPointerCallINTEL:
2029 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers)) {
2030 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_function_pointers);
2031 Reqs.addCapability(ToAdd: SPIRV::Capability::FunctionPointersINTEL);
2032 }
2033 break;
2034 case SPIRV::OpGroupNonUniformRotateKHR:
2035 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_subgroup_rotate))
2036 report_fatal_error(reason: "OpGroupNonUniformRotateKHR instruction requires the "
2037 "following SPIR-V extension: SPV_KHR_subgroup_rotate",
2038 gen_crash_diag: false);
2039 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_subgroup_rotate);
2040 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformRotateKHR);
2041 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniform);
2042 break;
2043 case SPIRV::OpFixedCosALTERA:
2044 case SPIRV::OpFixedSinALTERA:
2045 case SPIRV::OpFixedCosPiALTERA:
2046 case SPIRV::OpFixedSinPiALTERA:
2047 case SPIRV::OpFixedExpALTERA:
2048 case SPIRV::OpFixedLogALTERA:
2049 case SPIRV::OpFixedRecipALTERA:
2050 case SPIRV::OpFixedSqrtALTERA:
2051 case SPIRV::OpFixedSinCosALTERA:
2052 case SPIRV::OpFixedSinCosPiALTERA:
2053 case SPIRV::OpFixedRsqrtALTERA:
2054 if (!ST.canUseExtension(
2055 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point))
2056 report_fatal_error(reason: "This instruction requires the "
2057 "following SPIR-V extension: "
2058 "SPV_ALTERA_arbitrary_precision_fixed_point",
2059 gen_crash_diag: false);
2060 Reqs.addExtension(
2061 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point);
2062 Reqs.addCapability(ToAdd: SPIRV::Capability::ArbitraryPrecisionFixedPointALTERA);
2063 break;
2064 case SPIRV::OpGroupIMulKHR:
2065 case SPIRV::OpGroupFMulKHR:
2066 case SPIRV::OpGroupBitwiseAndKHR:
2067 case SPIRV::OpGroupBitwiseOrKHR:
2068 case SPIRV::OpGroupBitwiseXorKHR:
2069 case SPIRV::OpGroupLogicalAndKHR:
2070 case SPIRV::OpGroupLogicalOrKHR:
2071 case SPIRV::OpGroupLogicalXorKHR:
2072 if (ST.canUseExtension(
2073 E: SPIRV::Extension::SPV_KHR_uniform_group_instructions)) {
2074 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_uniform_group_instructions);
2075 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupUniformArithmeticKHR);
2076 }
2077 break;
2078 case SPIRV::OpReadClockKHR:
2079 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_shader_clock))
2080 report_fatal_error(reason: "OpReadClockKHR instruction requires the "
2081 "following SPIR-V extension: SPV_KHR_shader_clock",
2082 gen_crash_diag: false);
2083 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_shader_clock);
2084 Reqs.addCapability(ToAdd: SPIRV::Capability::ShaderClockKHR);
2085 break;
2086 case SPIRV::OpAbortKHR:
2087 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_abort))
2088 report_fatal_error(reason: "OpAbortKHR instruction requires the "
2089 "following SPIR-V extension: SPV_KHR_abort",
2090 gen_crash_diag: false);
2091 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_abort);
2092 Reqs.addCapability(ToAdd: SPIRV::Capability::AbortKHR);
2093 break;
2094 case SPIRV::OpPoisonKHR:
2095 case SPIRV::OpFreezeKHR:
2096 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze))
2097 report_fatal_error(reason: "OpPoisonKHR/OpFreezeKHR instruction requires the "
2098 "following SPIR-V extension: SPV_KHR_poison_freeze",
2099 gen_crash_diag: false);
2100 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_poison_freeze);
2101 Reqs.addCapability(ToAdd: SPIRV::Capability::PoisonFreezeKHR);
2102 break;
2103 case SPIRV::OpAtomicFAddEXT:
2104 case SPIRV::OpAtomicFMinEXT:
2105 case SPIRV::OpAtomicFMaxEXT:
2106 AddAtomicFloatRequirements(MI, Reqs, ST);
2107 break;
2108 case SPIRV::OpConvertBF16ToFINTEL:
2109 case SPIRV::OpConvertFToBF16INTEL:
2110 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bfloat16_conversion)) {
2111 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_conversion);
2112 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ConversionINTEL);
2113 }
2114 break;
2115 case SPIRV::OpRoundFToTF32INTEL:
2116 if (ST.canUseExtension(
2117 E: SPIRV::Extension::SPV_INTEL_tensor_float32_conversion)) {
2118 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_tensor_float32_conversion);
2119 Reqs.addCapability(ToAdd: SPIRV::Capability::TensorFloat32RoundingINTEL);
2120 }
2121 break;
2122 case SPIRV::OpVariableLengthArrayINTEL:
2123 case SPIRV::OpSaveMemoryINTEL:
2124 case SPIRV::OpRestoreMemoryINTEL:
2125 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array)) {
2126 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_variable_length_array);
2127 Reqs.addCapability(ToAdd: SPIRV::Capability::VariableLengthArrayINTEL);
2128 }
2129 break;
2130 case SPIRV::OpAsmTargetINTEL:
2131 case SPIRV::OpAsmINTEL:
2132 case SPIRV::OpAsmCallINTEL:
2133 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_inline_assembly)) {
2134 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_inline_assembly);
2135 Reqs.addCapability(ToAdd: SPIRV::Capability::AsmINTEL);
2136 }
2137 break;
2138 case SPIRV::OpTypeCooperativeMatrixKHR: {
2139 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
2140 report_fatal_error(
2141 reason: "OpTypeCooperativeMatrixKHR type requires the "
2142 "following SPIR-V extension: SPV_KHR_cooperative_matrix",
2143 gen_crash_diag: false);
2144 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
2145 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
2146 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2147 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
2148 if (isBFloat16Type(TypeDef))
2149 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16CooperativeMatrixKHR);
2150 break;
2151 }
2152 case SPIRV::OpArithmeticFenceEXT:
2153 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_arithmetic_fence))
2154 report_fatal_error(reason: "OpArithmeticFenceEXT requires the "
2155 "following SPIR-V extension: SPV_EXT_arithmetic_fence",
2156 gen_crash_diag: false);
2157 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_arithmetic_fence);
2158 Reqs.addCapability(ToAdd: SPIRV::Capability::ArithmeticFenceEXT);
2159 break;
2160 case SPIRV::OpControlBarrierArriveINTEL:
2161 case SPIRV::OpControlBarrierWaitINTEL:
2162 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_split_barrier)) {
2163 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_split_barrier);
2164 Reqs.addCapability(ToAdd: SPIRV::Capability::SplitBarrierINTEL);
2165 }
2166 break;
2167 case SPIRV::OpCooperativeMatrixMulAddKHR: {
2168 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
2169 report_fatal_error(reason: "Cooperative matrix instructions require the "
2170 "following SPIR-V extension: "
2171 "SPV_KHR_cooperative_matrix",
2172 gen_crash_diag: false);
2173 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
2174 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
2175 constexpr unsigned MulAddMaxSize = 6;
2176 if (MI.getNumOperands() != MulAddMaxSize)
2177 break;
2178 const int64_t CoopOperands = MI.getOperand(i: MulAddMaxSize - 1).getImm();
2179 if (CoopOperands &
2180 SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) {
2181 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2182 report_fatal_error(reason: "MatrixAAndBTF32ComponentsINTEL type interpretation "
2183 "require the following SPIR-V extension: "
2184 "SPV_INTEL_joint_matrix",
2185 gen_crash_diag: false);
2186 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2187 Reqs.addCapability(
2188 ToAdd: SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL);
2189 }
2190 if (CoopOperands & SPIRV::CooperativeMatrixOperands::
2191 MatrixAAndBBFloat16ComponentsINTEL ||
2192 CoopOperands &
2193 SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL ||
2194 CoopOperands & SPIRV::CooperativeMatrixOperands::
2195 MatrixResultBFloat16ComponentsINTEL) {
2196 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2197 report_fatal_error(reason: "***BF16ComponentsINTEL type interpretations "
2198 "require the following SPIR-V extension: "
2199 "SPV_INTEL_joint_matrix",
2200 gen_crash_diag: false);
2201 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2202 Reqs.addCapability(
2203 ToAdd: SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);
2204 }
2205 break;
2206 }
2207 case SPIRV::OpCooperativeMatrixLoadKHR:
2208 case SPIRV::OpCooperativeMatrixStoreKHR:
2209 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2210 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2211 case SPIRV::OpCooperativeMatrixPrefetchINTEL: {
2212 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
2213 report_fatal_error(reason: "Cooperative matrix instructions require the "
2214 "following SPIR-V extension: "
2215 "SPV_KHR_cooperative_matrix",
2216 gen_crash_diag: false);
2217 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
2218 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
2219
2220 // Check Layout operand in case if it's not a standard one and add the
2221 // appropriate capability.
2222 unsigned LayoutNum;
2223 switch (Op) {
2224 case SPIRV::OpCooperativeMatrixLoadKHR:
2225 LayoutNum = 3;
2226 break;
2227 case SPIRV::OpCooperativeMatrixStoreKHR:
2228 LayoutNum = 2;
2229 break;
2230 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2231 LayoutNum = 5;
2232 break;
2233 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2234 case SPIRV::OpCooperativeMatrixPrefetchINTEL:
2235 LayoutNum = 4;
2236 break;
2237 default:
2238 llvm_unreachable("unexpected cooperative matrix opcode");
2239 }
2240 Register RegLayout = MI.getOperand(i: LayoutNum).getReg();
2241 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2242 MachineInstr *MILayout = MRI.getUniqueVRegDef(Reg: RegLayout);
2243 if (MILayout->getOpcode() == SPIRV::OpConstantI) {
2244 const unsigned LayoutVal = MILayout->getOperand(i: 2).getImm();
2245 if (LayoutVal ==
2246 static_cast<unsigned>(SPIRV::CooperativeMatrixLayout::PackedINTEL)) {
2247 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2248 report_fatal_error(reason: "PackedINTEL layout require the following SPIR-V "
2249 "extension: SPV_INTEL_joint_matrix",
2250 gen_crash_diag: false);
2251 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2252 Reqs.addCapability(ToAdd: SPIRV::Capability::PackedCooperativeMatrixINTEL);
2253 }
2254 }
2255
2256 // Nothing to do.
2257 if (Op == SPIRV::OpCooperativeMatrixLoadKHR ||
2258 Op == SPIRV::OpCooperativeMatrixStoreKHR)
2259 break;
2260
2261 std::string InstName;
2262 switch (Op) {
2263 case SPIRV::OpCooperativeMatrixPrefetchINTEL:
2264 InstName = "OpCooperativeMatrixPrefetchINTEL";
2265 break;
2266 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2267 InstName = "OpCooperativeMatrixLoadCheckedINTEL";
2268 break;
2269 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2270 InstName = "OpCooperativeMatrixStoreCheckedINTEL";
2271 break;
2272 }
2273
2274 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix)) {
2275 const std::string ErrorMsg =
2276 InstName + " instruction requires the "
2277 "following SPIR-V extension: SPV_INTEL_joint_matrix";
2278 report_fatal_error(reason: ErrorMsg.c_str(), gen_crash_diag: false);
2279 }
2280 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2281 if (Op == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
2282 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixPrefetchINTEL);
2283 break;
2284 }
2285 Reqs.addCapability(
2286 ToAdd: SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
2287 break;
2288 }
2289 case SPIRV::OpCooperativeMatrixConstructCheckedINTEL:
2290 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2291 report_fatal_error(reason: "OpCooperativeMatrixConstructCheckedINTEL "
2292 "instructions require the following SPIR-V extension: "
2293 "SPV_INTEL_joint_matrix",
2294 gen_crash_diag: false);
2295 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2296 Reqs.addCapability(
2297 ToAdd: SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
2298 break;
2299 case SPIRV::OpReadPipeBlockingALTERA:
2300 case SPIRV::OpWritePipeBlockingALTERA:
2301 if (ST.canUseExtension(E: SPIRV::Extension::SPV_ALTERA_blocking_pipes)) {
2302 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_ALTERA_blocking_pipes);
2303 Reqs.addCapability(ToAdd: SPIRV::Capability::BlockingPipesALTERA);
2304 }
2305 break;
2306 case SPIRV::OpCooperativeMatrixGetElementCoordINTEL:
2307 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2308 report_fatal_error(reason: "OpCooperativeMatrixGetElementCoordINTEL requires the "
2309 "following SPIR-V extension: SPV_INTEL_joint_matrix",
2310 gen_crash_diag: false);
2311 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2312 Reqs.addCapability(
2313 ToAdd: SPIRV::Capability::CooperativeMatrixInvocationInstructionsINTEL);
2314 break;
2315 case SPIRV::OpConvertHandleToImageINTEL:
2316 case SPIRV::OpConvertHandleToSamplerINTEL:
2317 case SPIRV::OpConvertHandleToSampledImageINTEL: {
2318 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bindless_images))
2319 report_fatal_error(reason: "OpConvertHandleTo[Image/Sampler/SampledImage]INTEL "
2320 "instructions require the following SPIR-V extension: "
2321 "SPV_INTEL_bindless_images",
2322 gen_crash_diag: false);
2323 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
2324 SPIRV::AddressingModel::AddressingModel AddrModel = MAI.Addr;
2325 SPIRVTypeInst TyDef = GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg());
2326 if (Op == SPIRV::OpConvertHandleToImageINTEL &&
2327 TyDef->getOpcode() != SPIRV::OpTypeImage) {
2328 report_fatal_error(reason: "Incorrect return type for the instruction "
2329 "OpConvertHandleToImageINTEL",
2330 gen_crash_diag: false);
2331 } else if (Op == SPIRV::OpConvertHandleToSamplerINTEL &&
2332 TyDef->getOpcode() != SPIRV::OpTypeSampler) {
2333 report_fatal_error(reason: "Incorrect return type for the instruction "
2334 "OpConvertHandleToSamplerINTEL",
2335 gen_crash_diag: false);
2336 } else if (Op == SPIRV::OpConvertHandleToSampledImageINTEL &&
2337 TyDef->getOpcode() != SPIRV::OpTypeSampledImage) {
2338 report_fatal_error(reason: "Incorrect return type for the instruction "
2339 "OpConvertHandleToSampledImageINTEL",
2340 gen_crash_diag: false);
2341 }
2342 SPIRVTypeInst SpvTy = GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 2).getReg());
2343 unsigned Bitwidth = GR->getScalarOrVectorBitWidth(Type: SpvTy);
2344 if (!(Bitwidth == 32 && AddrModel == SPIRV::AddressingModel::Physical32) &&
2345 !(Bitwidth == 64 && AddrModel == SPIRV::AddressingModel::Physical64)) {
2346 report_fatal_error(
2347 reason: "Parameter value must be a 32-bit scalar in case of "
2348 "Physical32 addressing model or a 64-bit scalar in case of "
2349 "Physical64 addressing model",
2350 gen_crash_diag: false);
2351 }
2352 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bindless_images);
2353 Reqs.addCapability(ToAdd: SPIRV::Capability::BindlessImagesINTEL);
2354 break;
2355 }
2356 case SPIRV::OpSubgroup2DBlockLoadINTEL:
2357 case SPIRV::OpSubgroup2DBlockLoadTransposeINTEL:
2358 case SPIRV::OpSubgroup2DBlockLoadTransformINTEL:
2359 case SPIRV::OpSubgroup2DBlockPrefetchINTEL:
2360 case SPIRV::OpSubgroup2DBlockStoreINTEL: {
2361 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_2d_block_io))
2362 report_fatal_error(reason: "OpSubgroup2DBlock[Load/LoadTranspose/LoadTransform/"
2363 "Prefetch/Store]INTEL instructions require the "
2364 "following SPIR-V extension: SPV_INTEL_2d_block_io",
2365 gen_crash_diag: false);
2366 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_2d_block_io);
2367 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockIOINTEL);
2368
2369 if (Op == SPIRV::OpSubgroup2DBlockLoadTransposeINTEL) {
2370 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockTransposeINTEL);
2371 break;
2372 }
2373 if (Op == SPIRV::OpSubgroup2DBlockLoadTransformINTEL) {
2374 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockTransformINTEL);
2375 break;
2376 }
2377 break;
2378 }
2379 case SPIRV::OpKill: {
2380 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2381 } break;
2382 case SPIRV::OpDemoteToHelperInvocation:
2383 Reqs.addCapability(ToAdd: SPIRV::Capability::DemoteToHelperInvocation);
2384
2385 if (ST.canUseExtension(
2386 E: SPIRV::Extension::SPV_EXT_demote_to_helper_invocation)) {
2387 if (!ST.isAtLeastSPIRVVer(VerToCompareTo: llvm::VersionTuple(1, 6)))
2388 Reqs.addExtension(
2389 ToAdd: SPIRV::Extension::SPV_EXT_demote_to_helper_invocation);
2390 }
2391 break;
2392 case SPIRV::OpSDot:
2393 case SPIRV::OpUDot:
2394 case SPIRV::OpSUDot:
2395 case SPIRV::OpSDotAccSat:
2396 case SPIRV::OpUDotAccSat:
2397 case SPIRV::OpSUDotAccSat:
2398 AddDotProductRequirements(MI, Reqs, ST);
2399 break;
2400 case SPIRV::OpImageSampleImplicitLod:
2401 case SPIRV::OpImageFetch:
2402 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2403 addImageOperandReqs(MI, Reqs, ST, OpIdx: 4);
2404 break;
2405 case SPIRV::OpImageSampleExplicitLod:
2406 addImageOperandReqs(MI, Reqs, ST, OpIdx: 4);
2407 break;
2408 case SPIRV::OpImageSampleDrefImplicitLod:
2409 case SPIRV::OpImageSampleDrefExplicitLod:
2410 case SPIRV::OpImageDrefGather:
2411 case SPIRV::OpImageGather:
2412 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2413 addImageOperandReqs(MI, Reqs, ST, OpIdx: 5);
2414 break;
2415 case SPIRV::OpImageRead: {
2416 Register ImageReg = MI.getOperand(i: 2).getReg();
2417 SPIRVTypeInst TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(
2418 VReg: ImageReg, MF: const_cast<MachineFunction *>(MI.getMF()));
2419 // OpImageRead and OpImageWrite can use Unknown Image Formats
2420 // when the Kernel capability is declared. In the OpenCL environment we are
2421 // not allowed to produce
2422 // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see
2423 // https://github.com/KhronosGroup/SPIRV-Headers/issues/487
2424
2425 if (isImageTypeWithUnknownFormat(TypeInst: TypeDef) && ST.isShader())
2426 Reqs.addCapability(ToAdd: SPIRV::Capability::StorageImageReadWithoutFormat);
2427 break;
2428 }
2429 case SPIRV::OpImageWrite: {
2430 Register ImageReg = MI.getOperand(i: 0).getReg();
2431 SPIRVTypeInst TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(
2432 VReg: ImageReg, MF: const_cast<MachineFunction *>(MI.getMF()));
2433 // OpImageRead and OpImageWrite can use Unknown Image Formats
2434 // when the Kernel capability is declared. In the OpenCL environment we are
2435 // not allowed to produce
2436 // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see
2437 // https://github.com/KhronosGroup/SPIRV-Headers/issues/487
2438
2439 if (isImageTypeWithUnknownFormat(TypeInst: TypeDef) && ST.isShader())
2440 Reqs.addCapability(ToAdd: SPIRV::Capability::StorageImageWriteWithoutFormat);
2441 break;
2442 }
2443 case SPIRV::OpTypeStructContinuedINTEL:
2444 case SPIRV::OpConstantCompositeContinuedINTEL:
2445 case SPIRV::OpSpecConstantCompositeContinuedINTEL:
2446 case SPIRV::OpCompositeConstructContinuedINTEL: {
2447 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_long_composites))
2448 report_fatal_error(
2449 reason: "Continued instructions require the "
2450 "following SPIR-V extension: SPV_INTEL_long_composites",
2451 gen_crash_diag: false);
2452 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_long_composites);
2453 Reqs.addCapability(ToAdd: SPIRV::Capability::LongCompositesINTEL);
2454 break;
2455 }
2456 case SPIRV::OpArbitraryFloatEQALTERA:
2457 case SPIRV::OpArbitraryFloatGEALTERA:
2458 case SPIRV::OpArbitraryFloatGTALTERA:
2459 case SPIRV::OpArbitraryFloatLEALTERA:
2460 case SPIRV::OpArbitraryFloatLTALTERA:
2461 case SPIRV::OpArbitraryFloatCbrtALTERA:
2462 case SPIRV::OpArbitraryFloatCosALTERA:
2463 case SPIRV::OpArbitraryFloatCosPiALTERA:
2464 case SPIRV::OpArbitraryFloatExp10ALTERA:
2465 case SPIRV::OpArbitraryFloatExp2ALTERA:
2466 case SPIRV::OpArbitraryFloatExpALTERA:
2467 case SPIRV::OpArbitraryFloatExpm1ALTERA:
2468 case SPIRV::OpArbitraryFloatHypotALTERA:
2469 case SPIRV::OpArbitraryFloatLog10ALTERA:
2470 case SPIRV::OpArbitraryFloatLog1pALTERA:
2471 case SPIRV::OpArbitraryFloatLog2ALTERA:
2472 case SPIRV::OpArbitraryFloatLogALTERA:
2473 case SPIRV::OpArbitraryFloatRecipALTERA:
2474 case SPIRV::OpArbitraryFloatSinCosALTERA:
2475 case SPIRV::OpArbitraryFloatSinCosPiALTERA:
2476 case SPIRV::OpArbitraryFloatSinALTERA:
2477 case SPIRV::OpArbitraryFloatSinPiALTERA:
2478 case SPIRV::OpArbitraryFloatSqrtALTERA:
2479 case SPIRV::OpArbitraryFloatACosALTERA:
2480 case SPIRV::OpArbitraryFloatACosPiALTERA:
2481 case SPIRV::OpArbitraryFloatAddALTERA:
2482 case SPIRV::OpArbitraryFloatASinALTERA:
2483 case SPIRV::OpArbitraryFloatASinPiALTERA:
2484 case SPIRV::OpArbitraryFloatATan2ALTERA:
2485 case SPIRV::OpArbitraryFloatATanALTERA:
2486 case SPIRV::OpArbitraryFloatATanPiALTERA:
2487 case SPIRV::OpArbitraryFloatCastFromIntALTERA:
2488 case SPIRV::OpArbitraryFloatCastALTERA:
2489 case SPIRV::OpArbitraryFloatCastToIntALTERA:
2490 case SPIRV::OpArbitraryFloatDivALTERA:
2491 case SPIRV::OpArbitraryFloatMulALTERA:
2492 case SPIRV::OpArbitraryFloatPowALTERA:
2493 case SPIRV::OpArbitraryFloatPowNALTERA:
2494 case SPIRV::OpArbitraryFloatPowRALTERA:
2495 case SPIRV::OpArbitraryFloatRSqrtALTERA:
2496 case SPIRV::OpArbitraryFloatSubALTERA: {
2497 if (!ST.canUseExtension(
2498 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_floating_point))
2499 report_fatal_error(
2500 reason: "Floating point instructions can't be translated correctly without "
2501 "enabled SPV_ALTERA_arbitrary_precision_floating_point extension!",
2502 gen_crash_diag: false);
2503 Reqs.addExtension(
2504 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_floating_point);
2505 Reqs.addCapability(
2506 ToAdd: SPIRV::Capability::ArbitraryPrecisionFloatingPointALTERA);
2507 break;
2508 }
2509 case SPIRV::OpSubgroupMatrixMultiplyAccumulateINTEL: {
2510 if (!ST.canUseExtension(
2511 E: SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate))
2512 report_fatal_error(
2513 reason: "OpSubgroupMatrixMultiplyAccumulateINTEL instruction requires the "
2514 "following SPIR-V "
2515 "extension: SPV_INTEL_subgroup_matrix_multiply_accumulate",
2516 gen_crash_diag: false);
2517 Reqs.addExtension(
2518 ToAdd: SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate);
2519 Reqs.addCapability(
2520 ToAdd: SPIRV::Capability::SubgroupMatrixMultiplyAccumulateINTEL);
2521 break;
2522 }
2523 case SPIRV::OpBitwiseFunctionINTEL: {
2524 if (!ST.canUseExtension(
2525 E: SPIRV::Extension::SPV_INTEL_ternary_bitwise_function))
2526 report_fatal_error(
2527 reason: "OpBitwiseFunctionINTEL instruction requires the following SPIR-V "
2528 "extension: SPV_INTEL_ternary_bitwise_function",
2529 gen_crash_diag: false);
2530 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_ternary_bitwise_function);
2531 Reqs.addCapability(ToAdd: SPIRV::Capability::TernaryBitwiseFunctionINTEL);
2532 break;
2533 }
2534 case SPIRV::OpCopyMemorySized: {
2535 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
2536 // TODO: Add UntypedPointersKHR when implemented.
2537 break;
2538 }
2539 case SPIRV::OpTypeUntypedPointerKHR:
2540 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::StorageClassOperand,
2541 i: MI.getOperand(i: 1).getImm(), ST);
2542 [[fallthrough]];
2543 case SPIRV::OpUntypedVariableKHR:
2544 case SPIRV::OpUntypedAccessChainKHR:
2545 case SPIRV::OpUntypedInBoundsAccessChainKHR:
2546 case SPIRV::OpUntypedPtrAccessChainKHR:
2547 case SPIRV::OpUntypedInBoundsPtrAccessChainKHR:
2548 case SPIRV::OpUntypedPrefetchKHR:
2549 case SPIRV::OpUntypedGroupAsyncCopyKHR: {
2550 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_untyped_pointers))
2551 report_fatal_error(reason: "Untyped pointer instructions require the following "
2552 "SPIR-V extension: SPV_KHR_untyped_pointers",
2553 gen_crash_diag: false);
2554 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_untyped_pointers);
2555 Reqs.addCapability(ToAdd: SPIRV::Capability::UntypedPointersKHR);
2556 break;
2557 }
2558 case SPIRV::OpPredicatedLoadINTEL:
2559 case SPIRV::OpPredicatedStoreINTEL: {
2560 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_predicated_io))
2561 report_fatal_error(
2562 reason: "OpPredicated[Load/Store]INTEL instructions require "
2563 "the following SPIR-V extension: SPV_INTEL_predicated_io",
2564 gen_crash_diag: false);
2565 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_predicated_io);
2566 Reqs.addCapability(ToAdd: SPIRV::Capability::PredicatedIOINTEL);
2567 break;
2568 }
2569 case SPIRV::OpFAddS:
2570 case SPIRV::OpFSubS:
2571 case SPIRV::OpFMulS:
2572 case SPIRV::OpFDivS:
2573 case SPIRV::OpFRemS:
2574 case SPIRV::OpFMod:
2575 case SPIRV::OpFNegate:
2576 case SPIRV::OpFAddV:
2577 case SPIRV::OpFSubV:
2578 case SPIRV::OpFMulV:
2579 case SPIRV::OpFDivV:
2580 case SPIRV::OpFRemV:
2581 case SPIRV::OpFNegateV: {
2582 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2583 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
2584 if (isVectorType(SPVTy: TypeDef))
2585 TypeDef = MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
2586 if (isBFloat16Type(TypeDef)) {
2587 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic))
2588 report_fatal_error(
2589 reason: "Arithmetic instructions with bfloat16 arguments require the "
2590 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic",
2591 gen_crash_diag: false);
2592 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);
2593 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ArithmeticINTEL);
2594 }
2595 break;
2596 }
2597 case SPIRV::OpOrdered:
2598 case SPIRV::OpUnordered:
2599 case SPIRV::OpFOrdEqual:
2600 case SPIRV::OpFOrdNotEqual:
2601 case SPIRV::OpFOrdLessThan:
2602 case SPIRV::OpFOrdLessThanEqual:
2603 case SPIRV::OpFOrdGreaterThan:
2604 case SPIRV::OpFOrdGreaterThanEqual:
2605 case SPIRV::OpFUnordEqual:
2606 case SPIRV::OpFUnordNotEqual:
2607 case SPIRV::OpFUnordLessThan:
2608 case SPIRV::OpFUnordLessThanEqual:
2609 case SPIRV::OpFUnordGreaterThan:
2610 case SPIRV::OpFUnordGreaterThanEqual: {
2611 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2612 MachineInstr *OperandDef = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
2613 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: OperandDef->getOperand(i: 1).getReg());
2614 if (isVectorType(SPVTy: TypeDef))
2615 TypeDef = MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
2616 if (isBFloat16Type(TypeDef)) {
2617 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic))
2618 report_fatal_error(
2619 reason: "Relational instructions with bfloat16 arguments require the "
2620 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic",
2621 gen_crash_diag: false);
2622 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);
2623 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ArithmeticINTEL);
2624 }
2625 break;
2626 }
2627 case SPIRV::OpDPdxCoarse:
2628 case SPIRV::OpDPdyCoarse:
2629 case SPIRV::OpDPdxFine:
2630 case SPIRV::OpDPdyFine: {
2631 Reqs.addCapability(ToAdd: SPIRV::Capability::DerivativeControl);
2632 break;
2633 }
2634 case SPIRV::OpLoopControlINTEL: {
2635 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls);
2636 Reqs.addCapability(ToAdd: SPIRV::Capability::UnstructuredLoopControlsINTEL);
2637 break;
2638 }
2639
2640 default:
2641 break;
2642 }
2643
2644 // If we require capability Shader, then we can remove the requirement for
2645 // the BitInstructions capability, since Shader is a superset capability
2646 // of BitInstructions.
2647 Reqs.removeCapabilityIf(ToRemove: SPIRV::Capability::BitInstructions,
2648 IfPresent: SPIRV::Capability::Shader);
2649}
2650
2651static void collectReqs(const Module &M, SPIRV::ModuleAnalysisInfo &MAI,
2652 MachineModuleInfo *MMI, const SPIRVSubtarget &ST) {
2653 // Collect requirements for existing instructions.
2654 for (const Function &F : M) {
2655 MachineFunction *MF = MMI->getMachineFunction(F);
2656 if (!MF)
2657 continue;
2658 for (const MachineBasicBlock &MBB : *MF)
2659 for (const MachineInstr &MI : MBB)
2660 addInstrRequirements(MI, MAI, ST);
2661 }
2662 // Collect requirements for OpExecutionMode instructions.
2663 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
2664 if (Node) {
2665 bool RequireFloatControls = false, RequireIntelFloatControls2 = false,
2666 RequireKHRFloatControls2 = false,
2667 VerLower14 = !ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 4));
2668 bool HasIntelFloatControls2 =
2669 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_float_controls2);
2670 bool HasKHRFloatControls2 =
2671 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2672 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
2673 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
2674 const MDOperand &MDOp = MDN->getOperand(I: 1);
2675 if (auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: MDOp)) {
2676 Constant *C = CMeta->getValue();
2677 if (ConstantInt *Const = dyn_cast<ConstantInt>(Val: C)) {
2678 auto EM = Const->getZExtValue();
2679 // SPV_KHR_float_controls is not available until v1.4:
2680 // add SPV_KHR_float_controls if the version is too low
2681 switch (EM) {
2682 case SPIRV::ExecutionMode::DenormPreserve:
2683 case SPIRV::ExecutionMode::DenormFlushToZero:
2684 case SPIRV::ExecutionMode::RoundingModeRTE:
2685 case SPIRV::ExecutionMode::RoundingModeRTZ:
2686 RequireFloatControls = VerLower14;
2687 MAI.Reqs.getAndAddRequirements(
2688 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2689 break;
2690 case SPIRV::ExecutionMode::RoundingModeRTPINTEL:
2691 case SPIRV::ExecutionMode::RoundingModeRTNINTEL:
2692 case SPIRV::ExecutionMode::FloatingPointModeALTINTEL:
2693 case SPIRV::ExecutionMode::FloatingPointModeIEEEINTEL:
2694 if (HasIntelFloatControls2) {
2695 RequireIntelFloatControls2 = true;
2696 MAI.Reqs.getAndAddRequirements(
2697 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2698 }
2699 break;
2700 case SPIRV::ExecutionMode::FPFastMathDefault: {
2701 if (HasKHRFloatControls2) {
2702 RequireKHRFloatControls2 = true;
2703 MAI.Reqs.getAndAddRequirements(
2704 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2705 }
2706 break;
2707 }
2708 case SPIRV::ExecutionMode::ContractionOff:
2709 case SPIRV::ExecutionMode::SignedZeroInfNanPreserve:
2710 if (HasKHRFloatControls2) {
2711 RequireKHRFloatControls2 = true;
2712 MAI.Reqs.getAndAddRequirements(
2713 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2714 i: SPIRV::ExecutionMode::FPFastMathDefault, ST);
2715 } else {
2716 MAI.Reqs.getAndAddRequirements(
2717 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2718 }
2719 break;
2720 default:
2721 MAI.Reqs.getAndAddRequirements(
2722 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2723 }
2724 }
2725 }
2726 }
2727 if (RequireFloatControls &&
2728 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls))
2729 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls);
2730 if (RequireIntelFloatControls2)
2731 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_float_controls2);
2732 if (RequireKHRFloatControls2)
2733 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls2);
2734 }
2735 for (const Function &F : M) {
2736 if (F.isDeclaration())
2737 continue;
2738 if (F.getMetadata(Kind: "reqd_work_group_size"))
2739 MAI.Reqs.getAndAddRequirements(
2740 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2741 i: SPIRV::ExecutionMode::LocalSize, ST);
2742 if (F.getFnAttribute(Kind: "hlsl.numthreads").isValid()) {
2743 MAI.Reqs.getAndAddRequirements(
2744 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2745 i: SPIRV::ExecutionMode::LocalSize, ST);
2746 }
2747 if (F.getFnAttribute(Kind: "enable-maximal-reconvergence").getValueAsBool()) {
2748 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_maximal_reconvergence);
2749 }
2750 if (F.getMetadata(Kind: "work_group_size_hint"))
2751 MAI.Reqs.getAndAddRequirements(
2752 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2753 i: SPIRV::ExecutionMode::LocalSizeHint, ST);
2754 if (F.getMetadata(Kind: "intel_reqd_sub_group_size") ||
2755 F.getMetadata(Kind: "reqd_sub_group_size"))
2756 MAI.Reqs.getAndAddRequirements(
2757 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2758 i: SPIRV::ExecutionMode::SubgroupSize, ST);
2759 if (F.getMetadata(Kind: "max_work_group_size"))
2760 MAI.Reqs.getAndAddRequirements(
2761 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2762 i: SPIRV::ExecutionMode::MaxWorkgroupSizeINTEL, ST);
2763 if (F.getMetadata(Kind: "vec_type_hint"))
2764 MAI.Reqs.getAndAddRequirements(
2765 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2766 i: SPIRV::ExecutionMode::VecTypeHint, ST);
2767
2768 if (F.hasOptNone()) {
2769 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_optnone)) {
2770 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_optnone);
2771 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::OptNoneINTEL);
2772 } else if (ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_optnone)) {
2773 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_optnone);
2774 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::OptNoneEXT);
2775 }
2776 }
2777 }
2778}
2779
2780static unsigned getFastMathFlags(const MachineInstr &I,
2781 const SPIRVSubtarget &ST) {
2782 unsigned Flags = SPIRV::FPFastMathMode::None;
2783 bool CanUseKHRFloatControls2 =
2784 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2785 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNoNans))
2786 Flags |= SPIRV::FPFastMathMode::NotNaN;
2787 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNoInfs))
2788 Flags |= SPIRV::FPFastMathMode::NotInf;
2789 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNsz))
2790 Flags |= SPIRV::FPFastMathMode::NSZ;
2791 if (I.getFlag(Flag: MachineInstr::MIFlag::FmArcp))
2792 Flags |= SPIRV::FPFastMathMode::AllowRecip;
2793 if (I.getFlag(Flag: MachineInstr::MIFlag::FmContract) && CanUseKHRFloatControls2)
2794 Flags |= SPIRV::FPFastMathMode::AllowContract;
2795 if (I.getFlag(Flag: MachineInstr::MIFlag::FmReassoc)) {
2796 if (CanUseKHRFloatControls2)
2797 // LLVM reassoc maps to SPIRV transform, see
2798 // https://github.com/KhronosGroup/SPIRV-Registry/issues/326 for details.
2799 // Because we are enabling AllowTransform, we must enable AllowReassoc and
2800 // AllowContract too, as required by SPIRV spec. Also, we used to map
2801 // MIFlag::FmReassoc to FPFastMathMode::Fast, which now should instead by
2802 // replaced by turning all the other bits instead. Therefore, we're
2803 // enabling every bit here except None and Fast.
2804 Flags |= SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
2805 SPIRV::FPFastMathMode::NSZ | SPIRV::FPFastMathMode::AllowRecip |
2806 SPIRV::FPFastMathMode::AllowTransform |
2807 SPIRV::FPFastMathMode::AllowReassoc |
2808 SPIRV::FPFastMathMode::AllowContract;
2809 else
2810 Flags |= SPIRV::FPFastMathMode::Fast;
2811 }
2812
2813 if (CanUseKHRFloatControls2) {
2814 // Error out if SPIRV::FPFastMathMode::Fast is enabled.
2815 assert(!(Flags & SPIRV::FPFastMathMode::Fast) &&
2816 "SPIRV::FPFastMathMode::Fast is deprecated and should not be used "
2817 "anymore.");
2818
2819 // Error out if AllowTransform is enabled without AllowReassoc and
2820 // AllowContract.
2821 assert((!(Flags & SPIRV::FPFastMathMode::AllowTransform) ||
2822 ((Flags & SPIRV::FPFastMathMode::AllowReassoc &&
2823 Flags & SPIRV::FPFastMathMode::AllowContract))) &&
2824 "SPIRV::FPFastMathMode::AllowTransform requires AllowReassoc and "
2825 "AllowContract flags to be enabled as well.");
2826 }
2827
2828 return Flags;
2829}
2830
2831static bool isFastMathModeAvailable(const SPIRVSubtarget &ST) {
2832 if (ST.isKernel())
2833 return true;
2834 if (ST.getSPIRVVersion() < VersionTuple(1, 2))
2835 return false;
2836 return ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2837}
2838
2839static void handleMIFlagDecoration(
2840 MachineInstr &I, const SPIRVSubtarget &ST, const SPIRVInstrInfo &TII,
2841 SPIRV::RequirementHandler &Reqs, const SPIRVGlobalRegistry *GR,
2842 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec) {
2843 if (TII.canUseIntegerWrapDecoration(MI: I)) {
2844 if (I.getFlag(Flag: MachineInstr::MIFlag::NoSWrap) &&
2845 getSymbolicOperandRequirements(
2846 Category: SPIRV::OperandCategory::DecorationOperand,
2847 i: SPIRV::Decoration::NoSignedWrap, ST, Reqs)
2848 .IsSatisfiable)
2849 buildOpDecorate(Reg: I.getOperand(i: 0).getReg(), I, TII,
2850 Dec: SPIRV::Decoration::NoSignedWrap, DecArgs: {});
2851 if (I.getFlag(Flag: MachineInstr::MIFlag::NoUWrap) &&
2852 getSymbolicOperandRequirements(
2853 Category: SPIRV::OperandCategory::DecorationOperand,
2854 i: SPIRV::Decoration::NoUnsignedWrap, ST, Reqs)
2855 .IsSatisfiable)
2856 buildOpDecorate(Reg: I.getOperand(i: 0).getReg(), I, TII,
2857 Dec: SPIRV::Decoration::NoUnsignedWrap, DecArgs: {});
2858 }
2859 // In Kernel environments, FPFastMathMode on OpExtInst is valid per core
2860 // spec. For other instruction types, SPV_KHR_float_controls2 is required.
2861 bool CanUseFM =
2862 TII.canUseFastMathFlags(
2863 MI: I, KHRFloatControls2: ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2)) ||
2864 (ST.isKernel() && I.getOpcode() == SPIRV::OpExtInst);
2865 if (!CanUseFM)
2866 return;
2867
2868 unsigned FMFlags = getFastMathFlags(I, ST);
2869 if (FMFlags == SPIRV::FPFastMathMode::None) {
2870 // We also need to check if any FPFastMathDefault info was set for the
2871 // types used in this instruction.
2872 if (FPFastMathDefaultInfoVec.empty())
2873 return;
2874
2875 // There are three types of instructions that can use fast math flags:
2876 // 1. Arithmetic instructions (FAdd, FMul, FSub, FDiv, FRem, etc.)
2877 // 2. Relational instructions (FCmp, FOrd, FUnord, etc.)
2878 // 3. Extended instructions (ExtInst)
2879 // For arithmetic instructions, the floating point type can be in the
2880 // result type or in the operands, but they all must be the same.
2881 // For the relational and logical instructions, the floating point type
2882 // can only be in the operands 1 and 2, not the result type. Also, the
2883 // operands must have the same type. For the extended instructions, the
2884 // floating point type can be in the result type or in the operands. It's
2885 // unclear if the operands and the result type must be the same. Let's
2886 // assume they must be. Therefore, for 1. and 2., we can check the first
2887 // operand type, and for 3. we can check the result type.
2888 assert(I.getNumOperands() >= 3 && "Expected at least 3 operands");
2889 Register ResReg = I.getOpcode() == SPIRV::OpExtInst
2890 ? I.getOperand(i: 1).getReg()
2891 : I.getOperand(i: 2).getReg();
2892 SPIRVTypeInst ResType = GR->getSPIRVTypeForVReg(VReg: ResReg, MF: I.getMF());
2893 const Type *Ty = GR->getTypeForSPIRVType(Ty: ResType);
2894 Ty = Ty->isVectorTy() ? cast<VectorType>(Val: Ty)->getElementType() : Ty;
2895
2896 // Match instruction type with the FPFastMathDefaultInfoVec.
2897 bool Emit = false;
2898 for (SPIRV::FPFastMathDefaultInfo &Elem : FPFastMathDefaultInfoVec) {
2899 if (Ty == Elem.Ty) {
2900 FMFlags = Elem.FastMathFlags;
2901 Emit = Elem.ContractionOff || Elem.SignedZeroInfNanPreserve ||
2902 Elem.FPFastMathDefault;
2903 break;
2904 }
2905 }
2906
2907 if (FMFlags == SPIRV::FPFastMathMode::None && !Emit)
2908 return;
2909 }
2910 if (isFastMathModeAvailable(ST)) {
2911 Register DstReg = I.getOperand(i: 0).getReg();
2912 buildOpDecorate(Reg: DstReg, I, TII, Dec: SPIRV::Decoration::FPFastMathMode,
2913 DecArgs: {FMFlags});
2914 }
2915}
2916
2917// Walk all functions and add decorations related to MI flags.
2918static void addDecorations(const Module &M, const SPIRVInstrInfo &TII,
2919 MachineModuleInfo *MMI, const SPIRVSubtarget &ST,
2920 SPIRV::ModuleAnalysisInfo &MAI,
2921 const SPIRVGlobalRegistry *GR) {
2922 for (const Function &F : M) {
2923 MachineFunction *MF = MMI->getMachineFunction(F);
2924 if (!MF)
2925 continue;
2926
2927 for (auto &MBB : *MF)
2928 for (auto &MI : MBB)
2929 handleMIFlagDecoration(I&: MI, ST, TII, Reqs&: MAI.Reqs, GR,
2930 FPFastMathDefaultInfoVec&: MAI.FPFastMathDefaultInfoMap[&F]);
2931 }
2932}
2933
2934static void addMBBNames(const Module &M, const SPIRVInstrInfo &TII,
2935 MachineModuleInfo *MMI, const SPIRVSubtarget &ST,
2936 SPIRV::ModuleAnalysisInfo &MAI) {
2937 for (const Function &F : M) {
2938 MachineFunction *MF = MMI->getMachineFunction(F);
2939 if (!MF)
2940 continue;
2941 if (MF->getFunction()
2942 .getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME)
2943 .isValid())
2944 continue;
2945 MachineRegisterInfo &MRI = MF->getRegInfo();
2946 for (auto &MBB : *MF) {
2947 if (!MBB.hasName() || MBB.empty())
2948 continue;
2949 // Emit basic block names.
2950 Register Reg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2951 MRI.setRegClass(Reg, RC: &SPIRV::IDRegClass);
2952 buildOpName(Target: Reg, Name: MBB.getName(), I&: *std::prev(x: MBB.end()), TII);
2953 MCRegister GlobalReg = MAI.getOrCreateMBBRegister(MBB);
2954 MAI.setRegisterAlias(MF, Reg, AliasReg: GlobalReg);
2955 }
2956 }
2957}
2958
2959// patching Instruction::PHI to SPIRV::OpPhi
2960static void patchPhis(const Module &M, SPIRVGlobalRegistry *GR,
2961 const SPIRVInstrInfo &TII, MachineModuleInfo *MMI) {
2962 for (const Function &F : M) {
2963 MachineFunction *MF = MMI->getMachineFunction(F);
2964 if (!MF)
2965 continue;
2966 for (auto &MBB : *MF) {
2967 for (MachineInstr &MI : MBB.phis()) {
2968 MI.setDesc(TII.get(Opcode: SPIRV::OpPhi));
2969 Register ResTypeReg = GR->getSPIRVTypeID(
2970 SpirvType: GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 0).getReg(), MF));
2971 MI.insert(InsertBefore: MI.operands_begin() + 1,
2972 Ops: {MachineOperand::CreateReg(Reg: ResTypeReg, isDef: false)});
2973 }
2974 }
2975
2976 MF->getProperties().setNoPHIs();
2977 }
2978}
2979
2980static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
2981 const Module &M, SPIRV::ModuleAnalysisInfo &MAI, const Function *F) {
2982 auto it = MAI.FPFastMathDefaultInfoMap.find(Val: F);
2983 if (it != MAI.FPFastMathDefaultInfoMap.end())
2984 return it->second;
2985
2986 // If the map does not contain the entry, create a new one. Initialize it to
2987 // contain all 3 elements sorted by bit width of target type: {half, float,
2988 // double}.
2989 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
2990 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
2991 Args: SPIRV::FPFastMathMode::None);
2992 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
2993 Args: SPIRV::FPFastMathMode::None);
2994 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
2995 Args: SPIRV::FPFastMathMode::None);
2996 return MAI.FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
2997}
2998
2999static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
3000 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
3001 const Type *Ty) {
3002 size_t BitWidth = Ty->getScalarSizeInBits();
3003 int Index =
3004 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
3005 BitWidth);
3006 assert(Index >= 0 && Index < 3 &&
3007 "Expected FPFastMathDefaultInfo for half, float, or double");
3008 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3009 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3010 return FPFastMathDefaultInfoVec[Index];
3011}
3012
3013static void collectFPFastMathDefaults(const Module &M,
3014 SPIRV::ModuleAnalysisInfo &MAI,
3015 const SPIRVSubtarget &ST) {
3016 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
3017 return;
3018
3019 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
3020 // We need the entry point (function) as the key, and the target
3021 // type and flags as the value.
3022 // We also need to check ContractionOff and SignedZeroInfNanPreserve
3023 // execution modes, as they are now deprecated and must be replaced
3024 // with FPFastMathDefaultInfo.
3025 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
3026 if (!Node)
3027 return;
3028
3029 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
3030 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
3031 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
3032 const Function *F = cast<Function>(
3033 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
3034 const auto EM =
3035 cast<ConstantInt>(
3036 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
3037 ->getZExtValue();
3038 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
3039 assert(MDN->getNumOperands() == 4 &&
3040 "Expected 4 operands for FPFastMathDefault");
3041
3042 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
3043 unsigned Flags =
3044 cast<ConstantInt>(
3045 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
3046 ->getZExtValue();
3047 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3048 getOrCreateFPFastMathDefaultInfoVec(M, MAI, F);
3049 SPIRV::FPFastMathDefaultInfo &Info =
3050 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
3051 Info.FastMathFlags = Flags;
3052 Info.FPFastMathDefault = true;
3053 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
3054 assert(MDN->getNumOperands() == 2 &&
3055 "Expected no operands for ContractionOff");
3056
3057 // We need to save this info for every possible FP type, i.e. {half,
3058 // float, double, fp128}.
3059 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3060 getOrCreateFPFastMathDefaultInfoVec(M, MAI, F);
3061 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3062 Info.ContractionOff = true;
3063 }
3064 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
3065 assert(MDN->getNumOperands() == 3 &&
3066 "Expected 1 operand for SignedZeroInfNanPreserve");
3067 unsigned TargetWidth =
3068 cast<ConstantInt>(
3069 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
3070 ->getZExtValue();
3071 // We need to save this info only for the FP type with TargetWidth.
3072 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3073 getOrCreateFPFastMathDefaultInfoVec(M, MAI, F);
3074 int Index = SPIRV::FPFastMathDefaultInfoVector::
3075 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
3076 assert(Index >= 0 && Index < 3 &&
3077 "Expected FPFastMathDefaultInfo for half, float, or double");
3078 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3079 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3080 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
3081 }
3082 }
3083}
3084
3085void SPIRVModuleAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
3086 AU.addRequired<TargetPassConfig>();
3087 AU.addRequired<MachineModuleInfoWrapperPass>();
3088}
3089
3090bool SPIRVModuleAnalysis::runOnModule(Module &M) {
3091 SPIRVTargetMachine &TM =
3092 getAnalysis<TargetPassConfig>().getTM<SPIRVTargetMachine>();
3093 ST = TM.getSubtargetImpl();
3094 GR = ST->getSPIRVGlobalRegistry();
3095 TII = ST->getInstrInfo();
3096
3097 MMI = &getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
3098
3099 setBaseInfo(M);
3100
3101 patchPhis(M, GR, TII: *TII, MMI);
3102
3103 addMBBNames(M, TII: *TII, MMI, ST: *ST, MAI);
3104 collectFPFastMathDefaults(M, MAI, ST: *ST);
3105 addDecorations(M, TII: *TII, MMI, ST: *ST, MAI, GR);
3106
3107 collectReqs(M, MAI, MMI, ST: *ST);
3108
3109 // Process type/const/global var/func decl instructions, number their
3110 // destination registers from 0 to N, collect Extensions and Capabilities.
3111 collectDeclarations(M);
3112
3113 // Number rest of registers from N+1 onwards.
3114 numberRegistersGlobally(M);
3115
3116 // Collect OpName, OpEntryPoint, OpDecorate etc, process other instructions.
3117 processOtherInstrs(M);
3118
3119 // If there are no entry points, we need the Linkage capability.
3120 if (MAI.MS[SPIRV::MB_EntryPoints].empty())
3121 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::Linkage);
3122
3123 // Set maximum ID used.
3124 GR->setBound(MAI.MaxID);
3125
3126 return false;
3127}
3128