1//===- SPIRVModuleAnalysis.cpp - analysis of global instrs & regs - C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The analysis collects instructions that should be output at the module level
10// and performs the global register numbering.
11//
12// The results of this analysis are used in AsmPrinter to rename registers
13// globally and to output required instructions at the module level.
14//
15//===----------------------------------------------------------------------===//
16
17// TODO: Per LLVM best practices, the report_fatal_error (deprecated) /
18// ReportFatalUsageError calls in this file should be replaced with the
19// Diagnostic infrastructure (e.g. the reportUnsupported function below).
20
21#include "SPIRVModuleAnalysis.h"
22#include "MCTargetDesc/SPIRVBaseInfo.h"
23#include "MCTargetDesc/SPIRVMCTargetDesc.h"
24#include "SPIRV.h"
25#include "SPIRVSubtarget.h"
26#include "SPIRVTargetMachine.h"
27#include "SPIRVUtils.h"
28#include "llvm/ADT/STLExtras.h"
29#include "llvm/CodeGen/MachineModuleInfo.h"
30#include "llvm/CodeGen/TargetPassConfig.h"
31
32using namespace llvm;
33
34#define DEBUG_TYPE "spirv-module-analysis"
35
36static cl::opt<bool>
37 SPVDumpDeps("spv-dump-deps",
38 cl::desc("Dump MIR with SPIR-V dependencies info"),
39 cl::Optional, cl::init(Val: false));
40
41static cl::list<SPIRV::Capability::Capability>
42 AvoidCapabilities("avoid-spirv-capabilities",
43 cl::desc("SPIR-V capabilities to avoid if there are "
44 "other options enabling a feature"),
45 cl::Hidden,
46 cl::values(clEnumValN(SPIRV::Capability::Shader, "Shader",
47 "SPIR-V Shader capability")));
48// Use sets instead of cl::list to check "if contains" condition
49struct AvoidCapabilitiesSet {
50 SmallSet<SPIRV::Capability::Capability, 4> S;
51 AvoidCapabilitiesSet() { S.insert_range(R&: AvoidCapabilities); }
52};
53
54char llvm::SPIRVModuleAnalysis::ID = 0;
55
56INITIALIZE_PASS(SPIRVModuleAnalysis, DEBUG_TYPE, "SPIRV module analysis", true,
57 true)
58
59static void reportUnsupported(const MachineInstr &MI, const char *Msg) {
60 const Function &Func = MI.getMF()->getFunction();
61 Func.getContext().diagnose(
62 DI: DiagnosticInfoUnsupported(Func, Msg, MI.getDebugLoc()));
63}
64
65// Retrieve an unsigned from an MDNode with a list of them as operands.
66static unsigned getMetadataUInt(MDNode *MdNode, unsigned OpIndex,
67 unsigned DefaultVal = 0) {
68 if (MdNode && OpIndex < MdNode->getNumOperands()) {
69 const auto &Op = MdNode->getOperand(I: OpIndex);
70 return mdconst::extract<ConstantInt>(MD: Op)->getZExtValue();
71 }
72 return DefaultVal;
73}
74
75static SPIRV::Requirements
76getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
77 unsigned i, const SPIRVSubtarget &ST,
78 SPIRV::RequirementHandler &Reqs) {
79 // A set of capabilities to avoid if there is another option.
80 AvoidCapabilitiesSet AvoidCaps;
81 if (!ST.isShader())
82 AvoidCaps.S.insert(V: SPIRV::Capability::Shader);
83 else
84 AvoidCaps.S.insert(V: SPIRV::Capability::Kernel);
85
86 VersionTuple ReqMinVer = getSymbolicOperandMinVersion(Category, Value: i);
87 VersionTuple ReqMaxVer = getSymbolicOperandMaxVersion(Category, Value: i);
88 VersionTuple SPIRVVersion = ST.getSPIRVVersion();
89 bool MinVerOK = SPIRVVersion.empty() || SPIRVVersion >= ReqMinVer;
90 bool MaxVerOK =
91 ReqMaxVer.empty() || SPIRVVersion.empty() || SPIRVVersion <= ReqMaxVer;
92 CapabilityList ReqCaps = getSymbolicOperandCapabilities(Category, Value: i);
93 ExtensionList ReqExts = getSymbolicOperandExtensions(Category, Value: i);
94 if (ReqCaps.empty()) {
95 if (ReqExts.empty()) {
96 if (MinVerOK && MaxVerOK)
97 return {true, {}, {}, ReqMinVer, ReqMaxVer};
98 return {false, {}, {}, VersionTuple(), VersionTuple()};
99 }
100 } else if (MinVerOK && MaxVerOK) {
101 if (ReqCaps.size() == 1) {
102 auto Cap = ReqCaps[0];
103 if (Reqs.isCapabilityAvailable(Cap)) {
104 ReqExts.append(RHS: getSymbolicOperandExtensions(
105 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
106 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
107 }
108 } else {
109 // By SPIR-V specification: "If an instruction, enumerant, or other
110 // feature specifies multiple enabling capabilities, only one such
111 // capability needs to be declared to use the feature." However, one
112 // capability may be preferred over another. We use command line
113 // argument(s) and AvoidCapabilities to avoid selection of certain
114 // capabilities if there are other options.
115 CapabilityList UseCaps;
116 for (auto Cap : ReqCaps)
117 if (Reqs.isCapabilityAvailable(Cap))
118 UseCaps.push_back(Elt: Cap);
119 for (size_t i = 0, Sz = UseCaps.size(); i < Sz; ++i) {
120 auto Cap = UseCaps[i];
121 if (i == Sz - 1 || !AvoidCaps.S.contains(V: Cap)) {
122 ReqExts.append(RHS: getSymbolicOperandExtensions(
123 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
124 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
125 }
126 }
127 }
128 }
129 // If there are no capabilities, or we can't satisfy the version or
130 // capability requirements, use the list of extensions (if the subtarget
131 // can handle them all).
132 if (llvm::all_of(Range&: ReqExts, P: [&ST](const SPIRV::Extension::Extension &Ext) {
133 return ST.canUseExtension(E: Ext);
134 })) {
135 return {true,
136 {},
137 std::move(ReqExts),
138 VersionTuple(),
139 VersionTuple()}; // TODO: add versions to extensions.
140 }
141 return {false, {}, {}, VersionTuple(), VersionTuple()};
142}
143
144void SPIRVModuleAnalysis::setBaseInfo(const Module &M) {
145 MAI.MaxID = 0;
146 for (int i = 0; i < SPIRV::NUM_MODULE_SECTIONS; i++)
147 MAI.MS[i].clear();
148 MAI.RegisterAliasTable.clear();
149 MAI.InstrsToDelete.clear();
150 MAI.GlobalObjMap.clear();
151 MAI.GlobalVarList.clear();
152 MAI.ExtInstSetMap.clear();
153 MAI.Reqs.clear();
154 MAI.Reqs.initAvailableCapabilities(ST: *ST);
155
156 // TODO: determine memory model and source language from the configuratoin.
157 if (auto MemModel = M.getNamedMetadata(Name: "spirv.MemoryModel")) {
158 auto MemMD = MemModel->getOperand(i: 0);
159 MAI.Addr = static_cast<SPIRV::AddressingModel::AddressingModel>(
160 getMetadataUInt(MdNode: MemMD, OpIndex: 0));
161 MAI.Mem =
162 static_cast<SPIRV::MemoryModel::MemoryModel>(getMetadataUInt(MdNode: MemMD, OpIndex: 1));
163 } else {
164 // TODO: Add support for VulkanMemoryModel.
165 MAI.Mem = ST->isShader() ? SPIRV::MemoryModel::GLSL450
166 : SPIRV::MemoryModel::OpenCL;
167 if (MAI.Mem == SPIRV::MemoryModel::OpenCL) {
168 unsigned PtrSize = ST->getPointerSize();
169 MAI.Addr = PtrSize == 32 ? SPIRV::AddressingModel::Physical32
170 : PtrSize == 64 ? SPIRV::AddressingModel::Physical64
171 : SPIRV::AddressingModel::Logical;
172 } else {
173 // TODO: Add support for PhysicalStorageBufferAddress.
174 MAI.Addr = SPIRV::AddressingModel::Logical;
175 }
176 }
177 // Get the OpenCL version number from metadata.
178 // TODO: support other source languages.
179 if (auto VerNode = M.getNamedMetadata(Name: "opencl.ocl.version")) {
180 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_C;
181 // Construct version literal in accordance with SPIRV-LLVM-Translator.
182 // TODO: support multiple OCL version metadata.
183 assert(VerNode->getNumOperands() > 0 && "Invalid SPIR");
184 auto VersionMD = VerNode->getOperand(i: 0);
185 unsigned MajorNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 0, DefaultVal: 2);
186 unsigned MinorNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 1);
187 unsigned RevNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 2);
188 // Prevent Major part of OpenCL version to be 0
189 MAI.SrcLangVersion =
190 (std::max(a: 1U, b: MajorNum) * 100 + MinorNum) * 1000 + RevNum;
191 // When opencl.cxx.version is also present, validate compatibility
192 // and use C++ for OpenCL as source language with the C++ version.
193 if (auto *CxxVerNode = M.getNamedMetadata(Name: "opencl.cxx.version")) {
194 assert(CxxVerNode->getNumOperands() > 0 && "Invalid SPIR");
195 auto *CxxMD = CxxVerNode->getOperand(i: 0);
196 unsigned CxxVer =
197 (getMetadataUInt(MdNode: CxxMD, OpIndex: 0) * 100 + getMetadataUInt(MdNode: CxxMD, OpIndex: 1)) * 1000 +
198 getMetadataUInt(MdNode: CxxMD, OpIndex: 2);
199 if ((MAI.SrcLangVersion == 200000 && CxxVer == 100000) ||
200 (MAI.SrcLangVersion == 300000 && CxxVer == 202100000)) {
201 MAI.SrcLang = SPIRV::SourceLanguage::CPP_for_OpenCL;
202 MAI.SrcLangVersion = CxxVer;
203 } else {
204 report_fatal_error(
205 reason: "opencl cxx version is not compatible with opencl c version!");
206 }
207 }
208 } else {
209 // If there is no information about OpenCL version we are forced to generate
210 // OpenCL 1.0 by default for the OpenCL environment to avoid puzzling
211 // run-times with Unknown/0.0 version output. For a reference, LLVM-SPIRV
212 // Translator avoids potential issues with run-times in a similar manner.
213 if (!ST->isShader()) {
214 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_CPP;
215 MAI.SrcLangVersion = 100000;
216 } else {
217 MAI.SrcLang = SPIRV::SourceLanguage::Unknown;
218 MAI.SrcLangVersion = 0;
219 }
220 }
221
222 if (auto ExtNode = M.getNamedMetadata(Name: "opencl.used.extensions")) {
223 for (unsigned I = 0, E = ExtNode->getNumOperands(); I != E; ++I) {
224 MDNode *MD = ExtNode->getOperand(i: I);
225 if (!MD || MD->getNumOperands() == 0)
226 continue;
227 for (unsigned J = 0, N = MD->getNumOperands(); J != N; ++J)
228 MAI.SrcExt.insert(key: cast<MDString>(Val: MD->getOperand(I: J))->getString());
229 }
230 }
231
232 // Update required capabilities for this memory model, addressing model and
233 // source language.
234 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::MemoryModelOperand,
235 i: MAI.Mem, ST: *ST);
236 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::SourceLanguageOperand,
237 i: MAI.SrcLang, ST: *ST);
238 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::AddressingModelOperand,
239 i: MAI.Addr, ST: *ST);
240
241 if (MAI.Mem == SPIRV::MemoryModel::VulkanKHR)
242 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_vulkan_memory_model);
243
244 if (!ST->isShader()) {
245 // TODO: check if it's required by default.
246 MAI.ExtInstSetMap[static_cast<unsigned>(
247 SPIRV::InstructionSet::OpenCL_std)] = MAI.getNextIDRegister();
248 }
249}
250
251// Appends the signature of the decoration instructions that decorate R to
252// Signature.
253static void appendDecorationsForReg(const MachineRegisterInfo &MRI, Register R,
254 InstrSignature &Signature) {
255 for (MachineInstr &UseMI : MRI.use_instructions(Reg: R)) {
256 // We don't handle OpDecorateId because getting the register alias for the
257 // ID can cause problems, and we do not need it for now.
258 if (UseMI.getOpcode() != SPIRV::OpDecorate &&
259 UseMI.getOpcode() != SPIRV::OpMemberDecorate)
260 continue;
261
262 for (unsigned I = 0; I < UseMI.getNumOperands(); ++I) {
263 const MachineOperand &MO = UseMI.getOperand(i: I);
264 if (MO.isReg())
265 continue;
266 Signature.push_back(Elt: hash_value(MO));
267 }
268 }
269}
270
271// Returns a representation of an instruction as a vector of MachineOperand
272// hash values, see llvm::hash_value(const MachineOperand &MO) for details.
273// This creates a signature of the instruction with the same content
274// that MachineOperand::isIdenticalTo uses for comparison.
275static InstrSignature instrToSignature(const MachineInstr &MI,
276 SPIRV::ModuleAnalysisInfo &MAI,
277 bool UseDefReg) {
278 Register DefReg;
279 InstrSignature Signature{MI.getOpcode()};
280 for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
281 // The only decorations that can be applied more than once to a given <id>
282 // or structure member are FuncParamAttr (38), UserSemantic (5635),
283 // CacheControlLoadINTEL (6442), and CacheControlStoreINTEL (6443). For all
284 // the rest of decorations, we will only add to the signature the Opcode,
285 // the id to which it applies, and the decoration id, disregarding any
286 // decoration flags. This will ensure that any subsequent decoration with
287 // the same id will be deemed as a duplicate. Then, at the call site, we
288 // will be able to handle duplicates in the best way.
289 unsigned Opcode = MI.getOpcode();
290 if ((Opcode == SPIRV::OpDecorate) && i >= 2) {
291 unsigned DecorationID = MI.getOperand(i: 1).getImm();
292 if (DecorationID != SPIRV::Decoration::FuncParamAttr &&
293 DecorationID != SPIRV::Decoration::UserSemantic &&
294 DecorationID != SPIRV::Decoration::CacheControlLoadINTEL &&
295 DecorationID != SPIRV::Decoration::CacheControlStoreINTEL)
296 continue;
297 }
298 const MachineOperand &MO = MI.getOperand(i);
299 size_t h;
300 if (MO.isReg()) {
301 if (!UseDefReg && MO.isDef()) {
302 assert(!DefReg.isValid() && "Multiple def registers.");
303 DefReg = MO.getReg();
304 continue;
305 }
306 Register RegAlias = MAI.getRegisterAlias(MF: MI.getMF(), Reg: MO.getReg());
307 if (!RegAlias.isValid()) {
308 LLVM_DEBUG({
309 dbgs() << "Unexpectedly, no global id found for the operand ";
310 MO.print(dbgs());
311 dbgs() << "\nInstruction: ";
312 MI.print(dbgs());
313 dbgs() << "\n";
314 });
315 report_fatal_error(reason: "All v-regs must have been mapped to global id's");
316 }
317 // mimic llvm::hash_value(const MachineOperand &MO)
318 h = hash_combine(args: MO.getType(), args: (unsigned)RegAlias, args: MO.getSubReg(),
319 args: MO.isDef());
320 } else {
321 h = hash_value(MO);
322 }
323 Signature.push_back(Elt: h);
324 }
325
326 if (DefReg.isValid()) {
327 // Decorations change the semantics of the current instruction. So two
328 // identical instruction with different decorations cannot be merged. That
329 // is why we add the decorations to the signature.
330 appendDecorationsForReg(MRI: MI.getMF()->getRegInfo(), R: DefReg, Signature);
331 }
332 return Signature;
333}
334
335bool SPIRVModuleAnalysis::isDeclSection(const MachineRegisterInfo &MRI,
336 const MachineInstr &MI) {
337 unsigned Opcode = MI.getOpcode();
338 switch (Opcode) {
339 case SPIRV::OpTypeForwardPointer:
340 // omit now, collect later
341 return false;
342 case SPIRV::OpVariable:
343 return static_cast<SPIRV::StorageClass::StorageClass>(
344 MI.getOperand(i: 2).getImm()) != SPIRV::StorageClass::Function;
345 case SPIRV::OpFunction:
346 case SPIRV::OpFunctionParameter:
347 return true;
348 }
349 if (GR->hasConstFunPtr() && Opcode == SPIRV::OpUndef) {
350 // The OpUndef may be a placeholder for a function reference recorded by
351 // selectGlobalValue. Skip emitting it if any user consumes it as a
352 // function-pointer-like operand (OpConstantFunctionPointerINTEL operand 2,
353 // or OpEnqueueKernel's Invoke operand at index 8). The rewrite happens
354 // in visitFunPtrUse, which aliases the OpUndef's vreg to the function's
355 // global <id>.
356 Register DefReg = MI.getOperand(i: 0).getReg();
357 if (GR->getFunctionDefinitionByUse(Use: &MI.getOperand(i: 0))) {
358 for (MachineInstr &UseMI : MRI.use_instructions(Reg: DefReg)) {
359 unsigned UseOp = UseMI.getOpcode();
360 if (UseOp == SPIRV::OpConstantFunctionPointerINTEL ||
361 UseOp == SPIRV::OpEnqueueKernel) {
362 MAI.setSkipEmission(&MI);
363 return false;
364 }
365 }
366 }
367 for (MachineInstr &UseMI : MRI.use_instructions(Reg: DefReg)) {
368 if (UseMI.getOpcode() != SPIRV::OpConstantFunctionPointerINTEL)
369 continue;
370 // it's a dummy definition, FP constant refers to a function,
371 // and this is resolved in another way; let's skip this definition
372 assert(UseMI.getOperand(2).isReg() &&
373 UseMI.getOperand(2).getReg() == DefReg);
374 MAI.setSkipEmission(&MI);
375 return false;
376 }
377 }
378 return TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||
379 TII->isInlineAsmDefInstr(MI);
380}
381
382// This is a special case of a function pointer referring to a possibly
383// forward function declaration. The operand is a dummy OpUndef that
384// requires a special treatment.
385// FunPtrOp is the MachineOperand previously recorded via
386// SPIRVGlobalRegistry::recordFunctionPointer, identifying which Function
387// this placeholder refers to.
388void SPIRVModuleAnalysis::visitFunPtrUse(
389 Register OpReg, const MachineOperand *FunPtrOp,
390 InstrGRegsMap &SignatureToGReg,
391 std::map<const Value *, unsigned> &GlobalToGReg,
392 const MachineFunction *MF) {
393 const MachineOperand *OpFunDef = GR->getFunctionDefinitionByUse(Use: FunPtrOp);
394 assert(OpFunDef && OpFunDef->isReg());
395 // find the actual function definition and number it globally in advance
396 const MachineInstr *OpDefMI = OpFunDef->getParent();
397 assert(OpDefMI && OpDefMI->getOpcode() == SPIRV::OpFunction);
398 const MachineFunction *FunDefMF = OpDefMI->getParent()->getParent();
399 const MachineRegisterInfo &FunDefMRI = FunDefMF->getRegInfo();
400 do {
401 visitDecl(MRI: FunDefMRI, SignatureToGReg, GlobalToGReg, MF: FunDefMF, MI: *OpDefMI);
402 OpDefMI = OpDefMI->getNextNode();
403 } while (OpDefMI && (OpDefMI->getOpcode() == SPIRV::OpFunction ||
404 OpDefMI->getOpcode() == SPIRV::OpFunctionParameter));
405 // associate the function pointer with the newly assigned global number
406 MCRegister GlobalFunDefReg =
407 MAI.getRegisterAlias(MF: FunDefMF, Reg: OpFunDef->getReg());
408 assert(GlobalFunDefReg.isValid() &&
409 "Function definition must refer to a global register");
410 MAI.setRegisterAlias(MF, Reg: OpReg, AliasReg: GlobalFunDefReg);
411}
412
413// Depth first recursive traversal of dependencies. Repeated visits are guarded
414// by MAI.hasRegisterAlias().
415void SPIRVModuleAnalysis::visitDecl(
416 const MachineRegisterInfo &MRI, InstrGRegsMap &SignatureToGReg,
417 std::map<const Value *, unsigned> &GlobalToGReg, const MachineFunction *MF,
418 const MachineInstr &MI) {
419 unsigned Opcode = MI.getOpcode();
420
421 // Process each operand of the instruction to resolve dependencies
422 for (const MachineOperand &MO : MI.operands()) {
423 if (!MO.isReg() || MO.isDef())
424 continue;
425 Register OpReg = MO.getReg();
426 // Handle function pointers special case
427 if (Opcode == SPIRV::OpConstantFunctionPointerINTEL &&
428 MRI.getRegClass(Reg: OpReg) == &SPIRV::pIDRegClass) {
429 visitFunPtrUse(OpReg, FunPtrOp: &MI.getOperand(i: 2), SignatureToGReg, GlobalToGReg,
430 MF);
431 continue;
432 }
433 // Skip already processed instructions
434 if (MAI.hasRegisterAlias(MF, Reg: MO.getReg()))
435 continue;
436 // Recursively visit dependencies
437 if (const MachineInstr *OpDefMI = MRI.getUniqueVRegDef(Reg: OpReg)) {
438 if (isDeclSection(MRI, MI: *OpDefMI))
439 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, MI: *OpDefMI);
440 continue;
441 }
442 // Handle the unexpected case of no unique definition for the SPIR-V
443 // instruction
444 LLVM_DEBUG({
445 dbgs() << "Unexpectedly, no unique definition for the operand ";
446 MO.print(dbgs());
447 dbgs() << "\nInstruction: ";
448 MI.print(dbgs());
449 dbgs() << "\n";
450 });
451 report_fatal_error(
452 reason: "No unique definition is found for the virtual register");
453 }
454
455 MCRegister GReg;
456 bool IsFunDef = false;
457 if (TII->isSpecConstantInstr(MI)) {
458 GReg = MAI.getNextIDRegister();
459 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
460 } else if (Opcode == SPIRV::OpFunction ||
461 Opcode == SPIRV::OpFunctionParameter) {
462 GReg = handleFunctionOrParameter(MF, MI, GlobalToGReg, IsFunDef);
463 } else if (Opcode == SPIRV::OpTypeStruct ||
464 Opcode == SPIRV::OpConstantComposite) {
465 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);
466 const MachineInstr *NextInstr = MI.getNextNode();
467 while (NextInstr &&
468 ((Opcode == SPIRV::OpTypeStruct &&
469 NextInstr->getOpcode() == SPIRV::OpTypeStructContinuedINTEL) ||
470 (Opcode == SPIRV::OpConstantComposite &&
471 NextInstr->getOpcode() ==
472 SPIRV::OpConstantCompositeContinuedINTEL))) {
473 MCRegister Tmp = handleTypeDeclOrConstant(MI: *NextInstr, SignatureToGReg);
474 MAI.setRegisterAlias(MF, Reg: NextInstr->getOperand(i: 0).getReg(), AliasReg: Tmp);
475 MAI.setSkipEmission(NextInstr);
476 NextInstr = NextInstr->getNextNode();
477 }
478 } else if (TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||
479 TII->isInlineAsmDefInstr(MI)) {
480 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);
481 } else if (Opcode == SPIRV::OpVariable) {
482 GReg = handleVariable(MF, MI, GlobalToGReg);
483 } else {
484 LLVM_DEBUG({
485 dbgs() << "\nInstruction: ";
486 MI.print(dbgs());
487 dbgs() << "\n";
488 });
489 llvm_unreachable("Unexpected instruction is visited");
490 }
491 MAI.setRegisterAlias(MF, Reg: MI.getOperand(i: 0).getReg(), AliasReg: GReg);
492 if (!IsFunDef)
493 MAI.setSkipEmission(&MI);
494}
495
496MCRegister SPIRVModuleAnalysis::handleFunctionOrParameter(
497 const MachineFunction *MF, const MachineInstr &MI,
498 std::map<const Value *, unsigned> &GlobalToGReg, bool &IsFunDef) {
499 const Value *GObj = GR->getGlobalObject(MF, R: MI.getOperand(i: 0).getReg());
500 assert(GObj && "Unregistered global definition");
501 const Function *F = dyn_cast<Function>(Val: GObj);
502 if (!F)
503 F = dyn_cast<Argument>(Val: GObj)->getParent();
504 assert(F && "Expected a reference to a function or an argument");
505 IsFunDef = !F->isDeclaration();
506 auto [It, Inserted] = GlobalToGReg.try_emplace(k: GObj);
507 if (!Inserted)
508 return It->second;
509 MCRegister GReg = MAI.getNextIDRegister();
510 It->second = GReg;
511 if (!IsFunDef)
512 MAI.MS[SPIRV::MB_ExtFuncDecls].push_back(Elt: &MI);
513 return GReg;
514}
515
516MCRegister
517SPIRVModuleAnalysis::handleTypeDeclOrConstant(const MachineInstr &MI,
518 InstrGRegsMap &SignatureToGReg) {
519 InstrSignature MISign = instrToSignature(MI, MAI, UseDefReg: false);
520 auto [It, Inserted] = SignatureToGReg.try_emplace(k: MISign);
521 if (!Inserted)
522 return It->second;
523 MCRegister GReg = MAI.getNextIDRegister();
524 It->second = GReg;
525 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
526 return GReg;
527}
528
529MCRegister SPIRVModuleAnalysis::handleVariable(
530 const MachineFunction *MF, const MachineInstr &MI,
531 std::map<const Value *, unsigned> &GlobalToGReg) {
532 MAI.GlobalVarList.push_back(Elt: &MI);
533 const Value *GObj = GR->getGlobalObject(MF, R: MI.getOperand(i: 0).getReg());
534 assert(GObj && "Unregistered global definition");
535 auto [It, Inserted] = GlobalToGReg.try_emplace(k: GObj);
536 if (!Inserted)
537 return It->second;
538 MCRegister GReg = MAI.getNextIDRegister();
539 It->second = GReg;
540 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
541 if (const auto *GV = dyn_cast<GlobalVariable>(Val: GObj))
542 MAI.GlobalObjMap[GV] = GReg;
543 return GReg;
544}
545
546void SPIRVModuleAnalysis::collectDeclarations(const Module &M) {
547 InstrGRegsMap SignatureToGReg;
548 std::map<const Value *, unsigned> GlobalToGReg;
549 for (const Function &F : M) {
550 MachineFunction *MF = MMI->getMachineFunction(F);
551 if (!MF)
552 continue;
553 const MachineRegisterInfo &MRI = MF->getRegInfo();
554 unsigned PastHeader = 0;
555 for (MachineBasicBlock &MBB : *MF) {
556 for (MachineInstr &MI : MBB) {
557 if (MI.getNumOperands() == 0)
558 continue;
559 unsigned Opcode = MI.getOpcode();
560 if (Opcode == SPIRV::OpFunction) {
561 if (PastHeader == 0) {
562 PastHeader = 1;
563 continue;
564 }
565 } else if (Opcode == SPIRV::OpFunctionParameter) {
566 if (PastHeader < 2)
567 continue;
568 } else if (PastHeader > 0) {
569 PastHeader = 2;
570 }
571
572 const MachineOperand &DefMO = MI.getOperand(i: 0);
573 switch (Opcode) {
574 case SPIRV::OpExtension:
575 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::Extension(DefMO.getImm()));
576 MAI.setSkipEmission(&MI);
577 break;
578 case SPIRV::OpCapability:
579 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::Capability(DefMO.getImm()));
580 MAI.setSkipEmission(&MI);
581 if (PastHeader > 0)
582 PastHeader = 2;
583 break;
584 default:
585 if (DefMO.isReg() && isDeclSection(MRI, MI) &&
586 !MAI.hasRegisterAlias(MF, Reg: DefMO.getReg()))
587 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, MI);
588 // OpEnqueueKernel is not a decl, but its Invoke operand may be a
589 // function-pointer placeholder OpUndef recorded by selectGlobalValue.
590 // Resolve it to the OpFunction's global <id> via visitFunPtrUse.
591 if (Opcode == SPIRV::OpEnqueueKernel && MI.getNumOperands() > 8) {
592 const MachineOperand &InvokeMO = MI.getOperand(i: 8);
593 if (InvokeMO.isReg()) {
594 Register InvokeReg = InvokeMO.getReg();
595 if (!MAI.hasRegisterAlias(MF, Reg: InvokeReg)) {
596 if (const MachineInstr *DefMI =
597 MRI.getUniqueVRegDef(Reg: InvokeReg)) {
598 if (DefMI->getOpcode() == SPIRV::OpUndef) {
599 const MachineOperand *FunPtrOp = &DefMI->getOperand(i: 0);
600 if (GR->getFunctionDefinitionByUse(Use: FunPtrOp))
601 visitFunPtrUse(OpReg: InvokeReg, FunPtrOp, SignatureToGReg,
602 GlobalToGReg, MF);
603 }
604 }
605 }
606 }
607 }
608 }
609 }
610 }
611 }
612}
613
614// Look for IDs declared with Import linkage, and map the corresponding function
615// to the register defining that variable (which will usually be the result of
616// an OpFunction). This lets us call externally imported functions using
617// the correct ID registers.
618void SPIRVModuleAnalysis::collectFuncNames(MachineInstr &MI,
619 const Function *F) {
620 if (MI.getOpcode() == SPIRV::OpDecorate) {
621 // If it's got Import linkage.
622 auto Dec = MI.getOperand(i: 1).getImm();
623 if (Dec == SPIRV::Decoration::LinkageAttributes) {
624 auto Lnk = MI.getOperand(i: MI.getNumOperands() - 1).getImm();
625 if (Lnk == SPIRV::LinkageType::Import) {
626 // Map imported function name to function ID register.
627 const Function *ImportedFunc =
628 F->getParent()->getFunction(Name: getStringImm(MI, StartIndex: 2));
629 Register Target = MI.getOperand(i: 0).getReg();
630 MAI.GlobalObjMap[ImportedFunc] =
631 MAI.getRegisterAlias(MF: MI.getMF(), Reg: Target);
632 }
633 }
634 } else if (MI.getOpcode() == SPIRV::OpFunction) {
635 // Record all internal OpFunction declarations.
636 Register Reg = MI.defs().begin()->getReg();
637 MCRegister GlobalReg = MAI.getRegisterAlias(MF: MI.getMF(), Reg);
638 assert(GlobalReg.isValid());
639 MAI.GlobalObjMap[F] = GlobalReg;
640 }
641}
642
643// Collect the given instruction in the specified MS. We assume global register
644// numbering has already occurred by this point. We can directly compare reg
645// arguments when detecting duplicates.
646static void collectOtherInstr(MachineInstr &MI, SPIRV::ModuleAnalysisInfo &MAI,
647 SPIRV::ModuleSectionType MSType, InstrTraces &IS,
648 bool Append = true) {
649 MAI.setSkipEmission(&MI);
650 InstrSignature MISign = instrToSignature(MI, MAI, UseDefReg: true);
651 auto FoundMI = IS.insert(x: std::move(MISign));
652 if (!FoundMI.second) {
653 if (MI.getOpcode() == SPIRV::OpDecorate) {
654 assert(MI.getNumOperands() >= 2 &&
655 "Decoration instructions must have at least 2 operands");
656 assert(MSType == SPIRV::MB_Annotations &&
657 "Only OpDecorate instructions can be duplicates");
658 // For FPFastMathMode decoration, we need to merge the flags of the
659 // duplicate decoration with the original one, so we need to find the
660 // original instruction that has the same signature. For the rest of
661 // instructions, we will simply skip the duplicate.
662 if (MI.getOperand(i: 1).getImm() != SPIRV::Decoration::FPFastMathMode)
663 return; // Skip duplicates of other decorations.
664
665 const SPIRV::InstrList &Decorations = MAI.MS[MSType];
666 for (const MachineInstr *OrigMI : Decorations) {
667 if (instrToSignature(MI: *OrigMI, MAI, UseDefReg: true) == MISign) {
668 assert(OrigMI->getNumOperands() == MI.getNumOperands() &&
669 "Original instruction must have the same number of operands");
670 assert(
671 OrigMI->getNumOperands() == 3 &&
672 "FPFastMathMode decoration must have 3 operands for OpDecorate");
673 unsigned OrigFlags = OrigMI->getOperand(i: 2).getImm();
674 unsigned NewFlags = MI.getOperand(i: 2).getImm();
675 if (OrigFlags == NewFlags)
676 return; // No need to merge, the flags are the same.
677
678 // Emit warning about possible conflict between flags.
679 unsigned FinalFlags = OrigFlags | NewFlags;
680 llvm::errs()
681 << "Warning: Conflicting FPFastMathMode decoration flags "
682 "in instruction: "
683 << *OrigMI << "Original flags: " << OrigFlags
684 << ", new flags: " << NewFlags
685 << ". They will be merged on a best effort basis, but not "
686 "validated. Final flags: "
687 << FinalFlags << "\n";
688 MachineInstr *OrigMINonConst = const_cast<MachineInstr *>(OrigMI);
689 MachineOperand &OrigFlagsOp = OrigMINonConst->getOperand(i: 2);
690 OrigFlagsOp = MachineOperand::CreateImm(Val: FinalFlags);
691 return; // Merge done, so we found a duplicate; don't add it to MAI.MS
692 }
693 }
694 assert(false && "No original instruction found for the duplicate "
695 "OpDecorate, but we found one in IS.");
696 }
697 return; // insert failed, so we found a duplicate; don't add it to MAI.MS
698 }
699 // No duplicates, so add it.
700 if (Append)
701 MAI.MS[MSType].push_back(Elt: &MI);
702 else
703 MAI.MS[MSType].insert(I: MAI.MS[MSType].begin(), Elt: &MI);
704}
705
706// Some global instructions make reference to function-local ID regs, so cannot
707// be correctly collected until these registers are globally numbered.
708void SPIRVModuleAnalysis::processOtherInstrs(const Module &M) {
709 InstrTraces IS;
710 for (const Function &F : M) {
711 if (F.isDeclaration())
712 continue;
713 MachineFunction *MF = MMI->getMachineFunction(F);
714 assert(MF);
715
716 for (MachineBasicBlock &MBB : *MF)
717 for (MachineInstr &MI : MBB) {
718 if (MAI.getSkipEmission(MI: &MI))
719 continue;
720 const unsigned OpCode = MI.getOpcode();
721 if (OpCode == SPIRV::OpString) {
722 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_DebugStrings, IS);
723 } else if (OpCode == SPIRV::OpExtInst && MI.getOperand(i: 2).isImm() &&
724 MI.getOperand(i: 2).getImm() ==
725 SPIRV::InstructionSet::
726 NonSemantic_Shader_DebugInfo_100) {
727 // TODO: This branch is dead. SPIRVNonSemanticDebugHandler emits NSDI
728 // instructions directly as MCInsts at print time; no
729 // MachineInstructions with the NSDI ext set are created anymore.
730 // Remove this block and
731 // MB_NonSemanticGlobalDI once per-function NSDI emission is confirmed
732 // not to need MIR routing.
733 MachineOperand Ins = MI.getOperand(i: 3);
734 namespace NS = SPIRV::NonSemanticExtInst;
735 static constexpr int64_t GlobalNonSemanticDITy[] = {
736 NS::DebugSource, NS::DebugCompilationUnit, NS::DebugInfoNone,
737 NS::DebugTypeBasic, NS::DebugTypePointer};
738 bool IsGlobalDI = false;
739 for (unsigned Idx = 0; Idx < std::size(GlobalNonSemanticDITy); ++Idx)
740 IsGlobalDI |= Ins.getImm() == GlobalNonSemanticDITy[Idx];
741 if (IsGlobalDI)
742 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_NonSemanticGlobalDI, IS);
743 } else if (OpCode == SPIRV::OpName || OpCode == SPIRV::OpMemberName) {
744 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_DebugNames, IS);
745 } else if (OpCode == SPIRV::OpEntryPoint) {
746 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_EntryPoints, IS);
747 } else if (TII->isAliasingInstr(MI)) {
748 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_AliasingInsts, IS);
749 } else if (TII->isDecorationInstr(MI)) {
750 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_Annotations, IS);
751 collectFuncNames(MI, F: &F);
752 } else if (TII->isConstantInstr(MI)) {
753 // Now OpSpecConstant*s are not in DT,
754 // but they need to be collected anyway.
755 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_TypeConstVars, IS);
756 } else if (OpCode == SPIRV::OpFunction) {
757 collectFuncNames(MI, F: &F);
758 } else if (OpCode == SPIRV::OpTypeForwardPointer) {
759 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_TypeConstVars, IS, Append: false);
760 }
761 }
762 }
763 // Selection order can place a scope/list ahead of a domain/scope it
764 // references. The dependency meanwhile is domain -> scope -> list, so sort
765 // the def before its uses.
766 auto AliasingTier = [](const MachineInstr *MI) {
767 switch (MI->getOpcode()) {
768 case SPIRV::OpAliasDomainDeclINTEL:
769 return 0;
770 case SPIRV::OpAliasScopeDeclINTEL:
771 return 1;
772 case SPIRV::OpAliasScopeListDeclINTEL:
773 return 2;
774 default:
775 llvm_unreachable("unexpected aliasing instruction");
776 }
777 };
778 stable_sort(Range&: MAI.MS[SPIRV::MB_AliasingInsts],
779 C: [&](const MachineInstr *LHS, const MachineInstr *RHS) {
780 return AliasingTier(LHS) < AliasingTier(RHS);
781 });
782}
783
784// Number registers in all functions globally from 0 onwards and store
785// the result in global register alias table. Some registers are already
786// numbered.
787void SPIRVModuleAnalysis::numberRegistersGlobally(const Module &M) {
788 for (const Function &F : M) {
789 if (F.isDeclaration())
790 continue;
791 MachineFunction *MF = MMI->getMachineFunction(F);
792 assert(MF);
793 for (MachineBasicBlock &MBB : *MF) {
794 for (MachineInstr &MI : MBB) {
795 for (MachineOperand &Op : MI.operands()) {
796 if (!Op.isReg())
797 continue;
798 Register Reg = Op.getReg();
799 if (MAI.hasRegisterAlias(MF, Reg))
800 continue;
801 MCRegister NewReg = MAI.getNextIDRegister();
802 MAI.setRegisterAlias(MF, Reg, AliasReg: NewReg);
803 }
804 if (MI.getOpcode() != SPIRV::OpExtInst)
805 continue;
806 auto Set = MI.getOperand(i: 2).getImm();
807 auto [It, Inserted] = MAI.ExtInstSetMap.try_emplace(Key: Set);
808 if (Inserted)
809 It->second = MAI.getNextIDRegister();
810 }
811 }
812 }
813}
814
815// RequirementHandler implementations.
816void SPIRV::RequirementHandler::getAndAddRequirements(
817 SPIRV::OperandCategory::OperandCategory Category, uint32_t i,
818 const SPIRVSubtarget &ST) {
819 addRequirements(Req: getSymbolicOperandRequirements(Category, i, ST, Reqs&: *this));
820}
821
822void SPIRV::RequirementHandler::recursiveAddCapabilities(
823 const CapabilityList &ToPrune) {
824 for (const auto &Cap : ToPrune) {
825 AllCaps.insert(V: Cap);
826 CapabilityList ImplicitDecls =
827 getSymbolicOperandCapabilities(Category: OperandCategory::CapabilityOperand, Value: Cap);
828 recursiveAddCapabilities(ToPrune: ImplicitDecls);
829 }
830}
831
832void SPIRV::RequirementHandler::addCapabilities(const CapabilityList &ToAdd) {
833 for (const auto &Cap : ToAdd) {
834 bool IsNewlyInserted = AllCaps.insert(V: Cap).second;
835 if (!IsNewlyInserted) // Don't re-add if it's already been declared.
836 continue;
837 CapabilityList ImplicitDecls =
838 getSymbolicOperandCapabilities(Category: OperandCategory::CapabilityOperand, Value: Cap);
839 recursiveAddCapabilities(ToPrune: ImplicitDecls);
840 MinimalCaps.push_back(Elt: Cap);
841 }
842}
843
844void SPIRV::RequirementHandler::addRequirements(
845 const SPIRV::Requirements &Req) {
846 if (!Req.IsSatisfiable)
847 report_fatal_error(reason: "Adding SPIR-V requirements this target can't satisfy.");
848
849 if (Req.Cap.has_value())
850 addCapabilities(ToAdd: {Req.Cap.value()});
851
852 addExtensions(ToAdd: Req.Exts);
853
854 if (!Req.MinVer.empty()) {
855 if (!MaxVersion.empty() && Req.MinVer > MaxVersion) {
856 LLVM_DEBUG(dbgs() << "Conflicting version requirements: >= " << Req.MinVer
857 << " and <= " << MaxVersion << "\n");
858 report_fatal_error(reason: "Adding SPIR-V requirements that can't be satisfied.");
859 }
860
861 if (MinVersion.empty() || Req.MinVer > MinVersion)
862 MinVersion = Req.MinVer;
863 }
864
865 if (!Req.MaxVer.empty()) {
866 if (!MinVersion.empty() && Req.MaxVer < MinVersion) {
867 LLVM_DEBUG(dbgs() << "Conflicting version requirements: <= " << Req.MaxVer
868 << " and >= " << MinVersion << "\n");
869 report_fatal_error(reason: "Adding SPIR-V requirements that can't be satisfied.");
870 }
871
872 if (MaxVersion.empty() || Req.MaxVer < MaxVersion)
873 MaxVersion = Req.MaxVer;
874 }
875}
876
877void SPIRV::RequirementHandler::checkSatisfiable(
878 const SPIRVSubtarget &ST) const {
879 // Report as many errors as possible before aborting the compilation.
880 bool IsSatisfiable = true;
881 auto TargetVer = ST.getSPIRVVersion();
882
883 if (!MaxVersion.empty() && !TargetVer.empty() && MaxVersion < TargetVer) {
884 LLVM_DEBUG(
885 dbgs() << "Target SPIR-V version too high for required features\n"
886 << "Required max version: " << MaxVersion << " target version "
887 << TargetVer << "\n");
888 IsSatisfiable = false;
889 }
890
891 if (!MinVersion.empty() && !TargetVer.empty() && MinVersion > TargetVer) {
892 LLVM_DEBUG(dbgs() << "Target SPIR-V version too low for required features\n"
893 << "Required min version: " << MinVersion
894 << " target version " << TargetVer << "\n");
895 IsSatisfiable = false;
896 }
897
898 if (!MinVersion.empty() && !MaxVersion.empty() && MinVersion > MaxVersion) {
899 LLVM_DEBUG(
900 dbgs()
901 << "Version is too low for some features and too high for others.\n"
902 << "Required SPIR-V min version: " << MinVersion
903 << " required SPIR-V max version " << MaxVersion << "\n");
904 IsSatisfiable = false;
905 }
906
907 AvoidCapabilitiesSet AvoidCaps;
908 if (!ST.isShader())
909 AvoidCaps.S.insert(V: SPIRV::Capability::Shader);
910 else
911 AvoidCaps.S.insert(V: SPIRV::Capability::Kernel);
912
913 for (auto Cap : MinimalCaps) {
914 if (AvailableCaps.contains(V: Cap) && !AvoidCaps.S.contains(V: Cap))
915 continue;
916 LLVM_DEBUG(dbgs() << "Capability not supported: "
917 << getSymbolicOperandMnemonic(
918 OperandCategory::CapabilityOperand, Cap)
919 << "\n");
920 IsSatisfiable = false;
921 }
922
923 for (auto Ext : AllExtensions) {
924 if (ST.canUseExtension(E: Ext))
925 continue;
926 LLVM_DEBUG(dbgs() << "Extension not supported: "
927 << getSymbolicOperandMnemonic(
928 OperandCategory::ExtensionOperand, Ext)
929 << "\n");
930 IsSatisfiable = false;
931 }
932
933 if (!IsSatisfiable)
934 report_fatal_error(reason: "Unable to meet SPIR-V requirements for this target.");
935}
936
937// Add the given capabilities and all their implicitly defined capabilities too.
938void SPIRV::RequirementHandler::addAvailableCaps(const CapabilityList &ToAdd) {
939 for (const auto Cap : ToAdd)
940 if (AvailableCaps.insert(V: Cap).second)
941 addAvailableCaps(ToAdd: getSymbolicOperandCapabilities(
942 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
943}
944
945void SPIRV::RequirementHandler::removeCapabilityIf(
946 const Capability::Capability ToRemove,
947 const Capability::Capability IfPresent) {
948 if (AllCaps.contains(V: IfPresent)) {
949 AllCaps.erase(V: ToRemove);
950 llvm::erase(C&: MinimalCaps, V: ToRemove);
951 }
952}
953
954namespace llvm {
955namespace SPIRV {
956void RequirementHandler::initAvailableCapabilities(const SPIRVSubtarget &ST) {
957 // Provided by both all supported Vulkan versions and OpenCl.
958 addAvailableCaps(ToAdd: {Capability::Shader, Capability::Linkage, Capability::Int8,
959 Capability::Int16});
960
961 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 3)))
962 addAvailableCaps(ToAdd: {Capability::GroupNonUniform,
963 Capability::GroupNonUniformVote,
964 Capability::GroupNonUniformArithmetic,
965 Capability::GroupNonUniformBallot,
966 Capability::GroupNonUniformClustered,
967 Capability::GroupNonUniformShuffle,
968 Capability::GroupNonUniformShuffleRelative,
969 Capability::GroupNonUniformQuad});
970
971 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
972 addAvailableCaps(ToAdd: {Capability::DotProduct, Capability::DotProductInputAll,
973 Capability::DotProductInput4x8Bit,
974 Capability::DotProductInput4x8BitPacked,
975 Capability::DemoteToHelperInvocation});
976
977 // Add capabilities enabled by extensions.
978 for (auto Extension : ST.getAllAvailableExtensions()) {
979 CapabilityList EnabledCapabilities =
980 getCapabilitiesEnabledByExtension(Extension);
981 addAvailableCaps(ToAdd: EnabledCapabilities);
982 }
983
984 if (!ST.isShader()) {
985 initAvailableCapabilitiesForOpenCL(ST);
986 return;
987 }
988
989 if (ST.isShader()) {
990 initAvailableCapabilitiesForVulkan(ST);
991 return;
992 }
993
994 report_fatal_error(reason: "Unimplemented environment for SPIR-V generation.");
995}
996
997void RequirementHandler::initAvailableCapabilitiesForOpenCL(
998 const SPIRVSubtarget &ST) {
999 // Add the min requirements for different OpenCL and SPIR-V versions.
1000 addAvailableCaps(ToAdd: {Capability::Addresses, Capability::Float16Buffer,
1001 Capability::Kernel, Capability::Vector16,
1002 Capability::Groups, Capability::GenericPointer,
1003 Capability::StorageImageWriteWithoutFormat,
1004 Capability::StorageImageReadWithoutFormat});
1005 if (ST.hasOpenCLFullProfile())
1006 addAvailableCaps(ToAdd: {Capability::Int64, Capability::Int64Atomics});
1007 if (ST.hasOpenCLImageSupport()) {
1008 addAvailableCaps(ToAdd: {Capability::ImageBasic, Capability::LiteralSampler,
1009 Capability::Image1D, Capability::SampledBuffer,
1010 Capability::ImageBuffer});
1011 if (ST.isAtLeastOpenCLVer(VerToCompareTo: VersionTuple(2, 0)))
1012 addAvailableCaps(ToAdd: {Capability::ImageReadWrite});
1013 }
1014 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 1)) &&
1015 ST.isAtLeastOpenCLVer(VerToCompareTo: VersionTuple(2, 2)))
1016 addAvailableCaps(ToAdd: {Capability::SubgroupDispatch, Capability::PipeStorage});
1017 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 4)))
1018 addAvailableCaps(ToAdd: {Capability::DenormPreserve, Capability::DenormFlushToZero,
1019 Capability::SignedZeroInfNanPreserve,
1020 Capability::RoundingModeRTE,
1021 Capability::RoundingModeRTZ});
1022 // TODO: verify if this needs some checks.
1023 addAvailableCaps(ToAdd: {Capability::Float16, Capability::Float64});
1024
1025 // TODO: add OpenCL extensions.
1026}
1027
1028void RequirementHandler::initAvailableCapabilitiesForVulkan(
1029 const SPIRVSubtarget &ST) {
1030
1031 // Core in Vulkan 1.1 and earlier.
1032 addAvailableCaps(ToAdd: {Capability::Int64,
1033 Capability::Float16,
1034 Capability::Float64,
1035 Capability::GroupNonUniform,
1036 Capability::Image1D,
1037 Capability::SampledBuffer,
1038 Capability::ImageBuffer,
1039 Capability::UniformBufferArrayDynamicIndexing,
1040 Capability::SampledImageArrayDynamicIndexing,
1041 Capability::StorageBufferArrayDynamicIndexing,
1042 Capability::StorageImageArrayDynamicIndexing,
1043 Capability::DerivativeControl,
1044 Capability::MinLod,
1045 Capability::ImageQuery,
1046 Capability::ImageGatherExtended,
1047 Capability::Addresses,
1048 Capability::VulkanMemoryModelKHR,
1049 Capability::StorageImageExtendedFormats,
1050 Capability::StorageImageMultisample,
1051 Capability::ImageMSArray});
1052
1053 // Became core in Vulkan 1.2
1054 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 5))) {
1055 addAvailableCaps(
1056 ToAdd: {Capability::Int64Atomics, Capability::ShaderNonUniformEXT,
1057 Capability::RuntimeDescriptorArrayEXT,
1058 Capability::InputAttachmentArrayDynamicIndexingEXT,
1059 Capability::UniformTexelBufferArrayDynamicIndexingEXT,
1060 Capability::StorageTexelBufferArrayDynamicIndexingEXT,
1061 Capability::UniformBufferArrayNonUniformIndexingEXT,
1062 Capability::SampledImageArrayNonUniformIndexingEXT,
1063 Capability::StorageBufferArrayNonUniformIndexingEXT,
1064 Capability::StorageImageArrayNonUniformIndexingEXT,
1065 Capability::InputAttachmentArrayNonUniformIndexingEXT,
1066 Capability::UniformTexelBufferArrayNonUniformIndexingEXT,
1067 Capability::StorageTexelBufferArrayNonUniformIndexingEXT});
1068 }
1069
1070 // Became core in Vulkan 1.3
1071 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
1072 addAvailableCaps(ToAdd: {Capability::StorageImageWriteWithoutFormat,
1073 Capability::StorageImageReadWithoutFormat});
1074}
1075
1076} // namespace SPIRV
1077} // namespace llvm
1078
1079// Add the required capabilities from a decoration instruction (including
1080// BuiltIns).
1081static void addOpDecorateReqs(const MachineInstr &MI, unsigned DecIndex,
1082 SPIRV::RequirementHandler &Reqs,
1083 const SPIRVSubtarget &ST) {
1084 int64_t DecOp = MI.getOperand(i: DecIndex).getImm();
1085 auto Dec = static_cast<SPIRV::Decoration::Decoration>(DecOp);
1086 Reqs.addRequirements(Req: getSymbolicOperandRequirements(
1087 Category: SPIRV::OperandCategory::DecorationOperand, i: Dec, ST, Reqs));
1088
1089 if (Dec == SPIRV::Decoration::BuiltIn) {
1090 int64_t BuiltInOp = MI.getOperand(i: DecIndex + 1).getImm();
1091 auto BuiltIn = static_cast<SPIRV::BuiltIn::BuiltIn>(BuiltInOp);
1092 Reqs.addRequirements(Req: getSymbolicOperandRequirements(
1093 Category: SPIRV::OperandCategory::BuiltInOperand, i: BuiltIn, ST, Reqs));
1094 } else if (Dec == SPIRV::Decoration::LinkageAttributes) {
1095 int64_t LinkageOp = MI.getOperand(i: MI.getNumOperands() - 1).getImm();
1096 SPIRV::LinkageType::LinkageType LnkType =
1097 static_cast<SPIRV::LinkageType::LinkageType>(LinkageOp);
1098 if (LnkType == SPIRV::LinkageType::LinkOnceODR)
1099 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_linkonce_odr);
1100 else if (LnkType == SPIRV::LinkageType::WeakAMD) {
1101 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_AMD_weak_linkage);
1102 Reqs.addCapability(ToAdd: SPIRV::Capability::WeakLinkageAMD);
1103 }
1104 } else if (Dec == SPIRV::Decoration::CacheControlLoadINTEL ||
1105 Dec == SPIRV::Decoration::CacheControlStoreINTEL) {
1106 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_cache_controls);
1107 } else if (Dec == SPIRV::Decoration::HostAccessINTEL) {
1108 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_global_variable_host_access);
1109 } else if (Dec == SPIRV::Decoration::InitModeINTEL ||
1110 Dec == SPIRV::Decoration::ImplementInRegisterMapINTEL) {
1111 Reqs.addExtension(
1112 ToAdd: SPIRV::Extension::SPV_INTEL_global_variable_fpga_decorations);
1113 } else if (Dec == SPIRV::Decoration::NonUniformEXT) {
1114 Reqs.addRequirements(Req: SPIRV::Capability::ShaderNonUniformEXT);
1115 } else if (Dec == SPIRV::Decoration::FPMaxErrorDecorationINTEL) {
1116 Reqs.addRequirements(Req: SPIRV::Capability::FPMaxErrorINTEL);
1117 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_fp_max_error);
1118 } else if (Dec == SPIRV::Decoration::FPFastMathMode) {
1119 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2)) {
1120 Reqs.addRequirements(Req: SPIRV::Capability::FloatControls2);
1121 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls2);
1122 }
1123 }
1124}
1125
1126// Add requirements for image handling.
1127static void addOpTypeImageReqs(const MachineInstr &MI,
1128 SPIRV::RequirementHandler &Reqs,
1129 const SPIRVSubtarget &ST) {
1130 assert(MI.getNumOperands() >= 8 && "Insufficient operands for OpTypeImage");
1131 // The operand indices used here are based on the OpTypeImage layout, which
1132 // the MachineInstr follows as well.
1133 int64_t ImgFormatOp = MI.getOperand(i: 7).getImm();
1134 auto ImgFormat = static_cast<SPIRV::ImageFormat::ImageFormat>(ImgFormatOp);
1135 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ImageFormatOperand,
1136 i: ImgFormat, ST);
1137
1138 bool IsArrayed = MI.getOperand(i: 4).getImm() == 1;
1139 bool IsMultisampled = MI.getOperand(i: 5).getImm() == 1;
1140 bool NoSampler = MI.getOperand(i: 6).getImm() == 2;
1141 // Add dimension requirements.
1142 assert(MI.getOperand(2).isImm());
1143 switch (MI.getOperand(i: 2).getImm()) {
1144 case SPIRV::Dim::DIM_1D:
1145 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::Image1D
1146 : SPIRV::Capability::Sampled1D);
1147 break;
1148 case SPIRV::Dim::DIM_2D:
1149 if (IsMultisampled && NoSampler)
1150 Reqs.addRequirements(Req: SPIRV::Capability::StorageImageMultisample);
1151 if (IsMultisampled && IsArrayed)
1152 Reqs.addRequirements(Req: SPIRV::Capability::ImageMSArray);
1153 break;
1154 case SPIRV::Dim::DIM_3D:
1155 break;
1156 case SPIRV::Dim::DIM_Cube:
1157 Reqs.addRequirements(Req: SPIRV::Capability::Shader);
1158 if (IsArrayed)
1159 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageCubeArray
1160 : SPIRV::Capability::SampledCubeArray);
1161 break;
1162 case SPIRV::Dim::DIM_Rect:
1163 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageRect
1164 : SPIRV::Capability::SampledRect);
1165 break;
1166 case SPIRV::Dim::DIM_Buffer:
1167 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageBuffer
1168 : SPIRV::Capability::SampledBuffer);
1169 break;
1170 case SPIRV::Dim::DIM_SubpassData:
1171 Reqs.addRequirements(Req: SPIRV::Capability::InputAttachment);
1172 break;
1173 }
1174
1175 // Has optional access qualifier.
1176 if (!ST.isShader()) {
1177 if (MI.getNumOperands() > 8 &&
1178 MI.getOperand(i: 8).getImm() == SPIRV::AccessQualifier::ReadWrite)
1179 Reqs.addRequirements(Req: SPIRV::Capability::ImageReadWrite);
1180 else
1181 Reqs.addRequirements(Req: SPIRV::Capability::ImageBasic);
1182 }
1183}
1184
1185static bool isBFloat16Type(SPIRVTypeInst TypeDef) {
1186 return TypeDef && TypeDef->getNumOperands() == 3 &&
1187 TypeDef->getOpcode() == SPIRV::OpTypeFloat &&
1188 TypeDef->getOperand(i: 1).getImm() == 16 &&
1189 TypeDef->getOperand(i: 2).getImm() == SPIRV::FPEncoding::BFloat16KHR;
1190}
1191
1192// Add requirements for handling atomic float instructions
1193#define ATOM_FLT_REQ_EXT_MSG(ExtName) \
1194 "The atomic float instruction requires the following SPIR-V " \
1195 "extension: SPV_EXT_shader_atomic_float" ExtName
1196static void AddAtomicVectorFloatRequirements(const MachineInstr &MI,
1197 SPIRV::RequirementHandler &Reqs,
1198 const SPIRVSubtarget &ST) {
1199 SPIRVTypeInst VecTypeDef =
1200 MI.getMF()->getRegInfo().getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1201
1202 const unsigned Rank = VecTypeDef->getOperand(i: 2).getImm();
1203 if (Rank != 2 && Rank != 4)
1204 reportFatalUsageError(reason: "Result type of an atomic vector float instruction "
1205 "must be a 2-component or 4 component vector");
1206
1207 SPIRVTypeInst EltTypeDef =
1208 MI.getMF()->getRegInfo().getVRegDef(Reg: VecTypeDef->getOperand(i: 1).getReg());
1209
1210 if (EltTypeDef->getOpcode() != SPIRV::OpTypeFloat ||
1211 EltTypeDef->getOperand(i: 1).getImm() != 16)
1212 reportFatalUsageError(
1213 reason: "The element type for the result type of an atomic vector float "
1214 "instruction must be a 16-bit floating-point scalar");
1215
1216 // The extension is defined for fp16, but the AMD target lets a bf16 vector
1217 // use the same instruction so it can lower to a packed bf16 atomic.
1218 if (isBFloat16Type(TypeDef: EltTypeDef) &&
1219 ST.getTargetTriple().getVendor() != Triple::AMD)
1220 reportFatalUsageError(
1221 reason: "The element type for the result type of an atomic vector float "
1222 "instruction cannot be a bfloat16 scalar");
1223 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_NV_shader_atomic_fp16_vector))
1224 reportFatalUsageError(
1225 reason: "The atomic float16 vector instruction requires the following SPIR-V "
1226 "extension: SPV_NV_shader_atomic_fp16_vector");
1227
1228 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_NV_shader_atomic_fp16_vector);
1229 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16VectorNV);
1230}
1231
1232static void AddAtomicFloatRequirements(const MachineInstr &MI,
1233 SPIRV::RequirementHandler &Reqs,
1234 const SPIRVSubtarget &ST) {
1235 assert(MI.getOperand(1).isReg() &&
1236 "Expect register operand in atomic float instruction");
1237 Register TypeReg = MI.getOperand(i: 1).getReg();
1238 SPIRVTypeInst TypeDef = MI.getMF()->getRegInfo().getVRegDef(Reg: TypeReg);
1239
1240 if (TypeDef->getOpcode() == SPIRV::OpTypeVector)
1241 return AddAtomicVectorFloatRequirements(MI, Reqs, ST);
1242
1243 if (TypeDef->getOpcode() != SPIRV::OpTypeFloat)
1244 report_fatal_error(reason: "Result type of an atomic float instruction must be a "
1245 "floating-point type scalar");
1246
1247 unsigned BitWidth = TypeDef->getOperand(i: 1).getImm();
1248 unsigned Op = MI.getOpcode();
1249 if (Op == SPIRV::OpAtomicFAddEXT) {
1250 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_shader_atomic_float_add))
1251 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("_add"), gen_crash_diag: false);
1252 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float_add);
1253 switch (BitWidth) {
1254 case 16:
1255 if (isBFloat16Type(TypeDef)) {
1256 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1257 report_fatal_error(
1258 reason: "The atomic bfloat16 instruction requires the following SPIR-V "
1259 "extension: SPV_INTEL_16bit_atomics",
1260 gen_crash_diag: false);
1261 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1262 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicBFloat16AddINTEL);
1263 } else {
1264 if (!ST.canUseExtension(
1265 E: SPIRV::Extension::SPV_EXT_shader_atomic_float16_add))
1266 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("16_add"), gen_crash_diag: false);
1267 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float16_add);
1268 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16AddEXT);
1269 }
1270 break;
1271 case 32:
1272 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat32AddEXT);
1273 break;
1274 case 64:
1275 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat64AddEXT);
1276 break;
1277 default:
1278 report_fatal_error(
1279 reason: "Unexpected floating-point type width in atomic float instruction");
1280 }
1281 } else {
1282 if (!ST.canUseExtension(
1283 E: SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max))
1284 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("_min_max"), gen_crash_diag: false);
1285 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max);
1286 switch (BitWidth) {
1287 case 16:
1288 if (isBFloat16Type(TypeDef)) {
1289 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1290 report_fatal_error(
1291 reason: "The atomic bfloat16 instruction requires the following SPIR-V "
1292 "extension: SPV_INTEL_16bit_atomics",
1293 gen_crash_diag: false);
1294 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1295 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicBFloat16MinMaxINTEL);
1296 } else {
1297 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16MinMaxEXT);
1298 }
1299 break;
1300 case 32:
1301 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat32MinMaxEXT);
1302 break;
1303 case 64:
1304 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat64MinMaxEXT);
1305 break;
1306 default:
1307 report_fatal_error(
1308 reason: "Unexpected floating-point type width in atomic float instruction");
1309 }
1310 }
1311}
1312
1313bool isUniformTexelBuffer(MachineInstr *ImageInst) {
1314 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1315 return false;
1316 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1317 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1318 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 1;
1319}
1320
1321bool isStorageTexelBuffer(MachineInstr *ImageInst) {
1322 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1323 return false;
1324 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1325 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1326 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 2;
1327}
1328
1329bool isSampledImage(MachineInstr *ImageInst) {
1330 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1331 return false;
1332 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1333 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1334 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 1;
1335}
1336
1337bool isInputAttachment(MachineInstr *ImageInst) {
1338 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1339 return false;
1340 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1341 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1342 return Dim == SPIRV::Dim::DIM_SubpassData && Sampled == 2;
1343}
1344
1345bool isStorageImage(MachineInstr *ImageInst) {
1346 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1347 return false;
1348 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1349 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1350 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 2;
1351}
1352
1353bool isCombinedImageSampler(MachineInstr *SampledImageInst) {
1354 if (SampledImageInst->getOpcode() != SPIRV::OpTypeSampledImage)
1355 return false;
1356
1357 const MachineRegisterInfo &MRI = SampledImageInst->getMF()->getRegInfo();
1358 Register ImageReg = SampledImageInst->getOperand(i: 1).getReg();
1359 auto *ImageInst = MRI.getUniqueVRegDef(Reg: ImageReg);
1360 return isSampledImage(ImageInst);
1361}
1362
1363bool hasNonUniformDecoration(Register Reg, const MachineRegisterInfo &MRI) {
1364 for (const auto &MI : MRI.reg_instructions(Reg)) {
1365 if (MI.getOpcode() != SPIRV::OpDecorate)
1366 continue;
1367
1368 uint32_t Dec = MI.getOperand(i: 1).getImm();
1369 if (Dec == SPIRV::Decoration::NonUniformEXT)
1370 return true;
1371 }
1372 return false;
1373}
1374
1375void addOpAccessChainReqs(const MachineInstr &Instr,
1376 SPIRV::RequirementHandler &Handler,
1377 const SPIRVSubtarget &Subtarget) {
1378 const MachineRegisterInfo &MRI = Instr.getMF()->getRegInfo();
1379 // Get the result type. If it is an image type, then the shader uses
1380 // descriptor indexing. The appropriate capabilities will be added based
1381 // on the specifics of the image.
1382 Register ResTypeReg = Instr.getOperand(i: 1).getReg();
1383 MachineInstr *ResTypeInst = MRI.getUniqueVRegDef(Reg: ResTypeReg);
1384
1385 assert(ResTypeInst->getOpcode() == SPIRV::OpTypePointer);
1386 uint32_t StorageClass = ResTypeInst->getOperand(i: 1).getImm();
1387 if (StorageClass != SPIRV::StorageClass::StorageClass::UniformConstant &&
1388 StorageClass != SPIRV::StorageClass::StorageClass::Uniform &&
1389 StorageClass != SPIRV::StorageClass::StorageClass::StorageBuffer) {
1390 return;
1391 }
1392
1393 bool IsNonUniform =
1394 hasNonUniformDecoration(Reg: Instr.getOperand(i: 0).getReg(), MRI);
1395
1396 auto FirstIndexReg = Instr.getOperand(i: 3).getReg();
1397 bool FirstIndexIsConstant =
1398 Subtarget.getInstrInfo()->isConstantInstr(MI: *MRI.getVRegDef(Reg: FirstIndexReg));
1399
1400 if (StorageClass == SPIRV::StorageClass::StorageClass::StorageBuffer) {
1401 if (IsNonUniform)
1402 Handler.addRequirements(
1403 Req: SPIRV::Capability::StorageBufferArrayNonUniformIndexingEXT);
1404 else if (!FirstIndexIsConstant)
1405 Handler.addRequirements(
1406 Req: SPIRV::Capability::StorageBufferArrayDynamicIndexing);
1407 return;
1408 }
1409
1410 Register PointeeTypeReg = ResTypeInst->getOperand(i: 2).getReg();
1411 MachineInstr *PointeeType = MRI.getUniqueVRegDef(Reg: PointeeTypeReg);
1412 if (PointeeType->getOpcode() != SPIRV::OpTypeImage &&
1413 PointeeType->getOpcode() != SPIRV::OpTypeSampledImage &&
1414 PointeeType->getOpcode() != SPIRV::OpTypeSampler) {
1415 return;
1416 }
1417
1418 if (isUniformTexelBuffer(ImageInst: PointeeType)) {
1419 if (IsNonUniform)
1420 Handler.addRequirements(
1421 Req: SPIRV::Capability::UniformTexelBufferArrayNonUniformIndexingEXT);
1422 else if (!FirstIndexIsConstant)
1423 Handler.addRequirements(
1424 Req: SPIRV::Capability::UniformTexelBufferArrayDynamicIndexingEXT);
1425 } else if (isInputAttachment(ImageInst: PointeeType)) {
1426 if (IsNonUniform)
1427 Handler.addRequirements(
1428 Req: SPIRV::Capability::InputAttachmentArrayNonUniformIndexingEXT);
1429 else if (!FirstIndexIsConstant)
1430 Handler.addRequirements(
1431 Req: SPIRV::Capability::InputAttachmentArrayDynamicIndexingEXT);
1432 } else if (isStorageTexelBuffer(ImageInst: PointeeType)) {
1433 if (IsNonUniform)
1434 Handler.addRequirements(
1435 Req: SPIRV::Capability::StorageTexelBufferArrayNonUniformIndexingEXT);
1436 else if (!FirstIndexIsConstant)
1437 Handler.addRequirements(
1438 Req: SPIRV::Capability::StorageTexelBufferArrayDynamicIndexingEXT);
1439 } else if (isSampledImage(ImageInst: PointeeType) ||
1440 isCombinedImageSampler(SampledImageInst: PointeeType) ||
1441 PointeeType->getOpcode() == SPIRV::OpTypeSampler) {
1442 if (IsNonUniform)
1443 Handler.addRequirements(
1444 Req: SPIRV::Capability::SampledImageArrayNonUniformIndexingEXT);
1445 else if (!FirstIndexIsConstant)
1446 Handler.addRequirements(
1447 Req: SPIRV::Capability::SampledImageArrayDynamicIndexing);
1448 } else if (isStorageImage(ImageInst: PointeeType)) {
1449 if (IsNonUniform)
1450 Handler.addRequirements(
1451 Req: SPIRV::Capability::StorageImageArrayNonUniformIndexingEXT);
1452 else if (!FirstIndexIsConstant)
1453 Handler.addRequirements(
1454 Req: SPIRV::Capability::StorageImageArrayDynamicIndexing);
1455 }
1456}
1457
1458static bool isImageTypeWithUnknownFormat(SPIRVTypeInst TypeInst) {
1459 if (TypeInst->getOpcode() != SPIRV::OpTypeImage)
1460 return false;
1461 assert(TypeInst->getOperand(7).isImm() && "The image format must be an imm.");
1462 return TypeInst->getOperand(i: 7).getImm() == 0;
1463}
1464
1465static void AddDotProductRequirements(const MachineInstr &MI,
1466 SPIRV::RequirementHandler &Reqs,
1467 const SPIRVSubtarget &ST) {
1468 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product))
1469 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_integer_dot_product);
1470 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProduct);
1471
1472 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1473 assert(MI.getOperand(2).isReg() && "Unexpected operand in dot");
1474 // We do not consider what the previous instruction is. This is just used
1475 // to get the input register and to check the type.
1476 const MachineInstr *Input = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
1477 assert(Input->getOperand(1).isReg() && "Unexpected operand in dot input");
1478 Register InputReg = Input->getOperand(i: 1).getReg();
1479
1480 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: InputReg);
1481 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {
1482 assert(TypeDef->getOperand(1).getImm() == 32);
1483 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInput4x8BitPacked);
1484 } else if (TypeDef->getOpcode() == SPIRV::OpTypeVector) {
1485 SPIRVTypeInst ScalarTypeDef =
1486 MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
1487 assert(ScalarTypeDef->getOpcode() == SPIRV::OpTypeInt);
1488 if (ScalarTypeDef->getOperand(i: 1).getImm() == 8) {
1489 assert(TypeDef->getOperand(2).getImm() == 4 &&
1490 "Dot operand of 8-bit integer type requires 4 components");
1491 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInput4x8Bit);
1492 } else {
1493 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInputAll);
1494 }
1495 }
1496}
1497
1498void addPrintfRequirements(const MachineInstr &MI,
1499 SPIRV::RequirementHandler &Reqs,
1500 const SPIRVSubtarget &ST) {
1501 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1502 SPIRVTypeInst PtrType =
1503 GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 4).getReg(), MF: MI.getMF());
1504 if (PtrType) {
1505 MachineOperand ASOp = PtrType->getOperand(i: 1);
1506 if (ASOp.isImm()) {
1507 unsigned AddrSpace = ASOp.getImm();
1508 if (AddrSpace != SPIRV::StorageClass::UniformConstant) {
1509 if (!ST.canUseExtension(
1510 E: SPIRV::Extension::
1511 SPV_EXT_relaxed_printf_string_address_space)) {
1512 report_fatal_error(reason: "SPV_EXT_relaxed_printf_string_address_space is "
1513 "required because printf uses a format string not "
1514 "in constant address space.",
1515 gen_crash_diag: false);
1516 }
1517 Reqs.addExtension(
1518 ToAdd: SPIRV::Extension::SPV_EXT_relaxed_printf_string_address_space);
1519 }
1520 }
1521 }
1522}
1523
1524static void addImageOperandReqs(const MachineInstr &MI,
1525 SPIRV::RequirementHandler &Reqs,
1526 const SPIRVSubtarget &ST, unsigned OpIdx) {
1527 if (MI.getNumOperands() <= OpIdx)
1528 return;
1529 uint32_t Mask = MI.getOperand(i: OpIdx).getImm();
1530 for (uint32_t I = 0; I < 32; ++I)
1531 if (Mask & (1U << I))
1532 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ImageOperandOperand,
1533 i: 1U << I, ST);
1534}
1535
1536void addInstrRequirements(const MachineInstr &MI,
1537 SPIRV::ModuleAnalysisInfo &MAI,
1538 const SPIRVSubtarget &ST) {
1539 SPIRV::RequirementHandler &Reqs = MAI.Reqs;
1540 unsigned Op = MI.getOpcode();
1541 switch (Op) {
1542 case SPIRV::OpMemoryModel: {
1543 int64_t Addr = MI.getOperand(i: 0).getImm();
1544 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::AddressingModelOperand,
1545 i: Addr, ST);
1546 int64_t Mem = MI.getOperand(i: 1).getImm();
1547 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::MemoryModelOperand, i: Mem,
1548 ST);
1549 break;
1550 }
1551 case SPIRV::OpEntryPoint: {
1552 int64_t Exe = MI.getOperand(i: 0).getImm();
1553 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ExecutionModelOperand,
1554 i: Exe, ST);
1555 break;
1556 }
1557 case SPIRV::OpExecutionMode:
1558 case SPIRV::OpExecutionModeId: {
1559 int64_t Exe = MI.getOperand(i: 1).getImm();
1560 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ExecutionModeOperand,
1561 i: Exe, ST);
1562 break;
1563 }
1564 case SPIRV::OpTypeMatrix:
1565 Reqs.addCapability(ToAdd: SPIRV::Capability::Matrix);
1566 break;
1567 case SPIRV::OpTypeInt: {
1568 unsigned BitWidth = MI.getOperand(i: 1).getImm();
1569 if (BitWidth == 64)
1570 Reqs.addCapability(ToAdd: SPIRV::Capability::Int64);
1571 else if (BitWidth == 16)
1572 Reqs.addCapability(ToAdd: SPIRV::Capability::Int16);
1573 else if (BitWidth == 8)
1574 Reqs.addCapability(ToAdd: SPIRV::Capability::Int8);
1575 else if (BitWidth == 4 &&
1576 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4)) {
1577 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_int4);
1578 Reqs.addCapability(ToAdd: SPIRV::Capability::Int4TypeINTEL);
1579 } else if (BitWidth != 32) {
1580 if (!ST.canUseExtension(
1581 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers))
1582 reportFatalUsageError(
1583 reason: "OpTypeInt type with a width other than 8, 16, 32 or 64 bits "
1584 "requires the following SPIR-V extension: "
1585 "SPV_ALTERA_arbitrary_precision_integers");
1586 Reqs.addExtension(
1587 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
1588 Reqs.addCapability(ToAdd: SPIRV::Capability::ArbitraryPrecisionIntegersALTERA);
1589 }
1590 break;
1591 }
1592 case SPIRV::OpDot: {
1593 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1594 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1595 if (isBFloat16Type(TypeDef))
1596 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16DotProductKHR);
1597 break;
1598 }
1599 case SPIRV::OpTypeFloat: {
1600 unsigned BitWidth = MI.getOperand(i: 1).getImm();
1601 if (BitWidth == 64)
1602 Reqs.addCapability(ToAdd: SPIRV::Capability::Float64);
1603 else if (BitWidth == 16) {
1604 if (isBFloat16Type(TypeDef: &MI)) {
1605 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_bfloat16))
1606 report_fatal_error(reason: "OpTypeFloat type with bfloat requires the "
1607 "following SPIR-V extension: SPV_KHR_bfloat16",
1608 gen_crash_diag: false);
1609 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_bfloat16);
1610 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16TypeKHR);
1611 } else {
1612 Reqs.addCapability(ToAdd: SPIRV::Capability::Float16);
1613 }
1614 }
1615 break;
1616 }
1617 case SPIRV::OpTypeVector: {
1618 unsigned NumComponents = MI.getOperand(i: 2).getImm();
1619 if (NumComponents == 8 || NumComponents == 16)
1620 Reqs.addCapability(ToAdd: SPIRV::Capability::Vector16);
1621
1622 assert(MI.getOperand(1).isReg());
1623 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1624 SPIRVTypeInst ElemTypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1625 if (ElemTypeDef->getOpcode() == SPIRV::OpTypePointer &&
1626 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
1627 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_masked_gather_scatter);
1628 Reqs.addCapability(ToAdd: SPIRV::Capability::MaskedGatherScatterINTEL);
1629 }
1630 break;
1631 }
1632 case SPIRV::OpTypePointer: {
1633 auto SC = MI.getOperand(i: 1).getImm();
1634 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::StorageClassOperand, i: SC,
1635 ST);
1636 // If it's a type of pointer to float16 targeting OpenCL, add Float16Buffer
1637 // capability.
1638 if (ST.isShader())
1639 break;
1640 assert(MI.getOperand(2).isReg());
1641 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1642 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
1643 if ((TypeDef->getNumOperands() == 2) &&
1644 (TypeDef->getOpcode() == SPIRV::OpTypeFloat) &&
1645 (TypeDef->getOperand(i: 1).getImm() == 16))
1646 Reqs.addCapability(ToAdd: SPIRV::Capability::Float16Buffer);
1647 break;
1648 }
1649 case SPIRV::OpExtInst: {
1650 if (MI.getOperand(i: 2).getImm() ==
1651 static_cast<int64_t>(
1652 SPIRV::InstructionSet::NonSemantic_Shader_DebugInfo_100)) {
1653 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_non_semantic_info);
1654 break;
1655 }
1656 if (MI.getOperand(i: 3).getImm() ==
1657 static_cast<int64_t>(SPIRV::OpenCLExtInst::printf)) {
1658 addPrintfRequirements(MI, Reqs, ST);
1659 break;
1660 }
1661 if (MI.getOperand(i: 2).getImm() ==
1662 static_cast<int64_t>(SPIRV::InstructionSet::OpenCL_std)) {
1663 const MachineFunction *MF = MI.getMF();
1664 const MachineRegisterInfo &MRI = MF->getRegInfo();
1665 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1666
1667 auto IsBFloat16 = [&](SPIRVTypeInst TypeDef) {
1668 if (TypeDef && TypeDef->getOpcode() == SPIRV::OpTypeVector)
1669 TypeDef = MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
1670 return isBFloat16Type(TypeDef);
1671 };
1672
1673 // Result type is operand 1; arguments start at operand 4.
1674 bool UsesBFloat16 = IsBFloat16(MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg()));
1675 for (unsigned I = 4, E = MI.getNumOperands(); I < E && !UsesBFloat16;
1676 ++I) {
1677 const MachineOperand &MO = MI.getOperand(i: I);
1678 if (MO.isReg())
1679 UsesBFloat16 = IsBFloat16(GR->getResultType(
1680 VReg: MO.getReg(), MF: const_cast<MachineFunction *>(MF)));
1681 }
1682
1683 if (UsesBFloat16) {
1684 if (!ST.canUseExtension(
1685 E: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic)) {
1686 reportUnsupported(
1687 MI, Msg: "OpenCL Extended instructions with bfloat16 require the "
1688 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic");
1689 break;
1690 }
1691 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);
1692 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ArithmeticINTEL);
1693 }
1694 }
1695 break;
1696 }
1697 case SPIRV::OpAliasDomainDeclINTEL:
1698 case SPIRV::OpAliasScopeDeclINTEL:
1699 case SPIRV::OpAliasScopeListDeclINTEL: {
1700 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_memory_access_aliasing);
1701 Reqs.addCapability(ToAdd: SPIRV::Capability::MemoryAccessAliasingINTEL);
1702 break;
1703 }
1704 case SPIRV::OpBitReverse:
1705 case SPIRV::OpBitFieldInsert:
1706 case SPIRV::OpBitFieldSExtract:
1707 case SPIRV::OpBitFieldUExtract:
1708 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions)) {
1709 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
1710 break;
1711 }
1712 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_bit_instructions);
1713 Reqs.addCapability(ToAdd: SPIRV::Capability::BitInstructions);
1714 break;
1715 case SPIRV::OpTypeRuntimeArray:
1716 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
1717 break;
1718 case SPIRV::OpTypeOpaque:
1719 case SPIRV::OpTypeEvent:
1720 Reqs.addCapability(ToAdd: SPIRV::Capability::Kernel);
1721 break;
1722 case SPIRV::OpTypePipe:
1723 case SPIRV::OpTypeReserveId:
1724 Reqs.addCapability(ToAdd: SPIRV::Capability::Pipes);
1725 break;
1726 case SPIRV::OpTypeDeviceEvent:
1727 case SPIRV::OpTypeQueue:
1728 case SPIRV::OpBuildNDRange:
1729 case SPIRV::OpEnqueueKernel:
1730 Reqs.addCapability(ToAdd: SPIRV::Capability::DeviceEnqueue);
1731 break;
1732 case SPIRV::OpDecorate:
1733 case SPIRV::OpDecorateId:
1734 case SPIRV::OpDecorateString:
1735 addOpDecorateReqs(MI, DecIndex: 1, Reqs, ST);
1736 break;
1737 case SPIRV::OpMemberDecorate:
1738 case SPIRV::OpMemberDecorateString:
1739 addOpDecorateReqs(MI, DecIndex: 2, Reqs, ST);
1740 break;
1741 case SPIRV::OpInBoundsPtrAccessChain:
1742 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
1743 break;
1744 case SPIRV::OpConstantSampler:
1745 Reqs.addCapability(ToAdd: SPIRV::Capability::LiteralSampler);
1746 break;
1747 case SPIRV::OpInBoundsAccessChain:
1748 case SPIRV::OpAccessChain:
1749 addOpAccessChainReqs(Instr: MI, Handler&: Reqs, Subtarget: ST);
1750 break;
1751 case SPIRV::OpTypeImage:
1752 addOpTypeImageReqs(MI, Reqs, ST);
1753 break;
1754 case SPIRV::OpTypeSampler:
1755 if (!ST.isShader()) {
1756 Reqs.addCapability(ToAdd: SPIRV::Capability::ImageBasic);
1757 }
1758 break;
1759 case SPIRV::OpTypeForwardPointer:
1760 // TODO: check if it's OpenCL's kernel.
1761 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
1762 break;
1763 case SPIRV::OpAtomicFlagTestAndSet:
1764 case SPIRV::OpAtomicLoad:
1765 case SPIRV::OpAtomicStore:
1766 case SPIRV::OpAtomicExchange:
1767 case SPIRV::OpAtomicCompareExchange:
1768 case SPIRV::OpAtomicCompareExchangeWeak:
1769 case SPIRV::OpAtomicIIncrement:
1770 case SPIRV::OpAtomicIDecrement:
1771 case SPIRV::OpAtomicIAdd:
1772 case SPIRV::OpAtomicISub:
1773 case SPIRV::OpAtomicUMin:
1774 case SPIRV::OpAtomicUMax:
1775 case SPIRV::OpAtomicSMin:
1776 case SPIRV::OpAtomicSMax:
1777 case SPIRV::OpAtomicAnd:
1778 case SPIRV::OpAtomicOr:
1779 case SPIRV::OpAtomicXor: {
1780 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1781 const MachineInstr *InstrPtr = &MI;
1782 if (Op == SPIRV::OpAtomicStore) {
1783 assert(MI.getOperand(3).isReg());
1784 InstrPtr = MRI.getVRegDef(Reg: MI.getOperand(i: 3).getReg());
1785 assert(InstrPtr && "Unexpected type instruction for OpAtomicStore");
1786 }
1787 assert(InstrPtr->getOperand(1).isReg() && "Unexpected operand in atomic");
1788 Register TypeReg = InstrPtr->getOperand(i: 1).getReg();
1789 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: TypeReg);
1790
1791 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {
1792 unsigned BitWidth = TypeDef->getOperand(i: 1).getImm();
1793 if (BitWidth == 64)
1794 Reqs.addCapability(ToAdd: SPIRV::Capability::Int64Atomics);
1795 else if (BitWidth == 16) {
1796 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1797 report_fatal_error(
1798 reason: "16-bit integer atomic operations require the following SPIR-V "
1799 "extension: SPV_INTEL_16bit_atomics",
1800 gen_crash_diag: false);
1801 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1802 switch (Op) {
1803 case SPIRV::OpAtomicLoad:
1804 case SPIRV::OpAtomicStore:
1805 case SPIRV::OpAtomicExchange:
1806 case SPIRV::OpAtomicCompareExchange:
1807 case SPIRV::OpAtomicCompareExchangeWeak:
1808 Reqs.addCapability(
1809 ToAdd: SPIRV::Capability::AtomicInt16CompareExchangeINTEL);
1810 break;
1811 default:
1812 Reqs.addCapability(ToAdd: SPIRV::Capability::Int16AtomicsINTEL);
1813 break;
1814 }
1815 }
1816 } else if (isBFloat16Type(TypeDef)) {
1817 if (is_contained(Set: {SPIRV::OpAtomicLoad, SPIRV::OpAtomicStore,
1818 SPIRV::OpAtomicExchange},
1819 Element: Op)) {
1820 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1821 report_fatal_error(
1822 reason: "The atomic bfloat16 instruction requires the following SPIR-V "
1823 "extension: SPV_INTEL_16bit_atomics",
1824 gen_crash_diag: false);
1825 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1826 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicBFloat16LoadStoreINTEL);
1827 }
1828 }
1829 break;
1830 }
1831 case SPIRV::OpGroupNonUniformIAdd:
1832 case SPIRV::OpGroupNonUniformFAdd:
1833 case SPIRV::OpGroupNonUniformIMul:
1834 case SPIRV::OpGroupNonUniformFMul:
1835 case SPIRV::OpGroupNonUniformSMin:
1836 case SPIRV::OpGroupNonUniformUMin:
1837 case SPIRV::OpGroupNonUniformFMin:
1838 case SPIRV::OpGroupNonUniformSMax:
1839 case SPIRV::OpGroupNonUniformUMax:
1840 case SPIRV::OpGroupNonUniformFMax:
1841 case SPIRV::OpGroupNonUniformBitwiseAnd:
1842 case SPIRV::OpGroupNonUniformBitwiseOr:
1843 case SPIRV::OpGroupNonUniformBitwiseXor:
1844 case SPIRV::OpGroupNonUniformLogicalAnd:
1845 case SPIRV::OpGroupNonUniformLogicalOr:
1846 case SPIRV::OpGroupNonUniformLogicalXor: {
1847 assert(MI.getOperand(3).isImm());
1848 int64_t GroupOp = MI.getOperand(i: 3).getImm();
1849 switch (GroupOp) {
1850 case SPIRV::GroupOperation::Reduce:
1851 case SPIRV::GroupOperation::InclusiveScan:
1852 case SPIRV::GroupOperation::ExclusiveScan:
1853 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformArithmetic);
1854 break;
1855 case SPIRV::GroupOperation::ClusteredReduce:
1856 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformClustered);
1857 break;
1858 case SPIRV::GroupOperation::PartitionedReduceNV:
1859 case SPIRV::GroupOperation::PartitionedInclusiveScanNV:
1860 case SPIRV::GroupOperation::PartitionedExclusiveScanNV:
1861 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformPartitionedNV);
1862 break;
1863 }
1864 break;
1865 }
1866 case SPIRV::OpGroupNonUniformQuadSwap:
1867 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformQuad);
1868 break;
1869 case SPIRV::OpImageQueryLod:
1870 Reqs.addCapability(ToAdd: SPIRV::Capability::ImageQuery);
1871 break;
1872 case SPIRV::OpImageQuerySize:
1873 case SPIRV::OpImageQuerySizeLod:
1874 case SPIRV::OpImageQueryLevels:
1875 case SPIRV::OpImageQuerySamples:
1876 if (ST.isShader())
1877 Reqs.addCapability(ToAdd: SPIRV::Capability::ImageQuery);
1878 break;
1879 case SPIRV::OpImageQueryFormat: {
1880 Register ResultReg = MI.getOperand(i: 0).getReg();
1881 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1882 static const unsigned CompareOps[] = {
1883 SPIRV::OpIEqual, SPIRV::OpINotEqual,
1884 SPIRV::OpUGreaterThan, SPIRV::OpUGreaterThanEqual,
1885 SPIRV::OpULessThan, SPIRV::OpULessThanEqual,
1886 SPIRV::OpSGreaterThan, SPIRV::OpSGreaterThanEqual,
1887 SPIRV::OpSLessThan, SPIRV::OpSLessThanEqual};
1888
1889 auto CheckAndAddExtension = [&](int64_t ImmVal) {
1890 if (ImmVal == 4323 || ImmVal == 4324) {
1891 if (ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_image_raw10_raw12))
1892 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_image_raw10_raw12);
1893 else
1894 report_fatal_error(reason: "This requires the "
1895 "SPV_EXT_image_raw10_raw12 extension");
1896 }
1897 };
1898
1899 for (MachineInstr &UseInst : MRI.use_instructions(Reg: ResultReg)) {
1900 unsigned Opc = UseInst.getOpcode();
1901
1902 if (Opc == SPIRV::OpSwitch) {
1903 for (const MachineOperand &Op : UseInst.operands())
1904 if (Op.isImm())
1905 CheckAndAddExtension(Op.getImm());
1906 } else if (llvm::is_contained(Range: CompareOps, Element: Opc)) {
1907 for (unsigned i = 1; i < UseInst.getNumOperands(); ++i) {
1908 Register UseReg = UseInst.getOperand(i).getReg();
1909 MachineInstr *ConstInst = MRI.getVRegDef(Reg: UseReg);
1910 if (ConstInst && ConstInst->getOpcode() == SPIRV::OpConstantI) {
1911 int64_t ImmVal = ConstInst->getOperand(i: 2).getImm();
1912 if (ImmVal)
1913 CheckAndAddExtension(ImmVal);
1914 }
1915 }
1916 }
1917 }
1918 break;
1919 }
1920
1921 case SPIRV::OpGroupNonUniformShuffle:
1922 case SPIRV::OpGroupNonUniformShuffleXor:
1923 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformShuffle);
1924 break;
1925 case SPIRV::OpGroupNonUniformShuffleUp:
1926 case SPIRV::OpGroupNonUniformShuffleDown:
1927 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformShuffleRelative);
1928 break;
1929 case SPIRV::OpGroupAll:
1930 case SPIRV::OpGroupAny:
1931 case SPIRV::OpGroupBroadcast:
1932 case SPIRV::OpGroupIAdd:
1933 case SPIRV::OpGroupFAdd:
1934 case SPIRV::OpGroupFMin:
1935 case SPIRV::OpGroupUMin:
1936 case SPIRV::OpGroupSMin:
1937 case SPIRV::OpGroupFMax:
1938 case SPIRV::OpGroupUMax:
1939 case SPIRV::OpGroupSMax:
1940 Reqs.addCapability(ToAdd: SPIRV::Capability::Groups);
1941 break;
1942 case SPIRV::OpGroupNonUniformElect:
1943 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniform);
1944 break;
1945 case SPIRV::OpGroupNonUniformAll:
1946 case SPIRV::OpGroupNonUniformAny:
1947 case SPIRV::OpGroupNonUniformAllEqual:
1948 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformVote);
1949 break;
1950 case SPIRV::OpGroupNonUniformBroadcast:
1951 case SPIRV::OpGroupNonUniformBroadcastFirst:
1952 case SPIRV::OpGroupNonUniformBallot:
1953 case SPIRV::OpGroupNonUniformInverseBallot:
1954 case SPIRV::OpGroupNonUniformBallotBitExtract:
1955 case SPIRV::OpGroupNonUniformBallotBitCount:
1956 case SPIRV::OpGroupNonUniformBallotFindLSB:
1957 case SPIRV::OpGroupNonUniformBallotFindMSB:
1958 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformBallot);
1959 break;
1960 case SPIRV::OpSubgroupShuffleINTEL:
1961 case SPIRV::OpSubgroupShuffleDownINTEL:
1962 case SPIRV::OpSubgroupShuffleUpINTEL:
1963 case SPIRV::OpSubgroupShuffleXorINTEL:
1964 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
1965 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
1966 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupShuffleINTEL);
1967 }
1968 break;
1969 case SPIRV::OpSubgroupBlockReadINTEL:
1970 case SPIRV::OpSubgroupBlockWriteINTEL:
1971 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
1972 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
1973 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupBufferBlockIOINTEL);
1974 }
1975 break;
1976 case SPIRV::OpSubgroupImageBlockReadINTEL:
1977 case SPIRV::OpSubgroupImageBlockWriteINTEL:
1978 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
1979 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
1980 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupImageBlockIOINTEL);
1981 }
1982 break;
1983 case SPIRV::OpSubgroupImageMediaBlockReadINTEL:
1984 case SPIRV::OpSubgroupImageMediaBlockWriteINTEL:
1985 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_media_block_io)) {
1986 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_media_block_io);
1987 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupImageMediaBlockIOINTEL);
1988 }
1989 break;
1990 case SPIRV::OpAssumeTrueKHR:
1991 case SPIRV::OpExpectKHR:
1992 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume)) {
1993 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_expect_assume);
1994 Reqs.addCapability(ToAdd: SPIRV::Capability::ExpectAssumeKHR);
1995 }
1996 break;
1997 case SPIRV::OpFmaKHR:
1998 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_fma)) {
1999 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_fma);
2000 Reqs.addCapability(ToAdd: SPIRV::Capability::FmaKHR);
2001 }
2002 break;
2003 case SPIRV::OpPtrCastToCrossWorkgroupINTEL:
2004 case SPIRV::OpCrossWorkgroupCastToPtrINTEL:
2005 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)) {
2006 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_usm_storage_classes);
2007 Reqs.addCapability(ToAdd: SPIRV::Capability::USMStorageClassesINTEL);
2008 }
2009 break;
2010 case SPIRV::OpConstantFunctionPointerINTEL:
2011 case SPIRV::OpFunctionPointerCallINTEL:
2012 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers)) {
2013 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_function_pointers);
2014 Reqs.addCapability(ToAdd: SPIRV::Capability::FunctionPointersINTEL);
2015 }
2016 break;
2017 case SPIRV::OpGroupNonUniformRotateKHR:
2018 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_subgroup_rotate))
2019 report_fatal_error(reason: "OpGroupNonUniformRotateKHR instruction requires the "
2020 "following SPIR-V extension: SPV_KHR_subgroup_rotate",
2021 gen_crash_diag: false);
2022 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_subgroup_rotate);
2023 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformRotateKHR);
2024 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniform);
2025 break;
2026 case SPIRV::OpFixedCosALTERA:
2027 case SPIRV::OpFixedSinALTERA:
2028 case SPIRV::OpFixedCosPiALTERA:
2029 case SPIRV::OpFixedSinPiALTERA:
2030 case SPIRV::OpFixedExpALTERA:
2031 case SPIRV::OpFixedLogALTERA:
2032 case SPIRV::OpFixedRecipALTERA:
2033 case SPIRV::OpFixedSqrtALTERA:
2034 case SPIRV::OpFixedSinCosALTERA:
2035 case SPIRV::OpFixedSinCosPiALTERA:
2036 case SPIRV::OpFixedRsqrtALTERA:
2037 if (!ST.canUseExtension(
2038 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point))
2039 report_fatal_error(reason: "This instruction requires the "
2040 "following SPIR-V extension: "
2041 "SPV_ALTERA_arbitrary_precision_fixed_point",
2042 gen_crash_diag: false);
2043 Reqs.addExtension(
2044 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point);
2045 Reqs.addCapability(ToAdd: SPIRV::Capability::ArbitraryPrecisionFixedPointALTERA);
2046 break;
2047 case SPIRV::OpGroupIMulKHR:
2048 case SPIRV::OpGroupFMulKHR:
2049 case SPIRV::OpGroupBitwiseAndKHR:
2050 case SPIRV::OpGroupBitwiseOrKHR:
2051 case SPIRV::OpGroupBitwiseXorKHR:
2052 case SPIRV::OpGroupLogicalAndKHR:
2053 case SPIRV::OpGroupLogicalOrKHR:
2054 case SPIRV::OpGroupLogicalXorKHR:
2055 if (ST.canUseExtension(
2056 E: SPIRV::Extension::SPV_KHR_uniform_group_instructions)) {
2057 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_uniform_group_instructions);
2058 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupUniformArithmeticKHR);
2059 }
2060 break;
2061 case SPIRV::OpReadClockKHR:
2062 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_shader_clock))
2063 report_fatal_error(reason: "OpReadClockKHR instruction requires the "
2064 "following SPIR-V extension: SPV_KHR_shader_clock",
2065 gen_crash_diag: false);
2066 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_shader_clock);
2067 Reqs.addCapability(ToAdd: SPIRV::Capability::ShaderClockKHR);
2068 break;
2069 case SPIRV::OpAbortKHR:
2070 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_abort))
2071 report_fatal_error(reason: "OpAbortKHR instruction requires the "
2072 "following SPIR-V extension: SPV_KHR_abort",
2073 gen_crash_diag: false);
2074 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_abort);
2075 Reqs.addCapability(ToAdd: SPIRV::Capability::AbortKHR);
2076 break;
2077 case SPIRV::OpPoisonKHR:
2078 case SPIRV::OpFreezeKHR:
2079 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze))
2080 report_fatal_error(reason: "OpPoisonKHR/OpFreezeKHR instruction requires the "
2081 "following SPIR-V extension: SPV_KHR_poison_freeze",
2082 gen_crash_diag: false);
2083 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_poison_freeze);
2084 Reqs.addCapability(ToAdd: SPIRV::Capability::PoisonFreezeKHR);
2085 break;
2086 case SPIRV::OpAtomicFAddEXT:
2087 case SPIRV::OpAtomicFMinEXT:
2088 case SPIRV::OpAtomicFMaxEXT:
2089 AddAtomicFloatRequirements(MI, Reqs, ST);
2090 break;
2091 case SPIRV::OpConvertBF16ToFINTEL:
2092 case SPIRV::OpConvertFToBF16INTEL:
2093 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bfloat16_conversion)) {
2094 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_conversion);
2095 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ConversionINTEL);
2096 }
2097 break;
2098 case SPIRV::OpRoundFToTF32INTEL:
2099 if (ST.canUseExtension(
2100 E: SPIRV::Extension::SPV_INTEL_tensor_float32_conversion)) {
2101 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_tensor_float32_conversion);
2102 Reqs.addCapability(ToAdd: SPIRV::Capability::TensorFloat32RoundingINTEL);
2103 }
2104 break;
2105 case SPIRV::OpVariableLengthArrayINTEL:
2106 case SPIRV::OpSaveMemoryINTEL:
2107 case SPIRV::OpRestoreMemoryINTEL:
2108 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array)) {
2109 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_variable_length_array);
2110 Reqs.addCapability(ToAdd: SPIRV::Capability::VariableLengthArrayINTEL);
2111 }
2112 break;
2113 case SPIRV::OpAsmTargetINTEL:
2114 case SPIRV::OpAsmINTEL:
2115 case SPIRV::OpAsmCallINTEL:
2116 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_inline_assembly)) {
2117 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_inline_assembly);
2118 Reqs.addCapability(ToAdd: SPIRV::Capability::AsmINTEL);
2119 }
2120 break;
2121 case SPIRV::OpTypeCooperativeMatrixKHR: {
2122 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
2123 report_fatal_error(
2124 reason: "OpTypeCooperativeMatrixKHR type requires the "
2125 "following SPIR-V extension: SPV_KHR_cooperative_matrix",
2126 gen_crash_diag: false);
2127 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
2128 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
2129 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2130 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
2131 if (isBFloat16Type(TypeDef))
2132 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16CooperativeMatrixKHR);
2133 break;
2134 }
2135 case SPIRV::OpArithmeticFenceEXT:
2136 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_arithmetic_fence))
2137 report_fatal_error(reason: "OpArithmeticFenceEXT requires the "
2138 "following SPIR-V extension: SPV_EXT_arithmetic_fence",
2139 gen_crash_diag: false);
2140 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_arithmetic_fence);
2141 Reqs.addCapability(ToAdd: SPIRV::Capability::ArithmeticFenceEXT);
2142 break;
2143 case SPIRV::OpControlBarrierArriveINTEL:
2144 case SPIRV::OpControlBarrierWaitINTEL:
2145 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_split_barrier)) {
2146 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_split_barrier);
2147 Reqs.addCapability(ToAdd: SPIRV::Capability::SplitBarrierINTEL);
2148 }
2149 break;
2150 case SPIRV::OpCooperativeMatrixMulAddKHR: {
2151 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
2152 report_fatal_error(reason: "Cooperative matrix instructions require the "
2153 "following SPIR-V extension: "
2154 "SPV_KHR_cooperative_matrix",
2155 gen_crash_diag: false);
2156 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
2157 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
2158 constexpr unsigned MulAddMaxSize = 6;
2159 if (MI.getNumOperands() != MulAddMaxSize)
2160 break;
2161 const int64_t CoopOperands = MI.getOperand(i: MulAddMaxSize - 1).getImm();
2162 if (CoopOperands &
2163 SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) {
2164 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2165 report_fatal_error(reason: "MatrixAAndBTF32ComponentsINTEL type interpretation "
2166 "require the following SPIR-V extension: "
2167 "SPV_INTEL_joint_matrix",
2168 gen_crash_diag: false);
2169 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2170 Reqs.addCapability(
2171 ToAdd: SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL);
2172 }
2173 if (CoopOperands & SPIRV::CooperativeMatrixOperands::
2174 MatrixAAndBBFloat16ComponentsINTEL ||
2175 CoopOperands &
2176 SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL ||
2177 CoopOperands & SPIRV::CooperativeMatrixOperands::
2178 MatrixResultBFloat16ComponentsINTEL) {
2179 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2180 report_fatal_error(reason: "***BF16ComponentsINTEL type interpretations "
2181 "require the following SPIR-V extension: "
2182 "SPV_INTEL_joint_matrix",
2183 gen_crash_diag: false);
2184 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2185 Reqs.addCapability(
2186 ToAdd: SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);
2187 }
2188 break;
2189 }
2190 case SPIRV::OpCooperativeMatrixLoadKHR:
2191 case SPIRV::OpCooperativeMatrixStoreKHR:
2192 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2193 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2194 case SPIRV::OpCooperativeMatrixPrefetchINTEL: {
2195 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
2196 report_fatal_error(reason: "Cooperative matrix instructions require the "
2197 "following SPIR-V extension: "
2198 "SPV_KHR_cooperative_matrix",
2199 gen_crash_diag: false);
2200 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
2201 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
2202
2203 // Check Layout operand in case if it's not a standard one and add the
2204 // appropriate capability.
2205 unsigned LayoutNum;
2206 switch (Op) {
2207 case SPIRV::OpCooperativeMatrixLoadKHR:
2208 LayoutNum = 3;
2209 break;
2210 case SPIRV::OpCooperativeMatrixStoreKHR:
2211 LayoutNum = 2;
2212 break;
2213 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2214 LayoutNum = 5;
2215 break;
2216 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2217 case SPIRV::OpCooperativeMatrixPrefetchINTEL:
2218 LayoutNum = 4;
2219 break;
2220 default:
2221 llvm_unreachable("unexpected cooperative matrix opcode");
2222 }
2223 Register RegLayout = MI.getOperand(i: LayoutNum).getReg();
2224 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2225 MachineInstr *MILayout = MRI.getUniqueVRegDef(Reg: RegLayout);
2226 if (MILayout->getOpcode() == SPIRV::OpConstantI) {
2227 const unsigned LayoutVal = MILayout->getOperand(i: 2).getImm();
2228 if (LayoutVal ==
2229 static_cast<unsigned>(SPIRV::CooperativeMatrixLayout::PackedINTEL)) {
2230 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2231 report_fatal_error(reason: "PackedINTEL layout require the following SPIR-V "
2232 "extension: SPV_INTEL_joint_matrix",
2233 gen_crash_diag: false);
2234 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2235 Reqs.addCapability(ToAdd: SPIRV::Capability::PackedCooperativeMatrixINTEL);
2236 }
2237 }
2238
2239 // Nothing to do.
2240 if (Op == SPIRV::OpCooperativeMatrixLoadKHR ||
2241 Op == SPIRV::OpCooperativeMatrixStoreKHR)
2242 break;
2243
2244 std::string InstName;
2245 switch (Op) {
2246 case SPIRV::OpCooperativeMatrixPrefetchINTEL:
2247 InstName = "OpCooperativeMatrixPrefetchINTEL";
2248 break;
2249 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2250 InstName = "OpCooperativeMatrixLoadCheckedINTEL";
2251 break;
2252 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2253 InstName = "OpCooperativeMatrixStoreCheckedINTEL";
2254 break;
2255 }
2256
2257 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix)) {
2258 const std::string ErrorMsg =
2259 InstName + " instruction requires the "
2260 "following SPIR-V extension: SPV_INTEL_joint_matrix";
2261 report_fatal_error(reason: ErrorMsg.c_str(), gen_crash_diag: false);
2262 }
2263 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2264 if (Op == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
2265 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixPrefetchINTEL);
2266 break;
2267 }
2268 Reqs.addCapability(
2269 ToAdd: SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
2270 break;
2271 }
2272 case SPIRV::OpCooperativeMatrixConstructCheckedINTEL:
2273 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2274 report_fatal_error(reason: "OpCooperativeMatrixConstructCheckedINTEL "
2275 "instructions require the following SPIR-V extension: "
2276 "SPV_INTEL_joint_matrix",
2277 gen_crash_diag: false);
2278 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2279 Reqs.addCapability(
2280 ToAdd: SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
2281 break;
2282 case SPIRV::OpReadPipeBlockingALTERA:
2283 case SPIRV::OpWritePipeBlockingALTERA:
2284 if (ST.canUseExtension(E: SPIRV::Extension::SPV_ALTERA_blocking_pipes)) {
2285 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_ALTERA_blocking_pipes);
2286 Reqs.addCapability(ToAdd: SPIRV::Capability::BlockingPipesALTERA);
2287 }
2288 break;
2289 case SPIRV::OpCooperativeMatrixGetElementCoordINTEL:
2290 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2291 report_fatal_error(reason: "OpCooperativeMatrixGetElementCoordINTEL requires the "
2292 "following SPIR-V extension: SPV_INTEL_joint_matrix",
2293 gen_crash_diag: false);
2294 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2295 Reqs.addCapability(
2296 ToAdd: SPIRV::Capability::CooperativeMatrixInvocationInstructionsINTEL);
2297 break;
2298 case SPIRV::OpConvertHandleToImageINTEL:
2299 case SPIRV::OpConvertHandleToSamplerINTEL:
2300 case SPIRV::OpConvertHandleToSampledImageINTEL: {
2301 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bindless_images))
2302 report_fatal_error(reason: "OpConvertHandleTo[Image/Sampler/SampledImage]INTEL "
2303 "instructions require the following SPIR-V extension: "
2304 "SPV_INTEL_bindless_images",
2305 gen_crash_diag: false);
2306 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
2307 SPIRV::AddressingModel::AddressingModel AddrModel = MAI.Addr;
2308 SPIRVTypeInst TyDef = GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg());
2309 if (Op == SPIRV::OpConvertHandleToImageINTEL &&
2310 TyDef->getOpcode() != SPIRV::OpTypeImage) {
2311 report_fatal_error(reason: "Incorrect return type for the instruction "
2312 "OpConvertHandleToImageINTEL",
2313 gen_crash_diag: false);
2314 } else if (Op == SPIRV::OpConvertHandleToSamplerINTEL &&
2315 TyDef->getOpcode() != SPIRV::OpTypeSampler) {
2316 report_fatal_error(reason: "Incorrect return type for the instruction "
2317 "OpConvertHandleToSamplerINTEL",
2318 gen_crash_diag: false);
2319 } else if (Op == SPIRV::OpConvertHandleToSampledImageINTEL &&
2320 TyDef->getOpcode() != SPIRV::OpTypeSampledImage) {
2321 report_fatal_error(reason: "Incorrect return type for the instruction "
2322 "OpConvertHandleToSampledImageINTEL",
2323 gen_crash_diag: false);
2324 }
2325 SPIRVTypeInst SpvTy = GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 2).getReg());
2326 unsigned Bitwidth = GR->getScalarOrVectorBitWidth(Type: SpvTy);
2327 if (!(Bitwidth == 32 && AddrModel == SPIRV::AddressingModel::Physical32) &&
2328 !(Bitwidth == 64 && AddrModel == SPIRV::AddressingModel::Physical64)) {
2329 report_fatal_error(
2330 reason: "Parameter value must be a 32-bit scalar in case of "
2331 "Physical32 addressing model or a 64-bit scalar in case of "
2332 "Physical64 addressing model",
2333 gen_crash_diag: false);
2334 }
2335 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bindless_images);
2336 Reqs.addCapability(ToAdd: SPIRV::Capability::BindlessImagesINTEL);
2337 break;
2338 }
2339 case SPIRV::OpSubgroup2DBlockLoadINTEL:
2340 case SPIRV::OpSubgroup2DBlockLoadTransposeINTEL:
2341 case SPIRV::OpSubgroup2DBlockLoadTransformINTEL:
2342 case SPIRV::OpSubgroup2DBlockPrefetchINTEL:
2343 case SPIRV::OpSubgroup2DBlockStoreINTEL: {
2344 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_2d_block_io))
2345 report_fatal_error(reason: "OpSubgroup2DBlock[Load/LoadTranspose/LoadTransform/"
2346 "Prefetch/Store]INTEL instructions require the "
2347 "following SPIR-V extension: SPV_INTEL_2d_block_io",
2348 gen_crash_diag: false);
2349 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_2d_block_io);
2350 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockIOINTEL);
2351
2352 if (Op == SPIRV::OpSubgroup2DBlockLoadTransposeINTEL) {
2353 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockTransposeINTEL);
2354 break;
2355 }
2356 if (Op == SPIRV::OpSubgroup2DBlockLoadTransformINTEL) {
2357 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockTransformINTEL);
2358 break;
2359 }
2360 break;
2361 }
2362 case SPIRV::OpKill: {
2363 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2364 } break;
2365 case SPIRV::OpDemoteToHelperInvocation:
2366 Reqs.addCapability(ToAdd: SPIRV::Capability::DemoteToHelperInvocation);
2367
2368 if (ST.canUseExtension(
2369 E: SPIRV::Extension::SPV_EXT_demote_to_helper_invocation)) {
2370 if (!ST.isAtLeastSPIRVVer(VerToCompareTo: llvm::VersionTuple(1, 6)))
2371 Reqs.addExtension(
2372 ToAdd: SPIRV::Extension::SPV_EXT_demote_to_helper_invocation);
2373 }
2374 break;
2375 case SPIRV::OpSDot:
2376 case SPIRV::OpUDot:
2377 case SPIRV::OpSUDot:
2378 case SPIRV::OpSDotAccSat:
2379 case SPIRV::OpUDotAccSat:
2380 case SPIRV::OpSUDotAccSat:
2381 AddDotProductRequirements(MI, Reqs, ST);
2382 break;
2383 case SPIRV::OpImageSampleImplicitLod:
2384 case SPIRV::OpImageFetch:
2385 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2386 addImageOperandReqs(MI, Reqs, ST, OpIdx: 4);
2387 break;
2388 case SPIRV::OpImageSampleExplicitLod:
2389 addImageOperandReqs(MI, Reqs, ST, OpIdx: 4);
2390 break;
2391 case SPIRV::OpImageSampleDrefImplicitLod:
2392 case SPIRV::OpImageSampleDrefExplicitLod:
2393 case SPIRV::OpImageDrefGather:
2394 case SPIRV::OpImageGather:
2395 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2396 addImageOperandReqs(MI, Reqs, ST, OpIdx: 5);
2397 break;
2398 case SPIRV::OpImageRead: {
2399 Register ImageReg = MI.getOperand(i: 2).getReg();
2400 SPIRVTypeInst TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(
2401 VReg: ImageReg, MF: const_cast<MachineFunction *>(MI.getMF()));
2402 // OpImageRead and OpImageWrite can use Unknown Image Formats
2403 // when the Kernel capability is declared. In the OpenCL environment we are
2404 // not allowed to produce
2405 // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see
2406 // https://github.com/KhronosGroup/SPIRV-Headers/issues/487
2407
2408 if (isImageTypeWithUnknownFormat(TypeInst: TypeDef) && ST.isShader())
2409 Reqs.addCapability(ToAdd: SPIRV::Capability::StorageImageReadWithoutFormat);
2410 break;
2411 }
2412 case SPIRV::OpImageWrite: {
2413 Register ImageReg = MI.getOperand(i: 0).getReg();
2414 SPIRVTypeInst TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(
2415 VReg: ImageReg, MF: const_cast<MachineFunction *>(MI.getMF()));
2416 // OpImageRead and OpImageWrite can use Unknown Image Formats
2417 // when the Kernel capability is declared. In the OpenCL environment we are
2418 // not allowed to produce
2419 // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see
2420 // https://github.com/KhronosGroup/SPIRV-Headers/issues/487
2421
2422 if (isImageTypeWithUnknownFormat(TypeInst: TypeDef) && ST.isShader())
2423 Reqs.addCapability(ToAdd: SPIRV::Capability::StorageImageWriteWithoutFormat);
2424 break;
2425 }
2426 case SPIRV::OpTypeStructContinuedINTEL:
2427 case SPIRV::OpConstantCompositeContinuedINTEL:
2428 case SPIRV::OpSpecConstantCompositeContinuedINTEL:
2429 case SPIRV::OpCompositeConstructContinuedINTEL: {
2430 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_long_composites))
2431 report_fatal_error(
2432 reason: "Continued instructions require the "
2433 "following SPIR-V extension: SPV_INTEL_long_composites",
2434 gen_crash_diag: false);
2435 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_long_composites);
2436 Reqs.addCapability(ToAdd: SPIRV::Capability::LongCompositesINTEL);
2437 break;
2438 }
2439 case SPIRV::OpArbitraryFloatEQALTERA:
2440 case SPIRV::OpArbitraryFloatGEALTERA:
2441 case SPIRV::OpArbitraryFloatGTALTERA:
2442 case SPIRV::OpArbitraryFloatLEALTERA:
2443 case SPIRV::OpArbitraryFloatLTALTERA:
2444 case SPIRV::OpArbitraryFloatCbrtALTERA:
2445 case SPIRV::OpArbitraryFloatCosALTERA:
2446 case SPIRV::OpArbitraryFloatCosPiALTERA:
2447 case SPIRV::OpArbitraryFloatExp10ALTERA:
2448 case SPIRV::OpArbitraryFloatExp2ALTERA:
2449 case SPIRV::OpArbitraryFloatExpALTERA:
2450 case SPIRV::OpArbitraryFloatExpm1ALTERA:
2451 case SPIRV::OpArbitraryFloatHypotALTERA:
2452 case SPIRV::OpArbitraryFloatLog10ALTERA:
2453 case SPIRV::OpArbitraryFloatLog1pALTERA:
2454 case SPIRV::OpArbitraryFloatLog2ALTERA:
2455 case SPIRV::OpArbitraryFloatLogALTERA:
2456 case SPIRV::OpArbitraryFloatRecipALTERA:
2457 case SPIRV::OpArbitraryFloatSinCosALTERA:
2458 case SPIRV::OpArbitraryFloatSinCosPiALTERA:
2459 case SPIRV::OpArbitraryFloatSinALTERA:
2460 case SPIRV::OpArbitraryFloatSinPiALTERA:
2461 case SPIRV::OpArbitraryFloatSqrtALTERA:
2462 case SPIRV::OpArbitraryFloatACosALTERA:
2463 case SPIRV::OpArbitraryFloatACosPiALTERA:
2464 case SPIRV::OpArbitraryFloatAddALTERA:
2465 case SPIRV::OpArbitraryFloatASinALTERA:
2466 case SPIRV::OpArbitraryFloatASinPiALTERA:
2467 case SPIRV::OpArbitraryFloatATan2ALTERA:
2468 case SPIRV::OpArbitraryFloatATanALTERA:
2469 case SPIRV::OpArbitraryFloatATanPiALTERA:
2470 case SPIRV::OpArbitraryFloatCastFromIntALTERA:
2471 case SPIRV::OpArbitraryFloatCastALTERA:
2472 case SPIRV::OpArbitraryFloatCastToIntALTERA:
2473 case SPIRV::OpArbitraryFloatDivALTERA:
2474 case SPIRV::OpArbitraryFloatMulALTERA:
2475 case SPIRV::OpArbitraryFloatPowALTERA:
2476 case SPIRV::OpArbitraryFloatPowNALTERA:
2477 case SPIRV::OpArbitraryFloatPowRALTERA:
2478 case SPIRV::OpArbitraryFloatRSqrtALTERA:
2479 case SPIRV::OpArbitraryFloatSubALTERA: {
2480 if (!ST.canUseExtension(
2481 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_floating_point))
2482 report_fatal_error(
2483 reason: "Floating point instructions can't be translated correctly without "
2484 "enabled SPV_ALTERA_arbitrary_precision_floating_point extension!",
2485 gen_crash_diag: false);
2486 Reqs.addExtension(
2487 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_floating_point);
2488 Reqs.addCapability(
2489 ToAdd: SPIRV::Capability::ArbitraryPrecisionFloatingPointALTERA);
2490 break;
2491 }
2492 case SPIRV::OpSubgroupMatrixMultiplyAccumulateINTEL: {
2493 if (!ST.canUseExtension(
2494 E: SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate))
2495 report_fatal_error(
2496 reason: "OpSubgroupMatrixMultiplyAccumulateINTEL instruction requires the "
2497 "following SPIR-V "
2498 "extension: SPV_INTEL_subgroup_matrix_multiply_accumulate",
2499 gen_crash_diag: false);
2500 Reqs.addExtension(
2501 ToAdd: SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate);
2502 Reqs.addCapability(
2503 ToAdd: SPIRV::Capability::SubgroupMatrixMultiplyAccumulateINTEL);
2504 break;
2505 }
2506 case SPIRV::OpBitwiseFunctionINTEL: {
2507 if (!ST.canUseExtension(
2508 E: SPIRV::Extension::SPV_INTEL_ternary_bitwise_function))
2509 report_fatal_error(
2510 reason: "OpBitwiseFunctionINTEL instruction requires the following SPIR-V "
2511 "extension: SPV_INTEL_ternary_bitwise_function",
2512 gen_crash_diag: false);
2513 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_ternary_bitwise_function);
2514 Reqs.addCapability(ToAdd: SPIRV::Capability::TernaryBitwiseFunctionINTEL);
2515 break;
2516 }
2517 case SPIRV::OpCopyMemorySized: {
2518 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
2519 // TODO: Add UntypedPointersKHR when implemented.
2520 break;
2521 }
2522 case SPIRV::OpPredicatedLoadINTEL:
2523 case SPIRV::OpPredicatedStoreINTEL: {
2524 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_predicated_io))
2525 report_fatal_error(
2526 reason: "OpPredicated[Load/Store]INTEL instructions require "
2527 "the following SPIR-V extension: SPV_INTEL_predicated_io",
2528 gen_crash_diag: false);
2529 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_predicated_io);
2530 Reqs.addCapability(ToAdd: SPIRV::Capability::PredicatedIOINTEL);
2531 break;
2532 }
2533 case SPIRV::OpFAddS:
2534 case SPIRV::OpFSubS:
2535 case SPIRV::OpFMulS:
2536 case SPIRV::OpFDivS:
2537 case SPIRV::OpFRemS:
2538 case SPIRV::OpFMod:
2539 case SPIRV::OpFNegate:
2540 case SPIRV::OpFAddV:
2541 case SPIRV::OpFSubV:
2542 case SPIRV::OpFMulV:
2543 case SPIRV::OpFDivV:
2544 case SPIRV::OpFRemV:
2545 case SPIRV::OpFNegateV: {
2546 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2547 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
2548 if (TypeDef->getOpcode() == SPIRV::OpTypeVector)
2549 TypeDef = MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
2550 if (isBFloat16Type(TypeDef)) {
2551 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic))
2552 report_fatal_error(
2553 reason: "Arithmetic instructions with bfloat16 arguments require the "
2554 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic",
2555 gen_crash_diag: false);
2556 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);
2557 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ArithmeticINTEL);
2558 }
2559 break;
2560 }
2561 case SPIRV::OpOrdered:
2562 case SPIRV::OpUnordered:
2563 case SPIRV::OpFOrdEqual:
2564 case SPIRV::OpFOrdNotEqual:
2565 case SPIRV::OpFOrdLessThan:
2566 case SPIRV::OpFOrdLessThanEqual:
2567 case SPIRV::OpFOrdGreaterThan:
2568 case SPIRV::OpFOrdGreaterThanEqual:
2569 case SPIRV::OpFUnordEqual:
2570 case SPIRV::OpFUnordNotEqual:
2571 case SPIRV::OpFUnordLessThan:
2572 case SPIRV::OpFUnordLessThanEqual:
2573 case SPIRV::OpFUnordGreaterThan:
2574 case SPIRV::OpFUnordGreaterThanEqual: {
2575 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2576 MachineInstr *OperandDef = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
2577 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: OperandDef->getOperand(i: 1).getReg());
2578 if (TypeDef->getOpcode() == SPIRV::OpTypeVector)
2579 TypeDef = MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
2580 if (isBFloat16Type(TypeDef)) {
2581 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic))
2582 report_fatal_error(
2583 reason: "Relational instructions with bfloat16 arguments require the "
2584 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic",
2585 gen_crash_diag: false);
2586 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);
2587 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ArithmeticINTEL);
2588 }
2589 break;
2590 }
2591 case SPIRV::OpDPdxCoarse:
2592 case SPIRV::OpDPdyCoarse:
2593 case SPIRV::OpDPdxFine:
2594 case SPIRV::OpDPdyFine: {
2595 Reqs.addCapability(ToAdd: SPIRV::Capability::DerivativeControl);
2596 break;
2597 }
2598 case SPIRV::OpLoopControlINTEL: {
2599 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls);
2600 Reqs.addCapability(ToAdd: SPIRV::Capability::UnstructuredLoopControlsINTEL);
2601 break;
2602 }
2603
2604 default:
2605 break;
2606 }
2607
2608 // If we require capability Shader, then we can remove the requirement for
2609 // the BitInstructions capability, since Shader is a superset capability
2610 // of BitInstructions.
2611 Reqs.removeCapabilityIf(ToRemove: SPIRV::Capability::BitInstructions,
2612 IfPresent: SPIRV::Capability::Shader);
2613}
2614
2615static void collectReqs(const Module &M, SPIRV::ModuleAnalysisInfo &MAI,
2616 MachineModuleInfo *MMI, const SPIRVSubtarget &ST) {
2617 // Collect requirements for existing instructions.
2618 for (const Function &F : M) {
2619 MachineFunction *MF = MMI->getMachineFunction(F);
2620 if (!MF)
2621 continue;
2622 for (const MachineBasicBlock &MBB : *MF)
2623 for (const MachineInstr &MI : MBB)
2624 addInstrRequirements(MI, MAI, ST);
2625 }
2626 // Collect requirements for OpExecutionMode instructions.
2627 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
2628 if (Node) {
2629 bool RequireFloatControls = false, RequireIntelFloatControls2 = false,
2630 RequireKHRFloatControls2 = false,
2631 VerLower14 = !ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 4));
2632 bool HasIntelFloatControls2 =
2633 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_float_controls2);
2634 bool HasKHRFloatControls2 =
2635 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2636 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
2637 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
2638 const MDOperand &MDOp = MDN->getOperand(I: 1);
2639 if (auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: MDOp)) {
2640 Constant *C = CMeta->getValue();
2641 if (ConstantInt *Const = dyn_cast<ConstantInt>(Val: C)) {
2642 auto EM = Const->getZExtValue();
2643 // SPV_KHR_float_controls is not available until v1.4:
2644 // add SPV_KHR_float_controls if the version is too low
2645 switch (EM) {
2646 case SPIRV::ExecutionMode::DenormPreserve:
2647 case SPIRV::ExecutionMode::DenormFlushToZero:
2648 case SPIRV::ExecutionMode::RoundingModeRTE:
2649 case SPIRV::ExecutionMode::RoundingModeRTZ:
2650 RequireFloatControls = VerLower14;
2651 MAI.Reqs.getAndAddRequirements(
2652 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2653 break;
2654 case SPIRV::ExecutionMode::RoundingModeRTPINTEL:
2655 case SPIRV::ExecutionMode::RoundingModeRTNINTEL:
2656 case SPIRV::ExecutionMode::FloatingPointModeALTINTEL:
2657 case SPIRV::ExecutionMode::FloatingPointModeIEEEINTEL:
2658 if (HasIntelFloatControls2) {
2659 RequireIntelFloatControls2 = true;
2660 MAI.Reqs.getAndAddRequirements(
2661 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2662 }
2663 break;
2664 case SPIRV::ExecutionMode::FPFastMathDefault: {
2665 if (HasKHRFloatControls2) {
2666 RequireKHRFloatControls2 = true;
2667 MAI.Reqs.getAndAddRequirements(
2668 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2669 }
2670 break;
2671 }
2672 case SPIRV::ExecutionMode::ContractionOff:
2673 case SPIRV::ExecutionMode::SignedZeroInfNanPreserve:
2674 if (HasKHRFloatControls2) {
2675 RequireKHRFloatControls2 = true;
2676 MAI.Reqs.getAndAddRequirements(
2677 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2678 i: SPIRV::ExecutionMode::FPFastMathDefault, ST);
2679 } else {
2680 MAI.Reqs.getAndAddRequirements(
2681 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2682 }
2683 break;
2684 default:
2685 MAI.Reqs.getAndAddRequirements(
2686 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2687 }
2688 }
2689 }
2690 }
2691 if (RequireFloatControls &&
2692 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls))
2693 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls);
2694 if (RequireIntelFloatControls2)
2695 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_float_controls2);
2696 if (RequireKHRFloatControls2)
2697 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls2);
2698 }
2699 for (const Function &F : M) {
2700 if (F.isDeclaration())
2701 continue;
2702 if (F.getMetadata(Kind: "reqd_work_group_size"))
2703 MAI.Reqs.getAndAddRequirements(
2704 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2705 i: SPIRV::ExecutionMode::LocalSize, ST);
2706 if (F.getFnAttribute(Kind: "hlsl.numthreads").isValid()) {
2707 MAI.Reqs.getAndAddRequirements(
2708 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2709 i: SPIRV::ExecutionMode::LocalSize, ST);
2710 }
2711 if (F.getFnAttribute(Kind: "enable-maximal-reconvergence").getValueAsBool()) {
2712 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_maximal_reconvergence);
2713 }
2714 if (F.getMetadata(Kind: "work_group_size_hint"))
2715 MAI.Reqs.getAndAddRequirements(
2716 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2717 i: SPIRV::ExecutionMode::LocalSizeHint, ST);
2718 if (F.getMetadata(Kind: "intel_reqd_sub_group_size") ||
2719 F.getMetadata(Kind: "reqd_sub_group_size"))
2720 MAI.Reqs.getAndAddRequirements(
2721 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2722 i: SPIRV::ExecutionMode::SubgroupSize, ST);
2723 if (F.getMetadata(Kind: "max_work_group_size"))
2724 MAI.Reqs.getAndAddRequirements(
2725 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2726 i: SPIRV::ExecutionMode::MaxWorkgroupSizeINTEL, ST);
2727 if (F.getMetadata(Kind: "vec_type_hint"))
2728 MAI.Reqs.getAndAddRequirements(
2729 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2730 i: SPIRV::ExecutionMode::VecTypeHint, ST);
2731
2732 if (F.hasOptNone()) {
2733 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_optnone)) {
2734 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_optnone);
2735 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::OptNoneINTEL);
2736 } else if (ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_optnone)) {
2737 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_optnone);
2738 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::OptNoneEXT);
2739 }
2740 }
2741 }
2742}
2743
2744static unsigned getFastMathFlags(const MachineInstr &I,
2745 const SPIRVSubtarget &ST) {
2746 unsigned Flags = SPIRV::FPFastMathMode::None;
2747 bool CanUseKHRFloatControls2 =
2748 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2749 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNoNans))
2750 Flags |= SPIRV::FPFastMathMode::NotNaN;
2751 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNoInfs))
2752 Flags |= SPIRV::FPFastMathMode::NotInf;
2753 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNsz))
2754 Flags |= SPIRV::FPFastMathMode::NSZ;
2755 if (I.getFlag(Flag: MachineInstr::MIFlag::FmArcp))
2756 Flags |= SPIRV::FPFastMathMode::AllowRecip;
2757 if (I.getFlag(Flag: MachineInstr::MIFlag::FmContract) && CanUseKHRFloatControls2)
2758 Flags |= SPIRV::FPFastMathMode::AllowContract;
2759 if (I.getFlag(Flag: MachineInstr::MIFlag::FmReassoc)) {
2760 if (CanUseKHRFloatControls2)
2761 // LLVM reassoc maps to SPIRV transform, see
2762 // https://github.com/KhronosGroup/SPIRV-Registry/issues/326 for details.
2763 // Because we are enabling AllowTransform, we must enable AllowReassoc and
2764 // AllowContract too, as required by SPIRV spec. Also, we used to map
2765 // MIFlag::FmReassoc to FPFastMathMode::Fast, which now should instead by
2766 // replaced by turning all the other bits instead. Therefore, we're
2767 // enabling every bit here except None and Fast.
2768 Flags |= SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
2769 SPIRV::FPFastMathMode::NSZ | SPIRV::FPFastMathMode::AllowRecip |
2770 SPIRV::FPFastMathMode::AllowTransform |
2771 SPIRV::FPFastMathMode::AllowReassoc |
2772 SPIRV::FPFastMathMode::AllowContract;
2773 else
2774 Flags |= SPIRV::FPFastMathMode::Fast;
2775 }
2776
2777 if (CanUseKHRFloatControls2) {
2778 // Error out if SPIRV::FPFastMathMode::Fast is enabled.
2779 assert(!(Flags & SPIRV::FPFastMathMode::Fast) &&
2780 "SPIRV::FPFastMathMode::Fast is deprecated and should not be used "
2781 "anymore.");
2782
2783 // Error out if AllowTransform is enabled without AllowReassoc and
2784 // AllowContract.
2785 assert((!(Flags & SPIRV::FPFastMathMode::AllowTransform) ||
2786 ((Flags & SPIRV::FPFastMathMode::AllowReassoc &&
2787 Flags & SPIRV::FPFastMathMode::AllowContract))) &&
2788 "SPIRV::FPFastMathMode::AllowTransform requires AllowReassoc and "
2789 "AllowContract flags to be enabled as well.");
2790 }
2791
2792 return Flags;
2793}
2794
2795static bool isFastMathModeAvailable(const SPIRVSubtarget &ST) {
2796 if (ST.isKernel())
2797 return true;
2798 if (ST.getSPIRVVersion() < VersionTuple(1, 2))
2799 return false;
2800 return ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2801}
2802
2803static void handleMIFlagDecoration(
2804 MachineInstr &I, const SPIRVSubtarget &ST, const SPIRVInstrInfo &TII,
2805 SPIRV::RequirementHandler &Reqs, const SPIRVGlobalRegistry *GR,
2806 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec) {
2807 if (TII.canUseIntegerWrapDecoration(MI: I)) {
2808 if (I.getFlag(Flag: MachineInstr::MIFlag::NoSWrap) &&
2809 getSymbolicOperandRequirements(
2810 Category: SPIRV::OperandCategory::DecorationOperand,
2811 i: SPIRV::Decoration::NoSignedWrap, ST, Reqs)
2812 .IsSatisfiable)
2813 buildOpDecorate(Reg: I.getOperand(i: 0).getReg(), I, TII,
2814 Dec: SPIRV::Decoration::NoSignedWrap, DecArgs: {});
2815 if (I.getFlag(Flag: MachineInstr::MIFlag::NoUWrap) &&
2816 getSymbolicOperandRequirements(
2817 Category: SPIRV::OperandCategory::DecorationOperand,
2818 i: SPIRV::Decoration::NoUnsignedWrap, ST, Reqs)
2819 .IsSatisfiable)
2820 buildOpDecorate(Reg: I.getOperand(i: 0).getReg(), I, TII,
2821 Dec: SPIRV::Decoration::NoUnsignedWrap, DecArgs: {});
2822 }
2823 // In Kernel environments, FPFastMathMode on OpExtInst is valid per core
2824 // spec. For other instruction types, SPV_KHR_float_controls2 is required.
2825 bool CanUseFM =
2826 TII.canUseFastMathFlags(
2827 MI: I, KHRFloatControls2: ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2)) ||
2828 (ST.isKernel() && I.getOpcode() == SPIRV::OpExtInst);
2829 if (!CanUseFM)
2830 return;
2831
2832 unsigned FMFlags = getFastMathFlags(I, ST);
2833 if (FMFlags == SPIRV::FPFastMathMode::None) {
2834 // We also need to check if any FPFastMathDefault info was set for the
2835 // types used in this instruction.
2836 if (FPFastMathDefaultInfoVec.empty())
2837 return;
2838
2839 // There are three types of instructions that can use fast math flags:
2840 // 1. Arithmetic instructions (FAdd, FMul, FSub, FDiv, FRem, etc.)
2841 // 2. Relational instructions (FCmp, FOrd, FUnord, etc.)
2842 // 3. Extended instructions (ExtInst)
2843 // For arithmetic instructions, the floating point type can be in the
2844 // result type or in the operands, but they all must be the same.
2845 // For the relational and logical instructions, the floating point type
2846 // can only be in the operands 1 and 2, not the result type. Also, the
2847 // operands must have the same type. For the extended instructions, the
2848 // floating point type can be in the result type or in the operands. It's
2849 // unclear if the operands and the result type must be the same. Let's
2850 // assume they must be. Therefore, for 1. and 2., we can check the first
2851 // operand type, and for 3. we can check the result type.
2852 assert(I.getNumOperands() >= 3 && "Expected at least 3 operands");
2853 Register ResReg = I.getOpcode() == SPIRV::OpExtInst
2854 ? I.getOperand(i: 1).getReg()
2855 : I.getOperand(i: 2).getReg();
2856 SPIRVTypeInst ResType = GR->getSPIRVTypeForVReg(VReg: ResReg, MF: I.getMF());
2857 const Type *Ty = GR->getTypeForSPIRVType(Ty: ResType);
2858 Ty = Ty->isVectorTy() ? cast<VectorType>(Val: Ty)->getElementType() : Ty;
2859
2860 // Match instruction type with the FPFastMathDefaultInfoVec.
2861 bool Emit = false;
2862 for (SPIRV::FPFastMathDefaultInfo &Elem : FPFastMathDefaultInfoVec) {
2863 if (Ty == Elem.Ty) {
2864 FMFlags = Elem.FastMathFlags;
2865 Emit = Elem.ContractionOff || Elem.SignedZeroInfNanPreserve ||
2866 Elem.FPFastMathDefault;
2867 break;
2868 }
2869 }
2870
2871 if (FMFlags == SPIRV::FPFastMathMode::None && !Emit)
2872 return;
2873 }
2874 if (isFastMathModeAvailable(ST)) {
2875 Register DstReg = I.getOperand(i: 0).getReg();
2876 buildOpDecorate(Reg: DstReg, I, TII, Dec: SPIRV::Decoration::FPFastMathMode,
2877 DecArgs: {FMFlags});
2878 }
2879}
2880
2881// Walk all functions and add decorations related to MI flags.
2882static void addDecorations(const Module &M, const SPIRVInstrInfo &TII,
2883 MachineModuleInfo *MMI, const SPIRVSubtarget &ST,
2884 SPIRV::ModuleAnalysisInfo &MAI,
2885 const SPIRVGlobalRegistry *GR) {
2886 for (const Function &F : M) {
2887 MachineFunction *MF = MMI->getMachineFunction(F);
2888 if (!MF)
2889 continue;
2890
2891 for (auto &MBB : *MF)
2892 for (auto &MI : MBB)
2893 handleMIFlagDecoration(I&: MI, ST, TII, Reqs&: MAI.Reqs, GR,
2894 FPFastMathDefaultInfoVec&: MAI.FPFastMathDefaultInfoMap[&F]);
2895 }
2896}
2897
2898static void addMBBNames(const Module &M, const SPIRVInstrInfo &TII,
2899 MachineModuleInfo *MMI, const SPIRVSubtarget &ST,
2900 SPIRV::ModuleAnalysisInfo &MAI) {
2901 for (const Function &F : M) {
2902 MachineFunction *MF = MMI->getMachineFunction(F);
2903 if (!MF)
2904 continue;
2905 if (MF->getFunction()
2906 .getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME)
2907 .isValid())
2908 continue;
2909 MachineRegisterInfo &MRI = MF->getRegInfo();
2910 for (auto &MBB : *MF) {
2911 if (!MBB.hasName() || MBB.empty())
2912 continue;
2913 // Emit basic block names.
2914 Register Reg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2915 MRI.setRegClass(Reg, RC: &SPIRV::IDRegClass);
2916 buildOpName(Target: Reg, Name: MBB.getName(), I&: *std::prev(x: MBB.end()), TII);
2917 MCRegister GlobalReg = MAI.getOrCreateMBBRegister(MBB);
2918 MAI.setRegisterAlias(MF, Reg, AliasReg: GlobalReg);
2919 }
2920 }
2921}
2922
2923// patching Instruction::PHI to SPIRV::OpPhi
2924static void patchPhis(const Module &M, SPIRVGlobalRegistry *GR,
2925 const SPIRVInstrInfo &TII, MachineModuleInfo *MMI) {
2926 for (const Function &F : M) {
2927 MachineFunction *MF = MMI->getMachineFunction(F);
2928 if (!MF)
2929 continue;
2930 for (auto &MBB : *MF) {
2931 for (MachineInstr &MI : MBB.phis()) {
2932 MI.setDesc(TII.get(Opcode: SPIRV::OpPhi));
2933 Register ResTypeReg = GR->getSPIRVTypeID(
2934 SpirvType: GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 0).getReg(), MF));
2935 MI.insert(InsertBefore: MI.operands_begin() + 1,
2936 Ops: {MachineOperand::CreateReg(Reg: ResTypeReg, isDef: false)});
2937 }
2938 }
2939
2940 MF->getProperties().setNoPHIs();
2941 }
2942}
2943
2944static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
2945 const Module &M, SPIRV::ModuleAnalysisInfo &MAI, const Function *F) {
2946 auto it = MAI.FPFastMathDefaultInfoMap.find(Val: F);
2947 if (it != MAI.FPFastMathDefaultInfoMap.end())
2948 return it->second;
2949
2950 // If the map does not contain the entry, create a new one. Initialize it to
2951 // contain all 3 elements sorted by bit width of target type: {half, float,
2952 // double}.
2953 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
2954 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
2955 Args: SPIRV::FPFastMathMode::None);
2956 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
2957 Args: SPIRV::FPFastMathMode::None);
2958 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
2959 Args: SPIRV::FPFastMathMode::None);
2960 return MAI.FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
2961}
2962
2963static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
2964 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
2965 const Type *Ty) {
2966 size_t BitWidth = Ty->getScalarSizeInBits();
2967 int Index =
2968 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
2969 BitWidth);
2970 assert(Index >= 0 && Index < 3 &&
2971 "Expected FPFastMathDefaultInfo for half, float, or double");
2972 assert(FPFastMathDefaultInfoVec.size() == 3 &&
2973 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
2974 return FPFastMathDefaultInfoVec[Index];
2975}
2976
2977static void collectFPFastMathDefaults(const Module &M,
2978 SPIRV::ModuleAnalysisInfo &MAI,
2979 const SPIRVSubtarget &ST) {
2980 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
2981 return;
2982
2983 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
2984 // We need the entry point (function) as the key, and the target
2985 // type and flags as the value.
2986 // We also need to check ContractionOff and SignedZeroInfNanPreserve
2987 // execution modes, as they are now deprecated and must be replaced
2988 // with FPFastMathDefaultInfo.
2989 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
2990 if (!Node)
2991 return;
2992
2993 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
2994 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
2995 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
2996 const Function *F = cast<Function>(
2997 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
2998 const auto EM =
2999 cast<ConstantInt>(
3000 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
3001 ->getZExtValue();
3002 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
3003 assert(MDN->getNumOperands() == 4 &&
3004 "Expected 4 operands for FPFastMathDefault");
3005
3006 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
3007 unsigned Flags =
3008 cast<ConstantInt>(
3009 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
3010 ->getZExtValue();
3011 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3012 getOrCreateFPFastMathDefaultInfoVec(M, MAI, F);
3013 SPIRV::FPFastMathDefaultInfo &Info =
3014 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
3015 Info.FastMathFlags = Flags;
3016 Info.FPFastMathDefault = true;
3017 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
3018 assert(MDN->getNumOperands() == 2 &&
3019 "Expected no operands for ContractionOff");
3020
3021 // We need to save this info for every possible FP type, i.e. {half,
3022 // float, double, fp128}.
3023 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3024 getOrCreateFPFastMathDefaultInfoVec(M, MAI, F);
3025 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3026 Info.ContractionOff = true;
3027 }
3028 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
3029 assert(MDN->getNumOperands() == 3 &&
3030 "Expected 1 operand for SignedZeroInfNanPreserve");
3031 unsigned TargetWidth =
3032 cast<ConstantInt>(
3033 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
3034 ->getZExtValue();
3035 // We need to save this info only for the FP type with TargetWidth.
3036 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3037 getOrCreateFPFastMathDefaultInfoVec(M, MAI, F);
3038 int Index = SPIRV::FPFastMathDefaultInfoVector::
3039 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
3040 assert(Index >= 0 && Index < 3 &&
3041 "Expected FPFastMathDefaultInfo for half, float, or double");
3042 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3043 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3044 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
3045 }
3046 }
3047}
3048
3049void SPIRVModuleAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
3050 AU.addRequired<TargetPassConfig>();
3051 AU.addRequired<MachineModuleInfoWrapperPass>();
3052}
3053
3054bool SPIRVModuleAnalysis::runOnModule(Module &M) {
3055 SPIRVTargetMachine &TM =
3056 getAnalysis<TargetPassConfig>().getTM<SPIRVTargetMachine>();
3057 ST = TM.getSubtargetImpl();
3058 GR = ST->getSPIRVGlobalRegistry();
3059 TII = ST->getInstrInfo();
3060
3061 MMI = &getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
3062
3063 setBaseInfo(M);
3064
3065 patchPhis(M, GR, TII: *TII, MMI);
3066
3067 addMBBNames(M, TII: *TII, MMI, ST: *ST, MAI);
3068 collectFPFastMathDefaults(M, MAI, ST: *ST);
3069 addDecorations(M, TII: *TII, MMI, ST: *ST, MAI, GR);
3070
3071 collectReqs(M, MAI, MMI, ST: *ST);
3072
3073 // Process type/const/global var/func decl instructions, number their
3074 // destination registers from 0 to N, collect Extensions and Capabilities.
3075 collectDeclarations(M);
3076
3077 // Number rest of registers from N+1 onwards.
3078 numberRegistersGlobally(M);
3079
3080 // Collect OpName, OpEntryPoint, OpDecorate etc, process other instructions.
3081 processOtherInstrs(M);
3082
3083 // If there are no entry points, we need the Linkage capability.
3084 if (MAI.MS[SPIRV::MB_EntryPoints].empty())
3085 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::Linkage);
3086
3087 // Set maximum ID used.
3088 GR->setBound(MAI.MaxID);
3089
3090 return false;
3091}
3092