1//===- SPIRVModuleAnalysis.cpp - analysis of global instrs & regs - C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The analysis collects instructions that should be output at the module level
10// and performs the global register numbering.
11//
12// The results of this analysis are used in AsmPrinter to rename registers
13// globally and to output required instructions at the module level.
14//
15//===----------------------------------------------------------------------===//
16
17// TODO: uses or report_fatal_error (which is also deprecated) /
18// ReportFatalUsageError in this file should be refactored, as per LLVM
19// best practices, to rely on the Diagnostic infrastructure.
20
21#include "SPIRVModuleAnalysis.h"
22#include "MCTargetDesc/SPIRVBaseInfo.h"
23#include "MCTargetDesc/SPIRVMCTargetDesc.h"
24#include "SPIRV.h"
25#include "SPIRVSubtarget.h"
26#include "SPIRVTargetMachine.h"
27#include "SPIRVUtils.h"
28#include "llvm/ADT/STLExtras.h"
29#include "llvm/CodeGen/MachineModuleInfo.h"
30#include "llvm/CodeGen/TargetPassConfig.h"
31
32using namespace llvm;
33
34#define DEBUG_TYPE "spirv-module-analysis"
35
36static cl::opt<bool>
37 SPVDumpDeps("spv-dump-deps",
38 cl::desc("Dump MIR with SPIR-V dependencies info"),
39 cl::Optional, cl::init(Val: false));
40
41static cl::list<SPIRV::Capability::Capability>
42 AvoidCapabilities("avoid-spirv-capabilities",
43 cl::desc("SPIR-V capabilities to avoid if there are "
44 "other options enabling a feature"),
45 cl::ZeroOrMore, cl::Hidden,
46 cl::values(clEnumValN(SPIRV::Capability::Shader, "Shader",
47 "SPIR-V Shader capability")));
48// Use sets instead of cl::list to check "if contains" condition
49struct AvoidCapabilitiesSet {
50 SmallSet<SPIRV::Capability::Capability, 4> S;
51 AvoidCapabilitiesSet() { S.insert_range(R&: AvoidCapabilities); }
52};
53
54char llvm::SPIRVModuleAnalysis::ID = 0;
55
56INITIALIZE_PASS(SPIRVModuleAnalysis, DEBUG_TYPE, "SPIRV module analysis", true,
57 true)
58
59// Retrieve an unsigned from an MDNode with a list of them as operands.
60static unsigned getMetadataUInt(MDNode *MdNode, unsigned OpIndex,
61 unsigned DefaultVal = 0) {
62 if (MdNode && OpIndex < MdNode->getNumOperands()) {
63 const auto &Op = MdNode->getOperand(I: OpIndex);
64 return mdconst::extract<ConstantInt>(MD: Op)->getZExtValue();
65 }
66 return DefaultVal;
67}
68
69static SPIRV::Requirements
70getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
71 unsigned i, const SPIRVSubtarget &ST,
72 SPIRV::RequirementHandler &Reqs) {
73 // A set of capabilities to avoid if there is another option.
74 AvoidCapabilitiesSet AvoidCaps;
75 if (!ST.isShader())
76 AvoidCaps.S.insert(V: SPIRV::Capability::Shader);
77 else
78 AvoidCaps.S.insert(V: SPIRV::Capability::Kernel);
79
80 VersionTuple ReqMinVer = getSymbolicOperandMinVersion(Category, Value: i);
81 VersionTuple ReqMaxVer = getSymbolicOperandMaxVersion(Category, Value: i);
82 VersionTuple SPIRVVersion = ST.getSPIRVVersion();
83 bool MinVerOK = SPIRVVersion.empty() || SPIRVVersion >= ReqMinVer;
84 bool MaxVerOK =
85 ReqMaxVer.empty() || SPIRVVersion.empty() || SPIRVVersion <= ReqMaxVer;
86 CapabilityList ReqCaps = getSymbolicOperandCapabilities(Category, Value: i);
87 ExtensionList ReqExts = getSymbolicOperandExtensions(Category, Value: i);
88 if (ReqCaps.empty()) {
89 if (ReqExts.empty()) {
90 if (MinVerOK && MaxVerOK)
91 return {true, {}, {}, ReqMinVer, ReqMaxVer};
92 return {false, {}, {}, VersionTuple(), VersionTuple()};
93 }
94 } else if (MinVerOK && MaxVerOK) {
95 if (ReqCaps.size() == 1) {
96 auto Cap = ReqCaps[0];
97 if (Reqs.isCapabilityAvailable(Cap)) {
98 ReqExts.append(RHS: getSymbolicOperandExtensions(
99 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
100 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
101 }
102 } else {
103 // By SPIR-V specification: "If an instruction, enumerant, or other
104 // feature specifies multiple enabling capabilities, only one such
105 // capability needs to be declared to use the feature." However, one
106 // capability may be preferred over another. We use command line
107 // argument(s) and AvoidCapabilities to avoid selection of certain
108 // capabilities if there are other options.
109 CapabilityList UseCaps;
110 for (auto Cap : ReqCaps)
111 if (Reqs.isCapabilityAvailable(Cap))
112 UseCaps.push_back(Elt: Cap);
113 for (size_t i = 0, Sz = UseCaps.size(); i < Sz; ++i) {
114 auto Cap = UseCaps[i];
115 if (i == Sz - 1 || !AvoidCaps.S.contains(V: Cap)) {
116 ReqExts.append(RHS: getSymbolicOperandExtensions(
117 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
118 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
119 }
120 }
121 }
122 }
123 // If there are no capabilities, or we can't satisfy the version or
124 // capability requirements, use the list of extensions (if the subtarget
125 // can handle them all).
126 if (llvm::all_of(Range&: ReqExts, P: [&ST](const SPIRV::Extension::Extension &Ext) {
127 return ST.canUseExtension(E: Ext);
128 })) {
129 return {true,
130 {},
131 std::move(ReqExts),
132 VersionTuple(),
133 VersionTuple()}; // TODO: add versions to extensions.
134 }
135 return {false, {}, {}, VersionTuple(), VersionTuple()};
136}
137
138void SPIRVModuleAnalysis::setBaseInfo(const Module &M) {
139 MAI.MaxID = 0;
140 for (int i = 0; i < SPIRV::NUM_MODULE_SECTIONS; i++)
141 MAI.MS[i].clear();
142 MAI.RegisterAliasTable.clear();
143 MAI.InstrsToDelete.clear();
144 MAI.FuncMap.clear();
145 MAI.GlobalVarList.clear();
146 MAI.ExtInstSetMap.clear();
147 MAI.Reqs.clear();
148 MAI.Reqs.initAvailableCapabilities(ST: *ST);
149
150 // TODO: determine memory model and source language from the configuratoin.
151 if (auto MemModel = M.getNamedMetadata(Name: "spirv.MemoryModel")) {
152 auto MemMD = MemModel->getOperand(i: 0);
153 MAI.Addr = static_cast<SPIRV::AddressingModel::AddressingModel>(
154 getMetadataUInt(MdNode: MemMD, OpIndex: 0));
155 MAI.Mem =
156 static_cast<SPIRV::MemoryModel::MemoryModel>(getMetadataUInt(MdNode: MemMD, OpIndex: 1));
157 } else {
158 // TODO: Add support for VulkanMemoryModel.
159 MAI.Mem = ST->isShader() ? SPIRV::MemoryModel::GLSL450
160 : SPIRV::MemoryModel::OpenCL;
161 if (MAI.Mem == SPIRV::MemoryModel::OpenCL) {
162 unsigned PtrSize = ST->getPointerSize();
163 MAI.Addr = PtrSize == 32 ? SPIRV::AddressingModel::Physical32
164 : PtrSize == 64 ? SPIRV::AddressingModel::Physical64
165 : SPIRV::AddressingModel::Logical;
166 } else {
167 // TODO: Add support for PhysicalStorageBufferAddress.
168 MAI.Addr = SPIRV::AddressingModel::Logical;
169 }
170 }
171 // Get the OpenCL version number from metadata.
172 // TODO: support other source languages.
173 if (auto VerNode = M.getNamedMetadata(Name: "opencl.ocl.version")) {
174 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_C;
175 // Construct version literal in accordance with SPIRV-LLVM-Translator.
176 // TODO: support multiple OCL version metadata.
177 assert(VerNode->getNumOperands() > 0 && "Invalid SPIR");
178 auto VersionMD = VerNode->getOperand(i: 0);
179 unsigned MajorNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 0, DefaultVal: 2);
180 unsigned MinorNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 1);
181 unsigned RevNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 2);
182 // Prevent Major part of OpenCL version to be 0
183 MAI.SrcLangVersion =
184 (std::max(a: 1U, b: MajorNum) * 100 + MinorNum) * 1000 + RevNum;
185 } else {
186 // If there is no information about OpenCL version we are forced to generate
187 // OpenCL 1.0 by default for the OpenCL environment to avoid puzzling
188 // run-times with Unknown/0.0 version output. For a reference, LLVM-SPIRV
189 // Translator avoids potential issues with run-times in a similar manner.
190 if (!ST->isShader()) {
191 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_CPP;
192 MAI.SrcLangVersion = 100000;
193 } else {
194 MAI.SrcLang = SPIRV::SourceLanguage::Unknown;
195 MAI.SrcLangVersion = 0;
196 }
197 }
198
199 if (auto ExtNode = M.getNamedMetadata(Name: "opencl.used.extensions")) {
200 for (unsigned I = 0, E = ExtNode->getNumOperands(); I != E; ++I) {
201 MDNode *MD = ExtNode->getOperand(i: I);
202 if (!MD || MD->getNumOperands() == 0)
203 continue;
204 for (unsigned J = 0, N = MD->getNumOperands(); J != N; ++J)
205 MAI.SrcExt.insert(key: cast<MDString>(Val: MD->getOperand(I: J))->getString());
206 }
207 }
208
209 // Update required capabilities for this memory model, addressing model and
210 // source language.
211 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::MemoryModelOperand,
212 i: MAI.Mem, ST: *ST);
213 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::SourceLanguageOperand,
214 i: MAI.SrcLang, ST: *ST);
215 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::AddressingModelOperand,
216 i: MAI.Addr, ST: *ST);
217
218 if (!ST->isShader()) {
219 // TODO: check if it's required by default.
220 MAI.ExtInstSetMap[static_cast<unsigned>(
221 SPIRV::InstructionSet::OpenCL_std)] = MAI.getNextIDRegister();
222 }
223}
224
225// Appends the signature of the decoration instructions that decorate R to
226// Signature.
227static void appendDecorationsForReg(const MachineRegisterInfo &MRI, Register R,
228 InstrSignature &Signature) {
229 for (MachineInstr &UseMI : MRI.use_instructions(Reg: R)) {
230 // We don't handle OpDecorateId because getting the register alias for the
231 // ID can cause problems, and we do not need it for now.
232 if (UseMI.getOpcode() != SPIRV::OpDecorate &&
233 UseMI.getOpcode() != SPIRV::OpMemberDecorate)
234 continue;
235
236 for (unsigned I = 0; I < UseMI.getNumOperands(); ++I) {
237 const MachineOperand &MO = UseMI.getOperand(i: I);
238 if (MO.isReg())
239 continue;
240 Signature.push_back(Elt: hash_value(MO));
241 }
242 }
243}
244
245// Returns a representation of an instruction as a vector of MachineOperand
246// hash values, see llvm::hash_value(const MachineOperand &MO) for details.
247// This creates a signature of the instruction with the same content
248// that MachineOperand::isIdenticalTo uses for comparison.
249static InstrSignature instrToSignature(const MachineInstr &MI,
250 SPIRV::ModuleAnalysisInfo &MAI,
251 bool UseDefReg) {
252 Register DefReg;
253 InstrSignature Signature{MI.getOpcode()};
254 for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
255 // The only decorations that can be applied more than once to a given <id>
256 // or structure member are FuncParamAttr (38), UserSemantic (5635),
257 // CacheControlLoadINTEL (6442), and CacheControlStoreINTEL (6443). For all
258 // the rest of decorations, we will only add to the signature the Opcode,
259 // the id to which it applies, and the decoration id, disregarding any
260 // decoration flags. This will ensure that any subsequent decoration with
261 // the same id will be deemed as a duplicate. Then, at the call site, we
262 // will be able to handle duplicates in the best way.
263 unsigned Opcode = MI.getOpcode();
264 if ((Opcode == SPIRV::OpDecorate) && i >= 2) {
265 unsigned DecorationID = MI.getOperand(i: 1).getImm();
266 if (DecorationID != SPIRV::Decoration::FuncParamAttr &&
267 DecorationID != SPIRV::Decoration::UserSemantic &&
268 DecorationID != SPIRV::Decoration::CacheControlLoadINTEL &&
269 DecorationID != SPIRV::Decoration::CacheControlStoreINTEL)
270 continue;
271 }
272 const MachineOperand &MO = MI.getOperand(i);
273 size_t h;
274 if (MO.isReg()) {
275 if (!UseDefReg && MO.isDef()) {
276 assert(!DefReg.isValid() && "Multiple def registers.");
277 DefReg = MO.getReg();
278 continue;
279 }
280 Register RegAlias = MAI.getRegisterAlias(MF: MI.getMF(), Reg: MO.getReg());
281 if (!RegAlias.isValid()) {
282 LLVM_DEBUG({
283 dbgs() << "Unexpectedly, no global id found for the operand ";
284 MO.print(dbgs());
285 dbgs() << "\nInstruction: ";
286 MI.print(dbgs());
287 dbgs() << "\n";
288 });
289 report_fatal_error(reason: "All v-regs must have been mapped to global id's");
290 }
291 // mimic llvm::hash_value(const MachineOperand &MO)
292 h = hash_combine(args: MO.getType(), args: (unsigned)RegAlias, args: MO.getSubReg(),
293 args: MO.isDef());
294 } else {
295 h = hash_value(MO);
296 }
297 Signature.push_back(Elt: h);
298 }
299
300 if (DefReg.isValid()) {
301 // Decorations change the semantics of the current instruction. So two
302 // identical instruction with different decorations cannot be merged. That
303 // is why we add the decorations to the signature.
304 appendDecorationsForReg(MRI: MI.getMF()->getRegInfo(), R: DefReg, Signature);
305 }
306 return Signature;
307}
308
309bool SPIRVModuleAnalysis::isDeclSection(const MachineRegisterInfo &MRI,
310 const MachineInstr &MI) {
311 unsigned Opcode = MI.getOpcode();
312 switch (Opcode) {
313 case SPIRV::OpTypeForwardPointer:
314 // omit now, collect later
315 return false;
316 case SPIRV::OpVariable:
317 return static_cast<SPIRV::StorageClass::StorageClass>(
318 MI.getOperand(i: 2).getImm()) != SPIRV::StorageClass::Function;
319 case SPIRV::OpFunction:
320 case SPIRV::OpFunctionParameter:
321 return true;
322 }
323 if (GR->hasConstFunPtr() && Opcode == SPIRV::OpUndef) {
324 Register DefReg = MI.getOperand(i: 0).getReg();
325 for (MachineInstr &UseMI : MRI.use_instructions(Reg: DefReg)) {
326 if (UseMI.getOpcode() != SPIRV::OpConstantFunctionPointerINTEL)
327 continue;
328 // it's a dummy definition, FP constant refers to a function,
329 // and this is resolved in another way; let's skip this definition
330 assert(UseMI.getOperand(2).isReg() &&
331 UseMI.getOperand(2).getReg() == DefReg);
332 MAI.setSkipEmission(&MI);
333 return false;
334 }
335 }
336 return TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||
337 TII->isInlineAsmDefInstr(MI);
338}
339
340// This is a special case of a function pointer refering to a possibly
341// forward function declaration. The operand is a dummy OpUndef that
342// requires a special treatment.
343void SPIRVModuleAnalysis::visitFunPtrUse(
344 Register OpReg, InstrGRegsMap &SignatureToGReg,
345 std::map<const Value *, unsigned> &GlobalToGReg, const MachineFunction *MF,
346 const MachineInstr &MI) {
347 const MachineOperand *OpFunDef =
348 GR->getFunctionDefinitionByUse(Use: &MI.getOperand(i: 2));
349 assert(OpFunDef && OpFunDef->isReg());
350 // find the actual function definition and number it globally in advance
351 const MachineInstr *OpDefMI = OpFunDef->getParent();
352 assert(OpDefMI && OpDefMI->getOpcode() == SPIRV::OpFunction);
353 const MachineFunction *FunDefMF = OpDefMI->getParent()->getParent();
354 const MachineRegisterInfo &FunDefMRI = FunDefMF->getRegInfo();
355 do {
356 visitDecl(MRI: FunDefMRI, SignatureToGReg, GlobalToGReg, MF: FunDefMF, MI: *OpDefMI);
357 OpDefMI = OpDefMI->getNextNode();
358 } while (OpDefMI && (OpDefMI->getOpcode() == SPIRV::OpFunction ||
359 OpDefMI->getOpcode() == SPIRV::OpFunctionParameter));
360 // associate the function pointer with the newly assigned global number
361 MCRegister GlobalFunDefReg =
362 MAI.getRegisterAlias(MF: FunDefMF, Reg: OpFunDef->getReg());
363 assert(GlobalFunDefReg.isValid() &&
364 "Function definition must refer to a global register");
365 MAI.setRegisterAlias(MF, Reg: OpReg, AliasReg: GlobalFunDefReg);
366}
367
368// Depth first recursive traversal of dependencies. Repeated visits are guarded
369// by MAI.hasRegisterAlias().
370void SPIRVModuleAnalysis::visitDecl(
371 const MachineRegisterInfo &MRI, InstrGRegsMap &SignatureToGReg,
372 std::map<const Value *, unsigned> &GlobalToGReg, const MachineFunction *MF,
373 const MachineInstr &MI) {
374 unsigned Opcode = MI.getOpcode();
375
376 // Process each operand of the instruction to resolve dependencies
377 for (const MachineOperand &MO : MI.operands()) {
378 if (!MO.isReg() || MO.isDef())
379 continue;
380 Register OpReg = MO.getReg();
381 // Handle function pointers special case
382 if (Opcode == SPIRV::OpConstantFunctionPointerINTEL &&
383 MRI.getRegClass(Reg: OpReg) == &SPIRV::pIDRegClass) {
384 visitFunPtrUse(OpReg, SignatureToGReg, GlobalToGReg, MF, MI);
385 continue;
386 }
387 // Skip already processed instructions
388 if (MAI.hasRegisterAlias(MF, Reg: MO.getReg()))
389 continue;
390 // Recursively visit dependencies
391 if (const MachineInstr *OpDefMI = MRI.getUniqueVRegDef(Reg: OpReg)) {
392 if (isDeclSection(MRI, MI: *OpDefMI))
393 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, MI: *OpDefMI);
394 continue;
395 }
396 // Handle the unexpected case of no unique definition for the SPIR-V
397 // instruction
398 LLVM_DEBUG({
399 dbgs() << "Unexpectedly, no unique definition for the operand ";
400 MO.print(dbgs());
401 dbgs() << "\nInstruction: ";
402 MI.print(dbgs());
403 dbgs() << "\n";
404 });
405 report_fatal_error(
406 reason: "No unique definition is found for the virtual register");
407 }
408
409 MCRegister GReg;
410 bool IsFunDef = false;
411 if (TII->isSpecConstantInstr(MI)) {
412 GReg = MAI.getNextIDRegister();
413 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
414 } else if (Opcode == SPIRV::OpFunction ||
415 Opcode == SPIRV::OpFunctionParameter) {
416 GReg = handleFunctionOrParameter(MF, MI, GlobalToGReg, IsFunDef);
417 } else if (Opcode == SPIRV::OpTypeStruct ||
418 Opcode == SPIRV::OpConstantComposite) {
419 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);
420 const MachineInstr *NextInstr = MI.getNextNode();
421 while (NextInstr &&
422 ((Opcode == SPIRV::OpTypeStruct &&
423 NextInstr->getOpcode() == SPIRV::OpTypeStructContinuedINTEL) ||
424 (Opcode == SPIRV::OpConstantComposite &&
425 NextInstr->getOpcode() ==
426 SPIRV::OpConstantCompositeContinuedINTEL))) {
427 MCRegister Tmp = handleTypeDeclOrConstant(MI: *NextInstr, SignatureToGReg);
428 MAI.setRegisterAlias(MF, Reg: NextInstr->getOperand(i: 0).getReg(), AliasReg: Tmp);
429 MAI.setSkipEmission(NextInstr);
430 NextInstr = NextInstr->getNextNode();
431 }
432 } else if (TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||
433 TII->isInlineAsmDefInstr(MI)) {
434 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);
435 } else if (Opcode == SPIRV::OpVariable) {
436 GReg = handleVariable(MF, MI, GlobalToGReg);
437 } else {
438 LLVM_DEBUG({
439 dbgs() << "\nInstruction: ";
440 MI.print(dbgs());
441 dbgs() << "\n";
442 });
443 llvm_unreachable("Unexpected instruction is visited");
444 }
445 MAI.setRegisterAlias(MF, Reg: MI.getOperand(i: 0).getReg(), AliasReg: GReg);
446 if (!IsFunDef)
447 MAI.setSkipEmission(&MI);
448}
449
450MCRegister SPIRVModuleAnalysis::handleFunctionOrParameter(
451 const MachineFunction *MF, const MachineInstr &MI,
452 std::map<const Value *, unsigned> &GlobalToGReg, bool &IsFunDef) {
453 const Value *GObj = GR->getGlobalObject(MF, R: MI.getOperand(i: 0).getReg());
454 assert(GObj && "Unregistered global definition");
455 const Function *F = dyn_cast<Function>(Val: GObj);
456 if (!F)
457 F = dyn_cast<Argument>(Val: GObj)->getParent();
458 assert(F && "Expected a reference to a function or an argument");
459 IsFunDef = !F->isDeclaration();
460 auto [It, Inserted] = GlobalToGReg.try_emplace(k: GObj);
461 if (!Inserted)
462 return It->second;
463 MCRegister GReg = MAI.getNextIDRegister();
464 It->second = GReg;
465 if (!IsFunDef)
466 MAI.MS[SPIRV::MB_ExtFuncDecls].push_back(Elt: &MI);
467 return GReg;
468}
469
470MCRegister
471SPIRVModuleAnalysis::handleTypeDeclOrConstant(const MachineInstr &MI,
472 InstrGRegsMap &SignatureToGReg) {
473 InstrSignature MISign = instrToSignature(MI, MAI, UseDefReg: false);
474 auto [It, Inserted] = SignatureToGReg.try_emplace(k: MISign);
475 if (!Inserted)
476 return It->second;
477 MCRegister GReg = MAI.getNextIDRegister();
478 It->second = GReg;
479 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
480 return GReg;
481}
482
483MCRegister SPIRVModuleAnalysis::handleVariable(
484 const MachineFunction *MF, const MachineInstr &MI,
485 std::map<const Value *, unsigned> &GlobalToGReg) {
486 MAI.GlobalVarList.push_back(Elt: &MI);
487 const Value *GObj = GR->getGlobalObject(MF, R: MI.getOperand(i: 0).getReg());
488 assert(GObj && "Unregistered global definition");
489 auto [It, Inserted] = GlobalToGReg.try_emplace(k: GObj);
490 if (!Inserted)
491 return It->second;
492 MCRegister GReg = MAI.getNextIDRegister();
493 It->second = GReg;
494 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
495 return GReg;
496}
497
498void SPIRVModuleAnalysis::collectDeclarations(const Module &M) {
499 InstrGRegsMap SignatureToGReg;
500 std::map<const Value *, unsigned> GlobalToGReg;
501 for (const Function &F : M) {
502 MachineFunction *MF = MMI->getMachineFunction(F);
503 if (!MF)
504 continue;
505 const MachineRegisterInfo &MRI = MF->getRegInfo();
506 unsigned PastHeader = 0;
507 for (MachineBasicBlock &MBB : *MF) {
508 for (MachineInstr &MI : MBB) {
509 if (MI.getNumOperands() == 0)
510 continue;
511 unsigned Opcode = MI.getOpcode();
512 if (Opcode == SPIRV::OpFunction) {
513 if (PastHeader == 0) {
514 PastHeader = 1;
515 continue;
516 }
517 } else if (Opcode == SPIRV::OpFunctionParameter) {
518 if (PastHeader < 2)
519 continue;
520 } else if (PastHeader > 0) {
521 PastHeader = 2;
522 }
523
524 const MachineOperand &DefMO = MI.getOperand(i: 0);
525 switch (Opcode) {
526 case SPIRV::OpExtension:
527 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::Extension(DefMO.getImm()));
528 MAI.setSkipEmission(&MI);
529 break;
530 case SPIRV::OpCapability:
531 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::Capability(DefMO.getImm()));
532 MAI.setSkipEmission(&MI);
533 if (PastHeader > 0)
534 PastHeader = 2;
535 break;
536 default:
537 if (DefMO.isReg() && isDeclSection(MRI, MI) &&
538 !MAI.hasRegisterAlias(MF, Reg: DefMO.getReg()))
539 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, MI);
540 }
541 }
542 }
543 }
544}
545
546// Look for IDs declared with Import linkage, and map the corresponding function
547// to the register defining that variable (which will usually be the result of
548// an OpFunction). This lets us call externally imported functions using
549// the correct ID registers.
550void SPIRVModuleAnalysis::collectFuncNames(MachineInstr &MI,
551 const Function *F) {
552 if (MI.getOpcode() == SPIRV::OpDecorate) {
553 // If it's got Import linkage.
554 auto Dec = MI.getOperand(i: 1).getImm();
555 if (Dec == SPIRV::Decoration::LinkageAttributes) {
556 auto Lnk = MI.getOperand(i: MI.getNumOperands() - 1).getImm();
557 if (Lnk == SPIRV::LinkageType::Import) {
558 // Map imported function name to function ID register.
559 const Function *ImportedFunc =
560 F->getParent()->getFunction(Name: getStringImm(MI, StartIndex: 2));
561 Register Target = MI.getOperand(i: 0).getReg();
562 MAI.FuncMap[ImportedFunc] = MAI.getRegisterAlias(MF: MI.getMF(), Reg: Target);
563 }
564 }
565 } else if (MI.getOpcode() == SPIRV::OpFunction) {
566 // Record all internal OpFunction declarations.
567 Register Reg = MI.defs().begin()->getReg();
568 MCRegister GlobalReg = MAI.getRegisterAlias(MF: MI.getMF(), Reg);
569 assert(GlobalReg.isValid());
570 MAI.FuncMap[F] = GlobalReg;
571 }
572}
573
574// Collect the given instruction in the specified MS. We assume global register
575// numbering has already occurred by this point. We can directly compare reg
576// arguments when detecting duplicates.
577static void collectOtherInstr(MachineInstr &MI, SPIRV::ModuleAnalysisInfo &MAI,
578 SPIRV::ModuleSectionType MSType, InstrTraces &IS,
579 bool Append = true) {
580 MAI.setSkipEmission(&MI);
581 InstrSignature MISign = instrToSignature(MI, MAI, UseDefReg: true);
582 auto FoundMI = IS.insert(x: std::move(MISign));
583 if (!FoundMI.second) {
584 if (MI.getOpcode() == SPIRV::OpDecorate) {
585 assert(MI.getNumOperands() >= 2 &&
586 "Decoration instructions must have at least 2 operands");
587 assert(MSType == SPIRV::MB_Annotations &&
588 "Only OpDecorate instructions can be duplicates");
589 // For FPFastMathMode decoration, we need to merge the flags of the
590 // duplicate decoration with the original one, so we need to find the
591 // original instruction that has the same signature. For the rest of
592 // instructions, we will simply skip the duplicate.
593 if (MI.getOperand(i: 1).getImm() != SPIRV::Decoration::FPFastMathMode)
594 return; // Skip duplicates of other decorations.
595
596 const SPIRV::InstrList &Decorations = MAI.MS[MSType];
597 for (const MachineInstr *OrigMI : Decorations) {
598 if (instrToSignature(MI: *OrigMI, MAI, UseDefReg: true) == MISign) {
599 assert(OrigMI->getNumOperands() == MI.getNumOperands() &&
600 "Original instruction must have the same number of operands");
601 assert(
602 OrigMI->getNumOperands() == 3 &&
603 "FPFastMathMode decoration must have 3 operands for OpDecorate");
604 unsigned OrigFlags = OrigMI->getOperand(i: 2).getImm();
605 unsigned NewFlags = MI.getOperand(i: 2).getImm();
606 if (OrigFlags == NewFlags)
607 return; // No need to merge, the flags are the same.
608
609 // Emit warning about possible conflict between flags.
610 unsigned FinalFlags = OrigFlags | NewFlags;
611 llvm::errs()
612 << "Warning: Conflicting FPFastMathMode decoration flags "
613 "in instruction: "
614 << *OrigMI << "Original flags: " << OrigFlags
615 << ", new flags: " << NewFlags
616 << ". They will be merged on a best effort basis, but not "
617 "validated. Final flags: "
618 << FinalFlags << "\n";
619 MachineInstr *OrigMINonConst = const_cast<MachineInstr *>(OrigMI);
620 MachineOperand &OrigFlagsOp = OrigMINonConst->getOperand(i: 2);
621 OrigFlagsOp = MachineOperand::CreateImm(Val: FinalFlags);
622 return; // Merge done, so we found a duplicate; don't add it to MAI.MS
623 }
624 }
625 assert(false && "No original instruction found for the duplicate "
626 "OpDecorate, but we found one in IS.");
627 }
628 return; // insert failed, so we found a duplicate; don't add it to MAI.MS
629 }
630 // No duplicates, so add it.
631 if (Append)
632 MAI.MS[MSType].push_back(Elt: &MI);
633 else
634 MAI.MS[MSType].insert(I: MAI.MS[MSType].begin(), Elt: &MI);
635}
636
637// Some global instructions make reference to function-local ID regs, so cannot
638// be correctly collected until these registers are globally numbered.
639void SPIRVModuleAnalysis::processOtherInstrs(const Module &M) {
640 InstrTraces IS;
641 for (const Function &F : M) {
642 if (F.isDeclaration())
643 continue;
644 MachineFunction *MF = MMI->getMachineFunction(F);
645 assert(MF);
646
647 for (MachineBasicBlock &MBB : *MF)
648 for (MachineInstr &MI : MBB) {
649 if (MAI.getSkipEmission(MI: &MI))
650 continue;
651 const unsigned OpCode = MI.getOpcode();
652 if (OpCode == SPIRV::OpString) {
653 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_DebugStrings, IS);
654 } else if (OpCode == SPIRV::OpExtInst && MI.getOperand(i: 2).isImm() &&
655 MI.getOperand(i: 2).getImm() ==
656 SPIRV::InstructionSet::
657 NonSemantic_Shader_DebugInfo_100) {
658 MachineOperand Ins = MI.getOperand(i: 3);
659 namespace NS = SPIRV::NonSemanticExtInst;
660 static constexpr int64_t GlobalNonSemanticDITy[] = {
661 NS::DebugSource, NS::DebugCompilationUnit, NS::DebugInfoNone,
662 NS::DebugTypeBasic, NS::DebugTypePointer};
663 bool IsGlobalDI = false;
664 for (unsigned Idx = 0; Idx < std::size(GlobalNonSemanticDITy); ++Idx)
665 IsGlobalDI |= Ins.getImm() == GlobalNonSemanticDITy[Idx];
666 if (IsGlobalDI)
667 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_NonSemanticGlobalDI, IS);
668 } else if (OpCode == SPIRV::OpName || OpCode == SPIRV::OpMemberName) {
669 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_DebugNames, IS);
670 } else if (OpCode == SPIRV::OpEntryPoint) {
671 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_EntryPoints, IS);
672 } else if (TII->isAliasingInstr(MI)) {
673 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_AliasingInsts, IS);
674 } else if (TII->isDecorationInstr(MI)) {
675 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_Annotations, IS);
676 collectFuncNames(MI, F: &F);
677 } else if (TII->isConstantInstr(MI)) {
678 // Now OpSpecConstant*s are not in DT,
679 // but they need to be collected anyway.
680 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_TypeConstVars, IS);
681 } else if (OpCode == SPIRV::OpFunction) {
682 collectFuncNames(MI, F: &F);
683 } else if (OpCode == SPIRV::OpTypeForwardPointer) {
684 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_TypeConstVars, IS, Append: false);
685 }
686 }
687 }
688}
689
690// Number registers in all functions globally from 0 onwards and store
691// the result in global register alias table. Some registers are already
692// numbered.
693void SPIRVModuleAnalysis::numberRegistersGlobally(const Module &M) {
694 for (const Function &F : M) {
695 if (F.isDeclaration())
696 continue;
697 MachineFunction *MF = MMI->getMachineFunction(F);
698 assert(MF);
699 for (MachineBasicBlock &MBB : *MF) {
700 for (MachineInstr &MI : MBB) {
701 for (MachineOperand &Op : MI.operands()) {
702 if (!Op.isReg())
703 continue;
704 Register Reg = Op.getReg();
705 if (MAI.hasRegisterAlias(MF, Reg))
706 continue;
707 MCRegister NewReg = MAI.getNextIDRegister();
708 MAI.setRegisterAlias(MF, Reg, AliasReg: NewReg);
709 }
710 if (MI.getOpcode() != SPIRV::OpExtInst)
711 continue;
712 auto Set = MI.getOperand(i: 2).getImm();
713 auto [It, Inserted] = MAI.ExtInstSetMap.try_emplace(Key: Set);
714 if (Inserted)
715 It->second = MAI.getNextIDRegister();
716 }
717 }
718 }
719}
720
721// RequirementHandler implementations.
722void SPIRV::RequirementHandler::getAndAddRequirements(
723 SPIRV::OperandCategory::OperandCategory Category, uint32_t i,
724 const SPIRVSubtarget &ST) {
725 addRequirements(Req: getSymbolicOperandRequirements(Category, i, ST, Reqs&: *this));
726}
727
728void SPIRV::RequirementHandler::recursiveAddCapabilities(
729 const CapabilityList &ToPrune) {
730 for (const auto &Cap : ToPrune) {
731 AllCaps.insert(V: Cap);
732 CapabilityList ImplicitDecls =
733 getSymbolicOperandCapabilities(Category: OperandCategory::CapabilityOperand, Value: Cap);
734 recursiveAddCapabilities(ToPrune: ImplicitDecls);
735 }
736}
737
738void SPIRV::RequirementHandler::addCapabilities(const CapabilityList &ToAdd) {
739 for (const auto &Cap : ToAdd) {
740 bool IsNewlyInserted = AllCaps.insert(V: Cap).second;
741 if (!IsNewlyInserted) // Don't re-add if it's already been declared.
742 continue;
743 CapabilityList ImplicitDecls =
744 getSymbolicOperandCapabilities(Category: OperandCategory::CapabilityOperand, Value: Cap);
745 recursiveAddCapabilities(ToPrune: ImplicitDecls);
746 MinimalCaps.push_back(Elt: Cap);
747 }
748}
749
750void SPIRV::RequirementHandler::addRequirements(
751 const SPIRV::Requirements &Req) {
752 if (!Req.IsSatisfiable)
753 report_fatal_error(reason: "Adding SPIR-V requirements this target can't satisfy.");
754
755 if (Req.Cap.has_value())
756 addCapabilities(ToAdd: {Req.Cap.value()});
757
758 addExtensions(ToAdd: Req.Exts);
759
760 if (!Req.MinVer.empty()) {
761 if (!MaxVersion.empty() && Req.MinVer > MaxVersion) {
762 LLVM_DEBUG(dbgs() << "Conflicting version requirements: >= " << Req.MinVer
763 << " and <= " << MaxVersion << "\n");
764 report_fatal_error(reason: "Adding SPIR-V requirements that can't be satisfied.");
765 }
766
767 if (MinVersion.empty() || Req.MinVer > MinVersion)
768 MinVersion = Req.MinVer;
769 }
770
771 if (!Req.MaxVer.empty()) {
772 if (!MinVersion.empty() && Req.MaxVer < MinVersion) {
773 LLVM_DEBUG(dbgs() << "Conflicting version requirements: <= " << Req.MaxVer
774 << " and >= " << MinVersion << "\n");
775 report_fatal_error(reason: "Adding SPIR-V requirements that can't be satisfied.");
776 }
777
778 if (MaxVersion.empty() || Req.MaxVer < MaxVersion)
779 MaxVersion = Req.MaxVer;
780 }
781}
782
783void SPIRV::RequirementHandler::checkSatisfiable(
784 const SPIRVSubtarget &ST) const {
785 // Report as many errors as possible before aborting the compilation.
786 bool IsSatisfiable = true;
787 auto TargetVer = ST.getSPIRVVersion();
788
789 if (!MaxVersion.empty() && !TargetVer.empty() && MaxVersion < TargetVer) {
790 LLVM_DEBUG(
791 dbgs() << "Target SPIR-V version too high for required features\n"
792 << "Required max version: " << MaxVersion << " target version "
793 << TargetVer << "\n");
794 IsSatisfiable = false;
795 }
796
797 if (!MinVersion.empty() && !TargetVer.empty() && MinVersion > TargetVer) {
798 LLVM_DEBUG(dbgs() << "Target SPIR-V version too low for required features\n"
799 << "Required min version: " << MinVersion
800 << " target version " << TargetVer << "\n");
801 IsSatisfiable = false;
802 }
803
804 if (!MinVersion.empty() && !MaxVersion.empty() && MinVersion > MaxVersion) {
805 LLVM_DEBUG(
806 dbgs()
807 << "Version is too low for some features and too high for others.\n"
808 << "Required SPIR-V min version: " << MinVersion
809 << " required SPIR-V max version " << MaxVersion << "\n");
810 IsSatisfiable = false;
811 }
812
813 AvoidCapabilitiesSet AvoidCaps;
814 if (!ST.isShader())
815 AvoidCaps.S.insert(V: SPIRV::Capability::Shader);
816 else
817 AvoidCaps.S.insert(V: SPIRV::Capability::Kernel);
818
819 for (auto Cap : MinimalCaps) {
820 if (AvailableCaps.contains(V: Cap) && !AvoidCaps.S.contains(V: Cap))
821 continue;
822 LLVM_DEBUG(dbgs() << "Capability not supported: "
823 << getSymbolicOperandMnemonic(
824 OperandCategory::CapabilityOperand, Cap)
825 << "\n");
826 IsSatisfiable = false;
827 }
828
829 for (auto Ext : AllExtensions) {
830 if (ST.canUseExtension(E: Ext))
831 continue;
832 LLVM_DEBUG(dbgs() << "Extension not supported: "
833 << getSymbolicOperandMnemonic(
834 OperandCategory::ExtensionOperand, Ext)
835 << "\n");
836 IsSatisfiable = false;
837 }
838
839 if (!IsSatisfiable)
840 report_fatal_error(reason: "Unable to meet SPIR-V requirements for this target.");
841}
842
843// Add the given capabilities and all their implicitly defined capabilities too.
844void SPIRV::RequirementHandler::addAvailableCaps(const CapabilityList &ToAdd) {
845 for (const auto Cap : ToAdd)
846 if (AvailableCaps.insert(V: Cap).second)
847 addAvailableCaps(ToAdd: getSymbolicOperandCapabilities(
848 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
849}
850
851void SPIRV::RequirementHandler::removeCapabilityIf(
852 const Capability::Capability ToRemove,
853 const Capability::Capability IfPresent) {
854 if (AllCaps.contains(V: IfPresent))
855 AllCaps.erase(V: ToRemove);
856}
857
858namespace llvm {
859namespace SPIRV {
860void RequirementHandler::initAvailableCapabilities(const SPIRVSubtarget &ST) {
861 // Provided by both all supported Vulkan versions and OpenCl.
862 addAvailableCaps(ToAdd: {Capability::Shader, Capability::Linkage, Capability::Int8,
863 Capability::Int16});
864
865 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 3)))
866 addAvailableCaps(ToAdd: {Capability::GroupNonUniform,
867 Capability::GroupNonUniformVote,
868 Capability::GroupNonUniformArithmetic,
869 Capability::GroupNonUniformBallot,
870 Capability::GroupNonUniformClustered,
871 Capability::GroupNonUniformShuffle,
872 Capability::GroupNonUniformShuffleRelative});
873
874 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
875 addAvailableCaps(ToAdd: {Capability::DotProduct, Capability::DotProductInputAll,
876 Capability::DotProductInput4x8Bit,
877 Capability::DotProductInput4x8BitPacked,
878 Capability::DemoteToHelperInvocation});
879
880 // Add capabilities enabled by extensions.
881 for (auto Extension : ST.getAllAvailableExtensions()) {
882 CapabilityList EnabledCapabilities =
883 getCapabilitiesEnabledByExtension(Extension);
884 addAvailableCaps(ToAdd: EnabledCapabilities);
885 }
886
887 if (!ST.isShader()) {
888 initAvailableCapabilitiesForOpenCL(ST);
889 return;
890 }
891
892 if (ST.isShader()) {
893 initAvailableCapabilitiesForVulkan(ST);
894 return;
895 }
896
897 report_fatal_error(reason: "Unimplemented environment for SPIR-V generation.");
898}
899
900void RequirementHandler::initAvailableCapabilitiesForOpenCL(
901 const SPIRVSubtarget &ST) {
902 // Add the min requirements for different OpenCL and SPIR-V versions.
903 addAvailableCaps(ToAdd: {Capability::Addresses, Capability::Float16Buffer,
904 Capability::Kernel, Capability::Vector16,
905 Capability::Groups, Capability::GenericPointer,
906 Capability::StorageImageWriteWithoutFormat,
907 Capability::StorageImageReadWithoutFormat});
908 if (ST.hasOpenCLFullProfile())
909 addAvailableCaps(ToAdd: {Capability::Int64, Capability::Int64Atomics});
910 if (ST.hasOpenCLImageSupport()) {
911 addAvailableCaps(ToAdd: {Capability::ImageBasic, Capability::LiteralSampler,
912 Capability::Image1D, Capability::SampledBuffer,
913 Capability::ImageBuffer});
914 if (ST.isAtLeastOpenCLVer(VerToCompareTo: VersionTuple(2, 0)))
915 addAvailableCaps(ToAdd: {Capability::ImageReadWrite});
916 }
917 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 1)) &&
918 ST.isAtLeastOpenCLVer(VerToCompareTo: VersionTuple(2, 2)))
919 addAvailableCaps(ToAdd: {Capability::SubgroupDispatch, Capability::PipeStorage});
920 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 4)))
921 addAvailableCaps(ToAdd: {Capability::DenormPreserve, Capability::DenormFlushToZero,
922 Capability::SignedZeroInfNanPreserve,
923 Capability::RoundingModeRTE,
924 Capability::RoundingModeRTZ});
925 // TODO: verify if this needs some checks.
926 addAvailableCaps(ToAdd: {Capability::Float16, Capability::Float64});
927
928 // TODO: add OpenCL extensions.
929}
930
931void RequirementHandler::initAvailableCapabilitiesForVulkan(
932 const SPIRVSubtarget &ST) {
933
934 // Core in Vulkan 1.1 and earlier.
935 addAvailableCaps(ToAdd: {Capability::Int64, Capability::Float16, Capability::Float64,
936 Capability::GroupNonUniform, Capability::Image1D,
937 Capability::SampledBuffer, Capability::ImageBuffer,
938 Capability::UniformBufferArrayDynamicIndexing,
939 Capability::SampledImageArrayDynamicIndexing,
940 Capability::StorageBufferArrayDynamicIndexing,
941 Capability::StorageImageArrayDynamicIndexing,
942 Capability::DerivativeControl, Capability::MinLod});
943
944 // Became core in Vulkan 1.2
945 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 5))) {
946 addAvailableCaps(
947 ToAdd: {Capability::ShaderNonUniformEXT, Capability::RuntimeDescriptorArrayEXT,
948 Capability::InputAttachmentArrayDynamicIndexingEXT,
949 Capability::UniformTexelBufferArrayDynamicIndexingEXT,
950 Capability::StorageTexelBufferArrayDynamicIndexingEXT,
951 Capability::UniformBufferArrayNonUniformIndexingEXT,
952 Capability::SampledImageArrayNonUniformIndexingEXT,
953 Capability::StorageBufferArrayNonUniformIndexingEXT,
954 Capability::StorageImageArrayNonUniformIndexingEXT,
955 Capability::InputAttachmentArrayNonUniformIndexingEXT,
956 Capability::UniformTexelBufferArrayNonUniformIndexingEXT,
957 Capability::StorageTexelBufferArrayNonUniformIndexingEXT});
958 }
959
960 // Became core in Vulkan 1.3
961 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
962 addAvailableCaps(ToAdd: {Capability::StorageImageWriteWithoutFormat,
963 Capability::StorageImageReadWithoutFormat});
964}
965
966} // namespace SPIRV
967} // namespace llvm
968
969// Add the required capabilities from a decoration instruction (including
970// BuiltIns).
971static void addOpDecorateReqs(const MachineInstr &MI, unsigned DecIndex,
972 SPIRV::RequirementHandler &Reqs,
973 const SPIRVSubtarget &ST) {
974 int64_t DecOp = MI.getOperand(i: DecIndex).getImm();
975 auto Dec = static_cast<SPIRV::Decoration::Decoration>(DecOp);
976 Reqs.addRequirements(Req: getSymbolicOperandRequirements(
977 Category: SPIRV::OperandCategory::DecorationOperand, i: Dec, ST, Reqs));
978
979 if (Dec == SPIRV::Decoration::BuiltIn) {
980 int64_t BuiltInOp = MI.getOperand(i: DecIndex + 1).getImm();
981 auto BuiltIn = static_cast<SPIRV::BuiltIn::BuiltIn>(BuiltInOp);
982 Reqs.addRequirements(Req: getSymbolicOperandRequirements(
983 Category: SPIRV::OperandCategory::BuiltInOperand, i: BuiltIn, ST, Reqs));
984 } else if (Dec == SPIRV::Decoration::LinkageAttributes) {
985 int64_t LinkageOp = MI.getOperand(i: MI.getNumOperands() - 1).getImm();
986 SPIRV::LinkageType::LinkageType LnkType =
987 static_cast<SPIRV::LinkageType::LinkageType>(LinkageOp);
988 if (LnkType == SPIRV::LinkageType::LinkOnceODR)
989 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_linkonce_odr);
990 } else if (Dec == SPIRV::Decoration::CacheControlLoadINTEL ||
991 Dec == SPIRV::Decoration::CacheControlStoreINTEL) {
992 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_cache_controls);
993 } else if (Dec == SPIRV::Decoration::HostAccessINTEL) {
994 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_global_variable_host_access);
995 } else if (Dec == SPIRV::Decoration::InitModeINTEL ||
996 Dec == SPIRV::Decoration::ImplementInRegisterMapINTEL) {
997 Reqs.addExtension(
998 ToAdd: SPIRV::Extension::SPV_INTEL_global_variable_fpga_decorations);
999 } else if (Dec == SPIRV::Decoration::NonUniformEXT) {
1000 Reqs.addRequirements(Req: SPIRV::Capability::ShaderNonUniformEXT);
1001 } else if (Dec == SPIRV::Decoration::FPMaxErrorDecorationINTEL) {
1002 Reqs.addRequirements(Req: SPIRV::Capability::FPMaxErrorINTEL);
1003 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_fp_max_error);
1004 } else if (Dec == SPIRV::Decoration::FPFastMathMode) {
1005 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2)) {
1006 Reqs.addRequirements(Req: SPIRV::Capability::FloatControls2);
1007 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls2);
1008 }
1009 }
1010}
1011
1012// Add requirements for image handling.
1013static void addOpTypeImageReqs(const MachineInstr &MI,
1014 SPIRV::RequirementHandler &Reqs,
1015 const SPIRVSubtarget &ST) {
1016 assert(MI.getNumOperands() >= 8 && "Insufficient operands for OpTypeImage");
1017 // The operand indices used here are based on the OpTypeImage layout, which
1018 // the MachineInstr follows as well.
1019 int64_t ImgFormatOp = MI.getOperand(i: 7).getImm();
1020 auto ImgFormat = static_cast<SPIRV::ImageFormat::ImageFormat>(ImgFormatOp);
1021 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ImageFormatOperand,
1022 i: ImgFormat, ST);
1023
1024 bool IsArrayed = MI.getOperand(i: 4).getImm() == 1;
1025 bool IsMultisampled = MI.getOperand(i: 5).getImm() == 1;
1026 bool NoSampler = MI.getOperand(i: 6).getImm() == 2;
1027 // Add dimension requirements.
1028 assert(MI.getOperand(2).isImm());
1029 switch (MI.getOperand(i: 2).getImm()) {
1030 case SPIRV::Dim::DIM_1D:
1031 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::Image1D
1032 : SPIRV::Capability::Sampled1D);
1033 break;
1034 case SPIRV::Dim::DIM_2D:
1035 if (IsMultisampled && NoSampler)
1036 Reqs.addRequirements(Req: SPIRV::Capability::ImageMSArray);
1037 break;
1038 case SPIRV::Dim::DIM_Cube:
1039 Reqs.addRequirements(Req: SPIRV::Capability::Shader);
1040 if (IsArrayed)
1041 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageCubeArray
1042 : SPIRV::Capability::SampledCubeArray);
1043 break;
1044 case SPIRV::Dim::DIM_Rect:
1045 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageRect
1046 : SPIRV::Capability::SampledRect);
1047 break;
1048 case SPIRV::Dim::DIM_Buffer:
1049 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageBuffer
1050 : SPIRV::Capability::SampledBuffer);
1051 break;
1052 case SPIRV::Dim::DIM_SubpassData:
1053 Reqs.addRequirements(Req: SPIRV::Capability::InputAttachment);
1054 break;
1055 }
1056
1057 // Has optional access qualifier.
1058 if (!ST.isShader()) {
1059 if (MI.getNumOperands() > 8 &&
1060 MI.getOperand(i: 8).getImm() == SPIRV::AccessQualifier::ReadWrite)
1061 Reqs.addRequirements(Req: SPIRV::Capability::ImageReadWrite);
1062 else
1063 Reqs.addRequirements(Req: SPIRV::Capability::ImageBasic);
1064 }
1065}
1066
1067static bool isBFloat16Type(SPIRVTypeInst TypeDef) {
1068 return TypeDef && TypeDef->getNumOperands() == 3 &&
1069 TypeDef->getOpcode() == SPIRV::OpTypeFloat &&
1070 TypeDef->getOperand(i: 1).getImm() == 16 &&
1071 TypeDef->getOperand(i: 2).getImm() == SPIRV::FPEncoding::BFloat16KHR;
1072}
1073
1074// Add requirements for handling atomic float instructions
1075#define ATOM_FLT_REQ_EXT_MSG(ExtName) \
1076 "The atomic float instruction requires the following SPIR-V " \
1077 "extension: SPV_EXT_shader_atomic_float" ExtName
1078static void AddAtomicVectorFloatRequirements(const MachineInstr &MI,
1079 SPIRV::RequirementHandler &Reqs,
1080 const SPIRVSubtarget &ST) {
1081 SPIRVTypeInst VecTypeDef =
1082 MI.getMF()->getRegInfo().getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1083
1084 const unsigned Rank = VecTypeDef->getOperand(i: 2).getImm();
1085 if (Rank != 2 && Rank != 4)
1086 reportFatalUsageError(reason: "Result type of an atomic vector float instruction "
1087 "must be a 2-component or 4 component vector");
1088
1089 SPIRVTypeInst EltTypeDef =
1090 MI.getMF()->getRegInfo().getVRegDef(Reg: VecTypeDef->getOperand(i: 1).getReg());
1091
1092 if (EltTypeDef->getOpcode() != SPIRV::OpTypeFloat ||
1093 EltTypeDef->getOperand(i: 1).getImm() != 16)
1094 reportFatalUsageError(
1095 reason: "The element type for the result type of an atomic vector float "
1096 "instruction must be a 16-bit floating-point scalar");
1097
1098 if (isBFloat16Type(TypeDef: EltTypeDef))
1099 reportFatalUsageError(
1100 reason: "The element type for the result type of an atomic vector float "
1101 "instruction cannot be a bfloat16 scalar");
1102 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_NV_shader_atomic_fp16_vector))
1103 reportFatalUsageError(
1104 reason: "The atomic float16 vector instruction requires the following SPIR-V "
1105 "extension: SPV_NV_shader_atomic_fp16_vector");
1106
1107 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_NV_shader_atomic_fp16_vector);
1108 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16VectorNV);
1109}
1110
1111static void AddAtomicFloatRequirements(const MachineInstr &MI,
1112 SPIRV::RequirementHandler &Reqs,
1113 const SPIRVSubtarget &ST) {
1114 assert(MI.getOperand(1).isReg() &&
1115 "Expect register operand in atomic float instruction");
1116 Register TypeReg = MI.getOperand(i: 1).getReg();
1117 SPIRVTypeInst TypeDef = MI.getMF()->getRegInfo().getVRegDef(Reg: TypeReg);
1118
1119 if (TypeDef->getOpcode() == SPIRV::OpTypeVector)
1120 return AddAtomicVectorFloatRequirements(MI, Reqs, ST);
1121
1122 if (TypeDef->getOpcode() != SPIRV::OpTypeFloat)
1123 report_fatal_error(reason: "Result type of an atomic float instruction must be a "
1124 "floating-point type scalar");
1125
1126 unsigned BitWidth = TypeDef->getOperand(i: 1).getImm();
1127 unsigned Op = MI.getOpcode();
1128 if (Op == SPIRV::OpAtomicFAddEXT) {
1129 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_shader_atomic_float_add))
1130 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("_add"), gen_crash_diag: false);
1131 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float_add);
1132 switch (BitWidth) {
1133 case 16:
1134 if (isBFloat16Type(TypeDef)) {
1135 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1136 report_fatal_error(
1137 reason: "The atomic bfloat16 instruction requires the following SPIR-V "
1138 "extension: SPV_INTEL_16bit_atomics",
1139 gen_crash_diag: false);
1140 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1141 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicBFloat16AddINTEL);
1142 } else {
1143 if (!ST.canUseExtension(
1144 E: SPIRV::Extension::SPV_EXT_shader_atomic_float16_add))
1145 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("16_add"), gen_crash_diag: false);
1146 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float16_add);
1147 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16AddEXT);
1148 }
1149 break;
1150 case 32:
1151 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat32AddEXT);
1152 break;
1153 case 64:
1154 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat64AddEXT);
1155 break;
1156 default:
1157 report_fatal_error(
1158 reason: "Unexpected floating-point type width in atomic float instruction");
1159 }
1160 } else {
1161 if (!ST.canUseExtension(
1162 E: SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max))
1163 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("_min_max"), gen_crash_diag: false);
1164 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max);
1165 switch (BitWidth) {
1166 case 16:
1167 if (isBFloat16Type(TypeDef)) {
1168 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1169 report_fatal_error(
1170 reason: "The atomic bfloat16 instruction requires the following SPIR-V "
1171 "extension: SPV_INTEL_16bit_atomics",
1172 gen_crash_diag: false);
1173 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1174 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicBFloat16MinMaxINTEL);
1175 } else {
1176 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16MinMaxEXT);
1177 }
1178 break;
1179 case 32:
1180 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat32MinMaxEXT);
1181 break;
1182 case 64:
1183 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat64MinMaxEXT);
1184 break;
1185 default:
1186 report_fatal_error(
1187 reason: "Unexpected floating-point type width in atomic float instruction");
1188 }
1189 }
1190}
1191
1192bool isUniformTexelBuffer(MachineInstr *ImageInst) {
1193 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1194 return false;
1195 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1196 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1197 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 1;
1198}
1199
1200bool isStorageTexelBuffer(MachineInstr *ImageInst) {
1201 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1202 return false;
1203 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1204 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1205 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 2;
1206}
1207
1208bool isSampledImage(MachineInstr *ImageInst) {
1209 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1210 return false;
1211 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1212 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1213 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 1;
1214}
1215
1216bool isInputAttachment(MachineInstr *ImageInst) {
1217 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1218 return false;
1219 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1220 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1221 return Dim == SPIRV::Dim::DIM_SubpassData && Sampled == 2;
1222}
1223
1224bool isStorageImage(MachineInstr *ImageInst) {
1225 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1226 return false;
1227 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1228 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1229 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 2;
1230}
1231
1232bool isCombinedImageSampler(MachineInstr *SampledImageInst) {
1233 if (SampledImageInst->getOpcode() != SPIRV::OpTypeSampledImage)
1234 return false;
1235
1236 const MachineRegisterInfo &MRI = SampledImageInst->getMF()->getRegInfo();
1237 Register ImageReg = SampledImageInst->getOperand(i: 1).getReg();
1238 auto *ImageInst = MRI.getUniqueVRegDef(Reg: ImageReg);
1239 return isSampledImage(ImageInst);
1240}
1241
1242bool hasNonUniformDecoration(Register Reg, const MachineRegisterInfo &MRI) {
1243 for (const auto &MI : MRI.reg_instructions(Reg)) {
1244 if (MI.getOpcode() != SPIRV::OpDecorate)
1245 continue;
1246
1247 uint32_t Dec = MI.getOperand(i: 1).getImm();
1248 if (Dec == SPIRV::Decoration::NonUniformEXT)
1249 return true;
1250 }
1251 return false;
1252}
1253
1254void addOpAccessChainReqs(const MachineInstr &Instr,
1255 SPIRV::RequirementHandler &Handler,
1256 const SPIRVSubtarget &Subtarget) {
1257 const MachineRegisterInfo &MRI = Instr.getMF()->getRegInfo();
1258 // Get the result type. If it is an image type, then the shader uses
1259 // descriptor indexing. The appropriate capabilities will be added based
1260 // on the specifics of the image.
1261 Register ResTypeReg = Instr.getOperand(i: 1).getReg();
1262 MachineInstr *ResTypeInst = MRI.getUniqueVRegDef(Reg: ResTypeReg);
1263
1264 assert(ResTypeInst->getOpcode() == SPIRV::OpTypePointer);
1265 uint32_t StorageClass = ResTypeInst->getOperand(i: 1).getImm();
1266 if (StorageClass != SPIRV::StorageClass::StorageClass::UniformConstant &&
1267 StorageClass != SPIRV::StorageClass::StorageClass::Uniform &&
1268 StorageClass != SPIRV::StorageClass::StorageClass::StorageBuffer) {
1269 return;
1270 }
1271
1272 bool IsNonUniform =
1273 hasNonUniformDecoration(Reg: Instr.getOperand(i: 0).getReg(), MRI);
1274
1275 auto FirstIndexReg = Instr.getOperand(i: 3).getReg();
1276 bool FirstIndexIsConstant =
1277 Subtarget.getInstrInfo()->isConstantInstr(MI: *MRI.getVRegDef(Reg: FirstIndexReg));
1278
1279 if (StorageClass == SPIRV::StorageClass::StorageClass::StorageBuffer) {
1280 if (IsNonUniform)
1281 Handler.addRequirements(
1282 Req: SPIRV::Capability::StorageBufferArrayNonUniformIndexingEXT);
1283 else if (!FirstIndexIsConstant)
1284 Handler.addRequirements(
1285 Req: SPIRV::Capability::StorageBufferArrayDynamicIndexing);
1286 return;
1287 }
1288
1289 Register PointeeTypeReg = ResTypeInst->getOperand(i: 2).getReg();
1290 MachineInstr *PointeeType = MRI.getUniqueVRegDef(Reg: PointeeTypeReg);
1291 if (PointeeType->getOpcode() != SPIRV::OpTypeImage &&
1292 PointeeType->getOpcode() != SPIRV::OpTypeSampledImage &&
1293 PointeeType->getOpcode() != SPIRV::OpTypeSampler) {
1294 return;
1295 }
1296
1297 if (isUniformTexelBuffer(ImageInst: PointeeType)) {
1298 if (IsNonUniform)
1299 Handler.addRequirements(
1300 Req: SPIRV::Capability::UniformTexelBufferArrayNonUniformIndexingEXT);
1301 else if (!FirstIndexIsConstant)
1302 Handler.addRequirements(
1303 Req: SPIRV::Capability::UniformTexelBufferArrayDynamicIndexingEXT);
1304 } else if (isInputAttachment(ImageInst: PointeeType)) {
1305 if (IsNonUniform)
1306 Handler.addRequirements(
1307 Req: SPIRV::Capability::InputAttachmentArrayNonUniformIndexingEXT);
1308 else if (!FirstIndexIsConstant)
1309 Handler.addRequirements(
1310 Req: SPIRV::Capability::InputAttachmentArrayDynamicIndexingEXT);
1311 } else if (isStorageTexelBuffer(ImageInst: PointeeType)) {
1312 if (IsNonUniform)
1313 Handler.addRequirements(
1314 Req: SPIRV::Capability::StorageTexelBufferArrayNonUniformIndexingEXT);
1315 else if (!FirstIndexIsConstant)
1316 Handler.addRequirements(
1317 Req: SPIRV::Capability::StorageTexelBufferArrayDynamicIndexingEXT);
1318 } else if (isSampledImage(ImageInst: PointeeType) ||
1319 isCombinedImageSampler(SampledImageInst: PointeeType) ||
1320 PointeeType->getOpcode() == SPIRV::OpTypeSampler) {
1321 if (IsNonUniform)
1322 Handler.addRequirements(
1323 Req: SPIRV::Capability::SampledImageArrayNonUniformIndexingEXT);
1324 else if (!FirstIndexIsConstant)
1325 Handler.addRequirements(
1326 Req: SPIRV::Capability::SampledImageArrayDynamicIndexing);
1327 } else if (isStorageImage(ImageInst: PointeeType)) {
1328 if (IsNonUniform)
1329 Handler.addRequirements(
1330 Req: SPIRV::Capability::StorageImageArrayNonUniformIndexingEXT);
1331 else if (!FirstIndexIsConstant)
1332 Handler.addRequirements(
1333 Req: SPIRV::Capability::StorageImageArrayDynamicIndexing);
1334 }
1335}
1336
1337static bool isImageTypeWithUnknownFormat(SPIRVTypeInst TypeInst) {
1338 if (TypeInst->getOpcode() != SPIRV::OpTypeImage)
1339 return false;
1340 assert(TypeInst->getOperand(7).isImm() && "The image format must be an imm.");
1341 return TypeInst->getOperand(i: 7).getImm() == 0;
1342}
1343
1344static void AddDotProductRequirements(const MachineInstr &MI,
1345 SPIRV::RequirementHandler &Reqs,
1346 const SPIRVSubtarget &ST) {
1347 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product))
1348 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_integer_dot_product);
1349 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProduct);
1350
1351 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1352 assert(MI.getOperand(2).isReg() && "Unexpected operand in dot");
1353 // We do not consider what the previous instruction is. This is just used
1354 // to get the input register and to check the type.
1355 const MachineInstr *Input = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
1356 assert(Input->getOperand(1).isReg() && "Unexpected operand in dot input");
1357 Register InputReg = Input->getOperand(i: 1).getReg();
1358
1359 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: InputReg);
1360 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {
1361 assert(TypeDef->getOperand(1).getImm() == 32);
1362 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInput4x8BitPacked);
1363 } else if (TypeDef->getOpcode() == SPIRV::OpTypeVector) {
1364 SPIRVTypeInst ScalarTypeDef =
1365 MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
1366 assert(ScalarTypeDef->getOpcode() == SPIRV::OpTypeInt);
1367 if (ScalarTypeDef->getOperand(i: 1).getImm() == 8) {
1368 assert(TypeDef->getOperand(2).getImm() == 4 &&
1369 "Dot operand of 8-bit integer type requires 4 components");
1370 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInput4x8Bit);
1371 } else {
1372 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInputAll);
1373 }
1374 }
1375}
1376
1377void addPrintfRequirements(const MachineInstr &MI,
1378 SPIRV::RequirementHandler &Reqs,
1379 const SPIRVSubtarget &ST) {
1380 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1381 SPIRVTypeInst PtrType = GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 4).getReg());
1382 if (PtrType) {
1383 MachineOperand ASOp = PtrType->getOperand(i: 1);
1384 if (ASOp.isImm()) {
1385 unsigned AddrSpace = ASOp.getImm();
1386 if (AddrSpace != SPIRV::StorageClass::UniformConstant) {
1387 if (!ST.canUseExtension(
1388 E: SPIRV::Extension::
1389 SPV_EXT_relaxed_printf_string_address_space)) {
1390 report_fatal_error(reason: "SPV_EXT_relaxed_printf_string_address_space is "
1391 "required because printf uses a format string not "
1392 "in constant address space.",
1393 gen_crash_diag: false);
1394 }
1395 Reqs.addExtension(
1396 ToAdd: SPIRV::Extension::SPV_EXT_relaxed_printf_string_address_space);
1397 }
1398 }
1399 }
1400}
1401
1402static void addImageOperandReqs(const MachineInstr &MI,
1403 SPIRV::RequirementHandler &Reqs,
1404 const SPIRVSubtarget &ST, unsigned OpIdx) {
1405 if (MI.getNumOperands() <= OpIdx)
1406 return;
1407 uint32_t Mask = MI.getOperand(i: OpIdx).getImm();
1408 for (uint32_t I = 0; I < 32; ++I)
1409 if (Mask & (1U << I))
1410 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ImageOperandOperand,
1411 i: 1U << I, ST);
1412}
1413
1414void addInstrRequirements(const MachineInstr &MI,
1415 SPIRV::ModuleAnalysisInfo &MAI,
1416 const SPIRVSubtarget &ST) {
1417 SPIRV::RequirementHandler &Reqs = MAI.Reqs;
1418 switch (MI.getOpcode()) {
1419 case SPIRV::OpMemoryModel: {
1420 int64_t Addr = MI.getOperand(i: 0).getImm();
1421 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::AddressingModelOperand,
1422 i: Addr, ST);
1423 int64_t Mem = MI.getOperand(i: 1).getImm();
1424 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::MemoryModelOperand, i: Mem,
1425 ST);
1426 break;
1427 }
1428 case SPIRV::OpEntryPoint: {
1429 int64_t Exe = MI.getOperand(i: 0).getImm();
1430 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ExecutionModelOperand,
1431 i: Exe, ST);
1432 break;
1433 }
1434 case SPIRV::OpExecutionMode:
1435 case SPIRV::OpExecutionModeId: {
1436 int64_t Exe = MI.getOperand(i: 1).getImm();
1437 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ExecutionModeOperand,
1438 i: Exe, ST);
1439 break;
1440 }
1441 case SPIRV::OpTypeMatrix:
1442 Reqs.addCapability(ToAdd: SPIRV::Capability::Matrix);
1443 break;
1444 case SPIRV::OpTypeInt: {
1445 unsigned BitWidth = MI.getOperand(i: 1).getImm();
1446 if (BitWidth == 64)
1447 Reqs.addCapability(ToAdd: SPIRV::Capability::Int64);
1448 else if (BitWidth == 16)
1449 Reqs.addCapability(ToAdd: SPIRV::Capability::Int16);
1450 else if (BitWidth == 8)
1451 Reqs.addCapability(ToAdd: SPIRV::Capability::Int8);
1452 else if (BitWidth == 4 &&
1453 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4)) {
1454 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_int4);
1455 Reqs.addCapability(ToAdd: SPIRV::Capability::Int4TypeINTEL);
1456 } else if (BitWidth != 32) {
1457 if (!ST.canUseExtension(
1458 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers))
1459 reportFatalUsageError(
1460 reason: "OpTypeInt type with a width other than 8, 16, 32 or 64 bits "
1461 "requires the following SPIR-V extension: "
1462 "SPV_ALTERA_arbitrary_precision_integers");
1463 Reqs.addExtension(
1464 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
1465 Reqs.addCapability(ToAdd: SPIRV::Capability::ArbitraryPrecisionIntegersALTERA);
1466 }
1467 break;
1468 }
1469 case SPIRV::OpDot: {
1470 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1471 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1472 if (isBFloat16Type(TypeDef))
1473 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16DotProductKHR);
1474 break;
1475 }
1476 case SPIRV::OpTypeFloat: {
1477 unsigned BitWidth = MI.getOperand(i: 1).getImm();
1478 if (BitWidth == 64)
1479 Reqs.addCapability(ToAdd: SPIRV::Capability::Float64);
1480 else if (BitWidth == 16) {
1481 if (isBFloat16Type(TypeDef: &MI)) {
1482 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_bfloat16))
1483 report_fatal_error(reason: "OpTypeFloat type with bfloat requires the "
1484 "following SPIR-V extension: SPV_KHR_bfloat16",
1485 gen_crash_diag: false);
1486 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_bfloat16);
1487 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16TypeKHR);
1488 } else {
1489 Reqs.addCapability(ToAdd: SPIRV::Capability::Float16);
1490 }
1491 }
1492 break;
1493 }
1494 case SPIRV::OpTypeVector: {
1495 unsigned NumComponents = MI.getOperand(i: 2).getImm();
1496 if (NumComponents == 8 || NumComponents == 16)
1497 Reqs.addCapability(ToAdd: SPIRV::Capability::Vector16);
1498 break;
1499 }
1500 case SPIRV::OpTypePointer: {
1501 auto SC = MI.getOperand(i: 1).getImm();
1502 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::StorageClassOperand, i: SC,
1503 ST);
1504 // If it's a type of pointer to float16 targeting OpenCL, add Float16Buffer
1505 // capability.
1506 if (ST.isShader())
1507 break;
1508 assert(MI.getOperand(2).isReg());
1509 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1510 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
1511 if ((TypeDef->getNumOperands() == 2) &&
1512 (TypeDef->getOpcode() == SPIRV::OpTypeFloat) &&
1513 (TypeDef->getOperand(i: 1).getImm() == 16))
1514 Reqs.addCapability(ToAdd: SPIRV::Capability::Float16Buffer);
1515 break;
1516 }
1517 case SPIRV::OpExtInst: {
1518 if (MI.getOperand(i: 2).getImm() ==
1519 static_cast<int64_t>(
1520 SPIRV::InstructionSet::NonSemantic_Shader_DebugInfo_100)) {
1521 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_non_semantic_info);
1522 break;
1523 }
1524 if (MI.getOperand(i: 3).getImm() ==
1525 static_cast<int64_t>(SPIRV::OpenCLExtInst::printf)) {
1526 addPrintfRequirements(MI, Reqs, ST);
1527 break;
1528 }
1529 // TODO: handle bfloat16 extended instructions when
1530 // SPV_INTEL_bfloat16_arithmetic is enabled.
1531 break;
1532 }
1533 case SPIRV::OpAliasDomainDeclINTEL:
1534 case SPIRV::OpAliasScopeDeclINTEL:
1535 case SPIRV::OpAliasScopeListDeclINTEL: {
1536 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_memory_access_aliasing);
1537 Reqs.addCapability(ToAdd: SPIRV::Capability::MemoryAccessAliasingINTEL);
1538 break;
1539 }
1540 case SPIRV::OpBitReverse:
1541 case SPIRV::OpBitFieldInsert:
1542 case SPIRV::OpBitFieldSExtract:
1543 case SPIRV::OpBitFieldUExtract:
1544 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions)) {
1545 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
1546 break;
1547 }
1548 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_bit_instructions);
1549 Reqs.addCapability(ToAdd: SPIRV::Capability::BitInstructions);
1550 break;
1551 case SPIRV::OpTypeRuntimeArray:
1552 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
1553 break;
1554 case SPIRV::OpTypeOpaque:
1555 case SPIRV::OpTypeEvent:
1556 Reqs.addCapability(ToAdd: SPIRV::Capability::Kernel);
1557 break;
1558 case SPIRV::OpTypePipe:
1559 case SPIRV::OpTypeReserveId:
1560 Reqs.addCapability(ToAdd: SPIRV::Capability::Pipes);
1561 break;
1562 case SPIRV::OpTypeDeviceEvent:
1563 case SPIRV::OpTypeQueue:
1564 case SPIRV::OpBuildNDRange:
1565 Reqs.addCapability(ToAdd: SPIRV::Capability::DeviceEnqueue);
1566 break;
1567 case SPIRV::OpDecorate:
1568 case SPIRV::OpDecorateId:
1569 case SPIRV::OpDecorateString:
1570 addOpDecorateReqs(MI, DecIndex: 1, Reqs, ST);
1571 break;
1572 case SPIRV::OpMemberDecorate:
1573 case SPIRV::OpMemberDecorateString:
1574 addOpDecorateReqs(MI, DecIndex: 2, Reqs, ST);
1575 break;
1576 case SPIRV::OpInBoundsPtrAccessChain:
1577 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
1578 break;
1579 case SPIRV::OpConstantSampler:
1580 Reqs.addCapability(ToAdd: SPIRV::Capability::LiteralSampler);
1581 break;
1582 case SPIRV::OpInBoundsAccessChain:
1583 case SPIRV::OpAccessChain:
1584 addOpAccessChainReqs(Instr: MI, Handler&: Reqs, Subtarget: ST);
1585 break;
1586 case SPIRV::OpTypeImage:
1587 addOpTypeImageReqs(MI, Reqs, ST);
1588 break;
1589 case SPIRV::OpTypeSampler:
1590 if (!ST.isShader()) {
1591 Reqs.addCapability(ToAdd: SPIRV::Capability::ImageBasic);
1592 }
1593 break;
1594 case SPIRV::OpTypeForwardPointer:
1595 // TODO: check if it's OpenCL's kernel.
1596 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
1597 break;
1598 case SPIRV::OpAtomicFlagTestAndSet:
1599 case SPIRV::OpAtomicLoad:
1600 case SPIRV::OpAtomicStore:
1601 case SPIRV::OpAtomicExchange:
1602 case SPIRV::OpAtomicCompareExchange:
1603 case SPIRV::OpAtomicIIncrement:
1604 case SPIRV::OpAtomicIDecrement:
1605 case SPIRV::OpAtomicIAdd:
1606 case SPIRV::OpAtomicISub:
1607 case SPIRV::OpAtomicUMin:
1608 case SPIRV::OpAtomicUMax:
1609 case SPIRV::OpAtomicSMin:
1610 case SPIRV::OpAtomicSMax:
1611 case SPIRV::OpAtomicAnd:
1612 case SPIRV::OpAtomicOr:
1613 case SPIRV::OpAtomicXor: {
1614 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1615 const MachineInstr *InstrPtr = &MI;
1616 if (MI.getOpcode() == SPIRV::OpAtomicStore) {
1617 assert(MI.getOperand(3).isReg());
1618 InstrPtr = MRI.getVRegDef(Reg: MI.getOperand(i: 3).getReg());
1619 assert(InstrPtr && "Unexpected type instruction for OpAtomicStore");
1620 }
1621 assert(InstrPtr->getOperand(1).isReg() && "Unexpected operand in atomic");
1622 Register TypeReg = InstrPtr->getOperand(i: 1).getReg();
1623 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: TypeReg);
1624 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {
1625 unsigned BitWidth = TypeDef->getOperand(i: 1).getImm();
1626 if (BitWidth == 64)
1627 Reqs.addCapability(ToAdd: SPIRV::Capability::Int64Atomics);
1628 }
1629 break;
1630 }
1631 case SPIRV::OpGroupNonUniformIAdd:
1632 case SPIRV::OpGroupNonUniformFAdd:
1633 case SPIRV::OpGroupNonUniformIMul:
1634 case SPIRV::OpGroupNonUniformFMul:
1635 case SPIRV::OpGroupNonUniformSMin:
1636 case SPIRV::OpGroupNonUniformUMin:
1637 case SPIRV::OpGroupNonUniformFMin:
1638 case SPIRV::OpGroupNonUniformSMax:
1639 case SPIRV::OpGroupNonUniformUMax:
1640 case SPIRV::OpGroupNonUniformFMax:
1641 case SPIRV::OpGroupNonUniformBitwiseAnd:
1642 case SPIRV::OpGroupNonUniformBitwiseOr:
1643 case SPIRV::OpGroupNonUniformBitwiseXor:
1644 case SPIRV::OpGroupNonUniformLogicalAnd:
1645 case SPIRV::OpGroupNonUniformLogicalOr:
1646 case SPIRV::OpGroupNonUniformLogicalXor: {
1647 assert(MI.getOperand(3).isImm());
1648 int64_t GroupOp = MI.getOperand(i: 3).getImm();
1649 switch (GroupOp) {
1650 case SPIRV::GroupOperation::Reduce:
1651 case SPIRV::GroupOperation::InclusiveScan:
1652 case SPIRV::GroupOperation::ExclusiveScan:
1653 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformArithmetic);
1654 break;
1655 case SPIRV::GroupOperation::ClusteredReduce:
1656 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformClustered);
1657 break;
1658 case SPIRV::GroupOperation::PartitionedReduceNV:
1659 case SPIRV::GroupOperation::PartitionedInclusiveScanNV:
1660 case SPIRV::GroupOperation::PartitionedExclusiveScanNV:
1661 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformPartitionedNV);
1662 break;
1663 }
1664 break;
1665 }
1666 case SPIRV::OpImageQueryFormat: {
1667 Register ResultReg = MI.getOperand(i: 0).getReg();
1668 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1669 static const unsigned CompareOps[] = {
1670 SPIRV::OpIEqual, SPIRV::OpINotEqual,
1671 SPIRV::OpUGreaterThan, SPIRV::OpUGreaterThanEqual,
1672 SPIRV::OpULessThan, SPIRV::OpULessThanEqual,
1673 SPIRV::OpSGreaterThan, SPIRV::OpSGreaterThanEqual,
1674 SPIRV::OpSLessThan, SPIRV::OpSLessThanEqual};
1675
1676 auto CheckAndAddExtension = [&](int64_t ImmVal) {
1677 if (ImmVal == 4323 || ImmVal == 4324) {
1678 if (ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_image_raw10_raw12))
1679 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_image_raw10_raw12);
1680 else
1681 report_fatal_error(reason: "This requires the "
1682 "SPV_EXT_image_raw10_raw12 extension");
1683 }
1684 };
1685
1686 for (MachineInstr &UseInst : MRI.use_instructions(Reg: ResultReg)) {
1687 unsigned Opc = UseInst.getOpcode();
1688
1689 if (Opc == SPIRV::OpSwitch) {
1690 for (const MachineOperand &Op : UseInst.operands())
1691 if (Op.isImm())
1692 CheckAndAddExtension(Op.getImm());
1693 } else if (llvm::is_contained(Range: CompareOps, Element: Opc)) {
1694 for (unsigned i = 1; i < UseInst.getNumOperands(); ++i) {
1695 Register UseReg = UseInst.getOperand(i).getReg();
1696 MachineInstr *ConstInst = MRI.getVRegDef(Reg: UseReg);
1697 if (ConstInst && ConstInst->getOpcode() == SPIRV::OpConstantI) {
1698 int64_t ImmVal = ConstInst->getOperand(i: 2).getImm();
1699 if (ImmVal)
1700 CheckAndAddExtension(ImmVal);
1701 }
1702 }
1703 }
1704 }
1705 break;
1706 }
1707
1708 case SPIRV::OpGroupNonUniformShuffle:
1709 case SPIRV::OpGroupNonUniformShuffleXor:
1710 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformShuffle);
1711 break;
1712 case SPIRV::OpGroupNonUniformShuffleUp:
1713 case SPIRV::OpGroupNonUniformShuffleDown:
1714 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformShuffleRelative);
1715 break;
1716 case SPIRV::OpGroupAll:
1717 case SPIRV::OpGroupAny:
1718 case SPIRV::OpGroupBroadcast:
1719 case SPIRV::OpGroupIAdd:
1720 case SPIRV::OpGroupFAdd:
1721 case SPIRV::OpGroupFMin:
1722 case SPIRV::OpGroupUMin:
1723 case SPIRV::OpGroupSMin:
1724 case SPIRV::OpGroupFMax:
1725 case SPIRV::OpGroupUMax:
1726 case SPIRV::OpGroupSMax:
1727 Reqs.addCapability(ToAdd: SPIRV::Capability::Groups);
1728 break;
1729 case SPIRV::OpGroupNonUniformElect:
1730 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniform);
1731 break;
1732 case SPIRV::OpGroupNonUniformAll:
1733 case SPIRV::OpGroupNonUniformAny:
1734 case SPIRV::OpGroupNonUniformAllEqual:
1735 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformVote);
1736 break;
1737 case SPIRV::OpGroupNonUniformBroadcast:
1738 case SPIRV::OpGroupNonUniformBroadcastFirst:
1739 case SPIRV::OpGroupNonUniformBallot:
1740 case SPIRV::OpGroupNonUniformInverseBallot:
1741 case SPIRV::OpGroupNonUniformBallotBitExtract:
1742 case SPIRV::OpGroupNonUniformBallotBitCount:
1743 case SPIRV::OpGroupNonUniformBallotFindLSB:
1744 case SPIRV::OpGroupNonUniformBallotFindMSB:
1745 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformBallot);
1746 break;
1747 case SPIRV::OpSubgroupShuffleINTEL:
1748 case SPIRV::OpSubgroupShuffleDownINTEL:
1749 case SPIRV::OpSubgroupShuffleUpINTEL:
1750 case SPIRV::OpSubgroupShuffleXorINTEL:
1751 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
1752 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
1753 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupShuffleINTEL);
1754 }
1755 break;
1756 case SPIRV::OpSubgroupBlockReadINTEL:
1757 case SPIRV::OpSubgroupBlockWriteINTEL:
1758 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
1759 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
1760 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupBufferBlockIOINTEL);
1761 }
1762 break;
1763 case SPIRV::OpSubgroupImageBlockReadINTEL:
1764 case SPIRV::OpSubgroupImageBlockWriteINTEL:
1765 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
1766 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
1767 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupImageBlockIOINTEL);
1768 }
1769 break;
1770 case SPIRV::OpSubgroupImageMediaBlockReadINTEL:
1771 case SPIRV::OpSubgroupImageMediaBlockWriteINTEL:
1772 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_media_block_io)) {
1773 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_media_block_io);
1774 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupImageMediaBlockIOINTEL);
1775 }
1776 break;
1777 case SPIRV::OpAssumeTrueKHR:
1778 case SPIRV::OpExpectKHR:
1779 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume)) {
1780 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_expect_assume);
1781 Reqs.addCapability(ToAdd: SPIRV::Capability::ExpectAssumeKHR);
1782 }
1783 break;
1784 case SPIRV::OpFmaKHR:
1785 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_fma)) {
1786 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_fma);
1787 Reqs.addCapability(ToAdd: SPIRV::Capability::FmaKHR);
1788 }
1789 break;
1790 case SPIRV::OpPtrCastToCrossWorkgroupINTEL:
1791 case SPIRV::OpCrossWorkgroupCastToPtrINTEL:
1792 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)) {
1793 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_usm_storage_classes);
1794 Reqs.addCapability(ToAdd: SPIRV::Capability::USMStorageClassesINTEL);
1795 }
1796 break;
1797 case SPIRV::OpConstantFunctionPointerINTEL:
1798 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers)) {
1799 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_function_pointers);
1800 Reqs.addCapability(ToAdd: SPIRV::Capability::FunctionPointersINTEL);
1801 }
1802 break;
1803 case SPIRV::OpGroupNonUniformRotateKHR:
1804 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_subgroup_rotate))
1805 report_fatal_error(reason: "OpGroupNonUniformRotateKHR instruction requires the "
1806 "following SPIR-V extension: SPV_KHR_subgroup_rotate",
1807 gen_crash_diag: false);
1808 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_subgroup_rotate);
1809 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformRotateKHR);
1810 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniform);
1811 break;
1812 case SPIRV::OpFixedCosALTERA:
1813 case SPIRV::OpFixedSinALTERA:
1814 case SPIRV::OpFixedCosPiALTERA:
1815 case SPIRV::OpFixedSinPiALTERA:
1816 case SPIRV::OpFixedExpALTERA:
1817 case SPIRV::OpFixedLogALTERA:
1818 case SPIRV::OpFixedRecipALTERA:
1819 case SPIRV::OpFixedSqrtALTERA:
1820 case SPIRV::OpFixedSinCosALTERA:
1821 case SPIRV::OpFixedSinCosPiALTERA:
1822 case SPIRV::OpFixedRsqrtALTERA:
1823 if (!ST.canUseExtension(
1824 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point))
1825 report_fatal_error(reason: "This instruction requires the "
1826 "following SPIR-V extension: "
1827 "SPV_ALTERA_arbitrary_precision_fixed_point",
1828 gen_crash_diag: false);
1829 Reqs.addExtension(
1830 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point);
1831 Reqs.addCapability(ToAdd: SPIRV::Capability::ArbitraryPrecisionFixedPointALTERA);
1832 break;
1833 case SPIRV::OpGroupIMulKHR:
1834 case SPIRV::OpGroupFMulKHR:
1835 case SPIRV::OpGroupBitwiseAndKHR:
1836 case SPIRV::OpGroupBitwiseOrKHR:
1837 case SPIRV::OpGroupBitwiseXorKHR:
1838 case SPIRV::OpGroupLogicalAndKHR:
1839 case SPIRV::OpGroupLogicalOrKHR:
1840 case SPIRV::OpGroupLogicalXorKHR:
1841 if (ST.canUseExtension(
1842 E: SPIRV::Extension::SPV_KHR_uniform_group_instructions)) {
1843 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_uniform_group_instructions);
1844 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupUniformArithmeticKHR);
1845 }
1846 break;
1847 case SPIRV::OpReadClockKHR:
1848 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_shader_clock))
1849 report_fatal_error(reason: "OpReadClockKHR instruction requires the "
1850 "following SPIR-V extension: SPV_KHR_shader_clock",
1851 gen_crash_diag: false);
1852 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_shader_clock);
1853 Reqs.addCapability(ToAdd: SPIRV::Capability::ShaderClockKHR);
1854 break;
1855 case SPIRV::OpFunctionPointerCallINTEL:
1856 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers)) {
1857 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_function_pointers);
1858 Reqs.addCapability(ToAdd: SPIRV::Capability::FunctionPointersINTEL);
1859 }
1860 break;
1861 case SPIRV::OpAtomicFAddEXT:
1862 case SPIRV::OpAtomicFMinEXT:
1863 case SPIRV::OpAtomicFMaxEXT:
1864 AddAtomicFloatRequirements(MI, Reqs, ST);
1865 break;
1866 case SPIRV::OpConvertBF16ToFINTEL:
1867 case SPIRV::OpConvertFToBF16INTEL:
1868 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bfloat16_conversion)) {
1869 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_conversion);
1870 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ConversionINTEL);
1871 }
1872 break;
1873 case SPIRV::OpRoundFToTF32INTEL:
1874 if (ST.canUseExtension(
1875 E: SPIRV::Extension::SPV_INTEL_tensor_float32_conversion)) {
1876 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_tensor_float32_conversion);
1877 Reqs.addCapability(ToAdd: SPIRV::Capability::TensorFloat32RoundingINTEL);
1878 }
1879 break;
1880 case SPIRV::OpVariableLengthArrayINTEL:
1881 case SPIRV::OpSaveMemoryINTEL:
1882 case SPIRV::OpRestoreMemoryINTEL:
1883 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array)) {
1884 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_variable_length_array);
1885 Reqs.addCapability(ToAdd: SPIRV::Capability::VariableLengthArrayINTEL);
1886 }
1887 break;
1888 case SPIRV::OpAsmTargetINTEL:
1889 case SPIRV::OpAsmINTEL:
1890 case SPIRV::OpAsmCallINTEL:
1891 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_inline_assembly)) {
1892 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_inline_assembly);
1893 Reqs.addCapability(ToAdd: SPIRV::Capability::AsmINTEL);
1894 }
1895 break;
1896 case SPIRV::OpTypeCooperativeMatrixKHR: {
1897 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
1898 report_fatal_error(
1899 reason: "OpTypeCooperativeMatrixKHR type requires the "
1900 "following SPIR-V extension: SPV_KHR_cooperative_matrix",
1901 gen_crash_diag: false);
1902 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
1903 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
1904 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1905 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1906 if (isBFloat16Type(TypeDef))
1907 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16CooperativeMatrixKHR);
1908 break;
1909 }
1910 case SPIRV::OpArithmeticFenceEXT:
1911 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_arithmetic_fence))
1912 report_fatal_error(reason: "OpArithmeticFenceEXT requires the "
1913 "following SPIR-V extension: SPV_EXT_arithmetic_fence",
1914 gen_crash_diag: false);
1915 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_arithmetic_fence);
1916 Reqs.addCapability(ToAdd: SPIRV::Capability::ArithmeticFenceEXT);
1917 break;
1918 case SPIRV::OpControlBarrierArriveINTEL:
1919 case SPIRV::OpControlBarrierWaitINTEL:
1920 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_split_barrier)) {
1921 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_split_barrier);
1922 Reqs.addCapability(ToAdd: SPIRV::Capability::SplitBarrierINTEL);
1923 }
1924 break;
1925 case SPIRV::OpCooperativeMatrixMulAddKHR: {
1926 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
1927 report_fatal_error(reason: "Cooperative matrix instructions require the "
1928 "following SPIR-V extension: "
1929 "SPV_KHR_cooperative_matrix",
1930 gen_crash_diag: false);
1931 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
1932 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
1933 constexpr unsigned MulAddMaxSize = 6;
1934 if (MI.getNumOperands() != MulAddMaxSize)
1935 break;
1936 const int64_t CoopOperands = MI.getOperand(i: MulAddMaxSize - 1).getImm();
1937 if (CoopOperands &
1938 SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) {
1939 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
1940 report_fatal_error(reason: "MatrixAAndBTF32ComponentsINTEL type interpretation "
1941 "require the following SPIR-V extension: "
1942 "SPV_INTEL_joint_matrix",
1943 gen_crash_diag: false);
1944 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
1945 Reqs.addCapability(
1946 ToAdd: SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL);
1947 }
1948 if (CoopOperands & SPIRV::CooperativeMatrixOperands::
1949 MatrixAAndBBFloat16ComponentsINTEL ||
1950 CoopOperands &
1951 SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL ||
1952 CoopOperands & SPIRV::CooperativeMatrixOperands::
1953 MatrixResultBFloat16ComponentsINTEL) {
1954 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
1955 report_fatal_error(reason: "***BF16ComponentsINTEL type interpretations "
1956 "require the following SPIR-V extension: "
1957 "SPV_INTEL_joint_matrix",
1958 gen_crash_diag: false);
1959 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
1960 Reqs.addCapability(
1961 ToAdd: SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);
1962 }
1963 break;
1964 }
1965 case SPIRV::OpCooperativeMatrixLoadKHR:
1966 case SPIRV::OpCooperativeMatrixStoreKHR:
1967 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
1968 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
1969 case SPIRV::OpCooperativeMatrixPrefetchINTEL: {
1970 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
1971 report_fatal_error(reason: "Cooperative matrix instructions require the "
1972 "following SPIR-V extension: "
1973 "SPV_KHR_cooperative_matrix",
1974 gen_crash_diag: false);
1975 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
1976 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
1977
1978 // Check Layout operand in case if it's not a standard one and add the
1979 // appropriate capability.
1980 std::unordered_map<unsigned, unsigned> LayoutToInstMap = {
1981 {SPIRV::OpCooperativeMatrixLoadKHR, 3},
1982 {SPIRV::OpCooperativeMatrixStoreKHR, 2},
1983 {SPIRV::OpCooperativeMatrixLoadCheckedINTEL, 5},
1984 {SPIRV::OpCooperativeMatrixStoreCheckedINTEL, 4},
1985 {SPIRV::OpCooperativeMatrixPrefetchINTEL, 4}};
1986
1987 const auto OpCode = MI.getOpcode();
1988 const unsigned LayoutNum = LayoutToInstMap[OpCode];
1989 Register RegLayout = MI.getOperand(i: LayoutNum).getReg();
1990 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1991 MachineInstr *MILayout = MRI.getUniqueVRegDef(Reg: RegLayout);
1992 if (MILayout->getOpcode() == SPIRV::OpConstantI) {
1993 const unsigned LayoutVal = MILayout->getOperand(i: 2).getImm();
1994 if (LayoutVal ==
1995 static_cast<unsigned>(SPIRV::CooperativeMatrixLayout::PackedINTEL)) {
1996 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
1997 report_fatal_error(reason: "PackedINTEL layout require the following SPIR-V "
1998 "extension: SPV_INTEL_joint_matrix",
1999 gen_crash_diag: false);
2000 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2001 Reqs.addCapability(ToAdd: SPIRV::Capability::PackedCooperativeMatrixINTEL);
2002 }
2003 }
2004
2005 // Nothing to do.
2006 if (OpCode == SPIRV::OpCooperativeMatrixLoadKHR ||
2007 OpCode == SPIRV::OpCooperativeMatrixStoreKHR)
2008 break;
2009
2010 std::string InstName;
2011 switch (OpCode) {
2012 case SPIRV::OpCooperativeMatrixPrefetchINTEL:
2013 InstName = "OpCooperativeMatrixPrefetchINTEL";
2014 break;
2015 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2016 InstName = "OpCooperativeMatrixLoadCheckedINTEL";
2017 break;
2018 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2019 InstName = "OpCooperativeMatrixStoreCheckedINTEL";
2020 break;
2021 }
2022
2023 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix)) {
2024 const std::string ErrorMsg =
2025 InstName + " instruction requires the "
2026 "following SPIR-V extension: SPV_INTEL_joint_matrix";
2027 report_fatal_error(reason: ErrorMsg.c_str(), gen_crash_diag: false);
2028 }
2029 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2030 if (OpCode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
2031 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixPrefetchINTEL);
2032 break;
2033 }
2034 Reqs.addCapability(
2035 ToAdd: SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
2036 break;
2037 }
2038 case SPIRV::OpCooperativeMatrixConstructCheckedINTEL:
2039 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2040 report_fatal_error(reason: "OpCooperativeMatrixConstructCheckedINTEL "
2041 "instructions require the following SPIR-V extension: "
2042 "SPV_INTEL_joint_matrix",
2043 gen_crash_diag: false);
2044 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2045 Reqs.addCapability(
2046 ToAdd: SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
2047 break;
2048 case SPIRV::OpReadPipeBlockingALTERA:
2049 case SPIRV::OpWritePipeBlockingALTERA:
2050 if (ST.canUseExtension(E: SPIRV::Extension::SPV_ALTERA_blocking_pipes)) {
2051 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_ALTERA_blocking_pipes);
2052 Reqs.addCapability(ToAdd: SPIRV::Capability::BlockingPipesALTERA);
2053 }
2054 break;
2055 case SPIRV::OpCooperativeMatrixGetElementCoordINTEL:
2056 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2057 report_fatal_error(reason: "OpCooperativeMatrixGetElementCoordINTEL requires the "
2058 "following SPIR-V extension: SPV_INTEL_joint_matrix",
2059 gen_crash_diag: false);
2060 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2061 Reqs.addCapability(
2062 ToAdd: SPIRV::Capability::CooperativeMatrixInvocationInstructionsINTEL);
2063 break;
2064 case SPIRV::OpConvertHandleToImageINTEL:
2065 case SPIRV::OpConvertHandleToSamplerINTEL:
2066 case SPIRV::OpConvertHandleToSampledImageINTEL: {
2067 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bindless_images))
2068 report_fatal_error(reason: "OpConvertHandleTo[Image/Sampler/SampledImage]INTEL "
2069 "instructions require the following SPIR-V extension: "
2070 "SPV_INTEL_bindless_images",
2071 gen_crash_diag: false);
2072 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
2073 SPIRV::AddressingModel::AddressingModel AddrModel = MAI.Addr;
2074 SPIRVTypeInst TyDef = GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg());
2075 if (MI.getOpcode() == SPIRV::OpConvertHandleToImageINTEL &&
2076 TyDef->getOpcode() != SPIRV::OpTypeImage) {
2077 report_fatal_error(reason: "Incorrect return type for the instruction "
2078 "OpConvertHandleToImageINTEL",
2079 gen_crash_diag: false);
2080 } else if (MI.getOpcode() == SPIRV::OpConvertHandleToSamplerINTEL &&
2081 TyDef->getOpcode() != SPIRV::OpTypeSampler) {
2082 report_fatal_error(reason: "Incorrect return type for the instruction "
2083 "OpConvertHandleToSamplerINTEL",
2084 gen_crash_diag: false);
2085 } else if (MI.getOpcode() == SPIRV::OpConvertHandleToSampledImageINTEL &&
2086 TyDef->getOpcode() != SPIRV::OpTypeSampledImage) {
2087 report_fatal_error(reason: "Incorrect return type for the instruction "
2088 "OpConvertHandleToSampledImageINTEL",
2089 gen_crash_diag: false);
2090 }
2091 SPIRVTypeInst SpvTy = GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 2).getReg());
2092 unsigned Bitwidth = GR->getScalarOrVectorBitWidth(Type: SpvTy);
2093 if (!(Bitwidth == 32 && AddrModel == SPIRV::AddressingModel::Physical32) &&
2094 !(Bitwidth == 64 && AddrModel == SPIRV::AddressingModel::Physical64)) {
2095 report_fatal_error(
2096 reason: "Parameter value must be a 32-bit scalar in case of "
2097 "Physical32 addressing model or a 64-bit scalar in case of "
2098 "Physical64 addressing model",
2099 gen_crash_diag: false);
2100 }
2101 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bindless_images);
2102 Reqs.addCapability(ToAdd: SPIRV::Capability::BindlessImagesINTEL);
2103 break;
2104 }
2105 case SPIRV::OpSubgroup2DBlockLoadINTEL:
2106 case SPIRV::OpSubgroup2DBlockLoadTransposeINTEL:
2107 case SPIRV::OpSubgroup2DBlockLoadTransformINTEL:
2108 case SPIRV::OpSubgroup2DBlockPrefetchINTEL:
2109 case SPIRV::OpSubgroup2DBlockStoreINTEL: {
2110 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_2d_block_io))
2111 report_fatal_error(reason: "OpSubgroup2DBlock[Load/LoadTranspose/LoadTransform/"
2112 "Prefetch/Store]INTEL instructions require the "
2113 "following SPIR-V extension: SPV_INTEL_2d_block_io",
2114 gen_crash_diag: false);
2115 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_2d_block_io);
2116 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockIOINTEL);
2117
2118 const auto OpCode = MI.getOpcode();
2119 if (OpCode == SPIRV::OpSubgroup2DBlockLoadTransposeINTEL) {
2120 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockTransposeINTEL);
2121 break;
2122 }
2123 if (OpCode == SPIRV::OpSubgroup2DBlockLoadTransformINTEL) {
2124 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockTransformINTEL);
2125 break;
2126 }
2127 break;
2128 }
2129 case SPIRV::OpKill: {
2130 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2131 } break;
2132 case SPIRV::OpDemoteToHelperInvocation:
2133 Reqs.addCapability(ToAdd: SPIRV::Capability::DemoteToHelperInvocation);
2134
2135 if (ST.canUseExtension(
2136 E: SPIRV::Extension::SPV_EXT_demote_to_helper_invocation)) {
2137 if (!ST.isAtLeastSPIRVVer(VerToCompareTo: llvm::VersionTuple(1, 6)))
2138 Reqs.addExtension(
2139 ToAdd: SPIRV::Extension::SPV_EXT_demote_to_helper_invocation);
2140 }
2141 break;
2142 case SPIRV::OpSDot:
2143 case SPIRV::OpUDot:
2144 case SPIRV::OpSUDot:
2145 case SPIRV::OpSDotAccSat:
2146 case SPIRV::OpUDotAccSat:
2147 case SPIRV::OpSUDotAccSat:
2148 AddDotProductRequirements(MI, Reqs, ST);
2149 break;
2150 case SPIRV::OpImageSampleImplicitLod:
2151 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2152 addImageOperandReqs(MI, Reqs, ST, OpIdx: 4);
2153 break;
2154 case SPIRV::OpImageSampleExplicitLod:
2155 addImageOperandReqs(MI, Reqs, ST, OpIdx: 4);
2156 break;
2157 case SPIRV::OpImageSampleDrefImplicitLod:
2158 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2159 addImageOperandReqs(MI, Reqs, ST, OpIdx: 5);
2160 break;
2161 case SPIRV::OpImageSampleDrefExplicitLod:
2162 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2163 addImageOperandReqs(MI, Reqs, ST, OpIdx: 5);
2164 break;
2165 case SPIRV::OpImageRead: {
2166 Register ImageReg = MI.getOperand(i: 2).getReg();
2167 SPIRVTypeInst TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(
2168 VReg: ImageReg, MF: const_cast<MachineFunction *>(MI.getMF()));
2169 // OpImageRead and OpImageWrite can use Unknown Image Formats
2170 // when the Kernel capability is declared. In the OpenCL environment we are
2171 // not allowed to produce
2172 // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see
2173 // https://github.com/KhronosGroup/SPIRV-Headers/issues/487
2174
2175 if (isImageTypeWithUnknownFormat(TypeInst: TypeDef) && ST.isShader())
2176 Reqs.addCapability(ToAdd: SPIRV::Capability::StorageImageReadWithoutFormat);
2177 break;
2178 }
2179 case SPIRV::OpImageWrite: {
2180 Register ImageReg = MI.getOperand(i: 0).getReg();
2181 SPIRVTypeInst TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(
2182 VReg: ImageReg, MF: const_cast<MachineFunction *>(MI.getMF()));
2183 // OpImageRead and OpImageWrite can use Unknown Image Formats
2184 // when the Kernel capability is declared. In the OpenCL environment we are
2185 // not allowed to produce
2186 // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see
2187 // https://github.com/KhronosGroup/SPIRV-Headers/issues/487
2188
2189 if (isImageTypeWithUnknownFormat(TypeInst: TypeDef) && ST.isShader())
2190 Reqs.addCapability(ToAdd: SPIRV::Capability::StorageImageWriteWithoutFormat);
2191 break;
2192 }
2193 case SPIRV::OpTypeStructContinuedINTEL:
2194 case SPIRV::OpConstantCompositeContinuedINTEL:
2195 case SPIRV::OpSpecConstantCompositeContinuedINTEL:
2196 case SPIRV::OpCompositeConstructContinuedINTEL: {
2197 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_long_composites))
2198 report_fatal_error(
2199 reason: "Continued instructions require the "
2200 "following SPIR-V extension: SPV_INTEL_long_composites",
2201 gen_crash_diag: false);
2202 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_long_composites);
2203 Reqs.addCapability(ToAdd: SPIRV::Capability::LongCompositesINTEL);
2204 break;
2205 }
2206 case SPIRV::OpArbitraryFloatEQALTERA:
2207 case SPIRV::OpArbitraryFloatGEALTERA:
2208 case SPIRV::OpArbitraryFloatGTALTERA:
2209 case SPIRV::OpArbitraryFloatLEALTERA:
2210 case SPIRV::OpArbitraryFloatLTALTERA:
2211 case SPIRV::OpArbitraryFloatCbrtALTERA:
2212 case SPIRV::OpArbitraryFloatCosALTERA:
2213 case SPIRV::OpArbitraryFloatCosPiALTERA:
2214 case SPIRV::OpArbitraryFloatExp10ALTERA:
2215 case SPIRV::OpArbitraryFloatExp2ALTERA:
2216 case SPIRV::OpArbitraryFloatExpALTERA:
2217 case SPIRV::OpArbitraryFloatExpm1ALTERA:
2218 case SPIRV::OpArbitraryFloatHypotALTERA:
2219 case SPIRV::OpArbitraryFloatLog10ALTERA:
2220 case SPIRV::OpArbitraryFloatLog1pALTERA:
2221 case SPIRV::OpArbitraryFloatLog2ALTERA:
2222 case SPIRV::OpArbitraryFloatLogALTERA:
2223 case SPIRV::OpArbitraryFloatRecipALTERA:
2224 case SPIRV::OpArbitraryFloatSinCosALTERA:
2225 case SPIRV::OpArbitraryFloatSinCosPiALTERA:
2226 case SPIRV::OpArbitraryFloatSinALTERA:
2227 case SPIRV::OpArbitraryFloatSinPiALTERA:
2228 case SPIRV::OpArbitraryFloatSqrtALTERA:
2229 case SPIRV::OpArbitraryFloatACosALTERA:
2230 case SPIRV::OpArbitraryFloatACosPiALTERA:
2231 case SPIRV::OpArbitraryFloatAddALTERA:
2232 case SPIRV::OpArbitraryFloatASinALTERA:
2233 case SPIRV::OpArbitraryFloatASinPiALTERA:
2234 case SPIRV::OpArbitraryFloatATan2ALTERA:
2235 case SPIRV::OpArbitraryFloatATanALTERA:
2236 case SPIRV::OpArbitraryFloatATanPiALTERA:
2237 case SPIRV::OpArbitraryFloatCastFromIntALTERA:
2238 case SPIRV::OpArbitraryFloatCastALTERA:
2239 case SPIRV::OpArbitraryFloatCastToIntALTERA:
2240 case SPIRV::OpArbitraryFloatDivALTERA:
2241 case SPIRV::OpArbitraryFloatMulALTERA:
2242 case SPIRV::OpArbitraryFloatPowALTERA:
2243 case SPIRV::OpArbitraryFloatPowNALTERA:
2244 case SPIRV::OpArbitraryFloatPowRALTERA:
2245 case SPIRV::OpArbitraryFloatRSqrtALTERA:
2246 case SPIRV::OpArbitraryFloatSubALTERA: {
2247 if (!ST.canUseExtension(
2248 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_floating_point))
2249 report_fatal_error(
2250 reason: "Floating point instructions can't be translated correctly without "
2251 "enabled SPV_ALTERA_arbitrary_precision_floating_point extension!",
2252 gen_crash_diag: false);
2253 Reqs.addExtension(
2254 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_floating_point);
2255 Reqs.addCapability(
2256 ToAdd: SPIRV::Capability::ArbitraryPrecisionFloatingPointALTERA);
2257 break;
2258 }
2259 case SPIRV::OpSubgroupMatrixMultiplyAccumulateINTEL: {
2260 if (!ST.canUseExtension(
2261 E: SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate))
2262 report_fatal_error(
2263 reason: "OpSubgroupMatrixMultiplyAccumulateINTEL instruction requires the "
2264 "following SPIR-V "
2265 "extension: SPV_INTEL_subgroup_matrix_multiply_accumulate",
2266 gen_crash_diag: false);
2267 Reqs.addExtension(
2268 ToAdd: SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate);
2269 Reqs.addCapability(
2270 ToAdd: SPIRV::Capability::SubgroupMatrixMultiplyAccumulateINTEL);
2271 break;
2272 }
2273 case SPIRV::OpBitwiseFunctionINTEL: {
2274 if (!ST.canUseExtension(
2275 E: SPIRV::Extension::SPV_INTEL_ternary_bitwise_function))
2276 report_fatal_error(
2277 reason: "OpBitwiseFunctionINTEL instruction requires the following SPIR-V "
2278 "extension: SPV_INTEL_ternary_bitwise_function",
2279 gen_crash_diag: false);
2280 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_ternary_bitwise_function);
2281 Reqs.addCapability(ToAdd: SPIRV::Capability::TernaryBitwiseFunctionINTEL);
2282 break;
2283 }
2284 case SPIRV::OpCopyMemorySized: {
2285 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
2286 // TODO: Add UntypedPointersKHR when implemented.
2287 break;
2288 }
2289 case SPIRV::OpPredicatedLoadINTEL:
2290 case SPIRV::OpPredicatedStoreINTEL: {
2291 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_predicated_io))
2292 report_fatal_error(
2293 reason: "OpPredicated[Load/Store]INTEL instructions require "
2294 "the following SPIR-V extension: SPV_INTEL_predicated_io",
2295 gen_crash_diag: false);
2296 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_predicated_io);
2297 Reqs.addCapability(ToAdd: SPIRV::Capability::PredicatedIOINTEL);
2298 break;
2299 }
2300 case SPIRV::OpFAddS:
2301 case SPIRV::OpFSubS:
2302 case SPIRV::OpFMulS:
2303 case SPIRV::OpFDivS:
2304 case SPIRV::OpFRemS:
2305 case SPIRV::OpFMod:
2306 case SPIRV::OpFNegate:
2307 case SPIRV::OpFAddV:
2308 case SPIRV::OpFSubV:
2309 case SPIRV::OpFMulV:
2310 case SPIRV::OpFDivV:
2311 case SPIRV::OpFRemV:
2312 case SPIRV::OpFNegateV: {
2313 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2314 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
2315 if (TypeDef->getOpcode() == SPIRV::OpTypeVector)
2316 TypeDef = MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
2317 if (isBFloat16Type(TypeDef)) {
2318 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic))
2319 report_fatal_error(
2320 reason: "Arithmetic instructions with bfloat16 arguments require the "
2321 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic",
2322 gen_crash_diag: false);
2323 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);
2324 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ArithmeticINTEL);
2325 }
2326 break;
2327 }
2328 case SPIRV::OpOrdered:
2329 case SPIRV::OpUnordered:
2330 case SPIRV::OpFOrdEqual:
2331 case SPIRV::OpFOrdNotEqual:
2332 case SPIRV::OpFOrdLessThan:
2333 case SPIRV::OpFOrdLessThanEqual:
2334 case SPIRV::OpFOrdGreaterThan:
2335 case SPIRV::OpFOrdGreaterThanEqual:
2336 case SPIRV::OpFUnordEqual:
2337 case SPIRV::OpFUnordNotEqual:
2338 case SPIRV::OpFUnordLessThan:
2339 case SPIRV::OpFUnordLessThanEqual:
2340 case SPIRV::OpFUnordGreaterThan:
2341 case SPIRV::OpFUnordGreaterThanEqual: {
2342 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2343 MachineInstr *OperandDef = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
2344 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: OperandDef->getOperand(i: 1).getReg());
2345 if (TypeDef->getOpcode() == SPIRV::OpTypeVector)
2346 TypeDef = MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
2347 if (isBFloat16Type(TypeDef)) {
2348 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic))
2349 report_fatal_error(
2350 reason: "Relational instructions with bfloat16 arguments require the "
2351 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic",
2352 gen_crash_diag: false);
2353 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);
2354 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ArithmeticINTEL);
2355 }
2356 break;
2357 }
2358 case SPIRV::OpDPdxCoarse:
2359 case SPIRV::OpDPdyCoarse:
2360 case SPIRV::OpDPdxFine:
2361 case SPIRV::OpDPdyFine: {
2362 Reqs.addCapability(ToAdd: SPIRV::Capability::DerivativeControl);
2363 break;
2364 }
2365 case SPIRV::OpLoopControlINTEL: {
2366 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls);
2367 Reqs.addCapability(ToAdd: SPIRV::Capability::UnstructuredLoopControlsINTEL);
2368 break;
2369 }
2370
2371 default:
2372 break;
2373 }
2374
2375 // If we require capability Shader, then we can remove the requirement for
2376 // the BitInstructions capability, since Shader is a superset capability
2377 // of BitInstructions.
2378 Reqs.removeCapabilityIf(ToRemove: SPIRV::Capability::BitInstructions,
2379 IfPresent: SPIRV::Capability::Shader);
2380}
2381
2382static void collectReqs(const Module &M, SPIRV::ModuleAnalysisInfo &MAI,
2383 MachineModuleInfo *MMI, const SPIRVSubtarget &ST) {
2384 // Collect requirements for existing instructions.
2385 for (const Function &F : M) {
2386 MachineFunction *MF = MMI->getMachineFunction(F);
2387 if (!MF)
2388 continue;
2389 for (const MachineBasicBlock &MBB : *MF)
2390 for (const MachineInstr &MI : MBB)
2391 addInstrRequirements(MI, MAI, ST);
2392 }
2393 // Collect requirements for OpExecutionMode instructions.
2394 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
2395 if (Node) {
2396 bool RequireFloatControls = false, RequireIntelFloatControls2 = false,
2397 RequireKHRFloatControls2 = false,
2398 VerLower14 = !ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 4));
2399 bool HasIntelFloatControls2 =
2400 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_float_controls2);
2401 bool HasKHRFloatControls2 =
2402 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2403 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
2404 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
2405 const MDOperand &MDOp = MDN->getOperand(I: 1);
2406 if (auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: MDOp)) {
2407 Constant *C = CMeta->getValue();
2408 if (ConstantInt *Const = dyn_cast<ConstantInt>(Val: C)) {
2409 auto EM = Const->getZExtValue();
2410 // SPV_KHR_float_controls is not available until v1.4:
2411 // add SPV_KHR_float_controls if the version is too low
2412 switch (EM) {
2413 case SPIRV::ExecutionMode::DenormPreserve:
2414 case SPIRV::ExecutionMode::DenormFlushToZero:
2415 case SPIRV::ExecutionMode::RoundingModeRTE:
2416 case SPIRV::ExecutionMode::RoundingModeRTZ:
2417 RequireFloatControls = VerLower14;
2418 MAI.Reqs.getAndAddRequirements(
2419 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2420 break;
2421 case SPIRV::ExecutionMode::RoundingModeRTPINTEL:
2422 case SPIRV::ExecutionMode::RoundingModeRTNINTEL:
2423 case SPIRV::ExecutionMode::FloatingPointModeALTINTEL:
2424 case SPIRV::ExecutionMode::FloatingPointModeIEEEINTEL:
2425 if (HasIntelFloatControls2) {
2426 RequireIntelFloatControls2 = true;
2427 MAI.Reqs.getAndAddRequirements(
2428 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2429 }
2430 break;
2431 case SPIRV::ExecutionMode::FPFastMathDefault: {
2432 if (HasKHRFloatControls2) {
2433 RequireKHRFloatControls2 = true;
2434 MAI.Reqs.getAndAddRequirements(
2435 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2436 }
2437 break;
2438 }
2439 case SPIRV::ExecutionMode::ContractionOff:
2440 case SPIRV::ExecutionMode::SignedZeroInfNanPreserve:
2441 if (HasKHRFloatControls2) {
2442 RequireKHRFloatControls2 = true;
2443 MAI.Reqs.getAndAddRequirements(
2444 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2445 i: SPIRV::ExecutionMode::FPFastMathDefault, ST);
2446 } else {
2447 MAI.Reqs.getAndAddRequirements(
2448 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2449 }
2450 break;
2451 default:
2452 MAI.Reqs.getAndAddRequirements(
2453 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2454 }
2455 }
2456 }
2457 }
2458 if (RequireFloatControls &&
2459 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls))
2460 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls);
2461 if (RequireIntelFloatControls2)
2462 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_float_controls2);
2463 if (RequireKHRFloatControls2)
2464 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls2);
2465 }
2466 for (const Function &F : M) {
2467 if (F.isDeclaration())
2468 continue;
2469 if (F.getMetadata(Kind: "reqd_work_group_size"))
2470 MAI.Reqs.getAndAddRequirements(
2471 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2472 i: SPIRV::ExecutionMode::LocalSize, ST);
2473 if (F.getFnAttribute(Kind: "hlsl.numthreads").isValid()) {
2474 MAI.Reqs.getAndAddRequirements(
2475 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2476 i: SPIRV::ExecutionMode::LocalSize, ST);
2477 }
2478 if (F.getFnAttribute(Kind: "enable-maximal-reconvergence").getValueAsBool()) {
2479 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_maximal_reconvergence);
2480 }
2481 if (F.getMetadata(Kind: "work_group_size_hint"))
2482 MAI.Reqs.getAndAddRequirements(
2483 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2484 i: SPIRV::ExecutionMode::LocalSizeHint, ST);
2485 if (F.getMetadata(Kind: "intel_reqd_sub_group_size"))
2486 MAI.Reqs.getAndAddRequirements(
2487 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2488 i: SPIRV::ExecutionMode::SubgroupSize, ST);
2489 if (F.getMetadata(Kind: "max_work_group_size"))
2490 MAI.Reqs.getAndAddRequirements(
2491 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2492 i: SPIRV::ExecutionMode::MaxWorkgroupSizeINTEL, ST);
2493 if (F.getMetadata(Kind: "vec_type_hint"))
2494 MAI.Reqs.getAndAddRequirements(
2495 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2496 i: SPIRV::ExecutionMode::VecTypeHint, ST);
2497
2498 if (F.hasOptNone()) {
2499 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_optnone)) {
2500 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_optnone);
2501 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::OptNoneINTEL);
2502 } else if (ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_optnone)) {
2503 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_optnone);
2504 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::OptNoneEXT);
2505 }
2506 }
2507 }
2508}
2509
2510static unsigned getFastMathFlags(const MachineInstr &I,
2511 const SPIRVSubtarget &ST) {
2512 unsigned Flags = SPIRV::FPFastMathMode::None;
2513 bool CanUseKHRFloatControls2 =
2514 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2515 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNoNans))
2516 Flags |= SPIRV::FPFastMathMode::NotNaN;
2517 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNoInfs))
2518 Flags |= SPIRV::FPFastMathMode::NotInf;
2519 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNsz))
2520 Flags |= SPIRV::FPFastMathMode::NSZ;
2521 if (I.getFlag(Flag: MachineInstr::MIFlag::FmArcp))
2522 Flags |= SPIRV::FPFastMathMode::AllowRecip;
2523 if (I.getFlag(Flag: MachineInstr::MIFlag::FmContract) && CanUseKHRFloatControls2)
2524 Flags |= SPIRV::FPFastMathMode::AllowContract;
2525 if (I.getFlag(Flag: MachineInstr::MIFlag::FmReassoc)) {
2526 if (CanUseKHRFloatControls2)
2527 // LLVM reassoc maps to SPIRV transform, see
2528 // https://github.com/KhronosGroup/SPIRV-Registry/issues/326 for details.
2529 // Because we are enabling AllowTransform, we must enable AllowReassoc and
2530 // AllowContract too, as required by SPIRV spec. Also, we used to map
2531 // MIFlag::FmReassoc to FPFastMathMode::Fast, which now should instead by
2532 // replaced by turning all the other bits instead. Therefore, we're
2533 // enabling every bit here except None and Fast.
2534 Flags |= SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
2535 SPIRV::FPFastMathMode::NSZ | SPIRV::FPFastMathMode::AllowRecip |
2536 SPIRV::FPFastMathMode::AllowTransform |
2537 SPIRV::FPFastMathMode::AllowReassoc |
2538 SPIRV::FPFastMathMode::AllowContract;
2539 else
2540 Flags |= SPIRV::FPFastMathMode::Fast;
2541 }
2542
2543 if (CanUseKHRFloatControls2) {
2544 // Error out if SPIRV::FPFastMathMode::Fast is enabled.
2545 assert(!(Flags & SPIRV::FPFastMathMode::Fast) &&
2546 "SPIRV::FPFastMathMode::Fast is deprecated and should not be used "
2547 "anymore.");
2548
2549 // Error out if AllowTransform is enabled without AllowReassoc and
2550 // AllowContract.
2551 assert((!(Flags & SPIRV::FPFastMathMode::AllowTransform) ||
2552 ((Flags & SPIRV::FPFastMathMode::AllowReassoc &&
2553 Flags & SPIRV::FPFastMathMode::AllowContract))) &&
2554 "SPIRV::FPFastMathMode::AllowTransform requires AllowReassoc and "
2555 "AllowContract flags to be enabled as well.");
2556 }
2557
2558 return Flags;
2559}
2560
2561static bool isFastMathModeAvailable(const SPIRVSubtarget &ST) {
2562 if (ST.isKernel())
2563 return true;
2564 if (ST.getSPIRVVersion() < VersionTuple(1, 2))
2565 return false;
2566 return ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2567}
2568
2569static void handleMIFlagDecoration(
2570 MachineInstr &I, const SPIRVSubtarget &ST, const SPIRVInstrInfo &TII,
2571 SPIRV::RequirementHandler &Reqs, const SPIRVGlobalRegistry *GR,
2572 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec) {
2573 if (I.getFlag(Flag: MachineInstr::MIFlag::NoSWrap) && TII.canUseNSW(MI: I) &&
2574 getSymbolicOperandRequirements(Category: SPIRV::OperandCategory::DecorationOperand,
2575 i: SPIRV::Decoration::NoSignedWrap, ST, Reqs)
2576 .IsSatisfiable) {
2577 buildOpDecorate(Reg: I.getOperand(i: 0).getReg(), I, TII,
2578 Dec: SPIRV::Decoration::NoSignedWrap, DecArgs: {});
2579 }
2580 if (I.getFlag(Flag: MachineInstr::MIFlag::NoUWrap) && TII.canUseNUW(MI: I) &&
2581 getSymbolicOperandRequirements(Category: SPIRV::OperandCategory::DecorationOperand,
2582 i: SPIRV::Decoration::NoUnsignedWrap, ST,
2583 Reqs)
2584 .IsSatisfiable) {
2585 buildOpDecorate(Reg: I.getOperand(i: 0).getReg(), I, TII,
2586 Dec: SPIRV::Decoration::NoUnsignedWrap, DecArgs: {});
2587 }
2588 if (!TII.canUseFastMathFlags(
2589 MI: I, KHRFloatControls2: ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2)))
2590 return;
2591
2592 unsigned FMFlags = getFastMathFlags(I, ST);
2593 if (FMFlags == SPIRV::FPFastMathMode::None) {
2594 // We also need to check if any FPFastMathDefault info was set for the
2595 // types used in this instruction.
2596 if (FPFastMathDefaultInfoVec.empty())
2597 return;
2598
2599 // There are three types of instructions that can use fast math flags:
2600 // 1. Arithmetic instructions (FAdd, FMul, FSub, FDiv, FRem, etc.)
2601 // 2. Relational instructions (FCmp, FOrd, FUnord, etc.)
2602 // 3. Extended instructions (ExtInst)
2603 // For arithmetic instructions, the floating point type can be in the
2604 // result type or in the operands, but they all must be the same.
2605 // For the relational and logical instructions, the floating point type
2606 // can only be in the operands 1 and 2, not the result type. Also, the
2607 // operands must have the same type. For the extended instructions, the
2608 // floating point type can be in the result type or in the operands. It's
2609 // unclear if the operands and the result type must be the same. Let's
2610 // assume they must be. Therefore, for 1. and 2., we can check the first
2611 // operand type, and for 3. we can check the result type.
2612 assert(I.getNumOperands() >= 3 && "Expected at least 3 operands");
2613 Register ResReg = I.getOpcode() == SPIRV::OpExtInst
2614 ? I.getOperand(i: 1).getReg()
2615 : I.getOperand(i: 2).getReg();
2616 SPIRVTypeInst ResType = GR->getSPIRVTypeForVReg(VReg: ResReg, MF: I.getMF());
2617 const Type *Ty = GR->getTypeForSPIRVType(Ty: ResType);
2618 Ty = Ty->isVectorTy() ? cast<VectorType>(Val: Ty)->getElementType() : Ty;
2619
2620 // Match instruction type with the FPFastMathDefaultInfoVec.
2621 bool Emit = false;
2622 for (SPIRV::FPFastMathDefaultInfo &Elem : FPFastMathDefaultInfoVec) {
2623 if (Ty == Elem.Ty) {
2624 FMFlags = Elem.FastMathFlags;
2625 Emit = Elem.ContractionOff || Elem.SignedZeroInfNanPreserve ||
2626 Elem.FPFastMathDefault;
2627 break;
2628 }
2629 }
2630
2631 if (FMFlags == SPIRV::FPFastMathMode::None && !Emit)
2632 return;
2633 }
2634 if (isFastMathModeAvailable(ST)) {
2635 Register DstReg = I.getOperand(i: 0).getReg();
2636 buildOpDecorate(Reg: DstReg, I, TII, Dec: SPIRV::Decoration::FPFastMathMode,
2637 DecArgs: {FMFlags});
2638 }
2639}
2640
2641// Walk all functions and add decorations related to MI flags.
2642static void addDecorations(const Module &M, const SPIRVInstrInfo &TII,
2643 MachineModuleInfo *MMI, const SPIRVSubtarget &ST,
2644 SPIRV::ModuleAnalysisInfo &MAI,
2645 const SPIRVGlobalRegistry *GR) {
2646 for (const Function &F : M) {
2647 MachineFunction *MF = MMI->getMachineFunction(F);
2648 if (!MF)
2649 continue;
2650
2651 for (auto &MBB : *MF)
2652 for (auto &MI : MBB)
2653 handleMIFlagDecoration(I&: MI, ST, TII, Reqs&: MAI.Reqs, GR,
2654 FPFastMathDefaultInfoVec&: MAI.FPFastMathDefaultInfoMap[&F]);
2655 }
2656}
2657
2658static void addMBBNames(const Module &M, const SPIRVInstrInfo &TII,
2659 MachineModuleInfo *MMI, const SPIRVSubtarget &ST,
2660 SPIRV::ModuleAnalysisInfo &MAI) {
2661 for (const Function &F : M) {
2662 MachineFunction *MF = MMI->getMachineFunction(F);
2663 if (!MF)
2664 continue;
2665 MachineRegisterInfo &MRI = MF->getRegInfo();
2666 for (auto &MBB : *MF) {
2667 if (!MBB.hasName() || MBB.empty())
2668 continue;
2669 // Emit basic block names.
2670 Register Reg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 64));
2671 MRI.setRegClass(Reg, RC: &SPIRV::IDRegClass);
2672 buildOpName(Target: Reg, Name: MBB.getName(), I&: *std::prev(x: MBB.end()), TII);
2673 MCRegister GlobalReg = MAI.getOrCreateMBBRegister(MBB);
2674 MAI.setRegisterAlias(MF, Reg, AliasReg: GlobalReg);
2675 }
2676 }
2677}
2678
2679// patching Instruction::PHI to SPIRV::OpPhi
2680static void patchPhis(const Module &M, SPIRVGlobalRegistry *GR,
2681 const SPIRVInstrInfo &TII, MachineModuleInfo *MMI) {
2682 for (const Function &F : M) {
2683 MachineFunction *MF = MMI->getMachineFunction(F);
2684 if (!MF)
2685 continue;
2686 for (auto &MBB : *MF) {
2687 for (MachineInstr &MI : MBB.phis()) {
2688 MI.setDesc(TII.get(Opcode: SPIRV::OpPhi));
2689 Register ResTypeReg = GR->getSPIRVTypeID(
2690 SpirvType: GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 0).getReg(), MF));
2691 MI.insert(InsertBefore: MI.operands_begin() + 1,
2692 Ops: {MachineOperand::CreateReg(Reg: ResTypeReg, isDef: false)});
2693 }
2694 }
2695
2696 MF->getProperties().setNoPHIs();
2697 }
2698}
2699
2700static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
2701 const Module &M, SPIRV::ModuleAnalysisInfo &MAI, const Function *F) {
2702 auto it = MAI.FPFastMathDefaultInfoMap.find(Val: F);
2703 if (it != MAI.FPFastMathDefaultInfoMap.end())
2704 return it->second;
2705
2706 // If the map does not contain the entry, create a new one. Initialize it to
2707 // contain all 3 elements sorted by bit width of target type: {half, float,
2708 // double}.
2709 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
2710 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
2711 Args: SPIRV::FPFastMathMode::None);
2712 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
2713 Args: SPIRV::FPFastMathMode::None);
2714 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
2715 Args: SPIRV::FPFastMathMode::None);
2716 return MAI.FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
2717}
2718
2719static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
2720 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
2721 const Type *Ty) {
2722 size_t BitWidth = Ty->getScalarSizeInBits();
2723 int Index =
2724 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
2725 BitWidth);
2726 assert(Index >= 0 && Index < 3 &&
2727 "Expected FPFastMathDefaultInfo for half, float, or double");
2728 assert(FPFastMathDefaultInfoVec.size() == 3 &&
2729 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
2730 return FPFastMathDefaultInfoVec[Index];
2731}
2732
2733static void collectFPFastMathDefaults(const Module &M,
2734 SPIRV::ModuleAnalysisInfo &MAI,
2735 const SPIRVSubtarget &ST) {
2736 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
2737 return;
2738
2739 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
2740 // We need the entry point (function) as the key, and the target
2741 // type and flags as the value.
2742 // We also need to check ContractionOff and SignedZeroInfNanPreserve
2743 // execution modes, as they are now deprecated and must be replaced
2744 // with FPFastMathDefaultInfo.
2745 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
2746 if (!Node)
2747 return;
2748
2749 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
2750 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
2751 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
2752 const Function *F = cast<Function>(
2753 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
2754 const auto EM =
2755 cast<ConstantInt>(
2756 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
2757 ->getZExtValue();
2758 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
2759 assert(MDN->getNumOperands() == 4 &&
2760 "Expected 4 operands for FPFastMathDefault");
2761
2762 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
2763 unsigned Flags =
2764 cast<ConstantInt>(
2765 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
2766 ->getZExtValue();
2767 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2768 getOrCreateFPFastMathDefaultInfoVec(M, MAI, F);
2769 SPIRV::FPFastMathDefaultInfo &Info =
2770 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
2771 Info.FastMathFlags = Flags;
2772 Info.FPFastMathDefault = true;
2773 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
2774 assert(MDN->getNumOperands() == 2 &&
2775 "Expected no operands for ContractionOff");
2776
2777 // We need to save this info for every possible FP type, i.e. {half,
2778 // float, double, fp128}.
2779 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2780 getOrCreateFPFastMathDefaultInfoVec(M, MAI, F);
2781 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
2782 Info.ContractionOff = true;
2783 }
2784 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
2785 assert(MDN->getNumOperands() == 3 &&
2786 "Expected 1 operand for SignedZeroInfNanPreserve");
2787 unsigned TargetWidth =
2788 cast<ConstantInt>(
2789 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
2790 ->getZExtValue();
2791 // We need to save this info only for the FP type with TargetWidth.
2792 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
2793 getOrCreateFPFastMathDefaultInfoVec(M, MAI, F);
2794 int Index = SPIRV::FPFastMathDefaultInfoVector::
2795 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
2796 assert(Index >= 0 && Index < 3 &&
2797 "Expected FPFastMathDefaultInfo for half, float, or double");
2798 assert(FPFastMathDefaultInfoVec.size() == 3 &&
2799 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
2800 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
2801 }
2802 }
2803}
2804
2805struct SPIRV::ModuleAnalysisInfo SPIRVModuleAnalysis::MAI;
2806
2807void SPIRVModuleAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
2808 AU.addRequired<TargetPassConfig>();
2809 AU.addRequired<MachineModuleInfoWrapperPass>();
2810}
2811
2812bool SPIRVModuleAnalysis::runOnModule(Module &M) {
2813 SPIRVTargetMachine &TM =
2814 getAnalysis<TargetPassConfig>().getTM<SPIRVTargetMachine>();
2815 ST = TM.getSubtargetImpl();
2816 GR = ST->getSPIRVGlobalRegistry();
2817 TII = ST->getInstrInfo();
2818
2819 MMI = &getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
2820
2821 setBaseInfo(M);
2822
2823 patchPhis(M, GR, TII: *TII, MMI);
2824
2825 addMBBNames(M, TII: *TII, MMI, ST: *ST, MAI);
2826 collectFPFastMathDefaults(M, MAI, ST: *ST);
2827 addDecorations(M, TII: *TII, MMI, ST: *ST, MAI, GR);
2828
2829 collectReqs(M, MAI, MMI, ST: *ST);
2830
2831 // Process type/const/global var/func decl instructions, number their
2832 // destination registers from 0 to N, collect Extensions and Capabilities.
2833 collectReqs(M, MAI, MMI, ST: *ST);
2834 collectDeclarations(M);
2835
2836 // Number rest of registers from N+1 onwards.
2837 numberRegistersGlobally(M);
2838
2839 // Collect OpName, OpEntryPoint, OpDecorate etc, process other instructions.
2840 processOtherInstrs(M);
2841
2842 // If there are no entry points, we need the Linkage capability.
2843 if (MAI.MS[SPIRV::MB_EntryPoints].empty())
2844 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::Linkage);
2845
2846 // Set maximum ID used.
2847 GR->setBound(MAI.MaxID);
2848
2849 return false;
2850}
2851