1//===- SPIRVModuleAnalysis.cpp - analysis of global instrs & regs - C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The analysis collects instructions that should be output at the module level
10// and performs the global register numbering.
11//
12// The results of this analysis are used in AsmPrinter to rename registers
13// globally and to output required instructions at the module level.
14//
15//===----------------------------------------------------------------------===//
16
17// TODO: Per LLVM best practices, the report_fatal_error (deprecated) /
18// ReportFatalUsageError calls in this file should be replaced with the
19// Diagnostic infrastructure (e.g. the reportUnsupported function below).
20
21#include "SPIRVModuleAnalysis.h"
22#include "MCTargetDesc/SPIRVBaseInfo.h"
23#include "MCTargetDesc/SPIRVMCTargetDesc.h"
24#include "SPIRV.h"
25#include "SPIRVSubtarget.h"
26#include "SPIRVTargetMachine.h"
27#include "SPIRVUtils.h"
28#include "llvm/ADT/STLExtras.h"
29#include "llvm/ADT/StringSwitch.h"
30#include "llvm/CodeGen/MachineFunctionAnalysis.h"
31#include "llvm/CodeGen/MachineModuleInfo.h"
32#include "llvm/CodeGen/TargetPassConfig.h"
33
34using namespace llvm;
35
36#define DEBUG_TYPE "spirv-module-analysis"
37
38static cl::opt<bool>
39 SPVDumpDeps("spv-dump-deps",
40 cl::desc("Dump MIR with SPIR-V dependencies info"),
41 cl::init(Val: false));
42
43static cl::list<SPIRV::Capability::Capability>
44 AvoidCapabilities("avoid-spirv-capabilities",
45 cl::desc("SPIR-V capabilities to avoid if there are "
46 "other options enabling a feature"),
47 cl::Hidden,
48 cl::values(clEnumValN(SPIRV::Capability::Shader, "Shader",
49 "SPIR-V Shader capability")));
50// Use sets instead of cl::list to check "if contains" condition
51struct AvoidCapabilitiesSet {
52 SmallSet<SPIRV::Capability::Capability, 4> S;
53 AvoidCapabilitiesSet() { S.insert_range(R&: AvoidCapabilities); }
54};
55
56char llvm::SPIRVModuleAnalysisWrapperPass::ID = 0;
57
58INITIALIZE_PASS(SPIRVModuleAnalysisWrapperPass, DEBUG_TYPE,
59 "SPIRV module analysis", true, true)
60
61static void reportUnsupported(const MachineInstr &MI, const char *Msg) {
62 const Function &Func = MI.getMF()->getFunction();
63 Func.getContext().diagnose(
64 DI: DiagnosticInfoUnsupported(Func, Msg, MI.getDebugLoc()));
65}
66
67// Retrieve an unsigned from an MDNode with a list of them as operands.
68static unsigned getMetadataUInt(MDNode *MdNode, unsigned OpIndex,
69 unsigned DefaultVal = 0) {
70 if (MdNode && OpIndex < MdNode->getNumOperands()) {
71 const auto &Op = MdNode->getOperand(I: OpIndex);
72 return mdconst::extract<ConstantInt>(MD: Op)->getZExtValue();
73 }
74 return DefaultVal;
75}
76
77static SPIRV::Requirements
78getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
79 unsigned i, const SPIRVSubtarget &ST,
80 SPIRV::RequirementHandler &Reqs) {
81 // A set of capabilities to avoid if there is another option.
82 AvoidCapabilitiesSet AvoidCaps;
83 if (!ST.isShader())
84 AvoidCaps.S.insert(V: SPIRV::Capability::Shader);
85 else
86 AvoidCaps.S.insert(V: SPIRV::Capability::Kernel);
87
88 VersionTuple ReqMinVer = getSymbolicOperandMinVersion(Category, Value: i);
89 VersionTuple ReqMaxVer = getSymbolicOperandMaxVersion(Category, Value: i);
90 VersionTuple SPIRVVersion = ST.getSPIRVVersion();
91 bool MinVerOK = SPIRVVersion.empty() || SPIRVVersion >= ReqMinVer;
92 bool MaxVerOK =
93 ReqMaxVer.empty() || SPIRVVersion.empty() || SPIRVVersion <= ReqMaxVer;
94 CapabilityList ReqCaps = getSymbolicOperandCapabilities(Category, Value: i);
95 ExtensionList ReqExts = getSymbolicOperandExtensions(Category, Value: i);
96 if (ReqCaps.empty()) {
97 if (ReqExts.empty()) {
98 if (MinVerOK && MaxVerOK)
99 return {true, {}, {}, ReqMinVer, ReqMaxVer};
100 return {false, {}, {}, VersionTuple(), VersionTuple()};
101 }
102 } else if (MinVerOK && MaxVerOK) {
103 if (ReqCaps.size() == 1) {
104 auto Cap = ReqCaps[0];
105 if (Reqs.isCapabilityAvailable(Cap)) {
106 ReqExts.append(RHS: getSymbolicOperandExtensions(
107 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
108 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
109 }
110 } else {
111 // By SPIR-V specification: "If an instruction, enumerant, or other
112 // feature specifies multiple enabling capabilities, only one such
113 // capability needs to be declared to use the feature." However, one
114 // capability may be preferred over another. We use command line
115 // argument(s) and AvoidCapabilities to avoid selection of certain
116 // capabilities if there are other options.
117 CapabilityList UseCaps;
118 for (auto Cap : ReqCaps)
119 if (Reqs.isCapabilityAvailable(Cap))
120 UseCaps.push_back(Elt: Cap);
121 for (size_t i = 0, Sz = UseCaps.size(); i < Sz; ++i) {
122 auto Cap = UseCaps[i];
123 if (i == Sz - 1 || !AvoidCaps.S.contains(V: Cap)) {
124 ReqExts.append(RHS: getSymbolicOperandExtensions(
125 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
126 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
127 }
128 }
129 }
130 }
131 // If there are no capabilities, or we can't satisfy the version or
132 // capability requirements, use the list of extensions (if the subtarget
133 // can handle them all).
134 if (llvm::all_of(Range&: ReqExts, P: [&ST](const SPIRV::Extension::Extension &Ext) {
135 return ST.canUseExtension(E: Ext);
136 })) {
137 return {true,
138 {},
139 std::move(ReqExts),
140 VersionTuple(),
141 VersionTuple()}; // TODO: add versions to extensions.
142 }
143 return {false, {}, {}, VersionTuple(), VersionTuple()};
144}
145
146void SPIRVModuleAnalysisImpl::setBaseInfo(const Module &M) {
147 MAI.MaxID = 0;
148 for (int i = 0; i < SPIRV::NUM_MODULE_SECTIONS; i++)
149 MAI.MS[i].clear();
150 MAI.RegisterAliasTable.clear();
151 MAI.InstrsToDelete.clear();
152 MAI.MBBNames.clear();
153 MAI.GlobalObjMap.clear();
154 MAI.GlobalVarList.clear();
155 MAI.ExtInstSetMap.clear();
156 MAI.InstrAuxDataRecords.clear();
157 MAI.Reqs.clear();
158 MAI.Reqs.initAvailableCapabilities(ST: *ST);
159
160 // TODO: determine memory model and source language from the configuratoin.
161 if (auto MemModel = M.getNamedMetadata(Name: "spirv.MemoryModel")) {
162 auto MemMD = MemModel->getOperand(i: 0);
163 MAI.Addr = static_cast<SPIRV::AddressingModel::AddressingModel>(
164 getMetadataUInt(MdNode: MemMD, OpIndex: 0));
165 MAI.Mem =
166 static_cast<SPIRV::MemoryModel::MemoryModel>(getMetadataUInt(MdNode: MemMD, OpIndex: 1));
167 } else {
168 // TODO: Add support for VulkanMemoryModel.
169 MAI.Mem = ST->isShader() ? SPIRV::MemoryModel::GLSL450
170 : SPIRV::MemoryModel::OpenCL;
171 if (MAI.Mem == SPIRV::MemoryModel::OpenCL) {
172 unsigned PtrSize = ST->getPointerSize();
173 MAI.Addr = PtrSize == 32 ? SPIRV::AddressingModel::Physical32
174 : PtrSize == 64 ? SPIRV::AddressingModel::Physical64
175 : SPIRV::AddressingModel::Logical;
176 } else {
177 // TODO: Add support for PhysicalStorageBufferAddress.
178 MAI.Addr = SPIRV::AddressingModel::Logical;
179 }
180 }
181 // Get the OpenCL version number from metadata.
182 // TODO: support other source languages.
183 if (auto VerNode = M.getNamedMetadata(Name: "opencl.ocl.version")) {
184 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_C;
185 // Construct version literal in accordance with SPIRV-LLVM-Translator.
186 // TODO: support multiple OCL version metadata.
187 assert(VerNode->getNumOperands() > 0 && "Invalid SPIR");
188 auto VersionMD = VerNode->getOperand(i: 0);
189 unsigned MajorNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 0, DefaultVal: 2);
190 unsigned MinorNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 1);
191 unsigned RevNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 2);
192 // Prevent Major part of OpenCL version to be 0
193 MAI.SrcLangVersion =
194 (std::max(a: 1U, b: MajorNum) * 100 + MinorNum) * 1000 + RevNum;
195 // When opencl.cxx.version is also present, validate compatibility
196 // and use C++ for OpenCL as source language with the C++ version.
197 if (auto *CxxVerNode = M.getNamedMetadata(Name: "opencl.cxx.version")) {
198 assert(CxxVerNode->getNumOperands() > 0 && "Invalid SPIR");
199 auto *CxxMD = CxxVerNode->getOperand(i: 0);
200 unsigned CxxVer =
201 (getMetadataUInt(MdNode: CxxMD, OpIndex: 0) * 100 + getMetadataUInt(MdNode: CxxMD, OpIndex: 1)) * 1000 +
202 getMetadataUInt(MdNode: CxxMD, OpIndex: 2);
203 if ((MAI.SrcLangVersion == 200000 && CxxVer == 100000) ||
204 (MAI.SrcLangVersion == 300000 && CxxVer == 202100000)) {
205 MAI.SrcLang = SPIRV::SourceLanguage::CPP_for_OpenCL;
206 MAI.SrcLangVersion = CxxVer;
207 } else {
208 report_fatal_error(
209 reason: "opencl cxx version is not compatible with opencl c version!");
210 }
211 }
212 } else {
213 // If there is no information about OpenCL version we are forced to generate
214 // OpenCL 1.0 by default for the OpenCL environment to avoid puzzling
215 // run-times with Unknown/0.0 version output. For a reference, LLVM-SPIRV
216 // Translator avoids potential issues with run-times in a similar manner.
217 if (!ST->isShader()) {
218 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_CPP;
219 MAI.SrcLangVersion = 100000;
220 } else {
221 MAI.SrcLang = SPIRV::SourceLanguage::Unknown;
222 MAI.SrcLangVersion = 0;
223 }
224 }
225
226 if (auto ExtNode = M.getNamedMetadata(Name: "opencl.used.extensions")) {
227 for (unsigned I = 0, E = ExtNode->getNumOperands(); I != E; ++I) {
228 MDNode *MD = ExtNode->getOperand(i: I);
229 if (!MD || MD->getNumOperands() == 0)
230 continue;
231 for (unsigned J = 0, N = MD->getNumOperands(); J != N; ++J)
232 MAI.SrcExt.insert(key: cast<MDString>(Val: MD->getOperand(I: J))->getString());
233 }
234 }
235
236 // Update required capabilities for this memory model, addressing model and
237 // source language.
238 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::MemoryModelOperand,
239 i: MAI.Mem, ST: *ST);
240 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::SourceLanguageOperand,
241 i: MAI.SrcLang, ST: *ST);
242 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::AddressingModelOperand,
243 i: MAI.Addr, ST: *ST);
244
245 if (MAI.Mem == SPIRV::MemoryModel::VulkanKHR)
246 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_vulkan_memory_model);
247
248 if (!ST->isShader()) {
249 // TODO: check if it's required by default.
250 MAI.ExtInstSetMap[static_cast<unsigned>(
251 SPIRV::InstructionSet::OpenCL_std)] = MAI.getNextIDRegister();
252 }
253}
254
255// Appends the signature of the decoration instructions that decorate R to
256// Signature.
257static void appendDecorationsForReg(const MachineRegisterInfo &MRI, Register R,
258 InstrSignature &Signature) {
259 for (MachineInstr &UseMI : MRI.use_instructions(Reg: R)) {
260 // We don't handle OpDecorateId because getting the register alias for the
261 // ID can cause problems, and we do not need it for now.
262 if (UseMI.getOpcode() != SPIRV::OpDecorate &&
263 UseMI.getOpcode() != SPIRV::OpMemberDecorate)
264 continue;
265
266 for (unsigned I = 0; I < UseMI.getNumOperands(); ++I) {
267 const MachineOperand &MO = UseMI.getOperand(i: I);
268 if (MO.isReg())
269 continue;
270 Signature.push_back(Elt: hash_value(MO));
271 }
272 }
273}
274
275// Returns a representation of an instruction as a vector of MachineOperand
276// hash values, see llvm::hash_value(const MachineOperand &MO) for details.
277// This creates a signature of the instruction with the same content
278// that MachineOperand::isIdenticalTo uses for comparison.
279static InstrSignature instrToSignature(const MachineInstr &MI,
280 SPIRV::ModuleAnalysisInfo &MAI,
281 bool UseDefReg) {
282 Register DefReg;
283 InstrSignature Signature{MI.getOpcode()};
284 for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
285 // The only decorations that can be applied more than once to a given <id>
286 // or structure member are FuncParamAttr (38), UserSemantic (5635),
287 // CacheControlLoadINTEL (6442), and CacheControlStoreINTEL (6443). For all
288 // the rest of decorations, we will only add to the signature the Opcode,
289 // the id to which it applies, and the decoration id, disregarding any
290 // decoration flags. This will ensure that any subsequent decoration with
291 // the same id will be deemed as a duplicate. Then, at the call site, we
292 // will be able to handle duplicates in the best way.
293 unsigned Opcode = MI.getOpcode();
294 if ((Opcode == SPIRV::OpDecorate) && i >= 2) {
295 unsigned DecorationID = MI.getOperand(i: 1).getImm();
296 if (DecorationID != SPIRV::Decoration::FuncParamAttr &&
297 DecorationID != SPIRV::Decoration::UserSemantic &&
298 DecorationID != SPIRV::Decoration::CacheControlLoadINTEL &&
299 DecorationID != SPIRV::Decoration::CacheControlStoreINTEL)
300 continue;
301 }
302 const MachineOperand &MO = MI.getOperand(i);
303 size_t h;
304 if (MO.isReg()) {
305 if (!UseDefReg && MO.isDef()) {
306 assert(!DefReg.isValid() && "Multiple def registers.");
307 DefReg = MO.getReg();
308 continue;
309 }
310 Register RegAlias = MAI.getRegisterAlias(MF: MI.getMF(), Reg: MO.getReg());
311 if (!RegAlias.isValid()) {
312 LLVM_DEBUG({
313 dbgs() << "Unexpectedly, no global id found for the operand ";
314 MO.print(dbgs());
315 dbgs() << "\nInstruction: ";
316 MI.print(dbgs());
317 dbgs() << "\n";
318 });
319 report_fatal_error(reason: "All v-regs must have been mapped to global id's");
320 }
321 // mimic llvm::hash_value(const MachineOperand &MO)
322 h = hash_combine(args: MO.getType(), args: (unsigned)RegAlias, args: MO.getSubReg(),
323 args: MO.isDef());
324 } else {
325 h = hash_value(MO);
326 }
327 Signature.push_back(Elt: h);
328 }
329
330 if (DefReg.isValid()) {
331 // Decorations change the semantics of the current instruction. So two
332 // identical instruction with different decorations cannot be merged. That
333 // is why we add the decorations to the signature.
334 appendDecorationsForReg(MRI: MI.getMF()->getRegInfo(), R: DefReg, Signature);
335 }
336 return Signature;
337}
338
339// Operand index of Invoke in device enqueue instructions, 0 if none.
340static unsigned getInvokeOperandIdx(unsigned Opcode) {
341 switch (Opcode) {
342 case SPIRV::OpEnqueueKernel:
343 return 8;
344 case SPIRV::OpGetKernelNDrangeSubGroupCount:
345 case SPIRV::OpGetKernelNDrangeMaxSubGroupSize:
346 return 3;
347 case SPIRV::OpGetKernelWorkGroupSize:
348 case SPIRV::OpGetKernelPreferredWorkGroupSizeMultiple:
349 return 2;
350 default:
351 return 0;
352 }
353}
354
355bool SPIRVModuleAnalysisImpl::isDeclSection(const MachineRegisterInfo &MRI,
356 const MachineInstr &MI) {
357 unsigned Opcode = MI.getOpcode();
358 switch (Opcode) {
359 case SPIRV::OpTypeForwardPointer:
360 // omit now, collect later
361 return false;
362 case SPIRV::OpVariable:
363 case SPIRV::OpUntypedVariableKHR:
364 return static_cast<SPIRV::StorageClass::StorageClass>(
365 MI.getOperand(i: 2).getImm()) != SPIRV::StorageClass::Function;
366 case SPIRV::OpFunction:
367 case SPIRV::OpFunctionParameter:
368 return true;
369 }
370 if (GR->hasConstFunPtr() && Opcode == SPIRV::OpUndef) {
371 // The OpUndef may be a placeholder for a function reference recorded by
372 // selectGlobalValue. Skip emitting it if any user consumes it as a
373 // function-pointer-like operand (OpConstantFunctionPointerINTEL operand 2,
374 // or the Invoke operand of a device enqueue instruction). The rewrite
375 // happens in visitFunPtrUse, which aliases the OpUndef's vreg to the
376 // function's global <id>.
377 Register DefReg = MI.getOperand(i: 0).getReg();
378 if (GR->getFunctionDefinitionByUse(Use: &MI.getOperand(i: 0))) {
379 for (MachineInstr &UseMI : MRI.use_instructions(Reg: DefReg)) {
380 unsigned UseOp = UseMI.getOpcode();
381 if (UseOp == SPIRV::OpConstantFunctionPointerINTEL ||
382 getInvokeOperandIdx(Opcode: UseOp)) {
383 MAI.setSkipEmission(&MI);
384 return false;
385 }
386 }
387 }
388 for (MachineInstr &UseMI : MRI.use_instructions(Reg: DefReg)) {
389 if (UseMI.getOpcode() != SPIRV::OpConstantFunctionPointerINTEL)
390 continue;
391 // it's a dummy definition, FP constant refers to a function,
392 // and this is resolved in another way; let's skip this definition
393 assert(UseMI.getOperand(2).isReg() &&
394 UseMI.getOperand(2).getReg() == DefReg);
395 MAI.setSkipEmission(&MI);
396 return false;
397 }
398 }
399 return TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||
400 TII->isInlineAsmDefInstr(MI);
401}
402
403// This is a special case of a function pointer referring to a possibly
404// forward function declaration. The operand is a dummy OpUndef that
405// requires a special treatment.
406// FunPtrOp is the MachineOperand previously recorded via
407// SPIRVGlobalRegistry::recordFunctionPointer, identifying which Function
408// this placeholder refers to.
409void SPIRVModuleAnalysisImpl::visitFunPtrUse(
410 Register OpReg, const MachineOperand *FunPtrOp,
411 InstrGRegsMap &SignatureToGReg,
412 std::map<const Value *, unsigned> &GlobalToGReg,
413 const MachineFunction *MF) {
414 const MachineOperand *OpFunDef = GR->getFunctionDefinitionByUse(Use: FunPtrOp);
415 assert(OpFunDef && OpFunDef->isReg());
416 // find the actual function definition and number it globally in advance
417 const MachineInstr *OpDefMI = OpFunDef->getParent();
418 assert(OpDefMI && OpDefMI->getOpcode() == SPIRV::OpFunction);
419 const MachineFunction *FunDefMF = OpDefMI->getParent()->getParent();
420 const MachineRegisterInfo &FunDefMRI = FunDefMF->getRegInfo();
421 do {
422 visitDecl(MRI: FunDefMRI, SignatureToGReg, GlobalToGReg, MF: FunDefMF, MI: *OpDefMI);
423 OpDefMI = OpDefMI->getNextNode();
424 } while (OpDefMI && (OpDefMI->getOpcode() == SPIRV::OpFunction ||
425 OpDefMI->getOpcode() == SPIRV::OpFunctionParameter));
426 // associate the function pointer with the newly assigned global number
427 MCRegister GlobalFunDefReg =
428 MAI.getRegisterAlias(MF: FunDefMF, Reg: OpFunDef->getReg());
429 assert(GlobalFunDefReg.isValid() &&
430 "Function definition must refer to a global register");
431 MAI.setRegisterAlias(MF, Reg: OpReg, AliasReg: GlobalFunDefReg);
432}
433
434// Depth first recursive traversal of dependencies. Repeated visits are guarded
435// by MAI.hasRegisterAlias().
436void SPIRVModuleAnalysisImpl::visitDecl(
437 const MachineRegisterInfo &MRI, InstrGRegsMap &SignatureToGReg,
438 std::map<const Value *, unsigned> &GlobalToGReg, const MachineFunction *MF,
439 const MachineInstr &MI) {
440 unsigned Opcode = MI.getOpcode();
441
442 // Process each operand of the instruction to resolve dependencies
443 for (const MachineOperand &MO : MI.operands()) {
444 if (!MO.isReg() || MO.isDef())
445 continue;
446 Register OpReg = MO.getReg();
447 // Handle function pointers special case
448 if (Opcode == SPIRV::OpConstantFunctionPointerINTEL &&
449 MRI.getRegClass(Reg: OpReg) == &SPIRV::pIDRegClass) {
450 visitFunPtrUse(OpReg, FunPtrOp: &MI.getOperand(i: 2), SignatureToGReg, GlobalToGReg,
451 MF);
452 continue;
453 }
454 // Skip already processed instructions
455 if (MAI.hasRegisterAlias(MF, Reg: MO.getReg()))
456 continue;
457 // Recursively visit dependencies
458 if (const MachineInstr *OpDefMI = MRI.getUniqueVRegDef(Reg: OpReg)) {
459 if (isDeclSection(MRI, MI: *OpDefMI))
460 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, MI: *OpDefMI);
461 continue;
462 }
463 // Handle the unexpected case of no unique definition for the SPIR-V
464 // instruction
465 LLVM_DEBUG({
466 dbgs() << "Unexpectedly, no unique definition for the operand ";
467 MO.print(dbgs());
468 dbgs() << "\nInstruction: ";
469 MI.print(dbgs());
470 dbgs() << "\n";
471 });
472 report_fatal_error(
473 reason: "No unique definition is found for the virtual register");
474 }
475
476 MCRegister GReg;
477 bool IsFunDef = false;
478 if (TII->isSpecConstantInstr(MI)) {
479 GReg = MAI.getNextIDRegister();
480 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
481 } else if (Opcode == SPIRV::OpFunction ||
482 Opcode == SPIRV::OpFunctionParameter) {
483 GReg = handleFunctionOrParameter(MF, MI, GlobalToGReg, IsFunDef);
484 } else if (Opcode == SPIRV::OpTypeStruct ||
485 Opcode == SPIRV::OpConstantComposite) {
486 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);
487 const MachineInstr *NextInstr = MI.getNextNode();
488 while (NextInstr &&
489 ((Opcode == SPIRV::OpTypeStruct &&
490 NextInstr->getOpcode() == SPIRV::OpTypeStructContinuedINTEL) ||
491 (Opcode == SPIRV::OpConstantComposite &&
492 NextInstr->getOpcode() ==
493 SPIRV::OpConstantCompositeContinuedINTEL))) {
494 MCRegister Tmp = handleTypeDeclOrConstant(MI: *NextInstr, SignatureToGReg);
495 MAI.setRegisterAlias(MF, Reg: NextInstr->getOperand(i: 0).getReg(), AliasReg: Tmp);
496 MAI.setSkipEmission(NextInstr);
497 NextInstr = NextInstr->getNextNode();
498 }
499 } else if (TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||
500 TII->isInlineAsmDefInstr(MI)) {
501 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);
502 } else if (Opcode == SPIRV::OpVariable ||
503 Opcode == SPIRV::OpUntypedVariableKHR) {
504 GReg = handleVariable(MF, MI, GlobalToGReg);
505 } else {
506 LLVM_DEBUG({
507 dbgs() << "\nInstruction: ";
508 MI.print(dbgs());
509 dbgs() << "\n";
510 });
511 llvm_unreachable("Unexpected instruction is visited");
512 }
513 MAI.setRegisterAlias(MF, Reg: MI.getOperand(i: 0).getReg(), AliasReg: GReg);
514 if (!IsFunDef)
515 MAI.setSkipEmission(&MI);
516}
517
518MCRegister SPIRVModuleAnalysisImpl::handleFunctionOrParameter(
519 const MachineFunction *MF, const MachineInstr &MI,
520 std::map<const Value *, unsigned> &GlobalToGReg, bool &IsFunDef) {
521 const Value *GObj = GR->getGlobalObject(MF, R: MI.getOperand(i: 0).getReg());
522 assert(GObj && "Unregistered global definition");
523 const Function *F = dyn_cast<Function>(Val: GObj);
524 if (!F)
525 F = dyn_cast<Argument>(Val: GObj)->getParent();
526 assert(F && "Expected a reference to a function or an argument");
527 IsFunDef = !F->isDeclaration();
528 auto [It, Inserted] = GlobalToGReg.try_emplace(k: GObj);
529 if (!Inserted)
530 return It->second;
531 MCRegister GReg = MAI.getNextIDRegister();
532 It->second = GReg;
533 if (!IsFunDef)
534 MAI.MS[SPIRV::MB_ExtFuncDecls].push_back(Elt: &MI);
535 return GReg;
536}
537
538MCRegister SPIRVModuleAnalysisImpl::handleTypeDeclOrConstant(
539 const MachineInstr &MI, InstrGRegsMap &SignatureToGReg) {
540 InstrSignature MISign = instrToSignature(MI, MAI, UseDefReg: false);
541 auto [It, Inserted] = SignatureToGReg.try_emplace(k: MISign);
542 if (!Inserted)
543 return It->second;
544 MCRegister GReg = MAI.getNextIDRegister();
545 It->second = GReg;
546 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
547 return GReg;
548}
549
550MCRegister SPIRVModuleAnalysisImpl::handleVariable(
551 const MachineFunction *MF, const MachineInstr &MI,
552 std::map<const Value *, unsigned> &GlobalToGReg) {
553 MAI.GlobalVarList.push_back(Elt: &MI);
554 const Value *GObj = GR->getGlobalObject(MF, R: MI.getOperand(i: 0).getReg());
555 assert(GObj && "Unregistered global definition");
556 auto [It, Inserted] = GlobalToGReg.try_emplace(k: GObj);
557 if (!Inserted)
558 return It->second;
559 MCRegister GReg = MAI.getNextIDRegister();
560 It->second = GReg;
561 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
562 if (const auto *GV = dyn_cast<GlobalVariable>(Val: GObj))
563 MAI.GlobalObjMap[GV] = GReg;
564 return GReg;
565}
566
567void SPIRVModuleAnalysisImpl::collectDeclarations(const Module &M) {
568 InstrGRegsMap SignatureToGReg;
569 std::map<const Value *, unsigned> GlobalToGReg;
570 for (const Function &F : M) {
571 MachineFunction *MF = GetMF(F);
572 if (!MF)
573 continue;
574 const MachineRegisterInfo &MRI = MF->getRegInfo();
575 unsigned PastHeader = 0;
576 for (MachineBasicBlock &MBB : *MF) {
577 for (MachineInstr &MI : MBB) {
578 if (MI.getNumOperands() == 0)
579 continue;
580 unsigned Opcode = MI.getOpcode();
581 if (Opcode == SPIRV::OpFunction) {
582 if (PastHeader == 0) {
583 PastHeader = 1;
584 continue;
585 }
586 } else if (Opcode == SPIRV::OpFunctionParameter) {
587 if (PastHeader < 2)
588 continue;
589 } else if (PastHeader > 0) {
590 PastHeader = 2;
591 }
592
593 const MachineOperand &DefMO = MI.getOperand(i: 0);
594 switch (Opcode) {
595 case SPIRV::OpExtension:
596 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::Extension(DefMO.getImm()));
597 MAI.setSkipEmission(&MI);
598 break;
599 case SPIRV::OpCapability:
600 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::Capability(DefMO.getImm()));
601 MAI.setSkipEmission(&MI);
602 if (PastHeader > 0)
603 PastHeader = 2;
604 break;
605 default:
606 if (DefMO.isReg() && isDeclSection(MRI, MI) &&
607 !MAI.hasRegisterAlias(MF, Reg: DefMO.getReg()))
608 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, MI);
609 // Device enqueue instructions are not decls, but their Invoke
610 // operand may be a function-pointer placeholder OpUndef. Resolve it
611 // to the OpFunction's global <id> via visitFunPtrUse.
612 if (unsigned InvokeIdx = getInvokeOperandIdx(Opcode)) {
613 const MachineOperand &InvokeMO = MI.getOperand(i: InvokeIdx);
614 if (InvokeMO.isReg()) {
615 Register InvokeReg = InvokeMO.getReg();
616 if (!MAI.hasRegisterAlias(MF, Reg: InvokeReg)) {
617 if (const MachineInstr *DefMI =
618 MRI.getUniqueVRegDef(Reg: InvokeReg)) {
619 if (DefMI->getOpcode() == SPIRV::OpUndef) {
620 const MachineOperand *FunPtrOp = &DefMI->getOperand(i: 0);
621 if (GR->getFunctionDefinitionByUse(Use: FunPtrOp))
622 visitFunPtrUse(OpReg: InvokeReg, FunPtrOp, SignatureToGReg,
623 GlobalToGReg, MF);
624 }
625 }
626 }
627 }
628 }
629 }
630 }
631 }
632 }
633}
634
635// Look for IDs declared with Import linkage, and map the corresponding function
636// to the register defining that variable (which will usually be the result of
637// an OpFunction). This lets us call externally imported functions using
638// the correct ID registers.
639void SPIRVModuleAnalysisImpl::collectFuncNames(MachineInstr &MI,
640 const Function *F) {
641 if (MI.getOpcode() == SPIRV::OpDecorate) {
642 // If it's got Import linkage.
643 auto Dec = MI.getOperand(i: 1).getImm();
644 if (Dec == SPIRV::Decoration::LinkageAttributes) {
645 auto Lnk = MI.getOperand(i: MI.getNumOperands() - 1).getImm();
646 if (Lnk == SPIRV::LinkageType::Import) {
647 // Map imported function name to function ID register.
648 const Function *ImportedFunc =
649 F->getParent()->getFunction(Name: getStringImm(MI, StartIndex: 2));
650 Register Target = MI.getOperand(i: 0).getReg();
651 MAI.GlobalObjMap[ImportedFunc] =
652 MAI.getRegisterAlias(MF: MI.getMF(), Reg: Target);
653 }
654 }
655 } else if (MI.getOpcode() == SPIRV::OpFunction) {
656 // Record all internal OpFunction declarations.
657 Register Reg = MI.defs().begin()->getReg();
658 MCRegister GlobalReg = MAI.getRegisterAlias(MF: MI.getMF(), Reg);
659 assert(GlobalReg.isValid());
660 MAI.GlobalObjMap[F] = GlobalReg;
661 }
662}
663
664// Collect the given instruction in the specified MS. We assume global register
665// numbering has already occurred by this point. We can directly compare reg
666// arguments when detecting duplicates.
667static void collectOtherInstr(MachineInstr &MI, SPIRV::ModuleAnalysisInfo &MAI,
668 SPIRV::ModuleSectionType MSType, InstrTraces &IS,
669 bool Append = true) {
670 MAI.setSkipEmission(&MI);
671 InstrSignature MISign = instrToSignature(MI, MAI, UseDefReg: true);
672 auto FoundMI = IS.insert(x: std::move(MISign));
673 if (!FoundMI.second) {
674 if (MI.getOpcode() == SPIRV::OpDecorate) {
675 assert(MI.getNumOperands() >= 2 &&
676 "Decoration instructions must have at least 2 operands");
677 assert(MSType == SPIRV::MB_Annotations &&
678 "Only OpDecorate instructions can be duplicates");
679 // For FPFastMathMode decoration, we need to merge the flags of the
680 // duplicate decoration with the original one, so we need to find the
681 // original instruction that has the same signature. For the rest of
682 // instructions, we will simply skip the duplicate.
683 if (MI.getOperand(i: 1).getImm() != SPIRV::Decoration::FPFastMathMode)
684 return; // Skip duplicates of other decorations.
685
686 const SPIRV::InstrList &Decorations = MAI.MS[MSType];
687 for (const MachineInstr *OrigMI : Decorations) {
688 if (instrToSignature(MI: *OrigMI, MAI, UseDefReg: true) == MISign) {
689 assert(OrigMI->getNumOperands() == MI.getNumOperands() &&
690 "Original instruction must have the same number of operands");
691 assert(
692 OrigMI->getNumOperands() == 3 &&
693 "FPFastMathMode decoration must have 3 operands for OpDecorate");
694 unsigned OrigFlags = OrigMI->getOperand(i: 2).getImm();
695 unsigned NewFlags = MI.getOperand(i: 2).getImm();
696 if (OrigFlags == NewFlags)
697 return; // No need to merge, the flags are the same.
698
699 // Emit warning about possible conflict between flags.
700 unsigned FinalFlags = OrigFlags | NewFlags;
701 llvm::errs()
702 << "Warning: Conflicting FPFastMathMode decoration flags "
703 "in instruction: "
704 << *OrigMI << "Original flags: " << OrigFlags
705 << ", new flags: " << NewFlags
706 << ". They will be merged on a best effort basis, but not "
707 "validated. Final flags: "
708 << FinalFlags << "\n";
709 MachineInstr *OrigMINonConst = const_cast<MachineInstr *>(OrigMI);
710 MachineOperand &OrigFlagsOp = OrigMINonConst->getOperand(i: 2);
711 OrigFlagsOp = MachineOperand::CreateImm(Val: FinalFlags);
712 return; // Merge done, so we found a duplicate; don't add it to MAI.MS
713 }
714 }
715 assert(false && "No original instruction found for the duplicate "
716 "OpDecorate, but we found one in IS.");
717 }
718 return; // insert failed, so we found a duplicate; don't add it to MAI.MS
719 }
720 // No duplicates, so add it.
721 if (Append)
722 MAI.MS[MSType].push_back(Elt: &MI);
723 else
724 MAI.MS[MSType].insert(I: MAI.MS[MSType].begin(), Elt: &MI);
725}
726
727// Some global instructions make reference to function-local ID regs, so cannot
728// be correctly collected until these registers are globally numbered.
729void SPIRVModuleAnalysisImpl::processOtherInstrs(const Module &M) {
730 InstrTraces IS;
731 for (const Function &F : M) {
732 if (F.isDeclaration())
733 continue;
734 MachineFunction *MF = GetMF(F);
735 assert(MF);
736
737 for (MachineBasicBlock &MBB : *MF)
738 for (MachineInstr &MI : MBB) {
739 if (MAI.getSkipEmission(MI: &MI))
740 continue;
741 const unsigned OpCode = MI.getOpcode();
742 if (OpCode == SPIRV::OpString) {
743 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_DebugStrings, IS);
744 } else if (OpCode == SPIRV::OpExtInst && MI.getOperand(i: 2).isImm() &&
745 MI.getOperand(i: 2).getImm() ==
746 SPIRV::InstructionSet::
747 NonSemantic_Shader_DebugInfo_100) {
748 // TODO: This branch is dead. SPIRVNonSemanticDebugHandler emits NSDI
749 // instructions directly as MCInsts at print time; no
750 // MachineInstructions with the NSDI ext set are created anymore.
751 // Remove this block and
752 // MB_NonSemanticGlobalDI once per-function NSDI emission is confirmed
753 // not to need MIR routing.
754 MachineOperand Ins = MI.getOperand(i: 3);
755 namespace NS = SPIRV::NonSemanticExtInst;
756 static constexpr int64_t GlobalNonSemanticDITy[] = {
757 NS::DebugSource, NS::DebugCompilationUnit, NS::DebugInfoNone,
758 NS::DebugTypeBasic, NS::DebugTypePointer};
759 bool IsGlobalDI = false;
760 for (unsigned Idx = 0; Idx < std::size(GlobalNonSemanticDITy); ++Idx)
761 IsGlobalDI |= Ins.getImm() == GlobalNonSemanticDITy[Idx];
762 if (IsGlobalDI)
763 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_NonSemanticGlobalDI, IS);
764 } else if (OpCode == SPIRV::OpName || OpCode == SPIRV::OpMemberName) {
765 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_DebugNames, IS);
766 } else if (OpCode == SPIRV::OpEntryPoint) {
767 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_EntryPoints, IS);
768 } else if (TII->isAliasingInstr(MI)) {
769 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_AliasingInsts, IS);
770 } else if (TII->isDecorationInstr(MI)) {
771 if (MI.getOpcode() == SPIRV::OpDecorate &&
772 MI.getOperand(i: 1).getImm() ==
773 static_cast<unsigned>(
774 SPIRV::Decoration::AuxDataInstructionMetadata)) {
775 MAI.setSkipEmission(&MI);
776 std::string Str = getStringImm(MI, StartIndex: 2);
777 using AMDMD = SPIRV::ModuleAnalysisInfo::AMDGPUAtomicMDKind;
778 auto MaybeKind =
779 StringSwitch<std::optional<AMDMD>>(Str)
780 .Case(S: "amdgpu.no.fine.grained.memory",
781 Value: AMDMD::NoFineGrainedMemory)
782 .Case(S: "amdgpu.no.remote.memory", Value: AMDMD::NoRemoteMemory)
783 .Case(S: "atomic.ignore.denormal.mode",
784 Value: AMDMD::IgnoreDenormalMode)
785 .Default(Value: std::nullopt);
786 if (MaybeKind)
787 MAI.InstrAuxDataRecords.push_back(
788 Elt: {.MF: MF, .TargetReg: MI.getOperand(i: 0).getReg(), .Kind: *MaybeKind});
789 continue;
790 }
791 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_Annotations, IS);
792 collectFuncNames(MI, F: &F);
793 } else if (TII->isConstantInstr(MI)) {
794 // Now OpSpecConstant*s are not in DT,
795 // but they need to be collected anyway.
796 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_TypeConstVars, IS);
797 } else if (OpCode == SPIRV::OpFunction) {
798 collectFuncNames(MI, F: &F);
799 } else if (OpCode == SPIRV::OpTypeForwardPointer) {
800 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_TypeConstVars, IS, Append: false);
801 }
802 }
803 }
804 // Selection order can place a scope/list ahead of a domain/scope it
805 // references. The dependency meanwhile is domain -> scope -> list, so sort
806 // the def before its uses.
807 auto AliasingTier = [](const MachineInstr *MI) {
808 switch (MI->getOpcode()) {
809 case SPIRV::OpAliasDomainDeclINTEL:
810 return 0;
811 case SPIRV::OpAliasScopeDeclINTEL:
812 return 1;
813 case SPIRV::OpAliasScopeListDeclINTEL:
814 return 2;
815 default:
816 llvm_unreachable("unexpected aliasing instruction");
817 }
818 };
819 stable_sort(Range&: MAI.MS[SPIRV::MB_AliasingInsts],
820 C: [&](const MachineInstr *LHS, const MachineInstr *RHS) {
821 return AliasingTier(LHS) < AliasingTier(RHS);
822 });
823}
824
825// Number registers in all functions globally from 0 onwards and store
826// the result in global register alias table. Some registers are already
827// numbered.
828void SPIRVModuleAnalysisImpl::numberRegistersGlobally(const Module &M) {
829 for (const Function &F : M) {
830 if (F.isDeclaration())
831 continue;
832 MachineFunction *MF = GetMF(F);
833 assert(MF);
834 for (MachineBasicBlock &MBB : *MF) {
835 for (MachineInstr &MI : MBB) {
836 for (MachineOperand &Op : MI.operands()) {
837 if (!Op.isReg())
838 continue;
839 Register Reg = Op.getReg();
840 if (MAI.hasRegisterAlias(MF, Reg))
841 continue;
842 MCRegister NewReg = MAI.getNextIDRegister();
843 MAI.setRegisterAlias(MF, Reg, AliasReg: NewReg);
844 }
845 if (MI.getOpcode() != SPIRV::OpExtInst)
846 continue;
847 auto Set = MI.getOperand(i: 2).getImm();
848 auto [It, Inserted] = MAI.ExtInstSetMap.try_emplace(Key: Set);
849 if (Inserted)
850 It->second = MAI.getNextIDRegister();
851 }
852 }
853 }
854}
855
856// RequirementHandler implementations.
857void SPIRV::RequirementHandler::getAndAddRequirements(
858 SPIRV::OperandCategory::OperandCategory Category, uint32_t i,
859 const SPIRVSubtarget &ST) {
860 addRequirements(Req: getSymbolicOperandRequirements(Category, i, ST, Reqs&: *this));
861}
862
863void SPIRV::RequirementHandler::recursiveAddCapabilities(
864 const CapabilityList &ToPrune) {
865 for (const auto &Cap : ToPrune) {
866 AllCaps.insert(V: Cap);
867 CapabilityList ImplicitDecls =
868 getSymbolicOperandCapabilities(Category: OperandCategory::CapabilityOperand, Value: Cap);
869 recursiveAddCapabilities(ToPrune: ImplicitDecls);
870 }
871}
872
873void SPIRV::RequirementHandler::addCapabilities(const CapabilityList &ToAdd) {
874 for (const auto &Cap : ToAdd) {
875 bool IsNewlyInserted = AllCaps.insert(V: Cap).second;
876 if (!IsNewlyInserted) // Don't re-add if it's already been declared.
877 continue;
878 CapabilityList ImplicitDecls =
879 getSymbolicOperandCapabilities(Category: OperandCategory::CapabilityOperand, Value: Cap);
880 recursiveAddCapabilities(ToPrune: ImplicitDecls);
881 MinimalCaps.push_back(Elt: Cap);
882 }
883}
884
885void SPIRV::RequirementHandler::addRequirements(
886 const SPIRV::Requirements &Req) {
887 if (!Req.IsSatisfiable)
888 report_fatal_error(reason: "Adding SPIR-V requirements this target can't satisfy.");
889
890 if (Req.Cap.has_value())
891 addCapabilities(ToAdd: {Req.Cap.value()});
892
893 addExtensions(ToAdd: Req.Exts);
894
895 if (!Req.MinVer.empty()) {
896 if (!MaxVersion.empty() && Req.MinVer > MaxVersion) {
897 LLVM_DEBUG(dbgs() << "Conflicting version requirements: >= " << Req.MinVer
898 << " and <= " << MaxVersion << "\n");
899 report_fatal_error(reason: "Adding SPIR-V requirements that can't be satisfied.");
900 }
901
902 if (MinVersion.empty() || Req.MinVer > MinVersion)
903 MinVersion = Req.MinVer;
904 }
905
906 if (!Req.MaxVer.empty()) {
907 if (!MinVersion.empty() && Req.MaxVer < MinVersion) {
908 LLVM_DEBUG(dbgs() << "Conflicting version requirements: <= " << Req.MaxVer
909 << " and >= " << MinVersion << "\n");
910 report_fatal_error(reason: "Adding SPIR-V requirements that can't be satisfied.");
911 }
912
913 if (MaxVersion.empty() || Req.MaxVer < MaxVersion)
914 MaxVersion = Req.MaxVer;
915 }
916}
917
918void SPIRV::RequirementHandler::checkSatisfiable(
919 const SPIRVSubtarget &ST) const {
920 // Report as many errors as possible before aborting the compilation.
921 bool IsSatisfiable = true;
922 auto TargetVer = ST.getSPIRVVersion();
923
924 if (!MaxVersion.empty() && !TargetVer.empty() && MaxVersion < TargetVer) {
925 LLVM_DEBUG(
926 dbgs() << "Target SPIR-V version too high for required features\n"
927 << "Required max version: " << MaxVersion << " target version "
928 << TargetVer << "\n");
929 IsSatisfiable = false;
930 }
931
932 if (!MinVersion.empty() && !TargetVer.empty() && MinVersion > TargetVer) {
933 LLVM_DEBUG(dbgs() << "Target SPIR-V version too low for required features\n"
934 << "Required min version: " << MinVersion
935 << " target version " << TargetVer << "\n");
936 IsSatisfiable = false;
937 }
938
939 if (!MinVersion.empty() && !MaxVersion.empty() && MinVersion > MaxVersion) {
940 LLVM_DEBUG(
941 dbgs()
942 << "Version is too low for some features and too high for others.\n"
943 << "Required SPIR-V min version: " << MinVersion
944 << " required SPIR-V max version " << MaxVersion << "\n");
945 IsSatisfiable = false;
946 }
947
948 AvoidCapabilitiesSet AvoidCaps;
949 if (!ST.isShader())
950 AvoidCaps.S.insert(V: SPIRV::Capability::Shader);
951 else
952 AvoidCaps.S.insert(V: SPIRV::Capability::Kernel);
953
954 for (auto Cap : MinimalCaps) {
955 if (AvailableCaps.contains(V: Cap) && !AvoidCaps.S.contains(V: Cap))
956 continue;
957 LLVM_DEBUG(dbgs() << "Capability not supported: "
958 << getSymbolicOperandMnemonic(
959 OperandCategory::CapabilityOperand, Cap)
960 << "\n");
961 IsSatisfiable = false;
962 }
963
964 for (auto Ext : AllExtensions) {
965 if (ST.canUseExtension(E: Ext))
966 continue;
967 LLVM_DEBUG(dbgs() << "Extension not supported: "
968 << getSymbolicOperandMnemonic(
969 OperandCategory::ExtensionOperand, Ext)
970 << "\n");
971 IsSatisfiable = false;
972 }
973
974 if (!IsSatisfiable)
975 report_fatal_error(reason: "Unable to meet SPIR-V requirements for this target.");
976}
977
978// Add the given capabilities and all their implicitly defined capabilities too.
979void SPIRV::RequirementHandler::addAvailableCaps(const CapabilityList &ToAdd) {
980 for (const auto Cap : ToAdd)
981 if (AvailableCaps.insert(V: Cap).second)
982 addAvailableCaps(ToAdd: getSymbolicOperandCapabilities(
983 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
984}
985
986void SPIRV::RequirementHandler::removeCapabilityIf(
987 const Capability::Capability ToRemove,
988 const Capability::Capability IfPresent) {
989 if (AllCaps.contains(V: IfPresent)) {
990 AllCaps.erase(V: ToRemove);
991 llvm::erase(C&: MinimalCaps, V: ToRemove);
992 }
993}
994
995namespace llvm {
996namespace SPIRV {
997void RequirementHandler::initAvailableCapabilities(const SPIRVSubtarget &ST) {
998 // Provided by both all supported Vulkan versions and OpenCl.
999 addAvailableCaps(ToAdd: {Capability::Shader, Capability::Linkage, Capability::Int8,
1000 Capability::Int16});
1001
1002 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 3)))
1003 addAvailableCaps(ToAdd: {Capability::GroupNonUniform,
1004 Capability::GroupNonUniformVote,
1005 Capability::GroupNonUniformArithmetic,
1006 Capability::GroupNonUniformBallot,
1007 Capability::GroupNonUniformClustered,
1008 Capability::GroupNonUniformShuffle,
1009 Capability::GroupNonUniformShuffleRelative,
1010 Capability::GroupNonUniformQuad});
1011
1012 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
1013 addAvailableCaps(ToAdd: {Capability::DotProduct, Capability::DotProductInputAll,
1014 Capability::DotProductInput4x8Bit,
1015 Capability::DotProductInput4x8BitPacked,
1016 Capability::DemoteToHelperInvocation});
1017
1018 // Add capabilities enabled by extensions.
1019 for (auto Extension : ST.getAllAvailableExtensions()) {
1020 CapabilityList EnabledCapabilities =
1021 getCapabilitiesEnabledByExtension(Extension);
1022 addAvailableCaps(ToAdd: EnabledCapabilities);
1023 }
1024
1025 if (!ST.isShader()) {
1026 initAvailableCapabilitiesForOpenCL(ST);
1027 return;
1028 }
1029
1030 if (ST.isShader()) {
1031 initAvailableCapabilitiesForVulkan(ST);
1032 return;
1033 }
1034
1035 report_fatal_error(reason: "Unimplemented environment for SPIR-V generation.");
1036}
1037
1038void RequirementHandler::initAvailableCapabilitiesForOpenCL(
1039 const SPIRVSubtarget &ST) {
1040 // Add the min requirements for different OpenCL and SPIR-V versions.
1041 addAvailableCaps(ToAdd: {Capability::Addresses, Capability::Float16Buffer,
1042 Capability::Kernel, Capability::Vector16,
1043 Capability::Groups, Capability::GenericPointer,
1044 Capability::StorageImageWriteWithoutFormat,
1045 Capability::StorageImageReadWithoutFormat});
1046 if (ST.hasOpenCLFullProfile())
1047 addAvailableCaps(ToAdd: {Capability::Int64, Capability::Int64Atomics});
1048 if (ST.hasOpenCLImageSupport()) {
1049 addAvailableCaps(ToAdd: {Capability::ImageBasic, Capability::LiteralSampler,
1050 Capability::Image1D, Capability::SampledBuffer,
1051 Capability::ImageBuffer});
1052 if (ST.isAtLeastOpenCLVer(VerToCompareTo: VersionTuple(2, 0)))
1053 addAvailableCaps(ToAdd: {Capability::ImageReadWrite});
1054 }
1055 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 1)) &&
1056 ST.isAtLeastOpenCLVer(VerToCompareTo: VersionTuple(2, 2)))
1057 addAvailableCaps(ToAdd: {Capability::SubgroupDispatch, Capability::PipeStorage});
1058 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 4)))
1059 addAvailableCaps(ToAdd: {Capability::DenormPreserve, Capability::DenormFlushToZero,
1060 Capability::SignedZeroInfNanPreserve,
1061 Capability::RoundingModeRTE,
1062 Capability::RoundingModeRTZ});
1063 // TODO: verify if this needs some checks.
1064 addAvailableCaps(ToAdd: {Capability::Float16, Capability::Float64});
1065
1066 // TODO: add OpenCL extensions.
1067}
1068
1069void RequirementHandler::initAvailableCapabilitiesForVulkan(
1070 const SPIRVSubtarget &ST) {
1071
1072 // Core in Vulkan 1.1 and earlier.
1073 addAvailableCaps(ToAdd: {Capability::Int64,
1074 Capability::Float16,
1075 Capability::Float64,
1076 Capability::GroupNonUniform,
1077 Capability::Image1D,
1078 Capability::SampledBuffer,
1079 Capability::ImageBuffer,
1080 Capability::UniformBufferArrayDynamicIndexing,
1081 Capability::SampledImageArrayDynamicIndexing,
1082 Capability::StorageBufferArrayDynamicIndexing,
1083 Capability::StorageImageArrayDynamicIndexing,
1084 Capability::DerivativeControl,
1085 Capability::MinLod,
1086 Capability::ImageQuery,
1087 Capability::ImageGatherExtended,
1088 Capability::Addresses,
1089 Capability::VulkanMemoryModelKHR,
1090 Capability::StorageImageExtendedFormats,
1091 Capability::StorageImageMultisample,
1092 Capability::ImageMSArray});
1093
1094 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 3)) ||
1095 ST.canUseExtension(E: Extension::SPV_KHR_variable_pointers))
1096 addAvailableCaps(ToAdd: {Capability::VariablePointersStorageBuffer,
1097 Capability::VariablePointers});
1098
1099 // Became core in Vulkan 1.2
1100 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 5))) {
1101 addAvailableCaps(
1102 ToAdd: {Capability::Int64Atomics, Capability::ShaderNonUniformEXT,
1103 Capability::RuntimeDescriptorArrayEXT,
1104 Capability::InputAttachmentArrayDynamicIndexingEXT,
1105 Capability::UniformTexelBufferArrayDynamicIndexingEXT,
1106 Capability::StorageTexelBufferArrayDynamicIndexingEXT,
1107 Capability::UniformBufferArrayNonUniformIndexingEXT,
1108 Capability::SampledImageArrayNonUniformIndexingEXT,
1109 Capability::StorageBufferArrayNonUniformIndexingEXT,
1110 Capability::StorageImageArrayNonUniformIndexingEXT,
1111 Capability::InputAttachmentArrayNonUniformIndexingEXT,
1112 Capability::UniformTexelBufferArrayNonUniformIndexingEXT,
1113 Capability::StorageTexelBufferArrayNonUniformIndexingEXT});
1114 }
1115
1116 // Became core in Vulkan 1.3
1117 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
1118 addAvailableCaps(ToAdd: {Capability::StorageImageWriteWithoutFormat,
1119 Capability::StorageImageReadWithoutFormat});
1120}
1121
1122} // namespace SPIRV
1123} // namespace llvm
1124
1125// Add the required capabilities from a decoration instruction (including
1126// BuiltIns).
1127static void addOpDecorateReqs(const MachineInstr &MI, unsigned DecIndex,
1128 SPIRV::RequirementHandler &Reqs,
1129 const SPIRVSubtarget &ST) {
1130 int64_t DecOp = MI.getOperand(i: DecIndex).getImm();
1131 auto Dec = static_cast<SPIRV::Decoration::Decoration>(DecOp);
1132 Reqs.addRequirements(Req: getSymbolicOperandRequirements(
1133 Category: SPIRV::OperandCategory::DecorationOperand, i: Dec, ST, Reqs));
1134
1135 if (Dec == SPIRV::Decoration::BuiltIn) {
1136 int64_t BuiltInOp = MI.getOperand(i: DecIndex + 1).getImm();
1137 auto BuiltIn = static_cast<SPIRV::BuiltIn::BuiltIn>(BuiltInOp);
1138 Reqs.addRequirements(Req: getSymbolicOperandRequirements(
1139 Category: SPIRV::OperandCategory::BuiltInOperand, i: BuiltIn, ST, Reqs));
1140 } else if (Dec == SPIRV::Decoration::LinkageAttributes) {
1141 int64_t LinkageOp = MI.getOperand(i: MI.getNumOperands() - 1).getImm();
1142 SPIRV::LinkageType::LinkageType LnkType =
1143 static_cast<SPIRV::LinkageType::LinkageType>(LinkageOp);
1144 if (LnkType == SPIRV::LinkageType::LinkOnceODR)
1145 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_linkonce_odr);
1146 else if (LnkType == SPIRV::LinkageType::WeakAMD) {
1147 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_AMD_weak_linkage);
1148 Reqs.addCapability(ToAdd: SPIRV::Capability::WeakLinkageAMD);
1149 }
1150 } else if (Dec == SPIRV::Decoration::CacheControlLoadINTEL ||
1151 Dec == SPIRV::Decoration::CacheControlStoreINTEL) {
1152 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_cache_controls);
1153 } else if (Dec == SPIRV::Decoration::HostAccessINTEL) {
1154 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_global_variable_host_access);
1155 } else if (Dec == SPIRV::Decoration::InitModeINTEL ||
1156 Dec == SPIRV::Decoration::ImplementInRegisterMapINTEL) {
1157 Reqs.addExtension(
1158 ToAdd: SPIRV::Extension::SPV_INTEL_global_variable_fpga_decorations);
1159 } else if (Dec == SPIRV::Decoration::NonUniformEXT) {
1160 Reqs.addRequirements(Req: SPIRV::Capability::ShaderNonUniformEXT);
1161 } else if (Dec == SPIRV::Decoration::FPMaxErrorDecorationINTEL) {
1162 Reqs.addRequirements(Req: SPIRV::Capability::FPMaxErrorINTEL);
1163 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_fp_max_error);
1164 } else if (Dec == SPIRV::Decoration::FPFastMathMode) {
1165 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2)) {
1166 Reqs.addRequirements(Req: SPIRV::Capability::FloatControls2);
1167 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls2);
1168 }
1169 }
1170}
1171
1172// Add requirements for image handling.
1173static void addOpTypeImageReqs(const MachineInstr &MI,
1174 SPIRV::RequirementHandler &Reqs,
1175 const SPIRVSubtarget &ST) {
1176 assert(MI.getNumOperands() >= 8 && "Insufficient operands for OpTypeImage");
1177 // The operand indices used here are based on the OpTypeImage layout, which
1178 // the MachineInstr follows as well.
1179 int64_t ImgFormatOp = MI.getOperand(i: 7).getImm();
1180 auto ImgFormat = static_cast<SPIRV::ImageFormat::ImageFormat>(ImgFormatOp);
1181 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ImageFormatOperand,
1182 i: ImgFormat, ST);
1183
1184 bool IsArrayed = MI.getOperand(i: 4).getImm() == 1;
1185 bool IsMultisampled = MI.getOperand(i: 5).getImm() == 1;
1186 bool NoSampler = MI.getOperand(i: 6).getImm() == 2;
1187 // Add dimension requirements.
1188 assert(MI.getOperand(2).isImm());
1189 switch (MI.getOperand(i: 2).getImm()) {
1190 case SPIRV::Dim::DIM_1D:
1191 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::Image1D
1192 : SPIRV::Capability::Sampled1D);
1193 break;
1194 case SPIRV::Dim::DIM_2D:
1195 if (IsMultisampled && NoSampler)
1196 Reqs.addRequirements(Req: SPIRV::Capability::StorageImageMultisample);
1197 if (IsMultisampled && IsArrayed)
1198 Reqs.addRequirements(Req: SPIRV::Capability::ImageMSArray);
1199 break;
1200 case SPIRV::Dim::DIM_3D:
1201 break;
1202 case SPIRV::Dim::DIM_Cube:
1203 Reqs.addRequirements(Req: SPIRV::Capability::Shader);
1204 if (IsArrayed)
1205 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageCubeArray
1206 : SPIRV::Capability::SampledCubeArray);
1207 break;
1208 case SPIRV::Dim::DIM_Rect:
1209 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageRect
1210 : SPIRV::Capability::SampledRect);
1211 break;
1212 case SPIRV::Dim::DIM_Buffer:
1213 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageBuffer
1214 : SPIRV::Capability::SampledBuffer);
1215 break;
1216 case SPIRV::Dim::DIM_SubpassData:
1217 Reqs.addRequirements(Req: SPIRV::Capability::InputAttachment);
1218 break;
1219 }
1220
1221 // Has optional access qualifier.
1222 if (!ST.isShader()) {
1223 if (MI.getNumOperands() > 8 &&
1224 MI.getOperand(i: 8).getImm() == SPIRV::AccessQualifier::ReadWrite)
1225 Reqs.addRequirements(Req: SPIRV::Capability::ImageReadWrite);
1226 else
1227 Reqs.addRequirements(Req: SPIRV::Capability::ImageBasic);
1228 }
1229}
1230
1231static bool isBFloat16Type(SPIRVTypeInst TypeDef) {
1232 return TypeDef && TypeDef->getNumOperands() == 3 &&
1233 TypeDef->getOpcode() == SPIRV::OpTypeFloat &&
1234 TypeDef->getOperand(i: 1).getImm() == 16 &&
1235 TypeDef->getOperand(i: 2).getImm() == SPIRV::FPEncoding::BFloat16KHR;
1236}
1237
1238// Registers created after isel (e.g. by MachineCSE) may lack registry entries.
1239// Fall back to the defining instruction's result type.
1240static SPIRVTypeInst getRegType(Register Reg, const MachineFunction &MF,
1241 const SPIRVGlobalRegistry *GR) {
1242 if (SPIRVTypeInst Ty = GR->getSPIRVTypeForVReg(VReg: Reg, MF: &MF))
1243 return Ty;
1244 const MachineRegisterInfo &MRI = MF.getRegInfo();
1245 return MRI.getVRegDef(Reg: MRI.getVRegDef(Reg)->getOperand(i: 1).getReg());
1246}
1247
1248// Add requirements for handling atomic float instructions
1249#define ATOM_FLT_REQ_EXT_MSG(ExtName) \
1250 "The atomic float instruction requires the following SPIR-V " \
1251 "extension: SPV_EXT_shader_atomic_float" ExtName
1252static void AddAtomicVectorFloatRequirements(const MachineInstr &MI,
1253 SPIRV::RequirementHandler &Reqs,
1254 const SPIRVSubtarget &ST) {
1255 SPIRVTypeInst VecTypeDef =
1256 MI.getMF()->getRegInfo().getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1257
1258 const unsigned Rank = VecTypeDef->getOperand(i: 2).getImm();
1259 if (Rank != 2 && Rank != 4)
1260 reportFatalUsageError(reason: "Result type of an atomic vector float instruction "
1261 "must be a 2-component or 4 component vector");
1262
1263 SPIRVTypeInst EltTypeDef =
1264 MI.getMF()->getRegInfo().getVRegDef(Reg: VecTypeDef->getOperand(i: 1).getReg());
1265
1266 if (EltTypeDef->getOpcode() != SPIRV::OpTypeFloat ||
1267 EltTypeDef->getOperand(i: 1).getImm() != 16)
1268 reportFatalUsageError(
1269 reason: "The element type for the result type of an atomic vector float "
1270 "instruction must be a 16-bit floating-point scalar");
1271
1272 // The extension is defined for fp16, but the AMD target lets a bf16 vector
1273 // use the same instruction so it can lower to a packed bf16 atomic.
1274 if (isBFloat16Type(TypeDef: EltTypeDef) &&
1275 ST.getTargetTriple().getVendor() != Triple::AMD)
1276 reportFatalUsageError(
1277 reason: "The element type for the result type of an atomic vector float "
1278 "instruction cannot be a bfloat16 scalar");
1279 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_NV_shader_atomic_fp16_vector))
1280 reportFatalUsageError(
1281 reason: "The atomic float16 vector instruction requires the following SPIR-V "
1282 "extension: SPV_NV_shader_atomic_fp16_vector");
1283
1284 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_NV_shader_atomic_fp16_vector);
1285 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16VectorNV);
1286}
1287
1288static void AddAtomicFloatRequirements(const MachineInstr &MI,
1289 SPIRV::RequirementHandler &Reqs,
1290 const SPIRVSubtarget &ST) {
1291 assert(MI.getOperand(1).isReg() &&
1292 "Expect register operand in atomic float instruction");
1293 Register TypeReg = MI.getOperand(i: 1).getReg();
1294 SPIRVTypeInst TypeDef = MI.getMF()->getRegInfo().getVRegDef(Reg: TypeReg);
1295
1296 if (isVectorType(SPVTy: TypeDef))
1297 return AddAtomicVectorFloatRequirements(MI, Reqs, ST);
1298
1299 if (TypeDef->getOpcode() != SPIRV::OpTypeFloat)
1300 report_fatal_error(reason: "Result type of an atomic float instruction must be a "
1301 "floating-point type scalar");
1302
1303 unsigned BitWidth = TypeDef->getOperand(i: 1).getImm();
1304 unsigned Op = MI.getOpcode();
1305 if (Op == SPIRV::OpAtomicFAddEXT) {
1306 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_shader_atomic_float_add))
1307 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("_add"), gen_crash_diag: false);
1308 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float_add);
1309 switch (BitWidth) {
1310 case 16:
1311 if (isBFloat16Type(TypeDef)) {
1312 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1313 report_fatal_error(
1314 reason: "The atomic bfloat16 instruction requires the following SPIR-V "
1315 "extension: SPV_INTEL_16bit_atomics",
1316 gen_crash_diag: false);
1317 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1318 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicBFloat16AddINTEL);
1319 } else {
1320 if (!ST.canUseExtension(
1321 E: SPIRV::Extension::SPV_EXT_shader_atomic_float16_add))
1322 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("16_add"), gen_crash_diag: false);
1323 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float16_add);
1324 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16AddEXT);
1325 }
1326 break;
1327 case 32:
1328 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat32AddEXT);
1329 break;
1330 case 64:
1331 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat64AddEXT);
1332 break;
1333 default:
1334 report_fatal_error(
1335 reason: "Unexpected floating-point type width in atomic float instruction");
1336 }
1337 } else {
1338 if (!ST.canUseExtension(
1339 E: SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max))
1340 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("_min_max"), gen_crash_diag: false);
1341 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max);
1342 switch (BitWidth) {
1343 case 16:
1344 if (isBFloat16Type(TypeDef)) {
1345 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1346 report_fatal_error(
1347 reason: "The atomic bfloat16 instruction requires the following SPIR-V "
1348 "extension: SPV_INTEL_16bit_atomics",
1349 gen_crash_diag: false);
1350 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1351 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicBFloat16MinMaxINTEL);
1352 } else {
1353 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16MinMaxEXT);
1354 }
1355 break;
1356 case 32:
1357 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat32MinMaxEXT);
1358 break;
1359 case 64:
1360 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat64MinMaxEXT);
1361 break;
1362 default:
1363 report_fatal_error(
1364 reason: "Unexpected floating-point type width in atomic float instruction");
1365 }
1366 }
1367}
1368
1369bool isUniformTexelBuffer(MachineInstr *ImageInst) {
1370 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1371 return false;
1372 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1373 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1374 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 1;
1375}
1376
1377bool isStorageTexelBuffer(MachineInstr *ImageInst) {
1378 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1379 return false;
1380 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1381 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1382 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 2;
1383}
1384
1385bool isSampledImage(MachineInstr *ImageInst) {
1386 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1387 return false;
1388 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1389 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1390 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 1;
1391}
1392
1393bool isInputAttachment(MachineInstr *ImageInst) {
1394 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1395 return false;
1396 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1397 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1398 return Dim == SPIRV::Dim::DIM_SubpassData && Sampled == 2;
1399}
1400
1401bool isStorageImage(MachineInstr *ImageInst) {
1402 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1403 return false;
1404 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1405 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1406 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 2;
1407}
1408
1409bool isCombinedImageSampler(MachineInstr *SampledImageInst) {
1410 if (SampledImageInst->getOpcode() != SPIRV::OpTypeSampledImage)
1411 return false;
1412
1413 const MachineRegisterInfo &MRI = SampledImageInst->getMF()->getRegInfo();
1414 Register ImageReg = SampledImageInst->getOperand(i: 1).getReg();
1415 auto *ImageInst = MRI.getUniqueVRegDef(Reg: ImageReg);
1416 return isSampledImage(ImageInst);
1417}
1418
1419bool hasNonUniformDecoration(Register Reg, const MachineRegisterInfo &MRI) {
1420 for (const auto &MI : MRI.reg_instructions(Reg)) {
1421 if (MI.getOpcode() != SPIRV::OpDecorate)
1422 continue;
1423
1424 uint32_t Dec = MI.getOperand(i: 1).getImm();
1425 if (Dec == SPIRV::Decoration::NonUniformEXT)
1426 return true;
1427 }
1428 return false;
1429}
1430
1431void addOpAccessChainReqs(const MachineInstr &Instr,
1432 SPIRV::RequirementHandler &Handler,
1433 const SPIRVSubtarget &Subtarget) {
1434 const MachineRegisterInfo &MRI = Instr.getMF()->getRegInfo();
1435 // Get the result type. If it is an image type, then the shader uses
1436 // descriptor indexing. The appropriate capabilities will be added based
1437 // on the specifics of the image.
1438 Register ResTypeReg = Instr.getOperand(i: 1).getReg();
1439 MachineInstr *ResTypeInst = MRI.getUniqueVRegDef(Reg: ResTypeReg);
1440
1441 assert(ResTypeInst->getOpcode() == SPIRV::OpTypePointer);
1442 uint32_t StorageClass = ResTypeInst->getOperand(i: 1).getImm();
1443 if (StorageClass != SPIRV::StorageClass::StorageClass::UniformConstant &&
1444 StorageClass != SPIRV::StorageClass::StorageClass::Uniform &&
1445 StorageClass != SPIRV::StorageClass::StorageClass::StorageBuffer) {
1446 return;
1447 }
1448
1449 bool IsNonUniform =
1450 hasNonUniformDecoration(Reg: Instr.getOperand(i: 0).getReg(), MRI);
1451
1452 auto FirstIndexReg = Instr.getOperand(i: 3).getReg();
1453 bool FirstIndexIsConstant =
1454 Subtarget.getInstrInfo()->isConstantInstr(MI: *MRI.getVRegDef(Reg: FirstIndexReg));
1455
1456 if (StorageClass == SPIRV::StorageClass::StorageClass::StorageBuffer) {
1457 if (IsNonUniform)
1458 Handler.addRequirements(
1459 Req: SPIRV::Capability::StorageBufferArrayNonUniformIndexingEXT);
1460 else if (!FirstIndexIsConstant)
1461 Handler.addRequirements(
1462 Req: SPIRV::Capability::StorageBufferArrayDynamicIndexing);
1463 return;
1464 }
1465
1466 Register PointeeTypeReg = ResTypeInst->getOperand(i: 2).getReg();
1467 MachineInstr *PointeeType = MRI.getUniqueVRegDef(Reg: PointeeTypeReg);
1468 if (PointeeType->getOpcode() != SPIRV::OpTypeImage &&
1469 PointeeType->getOpcode() != SPIRV::OpTypeSampledImage &&
1470 PointeeType->getOpcode() != SPIRV::OpTypeSampler) {
1471 return;
1472 }
1473
1474 if (isUniformTexelBuffer(ImageInst: PointeeType)) {
1475 if (IsNonUniform)
1476 Handler.addRequirements(
1477 Req: SPIRV::Capability::UniformTexelBufferArrayNonUniformIndexingEXT);
1478 else if (!FirstIndexIsConstant)
1479 Handler.addRequirements(
1480 Req: SPIRV::Capability::UniformTexelBufferArrayDynamicIndexingEXT);
1481 } else if (isInputAttachment(ImageInst: PointeeType)) {
1482 if (IsNonUniform)
1483 Handler.addRequirements(
1484 Req: SPIRV::Capability::InputAttachmentArrayNonUniformIndexingEXT);
1485 else if (!FirstIndexIsConstant)
1486 Handler.addRequirements(
1487 Req: SPIRV::Capability::InputAttachmentArrayDynamicIndexingEXT);
1488 } else if (isStorageTexelBuffer(ImageInst: PointeeType)) {
1489 if (IsNonUniform)
1490 Handler.addRequirements(
1491 Req: SPIRV::Capability::StorageTexelBufferArrayNonUniformIndexingEXT);
1492 else if (!FirstIndexIsConstant)
1493 Handler.addRequirements(
1494 Req: SPIRV::Capability::StorageTexelBufferArrayDynamicIndexingEXT);
1495 } else if (isSampledImage(ImageInst: PointeeType) ||
1496 isCombinedImageSampler(SampledImageInst: PointeeType) ||
1497 PointeeType->getOpcode() == SPIRV::OpTypeSampler) {
1498 if (IsNonUniform)
1499 Handler.addRequirements(
1500 Req: SPIRV::Capability::SampledImageArrayNonUniformIndexingEXT);
1501 else if (!FirstIndexIsConstant)
1502 Handler.addRequirements(
1503 Req: SPIRV::Capability::SampledImageArrayDynamicIndexing);
1504 } else if (isStorageImage(ImageInst: PointeeType)) {
1505 if (IsNonUniform)
1506 Handler.addRequirements(
1507 Req: SPIRV::Capability::StorageImageArrayNonUniformIndexingEXT);
1508 else if (!FirstIndexIsConstant)
1509 Handler.addRequirements(
1510 Req: SPIRV::Capability::StorageImageArrayDynamicIndexing);
1511 }
1512}
1513
1514static bool isImageTypeWithUnknownFormat(SPIRVTypeInst TypeInst) {
1515 if (TypeInst->getOpcode() != SPIRV::OpTypeImage)
1516 return false;
1517 assert(TypeInst->getOperand(7).isImm() && "The image format must be an imm.");
1518 return TypeInst->getOperand(i: 7).getImm() == 0;
1519}
1520
1521static void AddDotProductRequirements(const MachineInstr &MI,
1522 SPIRV::RequirementHandler &Reqs,
1523 const SPIRVSubtarget &ST) {
1524 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product))
1525 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_integer_dot_product);
1526 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProduct);
1527
1528 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1529 assert(MI.getOperand(2).isReg() && "Unexpected operand in dot");
1530 SPIRVTypeInst TypeDef = getRegType(Reg: MI.getOperand(i: 2).getReg(), MF: *MI.getMF(),
1531 GR: ST.getSPIRVGlobalRegistry());
1532 assert(TypeDef && "Unexpected type instruction for dot input");
1533 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {
1534 assert(TypeDef->getOperand(1).getImm() == 32);
1535 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInput4x8BitPacked);
1536 } else if (isVectorType(SPVTy: TypeDef)) {
1537 SPIRVTypeInst ScalarTypeDef =
1538 MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
1539 assert(ScalarTypeDef->getOpcode() == SPIRV::OpTypeInt);
1540 if (ScalarTypeDef->getOperand(i: 1).getImm() == 8) {
1541 assert(TypeDef->getOperand(2).getImm() == 4 &&
1542 "Dot operand of 8-bit integer type requires 4 components");
1543 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInput4x8Bit);
1544 } else {
1545 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInputAll);
1546 }
1547 }
1548}
1549
1550void addPrintfRequirements(const MachineInstr &MI,
1551 SPIRV::RequirementHandler &Reqs,
1552 const SPIRVSubtarget &ST) {
1553 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1554 SPIRVTypeInst PtrType =
1555 GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 4).getReg(), MF: MI.getMF());
1556 if (PtrType) {
1557 MachineOperand ASOp = PtrType->getOperand(i: 1);
1558 if (ASOp.isImm()) {
1559 unsigned AddrSpace = ASOp.getImm();
1560 if (AddrSpace != SPIRV::StorageClass::UniformConstant) {
1561 if (!ST.canUseExtension(
1562 E: SPIRV::Extension::
1563 SPV_EXT_relaxed_printf_string_address_space)) {
1564 report_fatal_error(reason: "SPV_EXT_relaxed_printf_string_address_space is "
1565 "required because printf uses a format string not "
1566 "in constant address space.",
1567 gen_crash_diag: false);
1568 }
1569 Reqs.addExtension(
1570 ToAdd: SPIRV::Extension::SPV_EXT_relaxed_printf_string_address_space);
1571 }
1572 }
1573 }
1574}
1575
1576static void addImageOperandReqs(const MachineInstr &MI,
1577 SPIRV::RequirementHandler &Reqs,
1578 const SPIRVSubtarget &ST, unsigned OpIdx) {
1579 if (MI.getNumOperands() <= OpIdx)
1580 return;
1581 uint32_t Mask = MI.getOperand(i: OpIdx).getImm();
1582 for (uint32_t I = 0; I < 32; ++I)
1583 if (Mask & (1U << I))
1584 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ImageOperandOperand,
1585 i: 1U << I, ST);
1586}
1587
1588static inline void maybeAddScatterGatherReq(const MachineInstr &MI,
1589 SPIRV::RequirementHandler &Reqs,
1590 const SPIRVSubtarget &ST) {
1591 assert(MI.getOperand(1).isReg());
1592 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1593 SPIRVTypeInst ElemTypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1594 if (ElemTypeDef->getOpcode() == SPIRV::OpTypePointer &&
1595 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
1596 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_masked_gather_scatter);
1597 Reqs.addCapability(ToAdd: SPIRV::Capability::MaskedGatherScatterINTEL);
1598 }
1599}
1600
1601void addInstrRequirements(const MachineInstr &MI,
1602 SPIRV::ModuleAnalysisInfo &MAI,
1603 const SPIRVSubtarget &ST) {
1604 SPIRV::RequirementHandler &Reqs = MAI.Reqs;
1605 unsigned Op = MI.getOpcode();
1606 switch (Op) {
1607 case SPIRV::OpMemoryModel: {
1608 int64_t Addr = MI.getOperand(i: 0).getImm();
1609 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::AddressingModelOperand,
1610 i: Addr, ST);
1611 int64_t Mem = MI.getOperand(i: 1).getImm();
1612 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::MemoryModelOperand, i: Mem,
1613 ST);
1614 break;
1615 }
1616 case SPIRV::OpEntryPoint: {
1617 int64_t Exe = MI.getOperand(i: 0).getImm();
1618 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ExecutionModelOperand,
1619 i: Exe, ST);
1620 break;
1621 }
1622 case SPIRV::OpExecutionMode:
1623 case SPIRV::OpExecutionModeId: {
1624 int64_t Exe = MI.getOperand(i: 1).getImm();
1625 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ExecutionModeOperand,
1626 i: Exe, ST);
1627 break;
1628 }
1629 case SPIRV::OpTypeMatrix:
1630 Reqs.addCapability(ToAdd: SPIRV::Capability::Matrix);
1631 break;
1632 case SPIRV::OpTypeInt: {
1633 unsigned BitWidth = MI.getOperand(i: 1).getImm();
1634 if (BitWidth == 64)
1635 Reqs.addCapability(ToAdd: SPIRV::Capability::Int64);
1636 else if (BitWidth == 16)
1637 Reqs.addCapability(ToAdd: SPIRV::Capability::Int16);
1638 else if (BitWidth == 8)
1639 Reqs.addCapability(ToAdd: SPIRV::Capability::Int8);
1640 else if (BitWidth == 4 &&
1641 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4)) {
1642 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_int4);
1643 Reqs.addCapability(ToAdd: SPIRV::Capability::Int4TypeINTEL);
1644 } else if (BitWidth != 32) {
1645 if (!ST.canUseExtension(
1646 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers))
1647 reportFatalUsageError(
1648 reason: "OpTypeInt type with a width other than 8, 16, 32 or 64 bits "
1649 "requires the following SPIR-V extension: "
1650 "SPV_ALTERA_arbitrary_precision_integers");
1651 Reqs.addExtension(
1652 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
1653 Reqs.addCapability(ToAdd: SPIRV::Capability::ArbitraryPrecisionIntegersALTERA);
1654 }
1655 break;
1656 }
1657 case SPIRV::OpDot: {
1658 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1659 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1660 if (isBFloat16Type(TypeDef))
1661 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16DotProductKHR);
1662 break;
1663 }
1664 case SPIRV::OpTypeFloat: {
1665 unsigned BitWidth = MI.getOperand(i: 1).getImm();
1666 if (BitWidth == 64)
1667 Reqs.addCapability(ToAdd: SPIRV::Capability::Float64);
1668 else if (BitWidth == 16) {
1669 if (isBFloat16Type(TypeDef: &MI)) {
1670 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_bfloat16))
1671 report_fatal_error(reason: "OpTypeFloat type with bfloat requires the "
1672 "following SPIR-V extension: SPV_KHR_bfloat16",
1673 gen_crash_diag: false);
1674 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_bfloat16);
1675 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16TypeKHR);
1676 } else {
1677 Reqs.addCapability(ToAdd: SPIRV::Capability::Float16);
1678 }
1679 }
1680 break;
1681 }
1682 case SPIRV::OpTypeVector: {
1683 unsigned NumComponents = MI.getOperand(i: 2).getImm();
1684 if (NumComponents == 8 || NumComponents == 16)
1685 Reqs.addCapability(ToAdd: SPIRV::Capability::Vector16);
1686 else if (requiresLongVectorEXT(NumComponents))
1687 // Such widths are only expressible as OpTypeVectorIdEXT.
1688 reportFatalUsageError(
1689 reason: "OpTypeVector with " + Twine(NumComponents) +
1690 " components requires the following SPIR-V extension: "
1691 "SPV_EXT_long_vector");
1692
1693 maybeAddScatterGatherReq(MI, Reqs, ST);
1694 break;
1695 }
1696 case SPIRV::OpTypeVectorIdEXT: {
1697 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector))
1698 reportFatalUsageError(reason: "OpTypeVectorIdEXT requires the following SPIR-V "
1699 "extension: SPV_EXT_long_vector extension");
1700 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_long_vector);
1701 Reqs.addCapability(ToAdd: SPIRV::Capability::LongVectorEXT);
1702 maybeAddScatterGatherReq(MI, Reqs, ST);
1703 break;
1704 }
1705 case SPIRV::OpTypePointer: {
1706 auto SC = MI.getOperand(i: 1).getImm();
1707 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::StorageClassOperand, i: SC,
1708 ST);
1709 // If it's a type of pointer to float16 targeting OpenCL, add Float16Buffer
1710 // capability.
1711 if (ST.isShader())
1712 break;
1713 assert(MI.getOperand(2).isReg());
1714 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1715 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
1716 if ((TypeDef->getNumOperands() == 2) &&
1717 (TypeDef->getOpcode() == SPIRV::OpTypeFloat) &&
1718 (TypeDef->getOperand(i: 1).getImm() == 16))
1719 Reqs.addCapability(ToAdd: SPIRV::Capability::Float16Buffer);
1720 break;
1721 }
1722 case SPIRV::OpExtInst: {
1723 if (MI.getOperand(i: 2).getImm() ==
1724 static_cast<int64_t>(
1725 SPIRV::InstructionSet::NonSemantic_Shader_DebugInfo_100)) {
1726 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_non_semantic_info);
1727 break;
1728 }
1729 if (MI.getOperand(i: 3).getImm() ==
1730 static_cast<int64_t>(SPIRV::OpenCLExtInst::printf)) {
1731 addPrintfRequirements(MI, Reqs, ST);
1732 break;
1733 }
1734 if (MI.getOperand(i: 2).getImm() ==
1735 static_cast<int64_t>(SPIRV::InstructionSet::OpenCL_std)) {
1736 const MachineFunction *MF = MI.getMF();
1737 const MachineRegisterInfo &MRI = MF->getRegInfo();
1738 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1739
1740 auto IsBFloat16 = [&](SPIRVTypeInst TypeDef) {
1741 if (TypeDef && TypeDef->getOpcode() == SPIRV::OpTypeVector)
1742 TypeDef = MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
1743 return isBFloat16Type(TypeDef);
1744 };
1745
1746 // Result type is operand 1; arguments start at operand 4.
1747 bool UsesBFloat16 = IsBFloat16(MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg()));
1748 for (unsigned I = 4, E = MI.getNumOperands(); I < E && !UsesBFloat16;
1749 ++I) {
1750 const MachineOperand &MO = MI.getOperand(i: I);
1751 if (MO.isReg())
1752 UsesBFloat16 = IsBFloat16(GR->getResultType(
1753 VReg: MO.getReg(), MF: const_cast<MachineFunction *>(MF)));
1754 }
1755
1756 if (UsesBFloat16) {
1757 if (!ST.canUseExtension(
1758 E: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic)) {
1759 reportUnsupported(
1760 MI, Msg: "OpenCL Extended instructions with bfloat16 require the "
1761 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic");
1762 break;
1763 }
1764 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);
1765 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ArithmeticINTEL);
1766 }
1767 }
1768 break;
1769 }
1770 case SPIRV::OpAliasDomainDeclINTEL:
1771 case SPIRV::OpAliasScopeDeclINTEL:
1772 case SPIRV::OpAliasScopeListDeclINTEL: {
1773 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_memory_access_aliasing);
1774 Reqs.addCapability(ToAdd: SPIRV::Capability::MemoryAccessAliasingINTEL);
1775 break;
1776 }
1777 case SPIRV::OpBitReverse:
1778 case SPIRV::OpBitFieldInsert:
1779 case SPIRV::OpBitFieldSExtract:
1780 case SPIRV::OpBitFieldUExtract:
1781 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions)) {
1782 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
1783 break;
1784 }
1785 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_bit_instructions);
1786 Reqs.addCapability(ToAdd: SPIRV::Capability::BitInstructions);
1787 break;
1788 case SPIRV::OpTypeRuntimeArray:
1789 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
1790 break;
1791 case SPIRV::OpTypeOpaque:
1792 case SPIRV::OpTypeEvent:
1793 Reqs.addCapability(ToAdd: SPIRV::Capability::Kernel);
1794 break;
1795 case SPIRV::OpTypePipe:
1796 case SPIRV::OpTypeReserveId:
1797 Reqs.addCapability(ToAdd: SPIRV::Capability::Pipes);
1798 break;
1799 case SPIRV::OpTypeDeviceEvent:
1800 case SPIRV::OpTypeQueue:
1801 case SPIRV::OpBuildNDRange:
1802 case SPIRV::OpEnqueueKernel:
1803 case SPIRV::OpGetKernelNDrangeSubGroupCount:
1804 case SPIRV::OpGetKernelNDrangeMaxSubGroupSize:
1805 case SPIRV::OpGetKernelWorkGroupSize:
1806 case SPIRV::OpGetKernelPreferredWorkGroupSizeMultiple:
1807 Reqs.addCapability(ToAdd: SPIRV::Capability::DeviceEnqueue);
1808 break;
1809 case SPIRV::OpDecorate:
1810 case SPIRV::OpDecorateId:
1811 case SPIRV::OpDecorateString:
1812 addOpDecorateReqs(MI, DecIndex: 1, Reqs, ST);
1813 break;
1814 case SPIRV::OpMemberDecorate:
1815 case SPIRV::OpMemberDecorateString:
1816 addOpDecorateReqs(MI, DecIndex: 2, Reqs, ST);
1817 break;
1818 case SPIRV::OpInBoundsPtrAccessChain:
1819 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
1820 break;
1821 case SPIRV::OpConstantSampler:
1822 Reqs.addCapability(ToAdd: SPIRV::Capability::LiteralSampler);
1823 break;
1824 case SPIRV::OpInBoundsAccessChain:
1825 case SPIRV::OpAccessChain:
1826 addOpAccessChainReqs(Instr: MI, Handler&: Reqs, Subtarget: ST);
1827 break;
1828 case SPIRV::OpTypeImage:
1829 addOpTypeImageReqs(MI, Reqs, ST);
1830 break;
1831 case SPIRV::OpTypeSampler:
1832 if (!ST.isShader()) {
1833 Reqs.addCapability(ToAdd: SPIRV::Capability::ImageBasic);
1834 }
1835 break;
1836 case SPIRV::OpTypeForwardPointer:
1837 // TODO: check if it's OpenCL's kernel.
1838 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
1839 break;
1840 case SPIRV::OpAtomicFlagTestAndSet:
1841 case SPIRV::OpAtomicLoad:
1842 case SPIRV::OpAtomicStore:
1843 case SPIRV::OpAtomicExchange:
1844 case SPIRV::OpAtomicCompareExchange:
1845 case SPIRV::OpAtomicCompareExchangeWeak:
1846 case SPIRV::OpAtomicIIncrement:
1847 case SPIRV::OpAtomicIDecrement:
1848 case SPIRV::OpAtomicIAdd:
1849 case SPIRV::OpAtomicISub:
1850 case SPIRV::OpAtomicUMin:
1851 case SPIRV::OpAtomicUMax:
1852 case SPIRV::OpAtomicSMin:
1853 case SPIRV::OpAtomicSMax:
1854 case SPIRV::OpAtomicAnd:
1855 case SPIRV::OpAtomicOr:
1856 case SPIRV::OpAtomicXor: {
1857 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1858 SPIRVTypeInst TypeDef;
1859 if (Op == SPIRV::OpAtomicStore) {
1860 assert(MI.getOperand(3).isReg());
1861 TypeDef = getRegType(Reg: MI.getOperand(i: 3).getReg(), MF: *MI.getMF(),
1862 GR: ST.getSPIRVGlobalRegistry());
1863 assert(TypeDef && "Unexpected type instruction for OpAtomicStore");
1864 } else {
1865 assert(MI.getOperand(1).isReg() && "Unexpected operand in atomic");
1866 TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1867 }
1868
1869 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {
1870 unsigned BitWidth = TypeDef->getOperand(i: 1).getImm();
1871 if (BitWidth == 64)
1872 Reqs.addCapability(ToAdd: SPIRV::Capability::Int64Atomics);
1873 else if (BitWidth == 16) {
1874 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1875 report_fatal_error(
1876 reason: "16-bit integer atomic operations require the following SPIR-V "
1877 "extension: SPV_INTEL_16bit_atomics",
1878 gen_crash_diag: false);
1879 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1880 switch (Op) {
1881 case SPIRV::OpAtomicLoad:
1882 case SPIRV::OpAtomicStore:
1883 case SPIRV::OpAtomicExchange:
1884 case SPIRV::OpAtomicCompareExchange:
1885 case SPIRV::OpAtomicCompareExchangeWeak:
1886 Reqs.addCapability(
1887 ToAdd: SPIRV::Capability::AtomicInt16CompareExchangeINTEL);
1888 break;
1889 default:
1890 Reqs.addCapability(ToAdd: SPIRV::Capability::Int16AtomicsINTEL);
1891 break;
1892 }
1893 }
1894 } else if (isBFloat16Type(TypeDef)) {
1895 if (is_contained(Set: {SPIRV::OpAtomicLoad, SPIRV::OpAtomicStore,
1896 SPIRV::OpAtomicExchange},
1897 Element: Op)) {
1898 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1899 report_fatal_error(
1900 reason: "The atomic bfloat16 instruction requires the following SPIR-V "
1901 "extension: SPV_INTEL_16bit_atomics",
1902 gen_crash_diag: false);
1903 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1904 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicBFloat16LoadStoreINTEL);
1905 }
1906 }
1907 break;
1908 }
1909 case SPIRV::OpGroupNonUniformIAdd:
1910 case SPIRV::OpGroupNonUniformFAdd:
1911 case SPIRV::OpGroupNonUniformIMul:
1912 case SPIRV::OpGroupNonUniformFMul:
1913 case SPIRV::OpGroupNonUniformSMin:
1914 case SPIRV::OpGroupNonUniformUMin:
1915 case SPIRV::OpGroupNonUniformFMin:
1916 case SPIRV::OpGroupNonUniformSMax:
1917 case SPIRV::OpGroupNonUniformUMax:
1918 case SPIRV::OpGroupNonUniformFMax:
1919 case SPIRV::OpGroupNonUniformBitwiseAnd:
1920 case SPIRV::OpGroupNonUniformBitwiseOr:
1921 case SPIRV::OpGroupNonUniformBitwiseXor:
1922 case SPIRV::OpGroupNonUniformLogicalAnd:
1923 case SPIRV::OpGroupNonUniformLogicalOr:
1924 case SPIRV::OpGroupNonUniformLogicalXor: {
1925 assert(MI.getOperand(3).isImm());
1926 int64_t GroupOp = MI.getOperand(i: 3).getImm();
1927 switch (GroupOp) {
1928 case SPIRV::GroupOperation::Reduce:
1929 case SPIRV::GroupOperation::InclusiveScan:
1930 case SPIRV::GroupOperation::ExclusiveScan:
1931 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformArithmetic);
1932 break;
1933 case SPIRV::GroupOperation::ClusteredReduce:
1934 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformClustered);
1935 break;
1936 case SPIRV::GroupOperation::PartitionedReduceNV:
1937 case SPIRV::GroupOperation::PartitionedInclusiveScanNV:
1938 case SPIRV::GroupOperation::PartitionedExclusiveScanNV:
1939 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformPartitionedNV);
1940 break;
1941 }
1942 break;
1943 }
1944 case SPIRV::OpGroupNonUniformQuadSwap:
1945 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformQuad);
1946 break;
1947 case SPIRV::OpImageQueryLod:
1948 Reqs.addCapability(ToAdd: SPIRV::Capability::ImageQuery);
1949 break;
1950 case SPIRV::OpImageQuerySize:
1951 case SPIRV::OpImageQuerySizeLod:
1952 case SPIRV::OpImageQueryLevels:
1953 case SPIRV::OpImageQuerySamples:
1954 if (ST.isShader())
1955 Reqs.addCapability(ToAdd: SPIRV::Capability::ImageQuery);
1956 break;
1957 case SPIRV::OpImageQueryFormat: {
1958 Register ResultReg = MI.getOperand(i: 0).getReg();
1959 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1960 static const unsigned CompareOps[] = {
1961 SPIRV::OpIEqual, SPIRV::OpINotEqual,
1962 SPIRV::OpUGreaterThan, SPIRV::OpUGreaterThanEqual,
1963 SPIRV::OpULessThan, SPIRV::OpULessThanEqual,
1964 SPIRV::OpSGreaterThan, SPIRV::OpSGreaterThanEqual,
1965 SPIRV::OpSLessThan, SPIRV::OpSLessThanEqual};
1966
1967 auto CheckAndAddExtension = [&](int64_t ImmVal) {
1968 if (ImmVal == 4323 || ImmVal == 4324) {
1969 if (ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_image_raw10_raw12))
1970 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_image_raw10_raw12);
1971 else
1972 report_fatal_error(reason: "This requires the "
1973 "SPV_EXT_image_raw10_raw12 extension");
1974 }
1975 };
1976
1977 for (MachineInstr &UseInst : MRI.use_instructions(Reg: ResultReg)) {
1978 unsigned Opc = UseInst.getOpcode();
1979
1980 if (Opc == SPIRV::OpSwitch) {
1981 for (const MachineOperand &Op : UseInst.operands())
1982 if (Op.isImm())
1983 CheckAndAddExtension(Op.getImm());
1984 } else if (llvm::is_contained(Range: CompareOps, Element: Opc)) {
1985 for (unsigned i = 1; i < UseInst.getNumOperands(); ++i) {
1986 Register UseReg = UseInst.getOperand(i).getReg();
1987 MachineInstr *ConstInst = MRI.getVRegDef(Reg: UseReg);
1988 if (ConstInst && ConstInst->getOpcode() == SPIRV::OpConstantI) {
1989 int64_t ImmVal = ConstInst->getOperand(i: 2).getImm();
1990 if (ImmVal)
1991 CheckAndAddExtension(ImmVal);
1992 }
1993 }
1994 }
1995 }
1996 break;
1997 }
1998
1999 case SPIRV::OpGroupNonUniformShuffle:
2000 case SPIRV::OpGroupNonUniformShuffleXor:
2001 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformShuffle);
2002 break;
2003 case SPIRV::OpGroupNonUniformShuffleUp:
2004 case SPIRV::OpGroupNonUniformShuffleDown:
2005 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformShuffleRelative);
2006 break;
2007 case SPIRV::OpGroupAll:
2008 case SPIRV::OpGroupAny:
2009 case SPIRV::OpGroupBroadcast:
2010 case SPIRV::OpGroupIAdd:
2011 case SPIRV::OpGroupFAdd:
2012 case SPIRV::OpGroupFMin:
2013 case SPIRV::OpGroupUMin:
2014 case SPIRV::OpGroupSMin:
2015 case SPIRV::OpGroupFMax:
2016 case SPIRV::OpGroupUMax:
2017 case SPIRV::OpGroupSMax:
2018 Reqs.addCapability(ToAdd: SPIRV::Capability::Groups);
2019 break;
2020 case SPIRV::OpGroupNonUniformElect:
2021 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniform);
2022 break;
2023 case SPIRV::OpGroupNonUniformAll:
2024 case SPIRV::OpGroupNonUniformAny:
2025 case SPIRV::OpGroupNonUniformAllEqual:
2026 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformVote);
2027 break;
2028 case SPIRV::OpGroupNonUniformBroadcast:
2029 case SPIRV::OpGroupNonUniformBroadcastFirst:
2030 case SPIRV::OpGroupNonUniformBallot:
2031 case SPIRV::OpGroupNonUniformInverseBallot:
2032 case SPIRV::OpGroupNonUniformBallotBitExtract:
2033 case SPIRV::OpGroupNonUniformBallotBitCount:
2034 case SPIRV::OpGroupNonUniformBallotFindLSB:
2035 case SPIRV::OpGroupNonUniformBallotFindMSB:
2036 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformBallot);
2037 break;
2038 case SPIRV::OpSubgroupShuffleINTEL:
2039 case SPIRV::OpSubgroupShuffleDownINTEL:
2040 case SPIRV::OpSubgroupShuffleUpINTEL:
2041 case SPIRV::OpSubgroupShuffleXorINTEL:
2042 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
2043 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
2044 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupShuffleINTEL);
2045 }
2046 break;
2047 case SPIRV::OpSubgroupBlockReadINTEL:
2048 case SPIRV::OpSubgroupBlockWriteINTEL:
2049 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
2050 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
2051 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupBufferBlockIOINTEL);
2052 }
2053 break;
2054 case SPIRV::OpSubgroupImageBlockReadINTEL:
2055 case SPIRV::OpSubgroupImageBlockWriteINTEL:
2056 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
2057 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
2058 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupImageBlockIOINTEL);
2059 }
2060 break;
2061 case SPIRV::OpSubgroupImageMediaBlockReadINTEL:
2062 case SPIRV::OpSubgroupImageMediaBlockWriteINTEL:
2063 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_media_block_io)) {
2064 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_media_block_io);
2065 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupImageMediaBlockIOINTEL);
2066 }
2067 break;
2068 case SPIRV::OpAssumeTrueKHR:
2069 case SPIRV::OpExpectKHR:
2070 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume)) {
2071 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_expect_assume);
2072 Reqs.addCapability(ToAdd: SPIRV::Capability::ExpectAssumeKHR);
2073 }
2074 break;
2075 case SPIRV::OpFmaKHR:
2076 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_fma)) {
2077 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_fma);
2078 Reqs.addCapability(ToAdd: SPIRV::Capability::FmaKHR);
2079 }
2080 break;
2081 case SPIRV::OpPtrCastToCrossWorkgroupINTEL:
2082 case SPIRV::OpCrossWorkgroupCastToPtrINTEL:
2083 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)) {
2084 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_usm_storage_classes);
2085 Reqs.addCapability(ToAdd: SPIRV::Capability::USMStorageClassesINTEL);
2086 }
2087 break;
2088 case SPIRV::OpConstantFunctionPointerINTEL:
2089 case SPIRV::OpFunctionPointerCallINTEL:
2090 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers)) {
2091 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_function_pointers);
2092 Reqs.addCapability(ToAdd: SPIRV::Capability::FunctionPointersINTEL);
2093 }
2094 break;
2095 case SPIRV::OpGroupNonUniformRotateKHR:
2096 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_subgroup_rotate))
2097 report_fatal_error(reason: "OpGroupNonUniformRotateKHR instruction requires the "
2098 "following SPIR-V extension: SPV_KHR_subgroup_rotate",
2099 gen_crash_diag: false);
2100 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_subgroup_rotate);
2101 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformRotateKHR);
2102 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniform);
2103 break;
2104 case SPIRV::OpFixedCosALTERA:
2105 case SPIRV::OpFixedSinALTERA:
2106 case SPIRV::OpFixedCosPiALTERA:
2107 case SPIRV::OpFixedSinPiALTERA:
2108 case SPIRV::OpFixedExpALTERA:
2109 case SPIRV::OpFixedLogALTERA:
2110 case SPIRV::OpFixedRecipALTERA:
2111 case SPIRV::OpFixedSqrtALTERA:
2112 case SPIRV::OpFixedSinCosALTERA:
2113 case SPIRV::OpFixedSinCosPiALTERA:
2114 case SPIRV::OpFixedRsqrtALTERA:
2115 if (!ST.canUseExtension(
2116 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point))
2117 report_fatal_error(reason: "This instruction requires the "
2118 "following SPIR-V extension: "
2119 "SPV_ALTERA_arbitrary_precision_fixed_point",
2120 gen_crash_diag: false);
2121 Reqs.addExtension(
2122 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point);
2123 Reqs.addCapability(ToAdd: SPIRV::Capability::ArbitraryPrecisionFixedPointALTERA);
2124 break;
2125 case SPIRV::OpGroupIMulKHR:
2126 case SPIRV::OpGroupFMulKHR:
2127 case SPIRV::OpGroupBitwiseAndKHR:
2128 case SPIRV::OpGroupBitwiseOrKHR:
2129 case SPIRV::OpGroupBitwiseXorKHR:
2130 case SPIRV::OpGroupLogicalAndKHR:
2131 case SPIRV::OpGroupLogicalOrKHR:
2132 case SPIRV::OpGroupLogicalXorKHR:
2133 if (ST.canUseExtension(
2134 E: SPIRV::Extension::SPV_KHR_uniform_group_instructions)) {
2135 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_uniform_group_instructions);
2136 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupUniformArithmeticKHR);
2137 }
2138 break;
2139 case SPIRV::OpReadClockKHR:
2140 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_shader_clock))
2141 report_fatal_error(reason: "OpReadClockKHR instruction requires the "
2142 "following SPIR-V extension: SPV_KHR_shader_clock",
2143 gen_crash_diag: false);
2144 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_shader_clock);
2145 Reqs.addCapability(ToAdd: SPIRV::Capability::ShaderClockKHR);
2146 break;
2147 case SPIRV::OpAbortKHR:
2148 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_abort))
2149 report_fatal_error(reason: "OpAbortKHR instruction requires the "
2150 "following SPIR-V extension: SPV_KHR_abort",
2151 gen_crash_diag: false);
2152 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_abort);
2153 Reqs.addCapability(ToAdd: SPIRV::Capability::AbortKHR);
2154 break;
2155 case SPIRV::OpPoisonKHR:
2156 case SPIRV::OpFreezeKHR:
2157 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze))
2158 report_fatal_error(reason: "OpPoisonKHR/OpFreezeKHR instruction requires the "
2159 "following SPIR-V extension: SPV_KHR_poison_freeze",
2160 gen_crash_diag: false);
2161 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_poison_freeze);
2162 Reqs.addCapability(ToAdd: SPIRV::Capability::PoisonFreezeKHR);
2163 break;
2164 case SPIRV::OpAtomicFAddEXT:
2165 case SPIRV::OpAtomicFMinEXT:
2166 case SPIRV::OpAtomicFMaxEXT:
2167 AddAtomicFloatRequirements(MI, Reqs, ST);
2168 break;
2169 case SPIRV::OpConvertBF16ToFINTEL:
2170 case SPIRV::OpConvertFToBF16INTEL:
2171 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bfloat16_conversion)) {
2172 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_conversion);
2173 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ConversionINTEL);
2174 }
2175 break;
2176 case SPIRV::OpRoundFToTF32INTEL:
2177 if (ST.canUseExtension(
2178 E: SPIRV::Extension::SPV_INTEL_tensor_float32_conversion)) {
2179 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_tensor_float32_conversion);
2180 Reqs.addCapability(ToAdd: SPIRV::Capability::TensorFloat32RoundingINTEL);
2181 }
2182 break;
2183 case SPIRV::OpVariableLengthArrayINTEL:
2184 case SPIRV::OpSaveMemoryINTEL:
2185 case SPIRV::OpRestoreMemoryINTEL:
2186 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array)) {
2187 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_variable_length_array);
2188 Reqs.addCapability(ToAdd: SPIRV::Capability::VariableLengthArrayINTEL);
2189 }
2190 break;
2191 case SPIRV::OpUntypedVariableLengthArrayINTEL:
2192 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array)) {
2193 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_variable_length_array);
2194 Reqs.addCapability(ToAdd: SPIRV::Capability::UntypedVariableLengthArrayINTEL);
2195 }
2196 break;
2197 case SPIRV::OpAsmTargetINTEL:
2198 case SPIRV::OpAsmINTEL:
2199 case SPIRV::OpAsmCallINTEL:
2200 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_inline_assembly)) {
2201 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_inline_assembly);
2202 Reqs.addCapability(ToAdd: SPIRV::Capability::AsmINTEL);
2203 }
2204 break;
2205 case SPIRV::OpTypeCooperativeMatrixKHR: {
2206 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
2207 report_fatal_error(
2208 reason: "OpTypeCooperativeMatrixKHR type requires the "
2209 "following SPIR-V extension: SPV_KHR_cooperative_matrix",
2210 gen_crash_diag: false);
2211 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
2212 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
2213 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2214 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
2215 if (isBFloat16Type(TypeDef))
2216 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16CooperativeMatrixKHR);
2217 break;
2218 }
2219 case SPIRV::OpArithmeticFenceEXT:
2220 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_arithmetic_fence))
2221 report_fatal_error(reason: "OpArithmeticFenceEXT requires the "
2222 "following SPIR-V extension: SPV_EXT_arithmetic_fence",
2223 gen_crash_diag: false);
2224 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_arithmetic_fence);
2225 Reqs.addCapability(ToAdd: SPIRV::Capability::ArithmeticFenceEXT);
2226 break;
2227 case SPIRV::OpControlBarrierArriveINTEL:
2228 case SPIRV::OpControlBarrierWaitINTEL:
2229 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_split_barrier)) {
2230 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_split_barrier);
2231 Reqs.addCapability(ToAdd: SPIRV::Capability::SplitBarrierINTEL);
2232 }
2233 break;
2234 case SPIRV::OpCooperativeMatrixMulAddKHR: {
2235 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
2236 report_fatal_error(reason: "Cooperative matrix instructions require the "
2237 "following SPIR-V extension: "
2238 "SPV_KHR_cooperative_matrix",
2239 gen_crash_diag: false);
2240 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
2241 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
2242 constexpr unsigned MulAddMaxSize = 6;
2243 if (MI.getNumOperands() != MulAddMaxSize)
2244 break;
2245 const int64_t CoopOperands = MI.getOperand(i: MulAddMaxSize - 1).getImm();
2246 if (CoopOperands &
2247 SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) {
2248 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2249 report_fatal_error(reason: "MatrixAAndBTF32ComponentsINTEL type interpretation "
2250 "require the following SPIR-V extension: "
2251 "SPV_INTEL_joint_matrix",
2252 gen_crash_diag: false);
2253 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2254 Reqs.addCapability(
2255 ToAdd: SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL);
2256 }
2257 if (CoopOperands & SPIRV::CooperativeMatrixOperands::
2258 MatrixAAndBBFloat16ComponentsINTEL ||
2259 CoopOperands &
2260 SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL ||
2261 CoopOperands & SPIRV::CooperativeMatrixOperands::
2262 MatrixResultBFloat16ComponentsINTEL) {
2263 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2264 report_fatal_error(reason: "***BF16ComponentsINTEL type interpretations "
2265 "require the following SPIR-V extension: "
2266 "SPV_INTEL_joint_matrix",
2267 gen_crash_diag: false);
2268 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2269 Reqs.addCapability(
2270 ToAdd: SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);
2271 }
2272 break;
2273 }
2274 case SPIRV::OpCooperativeMatrixLoadKHR:
2275 case SPIRV::OpCooperativeMatrixStoreKHR:
2276 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2277 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2278 case SPIRV::OpCooperativeMatrixPrefetchINTEL: {
2279 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
2280 report_fatal_error(reason: "Cooperative matrix instructions require the "
2281 "following SPIR-V extension: "
2282 "SPV_KHR_cooperative_matrix",
2283 gen_crash_diag: false);
2284 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
2285 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
2286
2287 // Check Layout operand in case if it's not a standard one and add the
2288 // appropriate capability.
2289 unsigned LayoutNum;
2290 switch (Op) {
2291 case SPIRV::OpCooperativeMatrixLoadKHR:
2292 LayoutNum = 3;
2293 break;
2294 case SPIRV::OpCooperativeMatrixStoreKHR:
2295 LayoutNum = 2;
2296 break;
2297 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2298 LayoutNum = 5;
2299 break;
2300 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2301 case SPIRV::OpCooperativeMatrixPrefetchINTEL:
2302 LayoutNum = 4;
2303 break;
2304 default:
2305 llvm_unreachable("unexpected cooperative matrix opcode");
2306 }
2307 Register RegLayout = MI.getOperand(i: LayoutNum).getReg();
2308 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2309 MachineInstr *MILayout = MRI.getUniqueVRegDef(Reg: RegLayout);
2310 if (MILayout->getOpcode() == SPIRV::OpConstantI) {
2311 const unsigned LayoutVal = MILayout->getOperand(i: 2).getImm();
2312 if (LayoutVal ==
2313 static_cast<unsigned>(SPIRV::CooperativeMatrixLayout::PackedINTEL)) {
2314 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2315 report_fatal_error(reason: "PackedINTEL layout require the following SPIR-V "
2316 "extension: SPV_INTEL_joint_matrix",
2317 gen_crash_diag: false);
2318 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2319 Reqs.addCapability(ToAdd: SPIRV::Capability::PackedCooperativeMatrixINTEL);
2320 }
2321 }
2322
2323 // Nothing to do.
2324 if (Op == SPIRV::OpCooperativeMatrixLoadKHR ||
2325 Op == SPIRV::OpCooperativeMatrixStoreKHR)
2326 break;
2327
2328 std::string InstName;
2329 switch (Op) {
2330 case SPIRV::OpCooperativeMatrixPrefetchINTEL:
2331 InstName = "OpCooperativeMatrixPrefetchINTEL";
2332 break;
2333 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2334 InstName = "OpCooperativeMatrixLoadCheckedINTEL";
2335 break;
2336 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2337 InstName = "OpCooperativeMatrixStoreCheckedINTEL";
2338 break;
2339 }
2340
2341 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix)) {
2342 const std::string ErrorMsg =
2343 InstName + " instruction requires the "
2344 "following SPIR-V extension: SPV_INTEL_joint_matrix";
2345 report_fatal_error(reason: ErrorMsg.c_str(), gen_crash_diag: false);
2346 }
2347 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2348 if (Op == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
2349 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixPrefetchINTEL);
2350 break;
2351 }
2352 Reqs.addCapability(
2353 ToAdd: SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
2354 break;
2355 }
2356 case SPIRV::OpCooperativeMatrixConstructCheckedINTEL:
2357 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2358 report_fatal_error(reason: "OpCooperativeMatrixConstructCheckedINTEL "
2359 "instructions require the following SPIR-V extension: "
2360 "SPV_INTEL_joint_matrix",
2361 gen_crash_diag: false);
2362 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2363 Reqs.addCapability(
2364 ToAdd: SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
2365 break;
2366 case SPIRV::OpReadPipeBlockingALTERA:
2367 case SPIRV::OpWritePipeBlockingALTERA:
2368 if (ST.canUseExtension(E: SPIRV::Extension::SPV_ALTERA_blocking_pipes)) {
2369 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_ALTERA_blocking_pipes);
2370 Reqs.addCapability(ToAdd: SPIRV::Capability::BlockingPipesALTERA);
2371 }
2372 break;
2373 case SPIRV::OpCooperativeMatrixGetElementCoordINTEL:
2374 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2375 report_fatal_error(reason: "OpCooperativeMatrixGetElementCoordINTEL requires the "
2376 "following SPIR-V extension: SPV_INTEL_joint_matrix",
2377 gen_crash_diag: false);
2378 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2379 Reqs.addCapability(
2380 ToAdd: SPIRV::Capability::CooperativeMatrixInvocationInstructionsINTEL);
2381 break;
2382 case SPIRV::OpConvertHandleToImageINTEL:
2383 case SPIRV::OpConvertHandleToSamplerINTEL:
2384 case SPIRV::OpConvertHandleToSampledImageINTEL: {
2385 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bindless_images))
2386 report_fatal_error(reason: "OpConvertHandleTo[Image/Sampler/SampledImage]INTEL "
2387 "instructions require the following SPIR-V extension: "
2388 "SPV_INTEL_bindless_images",
2389 gen_crash_diag: false);
2390 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
2391 SPIRV::AddressingModel::AddressingModel AddrModel = MAI.Addr;
2392 SPIRVTypeInst TyDef = GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg());
2393 if (Op == SPIRV::OpConvertHandleToImageINTEL &&
2394 TyDef->getOpcode() != SPIRV::OpTypeImage) {
2395 report_fatal_error(reason: "Incorrect return type for the instruction "
2396 "OpConvertHandleToImageINTEL",
2397 gen_crash_diag: false);
2398 } else if (Op == SPIRV::OpConvertHandleToSamplerINTEL &&
2399 TyDef->getOpcode() != SPIRV::OpTypeSampler) {
2400 report_fatal_error(reason: "Incorrect return type for the instruction "
2401 "OpConvertHandleToSamplerINTEL",
2402 gen_crash_diag: false);
2403 } else if (Op == SPIRV::OpConvertHandleToSampledImageINTEL &&
2404 TyDef->getOpcode() != SPIRV::OpTypeSampledImage) {
2405 report_fatal_error(reason: "Incorrect return type for the instruction "
2406 "OpConvertHandleToSampledImageINTEL",
2407 gen_crash_diag: false);
2408 }
2409 SPIRVTypeInst SpvTy =
2410 getRegType(Reg: MI.getOperand(i: 2).getReg(), MF: *MI.getMF(), GR);
2411 unsigned Bitwidth = GR->getScalarOrVectorBitWidth(Type: SpvTy);
2412 if (!(Bitwidth == 32 && AddrModel == SPIRV::AddressingModel::Physical32) &&
2413 !(Bitwidth == 64 && AddrModel == SPIRV::AddressingModel::Physical64)) {
2414 report_fatal_error(
2415 reason: "Parameter value must be a 32-bit scalar in case of "
2416 "Physical32 addressing model or a 64-bit scalar in case of "
2417 "Physical64 addressing model",
2418 gen_crash_diag: false);
2419 }
2420 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bindless_images);
2421 Reqs.addCapability(ToAdd: SPIRV::Capability::BindlessImagesINTEL);
2422 break;
2423 }
2424 case SPIRV::OpSubgroup2DBlockLoadINTEL:
2425 case SPIRV::OpSubgroup2DBlockLoadTransposeINTEL:
2426 case SPIRV::OpSubgroup2DBlockLoadTransformINTEL:
2427 case SPIRV::OpSubgroup2DBlockPrefetchINTEL:
2428 case SPIRV::OpSubgroup2DBlockStoreINTEL: {
2429 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_2d_block_io))
2430 report_fatal_error(reason: "OpSubgroup2DBlock[Load/LoadTranspose/LoadTransform/"
2431 "Prefetch/Store]INTEL instructions require the "
2432 "following SPIR-V extension: SPV_INTEL_2d_block_io",
2433 gen_crash_diag: false);
2434 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_2d_block_io);
2435 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockIOINTEL);
2436
2437 if (Op == SPIRV::OpSubgroup2DBlockLoadTransposeINTEL) {
2438 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockTransposeINTEL);
2439 break;
2440 }
2441 if (Op == SPIRV::OpSubgroup2DBlockLoadTransformINTEL) {
2442 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockTransformINTEL);
2443 break;
2444 }
2445 break;
2446 }
2447 case SPIRV::OpKill: {
2448 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2449 } break;
2450 case SPIRV::OpDemoteToHelperInvocation:
2451 Reqs.addCapability(ToAdd: SPIRV::Capability::DemoteToHelperInvocation);
2452
2453 if (ST.canUseExtension(
2454 E: SPIRV::Extension::SPV_EXT_demote_to_helper_invocation)) {
2455 if (!ST.isAtLeastSPIRVVer(VerToCompareTo: llvm::VersionTuple(1, 6)))
2456 Reqs.addExtension(
2457 ToAdd: SPIRV::Extension::SPV_EXT_demote_to_helper_invocation);
2458 }
2459 break;
2460 case SPIRV::OpSDot:
2461 case SPIRV::OpUDot:
2462 case SPIRV::OpSUDot:
2463 case SPIRV::OpSDotAccSat:
2464 case SPIRV::OpUDotAccSat:
2465 case SPIRV::OpSUDotAccSat:
2466 AddDotProductRequirements(MI, Reqs, ST);
2467 break;
2468 case SPIRV::OpImageSampleImplicitLod:
2469 case SPIRV::OpImageFetch:
2470 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2471 addImageOperandReqs(MI, Reqs, ST, OpIdx: 4);
2472 break;
2473 case SPIRV::OpImageSampleExplicitLod:
2474 addImageOperandReqs(MI, Reqs, ST, OpIdx: 4);
2475 break;
2476 case SPIRV::OpImageSampleDrefImplicitLod:
2477 case SPIRV::OpImageSampleDrefExplicitLod:
2478 case SPIRV::OpImageDrefGather:
2479 case SPIRV::OpImageGather:
2480 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2481 addImageOperandReqs(MI, Reqs, ST, OpIdx: 5);
2482 break;
2483 case SPIRV::OpImageRead: {
2484 Register ImageReg = MI.getOperand(i: 2).getReg();
2485 SPIRVTypeInst TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(
2486 VReg: ImageReg, MF: const_cast<MachineFunction *>(MI.getMF()));
2487 // OpImageRead and OpImageWrite can use Unknown Image Formats
2488 // when the Kernel capability is declared. In the OpenCL environment we are
2489 // not allowed to produce
2490 // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see
2491 // https://github.com/KhronosGroup/SPIRV-Headers/issues/487
2492
2493 if (isImageTypeWithUnknownFormat(TypeInst: TypeDef) && ST.isShader())
2494 Reqs.addCapability(ToAdd: SPIRV::Capability::StorageImageReadWithoutFormat);
2495 break;
2496 }
2497 case SPIRV::OpImageWrite: {
2498 Register ImageReg = MI.getOperand(i: 0).getReg();
2499 SPIRVTypeInst TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(
2500 VReg: ImageReg, MF: const_cast<MachineFunction *>(MI.getMF()));
2501 // OpImageRead and OpImageWrite can use Unknown Image Formats
2502 // when the Kernel capability is declared. In the OpenCL environment we are
2503 // not allowed to produce
2504 // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see
2505 // https://github.com/KhronosGroup/SPIRV-Headers/issues/487
2506
2507 if (isImageTypeWithUnknownFormat(TypeInst: TypeDef) && ST.isShader())
2508 Reqs.addCapability(ToAdd: SPIRV::Capability::StorageImageWriteWithoutFormat);
2509 break;
2510 }
2511 case SPIRV::OpTypeStructContinuedINTEL:
2512 case SPIRV::OpConstantCompositeContinuedINTEL:
2513 case SPIRV::OpSpecConstantCompositeContinuedINTEL:
2514 case SPIRV::OpCompositeConstructContinuedINTEL: {
2515 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_long_composites))
2516 report_fatal_error(
2517 reason: "Continued instructions require the "
2518 "following SPIR-V extension: SPV_INTEL_long_composites",
2519 gen_crash_diag: false);
2520 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_long_composites);
2521 Reqs.addCapability(ToAdd: SPIRV::Capability::LongCompositesINTEL);
2522 break;
2523 }
2524 case SPIRV::OpArbitraryFloatEQALTERA:
2525 case SPIRV::OpArbitraryFloatGEALTERA:
2526 case SPIRV::OpArbitraryFloatGTALTERA:
2527 case SPIRV::OpArbitraryFloatLEALTERA:
2528 case SPIRV::OpArbitraryFloatLTALTERA:
2529 case SPIRV::OpArbitraryFloatCbrtALTERA:
2530 case SPIRV::OpArbitraryFloatCosALTERA:
2531 case SPIRV::OpArbitraryFloatCosPiALTERA:
2532 case SPIRV::OpArbitraryFloatExp10ALTERA:
2533 case SPIRV::OpArbitraryFloatExp2ALTERA:
2534 case SPIRV::OpArbitraryFloatExpALTERA:
2535 case SPIRV::OpArbitraryFloatExpm1ALTERA:
2536 case SPIRV::OpArbitraryFloatHypotALTERA:
2537 case SPIRV::OpArbitraryFloatLog10ALTERA:
2538 case SPIRV::OpArbitraryFloatLog1pALTERA:
2539 case SPIRV::OpArbitraryFloatLog2ALTERA:
2540 case SPIRV::OpArbitraryFloatLogALTERA:
2541 case SPIRV::OpArbitraryFloatRecipALTERA:
2542 case SPIRV::OpArbitraryFloatSinCosALTERA:
2543 case SPIRV::OpArbitraryFloatSinCosPiALTERA:
2544 case SPIRV::OpArbitraryFloatSinALTERA:
2545 case SPIRV::OpArbitraryFloatSinPiALTERA:
2546 case SPIRV::OpArbitraryFloatSqrtALTERA:
2547 case SPIRV::OpArbitraryFloatACosALTERA:
2548 case SPIRV::OpArbitraryFloatACosPiALTERA:
2549 case SPIRV::OpArbitraryFloatAddALTERA:
2550 case SPIRV::OpArbitraryFloatASinALTERA:
2551 case SPIRV::OpArbitraryFloatASinPiALTERA:
2552 case SPIRV::OpArbitraryFloatATan2ALTERA:
2553 case SPIRV::OpArbitraryFloatATanALTERA:
2554 case SPIRV::OpArbitraryFloatATanPiALTERA:
2555 case SPIRV::OpArbitraryFloatCastFromIntALTERA:
2556 case SPIRV::OpArbitraryFloatCastALTERA:
2557 case SPIRV::OpArbitraryFloatCastToIntALTERA:
2558 case SPIRV::OpArbitraryFloatDivALTERA:
2559 case SPIRV::OpArbitraryFloatMulALTERA:
2560 case SPIRV::OpArbitraryFloatPowALTERA:
2561 case SPIRV::OpArbitraryFloatPowNALTERA:
2562 case SPIRV::OpArbitraryFloatPowRALTERA:
2563 case SPIRV::OpArbitraryFloatRSqrtALTERA:
2564 case SPIRV::OpArbitraryFloatSubALTERA: {
2565 if (!ST.canUseExtension(
2566 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_floating_point))
2567 report_fatal_error(
2568 reason: "Floating point instructions can't be translated correctly without "
2569 "enabled SPV_ALTERA_arbitrary_precision_floating_point extension!",
2570 gen_crash_diag: false);
2571 Reqs.addExtension(
2572 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_floating_point);
2573 Reqs.addCapability(
2574 ToAdd: SPIRV::Capability::ArbitraryPrecisionFloatingPointALTERA);
2575 break;
2576 }
2577 case SPIRV::OpSubgroupMatrixMultiplyAccumulateINTEL: {
2578 if (!ST.canUseExtension(
2579 E: SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate))
2580 report_fatal_error(
2581 reason: "OpSubgroupMatrixMultiplyAccumulateINTEL instruction requires the "
2582 "following SPIR-V "
2583 "extension: SPV_INTEL_subgroup_matrix_multiply_accumulate",
2584 gen_crash_diag: false);
2585 Reqs.addExtension(
2586 ToAdd: SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate);
2587 Reqs.addCapability(
2588 ToAdd: SPIRV::Capability::SubgroupMatrixMultiplyAccumulateINTEL);
2589 break;
2590 }
2591 case SPIRV::OpBitwiseFunctionINTEL: {
2592 if (!ST.canUseExtension(
2593 E: SPIRV::Extension::SPV_INTEL_ternary_bitwise_function))
2594 report_fatal_error(
2595 reason: "OpBitwiseFunctionINTEL instruction requires the following SPIR-V "
2596 "extension: SPV_INTEL_ternary_bitwise_function",
2597 gen_crash_diag: false);
2598 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_ternary_bitwise_function);
2599 Reqs.addCapability(ToAdd: SPIRV::Capability::TernaryBitwiseFunctionINTEL);
2600 break;
2601 }
2602 case SPIRV::OpCopyMemorySized: {
2603 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
2604 // TODO: Add UntypedPointersKHR when implemented.
2605 break;
2606 }
2607 case SPIRV::OpTypeUntypedPointerKHR:
2608 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::StorageClassOperand,
2609 i: MI.getOperand(i: 1).getImm(), ST);
2610 [[fallthrough]];
2611 case SPIRV::OpUntypedVariableKHR:
2612 case SPIRV::OpUntypedAccessChainKHR:
2613 case SPIRV::OpUntypedInBoundsAccessChainKHR:
2614 case SPIRV::OpUntypedPtrAccessChainKHR:
2615 case SPIRV::OpUntypedInBoundsPtrAccessChainKHR:
2616 case SPIRV::OpUntypedPrefetchKHR:
2617 case SPIRV::OpUntypedGroupAsyncCopyKHR: {
2618 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_untyped_pointers))
2619 report_fatal_error(reason: "Untyped pointer instructions require the following "
2620 "SPIR-V extension: SPV_KHR_untyped_pointers",
2621 gen_crash_diag: false);
2622 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_untyped_pointers);
2623 Reqs.addCapability(ToAdd: SPIRV::Capability::UntypedPointersKHR);
2624 break;
2625 }
2626 case SPIRV::OpPredicatedLoadINTEL:
2627 case SPIRV::OpPredicatedStoreINTEL: {
2628 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_predicated_io))
2629 report_fatal_error(
2630 reason: "OpPredicated[Load/Store]INTEL instructions require "
2631 "the following SPIR-V extension: SPV_INTEL_predicated_io",
2632 gen_crash_diag: false);
2633 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_predicated_io);
2634 Reqs.addCapability(ToAdd: SPIRV::Capability::PredicatedIOINTEL);
2635 break;
2636 }
2637 case SPIRV::OpFAddS:
2638 case SPIRV::OpFSubS:
2639 case SPIRV::OpFMulS:
2640 case SPIRV::OpFDivS:
2641 case SPIRV::OpFRemS:
2642 case SPIRV::OpFMod:
2643 case SPIRV::OpFNegate:
2644 case SPIRV::OpFAddV:
2645 case SPIRV::OpFSubV:
2646 case SPIRV::OpFMulV:
2647 case SPIRV::OpFDivV:
2648 case SPIRV::OpFRemV:
2649 case SPIRV::OpFNegateV: {
2650 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2651 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
2652 if (isVectorType(SPVTy: TypeDef))
2653 TypeDef = MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
2654 if (isBFloat16Type(TypeDef)) {
2655 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic))
2656 report_fatal_error(
2657 reason: "Arithmetic instructions with bfloat16 arguments require the "
2658 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic",
2659 gen_crash_diag: false);
2660 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);
2661 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ArithmeticINTEL);
2662 }
2663 break;
2664 }
2665 case SPIRV::OpOrdered:
2666 case SPIRV::OpUnordered:
2667 case SPIRV::OpFOrdEqual:
2668 case SPIRV::OpFOrdNotEqual:
2669 case SPIRV::OpFOrdLessThan:
2670 case SPIRV::OpFOrdLessThanEqual:
2671 case SPIRV::OpFOrdGreaterThan:
2672 case SPIRV::OpFOrdGreaterThanEqual:
2673 case SPIRV::OpFUnordEqual:
2674 case SPIRV::OpFUnordNotEqual:
2675 case SPIRV::OpFUnordLessThan:
2676 case SPIRV::OpFUnordLessThanEqual:
2677 case SPIRV::OpFUnordGreaterThan:
2678 case SPIRV::OpFUnordGreaterThanEqual: {
2679 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2680 SPIRVTypeInst TypeDef = getRegType(Reg: MI.getOperand(i: 2).getReg(), MF: *MI.getMF(),
2681 GR: ST.getSPIRVGlobalRegistry());
2682 assert(TypeDef && "Unexpected type instruction for comparison operand");
2683 if (isVectorType(SPVTy: TypeDef))
2684 TypeDef = MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
2685 if (isBFloat16Type(TypeDef)) {
2686 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic))
2687 report_fatal_error(
2688 reason: "Relational instructions with bfloat16 arguments require the "
2689 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic",
2690 gen_crash_diag: false);
2691 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);
2692 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ArithmeticINTEL);
2693 }
2694 break;
2695 }
2696 case SPIRV::OpDPdxCoarse:
2697 case SPIRV::OpDPdyCoarse:
2698 case SPIRV::OpDPdxFine:
2699 case SPIRV::OpDPdyFine: {
2700 Reqs.addCapability(ToAdd: SPIRV::Capability::DerivativeControl);
2701 break;
2702 }
2703 case SPIRV::OpLoopControlINTEL: {
2704 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls);
2705 Reqs.addCapability(ToAdd: SPIRV::Capability::UnstructuredLoopControlsINTEL);
2706 break;
2707 }
2708
2709 default:
2710 break;
2711 }
2712
2713 // If we require capability Shader, then we can remove the requirement for
2714 // the BitInstructions capability, since Shader is a superset capability
2715 // of BitInstructions.
2716 Reqs.removeCapabilityIf(ToRemove: SPIRV::Capability::BitInstructions,
2717 IfPresent: SPIRV::Capability::Shader);
2718}
2719
2720static void collectReqs(const Module &M, SPIRV::ModuleAnalysisInfo &MAI,
2721 MachineFunctionGetter GetMF, const SPIRVSubtarget &ST) {
2722 // Collect requirements for existing instructions.
2723 for (const Function &F : M) {
2724 MachineFunction *MF = GetMF(F);
2725 if (!MF)
2726 continue;
2727 for (const MachineBasicBlock &MBB : *MF)
2728 for (const MachineInstr &MI : MBB)
2729 addInstrRequirements(MI, MAI, ST);
2730 }
2731 // Collect requirements for OpExecutionMode instructions.
2732 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
2733 if (Node) {
2734 bool RequireFloatControls = false, RequireIntelFloatControls2 = false,
2735 RequireKHRFloatControls2 = false,
2736 VerLower14 = !ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 4));
2737 bool HasIntelFloatControls2 =
2738 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_float_controls2);
2739 bool HasKHRFloatControls2 =
2740 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2741 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
2742 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
2743 const MDOperand &MDOp = MDN->getOperand(I: 1);
2744 if (auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: MDOp)) {
2745 Constant *C = CMeta->getValue();
2746 if (ConstantInt *Const = dyn_cast<ConstantInt>(Val: C)) {
2747 auto EM = Const->getZExtValue();
2748 // SPV_KHR_float_controls is not available until v1.4:
2749 // add SPV_KHR_float_controls if the version is too low
2750 switch (EM) {
2751 case SPIRV::ExecutionMode::DenormPreserve:
2752 case SPIRV::ExecutionMode::DenormFlushToZero:
2753 case SPIRV::ExecutionMode::RoundingModeRTE:
2754 case SPIRV::ExecutionMode::RoundingModeRTZ:
2755 RequireFloatControls = VerLower14;
2756 MAI.Reqs.getAndAddRequirements(
2757 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2758 break;
2759 case SPIRV::ExecutionMode::RoundingModeRTPINTEL:
2760 case SPIRV::ExecutionMode::RoundingModeRTNINTEL:
2761 case SPIRV::ExecutionMode::FloatingPointModeALTINTEL:
2762 case SPIRV::ExecutionMode::FloatingPointModeIEEEINTEL:
2763 if (HasIntelFloatControls2) {
2764 RequireIntelFloatControls2 = true;
2765 MAI.Reqs.getAndAddRequirements(
2766 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2767 }
2768 break;
2769 case SPIRV::ExecutionMode::FPFastMathDefault: {
2770 if (HasKHRFloatControls2) {
2771 RequireKHRFloatControls2 = true;
2772 MAI.Reqs.getAndAddRequirements(
2773 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2774 }
2775 break;
2776 }
2777 case SPIRV::ExecutionMode::ContractionOff:
2778 case SPIRV::ExecutionMode::SignedZeroInfNanPreserve:
2779 if (HasKHRFloatControls2) {
2780 RequireKHRFloatControls2 = true;
2781 MAI.Reqs.getAndAddRequirements(
2782 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2783 i: SPIRV::ExecutionMode::FPFastMathDefault, ST);
2784 } else {
2785 MAI.Reqs.getAndAddRequirements(
2786 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2787 }
2788 break;
2789 default:
2790 MAI.Reqs.getAndAddRequirements(
2791 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2792 }
2793 }
2794 }
2795 }
2796 if (RequireFloatControls &&
2797 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls))
2798 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls);
2799 if (RequireIntelFloatControls2)
2800 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_float_controls2);
2801 if (RequireKHRFloatControls2)
2802 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls2);
2803 }
2804 for (const Function &F : M) {
2805 if (F.isDeclaration())
2806 continue;
2807 if (F.getMetadata(Kind: "reqd_work_group_size"))
2808 MAI.Reqs.getAndAddRequirements(
2809 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2810 i: SPIRV::ExecutionMode::LocalSize, ST);
2811 if (F.getFnAttribute(Kind: "hlsl.numthreads").isValid()) {
2812 MAI.Reqs.getAndAddRequirements(
2813 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2814 i: SPIRV::ExecutionMode::LocalSize, ST);
2815 }
2816 if (F.getFnAttribute(Kind: "enable-maximal-reconvergence").getValueAsBool()) {
2817 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_maximal_reconvergence);
2818 }
2819 if (F.getMetadata(Kind: "work_group_size_hint"))
2820 MAI.Reqs.getAndAddRequirements(
2821 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2822 i: SPIRV::ExecutionMode::LocalSizeHint, ST);
2823 if (F.getMetadata(Kind: "intel_reqd_sub_group_size") ||
2824 F.getMetadata(Kind: "reqd_sub_group_size"))
2825 MAI.Reqs.getAndAddRequirements(
2826 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2827 i: SPIRV::ExecutionMode::SubgroupSize, ST);
2828 if (F.getMetadata(Kind: "max_work_group_size"))
2829 MAI.Reqs.getAndAddRequirements(
2830 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2831 i: SPIRV::ExecutionMode::MaxWorkgroupSizeINTEL, ST);
2832 if (F.getMetadata(Kind: "vec_type_hint"))
2833 MAI.Reqs.getAndAddRequirements(
2834 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2835 i: SPIRV::ExecutionMode::VecTypeHint, ST);
2836
2837 if (F.hasOptNone()) {
2838 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_optnone)) {
2839 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_optnone);
2840 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::OptNoneINTEL);
2841 } else if (ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_optnone)) {
2842 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_optnone);
2843 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::OptNoneEXT);
2844 }
2845 }
2846 }
2847}
2848
2849static unsigned getFastMathFlags(const MachineInstr &I,
2850 const SPIRVSubtarget &ST) {
2851 unsigned Flags = SPIRV::FPFastMathMode::None;
2852 bool CanUseKHRFloatControls2 =
2853 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2854 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNoNans))
2855 Flags |= SPIRV::FPFastMathMode::NotNaN;
2856 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNoInfs))
2857 Flags |= SPIRV::FPFastMathMode::NotInf;
2858 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNsz))
2859 Flags |= SPIRV::FPFastMathMode::NSZ;
2860 if (I.getFlag(Flag: MachineInstr::MIFlag::FmArcp))
2861 Flags |= SPIRV::FPFastMathMode::AllowRecip;
2862 if (I.getFlag(Flag: MachineInstr::MIFlag::FmContract) && CanUseKHRFloatControls2)
2863 Flags |= SPIRV::FPFastMathMode::AllowContract;
2864 if (I.getFlag(Flag: MachineInstr::MIFlag::FmReassoc)) {
2865 if (CanUseKHRFloatControls2)
2866 // LLVM reassoc maps to SPIRV transform, see
2867 // https://github.com/KhronosGroup/SPIRV-Registry/issues/326 for details.
2868 // Because we are enabling AllowTransform, we must enable AllowReassoc and
2869 // AllowContract too, as required by SPIRV spec. Also, we used to map
2870 // MIFlag::FmReassoc to FPFastMathMode::Fast, which now should instead by
2871 // replaced by turning all the other bits instead. Therefore, we're
2872 // enabling every bit here except None and Fast.
2873 Flags |= SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
2874 SPIRV::FPFastMathMode::NSZ | SPIRV::FPFastMathMode::AllowRecip |
2875 SPIRV::FPFastMathMode::AllowTransform |
2876 SPIRV::FPFastMathMode::AllowReassoc |
2877 SPIRV::FPFastMathMode::AllowContract;
2878 else
2879 Flags |= SPIRV::FPFastMathMode::Fast;
2880 }
2881
2882 if (CanUseKHRFloatControls2) {
2883 // Error out if SPIRV::FPFastMathMode::Fast is enabled.
2884 assert(!(Flags & SPIRV::FPFastMathMode::Fast) &&
2885 "SPIRV::FPFastMathMode::Fast is deprecated and should not be used "
2886 "anymore.");
2887
2888 // Error out if AllowTransform is enabled without AllowReassoc and
2889 // AllowContract.
2890 assert((!(Flags & SPIRV::FPFastMathMode::AllowTransform) ||
2891 ((Flags & SPIRV::FPFastMathMode::AllowReassoc &&
2892 Flags & SPIRV::FPFastMathMode::AllowContract))) &&
2893 "SPIRV::FPFastMathMode::AllowTransform requires AllowReassoc and "
2894 "AllowContract flags to be enabled as well.");
2895 }
2896
2897 return Flags;
2898}
2899
2900static bool isFastMathModeAvailable(const SPIRVSubtarget &ST) {
2901 if (ST.isKernel())
2902 return true;
2903 if (ST.getSPIRVVersion() < VersionTuple(1, 2))
2904 return false;
2905 return ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2906}
2907
2908static bool handleMIFlagDecoration(
2909 MachineInstr &I, const SPIRVSubtarget &ST, const SPIRVInstrInfo &TII,
2910 SPIRV::RequirementHandler &Reqs, const SPIRVGlobalRegistry *GR,
2911 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec) {
2912 bool Changed = false;
2913 // Insert after I so that the decorated register is defined before its use.
2914 MachineBasicBlock::iterator InsertPt = std::next(x: I.getIterator());
2915 auto Decorate = [&](SPIRV::Decoration::Decoration Dec,
2916 ArrayRef<uint32_t> DecArgs) {
2917 MachineIRBuilder MIRBuilder(*I.getParent(), InsertPt);
2918 MIRBuilder.setDebugLoc(I.getDebugLoc());
2919 buildOpDecorate(Reg: I.getOperand(i: 0).getReg(), MIRBuilder, Dec, DecArgs);
2920 Changed = true;
2921 };
2922 if (TII.canUseIntegerWrapDecoration(MI: I)) {
2923 if (I.getFlag(Flag: MachineInstr::MIFlag::NoSWrap) &&
2924 getSymbolicOperandRequirements(
2925 Category: SPIRV::OperandCategory::DecorationOperand,
2926 i: SPIRV::Decoration::NoSignedWrap, ST, Reqs)
2927 .IsSatisfiable)
2928 Decorate(SPIRV::Decoration::NoSignedWrap, {});
2929 if (I.getFlag(Flag: MachineInstr::MIFlag::NoUWrap) &&
2930 getSymbolicOperandRequirements(
2931 Category: SPIRV::OperandCategory::DecorationOperand,
2932 i: SPIRV::Decoration::NoUnsignedWrap, ST, Reqs)
2933 .IsSatisfiable)
2934 Decorate(SPIRV::Decoration::NoUnsignedWrap, {});
2935 }
2936 // In Kernel environments, FPFastMathMode on OpExtInst is valid per core
2937 // spec. For other instruction types, SPV_KHR_float_controls2 is required.
2938 bool CanUseFM =
2939 TII.canUseFastMathFlags(
2940 MI: I, KHRFloatControls2: ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2)) ||
2941 (ST.isKernel() && I.getOpcode() == SPIRV::OpExtInst);
2942 if (!CanUseFM)
2943 return Changed;
2944
2945 unsigned FMFlags = getFastMathFlags(I, ST);
2946 if (FMFlags == SPIRV::FPFastMathMode::None) {
2947 // We also need to check if any FPFastMathDefault info was set for the
2948 // types used in this instruction.
2949 if (FPFastMathDefaultInfoVec.empty())
2950 return Changed;
2951
2952 // Use the first operand's FP type for arithmetic and comparisons,
2953 // and the result type for calls and extended instructions.
2954 SPIRVTypeInst FPType =
2955 I.getOpcode() == SPIRV::OpExtInst ||
2956 I.getOpcode() == SPIRV::OpFunctionCall
2957 ? I.getMF()->getRegInfo().getVRegDef(Reg: I.getOperand(i: 1).getReg())
2958 : getRegType(Reg: I.getOperand(i: 2).getReg(), MF: *I.getMF(), GR);
2959 const Type *Ty = GR->getTypeForSPIRVType(Ty: FPType);
2960 Ty = Ty->isVectorTy() ? cast<VectorType>(Val: Ty)->getElementType() : Ty;
2961
2962 // Match instruction type with the FPFastMathDefaultInfoVec.
2963 bool Emit = false;
2964 for (SPIRV::FPFastMathDefaultInfo &Elem : FPFastMathDefaultInfoVec) {
2965 if (Ty == Elem.Ty) {
2966 FMFlags = Elem.FastMathFlags;
2967 Emit = Elem.ContractionOff || Elem.SignedZeroInfNanPreserve ||
2968 Elem.FPFastMathDefault;
2969 break;
2970 }
2971 }
2972
2973 if (FMFlags == SPIRV::FPFastMathMode::None && !Emit)
2974 return Changed;
2975 }
2976 if (isFastMathModeAvailable(ST))
2977 Decorate(SPIRV::Decoration::FPFastMathMode, {FMFlags});
2978 return Changed;
2979}
2980
2981static void addMBBNames(const Module &M, MachineFunctionGetter GetMF,
2982 SPIRV::ModuleAnalysisInfo &MAI) {
2983 for (const Function &F : M) {
2984 MachineFunction *MF = GetMF(F);
2985 if (!MF)
2986 continue;
2987 if (MF->getFunction()
2988 .getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME)
2989 .isValid())
2990 continue;
2991 for (const auto &MBB : *MF) {
2992 if (!MBB.hasName() || MBB.empty())
2993 continue;
2994 MAI.MBBNames.emplace_back(Args: MAI.getOrCreateMBBRegister(MBB), Args: MBB.getName());
2995 }
2996 }
2997}
2998
2999static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
3000 const Module &M,
3001 SPIRV::FPFastMathDefaultInfoMapTy &FPFastMathDefaultInfoMap,
3002 const Function *F) {
3003 auto it = FPFastMathDefaultInfoMap.find(Val: F);
3004 if (it != FPFastMathDefaultInfoMap.end())
3005 return it->second;
3006
3007 // If the map does not contain the entry, create a new one. Initialize it to
3008 // contain all 3 elements sorted by bit width of target type: {half, float,
3009 // double}.
3010 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
3011 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
3012 Args: SPIRV::FPFastMathMode::None);
3013 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
3014 Args: SPIRV::FPFastMathMode::None);
3015 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
3016 Args: SPIRV::FPFastMathMode::None);
3017 return FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
3018}
3019
3020static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
3021 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
3022 const Type *Ty) {
3023 size_t BitWidth = Ty->getScalarSizeInBits();
3024 int Index =
3025 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
3026 BitWidth);
3027 assert(Index >= 0 && Index < 3 &&
3028 "Expected FPFastMathDefaultInfo for half, float, or double");
3029 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3030 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3031 return FPFastMathDefaultInfoVec[Index];
3032}
3033
3034static void collectFPFastMathDefaults(
3035 const Module &M,
3036 SPIRV::FPFastMathDefaultInfoMapTy &FPFastMathDefaultInfoMap,
3037 const SPIRVSubtarget &ST, const Function *OnlyF = nullptr) {
3038 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
3039 return;
3040
3041 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
3042 // We need the entry point (function) as the key, and the target
3043 // type and flags as the value.
3044 // We also need to check ContractionOff and SignedZeroInfNanPreserve
3045 // execution modes, as they are now deprecated and must be replaced
3046 // with FPFastMathDefaultInfo.
3047 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
3048 if (!Node)
3049 return;
3050
3051 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
3052 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
3053 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
3054 const Function *F = cast<Function>(
3055 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
3056 if (OnlyF && F != OnlyF)
3057 continue;
3058 const auto EM =
3059 cast<ConstantInt>(
3060 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
3061 ->getZExtValue();
3062 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
3063 assert(MDN->getNumOperands() == 4 &&
3064 "Expected 4 operands for FPFastMathDefault");
3065
3066 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
3067 unsigned Flags =
3068 cast<ConstantInt>(
3069 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
3070 ->getZExtValue();
3071 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3072 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3073 SPIRV::FPFastMathDefaultInfo &Info =
3074 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
3075 Info.FastMathFlags = Flags;
3076 Info.FPFastMathDefault = true;
3077 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
3078 assert(MDN->getNumOperands() == 2 &&
3079 "Expected no operands for ContractionOff");
3080
3081 // We need to save this info for every possible FP type, i.e. {half,
3082 // float, double, fp128}.
3083 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3084 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3085 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3086 Info.ContractionOff = true;
3087 }
3088 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
3089 assert(MDN->getNumOperands() == 3 &&
3090 "Expected 1 operand for SignedZeroInfNanPreserve");
3091 unsigned TargetWidth =
3092 cast<ConstantInt>(
3093 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
3094 ->getZExtValue();
3095 // We need to save this info only for the FP type with TargetWidth.
3096 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3097 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3098 int Index = SPIRV::FPFastMathDefaultInfoVector::
3099 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
3100 assert(Index >= 0 && Index < 3 &&
3101 "Expected FPFastMathDefaultInfo for half, float, or double");
3102 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3103 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3104 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
3105 }
3106 }
3107}
3108
3109SPIRVModuleAnalysisImpl::SPIRVModuleAnalysisImpl(const SPIRVSubtarget &ST,
3110 SPIRV::ModuleAnalysisInfo &MAI,
3111 MachineFunctionGetter GetMF)
3112 : ST(&ST), GR(ST.getSPIRVGlobalRegistry()), TII(ST.getInstrInfo()),
3113 MAI(MAI), GetMF(GetMF) {}
3114
3115void SPIRVModuleAnalysisImpl::run(const Module &M) {
3116 setBaseInfo(M);
3117
3118 addMBBNames(M, GetMF, MAI);
3119 collectFPFastMathDefaults(M, FPFastMathDefaultInfoMap&: MAI.FPFastMathDefaultInfoMap, ST: *ST);
3120
3121 collectReqs(M, MAI, GetMF, ST: *ST);
3122
3123 // Process type/const/global var/func decl instructions, number their
3124 // destination registers from 0 to N, collect Extensions and Capabilities.
3125 collectDeclarations(M);
3126
3127 // Number rest of registers from N+1 onwards.
3128 numberRegistersGlobally(M);
3129
3130 // Collect OpName, OpEntryPoint, OpDecorate etc, process other instructions.
3131 processOtherInstrs(M);
3132
3133 // If there are no entry points, we need the Linkage capability.
3134 if (MAI.MS[SPIRV::MB_EntryPoints].empty())
3135 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::Linkage);
3136
3137 // Set maximum ID used.
3138 GR->setBound(MAI.MaxID);
3139}
3140
3141void SPIRVModuleAnalysisWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
3142 AU.addRequired<TargetPassConfig>();
3143 AU.addRequired<MachineModuleInfoWrapperPass>();
3144}
3145
3146bool SPIRVModuleAnalysisWrapperPass::runOnModule(Module &M) {
3147 SPIRVTargetMachine &TM =
3148 getAnalysis<TargetPassConfig>().getTM<SPIRVTargetMachine>();
3149 MachineModuleInfo &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
3150 SPIRVModuleAnalysisImpl(
3151 *TM.getSubtargetImpl(), MAI,
3152 [&MMI](const Function &F) { return MMI.getMachineFunction(F); })
3153 .run(M);
3154 return false;
3155}
3156
3157AnalysisKey SPIRVModuleAnalysis::Key;
3158
3159SPIRVModuleAnalysis::Result
3160SPIRVModuleAnalysis::run(Module &M, ModuleAnalysisManager &MAM) {
3161 const auto &TM = static_cast<const SPIRVTargetMachine &>(
3162 MAM.getResult<MachineModuleAnalysis>(IR&: M).getMMI().getTarget());
3163 FunctionAnalysisManager &FAM =
3164 MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
3165 Result MAI;
3166 SPIRVModuleAnalysisImpl(*TM.getSubtargetImpl(), MAI,
3167 [&FAM](const Function &F) -> MachineFunction * {
3168 MachineFunctionAnalysis::Result *MFA =
3169 FAM.getCachedResult<MachineFunctionAnalysis>(
3170 IR&: const_cast<Function &>(F));
3171 assert((MFA || F.isDeclaration()) &&
3172 "Missing MachineFunction for definition");
3173 return MFA ? &MFA->getMF() : nullptr;
3174 })
3175 .run(M);
3176 return MAI;
3177}
3178
3179static bool prepareModuleAnalysis(MachineFunction &MF) {
3180 const auto &ST = MF.getSubtarget<SPIRVSubtarget>();
3181 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
3182 const SPIRVInstrInfo &TII = *ST.getInstrInfo();
3183 SPIRV::RequirementHandler Reqs;
3184 Reqs.initAvailableCapabilities(ST);
3185 SPIRV::FPFastMathDefaultInfoMapTy FPFastMathDefaultInfoMap;
3186 const Function &F = MF.getFunction();
3187 collectFPFastMathDefaults(M: *F.getParent(), FPFastMathDefaultInfoMap, ST, OnlyF: &F);
3188 // Add decorations related to MI flags.
3189 bool Changed = false;
3190 for (auto &MBB : MF)
3191 for (auto &MI : make_early_inc_range(Range&: MBB))
3192 Changed |= handleMIFlagDecoration(I&: MI, ST, TII, Reqs, GR,
3193 FPFastMathDefaultInfoVec&: FPFastMathDefaultInfoMap[&F]);
3194 return Changed;
3195}
3196
3197namespace {
3198class SPIRVPrepareModuleAnalysisLegacy : public MachineFunctionPass {
3199public:
3200 static char ID;
3201 SPIRVPrepareModuleAnalysisLegacy() : MachineFunctionPass(ID) {}
3202 bool runOnMachineFunction(MachineFunction &MF) override {
3203 return prepareModuleAnalysis(MF);
3204 }
3205};
3206} // namespace
3207
3208char SPIRVPrepareModuleAnalysisLegacy::ID = 0;
3209
3210INITIALIZE_PASS(SPIRVPrepareModuleAnalysisLegacy,
3211 "spirv-prepare-module-analysis",
3212 "SPIRV prepare module analysis", false, false)
3213
3214FunctionPass *llvm::createSPIRVPrepareModuleAnalysisLegacyPass() {
3215 return new SPIRVPrepareModuleAnalysisLegacy();
3216}
3217
3218PreservedAnalyses
3219SPIRVPrepareModuleAnalysisPass::run(MachineFunction &MF,
3220 MachineFunctionAnalysisManager &MFAM) {
3221 return prepareModuleAnalysis(MF) ? getMachineFunctionPassPreservedAnalyses()
3222 : PreservedAnalyses::all();
3223}
3224