1//===- SPIRVModuleAnalysis.cpp - analysis of global instrs & regs - C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The analysis collects instructions that should be output at the module level
10// and performs the global register numbering.
11//
12// The results of this analysis are used in AsmPrinter to rename registers
13// globally and to output required instructions at the module level.
14//
15//===----------------------------------------------------------------------===//
16
17// TODO: Per LLVM best practices, the report_fatal_error (deprecated) /
18// ReportFatalUsageError calls in this file should be replaced with the
19// Diagnostic infrastructure (e.g. the reportUnsupported function below).
20
21#include "SPIRVModuleAnalysis.h"
22#include "MCTargetDesc/SPIRVBaseInfo.h"
23#include "MCTargetDesc/SPIRVMCTargetDesc.h"
24#include "SPIRV.h"
25#include "SPIRVSubtarget.h"
26#include "SPIRVTargetMachine.h"
27#include "SPIRVUtils.h"
28#include "llvm/ADT/STLExtras.h"
29#include "llvm/ADT/StringSwitch.h"
30#include "llvm/CodeGen/MachineFunctionAnalysis.h"
31#include "llvm/CodeGen/MachineModuleInfo.h"
32#include "llvm/CodeGen/TargetPassConfig.h"
33
34using namespace llvm;
35
36#define DEBUG_TYPE "spirv-module-analysis"
37
38char llvm::SPIRVModuleAnalysisWrapperPass::ID = 0;
39
40INITIALIZE_PASS(SPIRVModuleAnalysisWrapperPass, DEBUG_TYPE,
41 "SPIRV module analysis", true, true)
42
43static void reportUnsupported(const MachineInstr &MI, const char *Msg) {
44 const Function &Func = MI.getMF()->getFunction();
45 Func.getContext().diagnose(
46 DI: DiagnosticInfoUnsupported(Func, Msg, MI.getDebugLoc()));
47}
48
49// Retrieve an unsigned from an MDNode with a list of them as operands.
50static unsigned getMetadataUInt(MDNode *MdNode, unsigned OpIndex,
51 unsigned DefaultVal = 0) {
52 if (MdNode && OpIndex < MdNode->getNumOperands()) {
53 const auto &Op = MdNode->getOperand(I: OpIndex);
54 return mdconst::extract<ConstantInt>(MD: Op)->getZExtValue();
55 }
56 return DefaultVal;
57}
58
59// Return true if Cap should be avoided when another option enables the feature.
60static bool shouldAvoidCapability(const SPIRVSubtarget &ST,
61 SPIRV::Capability::Capability Cap) {
62 return Cap == (ST.isShader() ? SPIRV::Capability::Kernel
63 : SPIRV::Capability::Shader) ||
64 is_contained(Range: ST.getCLOpts().avoid_capabilities, Element: Cap);
65}
66
67static SPIRV::Requirements
68getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
69 unsigned i, const SPIRVSubtarget &ST,
70 SPIRV::RequirementHandler &Reqs) {
71 VersionTuple ReqMinVer = getSymbolicOperandMinVersion(Category, Value: i);
72 VersionTuple ReqMaxVer = getSymbolicOperandMaxVersion(Category, Value: i);
73 VersionTuple SPIRVVersion = ST.getSPIRVVersion();
74 bool MinVerOK = SPIRVVersion.empty() || SPIRVVersion >= ReqMinVer;
75 bool MaxVerOK =
76 ReqMaxVer.empty() || SPIRVVersion.empty() || SPIRVVersion <= ReqMaxVer;
77 bool VerOK = MinVerOK && MaxVerOK;
78 CapabilityList ReqCaps = getSymbolicOperandCapabilities(Category, Value: i);
79 ExtensionList ReqExts = getSymbolicOperandExtensions(Category, Value: i);
80 // A met minimum version makes the extension unnecessary.
81 if (VerOK && !ReqMinVer.empty())
82 ReqExts.clear();
83 if (ReqCaps.empty()) {
84 if (ReqExts.empty()) {
85 if (VerOK)
86 return {true, {}, {}, ReqMinVer, ReqMaxVer};
87 return {false, {}, {}, VersionTuple(), VersionTuple()};
88 }
89 } else if (VerOK) {
90 if (ReqCaps.size() == 1) {
91 auto Cap = ReqCaps[0];
92 if (Reqs.isCapabilityAvailable(Cap)) {
93 ReqExts.append(RHS: getSymbolicOperandExtensions(
94 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
95 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
96 }
97 } else {
98 // By SPIR-V specification: "If an instruction, enumerant, or other
99 // feature specifies multiple enabling capabilities, only one such
100 // capability needs to be declared to use the feature." However, one
101 // capability may be preferred over another. We use command line
102 // argument(s) and AvoidCapabilities to avoid selection of certain
103 // capabilities if there are other options.
104 CapabilityList UseCaps;
105 for (auto Cap : ReqCaps)
106 if (Reqs.isCapabilityAvailable(Cap))
107 UseCaps.push_back(Elt: Cap);
108 for (size_t i = 0, Sz = UseCaps.size(); i < Sz; ++i) {
109 auto Cap = UseCaps[i];
110 if (i == Sz - 1 || !shouldAvoidCapability(ST, Cap)) {
111 ReqExts.append(RHS: getSymbolicOperandExtensions(
112 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
113 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
114 }
115 }
116 }
117 }
118 // If there are no capabilities, or we can't satisfy the version or
119 // capability requirements, use the list of extensions (if the subtarget
120 // can handle them all). Without an extension the version must hold. An
121 // extension stands in for an unmet version.
122 if ((!ReqExts.empty() || VerOK) &&
123 llvm::all_of(Range&: ReqExts, P: [&ST](const SPIRV::Extension::Extension &Ext) {
124 return ST.canUseExtension(E: Ext);
125 })) {
126 if (VerOK)
127 return {true, {}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
128 return {true, {}, std::move(ReqExts), VersionTuple(), VersionTuple()};
129 }
130 return {false, {}, {}, VersionTuple(), VersionTuple()};
131}
132
133void SPIRVModuleAnalysisImpl::setBaseInfo(const Module &M) {
134 MAI.MaxID = 0;
135 for (int i = 0; i < SPIRV::NUM_MODULE_SECTIONS; i++)
136 MAI.MS[i].clear();
137 MAI.RegisterAliasTable.clear();
138 MAI.InstrsToDelete.clear();
139 MAI.MBBNames.clear();
140 MAI.GlobalObjMap.clear();
141 MAI.GlobalVarList.clear();
142 MAI.ExtInstSetMap.clear();
143 MAI.InstrAuxDataRecords.clear();
144 MAI.DebugValueMIs.clear();
145 MAI.Reqs.clear();
146 MAI.Reqs.initAvailableCapabilities(ST: *ST);
147
148 // TODO: determine memory model and source language from the configuratoin.
149 if (auto MemModel = M.getNamedMetadata(Name: "spirv.MemoryModel")) {
150 auto MemMD = MemModel->getOperand(i: 0);
151 MAI.Addr = static_cast<SPIRV::AddressingModel::AddressingModel>(
152 getMetadataUInt(MdNode: MemMD, OpIndex: 0));
153 MAI.Mem =
154 static_cast<SPIRV::MemoryModel::MemoryModel>(getMetadataUInt(MdNode: MemMD, OpIndex: 1));
155 } else {
156 // TODO: Add support for VulkanMemoryModel.
157 MAI.Mem = ST->isShader() ? SPIRV::MemoryModel::GLSL450
158 : SPIRV::MemoryModel::OpenCL;
159 if (MAI.Mem == SPIRV::MemoryModel::OpenCL) {
160 unsigned PtrSize = ST->getPointerSize();
161 MAI.Addr = PtrSize == 32 ? SPIRV::AddressingModel::Physical32
162 : PtrSize == 64 ? SPIRV::AddressingModel::Physical64
163 : SPIRV::AddressingModel::Logical;
164 } else {
165 // TODO: Add support for PhysicalStorageBufferAddress.
166 MAI.Addr = SPIRV::AddressingModel::Logical;
167 }
168 }
169 // Get the OpenCL version number from metadata.
170 // TODO: support other source languages.
171 if (auto VerNode = M.getNamedMetadata(Name: "opencl.ocl.version")) {
172 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_C;
173 // Construct version literal in accordance with SPIRV-LLVM-Translator.
174 // TODO: support multiple OCL version metadata.
175 assert(VerNode->getNumOperands() > 0 && "Invalid SPIR");
176 auto VersionMD = VerNode->getOperand(i: 0);
177 unsigned MajorNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 0, DefaultVal: 2);
178 unsigned MinorNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 1);
179 unsigned RevNum = getMetadataUInt(MdNode: VersionMD, OpIndex: 2);
180 // Prevent Major part of OpenCL version to be 0
181 MAI.SrcLangVersion =
182 (std::max(a: 1U, b: MajorNum) * 100 + MinorNum) * 1000 + RevNum;
183 // When opencl.cxx.version is also present, validate compatibility
184 // and use C++ for OpenCL as source language with the C++ version.
185 if (auto *CxxVerNode = M.getNamedMetadata(Name: "opencl.cxx.version")) {
186 assert(CxxVerNode->getNumOperands() > 0 && "Invalid SPIR");
187 auto *CxxMD = CxxVerNode->getOperand(i: 0);
188 unsigned CxxVer =
189 (getMetadataUInt(MdNode: CxxMD, OpIndex: 0) * 100 + getMetadataUInt(MdNode: CxxMD, OpIndex: 1)) * 1000 +
190 getMetadataUInt(MdNode: CxxMD, OpIndex: 2);
191 if ((MAI.SrcLangVersion == 200000 && CxxVer == 100000) ||
192 (MAI.SrcLangVersion == 300000 && CxxVer == 202100000)) {
193 MAI.SrcLang = SPIRV::SourceLanguage::CPP_for_OpenCL;
194 MAI.SrcLangVersion = CxxVer;
195 } else {
196 report_fatal_error(
197 reason: "opencl cxx version is not compatible with opencl c version!");
198 }
199 }
200 } else {
201 // If there is no information about OpenCL version we are forced to generate
202 // OpenCL 1.0 by default for the OpenCL environment to avoid puzzling
203 // run-times with Unknown/0.0 version output. For a reference, LLVM-SPIRV
204 // Translator avoids potential issues with run-times in a similar manner.
205 if (!ST->isShader()) {
206 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_CPP;
207 MAI.SrcLangVersion = 100000;
208 } else {
209 MAI.SrcLang = SPIRV::SourceLanguage::Unknown;
210 MAI.SrcLangVersion = 0;
211 }
212 }
213
214 if (auto ExtNode = M.getNamedMetadata(Name: "opencl.used.extensions")) {
215 for (unsigned I = 0, E = ExtNode->getNumOperands(); I != E; ++I) {
216 MDNode *MD = ExtNode->getOperand(i: I);
217 if (!MD || MD->getNumOperands() == 0)
218 continue;
219 for (unsigned J = 0, N = MD->getNumOperands(); J != N; ++J)
220 MAI.SrcExt.insert(key: cast<MDString>(Val: MD->getOperand(I: J))->getString());
221 }
222 }
223
224 // Update required capabilities for this memory model, addressing model and
225 // source language.
226 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::MemoryModelOperand,
227 i: MAI.Mem, ST: *ST);
228 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::SourceLanguageOperand,
229 i: MAI.SrcLang, ST: *ST);
230 MAI.Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::AddressingModelOperand,
231 i: MAI.Addr, ST: *ST);
232
233 if (MAI.Mem == SPIRV::MemoryModel::VulkanKHR)
234 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_vulkan_memory_model);
235
236 if (!ST->isShader()) {
237 // TODO: check if it's required by default.
238 MAI.ExtInstSetMap[static_cast<unsigned>(
239 SPIRV::InstructionSet::OpenCL_std)] = MAI.getNextIDRegister();
240 }
241}
242
243// Appends the signature of the decoration instructions that decorate R to
244// Signature.
245static void appendDecorationsForReg(const MachineRegisterInfo &MRI, Register R,
246 InstrSignature &Signature) {
247 for (MachineInstr &UseMI : MRI.use_instructions(Reg: R)) {
248 // We don't handle OpDecorateId because getting the register alias for the
249 // ID can cause problems, and we do not need it for now.
250 if (UseMI.getOpcode() != SPIRV::OpDecorate &&
251 UseMI.getOpcode() != SPIRV::OpMemberDecorate)
252 continue;
253
254 for (unsigned I = 0; I < UseMI.getNumOperands(); ++I) {
255 const MachineOperand &MO = UseMI.getOperand(i: I);
256 if (MO.isReg())
257 continue;
258 Signature.push_back(Elt: hash_value(MO));
259 }
260 }
261}
262
263// Returns a representation of an instruction as a vector of MachineOperand
264// hash values, see llvm::hash_value(const MachineOperand &MO) for details.
265// This creates a signature of the instruction with the same content
266// that MachineOperand::isIdenticalTo uses for comparison.
267static InstrSignature instrToSignature(const MachineInstr &MI,
268 SPIRV::ModuleAnalysisInfo &MAI,
269 bool UseDefReg) {
270 Register DefReg;
271 InstrSignature Signature{MI.getOpcode()};
272 for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
273 // The only decorations that can be applied more than once to a given <id>
274 // or structure member are FuncParamAttr (38), UserSemantic (5635),
275 // CacheControlLoadINTEL (6442), and CacheControlStoreINTEL (6443). For all
276 // the rest of decorations, we will only add to the signature the Opcode,
277 // the id to which it applies, and the decoration id, disregarding any
278 // decoration flags. This will ensure that any subsequent decoration with
279 // the same id will be deemed as a duplicate. Then, at the call site, we
280 // will be able to handle duplicates in the best way.
281 unsigned Opcode = MI.getOpcode();
282 if ((Opcode == SPIRV::OpDecorate) && i >= 2) {
283 unsigned DecorationID = MI.getOperand(i: 1).getImm();
284 if (DecorationID != SPIRV::Decoration::FuncParamAttr &&
285 DecorationID != SPIRV::Decoration::UserSemantic &&
286 DecorationID != SPIRV::Decoration::CacheControlLoadINTEL &&
287 DecorationID != SPIRV::Decoration::CacheControlStoreINTEL)
288 continue;
289 }
290 const MachineOperand &MO = MI.getOperand(i);
291 size_t h;
292 if (MO.isReg()) {
293 if (!UseDefReg && MO.isDef()) {
294 assert(!DefReg.isValid() && "Multiple def registers.");
295 DefReg = MO.getReg();
296 continue;
297 }
298 Register RegAlias = MAI.getRegisterAlias(MF: MI.getMF(), Reg: MO.getReg());
299 if (!RegAlias.isValid()) {
300 LLVM_DEBUG({
301 dbgs() << "Unexpectedly, no global id found for the operand ";
302 MO.print(dbgs());
303 dbgs() << "\nInstruction: ";
304 MI.print(dbgs());
305 dbgs() << "\n";
306 });
307 report_fatal_error(reason: "All v-regs must have been mapped to global id's");
308 }
309 // mimic llvm::hash_value(const MachineOperand &MO)
310 h = hash_combine(args: MO.getType(), args: (unsigned)RegAlias, args: MO.getSubReg(),
311 args: MO.isDef());
312 } else {
313 h = hash_value(MO);
314 }
315 Signature.push_back(Elt: h);
316 }
317
318 if (DefReg.isValid()) {
319 // Decorations change the semantics of the current instruction. So two
320 // identical instruction with different decorations cannot be merged. That
321 // is why we add the decorations to the signature.
322 appendDecorationsForReg(MRI: MI.getMF()->getRegInfo(), R: DefReg, Signature);
323 }
324 return Signature;
325}
326
327// Operand index of Invoke in device enqueue instructions, 0 if none.
328static unsigned getInvokeOperandIdx(unsigned Opcode) {
329 switch (Opcode) {
330 case SPIRV::OpEnqueueKernel:
331 return 8;
332 case SPIRV::OpGetKernelNDrangeSubGroupCount:
333 case SPIRV::OpGetKernelNDrangeMaxSubGroupSize:
334 return 3;
335 case SPIRV::OpGetKernelWorkGroupSize:
336 case SPIRV::OpGetKernelPreferredWorkGroupSizeMultiple:
337 return 2;
338 default:
339 return 0;
340 }
341}
342
343bool SPIRVModuleAnalysisImpl::isDeclSection(const MachineRegisterInfo &MRI,
344 const MachineInstr &MI) {
345 unsigned Opcode = MI.getOpcode();
346 switch (Opcode) {
347 case SPIRV::OpTypeForwardPointer:
348 // omit now, collect later
349 return false;
350 case SPIRV::OpVariable:
351 case SPIRV::OpUntypedVariableKHR:
352 return static_cast<SPIRV::StorageClass::StorageClass>(
353 MI.getOperand(i: 2).getImm()) != SPIRV::StorageClass::Function;
354 case SPIRV::OpFunction:
355 case SPIRV::OpFunctionParameter:
356 return true;
357 }
358 if (GR->hasConstFunPtr() && Opcode == SPIRV::OpUndef) {
359 // The OpUndef may be a placeholder for a function reference recorded by
360 // selectGlobalValue. Skip emitting it if any user consumes it as a
361 // function-pointer-like operand (OpConstantFunctionPointerINTEL operand 2,
362 // or the Invoke operand of a device enqueue instruction). The rewrite
363 // happens in visitFunPtrUse, which aliases the OpUndef's vreg to the
364 // function's global <id>.
365 Register DefReg = MI.getOperand(i: 0).getReg();
366 if (GR->getFunctionDefinitionByUse(Use: &MI.getOperand(i: 0))) {
367 for (MachineInstr &UseMI : MRI.use_instructions(Reg: DefReg)) {
368 unsigned UseOp = UseMI.getOpcode();
369 if (UseOp == SPIRV::OpConstantFunctionPointerINTEL ||
370 getInvokeOperandIdx(Opcode: UseOp)) {
371 MAI.setSkipEmission(&MI);
372 return false;
373 }
374 }
375 }
376 for (MachineInstr &UseMI : MRI.use_instructions(Reg: DefReg)) {
377 if (UseMI.getOpcode() != SPIRV::OpConstantFunctionPointerINTEL)
378 continue;
379 // it's a dummy definition, FP constant refers to a function,
380 // and this is resolved in another way; let's skip this definition
381 assert(UseMI.getOperand(2).isReg() &&
382 UseMI.getOperand(2).getReg() == DefReg);
383 MAI.setSkipEmission(&MI);
384 return false;
385 }
386 }
387 return TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||
388 TII->isInlineAsmDefInstr(MI);
389}
390
391// This is a special case of a function pointer referring to a possibly
392// forward function declaration. The operand is a dummy OpUndef that
393// requires a special treatment.
394// FunPtrOp is the MachineOperand previously recorded via
395// SPIRVGlobalRegistry::recordFunctionPointer, identifying which Function
396// this placeholder refers to.
397void SPIRVModuleAnalysisImpl::visitFunPtrUse(
398 Register OpReg, const MachineOperand *FunPtrOp,
399 InstrGRegsMap &SignatureToGReg,
400 std::map<const Value *, unsigned> &GlobalToGReg,
401 const MachineFunction *MF) {
402 const MachineOperand *OpFunDef = GR->getFunctionDefinitionByUse(Use: FunPtrOp);
403 assert(OpFunDef && OpFunDef->isReg());
404 // find the actual function definition and number it globally in advance
405 const MachineInstr *OpDefMI = OpFunDef->getParent();
406 assert(OpDefMI && OpDefMI->getOpcode() == SPIRV::OpFunction);
407 const MachineFunction *FunDefMF = OpDefMI->getParent()->getParent();
408 const MachineRegisterInfo &FunDefMRI = FunDefMF->getRegInfo();
409 do {
410 visitDecl(MRI: FunDefMRI, SignatureToGReg, GlobalToGReg, MF: FunDefMF, MI: *OpDefMI);
411 OpDefMI = OpDefMI->getNextNode();
412 } while (OpDefMI && (OpDefMI->getOpcode() == SPIRV::OpFunction ||
413 OpDefMI->getOpcode() == SPIRV::OpFunctionParameter));
414 // associate the function pointer with the newly assigned global number
415 MCRegister GlobalFunDefReg =
416 MAI.getRegisterAlias(MF: FunDefMF, Reg: OpFunDef->getReg());
417 assert(GlobalFunDefReg.isValid() &&
418 "Function definition must refer to a global register");
419 MAI.setRegisterAlias(MF, Reg: OpReg, AliasReg: GlobalFunDefReg);
420}
421
422// Depth first recursive traversal of dependencies. Repeated visits are guarded
423// by MAI.hasRegisterAlias().
424void SPIRVModuleAnalysisImpl::visitDecl(
425 const MachineRegisterInfo &MRI, InstrGRegsMap &SignatureToGReg,
426 std::map<const Value *, unsigned> &GlobalToGReg, const MachineFunction *MF,
427 const MachineInstr &MI) {
428 unsigned Opcode = MI.getOpcode();
429
430 // Process each operand of the instruction to resolve dependencies
431 for (const MachineOperand &MO : MI.operands()) {
432 if (!MO.isReg() || MO.isDef())
433 continue;
434 Register OpReg = MO.getReg();
435 // Handle function pointers special case
436 if (Opcode == SPIRV::OpConstantFunctionPointerINTEL &&
437 MRI.getRegClass(Reg: OpReg) == &SPIRV::pIDRegClass) {
438 visitFunPtrUse(OpReg, FunPtrOp: &MI.getOperand(i: 2), SignatureToGReg, GlobalToGReg,
439 MF);
440 continue;
441 }
442 // Skip already processed instructions
443 if (MAI.hasRegisterAlias(MF, Reg: MO.getReg()))
444 continue;
445 // Recursively visit dependencies
446 if (const MachineInstr *OpDefMI = MRI.getUniqueVRegDef(Reg: OpReg)) {
447 if (isDeclSection(MRI, MI: *OpDefMI))
448 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, MI: *OpDefMI);
449 continue;
450 }
451 // Handle the unexpected case of no unique definition for the SPIR-V
452 // instruction
453 LLVM_DEBUG({
454 dbgs() << "Unexpectedly, no unique definition for the operand ";
455 MO.print(dbgs());
456 dbgs() << "\nInstruction: ";
457 MI.print(dbgs());
458 dbgs() << "\n";
459 });
460 report_fatal_error(
461 reason: "No unique definition is found for the virtual register");
462 }
463
464 MCRegister GReg;
465 bool IsFunDef = false;
466 if (TII->isSpecConstantInstr(MI)) {
467 GReg = MAI.getNextIDRegister();
468 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
469 } else if (Opcode == SPIRV::OpFunction ||
470 Opcode == SPIRV::OpFunctionParameter) {
471 GReg = handleFunctionOrParameter(MF, MI, GlobalToGReg, IsFunDef);
472 } else if (Opcode == SPIRV::OpTypeStruct ||
473 Opcode == SPIRV::OpConstantComposite) {
474 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);
475 const MachineInstr *NextInstr = MI.getNextNode();
476 while (NextInstr &&
477 ((Opcode == SPIRV::OpTypeStruct &&
478 NextInstr->getOpcode() == SPIRV::OpTypeStructContinuedINTEL) ||
479 (Opcode == SPIRV::OpConstantComposite &&
480 NextInstr->getOpcode() ==
481 SPIRV::OpConstantCompositeContinuedINTEL))) {
482 MCRegister Tmp = handleTypeDeclOrConstant(MI: *NextInstr, SignatureToGReg);
483 MAI.setRegisterAlias(MF, Reg: NextInstr->getOperand(i: 0).getReg(), AliasReg: Tmp);
484 MAI.setSkipEmission(NextInstr);
485 NextInstr = NextInstr->getNextNode();
486 }
487 } else if (TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||
488 TII->isInlineAsmDefInstr(MI)) {
489 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);
490 } else if (Opcode == SPIRV::OpVariable ||
491 Opcode == SPIRV::OpUntypedVariableKHR) {
492 GReg = handleVariable(MF, MI, GlobalToGReg);
493 } else {
494 LLVM_DEBUG({
495 dbgs() << "\nInstruction: ";
496 MI.print(dbgs());
497 dbgs() << "\n";
498 });
499 llvm_unreachable("Unexpected instruction is visited");
500 }
501 MAI.setRegisterAlias(MF, Reg: MI.getOperand(i: 0).getReg(), AliasReg: GReg);
502 if (!IsFunDef)
503 MAI.setSkipEmission(&MI);
504}
505
506MCRegister SPIRVModuleAnalysisImpl::handleFunctionOrParameter(
507 const MachineFunction *MF, const MachineInstr &MI,
508 std::map<const Value *, unsigned> &GlobalToGReg, bool &IsFunDef) {
509 const Value *GObj = GR->getGlobalObject(MF, R: MI.getOperand(i: 0).getReg());
510 assert(GObj && "Unregistered global definition");
511 const Function *F = dyn_cast<Function>(Val: GObj);
512 if (!F)
513 F = dyn_cast<Argument>(Val: GObj)->getParent();
514 assert(F && "Expected a reference to a function or an argument");
515 IsFunDef = !F->isDeclaration();
516 auto [It, Inserted] = GlobalToGReg.try_emplace(k: GObj);
517 if (!Inserted)
518 return It->second;
519 MCRegister GReg = MAI.getNextIDRegister();
520 It->second = GReg;
521 if (!IsFunDef)
522 MAI.MS[SPIRV::MB_ExtFuncDecls].push_back(Elt: &MI);
523 return GReg;
524}
525
526MCRegister SPIRVModuleAnalysisImpl::handleTypeDeclOrConstant(
527 const MachineInstr &MI, InstrGRegsMap &SignatureToGReg) {
528 InstrSignature MISign = instrToSignature(MI, MAI, UseDefReg: false);
529 auto [It, Inserted] = SignatureToGReg.try_emplace(k: MISign);
530 if (!Inserted)
531 return It->second;
532 MCRegister GReg = MAI.getNextIDRegister();
533 It->second = GReg;
534 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
535 return GReg;
536}
537
538MCRegister SPIRVModuleAnalysisImpl::handleVariable(
539 const MachineFunction *MF, const MachineInstr &MI,
540 std::map<const Value *, unsigned> &GlobalToGReg) {
541 MAI.GlobalVarList.push_back(Elt: &MI);
542 const Value *GObj = GR->getGlobalObject(MF, R: MI.getOperand(i: 0).getReg());
543 assert(GObj && "Unregistered global definition");
544 auto [It, Inserted] = GlobalToGReg.try_emplace(k: GObj);
545 if (!Inserted)
546 return It->second;
547 MCRegister GReg = MAI.getNextIDRegister();
548 It->second = GReg;
549 MAI.MS[SPIRV::MB_TypeConstVars].push_back(Elt: &MI);
550 if (const auto *GV = dyn_cast<GlobalVariable>(Val: GObj))
551 MAI.GlobalObjMap[GV] = GReg;
552 return GReg;
553}
554
555void SPIRVModuleAnalysisImpl::collectDeclarations(const Module &M) {
556 InstrGRegsMap SignatureToGReg;
557 std::map<const Value *, unsigned> GlobalToGReg;
558 for (const Function &F : M) {
559 MachineFunction *MF = GetMF(F);
560 if (!MF)
561 continue;
562 const MachineRegisterInfo &MRI = MF->getRegInfo();
563 unsigned PastHeader = 0;
564 for (MachineBasicBlock &MBB : *MF) {
565 for (MachineInstr &MI : MBB) {
566 if (MI.getNumOperands() == 0)
567 continue;
568 unsigned Opcode = MI.getOpcode();
569 if (Opcode == SPIRV::OpFunction) {
570 if (PastHeader == 0) {
571 PastHeader = 1;
572 continue;
573 }
574 } else if (Opcode == SPIRV::OpFunctionParameter) {
575 if (PastHeader < 2)
576 continue;
577 } else if (PastHeader > 0) {
578 PastHeader = 2;
579 }
580
581 const MachineOperand &DefMO = MI.getOperand(i: 0);
582 switch (Opcode) {
583 case SPIRV::OpExtension:
584 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::Extension(DefMO.getImm()));
585 MAI.setSkipEmission(&MI);
586 break;
587 case SPIRV::OpCapability:
588 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::Capability(DefMO.getImm()));
589 MAI.setSkipEmission(&MI);
590 if (PastHeader > 0)
591 PastHeader = 2;
592 break;
593 default:
594 if (DefMO.isReg() && isDeclSection(MRI, MI) &&
595 !MAI.hasRegisterAlias(MF, Reg: DefMO.getReg()))
596 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, MI);
597 // Device enqueue instructions are not decls, but their Invoke
598 // operand may be a function-pointer placeholder OpUndef. Resolve it
599 // to the OpFunction's global <id> via visitFunPtrUse.
600 if (unsigned InvokeIdx = getInvokeOperandIdx(Opcode)) {
601 const MachineOperand &InvokeMO = MI.getOperand(i: InvokeIdx);
602 if (InvokeMO.isReg()) {
603 Register InvokeReg = InvokeMO.getReg();
604 if (!MAI.hasRegisterAlias(MF, Reg: InvokeReg)) {
605 if (const MachineInstr *DefMI =
606 MRI.getUniqueVRegDef(Reg: InvokeReg)) {
607 if (DefMI->getOpcode() == SPIRV::OpUndef) {
608 const MachineOperand *FunPtrOp = &DefMI->getOperand(i: 0);
609 if (GR->getFunctionDefinitionByUse(Use: FunPtrOp))
610 visitFunPtrUse(OpReg: InvokeReg, FunPtrOp, SignatureToGReg,
611 GlobalToGReg, MF);
612 }
613 }
614 }
615 }
616 }
617 }
618 }
619 }
620 }
621}
622
623// Look for IDs declared with Import linkage, and map the corresponding function
624// to the register defining that variable (which will usually be the result of
625// an OpFunction). This lets us call externally imported functions using
626// the correct ID registers.
627void SPIRVModuleAnalysisImpl::collectFuncNames(MachineInstr &MI,
628 const Function *F) {
629 if (MI.getOpcode() == SPIRV::OpDecorate) {
630 // If it's got Import linkage.
631 auto Dec = MI.getOperand(i: 1).getImm();
632 if (Dec == SPIRV::Decoration::LinkageAttributes) {
633 auto Lnk = MI.getOperand(i: MI.getNumOperands() - 1).getImm();
634 if (Lnk == SPIRV::LinkageType::Import) {
635 // Map imported function name to function ID register.
636 const Function *ImportedFunc =
637 F->getParent()->getFunction(Name: getStringImm(MI, StartIndex: 2));
638 Register Target = MI.getOperand(i: 0).getReg();
639 MAI.GlobalObjMap[ImportedFunc] =
640 MAI.getRegisterAlias(MF: MI.getMF(), Reg: Target);
641 }
642 }
643 } else if (MI.getOpcode() == SPIRV::OpFunction) {
644 // Record all internal OpFunction declarations.
645 Register Reg = MI.defs().begin()->getReg();
646 MCRegister GlobalReg = MAI.getRegisterAlias(MF: MI.getMF(), Reg);
647 assert(GlobalReg.isValid());
648 MAI.GlobalObjMap[F] = GlobalReg;
649 }
650}
651
652// Collect the given instruction in the specified MS. We assume global register
653// numbering has already occurred by this point. We can directly compare reg
654// arguments when detecting duplicates.
655static void collectOtherInstr(MachineInstr &MI, SPIRV::ModuleAnalysisInfo &MAI,
656 SPIRV::ModuleSectionType MSType, InstrTraces &IS,
657 bool Append = true) {
658 MAI.setSkipEmission(&MI);
659 InstrSignature MISign = instrToSignature(MI, MAI, UseDefReg: true);
660 auto FoundMI = IS.insert(x: std::move(MISign));
661 if (!FoundMI.second) {
662 if (MI.getOpcode() == SPIRV::OpDecorate) {
663 assert(MI.getNumOperands() >= 2 &&
664 "Decoration instructions must have at least 2 operands");
665 assert(MSType == SPIRV::MB_Annotations &&
666 "Only OpDecorate instructions can be duplicates");
667 // For FPFastMathMode decoration, we need to merge the flags of the
668 // duplicate decoration with the original one, so we need to find the
669 // original instruction that has the same signature. For the rest of
670 // instructions, we will simply skip the duplicate.
671 if (MI.getOperand(i: 1).getImm() != SPIRV::Decoration::FPFastMathMode)
672 return; // Skip duplicates of other decorations.
673
674 const SPIRV::InstrList &Decorations = MAI.MS[MSType];
675 for (const MachineInstr *OrigMI : Decorations) {
676 if (instrToSignature(MI: *OrigMI, MAI, UseDefReg: true) == MISign) {
677 assert(OrigMI->getNumOperands() == MI.getNumOperands() &&
678 "Original instruction must have the same number of operands");
679 assert(
680 OrigMI->getNumOperands() == 3 &&
681 "FPFastMathMode decoration must have 3 operands for OpDecorate");
682 unsigned OrigFlags = OrigMI->getOperand(i: 2).getImm();
683 unsigned NewFlags = MI.getOperand(i: 2).getImm();
684 if (OrigFlags == NewFlags)
685 return; // No need to merge, the flags are the same.
686
687 // Emit warning about possible conflict between flags.
688 unsigned FinalFlags = OrigFlags | NewFlags;
689 llvm::errs()
690 << "Warning: Conflicting FPFastMathMode decoration flags "
691 "in instruction: "
692 << *OrigMI << "Original flags: " << OrigFlags
693 << ", new flags: " << NewFlags
694 << ". They will be merged on a best effort basis, but not "
695 "validated. Final flags: "
696 << FinalFlags << "\n";
697 MachineInstr *OrigMINonConst = const_cast<MachineInstr *>(OrigMI);
698 MachineOperand &OrigFlagsOp = OrigMINonConst->getOperand(i: 2);
699 OrigFlagsOp = MachineOperand::CreateImm(Val: FinalFlags);
700 return; // Merge done, so we found a duplicate; don't add it to MAI.MS
701 }
702 }
703 assert(false && "No original instruction found for the duplicate "
704 "OpDecorate, but we found one in IS.");
705 }
706 return; // insert failed, so we found a duplicate; don't add it to MAI.MS
707 }
708 // No duplicates, so add it.
709 if (Append)
710 MAI.MS[MSType].push_back(Elt: &MI);
711 else
712 MAI.MS[MSType].insert(I: MAI.MS[MSType].begin(), Elt: &MI);
713}
714
715// Some global instructions make reference to function-local ID regs, so cannot
716// be correctly collected until these registers are globally numbered.
717void SPIRVModuleAnalysisImpl::processOtherInstrs(const Module &M) {
718 InstrTraces IS;
719 for (const Function &F : M) {
720 if (F.isDeclaration())
721 continue;
722 MachineFunction *MF = GetMF(F);
723 assert(MF);
724
725 for (MachineBasicBlock &MBB : *MF)
726 for (MachineInstr &MI : MBB) {
727 if (MI.isDebugValueLike()) {
728 MAI.DebugValueMIs.push_back(Elt: &MI);
729 continue;
730 }
731 if (MAI.getSkipEmission(MI: &MI))
732 continue;
733 const unsigned OpCode = MI.getOpcode();
734 if (OpCode == SPIRV::OpString) {
735 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_DebugStrings, IS);
736 } else if (OpCode == SPIRV::OpExtInst && MI.getOperand(i: 2).isImm() &&
737 MI.getOperand(i: 2).getImm() ==
738 SPIRV::InstructionSet::
739 NonSemantic_Shader_DebugInfo_100) {
740 // TODO: This branch is dead. SPIRVNonSemanticDebugHandler emits NSDI
741 // instructions directly as MCInsts at print time; no
742 // MachineInstructions with the NSDI ext set are created anymore.
743 // Remove this block and
744 // MB_NonSemanticGlobalDI once per-function NSDI emission is confirmed
745 // not to need MIR routing.
746 MachineOperand Ins = MI.getOperand(i: 3);
747 namespace NS = SPIRV::NonSemanticExtInst;
748 static constexpr int64_t GlobalNonSemanticDITy[] = {
749 NS::DebugSource, NS::DebugCompilationUnit, NS::DebugInfoNone,
750 NS::DebugTypeBasic, NS::DebugTypePointer};
751 bool IsGlobalDI = false;
752 for (unsigned Idx = 0; Idx < std::size(GlobalNonSemanticDITy); ++Idx)
753 IsGlobalDI |= Ins.getImm() == GlobalNonSemanticDITy[Idx];
754 if (IsGlobalDI)
755 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_NonSemanticGlobalDI, IS);
756 } else if (OpCode == SPIRV::OpName || OpCode == SPIRV::OpMemberName) {
757 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_DebugNames, IS);
758 } else if (OpCode == SPIRV::OpEntryPoint) {
759 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_EntryPoints, IS);
760 } else if (TII->isAliasingInstr(MI)) {
761 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_AliasingInsts, IS);
762 } else if (TII->isDecorationInstr(MI)) {
763 if (MI.getOpcode() == SPIRV::OpDecorate &&
764 MI.getOperand(i: 1).getImm() ==
765 static_cast<unsigned>(
766 SPIRV::Decoration::AuxDataInstructionMetadata)) {
767 MAI.setSkipEmission(&MI);
768 std::string Str = getStringImm(MI, StartIndex: 2);
769 using AMDMD = SPIRV::ModuleAnalysisInfo::AMDGPUAtomicMDKind;
770 auto MaybeKind =
771 StringSwitch<std::optional<AMDMD>>(Str)
772 .Case(S: "amdgpu.no.fine.grained.memory",
773 Value: AMDMD::NoFineGrainedMemory)
774 .Case(S: "amdgpu.no.remote.memory", Value: AMDMD::NoRemoteMemory)
775 .Case(S: "atomic.ignore.denormal.mode",
776 Value: AMDMD::IgnoreDenormalMode)
777 .Default(Value: std::nullopt);
778 if (MaybeKind)
779 MAI.InstrAuxDataRecords.push_back(
780 Elt: {.MF: MF, .TargetReg: MI.getOperand(i: 0).getReg(), .Kind: *MaybeKind});
781 continue;
782 }
783 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_Annotations, IS);
784 collectFuncNames(MI, F: &F);
785 } else if (TII->isConstantInstr(MI)) {
786 // Now OpSpecConstant*s are not in DT,
787 // but they need to be collected anyway.
788 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_TypeConstVars, IS);
789 } else if (OpCode == SPIRV::OpFunction) {
790 collectFuncNames(MI, F: &F);
791 } else if (OpCode == SPIRV::OpTypeForwardPointer) {
792 collectOtherInstr(MI, MAI, MSType: SPIRV::MB_TypeConstVars, IS, Append: false);
793 }
794 }
795 }
796 // Selection order can place a scope/list ahead of a domain/scope it
797 // references. The dependency meanwhile is domain -> scope -> list, so sort
798 // the def before its uses.
799 auto AliasingTier = [](const MachineInstr *MI) {
800 switch (MI->getOpcode()) {
801 case SPIRV::OpAliasDomainDeclINTEL:
802 return 0;
803 case SPIRV::OpAliasScopeDeclINTEL:
804 return 1;
805 case SPIRV::OpAliasScopeListDeclINTEL:
806 return 2;
807 default:
808 llvm_unreachable("unexpected aliasing instruction");
809 }
810 };
811 stable_sort(Range&: MAI.MS[SPIRV::MB_AliasingInsts],
812 C: [&](const MachineInstr *LHS, const MachineInstr *RHS) {
813 return AliasingTier(LHS) < AliasingTier(RHS);
814 });
815}
816
817// Number registers in all functions globally from 0 onwards and store
818// the result in global register alias table. Some registers are already
819// numbered.
820void SPIRVModuleAnalysisImpl::numberRegistersGlobally(const Module &M) {
821 for (const Function &F : M) {
822 if (F.isDeclaration())
823 continue;
824 MachineFunction *MF = GetMF(F);
825 assert(MF);
826 for (MachineBasicBlock &MBB : *MF) {
827 for (MachineInstr &MI : MBB) {
828 for (MachineOperand &Op : MI.operands()) {
829 if (!Op.isReg())
830 continue;
831 Register Reg = Op.getReg();
832 if (MAI.hasRegisterAlias(MF, Reg))
833 continue;
834 MCRegister NewReg = MAI.getNextIDRegister();
835 MAI.setRegisterAlias(MF, Reg, AliasReg: NewReg);
836 }
837 if (MI.getOpcode() != SPIRV::OpExtInst)
838 continue;
839 auto Set = MI.getOperand(i: 2).getImm();
840 auto [It, Inserted] = MAI.ExtInstSetMap.try_emplace(Key: Set);
841 if (Inserted)
842 It->second = MAI.getNextIDRegister();
843 }
844 }
845 }
846}
847
848static void addQuadControlRequirements(SPIRV::RequirementHandler &Reqs,
849 const SPIRVSubtarget &ST,
850 StringRef Feature) {
851 if (!ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 3)))
852 report_fatal_error(reason: Twine(Feature) + " requires SPIR-V 1.3", gen_crash_diag: false);
853 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_quad_control))
854 report_fatal_error(reason: Twine(Feature) + " requires the following SPIR-V "
855 "extension: SPV_KHR_quad_control",
856 gen_crash_diag: false);
857 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_maximal_reconvergence))
858 report_fatal_error(reason: "SPV_KHR_quad_control requires the following SPIR-V "
859 "extension: SPV_KHR_maximal_reconvergence",
860 gen_crash_diag: false);
861 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_quad_control);
862 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_maximal_reconvergence);
863}
864
865// RequirementHandler implementations.
866void SPIRV::RequirementHandler::getAndAddRequirements(
867 SPIRV::OperandCategory::OperandCategory Category, uint32_t i,
868 const SPIRVSubtarget &ST) {
869 if (Category == SPIRV::OperandCategory::ExecutionModeOperand &&
870 (i == SPIRV::ExecutionMode::QuadDerivativesKHR ||
871 i == SPIRV::ExecutionMode::RequireFullQuadsKHR))
872 addQuadControlRequirements(Reqs&: *this, ST,
873 Feature: getSymbolicOperandMnemonic(Category, Value: i));
874 if (Category == SPIRV::OperandCategory::ExecutionModeOperand &&
875 (i == SPIRV::ExecutionMode::DerivativeGroupQuadsKHR ||
876 i == SPIRV::ExecutionMode::DerivativeGroupLinearKHR) &&
877 !ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_compute_shader_derivatives))
878 report_fatal_error(reason: Twine(getSymbolicOperandMnemonic(Category, Value: i)) +
879 " requires the following SPIR-V extension: "
880 "SPV_KHR_compute_shader_derivatives",
881 gen_crash_diag: false);
882 addRequirements(Req: getSymbolicOperandRequirements(Category, i, ST, Reqs&: *this));
883}
884
885void SPIRV::RequirementHandler::recursiveAddCapabilities(
886 const CapabilityList &ToPrune) {
887 for (const auto &Cap : ToPrune) {
888 AllCaps.insert(V: Cap);
889 CapabilityList ImplicitDecls =
890 getSymbolicOperandCapabilities(Category: OperandCategory::CapabilityOperand, Value: Cap);
891 recursiveAddCapabilities(ToPrune: ImplicitDecls);
892 }
893}
894
895void SPIRV::RequirementHandler::addCapabilities(const CapabilityList &ToAdd) {
896 for (const auto &Cap : ToAdd) {
897 bool IsNewlyInserted = AllCaps.insert(V: Cap).second;
898 if (!IsNewlyInserted) // Don't re-add if it's already been declared.
899 continue;
900 CapabilityList ImplicitDecls =
901 getSymbolicOperandCapabilities(Category: OperandCategory::CapabilityOperand, Value: Cap);
902 recursiveAddCapabilities(ToPrune: ImplicitDecls);
903 MinimalCaps.push_back(Elt: Cap);
904 }
905}
906
907void SPIRV::RequirementHandler::addRequirements(
908 const SPIRV::Requirements &Req) {
909 if (!Req.IsSatisfiable)
910 report_fatal_error(reason: "Adding SPIR-V requirements this target can't satisfy.");
911
912 if (Req.Cap.has_value())
913 addCapabilities(ToAdd: {Req.Cap.value()});
914
915 addExtensions(ToAdd: Req.Exts);
916
917 if (!Req.MinVer.empty()) {
918 if (!MaxVersion.empty() && Req.MinVer > MaxVersion) {
919 LLVM_DEBUG(dbgs() << "Conflicting version requirements: >= " << Req.MinVer
920 << " and <= " << MaxVersion << "\n");
921 report_fatal_error(reason: "Adding SPIR-V requirements that can't be satisfied.");
922 }
923
924 if (MinVersion.empty() || Req.MinVer > MinVersion)
925 MinVersion = Req.MinVer;
926 }
927
928 if (!Req.MaxVer.empty()) {
929 if (!MinVersion.empty() && Req.MaxVer < MinVersion) {
930 LLVM_DEBUG(dbgs() << "Conflicting version requirements: <= " << Req.MaxVer
931 << " and >= " << MinVersion << "\n");
932 report_fatal_error(reason: "Adding SPIR-V requirements that can't be satisfied.");
933 }
934
935 if (MaxVersion.empty() || Req.MaxVer < MaxVersion)
936 MaxVersion = Req.MaxVer;
937 }
938}
939
940void SPIRV::RequirementHandler::checkSatisfiable(
941 const SPIRVSubtarget &ST) const {
942 // Report as many errors as possible before aborting the compilation.
943 bool IsSatisfiable = true;
944 auto TargetVer = ST.getSPIRVVersion();
945
946 if (!MaxVersion.empty() && !TargetVer.empty() && MaxVersion < TargetVer) {
947 LLVM_DEBUG(
948 dbgs() << "Target SPIR-V version too high for required features\n"
949 << "Required max version: " << MaxVersion << " target version "
950 << TargetVer << "\n");
951 IsSatisfiable = false;
952 }
953
954 if (!MinVersion.empty() && !TargetVer.empty() && MinVersion > TargetVer) {
955 LLVM_DEBUG(dbgs() << "Target SPIR-V version too low for required features\n"
956 << "Required min version: " << MinVersion
957 << " target version " << TargetVer << "\n");
958 IsSatisfiable = false;
959 }
960
961 if (!MinVersion.empty() && !MaxVersion.empty() && MinVersion > MaxVersion) {
962 LLVM_DEBUG(
963 dbgs()
964 << "Version is too low for some features and too high for others.\n"
965 << "Required SPIR-V min version: " << MinVersion
966 << " required SPIR-V max version " << MaxVersion << "\n");
967 IsSatisfiable = false;
968 }
969
970 for (auto Cap : MinimalCaps) {
971 if (AvailableCaps.contains(V: Cap) && !shouldAvoidCapability(ST, Cap))
972 continue;
973 LLVM_DEBUG(dbgs() << "Capability not supported: "
974 << getSymbolicOperandMnemonic(
975 OperandCategory::CapabilityOperand, Cap)
976 << "\n");
977 IsSatisfiable = false;
978 }
979
980 for (auto Ext : AllExtensions) {
981 if (ST.canUseExtension(E: Ext))
982 continue;
983 LLVM_DEBUG(dbgs() << "Extension not supported: "
984 << getSymbolicOperandMnemonic(
985 OperandCategory::ExtensionOperand, Ext)
986 << "\n");
987 IsSatisfiable = false;
988 }
989
990 if (!IsSatisfiable)
991 report_fatal_error(reason: "Unable to meet SPIR-V requirements for this target.");
992}
993
994// Add the given capabilities and all their implicitly defined capabilities too.
995void SPIRV::RequirementHandler::addAvailableCaps(const CapabilityList &ToAdd) {
996 for (const auto Cap : ToAdd)
997 if (AvailableCaps.insert(V: Cap).second)
998 addAvailableCaps(ToAdd: getSymbolicOperandCapabilities(
999 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap));
1000}
1001
1002void SPIRV::RequirementHandler::removeCapabilityIf(
1003 const Capability::Capability ToRemove,
1004 const Capability::Capability IfPresent) {
1005 if (AllCaps.contains(V: IfPresent)) {
1006 AllCaps.erase(V: ToRemove);
1007 llvm::erase(C&: MinimalCaps, V: ToRemove);
1008 }
1009}
1010
1011namespace llvm {
1012namespace SPIRV {
1013void RequirementHandler::initAvailableCapabilities(const SPIRVSubtarget &ST) {
1014 // Provided by both all supported Vulkan versions and OpenCl.
1015 addAvailableCaps(ToAdd: {Capability::Shader, Capability::Linkage, Capability::Int8,
1016 Capability::Int16});
1017
1018 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 3)))
1019 addAvailableCaps(ToAdd: {Capability::GroupNonUniform,
1020 Capability::GroupNonUniformVote,
1021 Capability::GroupNonUniformArithmetic,
1022 Capability::GroupNonUniformBallot,
1023 Capability::GroupNonUniformClustered,
1024 Capability::GroupNonUniformShuffle,
1025 Capability::GroupNonUniformShuffleRelative,
1026 Capability::GroupNonUniformQuad});
1027
1028 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
1029 addAvailableCaps(ToAdd: {Capability::DotProduct, Capability::DotProductInputAll,
1030 Capability::DotProductInput4x8Bit,
1031 Capability::DotProductInput4x8BitPacked,
1032 Capability::DemoteToHelperInvocation});
1033
1034 // Add capabilities enabled by extensions.
1035 for (auto Extension : ST.getAllAvailableExtensions()) {
1036 CapabilityList EnabledCapabilities =
1037 getCapabilitiesEnabledByExtension(Extension);
1038 addAvailableCaps(ToAdd: EnabledCapabilities);
1039 }
1040
1041 // EXT and NV use the same capability.
1042 if (ST.canUseExtension(E: Extension::SPV_EXT_shader_subgroup_partitioned))
1043 addAvailableCaps(ToAdd: {Capability::GroupNonUniformPartitionedNV});
1044
1045 if (!ST.isShader()) {
1046 initAvailableCapabilitiesForOpenCL(ST);
1047 return;
1048 }
1049
1050 if (ST.isShader()) {
1051 initAvailableCapabilitiesForVulkan(ST);
1052 return;
1053 }
1054
1055 report_fatal_error(reason: "Unimplemented environment for SPIR-V generation.");
1056}
1057
1058void RequirementHandler::initAvailableCapabilitiesForOpenCL(
1059 const SPIRVSubtarget &ST) {
1060 // Add the min requirements for different OpenCL and SPIR-V versions.
1061 addAvailableCaps(ToAdd: {Capability::Addresses, Capability::Float16Buffer,
1062 Capability::Kernel, Capability::Vector16,
1063 Capability::Groups, Capability::GenericPointer,
1064 Capability::StorageImageWriteWithoutFormat,
1065 Capability::StorageImageReadWithoutFormat});
1066 if (ST.hasOpenCLFullProfile())
1067 addAvailableCaps(ToAdd: {Capability::Int64, Capability::Int64Atomics});
1068 if (ST.hasOpenCLImageSupport()) {
1069 addAvailableCaps(ToAdd: {Capability::ImageBasic, Capability::LiteralSampler,
1070 Capability::Image1D, Capability::SampledBuffer,
1071 Capability::ImageBuffer});
1072 if (ST.isAtLeastOpenCLVer(VerToCompareTo: VersionTuple(2, 0)))
1073 addAvailableCaps(ToAdd: {Capability::ImageReadWrite});
1074 }
1075 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 1)) &&
1076 ST.isAtLeastOpenCLVer(VerToCompareTo: VersionTuple(2, 2)))
1077 addAvailableCaps(ToAdd: {Capability::SubgroupDispatch, Capability::PipeStorage});
1078 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 4)))
1079 addAvailableCaps(ToAdd: {Capability::DenormPreserve, Capability::DenormFlushToZero,
1080 Capability::SignedZeroInfNanPreserve,
1081 Capability::RoundingModeRTE,
1082 Capability::RoundingModeRTZ});
1083 // TODO: verify if this needs some checks.
1084 addAvailableCaps(ToAdd: {Capability::Float16, Capability::Float64});
1085
1086 // TODO: add OpenCL extensions.
1087}
1088
1089void RequirementHandler::initAvailableCapabilitiesForVulkan(
1090 const SPIRVSubtarget &ST) {
1091
1092 // Core in Vulkan 1.1 and earlier.
1093 addAvailableCaps(ToAdd: {Capability::Int64,
1094 Capability::Geometry,
1095 Capability::Float16,
1096 Capability::Float64,
1097 Capability::GroupNonUniform,
1098 Capability::Image1D,
1099 Capability::SampledBuffer,
1100 Capability::ImageBuffer,
1101 Capability::UniformBufferArrayDynamicIndexing,
1102 Capability::SampledImageArrayDynamicIndexing,
1103 Capability::StorageBufferArrayDynamicIndexing,
1104 Capability::StorageImageArrayDynamicIndexing,
1105 Capability::DerivativeControl,
1106 Capability::SampleRateShading,
1107 Capability::MinLod,
1108 Capability::ImageQuery,
1109 Capability::ImageGatherExtended,
1110 Capability::Addresses,
1111 Capability::VulkanMemoryModelKHR,
1112 Capability::StorageImageExtendedFormats,
1113 Capability::StorageImageMultisample,
1114 Capability::ImageMSArray});
1115
1116 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 3)) ||
1117 ST.canUseExtension(E: Extension::SPV_KHR_variable_pointers))
1118 addAvailableCaps(ToAdd: {Capability::VariablePointersStorageBuffer,
1119 Capability::VariablePointers});
1120
1121 // Became core in Vulkan 1.1.
1122 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 3)))
1123 addAvailableCaps(ToAdd: {Capability::DrawParameters, Capability::MultiView});
1124
1125 // Became core in Vulkan 1.2
1126 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 5))) {
1127 addAvailableCaps(
1128 ToAdd: {Capability::Int64Atomics, Capability::ShaderNonUniformEXT,
1129 Capability::RuntimeDescriptorArrayEXT,
1130 Capability::InputAttachmentArrayDynamicIndexingEXT,
1131 Capability::UniformTexelBufferArrayDynamicIndexingEXT,
1132 Capability::StorageTexelBufferArrayDynamicIndexingEXT,
1133 Capability::UniformBufferArrayNonUniformIndexingEXT,
1134 Capability::SampledImageArrayNonUniformIndexingEXT,
1135 Capability::StorageBufferArrayNonUniformIndexingEXT,
1136 Capability::StorageImageArrayNonUniformIndexingEXT,
1137 Capability::InputAttachmentArrayNonUniformIndexingEXT,
1138 Capability::UniformTexelBufferArrayNonUniformIndexingEXT,
1139 Capability::StorageTexelBufferArrayNonUniformIndexingEXT});
1140 }
1141
1142 // Became core in Vulkan 1.3
1143 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
1144 addAvailableCaps(ToAdd: {Capability::StorageImageWriteWithoutFormat,
1145 Capability::StorageImageReadWithoutFormat});
1146}
1147
1148} // namespace SPIRV
1149} // namespace llvm
1150
1151// Add a capability that became core in SPIR-V 1.3. Earlier versions require
1152// the extension that introduced it.
1153static void addCoreInSPIRV13Capability(SPIRV::RequirementHandler &Reqs,
1154 const SPIRVSubtarget &ST,
1155 SPIRV::Capability::Capability Cap,
1156 SPIRV::Extension::Extension Ext) {
1157 Reqs.addCapability(ToAdd: Cap);
1158 if (ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 3)))
1159 return;
1160 if (!ST.canUseExtension(E: Ext))
1161 report_fatal_error(
1162 reason: Twine("capability ") +
1163 getSymbolicOperandMnemonic(
1164 Category: SPIRV::OperandCategory::CapabilityOperand, Value: Cap) +
1165 " requires SPIR-V 1.3 or the following SPIR-V extension: " +
1166 getSymbolicOperandMnemonic(Category: SPIRV::OperandCategory::ExtensionOperand,
1167 Value: Ext),
1168 gen_crash_diag: false);
1169 Reqs.addExtension(ToAdd: Ext);
1170}
1171
1172// Add the required capabilities from a decoration instruction (including
1173// BuiltIns).
1174static void addOpDecorateReqs(const MachineInstr &MI, unsigned DecIndex,
1175 SPIRV::RequirementHandler &Reqs,
1176 const SPIRVSubtarget &ST) {
1177 int64_t DecOp = MI.getOperand(i: DecIndex).getImm();
1178 auto Dec = static_cast<SPIRV::Decoration::Decoration>(DecOp);
1179 Reqs.addRequirements(Req: getSymbolicOperandRequirements(
1180 Category: SPIRV::OperandCategory::DecorationOperand, i: Dec, ST, Reqs));
1181
1182 if (Dec == SPIRV::Decoration::BuiltIn) {
1183 int64_t BuiltInOp = MI.getOperand(i: DecIndex + 1).getImm();
1184 auto BuiltIn = static_cast<SPIRV::BuiltIn::BuiltIn>(BuiltInOp);
1185 if ((BuiltIn == SPIRV::BuiltIn::BaryCoordKHR ||
1186 BuiltIn == SPIRV::BuiltIn::BaryCoordNoPerspKHR) &&
1187 !ST.canUseExtension(
1188 E: SPIRV::Extension::SPV_KHR_fragment_shader_barycentric))
1189 report_fatal_error(
1190 reason: "Barycentric built-ins require the following SPIR-V extension: "
1191 "SPV_KHR_fragment_shader_barycentric",
1192 gen_crash_diag: false);
1193 switch (BuiltIn) {
1194 case SPIRV::BuiltIn::BaseVertex:
1195 case SPIRV::BuiltIn::BaseInstance:
1196 addCoreInSPIRV13Capability(
1197 Reqs, ST, Cap: SPIRV::Capability::DrawParameters,
1198 Ext: SPIRV::Extension::SPV_KHR_shader_draw_parameters);
1199 break;
1200 case SPIRV::BuiltIn::ViewIndex:
1201 addCoreInSPIRV13Capability(Reqs, ST, Cap: SPIRV::Capability::MultiView,
1202 Ext: SPIRV::Extension::SPV_KHR_multiview);
1203 break;
1204 default:
1205 Reqs.addRequirements(Req: getSymbolicOperandRequirements(
1206 Category: SPIRV::OperandCategory::BuiltInOperand, i: BuiltIn, ST, Reqs));
1207 break;
1208 }
1209 } else if (Dec == SPIRV::Decoration::LinkageAttributes) {
1210 int64_t LinkageOp = MI.getOperand(i: MI.getNumOperands() - 1).getImm();
1211 SPIRV::LinkageType::LinkageType LnkType =
1212 static_cast<SPIRV::LinkageType::LinkageType>(LinkageOp);
1213 if (LnkType == SPIRV::LinkageType::LinkOnceODR)
1214 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_linkonce_odr);
1215 else if (LnkType == SPIRV::LinkageType::WeakAMD) {
1216 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_AMD_weak_linkage);
1217 Reqs.addCapability(ToAdd: SPIRV::Capability::WeakLinkageAMD);
1218 }
1219 } else if (Dec == SPIRV::Decoration::CacheControlLoadINTEL ||
1220 Dec == SPIRV::Decoration::CacheControlStoreINTEL) {
1221 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_cache_controls);
1222 } else if (Dec == SPIRV::Decoration::HostAccessINTEL) {
1223 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_global_variable_host_access);
1224 } else if (Dec == SPIRV::Decoration::InitModeINTEL ||
1225 Dec == SPIRV::Decoration::ImplementInRegisterMapINTEL) {
1226 Reqs.addExtension(
1227 ToAdd: SPIRV::Extension::SPV_INTEL_global_variable_fpga_decorations);
1228 } else if (Dec == SPIRV::Decoration::NonUniformEXT) {
1229 Reqs.addRequirements(Req: SPIRV::Capability::ShaderNonUniformEXT);
1230 } else if (Dec == SPIRV::Decoration::FPMaxErrorDecorationINTEL) {
1231 Reqs.addRequirements(Req: SPIRV::Capability::FPMaxErrorINTEL);
1232 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_fp_max_error);
1233 } else if (Dec == SPIRV::Decoration::FPFastMathMode) {
1234 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2)) {
1235 Reqs.addRequirements(Req: SPIRV::Capability::FloatControls2);
1236 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls2);
1237 }
1238 }
1239}
1240
1241// Add requirements for image handling.
1242static void addOpTypeImageReqs(const MachineInstr &MI,
1243 SPIRV::RequirementHandler &Reqs,
1244 const SPIRVSubtarget &ST) {
1245 assert(MI.getNumOperands() >= 8 && "Insufficient operands for OpTypeImage");
1246 // The operand indices used here are based on the OpTypeImage layout, which
1247 // the MachineInstr follows as well.
1248 int64_t ImgFormatOp = MI.getOperand(i: 7).getImm();
1249 auto ImgFormat = static_cast<SPIRV::ImageFormat::ImageFormat>(ImgFormatOp);
1250 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ImageFormatOperand,
1251 i: ImgFormat, ST);
1252
1253 bool IsArrayed = MI.getOperand(i: 4).getImm() == 1;
1254 bool IsMultisampled = MI.getOperand(i: 5).getImm() == 1;
1255 bool NoSampler = MI.getOperand(i: 6).getImm() == 2;
1256 // Add dimension requirements.
1257 assert(MI.getOperand(2).isImm());
1258 switch (MI.getOperand(i: 2).getImm()) {
1259 case SPIRV::Dim::DIM_1D:
1260 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::Image1D
1261 : SPIRV::Capability::Sampled1D);
1262 break;
1263 case SPIRV::Dim::DIM_2D:
1264 if (IsMultisampled && NoSampler)
1265 Reqs.addRequirements(Req: SPIRV::Capability::StorageImageMultisample);
1266 if (IsMultisampled && IsArrayed)
1267 Reqs.addRequirements(Req: SPIRV::Capability::ImageMSArray);
1268 break;
1269 case SPIRV::Dim::DIM_3D:
1270 break;
1271 case SPIRV::Dim::DIM_Cube:
1272 Reqs.addRequirements(Req: SPIRV::Capability::Shader);
1273 if (IsArrayed)
1274 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageCubeArray
1275 : SPIRV::Capability::SampledCubeArray);
1276 break;
1277 case SPIRV::Dim::DIM_Rect:
1278 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageRect
1279 : SPIRV::Capability::SampledRect);
1280 break;
1281 case SPIRV::Dim::DIM_Buffer:
1282 Reqs.addRequirements(Req: NoSampler ? SPIRV::Capability::ImageBuffer
1283 : SPIRV::Capability::SampledBuffer);
1284 break;
1285 case SPIRV::Dim::DIM_SubpassData:
1286 Reqs.addRequirements(Req: SPIRV::Capability::InputAttachment);
1287 break;
1288 }
1289
1290 // Has optional access qualifier.
1291 if (!ST.isShader()) {
1292 if (MI.getNumOperands() > 8 &&
1293 MI.getOperand(i: 8).getImm() == SPIRV::AccessQualifier::ReadWrite)
1294 Reqs.addRequirements(Req: SPIRV::Capability::ImageReadWrite);
1295 else
1296 Reqs.addRequirements(Req: SPIRV::Capability::ImageBasic);
1297 }
1298}
1299
1300static bool isBFloat16Type(SPIRVTypeInst TypeDef) {
1301 return TypeDef && TypeDef->getNumOperands() == 3 &&
1302 TypeDef->getOpcode() == SPIRV::OpTypeFloat &&
1303 TypeDef->getOperand(i: 1).getImm() == 16 &&
1304 TypeDef->getOperand(i: 2).getImm() == SPIRV::FPEncoding::BFloat16KHR;
1305}
1306
1307// Adds the requirements for bfloat16 arithmetic. Prefers
1308// SPV_EXT_bfloat16_arithmetic and falls back to the legacy
1309// SPV_INTEL_bfloat16_arithmetic which enables the same capability.
1310static bool addBFloat16ArithmeticRequirements(SPIRV::RequirementHandler &Reqs,
1311 const SPIRVSubtarget &ST) {
1312 if (ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_bfloat16_arithmetic)) {
1313 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_bfloat16_arithmetic);
1314 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ArithmeticEXT);
1315 return true;
1316 }
1317 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic)) {
1318 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);
1319 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ArithmeticINTEL);
1320 return true;
1321 }
1322 return false;
1323}
1324
1325// Registers created after isel (e.g. by MachineCSE) may lack registry entries.
1326// Fall back to the defining instruction's result type.
1327static SPIRVTypeInst getRegType(Register Reg, const MachineFunction &MF,
1328 const SPIRVGlobalRegistry *GR) {
1329 if (SPIRVTypeInst Ty = GR->getSPIRVTypeForVReg(VReg: Reg, MF: &MF))
1330 return Ty;
1331 const MachineRegisterInfo &MRI = MF.getRegInfo();
1332 return MRI.getVRegDef(Reg: MRI.getVRegDef(Reg)->getOperand(i: 1).getReg());
1333}
1334
1335// Add requirements for handling atomic float instructions
1336#define ATOM_FLT_REQ_EXT_MSG(ExtName) \
1337 "The atomic float instruction requires the following SPIR-V " \
1338 "extension: SPV_EXT_shader_atomic_float" ExtName
1339static void AddAtomicVectorFloatRequirements(const MachineInstr &MI,
1340 SPIRV::RequirementHandler &Reqs,
1341 const SPIRVSubtarget &ST) {
1342 SPIRVTypeInst VecTypeDef =
1343 MI.getMF()->getRegInfo().getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1344
1345 const unsigned Rank = VecTypeDef->getOperand(i: 2).getImm();
1346 if (Rank != 2 && Rank != 4)
1347 reportFatalUsageError(reason: "Result type of an atomic vector float instruction "
1348 "must be a 2-component or 4 component vector");
1349
1350 SPIRVTypeInst EltTypeDef =
1351 MI.getMF()->getRegInfo().getVRegDef(Reg: VecTypeDef->getOperand(i: 1).getReg());
1352
1353 if (EltTypeDef->getOpcode() != SPIRV::OpTypeFloat ||
1354 EltTypeDef->getOperand(i: 1).getImm() != 16)
1355 reportFatalUsageError(
1356 reason: "The element type for the result type of an atomic vector float "
1357 "instruction must be a 16-bit floating-point scalar");
1358
1359 // The extension is defined for fp16, but the AMD target lets a bf16 vector
1360 // use the same instruction so it can lower to a packed bf16 atomic.
1361 if (isBFloat16Type(TypeDef: EltTypeDef) &&
1362 ST.getTargetTriple().getVendor() != Triple::AMD)
1363 reportFatalUsageError(
1364 reason: "The element type for the result type of an atomic vector float "
1365 "instruction cannot be a bfloat16 scalar");
1366 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_NV_shader_atomic_fp16_vector))
1367 reportFatalUsageError(
1368 reason: "The atomic float16 vector instruction requires the following SPIR-V "
1369 "extension: SPV_NV_shader_atomic_fp16_vector");
1370
1371 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_NV_shader_atomic_fp16_vector);
1372 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16VectorNV);
1373}
1374
1375static void AddAtomicFloatRequirements(const MachineInstr &MI,
1376 SPIRV::RequirementHandler &Reqs,
1377 const SPIRVSubtarget &ST) {
1378 assert(MI.getOperand(1).isReg() &&
1379 "Expect register operand in atomic float instruction");
1380 Register TypeReg = MI.getOperand(i: 1).getReg();
1381 SPIRVTypeInst TypeDef = MI.getMF()->getRegInfo().getVRegDef(Reg: TypeReg);
1382
1383 if (isVectorType(SPVTy: TypeDef))
1384 return AddAtomicVectorFloatRequirements(MI, Reqs, ST);
1385
1386 if (TypeDef->getOpcode() != SPIRV::OpTypeFloat)
1387 report_fatal_error(reason: "Result type of an atomic float instruction must be a "
1388 "floating-point type scalar");
1389
1390 unsigned BitWidth = TypeDef->getOperand(i: 1).getImm();
1391 unsigned Op = MI.getOpcode();
1392 if (Op == SPIRV::OpAtomicFAddEXT) {
1393 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_shader_atomic_float_add))
1394 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("_add"), gen_crash_diag: false);
1395 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float_add);
1396 switch (BitWidth) {
1397 case 16:
1398 if (isBFloat16Type(TypeDef)) {
1399 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1400 report_fatal_error(
1401 reason: "The atomic bfloat16 instruction requires the following SPIR-V "
1402 "extension: SPV_INTEL_16bit_atomics",
1403 gen_crash_diag: false);
1404 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1405 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicBFloat16AddINTEL);
1406 } else {
1407 if (!ST.canUseExtension(
1408 E: SPIRV::Extension::SPV_EXT_shader_atomic_float16_add))
1409 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("16_add"), gen_crash_diag: false);
1410 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float16_add);
1411 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16AddEXT);
1412 }
1413 break;
1414 case 32:
1415 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat32AddEXT);
1416 break;
1417 case 64:
1418 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat64AddEXT);
1419 break;
1420 default:
1421 report_fatal_error(
1422 reason: "Unexpected floating-point type width in atomic float instruction");
1423 }
1424 } else {
1425 if (!ST.canUseExtension(
1426 E: SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max))
1427 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("_min_max"), gen_crash_diag: false);
1428 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max);
1429 switch (BitWidth) {
1430 case 16:
1431 if (isBFloat16Type(TypeDef)) {
1432 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1433 report_fatal_error(
1434 reason: "The atomic bfloat16 instruction requires the following SPIR-V "
1435 "extension: SPV_INTEL_16bit_atomics",
1436 gen_crash_diag: false);
1437 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1438 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicBFloat16MinMaxINTEL);
1439 } else {
1440 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat16MinMaxEXT);
1441 }
1442 break;
1443 case 32:
1444 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat32MinMaxEXT);
1445 break;
1446 case 64:
1447 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicFloat64MinMaxEXT);
1448 break;
1449 default:
1450 report_fatal_error(
1451 reason: "Unexpected floating-point type width in atomic float instruction");
1452 }
1453 }
1454}
1455
1456bool isUniformTexelBuffer(MachineInstr *ImageInst) {
1457 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1458 return false;
1459 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1460 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1461 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 1;
1462}
1463
1464bool isStorageTexelBuffer(MachineInstr *ImageInst) {
1465 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1466 return false;
1467 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1468 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1469 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 2;
1470}
1471
1472bool isSampledImage(MachineInstr *ImageInst) {
1473 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1474 return false;
1475 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1476 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1477 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 1;
1478}
1479
1480bool isInputAttachment(MachineInstr *ImageInst) {
1481 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1482 return false;
1483 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1484 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1485 return Dim == SPIRV::Dim::DIM_SubpassData && Sampled == 2;
1486}
1487
1488bool isStorageImage(MachineInstr *ImageInst) {
1489 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1490 return false;
1491 uint32_t Dim = ImageInst->getOperand(i: 2).getImm();
1492 uint32_t Sampled = ImageInst->getOperand(i: 6).getImm();
1493 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 2;
1494}
1495
1496bool isCombinedImageSampler(MachineInstr *SampledImageInst) {
1497 if (SampledImageInst->getOpcode() != SPIRV::OpTypeSampledImage)
1498 return false;
1499
1500 const MachineRegisterInfo &MRI = SampledImageInst->getMF()->getRegInfo();
1501 Register ImageReg = SampledImageInst->getOperand(i: 1).getReg();
1502 auto *ImageInst = MRI.getUniqueVRegDef(Reg: ImageReg);
1503 return isSampledImage(ImageInst);
1504}
1505
1506bool hasNonUniformDecoration(Register Reg, const MachineRegisterInfo &MRI) {
1507 for (const auto &MI : MRI.reg_instructions(Reg)) {
1508 if (MI.getOpcode() != SPIRV::OpDecorate)
1509 continue;
1510
1511 uint32_t Dec = MI.getOperand(i: 1).getImm();
1512 if (Dec == SPIRV::Decoration::NonUniformEXT)
1513 return true;
1514 }
1515 return false;
1516}
1517
1518void addOpAccessChainReqs(const MachineInstr &Instr,
1519 SPIRV::RequirementHandler &Handler,
1520 const SPIRVSubtarget &Subtarget) {
1521 const MachineRegisterInfo &MRI = Instr.getMF()->getRegInfo();
1522 // Get the result type. If it is an image type, then the shader uses
1523 // descriptor indexing. The appropriate capabilities will be added based
1524 // on the specifics of the image.
1525 Register ResTypeReg = Instr.getOperand(i: 1).getReg();
1526 MachineInstr *ResTypeInst = MRI.getUniqueVRegDef(Reg: ResTypeReg);
1527
1528 assert(ResTypeInst->getOpcode() == SPIRV::OpTypePointer);
1529 uint32_t StorageClass = ResTypeInst->getOperand(i: 1).getImm();
1530 if (StorageClass != SPIRV::StorageClass::StorageClass::UniformConstant &&
1531 StorageClass != SPIRV::StorageClass::StorageClass::Uniform &&
1532 StorageClass != SPIRV::StorageClass::StorageClass::StorageBuffer) {
1533 return;
1534 }
1535
1536 bool IsNonUniform =
1537 hasNonUniformDecoration(Reg: Instr.getOperand(i: 0).getReg(), MRI);
1538
1539 auto FirstIndexReg = Instr.getOperand(i: 3).getReg();
1540 bool FirstIndexIsConstant =
1541 Subtarget.getInstrInfo()->isConstantInstr(MI: *MRI.getVRegDef(Reg: FirstIndexReg));
1542
1543 if (StorageClass == SPIRV::StorageClass::StorageClass::StorageBuffer) {
1544 if (IsNonUniform)
1545 Handler.addRequirements(
1546 Req: SPIRV::Capability::StorageBufferArrayNonUniformIndexingEXT);
1547 else if (!FirstIndexIsConstant)
1548 Handler.addRequirements(
1549 Req: SPIRV::Capability::StorageBufferArrayDynamicIndexing);
1550 return;
1551 }
1552
1553 Register PointeeTypeReg = ResTypeInst->getOperand(i: 2).getReg();
1554 MachineInstr *PointeeType = MRI.getUniqueVRegDef(Reg: PointeeTypeReg);
1555 if (PointeeType->getOpcode() != SPIRV::OpTypeImage &&
1556 PointeeType->getOpcode() != SPIRV::OpTypeSampledImage &&
1557 PointeeType->getOpcode() != SPIRV::OpTypeSampler) {
1558 return;
1559 }
1560
1561 if (isUniformTexelBuffer(ImageInst: PointeeType)) {
1562 if (IsNonUniform)
1563 Handler.addRequirements(
1564 Req: SPIRV::Capability::UniformTexelBufferArrayNonUniformIndexingEXT);
1565 else if (!FirstIndexIsConstant)
1566 Handler.addRequirements(
1567 Req: SPIRV::Capability::UniformTexelBufferArrayDynamicIndexingEXT);
1568 } else if (isInputAttachment(ImageInst: PointeeType)) {
1569 if (IsNonUniform)
1570 Handler.addRequirements(
1571 Req: SPIRV::Capability::InputAttachmentArrayNonUniformIndexingEXT);
1572 else if (!FirstIndexIsConstant)
1573 Handler.addRequirements(
1574 Req: SPIRV::Capability::InputAttachmentArrayDynamicIndexingEXT);
1575 } else if (isStorageTexelBuffer(ImageInst: PointeeType)) {
1576 if (IsNonUniform)
1577 Handler.addRequirements(
1578 Req: SPIRV::Capability::StorageTexelBufferArrayNonUniformIndexingEXT);
1579 else if (!FirstIndexIsConstant)
1580 Handler.addRequirements(
1581 Req: SPIRV::Capability::StorageTexelBufferArrayDynamicIndexingEXT);
1582 } else if (isSampledImage(ImageInst: PointeeType) ||
1583 isCombinedImageSampler(SampledImageInst: PointeeType) ||
1584 PointeeType->getOpcode() == SPIRV::OpTypeSampler) {
1585 if (IsNonUniform)
1586 Handler.addRequirements(
1587 Req: SPIRV::Capability::SampledImageArrayNonUniformIndexingEXT);
1588 else if (!FirstIndexIsConstant)
1589 Handler.addRequirements(
1590 Req: SPIRV::Capability::SampledImageArrayDynamicIndexing);
1591 } else if (isStorageImage(ImageInst: PointeeType)) {
1592 if (IsNonUniform)
1593 Handler.addRequirements(
1594 Req: SPIRV::Capability::StorageImageArrayNonUniformIndexingEXT);
1595 else if (!FirstIndexIsConstant)
1596 Handler.addRequirements(
1597 Req: SPIRV::Capability::StorageImageArrayDynamicIndexing);
1598 }
1599}
1600
1601static bool isImageTypeWithUnknownFormat(SPIRVTypeInst TypeInst) {
1602 if (TypeInst->getOpcode() != SPIRV::OpTypeImage)
1603 return false;
1604 assert(TypeInst->getOperand(7).isImm() && "The image format must be an imm.");
1605 return TypeInst->getOperand(i: 7).getImm() == 0;
1606}
1607
1608static void AddDotProductRequirements(const MachineInstr &MI,
1609 SPIRV::RequirementHandler &Reqs,
1610 const SPIRVSubtarget &ST) {
1611 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_integer_dot_product))
1612 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_integer_dot_product);
1613 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProduct);
1614
1615 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1616 assert(MI.getOperand(2).isReg() && "Unexpected operand in dot");
1617 SPIRVTypeInst TypeDef = getRegType(Reg: MI.getOperand(i: 2).getReg(), MF: *MI.getMF(),
1618 GR: ST.getSPIRVGlobalRegistry());
1619 assert(TypeDef && "Unexpected type instruction for dot input");
1620 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {
1621 assert(TypeDef->getOperand(1).getImm() == 32);
1622 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInput4x8BitPacked);
1623 } else if (isVectorType(SPVTy: TypeDef)) {
1624 SPIRVTypeInst ScalarTypeDef =
1625 MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
1626 assert(ScalarTypeDef->getOpcode() == SPIRV::OpTypeInt);
1627 if (ScalarTypeDef->getOperand(i: 1).getImm() == 8) {
1628 assert(TypeDef->getOperand(2).getImm() == 4 &&
1629 "Dot operand of 8-bit integer type requires 4 components");
1630 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInput4x8Bit);
1631 } else {
1632 Reqs.addCapability(ToAdd: SPIRV::Capability::DotProductInputAll);
1633 }
1634 }
1635}
1636
1637void addPrintfRequirements(const MachineInstr &MI,
1638 SPIRV::RequirementHandler &Reqs,
1639 const SPIRVSubtarget &ST) {
1640 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1641 SPIRVTypeInst PtrType =
1642 GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 4).getReg(), MF: MI.getMF());
1643 if (PtrType) {
1644 MachineOperand ASOp = PtrType->getOperand(i: 1);
1645 if (ASOp.isImm()) {
1646 unsigned AddrSpace = ASOp.getImm();
1647 if (AddrSpace != SPIRV::StorageClass::UniformConstant) {
1648 if (!ST.canUseExtension(
1649 E: SPIRV::Extension::
1650 SPV_EXT_relaxed_printf_string_address_space)) {
1651 report_fatal_error(reason: "SPV_EXT_relaxed_printf_string_address_space is "
1652 "required because printf uses a format string not "
1653 "in constant address space.",
1654 gen_crash_diag: false);
1655 }
1656 Reqs.addExtension(
1657 ToAdd: SPIRV::Extension::SPV_EXT_relaxed_printf_string_address_space);
1658 }
1659 }
1660 }
1661}
1662
1663static void addImageOperandReqs(const MachineInstr &MI,
1664 SPIRV::RequirementHandler &Reqs,
1665 const SPIRVSubtarget &ST, unsigned OpIdx) {
1666 if (MI.getNumOperands() <= OpIdx)
1667 return;
1668 uint32_t Mask = MI.getOperand(i: OpIdx).getImm();
1669 for (uint32_t I = 0; I < 32; ++I)
1670 if (Mask & (1U << I))
1671 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ImageOperandOperand,
1672 i: 1U << I, ST);
1673}
1674
1675static inline void maybeAddScatterGatherReq(const MachineInstr &MI,
1676 SPIRV::RequirementHandler &Reqs,
1677 const SPIRVSubtarget &ST) {
1678 assert(MI.getOperand(1).isReg());
1679 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1680 SPIRVTypeInst ElemTypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1681 if (ElemTypeDef->getOpcode() == SPIRV::OpTypePointer &&
1682 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) {
1683 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_masked_gather_scatter);
1684 Reqs.addCapability(ToAdd: SPIRV::Capability::MaskedGatherScatterINTEL);
1685 }
1686}
1687
1688void addInstrRequirements(const MachineInstr &MI,
1689 SPIRV::ModuleAnalysisInfo &MAI,
1690 const SPIRVSubtarget &ST) {
1691 SPIRV::RequirementHandler &Reqs = MAI.Reqs;
1692 unsigned Op = MI.getOpcode();
1693 switch (Op) {
1694 case SPIRV::OpMemoryModel: {
1695 int64_t Addr = MI.getOperand(i: 0).getImm();
1696 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::AddressingModelOperand,
1697 i: Addr, ST);
1698 int64_t Mem = MI.getOperand(i: 1).getImm();
1699 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::MemoryModelOperand, i: Mem,
1700 ST);
1701 break;
1702 }
1703 case SPIRV::OpEntryPoint: {
1704 int64_t Exe = MI.getOperand(i: 0).getImm();
1705 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ExecutionModelOperand,
1706 i: Exe, ST);
1707 break;
1708 }
1709 case SPIRV::OpExecutionMode:
1710 case SPIRV::OpExecutionModeId: {
1711 int64_t Exe = MI.getOperand(i: 1).getImm();
1712 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::ExecutionModeOperand,
1713 i: Exe, ST);
1714 break;
1715 }
1716 case SPIRV::OpTypeMatrix:
1717 Reqs.addCapability(ToAdd: SPIRV::Capability::Matrix);
1718 break;
1719 case SPIRV::OpTypeInt: {
1720 unsigned BitWidth = MI.getOperand(i: 1).getImm();
1721 if (BitWidth == 64)
1722 Reqs.addCapability(ToAdd: SPIRV::Capability::Int64);
1723 else if (BitWidth == 16)
1724 Reqs.addCapability(ToAdd: SPIRV::Capability::Int16);
1725 else if (BitWidth == 8)
1726 Reqs.addCapability(ToAdd: SPIRV::Capability::Int8);
1727 else if (BitWidth == 4 &&
1728 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4)) {
1729 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_int4);
1730 Reqs.addCapability(ToAdd: SPIRV::Capability::Int4TypeINTEL);
1731 } else if (BitWidth != 32) {
1732 if (!ST.canUseExtension(
1733 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers))
1734 reportFatalUsageError(
1735 reason: "OpTypeInt type with a width other than 8, 16, 32 or 64 bits "
1736 "requires the following SPIR-V extension: "
1737 "SPV_ALTERA_arbitrary_precision_integers");
1738 Reqs.addExtension(
1739 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
1740 Reqs.addCapability(ToAdd: SPIRV::Capability::ArbitraryPrecisionIntegersALTERA);
1741 }
1742 break;
1743 }
1744 case SPIRV::OpDot: {
1745 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1746 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1747 if (isBFloat16Type(TypeDef))
1748 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16DotProductKHR);
1749 break;
1750 }
1751 case SPIRV::OpTypeFloat: {
1752 unsigned BitWidth = MI.getOperand(i: 1).getImm();
1753 if (BitWidth == 64)
1754 Reqs.addCapability(ToAdd: SPIRV::Capability::Float64);
1755 else if (BitWidth == 16) {
1756 if (isBFloat16Type(TypeDef: &MI)) {
1757 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_bfloat16))
1758 report_fatal_error(reason: "OpTypeFloat type with bfloat requires the "
1759 "following SPIR-V extension: SPV_KHR_bfloat16",
1760 gen_crash_diag: false);
1761 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_bfloat16);
1762 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16TypeKHR);
1763 } else {
1764 Reqs.addCapability(ToAdd: SPIRV::Capability::Float16);
1765 }
1766 }
1767 break;
1768 }
1769 case SPIRV::OpTypeVector: {
1770 unsigned NumComponents = MI.getOperand(i: 2).getImm();
1771 if (NumComponents == 8 || NumComponents == 16)
1772 Reqs.addCapability(ToAdd: SPIRV::Capability::Vector16);
1773 else if (requiresLongVectorEXT(NumComponents))
1774 // Such widths are only expressible as OpTypeVectorIdEXT.
1775 reportFatalUsageError(
1776 reason: "OpTypeVector with " + Twine(NumComponents) +
1777 " components requires the following SPIR-V extension: "
1778 "SPV_EXT_long_vector");
1779
1780 maybeAddScatterGatherReq(MI, Reqs, ST);
1781 break;
1782 }
1783 case SPIRV::OpTypeVectorIdEXT: {
1784 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector))
1785 reportFatalUsageError(reason: "OpTypeVectorIdEXT requires the following SPIR-V "
1786 "extension: SPV_EXT_long_vector extension");
1787 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_long_vector);
1788 Reqs.addCapability(ToAdd: SPIRV::Capability::LongVectorEXT);
1789 maybeAddScatterGatherReq(MI, Reqs, ST);
1790 break;
1791 }
1792 case SPIRV::OpTypePointer: {
1793 auto SC = MI.getOperand(i: 1).getImm();
1794 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::StorageClassOperand, i: SC,
1795 ST);
1796 // If it's a type of pointer to float16 targeting OpenCL, add Float16Buffer
1797 // capability.
1798 if (ST.isShader())
1799 break;
1800 assert(MI.getOperand(2).isReg());
1801 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1802 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg());
1803 if ((TypeDef->getNumOperands() == 2) &&
1804 (TypeDef->getOpcode() == SPIRV::OpTypeFloat) &&
1805 (TypeDef->getOperand(i: 1).getImm() == 16))
1806 Reqs.addCapability(ToAdd: SPIRV::Capability::Float16Buffer);
1807 break;
1808 }
1809 case SPIRV::OpExtInst: {
1810 if (MI.getOperand(i: 2).getImm() ==
1811 static_cast<int64_t>(
1812 SPIRV::InstructionSet::NonSemantic_Shader_DebugInfo_100)) {
1813 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_non_semantic_info);
1814 break;
1815 }
1816 if (MI.getOperand(i: 2).getImm() ==
1817 static_cast<int64_t>(SPIRV::InstructionSet::NonSemantic_DebugBreak)) {
1818 // Non-semantic instruction sets no longer require
1819 // SPV_KHR_non_semantic_info as of SPIR-V 1.6
1820 if (!ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 6)))
1821 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_non_semantic_info);
1822 break;
1823 }
1824 if (MI.getOperand(i: 3).getImm() ==
1825 static_cast<int64_t>(SPIRV::OpenCLExtInst::printf)) {
1826 addPrintfRequirements(MI, Reqs, ST);
1827 break;
1828 }
1829 if (MI.getOperand(i: 2).getImm() ==
1830 static_cast<int64_t>(SPIRV::InstructionSet::OpenCL_std)) {
1831 const MachineFunction *MF = MI.getMF();
1832 const MachineRegisterInfo &MRI = MF->getRegInfo();
1833 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
1834
1835 auto IsBFloat16 = [&](SPIRVTypeInst TypeDef) {
1836 if (TypeDef && TypeDef->getOpcode() == SPIRV::OpTypeVector)
1837 TypeDef = MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
1838 return isBFloat16Type(TypeDef);
1839 };
1840
1841 // Result type is operand 1; arguments start at operand 4.
1842 bool UsesBFloat16 = IsBFloat16(MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg()));
1843 for (unsigned I = 4, E = MI.getNumOperands(); I < E && !UsesBFloat16;
1844 ++I) {
1845 const MachineOperand &MO = MI.getOperand(i: I);
1846 if (MO.isReg())
1847 UsesBFloat16 = IsBFloat16(GR->getResultType(
1848 VReg: MO.getReg(), MF: const_cast<MachineFunction *>(MF)));
1849 }
1850
1851 if (UsesBFloat16 && !addBFloat16ArithmeticRequirements(Reqs, ST))
1852 reportUnsupported(
1853 MI, Msg: "OpenCL Extended instructions with bfloat16 require the "
1854 "following SPIR-V extension: SPV_EXT_bfloat16_arithmetic");
1855 }
1856 break;
1857 }
1858 case SPIRV::OpAliasDomainDeclINTEL:
1859 case SPIRV::OpAliasScopeDeclINTEL:
1860 case SPIRV::OpAliasScopeListDeclINTEL: {
1861 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_memory_access_aliasing);
1862 Reqs.addCapability(ToAdd: SPIRV::Capability::MemoryAccessAliasingINTEL);
1863 break;
1864 }
1865 case SPIRV::OpBitReverse:
1866 case SPIRV::OpBitFieldInsert:
1867 case SPIRV::OpBitFieldSExtract:
1868 case SPIRV::OpBitFieldUExtract:
1869 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions)) {
1870 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
1871 break;
1872 }
1873 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_bit_instructions);
1874 Reqs.addCapability(ToAdd: SPIRV::Capability::BitInstructions);
1875 break;
1876 case SPIRV::OpTypeRuntimeArray:
1877 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
1878 break;
1879 case SPIRV::OpTypeOpaque:
1880 case SPIRV::OpTypeEvent:
1881 Reqs.addCapability(ToAdd: SPIRV::Capability::Kernel);
1882 break;
1883 case SPIRV::OpTypePipe:
1884 case SPIRV::OpTypeReserveId:
1885 Reqs.addCapability(ToAdd: SPIRV::Capability::Pipes);
1886 break;
1887 case SPIRV::OpTypePipeStorage:
1888 Reqs.addCapability(ToAdd: SPIRV::Capability::PipeStorage);
1889 break;
1890 case SPIRV::OpTypeDeviceEvent:
1891 case SPIRV::OpTypeQueue:
1892 case SPIRV::OpBuildNDRange:
1893 case SPIRV::OpEnqueueKernel:
1894 case SPIRV::OpGetKernelNDrangeSubGroupCount:
1895 case SPIRV::OpGetKernelNDrangeMaxSubGroupSize:
1896 case SPIRV::OpGetKernelWorkGroupSize:
1897 case SPIRV::OpGetKernelPreferredWorkGroupSizeMultiple:
1898 Reqs.addCapability(ToAdd: SPIRV::Capability::DeviceEnqueue);
1899 break;
1900 case SPIRV::OpDecorate:
1901 case SPIRV::OpDecorateId:
1902 case SPIRV::OpDecorateString:
1903 addOpDecorateReqs(MI, DecIndex: 1, Reqs, ST);
1904 break;
1905 case SPIRV::OpMemberDecorate:
1906 case SPIRV::OpMemberDecorateString:
1907 addOpDecorateReqs(MI, DecIndex: 2, Reqs, ST);
1908 break;
1909 case SPIRV::OpInBoundsPtrAccessChain:
1910 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
1911 break;
1912 case SPIRV::OpConstantSampler:
1913 Reqs.addCapability(ToAdd: SPIRV::Capability::LiteralSampler);
1914 break;
1915 case SPIRV::OpInBoundsAccessChain:
1916 case SPIRV::OpAccessChain:
1917 addOpAccessChainReqs(Instr: MI, Handler&: Reqs, Subtarget: ST);
1918 break;
1919 case SPIRV::OpTypeImage:
1920 addOpTypeImageReqs(MI, Reqs, ST);
1921 break;
1922 case SPIRV::OpTypeSampler:
1923 if (!ST.isShader()) {
1924 Reqs.addCapability(ToAdd: SPIRV::Capability::ImageBasic);
1925 }
1926 break;
1927 case SPIRV::OpTypeForwardPointer:
1928 // TODO: check if it's OpenCL's kernel.
1929 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
1930 break;
1931 case SPIRV::OpAtomicFlagTestAndSet:
1932 case SPIRV::OpAtomicLoad:
1933 case SPIRV::OpAtomicStore:
1934 case SPIRV::OpAtomicExchange:
1935 case SPIRV::OpAtomicCompareExchange:
1936 case SPIRV::OpAtomicCompareExchangeWeak:
1937 case SPIRV::OpAtomicIIncrement:
1938 case SPIRV::OpAtomicIDecrement:
1939 case SPIRV::OpAtomicIAdd:
1940 case SPIRV::OpAtomicISub:
1941 case SPIRV::OpAtomicUMin:
1942 case SPIRV::OpAtomicUMax:
1943 case SPIRV::OpAtomicSMin:
1944 case SPIRV::OpAtomicSMax:
1945 case SPIRV::OpAtomicAnd:
1946 case SPIRV::OpAtomicOr:
1947 case SPIRV::OpAtomicXor: {
1948 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1949 SPIRVTypeInst TypeDef;
1950 if (Op == SPIRV::OpAtomicStore) {
1951 assert(MI.getOperand(3).isReg());
1952 TypeDef = getRegType(Reg: MI.getOperand(i: 3).getReg(), MF: *MI.getMF(),
1953 GR: ST.getSPIRVGlobalRegistry());
1954 assert(TypeDef && "Unexpected type instruction for OpAtomicStore");
1955 } else {
1956 assert(MI.getOperand(1).isReg() && "Unexpected operand in atomic");
1957 TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
1958 }
1959
1960 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {
1961 unsigned BitWidth = TypeDef->getOperand(i: 1).getImm();
1962 if (BitWidth == 64)
1963 Reqs.addCapability(ToAdd: SPIRV::Capability::Int64Atomics);
1964 else if (BitWidth == 16) {
1965 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1966 report_fatal_error(
1967 reason: "16-bit integer atomic operations require the following SPIR-V "
1968 "extension: SPV_INTEL_16bit_atomics",
1969 gen_crash_diag: false);
1970 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1971 switch (Op) {
1972 case SPIRV::OpAtomicLoad:
1973 case SPIRV::OpAtomicStore:
1974 case SPIRV::OpAtomicExchange:
1975 case SPIRV::OpAtomicCompareExchange:
1976 case SPIRV::OpAtomicCompareExchangeWeak:
1977 Reqs.addCapability(
1978 ToAdd: SPIRV::Capability::AtomicInt16CompareExchangeINTEL);
1979 break;
1980 default:
1981 Reqs.addCapability(ToAdd: SPIRV::Capability::Int16AtomicsINTEL);
1982 break;
1983 }
1984 }
1985 } else if (isBFloat16Type(TypeDef)) {
1986 if (is_contained(Set: {SPIRV::OpAtomicLoad, SPIRV::OpAtomicStore,
1987 SPIRV::OpAtomicExchange},
1988 Element: Op)) {
1989 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_16bit_atomics))
1990 report_fatal_error(
1991 reason: "The atomic bfloat16 instruction requires the following SPIR-V "
1992 "extension: SPV_INTEL_16bit_atomics",
1993 gen_crash_diag: false);
1994 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_16bit_atomics);
1995 Reqs.addCapability(ToAdd: SPIRV::Capability::AtomicBFloat16LoadStoreINTEL);
1996 }
1997 }
1998 break;
1999 }
2000 case SPIRV::OpGroupNonUniformIAdd:
2001 case SPIRV::OpGroupNonUniformFAdd:
2002 case SPIRV::OpGroupNonUniformIMul:
2003 case SPIRV::OpGroupNonUniformFMul:
2004 case SPIRV::OpGroupNonUniformSMin:
2005 case SPIRV::OpGroupNonUniformUMin:
2006 case SPIRV::OpGroupNonUniformFMin:
2007 case SPIRV::OpGroupNonUniformSMax:
2008 case SPIRV::OpGroupNonUniformUMax:
2009 case SPIRV::OpGroupNonUniformFMax:
2010 case SPIRV::OpGroupNonUniformBitwiseAnd:
2011 case SPIRV::OpGroupNonUniformBitwiseOr:
2012 case SPIRV::OpGroupNonUniformBitwiseXor:
2013 case SPIRV::OpGroupNonUniformLogicalAnd:
2014 case SPIRV::OpGroupNonUniformLogicalOr:
2015 case SPIRV::OpGroupNonUniformLogicalXor: {
2016 assert(MI.getOperand(3).isImm());
2017 int64_t GroupOp = MI.getOperand(i: 3).getImm();
2018 switch (GroupOp) {
2019 case SPIRV::GroupOperation::Reduce:
2020 case SPIRV::GroupOperation::InclusiveScan:
2021 case SPIRV::GroupOperation::ExclusiveScan:
2022 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformArithmetic);
2023 break;
2024 case SPIRV::GroupOperation::ClusteredReduce:
2025 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformClustered);
2026 break;
2027 case SPIRV::GroupOperation::PartitionedReduceNV:
2028 case SPIRV::GroupOperation::PartitionedInclusiveScanNV:
2029 case SPIRV::GroupOperation::PartitionedExclusiveScanNV:
2030 if (ST.canUseExtension(
2031 E: SPIRV::Extension::SPV_EXT_shader_subgroup_partitioned))
2032 Reqs.addExtension(
2033 ToAdd: SPIRV::Extension::SPV_EXT_shader_subgroup_partitioned);
2034 else
2035 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_NV_shader_subgroup_partitioned);
2036 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformPartitionedNV);
2037 break;
2038 }
2039 break;
2040 }
2041 case SPIRV::OpGroupNonUniformQuadSwap:
2042 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformQuad);
2043 break;
2044 case SPIRV::OpImageQueryLod:
2045 Reqs.addCapability(ToAdd: SPIRV::Capability::ImageQuery);
2046 break;
2047 case SPIRV::OpImageQuerySize:
2048 case SPIRV::OpImageQuerySizeLod:
2049 case SPIRV::OpImageQueryLevels:
2050 case SPIRV::OpImageQuerySamples:
2051 if (ST.isShader())
2052 Reqs.addCapability(ToAdd: SPIRV::Capability::ImageQuery);
2053 break;
2054 case SPIRV::OpImageQueryFormat: {
2055 Register ResultReg = MI.getOperand(i: 0).getReg();
2056 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2057 static const unsigned CompareOps[] = {
2058 SPIRV::OpIEqual, SPIRV::OpINotEqual,
2059 SPIRV::OpUGreaterThan, SPIRV::OpUGreaterThanEqual,
2060 SPIRV::OpULessThan, SPIRV::OpULessThanEqual,
2061 SPIRV::OpSGreaterThan, SPIRV::OpSGreaterThanEqual,
2062 SPIRV::OpSLessThan, SPIRV::OpSLessThanEqual};
2063
2064 auto CheckAndAddExtension = [&](int64_t ImmVal) {
2065 if (ImmVal == 4323 || ImmVal == 4324) {
2066 if (ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_image_raw10_raw12))
2067 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_image_raw10_raw12);
2068 else
2069 report_fatal_error(reason: "This requires the "
2070 "SPV_EXT_image_raw10_raw12 extension");
2071 }
2072 };
2073
2074 for (MachineInstr &UseInst : MRI.use_instructions(Reg: ResultReg)) {
2075 unsigned Opc = UseInst.getOpcode();
2076
2077 if (Opc == SPIRV::OpSwitch) {
2078 for (const MachineOperand &Op : UseInst.operands())
2079 if (Op.isImm())
2080 CheckAndAddExtension(Op.getImm());
2081 } else if (llvm::is_contained(Range: CompareOps, Element: Opc)) {
2082 for (unsigned i = 1; i < UseInst.getNumOperands(); ++i) {
2083 Register UseReg = UseInst.getOperand(i).getReg();
2084 MachineInstr *ConstInst = MRI.getVRegDef(Reg: UseReg);
2085 if (ConstInst && ConstInst->getOpcode() == SPIRV::OpConstantI) {
2086 int64_t ImmVal = ConstInst->getOperand(i: 2).getImm();
2087 if (ImmVal)
2088 CheckAndAddExtension(ImmVal);
2089 }
2090 }
2091 }
2092 }
2093 break;
2094 }
2095
2096 case SPIRV::OpGroupNonUniformShuffle:
2097 case SPIRV::OpGroupNonUniformShuffleXor:
2098 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformShuffle);
2099 break;
2100 case SPIRV::OpGroupNonUniformShuffleUp:
2101 case SPIRV::OpGroupNonUniformShuffleDown:
2102 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformShuffleRelative);
2103 break;
2104 case SPIRV::OpGroupAll:
2105 case SPIRV::OpGroupAny:
2106 case SPIRV::OpGroupBroadcast:
2107 case SPIRV::OpGroupIAdd:
2108 case SPIRV::OpGroupFAdd:
2109 case SPIRV::OpGroupFMin:
2110 case SPIRV::OpGroupUMin:
2111 case SPIRV::OpGroupSMin:
2112 case SPIRV::OpGroupFMax:
2113 case SPIRV::OpGroupUMax:
2114 case SPIRV::OpGroupSMax:
2115 Reqs.addCapability(ToAdd: SPIRV::Capability::Groups);
2116 break;
2117 case SPIRV::OpGroupNonUniformElect:
2118 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniform);
2119 break;
2120 case SPIRV::OpGroupNonUniformAll:
2121 case SPIRV::OpGroupNonUniformAny:
2122 case SPIRV::OpGroupNonUniformAllEqual:
2123 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformVote);
2124 break;
2125 case SPIRV::OpGroupNonUniformBroadcast:
2126 case SPIRV::OpGroupNonUniformBroadcastFirst:
2127 case SPIRV::OpGroupNonUniformBallot:
2128 case SPIRV::OpGroupNonUniformInverseBallot:
2129 case SPIRV::OpGroupNonUniformBallotBitExtract:
2130 case SPIRV::OpGroupNonUniformBallotBitCount:
2131 case SPIRV::OpGroupNonUniformBallotFindLSB:
2132 case SPIRV::OpGroupNonUniformBallotFindMSB:
2133 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformBallot);
2134 break;
2135 case SPIRV::OpSubgroupShuffleINTEL:
2136 case SPIRV::OpSubgroupShuffleDownINTEL:
2137 case SPIRV::OpSubgroupShuffleUpINTEL:
2138 case SPIRV::OpSubgroupShuffleXorINTEL:
2139 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
2140 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
2141 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupShuffleINTEL);
2142 }
2143 break;
2144 case SPIRV::OpSubgroupBlockReadINTEL:
2145 case SPIRV::OpSubgroupBlockWriteINTEL:
2146 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
2147 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
2148 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupBufferBlockIOINTEL);
2149 }
2150 break;
2151 case SPIRV::OpSubgroupImageBlockReadINTEL:
2152 case SPIRV::OpSubgroupImageBlockWriteINTEL:
2153 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_subgroups)) {
2154 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_subgroups);
2155 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupImageBlockIOINTEL);
2156 }
2157 break;
2158 case SPIRV::OpSubgroupImageMediaBlockReadINTEL:
2159 case SPIRV::OpSubgroupImageMediaBlockWriteINTEL:
2160 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_media_block_io)) {
2161 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_media_block_io);
2162 Reqs.addCapability(ToAdd: SPIRV::Capability::SubgroupImageMediaBlockIOINTEL);
2163 }
2164 break;
2165 case SPIRV::OpAssumeTrueKHR:
2166 case SPIRV::OpExpectKHR:
2167 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_expect_assume)) {
2168 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_expect_assume);
2169 Reqs.addCapability(ToAdd: SPIRV::Capability::ExpectAssumeKHR);
2170 }
2171 break;
2172 case SPIRV::OpFmaKHR:
2173 if (ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_fma)) {
2174 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_fma);
2175 Reqs.addCapability(ToAdd: SPIRV::Capability::FmaKHR);
2176 }
2177 break;
2178 case SPIRV::OpPtrCastToCrossWorkgroupINTEL:
2179 case SPIRV::OpCrossWorkgroupCastToPtrINTEL:
2180 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_usm_storage_classes)) {
2181 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_usm_storage_classes);
2182 Reqs.addCapability(ToAdd: SPIRV::Capability::USMStorageClassesINTEL);
2183 }
2184 break;
2185 case SPIRV::OpConstantFunctionPointerINTEL:
2186 case SPIRV::OpFunctionPointerCallINTEL:
2187 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_function_pointers)) {
2188 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_function_pointers);
2189 Reqs.addCapability(ToAdd: SPIRV::Capability::FunctionPointersINTEL);
2190 }
2191 break;
2192 case SPIRV::OpGroupNonUniformQuadAllKHR:
2193 addQuadControlRequirements(Reqs, ST, Feature: "OpGroupNonUniformQuadAllKHR");
2194 Reqs.addCapability(ToAdd: SPIRV::Capability::QuadControlKHR);
2195 break;
2196 case SPIRV::OpGroupNonUniformRotateKHR:
2197 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_subgroup_rotate))
2198 report_fatal_error(reason: "OpGroupNonUniformRotateKHR instruction requires the "
2199 "following SPIR-V extension: SPV_KHR_subgroup_rotate",
2200 gen_crash_diag: false);
2201 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_subgroup_rotate);
2202 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniformRotateKHR);
2203 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupNonUniform);
2204 break;
2205 case SPIRV::OpFixedCosALTERA:
2206 case SPIRV::OpFixedSinALTERA:
2207 case SPIRV::OpFixedCosPiALTERA:
2208 case SPIRV::OpFixedSinPiALTERA:
2209 case SPIRV::OpFixedExpALTERA:
2210 case SPIRV::OpFixedLogALTERA:
2211 case SPIRV::OpFixedRecipALTERA:
2212 case SPIRV::OpFixedSqrtALTERA:
2213 case SPIRV::OpFixedSinCosALTERA:
2214 case SPIRV::OpFixedSinCosPiALTERA:
2215 case SPIRV::OpFixedRsqrtALTERA:
2216 if (!ST.canUseExtension(
2217 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point))
2218 report_fatal_error(reason: "This instruction requires the "
2219 "following SPIR-V extension: "
2220 "SPV_ALTERA_arbitrary_precision_fixed_point",
2221 gen_crash_diag: false);
2222 Reqs.addExtension(
2223 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point);
2224 Reqs.addCapability(ToAdd: SPIRV::Capability::ArbitraryPrecisionFixedPointALTERA);
2225 break;
2226 case SPIRV::OpGroupIMulKHR:
2227 case SPIRV::OpGroupFMulKHR:
2228 case SPIRV::OpGroupBitwiseAndKHR:
2229 case SPIRV::OpGroupBitwiseOrKHR:
2230 case SPIRV::OpGroupBitwiseXorKHR:
2231 case SPIRV::OpGroupLogicalAndKHR:
2232 case SPIRV::OpGroupLogicalOrKHR:
2233 case SPIRV::OpGroupLogicalXorKHR:
2234 if (ST.canUseExtension(
2235 E: SPIRV::Extension::SPV_KHR_uniform_group_instructions)) {
2236 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_uniform_group_instructions);
2237 Reqs.addCapability(ToAdd: SPIRV::Capability::GroupUniformArithmeticKHR);
2238 }
2239 break;
2240 case SPIRV::OpReadClockKHR:
2241 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_shader_clock))
2242 report_fatal_error(reason: "OpReadClockKHR instruction requires the "
2243 "following SPIR-V extension: SPV_KHR_shader_clock",
2244 gen_crash_diag: false);
2245 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_shader_clock);
2246 Reqs.addCapability(ToAdd: SPIRV::Capability::ShaderClockKHR);
2247 break;
2248 case SPIRV::OpAbortKHR:
2249 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_abort))
2250 report_fatal_error(reason: "OpAbortKHR instruction requires the "
2251 "following SPIR-V extension: SPV_KHR_abort",
2252 gen_crash_diag: false);
2253 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_abort);
2254 Reqs.addCapability(ToAdd: SPIRV::Capability::AbortKHR);
2255 break;
2256 case SPIRV::OpPoisonKHR:
2257 case SPIRV::OpFreezeKHR:
2258 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_poison_freeze))
2259 report_fatal_error(reason: "OpPoisonKHR/OpFreezeKHR instruction requires the "
2260 "following SPIR-V extension: SPV_KHR_poison_freeze",
2261 gen_crash_diag: false);
2262 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_poison_freeze);
2263 Reqs.addCapability(ToAdd: SPIRV::Capability::PoisonFreezeKHR);
2264 break;
2265 case SPIRV::OpAtomicFAddEXT:
2266 case SPIRV::OpAtomicFMinEXT:
2267 case SPIRV::OpAtomicFMaxEXT:
2268 AddAtomicFloatRequirements(MI, Reqs, ST);
2269 break;
2270 case SPIRV::OpConvertBF16ToFINTEL:
2271 case SPIRV::OpConvertFToBF16INTEL:
2272 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bfloat16_conversion)) {
2273 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bfloat16_conversion);
2274 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16ConversionINTEL);
2275 }
2276 break;
2277 case SPIRV::OpRoundFToTF32INTEL:
2278 if (ST.canUseExtension(
2279 E: SPIRV::Extension::SPV_INTEL_tensor_float32_conversion)) {
2280 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_tensor_float32_conversion);
2281 Reqs.addCapability(ToAdd: SPIRV::Capability::TensorFloat32RoundingINTEL);
2282 }
2283 break;
2284 case SPIRV::OpVariableLengthArrayINTEL:
2285 case SPIRV::OpSaveMemoryINTEL:
2286 case SPIRV::OpRestoreMemoryINTEL:
2287 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array)) {
2288 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_variable_length_array);
2289 Reqs.addCapability(ToAdd: SPIRV::Capability::VariableLengthArrayINTEL);
2290 }
2291 break;
2292 case SPIRV::OpUntypedVariableLengthArrayINTEL:
2293 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_variable_length_array)) {
2294 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_variable_length_array);
2295 Reqs.addCapability(ToAdd: SPIRV::Capability::UntypedVariableLengthArrayINTEL);
2296 }
2297 break;
2298 case SPIRV::OpAsmTargetINTEL:
2299 case SPIRV::OpAsmINTEL:
2300 case SPIRV::OpAsmCallINTEL:
2301 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_inline_assembly)) {
2302 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_inline_assembly);
2303 Reqs.addCapability(ToAdd: SPIRV::Capability::AsmINTEL);
2304 }
2305 break;
2306 case SPIRV::OpTypeCooperativeMatrixKHR: {
2307 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
2308 report_fatal_error(
2309 reason: "OpTypeCooperativeMatrixKHR type requires the "
2310 "following SPIR-V extension: SPV_KHR_cooperative_matrix",
2311 gen_crash_diag: false);
2312 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
2313 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
2314 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2315 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
2316 if (isBFloat16Type(TypeDef))
2317 Reqs.addCapability(ToAdd: SPIRV::Capability::BFloat16CooperativeMatrixKHR);
2318 break;
2319 }
2320 case SPIRV::OpArithmeticFenceEXT:
2321 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_arithmetic_fence))
2322 report_fatal_error(reason: "OpArithmeticFenceEXT requires the "
2323 "following SPIR-V extension: SPV_EXT_arithmetic_fence",
2324 gen_crash_diag: false);
2325 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_arithmetic_fence);
2326 Reqs.addCapability(ToAdd: SPIRV::Capability::ArithmeticFenceEXT);
2327 break;
2328 case SPIRV::OpControlBarrierArriveINTEL:
2329 case SPIRV::OpControlBarrierWaitINTEL:
2330 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_split_barrier)) {
2331 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_split_barrier);
2332 Reqs.addCapability(ToAdd: SPIRV::Capability::SplitBarrierINTEL);
2333 }
2334 break;
2335 case SPIRV::OpCooperativeMatrixMulAddKHR: {
2336 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
2337 report_fatal_error(reason: "Cooperative matrix instructions require the "
2338 "following SPIR-V extension: "
2339 "SPV_KHR_cooperative_matrix",
2340 gen_crash_diag: false);
2341 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
2342 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
2343 constexpr unsigned MulAddMaxSize = 6;
2344 if (MI.getNumOperands() != MulAddMaxSize)
2345 break;
2346 const int64_t CoopOperands = MI.getOperand(i: MulAddMaxSize - 1).getImm();
2347 if (CoopOperands &
2348 SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) {
2349 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2350 report_fatal_error(reason: "MatrixAAndBTF32ComponentsINTEL type interpretation "
2351 "require the following SPIR-V extension: "
2352 "SPV_INTEL_joint_matrix",
2353 gen_crash_diag: false);
2354 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2355 Reqs.addCapability(
2356 ToAdd: SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL);
2357 }
2358 if (CoopOperands & SPIRV::CooperativeMatrixOperands::
2359 MatrixAAndBBFloat16ComponentsINTEL ||
2360 CoopOperands &
2361 SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL ||
2362 CoopOperands & SPIRV::CooperativeMatrixOperands::
2363 MatrixResultBFloat16ComponentsINTEL) {
2364 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2365 report_fatal_error(reason: "***BF16ComponentsINTEL type interpretations "
2366 "require the following SPIR-V extension: "
2367 "SPV_INTEL_joint_matrix",
2368 gen_crash_diag: false);
2369 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2370 Reqs.addCapability(
2371 ToAdd: SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);
2372 }
2373 break;
2374 }
2375 case SPIRV::OpCooperativeMatrixLoadKHR:
2376 case SPIRV::OpCooperativeMatrixStoreKHR:
2377 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2378 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2379 case SPIRV::OpCooperativeMatrixPrefetchINTEL: {
2380 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_cooperative_matrix))
2381 report_fatal_error(reason: "Cooperative matrix instructions require the "
2382 "following SPIR-V extension: "
2383 "SPV_KHR_cooperative_matrix",
2384 gen_crash_diag: false);
2385 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_cooperative_matrix);
2386 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixKHR);
2387
2388 // Check Layout operand in case if it's not a standard one and add the
2389 // appropriate capability.
2390 unsigned LayoutNum;
2391 switch (Op) {
2392 case SPIRV::OpCooperativeMatrixLoadKHR:
2393 LayoutNum = 3;
2394 break;
2395 case SPIRV::OpCooperativeMatrixStoreKHR:
2396 LayoutNum = 2;
2397 break;
2398 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2399 LayoutNum = 5;
2400 break;
2401 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2402 case SPIRV::OpCooperativeMatrixPrefetchINTEL:
2403 LayoutNum = 4;
2404 break;
2405 default:
2406 llvm_unreachable("unexpected cooperative matrix opcode");
2407 }
2408 Register RegLayout = MI.getOperand(i: LayoutNum).getReg();
2409 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2410 MachineInstr *MILayout = MRI.getUniqueVRegDef(Reg: RegLayout);
2411 if (MILayout->getOpcode() == SPIRV::OpConstantI) {
2412 const unsigned LayoutVal = MILayout->getOperand(i: 2).getImm();
2413 if (LayoutVal ==
2414 static_cast<unsigned>(SPIRV::CooperativeMatrixLayout::PackedINTEL)) {
2415 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2416 report_fatal_error(reason: "PackedINTEL layout require the following SPIR-V "
2417 "extension: SPV_INTEL_joint_matrix",
2418 gen_crash_diag: false);
2419 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2420 Reqs.addCapability(ToAdd: SPIRV::Capability::PackedCooperativeMatrixINTEL);
2421 }
2422 }
2423
2424 // Nothing to do.
2425 if (Op == SPIRV::OpCooperativeMatrixLoadKHR ||
2426 Op == SPIRV::OpCooperativeMatrixStoreKHR)
2427 break;
2428
2429 std::string InstName;
2430 switch (Op) {
2431 case SPIRV::OpCooperativeMatrixPrefetchINTEL:
2432 InstName = "OpCooperativeMatrixPrefetchINTEL";
2433 break;
2434 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
2435 InstName = "OpCooperativeMatrixLoadCheckedINTEL";
2436 break;
2437 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
2438 InstName = "OpCooperativeMatrixStoreCheckedINTEL";
2439 break;
2440 }
2441
2442 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix)) {
2443 const std::string ErrorMsg =
2444 InstName + " instruction requires the "
2445 "following SPIR-V extension: SPV_INTEL_joint_matrix";
2446 report_fatal_error(reason: ErrorMsg.c_str(), gen_crash_diag: false);
2447 }
2448 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2449 if (Op == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
2450 Reqs.addCapability(ToAdd: SPIRV::Capability::CooperativeMatrixPrefetchINTEL);
2451 break;
2452 }
2453 Reqs.addCapability(
2454 ToAdd: SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
2455 break;
2456 }
2457 case SPIRV::OpCooperativeMatrixConstructCheckedINTEL:
2458 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2459 report_fatal_error(reason: "OpCooperativeMatrixConstructCheckedINTEL "
2460 "instructions require the following SPIR-V extension: "
2461 "SPV_INTEL_joint_matrix",
2462 gen_crash_diag: false);
2463 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2464 Reqs.addCapability(
2465 ToAdd: SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
2466 break;
2467 case SPIRV::OpReadPipeBlockingALTERA:
2468 case SPIRV::OpWritePipeBlockingALTERA:
2469 if (ST.canUseExtension(E: SPIRV::Extension::SPV_ALTERA_blocking_pipes)) {
2470 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_ALTERA_blocking_pipes);
2471 Reqs.addCapability(ToAdd: SPIRV::Capability::BlockingPipesALTERA);
2472 }
2473 break;
2474 case SPIRV::OpCooperativeMatrixGetElementCoordINTEL:
2475 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_joint_matrix))
2476 report_fatal_error(reason: "OpCooperativeMatrixGetElementCoordINTEL requires the "
2477 "following SPIR-V extension: SPV_INTEL_joint_matrix",
2478 gen_crash_diag: false);
2479 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_joint_matrix);
2480 Reqs.addCapability(
2481 ToAdd: SPIRV::Capability::CooperativeMatrixInvocationInstructionsINTEL);
2482 break;
2483 case SPIRV::OpConvertHandleToImageINTEL:
2484 case SPIRV::OpConvertHandleToSamplerINTEL:
2485 case SPIRV::OpConvertHandleToSampledImageINTEL: {
2486 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_bindless_images))
2487 report_fatal_error(reason: "OpConvertHandleTo[Image/Sampler/SampledImage]INTEL "
2488 "instructions require the following SPIR-V extension: "
2489 "SPV_INTEL_bindless_images",
2490 gen_crash_diag: false);
2491 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
2492 SPIRV::AddressingModel::AddressingModel AddrModel = MAI.Addr;
2493 SPIRVTypeInst TyDef = GR->getSPIRVTypeForVReg(VReg: MI.getOperand(i: 1).getReg());
2494 if (Op == SPIRV::OpConvertHandleToImageINTEL &&
2495 TyDef->getOpcode() != SPIRV::OpTypeImage) {
2496 report_fatal_error(reason: "Incorrect return type for the instruction "
2497 "OpConvertHandleToImageINTEL",
2498 gen_crash_diag: false);
2499 } else if (Op == SPIRV::OpConvertHandleToSamplerINTEL &&
2500 TyDef->getOpcode() != SPIRV::OpTypeSampler) {
2501 report_fatal_error(reason: "Incorrect return type for the instruction "
2502 "OpConvertHandleToSamplerINTEL",
2503 gen_crash_diag: false);
2504 } else if (Op == SPIRV::OpConvertHandleToSampledImageINTEL &&
2505 TyDef->getOpcode() != SPIRV::OpTypeSampledImage) {
2506 report_fatal_error(reason: "Incorrect return type for the instruction "
2507 "OpConvertHandleToSampledImageINTEL",
2508 gen_crash_diag: false);
2509 }
2510 SPIRVTypeInst SpvTy =
2511 getRegType(Reg: MI.getOperand(i: 2).getReg(), MF: *MI.getMF(), GR);
2512 unsigned Bitwidth = GR->getScalarOrVectorBitWidth(Type: SpvTy);
2513 if (!(Bitwidth == 32 && AddrModel == SPIRV::AddressingModel::Physical32) &&
2514 !(Bitwidth == 64 && AddrModel == SPIRV::AddressingModel::Physical64)) {
2515 report_fatal_error(
2516 reason: "Parameter value must be a 32-bit scalar in case of "
2517 "Physical32 addressing model or a 64-bit scalar in case of "
2518 "Physical64 addressing model",
2519 gen_crash_diag: false);
2520 }
2521 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_bindless_images);
2522 Reqs.addCapability(ToAdd: SPIRV::Capability::BindlessImagesINTEL);
2523 break;
2524 }
2525 case SPIRV::OpSubgroup2DBlockLoadINTEL:
2526 case SPIRV::OpSubgroup2DBlockLoadTransposeINTEL:
2527 case SPIRV::OpSubgroup2DBlockLoadTransformINTEL:
2528 case SPIRV::OpSubgroup2DBlockPrefetchINTEL:
2529 case SPIRV::OpSubgroup2DBlockStoreINTEL: {
2530 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_2d_block_io))
2531 report_fatal_error(reason: "OpSubgroup2DBlock[Load/LoadTranspose/LoadTransform/"
2532 "Prefetch/Store]INTEL instructions require the "
2533 "following SPIR-V extension: SPV_INTEL_2d_block_io",
2534 gen_crash_diag: false);
2535 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_2d_block_io);
2536 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockIOINTEL);
2537
2538 if (Op == SPIRV::OpSubgroup2DBlockLoadTransposeINTEL) {
2539 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockTransposeINTEL);
2540 break;
2541 }
2542 if (Op == SPIRV::OpSubgroup2DBlockLoadTransformINTEL) {
2543 Reqs.addCapability(ToAdd: SPIRV::Capability::Subgroup2DBlockTransformINTEL);
2544 break;
2545 }
2546 break;
2547 }
2548 case SPIRV::OpKill: {
2549 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2550 } break;
2551 case SPIRV::OpDemoteToHelperInvocation:
2552 Reqs.addCapability(ToAdd: SPIRV::Capability::DemoteToHelperInvocation);
2553
2554 if (ST.canUseExtension(
2555 E: SPIRV::Extension::SPV_EXT_demote_to_helper_invocation)) {
2556 if (!ST.isAtLeastSPIRVVer(VerToCompareTo: llvm::VersionTuple(1, 6)))
2557 Reqs.addExtension(
2558 ToAdd: SPIRV::Extension::SPV_EXT_demote_to_helper_invocation);
2559 }
2560 break;
2561 case SPIRV::OpSDot:
2562 case SPIRV::OpUDot:
2563 case SPIRV::OpSUDot:
2564 case SPIRV::OpSDotAccSat:
2565 case SPIRV::OpUDotAccSat:
2566 case SPIRV::OpSUDotAccSat:
2567 AddDotProductRequirements(MI, Reqs, ST);
2568 break;
2569 case SPIRV::OpImageSampleImplicitLod:
2570 case SPIRV::OpImageFetch:
2571 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2572 addImageOperandReqs(MI, Reqs, ST, OpIdx: 4);
2573 break;
2574 case SPIRV::OpImageSampleExplicitLod:
2575 addImageOperandReqs(MI, Reqs, ST, OpIdx: 4);
2576 break;
2577 case SPIRV::OpImageSampleDrefImplicitLod:
2578 case SPIRV::OpImageSampleDrefExplicitLod:
2579 case SPIRV::OpImageDrefGather:
2580 case SPIRV::OpImageGather:
2581 Reqs.addCapability(ToAdd: SPIRV::Capability::Shader);
2582 addImageOperandReqs(MI, Reqs, ST, OpIdx: 5);
2583 break;
2584 case SPIRV::OpImageRead: {
2585 Register ImageReg = MI.getOperand(i: 2).getReg();
2586 SPIRVTypeInst TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(
2587 VReg: ImageReg, MF: const_cast<MachineFunction *>(MI.getMF()));
2588 // OpImageRead and OpImageWrite can use Unknown Image Formats
2589 // when the Kernel capability is declared. In the OpenCL environment we are
2590 // not allowed to produce
2591 // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see
2592 // https://github.com/KhronosGroup/SPIRV-Headers/issues/487
2593
2594 if (isImageTypeWithUnknownFormat(TypeInst: TypeDef) && ST.isShader())
2595 Reqs.addCapability(ToAdd: SPIRV::Capability::StorageImageReadWithoutFormat);
2596 break;
2597 }
2598 case SPIRV::OpImageWrite: {
2599 Register ImageReg = MI.getOperand(i: 0).getReg();
2600 SPIRVTypeInst TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(
2601 VReg: ImageReg, MF: const_cast<MachineFunction *>(MI.getMF()));
2602 // OpImageRead and OpImageWrite can use Unknown Image Formats
2603 // when the Kernel capability is declared. In the OpenCL environment we are
2604 // not allowed to produce
2605 // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see
2606 // https://github.com/KhronosGroup/SPIRV-Headers/issues/487
2607
2608 if (isImageTypeWithUnknownFormat(TypeInst: TypeDef) && ST.isShader())
2609 Reqs.addCapability(ToAdd: SPIRV::Capability::StorageImageWriteWithoutFormat);
2610 break;
2611 }
2612 case SPIRV::OpTypeStructContinuedINTEL:
2613 case SPIRV::OpConstantCompositeContinuedINTEL:
2614 case SPIRV::OpSpecConstantCompositeContinuedINTEL:
2615 case SPIRV::OpCompositeConstructContinuedINTEL: {
2616 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_long_composites))
2617 report_fatal_error(
2618 reason: "Continued instructions require the "
2619 "following SPIR-V extension: SPV_INTEL_long_composites",
2620 gen_crash_diag: false);
2621 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_long_composites);
2622 Reqs.addCapability(ToAdd: SPIRV::Capability::LongCompositesINTEL);
2623 break;
2624 }
2625 case SPIRV::OpArbitraryFloatEQALTERA:
2626 case SPIRV::OpArbitraryFloatGEALTERA:
2627 case SPIRV::OpArbitraryFloatGTALTERA:
2628 case SPIRV::OpArbitraryFloatLEALTERA:
2629 case SPIRV::OpArbitraryFloatLTALTERA:
2630 case SPIRV::OpArbitraryFloatCbrtALTERA:
2631 case SPIRV::OpArbitraryFloatCosALTERA:
2632 case SPIRV::OpArbitraryFloatCosPiALTERA:
2633 case SPIRV::OpArbitraryFloatExp10ALTERA:
2634 case SPIRV::OpArbitraryFloatExp2ALTERA:
2635 case SPIRV::OpArbitraryFloatExpALTERA:
2636 case SPIRV::OpArbitraryFloatExpm1ALTERA:
2637 case SPIRV::OpArbitraryFloatHypotALTERA:
2638 case SPIRV::OpArbitraryFloatLog10ALTERA:
2639 case SPIRV::OpArbitraryFloatLog1pALTERA:
2640 case SPIRV::OpArbitraryFloatLog2ALTERA:
2641 case SPIRV::OpArbitraryFloatLogALTERA:
2642 case SPIRV::OpArbitraryFloatRecipALTERA:
2643 case SPIRV::OpArbitraryFloatSinCosALTERA:
2644 case SPIRV::OpArbitraryFloatSinCosPiALTERA:
2645 case SPIRV::OpArbitraryFloatSinALTERA:
2646 case SPIRV::OpArbitraryFloatSinPiALTERA:
2647 case SPIRV::OpArbitraryFloatSqrtALTERA:
2648 case SPIRV::OpArbitraryFloatACosALTERA:
2649 case SPIRV::OpArbitraryFloatACosPiALTERA:
2650 case SPIRV::OpArbitraryFloatAddALTERA:
2651 case SPIRV::OpArbitraryFloatASinALTERA:
2652 case SPIRV::OpArbitraryFloatASinPiALTERA:
2653 case SPIRV::OpArbitraryFloatATan2ALTERA:
2654 case SPIRV::OpArbitraryFloatATanALTERA:
2655 case SPIRV::OpArbitraryFloatATanPiALTERA:
2656 case SPIRV::OpArbitraryFloatCastFromIntALTERA:
2657 case SPIRV::OpArbitraryFloatCastALTERA:
2658 case SPIRV::OpArbitraryFloatCastToIntALTERA:
2659 case SPIRV::OpArbitraryFloatDivALTERA:
2660 case SPIRV::OpArbitraryFloatMulALTERA:
2661 case SPIRV::OpArbitraryFloatPowALTERA:
2662 case SPIRV::OpArbitraryFloatPowNALTERA:
2663 case SPIRV::OpArbitraryFloatPowRALTERA:
2664 case SPIRV::OpArbitraryFloatRSqrtALTERA:
2665 case SPIRV::OpArbitraryFloatSubALTERA: {
2666 if (!ST.canUseExtension(
2667 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_floating_point))
2668 report_fatal_error(
2669 reason: "Floating point instructions can't be translated correctly without "
2670 "enabled SPV_ALTERA_arbitrary_precision_floating_point extension!",
2671 gen_crash_diag: false);
2672 Reqs.addExtension(
2673 ToAdd: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_floating_point);
2674 Reqs.addCapability(
2675 ToAdd: SPIRV::Capability::ArbitraryPrecisionFloatingPointALTERA);
2676 break;
2677 }
2678 case SPIRV::OpSubgroupMatrixMultiplyAccumulateINTEL: {
2679 if (!ST.canUseExtension(
2680 E: SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate))
2681 report_fatal_error(
2682 reason: "OpSubgroupMatrixMultiplyAccumulateINTEL instruction requires the "
2683 "following SPIR-V "
2684 "extension: SPV_INTEL_subgroup_matrix_multiply_accumulate",
2685 gen_crash_diag: false);
2686 Reqs.addExtension(
2687 ToAdd: SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate);
2688 Reqs.addCapability(
2689 ToAdd: SPIRV::Capability::SubgroupMatrixMultiplyAccumulateINTEL);
2690 break;
2691 }
2692 case SPIRV::OpBitwiseFunctionINTEL: {
2693 if (!ST.canUseExtension(
2694 E: SPIRV::Extension::SPV_INTEL_ternary_bitwise_function))
2695 report_fatal_error(
2696 reason: "OpBitwiseFunctionINTEL instruction requires the following SPIR-V "
2697 "extension: SPV_INTEL_ternary_bitwise_function",
2698 gen_crash_diag: false);
2699 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_ternary_bitwise_function);
2700 Reqs.addCapability(ToAdd: SPIRV::Capability::TernaryBitwiseFunctionINTEL);
2701 break;
2702 }
2703 case SPIRV::OpCopyMemorySized: {
2704 Reqs.addCapability(ToAdd: SPIRV::Capability::Addresses);
2705 // TODO: Add UntypedPointersKHR when implemented.
2706 break;
2707 }
2708 case SPIRV::OpTypeUntypedPointerKHR:
2709 Reqs.getAndAddRequirements(Category: SPIRV::OperandCategory::StorageClassOperand,
2710 i: MI.getOperand(i: 1).getImm(), ST);
2711 [[fallthrough]];
2712 case SPIRV::OpUntypedVariableKHR:
2713 case SPIRV::OpUntypedAccessChainKHR:
2714 case SPIRV::OpUntypedInBoundsAccessChainKHR:
2715 case SPIRV::OpUntypedPtrAccessChainKHR:
2716 case SPIRV::OpUntypedInBoundsPtrAccessChainKHR:
2717 case SPIRV::OpUntypedPrefetchKHR:
2718 case SPIRV::OpUntypedGroupAsyncCopyKHR: {
2719 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_untyped_pointers))
2720 report_fatal_error(reason: "Untyped pointer instructions require the following "
2721 "SPIR-V extension: SPV_KHR_untyped_pointers",
2722 gen_crash_diag: false);
2723 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_untyped_pointers);
2724 Reqs.addCapability(ToAdd: SPIRV::Capability::UntypedPointersKHR);
2725 break;
2726 }
2727 case SPIRV::OpPredicatedLoadINTEL:
2728 case SPIRV::OpPredicatedStoreINTEL: {
2729 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_predicated_io))
2730 report_fatal_error(
2731 reason: "OpPredicated[Load/Store]INTEL instructions require "
2732 "the following SPIR-V extension: SPV_INTEL_predicated_io",
2733 gen_crash_diag: false);
2734 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_predicated_io);
2735 Reqs.addCapability(ToAdd: SPIRV::Capability::PredicatedIOINTEL);
2736 break;
2737 }
2738 case SPIRV::OpFAddS:
2739 case SPIRV::OpFSubS:
2740 case SPIRV::OpFMulS:
2741 case SPIRV::OpFDivS:
2742 case SPIRV::OpFRemS:
2743 case SPIRV::OpFMod:
2744 case SPIRV::OpFNegate:
2745 case SPIRV::OpFAddV:
2746 case SPIRV::OpFSubV:
2747 case SPIRV::OpFMulV:
2748 case SPIRV::OpFDivV:
2749 case SPIRV::OpFRemV:
2750 case SPIRV::OpFNegateV: {
2751 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2752 SPIRVTypeInst TypeDef = MRI.getVRegDef(Reg: MI.getOperand(i: 1).getReg());
2753 if (isVectorType(SPVTy: TypeDef))
2754 TypeDef = MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
2755 if (isBFloat16Type(TypeDef) && !addBFloat16ArithmeticRequirements(Reqs, ST))
2756 report_fatal_error(
2757 reason: "Arithmetic instructions with bfloat16 arguments require the "
2758 "following SPIR-V extension: SPV_EXT_bfloat16_arithmetic",
2759 gen_crash_diag: false);
2760 break;
2761 }
2762 case SPIRV::OpOrdered:
2763 case SPIRV::OpUnordered:
2764 case SPIRV::OpFOrdEqual:
2765 case SPIRV::OpFOrdNotEqual:
2766 case SPIRV::OpFOrdLessThan:
2767 case SPIRV::OpFOrdLessThanEqual:
2768 case SPIRV::OpFOrdGreaterThan:
2769 case SPIRV::OpFOrdGreaterThanEqual:
2770 case SPIRV::OpFUnordEqual:
2771 case SPIRV::OpFUnordNotEqual:
2772 case SPIRV::OpFUnordLessThan:
2773 case SPIRV::OpFUnordLessThanEqual:
2774 case SPIRV::OpFUnordGreaterThan:
2775 case SPIRV::OpFUnordGreaterThanEqual: {
2776 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
2777 SPIRVTypeInst TypeDef = getRegType(Reg: MI.getOperand(i: 2).getReg(), MF: *MI.getMF(),
2778 GR: ST.getSPIRVGlobalRegistry());
2779 assert(TypeDef && "Unexpected type instruction for comparison operand");
2780 if (isVectorType(SPVTy: TypeDef))
2781 TypeDef = MRI.getVRegDef(Reg: TypeDef->getOperand(i: 1).getReg());
2782 if (isBFloat16Type(TypeDef) && !addBFloat16ArithmeticRequirements(Reqs, ST))
2783 report_fatal_error(
2784 reason: "Relational instructions with bfloat16 arguments require the "
2785 "following SPIR-V extension: SPV_EXT_bfloat16_arithmetic",
2786 gen_crash_diag: false);
2787 break;
2788 }
2789 case SPIRV::OpDPdxCoarse:
2790 case SPIRV::OpDPdyCoarse:
2791 case SPIRV::OpDPdxFine:
2792 case SPIRV::OpDPdyFine: {
2793 Reqs.addCapability(ToAdd: SPIRV::Capability::DerivativeControl);
2794 break;
2795 }
2796 case SPIRV::OpLoopControlINTEL: {
2797 Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_unstructured_loop_controls);
2798 Reqs.addCapability(ToAdd: SPIRV::Capability::UnstructuredLoopControlsINTEL);
2799 break;
2800 }
2801
2802 default:
2803 break;
2804 }
2805
2806 // If we require capability Shader, then we can remove the requirement for
2807 // the BitInstructions capability, since Shader is a superset capability
2808 // of BitInstructions.
2809 Reqs.removeCapabilityIf(ToRemove: SPIRV::Capability::BitInstructions,
2810 IfPresent: SPIRV::Capability::Shader);
2811}
2812
2813static void collectReqs(const Module &M, SPIRV::ModuleAnalysisInfo &MAI,
2814 MachineFunctionGetter GetMF, const SPIRVSubtarget &ST) {
2815 // Collect requirements for existing instructions.
2816 for (const Function &F : M) {
2817 MachineFunction *MF = GetMF(F);
2818 if (!MF)
2819 continue;
2820 for (const MachineBasicBlock &MBB : *MF)
2821 for (const MachineInstr &MI : MBB)
2822 addInstrRequirements(MI, MAI, ST);
2823 }
2824 // Collect requirements for OpExecutionMode instructions.
2825 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
2826 if (Node) {
2827 bool RequireFloatControls = false, RequireIntelFloatControls2 = false,
2828 RequireKHRFloatControls2 = false,
2829 VerLower14 = !ST.isAtLeastSPIRVVer(VerToCompareTo: VersionTuple(1, 4));
2830 bool HasIntelFloatControls2 =
2831 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_float_controls2);
2832 bool HasKHRFloatControls2 =
2833 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2834 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
2835 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
2836 const MDOperand &MDOp = MDN->getOperand(I: 1);
2837 if (auto *CMeta = dyn_cast<ConstantAsMetadata>(Val: MDOp)) {
2838 Constant *C = CMeta->getValue();
2839 if (ConstantInt *Const = dyn_cast<ConstantInt>(Val: C)) {
2840 auto EM = Const->getZExtValue();
2841 // SPV_KHR_float_controls is not available until v1.4:
2842 // add SPV_KHR_float_controls if the version is too low
2843 switch (EM) {
2844 case SPIRV::ExecutionMode::DenormPreserve:
2845 case SPIRV::ExecutionMode::DenormFlushToZero:
2846 case SPIRV::ExecutionMode::RoundingModeRTE:
2847 case SPIRV::ExecutionMode::RoundingModeRTZ:
2848 RequireFloatControls = VerLower14;
2849 MAI.Reqs.getAndAddRequirements(
2850 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2851 break;
2852 case SPIRV::ExecutionMode::RoundingModeRTPINTEL:
2853 case SPIRV::ExecutionMode::RoundingModeRTNINTEL:
2854 case SPIRV::ExecutionMode::FloatingPointModeALTINTEL:
2855 case SPIRV::ExecutionMode::FloatingPointModeIEEEINTEL:
2856 if (HasIntelFloatControls2) {
2857 RequireIntelFloatControls2 = true;
2858 MAI.Reqs.getAndAddRequirements(
2859 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2860 }
2861 break;
2862 case SPIRV::ExecutionMode::FPFastMathDefault: {
2863 if (HasKHRFloatControls2) {
2864 RequireKHRFloatControls2 = true;
2865 MAI.Reqs.getAndAddRequirements(
2866 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2867 }
2868 break;
2869 }
2870 case SPIRV::ExecutionMode::ContractionOff:
2871 case SPIRV::ExecutionMode::SignedZeroInfNanPreserve:
2872 if (HasKHRFloatControls2) {
2873 RequireKHRFloatControls2 = true;
2874 MAI.Reqs.getAndAddRequirements(
2875 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2876 i: SPIRV::ExecutionMode::FPFastMathDefault, ST);
2877 } else {
2878 MAI.Reqs.getAndAddRequirements(
2879 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2880 }
2881 break;
2882 default:
2883 MAI.Reqs.getAndAddRequirements(
2884 Category: SPIRV::OperandCategory::ExecutionModeOperand, i: EM, ST);
2885 }
2886 }
2887 }
2888 }
2889 if (RequireFloatControls &&
2890 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls))
2891 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls);
2892 if (RequireIntelFloatControls2)
2893 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_float_controls2);
2894 if (RequireKHRFloatControls2)
2895 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_float_controls2);
2896 }
2897 for (const Function &F : M) {
2898 if (F.isDeclaration())
2899 continue;
2900 if (F.getMetadata(Kind: "reqd_work_group_size"))
2901 MAI.Reqs.getAndAddRequirements(
2902 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2903 i: SPIRV::ExecutionMode::LocalSize, ST);
2904 if (F.getFnAttribute(Kind: "hlsl.numthreads").isValid()) {
2905 MAI.Reqs.getAndAddRequirements(
2906 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2907 i: SPIRV::ExecutionMode::LocalSize, ST);
2908 }
2909 if (F.getFnAttribute(Kind: "enable-maximal-reconvergence").getValueAsBool()) {
2910 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_KHR_maximal_reconvergence);
2911 }
2912 if (F.getMetadata(Kind: "work_group_size_hint"))
2913 MAI.Reqs.getAndAddRequirements(
2914 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2915 i: SPIRV::ExecutionMode::LocalSizeHint, ST);
2916 if (F.getMetadata(Kind: "intel_reqd_sub_group_size") ||
2917 F.getMetadata(Kind: "reqd_sub_group_size"))
2918 MAI.Reqs.getAndAddRequirements(
2919 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2920 i: SPIRV::ExecutionMode::SubgroupSize, ST);
2921 if (F.getMetadata(Kind: "max_work_group_size"))
2922 MAI.Reqs.getAndAddRequirements(
2923 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2924 i: SPIRV::ExecutionMode::MaxWorkgroupSizeINTEL, ST);
2925 if (F.getMetadata(Kind: "vec_type_hint"))
2926 MAI.Reqs.getAndAddRequirements(
2927 Category: SPIRV::OperandCategory::ExecutionModeOperand,
2928 i: SPIRV::ExecutionMode::VecTypeHint, ST);
2929
2930 if (F.hasOptNone()) {
2931 if (ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_optnone)) {
2932 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_INTEL_optnone);
2933 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::OptNoneINTEL);
2934 } else if (ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_optnone)) {
2935 MAI.Reqs.addExtension(ToAdd: SPIRV::Extension::SPV_EXT_optnone);
2936 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::OptNoneEXT);
2937 }
2938 }
2939 }
2940}
2941
2942static unsigned getFastMathFlags(const MachineInstr &I,
2943 const SPIRVSubtarget &ST) {
2944 unsigned Flags = SPIRV::FPFastMathMode::None;
2945 bool CanUseKHRFloatControls2 =
2946 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2947 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNoNans))
2948 Flags |= SPIRV::FPFastMathMode::NotNaN;
2949 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNoInfs))
2950 Flags |= SPIRV::FPFastMathMode::NotInf;
2951 if (I.getFlag(Flag: MachineInstr::MIFlag::FmNsz))
2952 Flags |= SPIRV::FPFastMathMode::NSZ;
2953 if (I.getFlag(Flag: MachineInstr::MIFlag::FmArcp))
2954 Flags |= SPIRV::FPFastMathMode::AllowRecip;
2955 if (I.getFlag(Flag: MachineInstr::MIFlag::FmContract) && CanUseKHRFloatControls2)
2956 Flags |= SPIRV::FPFastMathMode::AllowContract;
2957 if (I.getFlag(Flag: MachineInstr::MIFlag::FmReassoc)) {
2958 if (CanUseKHRFloatControls2)
2959 // LLVM reassoc maps to SPIRV transform, see
2960 // https://github.com/KhronosGroup/SPIRV-Registry/issues/326 for details.
2961 // Because we are enabling AllowTransform, we must enable AllowReassoc and
2962 // AllowContract too, as required by SPIRV spec. Also, we used to map
2963 // MIFlag::FmReassoc to FPFastMathMode::Fast, which now should instead by
2964 // replaced by turning all the other bits instead. Therefore, we're
2965 // enabling every bit here except None and Fast.
2966 Flags |= SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
2967 SPIRV::FPFastMathMode::NSZ | SPIRV::FPFastMathMode::AllowRecip |
2968 SPIRV::FPFastMathMode::AllowTransform |
2969 SPIRV::FPFastMathMode::AllowReassoc |
2970 SPIRV::FPFastMathMode::AllowContract;
2971 else
2972 Flags |= SPIRV::FPFastMathMode::Fast;
2973 }
2974
2975 if (CanUseKHRFloatControls2) {
2976 // Error out if SPIRV::FPFastMathMode::Fast is enabled.
2977 assert(!(Flags & SPIRV::FPFastMathMode::Fast) &&
2978 "SPIRV::FPFastMathMode::Fast is deprecated and should not be used "
2979 "anymore.");
2980
2981 // Error out if AllowTransform is enabled without AllowReassoc and
2982 // AllowContract.
2983 assert((!(Flags & SPIRV::FPFastMathMode::AllowTransform) ||
2984 ((Flags & SPIRV::FPFastMathMode::AllowReassoc &&
2985 Flags & SPIRV::FPFastMathMode::AllowContract))) &&
2986 "SPIRV::FPFastMathMode::AllowTransform requires AllowReassoc and "
2987 "AllowContract flags to be enabled as well.");
2988 }
2989
2990 return Flags;
2991}
2992
2993static bool isFastMathModeAvailable(const SPIRVSubtarget &ST) {
2994 if (ST.isKernel())
2995 return true;
2996 if (ST.getSPIRVVersion() < VersionTuple(1, 2))
2997 return false;
2998 return ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2);
2999}
3000
3001static bool handleMIFlagDecoration(
3002 MachineInstr &I, const SPIRVSubtarget &ST, const SPIRVInstrInfo &TII,
3003 SPIRV::RequirementHandler &Reqs, const SPIRVGlobalRegistry *GR,
3004 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec) {
3005 bool Changed = false;
3006 // Insert after I so that the decorated register is defined before its use.
3007 MachineBasicBlock::iterator InsertPt = std::next(x: I.getIterator());
3008 auto Decorate = [&](SPIRV::Decoration::Decoration Dec,
3009 ArrayRef<uint32_t> DecArgs) {
3010 MachineIRBuilder MIRBuilder(*I.getParent(), InsertPt);
3011 MIRBuilder.setDebugLoc(I.getDebugLoc());
3012 buildOpDecorate(Reg: I.getOperand(i: 0).getReg(), MIRBuilder, Dec, DecArgs);
3013 Changed = true;
3014 };
3015 if (TII.canUseIntegerWrapDecoration(MI: I)) {
3016 if (I.getFlag(Flag: MachineInstr::MIFlag::NoSWrap) &&
3017 getSymbolicOperandRequirements(
3018 Category: SPIRV::OperandCategory::DecorationOperand,
3019 i: SPIRV::Decoration::NoSignedWrap, ST, Reqs)
3020 .IsSatisfiable)
3021 Decorate(SPIRV::Decoration::NoSignedWrap, {});
3022 if (I.getFlag(Flag: MachineInstr::MIFlag::NoUWrap) &&
3023 getSymbolicOperandRequirements(
3024 Category: SPIRV::OperandCategory::DecorationOperand,
3025 i: SPIRV::Decoration::NoUnsignedWrap, ST, Reqs)
3026 .IsSatisfiable)
3027 Decorate(SPIRV::Decoration::NoUnsignedWrap, {});
3028 }
3029 // In Kernel environments, FPFastMathMode on OpExtInst is valid per core
3030 // spec. For other instruction types, SPV_KHR_float_controls2 is required.
3031 bool CanUseFM =
3032 TII.canUseFastMathFlags(
3033 MI: I, KHRFloatControls2: ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2)) ||
3034 (ST.isKernel() && I.getOpcode() == SPIRV::OpExtInst);
3035 if (!CanUseFM)
3036 return Changed;
3037
3038 unsigned FMFlags = getFastMathFlags(I, ST);
3039 if (FMFlags == SPIRV::FPFastMathMode::None) {
3040 // We also need to check if any FPFastMathDefault info was set for the
3041 // types used in this instruction.
3042 if (FPFastMathDefaultInfoVec.empty())
3043 return Changed;
3044
3045 // Use the first operand's FP type for arithmetic and comparisons,
3046 // and the result type for calls and extended instructions.
3047 SPIRVTypeInst FPType =
3048 I.getOpcode() == SPIRV::OpExtInst ||
3049 I.getOpcode() == SPIRV::OpFunctionCall
3050 ? I.getMF()->getRegInfo().getVRegDef(Reg: I.getOperand(i: 1).getReg())
3051 : getRegType(Reg: I.getOperand(i: 2).getReg(), MF: *I.getMF(), GR);
3052 const Type *Ty = GR->getTypeForSPIRVType(Ty: FPType);
3053 Ty = Ty->isVectorTy() ? cast<VectorType>(Val: Ty)->getElementType() : Ty;
3054
3055 // Match instruction type with the FPFastMathDefaultInfoVec.
3056 bool Emit = false;
3057 for (SPIRV::FPFastMathDefaultInfo &Elem : FPFastMathDefaultInfoVec) {
3058 if (Ty == Elem.Ty) {
3059 FMFlags = Elem.FastMathFlags;
3060 Emit = Elem.ContractionOff || Elem.SignedZeroInfNanPreserve ||
3061 Elem.FPFastMathDefault;
3062 break;
3063 }
3064 }
3065
3066 if (FMFlags == SPIRV::FPFastMathMode::None && !Emit)
3067 return Changed;
3068 }
3069 if (isFastMathModeAvailable(ST))
3070 Decorate(SPIRV::Decoration::FPFastMathMode, {FMFlags});
3071 return Changed;
3072}
3073
3074static void addMBBNames(const Module &M, MachineFunctionGetter GetMF,
3075 SPIRV::ModuleAnalysisInfo &MAI) {
3076 for (const Function &F : M) {
3077 MachineFunction *MF = GetMF(F);
3078 if (!MF)
3079 continue;
3080 if (MF->getFunction()
3081 .getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME)
3082 .isValid())
3083 continue;
3084 for (const auto &MBB : *MF) {
3085 if (!MBB.hasName() || MBB.empty())
3086 continue;
3087 MAI.MBBNames.emplace_back(Args: MAI.getOrCreateMBBRegister(MBB), Args: MBB.getName());
3088 }
3089 }
3090}
3091
3092static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(
3093 const Module &M,
3094 SPIRV::FPFastMathDefaultInfoMapTy &FPFastMathDefaultInfoMap,
3095 const Function *F) {
3096 auto it = FPFastMathDefaultInfoMap.find(Key: F);
3097 if (it != FPFastMathDefaultInfoMap.end())
3098 return it->second;
3099
3100 // If the map does not contain the entry, create a new one. Initialize it to
3101 // contain all 3 elements sorted by bit width of target type: {half, float,
3102 // double}.
3103 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;
3104 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getHalfTy(C&: M.getContext()),
3105 Args: SPIRV::FPFastMathMode::None);
3106 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getFloatTy(C&: M.getContext()),
3107 Args: SPIRV::FPFastMathMode::None);
3108 FPFastMathDefaultInfoVec.emplace_back(Args: Type::getDoubleTy(C&: M.getContext()),
3109 Args: SPIRV::FPFastMathMode::None);
3110 return FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);
3111}
3112
3113static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(
3114 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,
3115 const Type *Ty) {
3116 size_t BitWidth = Ty->getScalarSizeInBits();
3117 int Index =
3118 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(
3119 BitWidth);
3120 assert(Index >= 0 && Index < 3 &&
3121 "Expected FPFastMathDefaultInfo for half, float, or double");
3122 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3123 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3124 return FPFastMathDefaultInfoVec[Index];
3125}
3126
3127static void collectFPFastMathDefaults(
3128 const Module &M,
3129 SPIRV::FPFastMathDefaultInfoMapTy &FPFastMathDefaultInfoMap,
3130 const SPIRVSubtarget &ST, const Function *OnlyF = nullptr) {
3131 if (!ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_float_controls2))
3132 return;
3133
3134 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.
3135 // We need the entry point (function) as the key, and the target
3136 // type and flags as the value.
3137 // We also need to check ContractionOff and SignedZeroInfNanPreserve
3138 // execution modes, as they are now deprecated and must be replaced
3139 // with FPFastMathDefaultInfo.
3140 auto Node = M.getNamedMetadata(Name: "spirv.ExecutionMode");
3141 if (!Node)
3142 return;
3143
3144 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
3145 MDNode *MDN = cast<MDNode>(Val: Node->getOperand(i));
3146 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");
3147 const Function *F = cast<Function>(
3148 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 0))->getValue());
3149 if (OnlyF && F != OnlyF)
3150 continue;
3151 const auto EM =
3152 cast<ConstantInt>(
3153 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 1))->getValue())
3154 ->getZExtValue();
3155 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
3156 assert(MDN->getNumOperands() == 4 &&
3157 "Expected 4 operands for FPFastMathDefault");
3158
3159 const Type *T = cast<ValueAsMetadata>(Val: MDN->getOperand(I: 2))->getType();
3160 unsigned Flags =
3161 cast<ConstantInt>(
3162 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 3))->getValue())
3163 ->getZExtValue();
3164 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3165 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3166 SPIRV::FPFastMathDefaultInfo &Info =
3167 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, Ty: T);
3168 Info.FastMathFlags = Flags;
3169 Info.FPFastMathDefault = true;
3170 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
3171 assert(MDN->getNumOperands() == 2 &&
3172 "Expected no operands for ContractionOff");
3173
3174 // We need to save this info for every possible FP type, i.e. {half,
3175 // float, double, fp128}.
3176 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3177 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3178 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {
3179 Info.ContractionOff = true;
3180 }
3181 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
3182 assert(MDN->getNumOperands() == 3 &&
3183 "Expected 1 operand for SignedZeroInfNanPreserve");
3184 unsigned TargetWidth =
3185 cast<ConstantInt>(
3186 Val: cast<ConstantAsMetadata>(Val: MDN->getOperand(I: 2))->getValue())
3187 ->getZExtValue();
3188 // We need to save this info only for the FP type with TargetWidth.
3189 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =
3190 getOrCreateFPFastMathDefaultInfoVec(M, FPFastMathDefaultInfoMap, F);
3191 int Index = SPIRV::FPFastMathDefaultInfoVector::
3192 computeFPFastMathDefaultInfoVecIndex(BitWidth: TargetWidth);
3193 assert(Index >= 0 && Index < 3 &&
3194 "Expected FPFastMathDefaultInfo for half, float, or double");
3195 assert(FPFastMathDefaultInfoVec.size() == 3 &&
3196 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
3197 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
3198 }
3199 }
3200}
3201
3202SPIRVModuleAnalysisImpl::SPIRVModuleAnalysisImpl(const SPIRVSubtarget &ST,
3203 SPIRV::ModuleAnalysisInfo &MAI,
3204 MachineFunctionGetter GetMF)
3205 : ST(&ST), GR(ST.getSPIRVGlobalRegistry()), TII(ST.getInstrInfo()),
3206 MAI(MAI), GetMF(GetMF) {}
3207
3208void SPIRVModuleAnalysisImpl::run(const Module &M) {
3209 setBaseInfo(M);
3210
3211 addMBBNames(M, GetMF, MAI);
3212 collectFPFastMathDefaults(M, FPFastMathDefaultInfoMap&: MAI.FPFastMathDefaultInfoMap, ST: *ST);
3213
3214 collectReqs(M, MAI, GetMF, ST: *ST);
3215
3216 // Process type/const/global var/func decl instructions, number their
3217 // destination registers from 0 to N, collect Extensions and Capabilities.
3218 collectDeclarations(M);
3219
3220 // Number rest of registers from N+1 onwards.
3221 numberRegistersGlobally(M);
3222
3223 // Collect OpName, OpEntryPoint, OpDecorate etc, process other instructions.
3224 processOtherInstrs(M);
3225
3226 // If there are no entry points, we need the Linkage capability.
3227 if (MAI.MS[SPIRV::MB_EntryPoints].empty())
3228 MAI.Reqs.addCapability(ToAdd: SPIRV::Capability::Linkage);
3229
3230 // Set maximum ID used.
3231 GR->setBound(MAI.MaxID);
3232}
3233
3234void SPIRVModuleAnalysisWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
3235 AU.addRequired<TargetPassConfig>();
3236 AU.addRequired<MachineModuleInfoWrapperPass>();
3237}
3238
3239bool SPIRVModuleAnalysisWrapperPass::runOnModule(Module &M) {
3240 SPIRVTargetMachine &TM =
3241 getAnalysis<TargetPassConfig>().getTM<SPIRVTargetMachine>();
3242 MachineModuleInfo &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
3243 SPIRVModuleAnalysisImpl(
3244 *TM.getSubtargetImpl(), MAI,
3245 [&MMI](const Function &F) { return MMI.getMachineFunction(F); })
3246 .run(M);
3247 return false;
3248}
3249
3250AnalysisKey SPIRVModuleAnalysis::Key;
3251
3252SPIRVModuleAnalysis::Result
3253SPIRVModuleAnalysis::run(Module &M, ModuleAnalysisManager &MAM) {
3254 const auto &TM = static_cast<const SPIRVTargetMachine &>(
3255 MAM.getResult<MachineModuleAnalysis>(IR&: M).getMMI().getTarget());
3256 FunctionAnalysisManager &FAM =
3257 MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
3258 Result MAI;
3259 SPIRVModuleAnalysisImpl(*TM.getSubtargetImpl(), MAI,
3260 [&FAM](const Function &F) -> MachineFunction * {
3261 MachineFunctionAnalysis::Result *MFA =
3262 FAM.getCachedResult<MachineFunctionAnalysis>(
3263 IR&: const_cast<Function &>(F));
3264 assert((MFA || F.isDeclaration()) &&
3265 "Missing MachineFunction for definition");
3266 return MFA ? &MFA->getMF() : nullptr;
3267 })
3268 .run(M);
3269 return MAI;
3270}
3271
3272static bool prepareModuleAnalysis(MachineFunction &MF) {
3273 const auto &ST = MF.getSubtarget<SPIRVSubtarget>();
3274 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();
3275 const SPIRVInstrInfo &TII = *ST.getInstrInfo();
3276 SPIRV::RequirementHandler Reqs;
3277 Reqs.initAvailableCapabilities(ST);
3278 SPIRV::FPFastMathDefaultInfoMapTy FPFastMathDefaultInfoMap;
3279 const Function &F = MF.getFunction();
3280 collectFPFastMathDefaults(M: *F.getParent(), FPFastMathDefaultInfoMap, ST, OnlyF: &F);
3281 // Add decorations related to MI flags.
3282 bool Changed = false;
3283 for (auto &MBB : MF)
3284 for (auto &MI : make_early_inc_range(Range&: MBB))
3285 Changed |= handleMIFlagDecoration(I&: MI, ST, TII, Reqs, GR,
3286 FPFastMathDefaultInfoVec&: FPFastMathDefaultInfoMap[&F]);
3287 return Changed;
3288}
3289
3290namespace {
3291class SPIRVPrepareModuleAnalysisLegacy : public MachineFunctionPass {
3292public:
3293 static char ID;
3294 SPIRVPrepareModuleAnalysisLegacy() : MachineFunctionPass(ID) {}
3295 bool runOnMachineFunction(MachineFunction &MF) override {
3296 return prepareModuleAnalysis(MF);
3297 }
3298};
3299} // namespace
3300
3301char SPIRVPrepareModuleAnalysisLegacy::ID = 0;
3302
3303INITIALIZE_PASS(SPIRVPrepareModuleAnalysisLegacy,
3304 "spirv-prepare-module-analysis",
3305 "SPIRV prepare module analysis", false, false)
3306
3307FunctionPass *llvm::createSPIRVPrepareModuleAnalysisLegacyPass() {
3308 return new SPIRVPrepareModuleAnalysisLegacy();
3309}
3310
3311PreservedAnalyses
3312SPIRVPrepareModuleAnalysisPass::run(MachineFunction &MF,
3313 MachineFunctionAnalysisManager &MFAM) {
3314 return prepareModuleAnalysis(MF) ? getMachineFunctionPassPreservedAnalyses()
3315 : PreservedAnalyses::all();
3316}
3317