1//===- Serializer.cpp - MLIR SPIR-V Serializer ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the MLIR SPIR-V module to SPIR-V binary serializer.
10//
11//===----------------------------------------------------------------------===//
12
13#include "Serializer.h"
14
15#include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h"
16#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
17#include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
18#include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
19#include "mlir/Dialect/SPIRV/IR/TargetAndABI.h"
20#include "mlir/Target/SPIRV/SPIRVBinaryUtils.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/Sequence.h"
23#include "llvm/ADT/StringExtras.h"
24#include "llvm/ADT/TypeSwitch.h"
25#include "llvm/ADT/bit.h"
26#include "llvm/Support/Debug.h"
27#include <cstdint>
28#include <optional>
29
30#define DEBUG_TYPE "spirv-serialization"
31
32using namespace mlir;
33
34/// Returns the merge block if the given `op` is a structured control flow op.
35/// Otherwise returns nullptr.
36static Block *getStructuredControlFlowOpMergeBlock(Operation *op) {
37 if (auto selectionOp = dyn_cast<spirv::SelectionOp>(Val: op))
38 return selectionOp.getMergeBlock();
39 if (auto loopOp = dyn_cast<spirv::LoopOp>(Val: op))
40 return loopOp.getMergeBlock();
41 return nullptr;
42}
43
44/// Given a predecessor `block` for a block with arguments, returns the block
45/// that should be used as the parent block for SPIR-V OpPhi instructions
46/// corresponding to the block arguments.
47static Block *getPhiIncomingBlock(Block *block) {
48 // If the predecessor block in question is the entry block for a
49 // spirv.mlir.loop, we jump to this spirv.mlir.loop from its enclosing block.
50 if (block->isEntryBlock()) {
51 if (auto loopOp = dyn_cast<spirv::LoopOp>(Val: block->getParentOp())) {
52 // Then the incoming parent block for OpPhi should be the merge block of
53 // the structured control flow op before this loop.
54 Operation *op = loopOp.getOperation();
55 while ((op = op->getPrevNode()) != nullptr)
56 if (Block *incomingBlock = getStructuredControlFlowOpMergeBlock(op))
57 return incomingBlock;
58 // Or the enclosing block itself if no structured control flow ops
59 // exists before this loop.
60 return loopOp->getBlock();
61 }
62 }
63
64 // Otherwise, we jump from the given predecessor block. Try to see if there is
65 // a structured control flow op inside it.
66 for (Operation &op : llvm::reverse(C&: block->getOperations())) {
67 if (Block *incomingBlock = getStructuredControlFlowOpMergeBlock(op: &op))
68 return incomingBlock;
69 }
70 return block;
71}
72
73static bool isZeroValue(Attribute attr) {
74 if (auto floatAttr = dyn_cast<FloatAttr>(Val&: attr)) {
75 return floatAttr.getValue().isZero();
76 }
77 if (auto boolAttr = dyn_cast<BoolAttr>(Val&: attr)) {
78 return !boolAttr.getValue();
79 }
80 if (auto intAttr = dyn_cast<IntegerAttr>(Val&: attr)) {
81 return intAttr.getValue().isZero();
82 }
83 if (auto splatElemAttr = dyn_cast<SplatElementsAttr>(Val&: attr)) {
84 return isZeroValue(attr: splatElemAttr.getSplatValue<Attribute>());
85 }
86 if (auto denseElemAttr = dyn_cast<DenseElementsAttr>(Val&: attr)) {
87 return all_of(Range: denseElemAttr.getValues<Attribute>(), P: isZeroValue);
88 }
89 return false;
90}
91
92/// Move all functions declaration before functions definitions. In SPIR-V
93/// "declarations" are functions without a body and "definitions" functions
94/// with a body. This is stronger than necessary. It should be sufficient to
95/// ensure any declarations precede their uses and not all definitions, however
96/// this allows to avoid analysing every function in the module this way.
97static void moveFuncDeclarationsToTop(spirv::ModuleOp moduleOp) {
98 Block::OpListType &ops = moduleOp.getBody()->getOperations();
99 if (ops.empty())
100 return;
101 Operation &firstOp = ops.front();
102 for (Operation &op : llvm::drop_begin(RangeOrContainer&: ops))
103 if (auto funcOp = dyn_cast<spirv::FuncOp>(Val&: op))
104 if (funcOp.getBody().empty())
105 funcOp->moveBefore(existingOp: &firstOp);
106}
107
108namespace mlir {
109namespace spirv {
110
111/// Encodes an SPIR-V instruction with the given `opcode` and `operands` into
112/// the given `binary` vector.
113void encodeInstructionInto(SmallVectorImpl<uint32_t> &binary, spirv::Opcode op,
114 ArrayRef<uint32_t> operands) {
115 uint32_t wordCount = 1 + operands.size();
116 binary.push_back(Elt: spirv::getPrefixedOpcode(wordCount, opcode: op));
117 binary.append(in_start: operands.begin(), in_end: operands.end());
118}
119
120Serializer::Serializer(spirv::ModuleOp module,
121 const SerializationOptions &options)
122 : module(module), mlirBuilder(module.getContext()), options(options) {}
123
124LogicalResult Serializer::serialize() {
125 LLVM_DEBUG(llvm::dbgs() << "+++ starting serialization +++\n");
126
127 if (failed(Result: module.verifyInvariants()))
128 return failure();
129
130 // TODO: handle the other sections
131 processCapability();
132 if (failed(Result: processExtension())) {
133 return failure();
134 }
135 processMemoryModel();
136 processDebugInfo();
137
138 moveFuncDeclarationsToTop(moduleOp: module);
139
140 // Iterate over the module body to serialize it. Assumptions are that there is
141 // only one basic block in the moduleOp
142 for (auto &op : *module.getBody()) {
143 if (failed(Result: processOperation(op: &op))) {
144 return failure();
145 }
146 }
147
148 LLVM_DEBUG(llvm::dbgs() << "+++ completed serialization +++\n");
149 return success();
150}
151
152void Serializer::collect(SmallVectorImpl<uint32_t> &binary) {
153 auto moduleSize = spirv::kHeaderWordCount + capabilities.size() +
154 extensions.size() + extendedSets.size() +
155 memoryModel.size() + entryPoints.size() +
156 executionModes.size() + decorations.size() +
157 typesGlobalValues.size() + functions.size() + graphs.size();
158
159 binary.clear();
160 binary.reserve(N: moduleSize);
161
162 spirv::appendModuleHeader(header&: binary, version: module.getVceTriple()->getVersion(),
163 idBound: nextID);
164 binary.append(in_start: capabilities.begin(), in_end: capabilities.end());
165 binary.append(in_start: extensions.begin(), in_end: extensions.end());
166 binary.append(in_start: extendedSets.begin(), in_end: extendedSets.end());
167 binary.append(in_start: memoryModel.begin(), in_end: memoryModel.end());
168 binary.append(in_start: entryPoints.begin(), in_end: entryPoints.end());
169 binary.append(in_start: executionModes.begin(), in_end: executionModes.end());
170 binary.append(in_start: debug.begin(), in_end: debug.end());
171 binary.append(in_start: names.begin(), in_end: names.end());
172 binary.append(in_start: decorations.begin(), in_end: decorations.end());
173 binary.append(in_start: typesGlobalValues.begin(), in_end: typesGlobalValues.end());
174 binary.append(in_start: functions.begin(), in_end: functions.end());
175 binary.append(in_start: graphs.begin(), in_end: graphs.end());
176 binary.append(in_start: graphsDebugInfo.begin(), in_end: graphsDebugInfo.end());
177}
178
179#ifndef NDEBUG
180void Serializer::printValueIDMap(raw_ostream &os) {
181 os << "\n= Value <id> Map =\n\n";
182 for (auto valueIDPair : valueIDMap) {
183 Value val = valueIDPair.first;
184 os << " " << val << " "
185 << "id = " << valueIDPair.second << ' ';
186 if (auto *op = val.getDefiningOp()) {
187 os << "from op '" << op->getName() << "'";
188 } else if (auto arg = dyn_cast<BlockArgument>(val)) {
189 Block *block = arg.getOwner();
190 os << "from argument of block " << block << ' ';
191 os << " in op '" << block->getParentOp()->getName() << "'";
192 }
193 os << '\n';
194 }
195}
196#endif
197
198//===----------------------------------------------------------------------===//
199// Module structure
200//===----------------------------------------------------------------------===//
201
202uint32_t Serializer::getOrCreateFunctionID(StringRef fnName) {
203 auto funcID = funcIDMap.lookup(Key: fnName);
204 if (!funcID) {
205 funcID = getNextID();
206 funcIDMap[fnName] = funcID;
207 }
208 return funcID;
209}
210
211void Serializer::processCapability() {
212 for (auto cap : module.getVceTriple()->getCapabilities())
213 encodeInstructionInto(binary&: capabilities, op: spirv::Opcode::OpCapability,
214 operands: {static_cast<uint32_t>(cap)});
215}
216
217void Serializer::addLongCompositesCapability() {
218 if (longCompositesEmitted)
219 return;
220 longCompositesEmitted = true;
221 auto vceTriple = module.getVceTriple();
222 if (!llvm::is_contained(Range: vceTriple->getCapabilities(),
223 Element: spirv::Capability::LongCompositesINTEL))
224 encodeInstructionInto(
225 binary&: capabilities, op: spirv::Opcode::OpCapability,
226 operands: {static_cast<uint32_t>(spirv::Capability::LongCompositesINTEL)});
227 if (!llvm::is_contained(Range: vceTriple->getExtensions(),
228 Element: spirv::Extension::SPV_INTEL_long_composites)) {
229 SmallVector<uint32_t, 8> extName;
230 spirv::encodeStringLiteralInto(
231 binary&: extName,
232 literal: spirv::stringifyExtension(spirv::Extension::SPV_INTEL_long_composites));
233 encodeInstructionInto(binary&: extensions, op: spirv::Opcode::OpExtension, operands: extName);
234 }
235}
236
237void Serializer::encodeInstructionWithContinuationInto(
238 SmallVectorImpl<uint32_t> &binary, spirv::Opcode op,
239 ArrayRef<uint32_t> operands) {
240 if (1 + operands.size() <= spirv::kMaxWordCount) {
241 encodeInstructionInto(binary, op, operands);
242 return;
243 }
244
245 std::optional<spirv::Opcode> continuationOp =
246 spirv::getContinuationOpcode(parent: op);
247 assert(continuationOp && "op is not a splittable composite/struct opcode");
248
249 const unsigned chunk = spirv::kMaxWordCount - 1;
250 encodeInstructionInto(binary, op, operands: operands.take_front(N: chunk));
251 for (ArrayRef<uint32_t> rest = operands.drop_front(N: chunk); !rest.empty();
252 rest = rest.drop_front(N: std::min<size_t>(a: rest.size(), b: chunk))) {
253 encodeInstructionInto(binary, op: *continuationOp, operands: rest.take_front(N: chunk));
254 }
255
256 addLongCompositesCapability();
257}
258
259void Serializer::processDebugInfo() {
260 if (!options.emitDebugInfo)
261 return;
262 auto fileLoc = dyn_cast<FileLineColLoc>(Val: module.getLoc());
263 auto fileName = fileLoc ? fileLoc.getFilename().strref() : "<unknown>";
264 fileID = getNextID();
265 SmallVector<uint32_t, 16> operands;
266 operands.push_back(Elt: fileID);
267 spirv::encodeStringLiteralInto(binary&: operands, literal: fileName);
268 encodeInstructionInto(binary&: debug, op: spirv::Opcode::OpString, operands);
269 // TODO: Encode more debug instructions.
270}
271
272LogicalResult Serializer::processExtension() {
273 llvm::SmallVector<uint32_t, 16> extName;
274 llvm::SmallSet<Extension, 4> deducedExts(
275 llvm::from_range, module.getVceTriple()->getExtensions());
276 auto nonSemanticInfoExt = spirv::Extension::SPV_KHR_non_semantic_info;
277 if (options.emitDebugInfo && !deducedExts.contains(V: nonSemanticInfoExt)) {
278 TargetEnvAttr targetEnvAttr = lookupTargetEnvOrDefault(op: module);
279 if (!is_contained(Range: targetEnvAttr.getExtensions(), Element: nonSemanticInfoExt))
280 return module.emitError(
281 message: "SPV_KHR_non_semantic_info extension not available");
282 deducedExts.insert(V: nonSemanticInfoExt);
283 }
284 for (spirv::Extension ext : deducedExts) {
285 extName.clear();
286 spirv::encodeStringLiteralInto(binary&: extName, literal: spirv::stringifyExtension(ext));
287 encodeInstructionInto(binary&: extensions, op: spirv::Opcode::OpExtension, operands: extName);
288 }
289 return success();
290}
291
292void Serializer::processMemoryModel() {
293 auto mm = static_cast<uint32_t>(module.getMemoryModel());
294 auto am = static_cast<uint32_t>(module.getAddressingModel());
295
296 encodeInstructionInto(binary&: memoryModel, op: spirv::Opcode::OpMemoryModel, operands: {am, mm});
297}
298
299static std::string getDecorationName(StringRef attrName) {
300 // convertToCamelFromSnakeCase will convert this to FpFastMathMode instead of
301 // expected FPFastMathMode.
302 if (attrName == "fp_fast_math_mode")
303 return "FPFastMathMode";
304 // similar here
305 if (attrName == "fp_rounding_mode")
306 return "FPRoundingMode";
307 // convertToCamelFromSnakeCase will not capitalize "INTEL".
308 if (attrName == "cache_control_load_intel")
309 return "CacheControlLoadINTEL";
310 if (attrName == "cache_control_store_intel")
311 return "CacheControlStoreINTEL";
312
313 return llvm::convertToCamelFromSnakeCase(input: attrName, /*capitalizeFirst=*/true);
314}
315
316template <typename AttrTy, typename EmitF>
317static LogicalResult processDecorationList(Location loc, Decoration decoration,
318 Attribute attrList,
319 StringRef attrName, EmitF emitter) {
320 auto arrayAttr = dyn_cast<ArrayAttr>(Val&: attrList);
321 if (!arrayAttr) {
322 return emitError(loc, message: "expecting array attribute of ")
323 << attrName << " for " << stringifyDecoration(decoration);
324 }
325 if (arrayAttr.empty()) {
326 return emitError(loc, message: "expecting non-empty array attribute of ")
327 << attrName << " for " << stringifyDecoration(decoration);
328 }
329 for (Attribute attr : arrayAttr.getValue()) {
330 auto cacheControlAttr = dyn_cast<AttrTy>(attr);
331 if (!cacheControlAttr) {
332 return emitError(loc, message: "expecting array attribute of ")
333 << attrName << " for " << stringifyDecoration(decoration);
334 }
335 // This named attribute encodes several decorations. Emit one per
336 // element in the array.
337 if (failed(emitter(cacheControlAttr)))
338 return failure();
339 }
340 return success();
341}
342
343LogicalResult Serializer::processDecorationAttr(Location loc, uint32_t resultID,
344 Decoration decoration,
345 Attribute attr) {
346 SmallVector<uint32_t, 1> args;
347 switch (decoration) {
348 case spirv::Decoration::LinkageAttributes: {
349 // Get the value of the Linkage Attributes
350 // e.g., LinkageAttributes=["linkageName", linkageType].
351 auto linkageAttr = dyn_cast<spirv::LinkageAttributesAttr>(Val&: attr);
352 auto linkageName = linkageAttr.getLinkageName();
353 auto linkageType = linkageAttr.getLinkageType().getValue();
354 // Encode the Linkage Name (string literal to uint32_t).
355 spirv::encodeStringLiteralInto(binary&: args, literal: linkageName);
356 // Encode LinkageType & Add the Linkagetype to the args.
357 args.push_back(Elt: static_cast<uint32_t>(linkageType));
358 break;
359 }
360 case spirv::Decoration::FPFastMathMode:
361 if (auto intAttr = dyn_cast<FPFastMathModeAttr>(Val&: attr)) {
362 args.push_back(Elt: static_cast<uint32_t>(intAttr.getValue()));
363 break;
364 }
365 return emitError(loc, message: "expected FPFastMathModeAttr attribute for ")
366 << stringifyDecoration(decoration);
367 case spirv::Decoration::FPRoundingMode:
368 if (auto intAttr = dyn_cast<FPRoundingModeAttr>(Val&: attr)) {
369 args.push_back(Elt: static_cast<uint32_t>(intAttr.getValue()));
370 break;
371 }
372 return emitError(loc, message: "expected FPRoundingModeAttr attribute for ")
373 << stringifyDecoration(decoration);
374 case spirv::Decoration::FuncParamAttr:
375 return processDecorationList<FunctionParameterAttributeAttr>(
376 loc, decoration, attrList: attr, attrName: "FunctionParameterAttribute",
377 emitter: [&](FunctionParameterAttributeAttr attr) {
378 return emitDecoration(target: resultID, decoration,
379 params: {static_cast<uint32_t>(attr.getValue())});
380 });
381 case spirv::Decoration::Alignment:
382 case spirv::Decoration::MaxByteOffset:
383 case spirv::Decoration::Binding:
384 case spirv::Decoration::DescriptorSet:
385 case spirv::Decoration::Location:
386 case spirv::Decoration::Index:
387 case spirv::Decoration::Offset:
388 case spirv::Decoration::XfbBuffer:
389 case spirv::Decoration::XfbStride:
390 if (auto intAttr = dyn_cast<IntegerAttr>(Val&: attr)) {
391 args.push_back(Elt: intAttr.getValue().getZExtValue());
392 break;
393 }
394 return emitError(loc, message: "expected integer attribute for ")
395 << stringifyDecoration(decoration);
396 case spirv::Decoration::BuiltIn:
397 if (auto strAttr = dyn_cast<StringAttr>(Val&: attr)) {
398 auto enumVal = spirv::symbolizeBuiltIn(strAttr.getValue());
399 if (enumVal) {
400 args.push_back(Elt: static_cast<uint32_t>(*enumVal));
401 break;
402 }
403 return emitError(loc, message: "invalid ")
404 << stringifyDecoration(decoration) << " decoration attribute "
405 << strAttr.getValue();
406 }
407 return emitError(loc, message: "expected string attribute for ")
408 << stringifyDecoration(decoration);
409 case spirv::Decoration::Aliased:
410 case spirv::Decoration::AliasedPointer:
411 case spirv::Decoration::Flat:
412 case spirv::Decoration::NonReadable:
413 case spirv::Decoration::NonWritable:
414 case spirv::Decoration::NoPerspective:
415 case spirv::Decoration::NoSignedWrap:
416 case spirv::Decoration::NoUnsignedWrap:
417 case spirv::Decoration::RelaxedPrecision:
418 case spirv::Decoration::Restrict:
419 case spirv::Decoration::RestrictPointer:
420 case spirv::Decoration::NoContraction:
421 case spirv::Decoration::Constant:
422 case spirv::Decoration::Block:
423 case spirv::Decoration::BufferBlock:
424 case spirv::Decoration::CPacked:
425 case spirv::Decoration::Invariant:
426 case spirv::Decoration::Patch:
427 case spirv::Decoration::Coherent:
428 case spirv::Decoration::Volatile:
429 case spirv::Decoration::SaturatedConversion:
430 // For unit attributes and decoration attributes, the args list
431 // has no values so we do nothing.
432 if (isa<UnitAttr, DecorationAttr>(Val: attr))
433 break;
434 return emitError(loc,
435 message: "expected unit attribute or decoration attribute for ")
436 << stringifyDecoration(decoration);
437 case spirv::Decoration::CacheControlLoadINTEL:
438 return processDecorationList<CacheControlLoadINTELAttr>(
439 loc, decoration, attrList: attr, attrName: "CacheControlLoadINTEL",
440 emitter: [&](CacheControlLoadINTELAttr attr) {
441 unsigned cacheLevel = attr.getCacheLevel();
442 LoadCacheControl loadCacheControl = attr.getLoadCacheControl();
443 return emitDecoration(
444 target: resultID, decoration,
445 params: {cacheLevel, static_cast<uint32_t>(loadCacheControl)});
446 });
447 case spirv::Decoration::CacheControlStoreINTEL:
448 return processDecorationList<CacheControlStoreINTELAttr>(
449 loc, decoration, attrList: attr, attrName: "CacheControlStoreINTEL",
450 emitter: [&](CacheControlStoreINTELAttr attr) {
451 unsigned cacheLevel = attr.getCacheLevel();
452 StoreCacheControl storeCacheControl = attr.getStoreCacheControl();
453 return emitDecoration(
454 target: resultID, decoration,
455 params: {cacheLevel, static_cast<uint32_t>(storeCacheControl)});
456 });
457 case spirv::Decoration::AlignmentId:
458 case spirv::Decoration::MaxByteOffsetId:
459 case spirv::Decoration::CounterBuffer: {
460 auto symRef = dyn_cast<FlatSymbolRefAttr>(Val&: attr);
461 if (!symRef)
462 return emitError(loc, message: "expected symbol reference for ")
463 << stringifyDecoration(decoration);
464 StringRef symName = symRef.getValue();
465 uint32_t operandID = getVariableID(varName: symName);
466 if (!operandID)
467 operandID = getSpecConstID(constName: symName);
468 if (!operandID)
469 return emitError(loc, message: "could not find <id> for symbol '")
470 << symName << "' referenced by "
471 << stringifyDecoration(decoration);
472 return emitDecorationId(target: resultID, decoration, operandIds: {operandID});
473 }
474 default:
475 return emitError(loc, message: "unhandled decoration ")
476 << stringifyDecoration(decoration);
477 }
478 return emitDecoration(target: resultID, decoration, params: args);
479}
480
481LogicalResult Serializer::processDecoration(Location loc, uint32_t resultID,
482 NamedAttribute attr) {
483 StringRef attrName = attr.getName().strref();
484 std::string decorationName = getDecorationName(attrName);
485 std::optional<Decoration> decoration =
486 spirv::symbolizeDecoration(decorationName);
487 if (!decoration) {
488 return emitError(
489 loc, message: "non-argument attributes expected to have snake-case-ified "
490 "decoration name, unhandled attribute with name : ")
491 << attrName;
492 }
493 return processDecorationAttr(loc, resultID, decoration: *decoration, attr: attr.getValue());
494}
495
496LogicalResult Serializer::processName(uint32_t resultID, StringRef name) {
497 assert(!name.empty() && "unexpected empty string for OpName");
498 if (!options.emitSymbolName)
499 return success();
500
501 SmallVector<uint32_t, 4> nameOperands;
502 nameOperands.push_back(Elt: resultID);
503 spirv::encodeStringLiteralInto(binary&: nameOperands, literal: name);
504 encodeInstructionInto(binary&: names, op: spirv::Opcode::OpName, operands: nameOperands);
505 return success();
506}
507
508template <>
509LogicalResult Serializer::processTypeDecoration<spirv::ArrayType>(
510 Location loc, spirv::ArrayType type, uint32_t resultID) {
511 if (unsigned stride = type.getArrayStride()) {
512 // OpDecorate %arrayTypeSSA ArrayStride strideLiteral
513 return emitDecoration(target: resultID, decoration: spirv::Decoration::ArrayStride, params: {stride});
514 }
515 return success();
516}
517
518template <>
519LogicalResult Serializer::processTypeDecoration<spirv::RuntimeArrayType>(
520 Location loc, spirv::RuntimeArrayType type, uint32_t resultID) {
521 if (unsigned stride = type.getArrayStride()) {
522 // OpDecorate %arrayTypeSSA ArrayStride strideLiteral
523 return emitDecoration(target: resultID, decoration: spirv::Decoration::ArrayStride, params: {stride});
524 }
525 return success();
526}
527
528LogicalResult Serializer::processMemberDecoration(
529 uint32_t structID,
530 const spirv::StructType::MemberDecorationInfo &memberDecoration) {
531 SmallVector<uint32_t, 4> args(
532 {structID, memberDecoration.memberIndex,
533 static_cast<uint32_t>(memberDecoration.decoration)});
534 if (memberDecoration.hasValue()) {
535 args.push_back(
536 Elt: cast<IntegerAttr>(Val: memberDecoration.decorationValue).getInt());
537 }
538 encodeInstructionInto(binary&: decorations, op: spirv::Opcode::OpMemberDecorate, operands: args);
539 return success();
540}
541
542//===----------------------------------------------------------------------===//
543// Type
544//===----------------------------------------------------------------------===//
545
546// According to the SPIR-V spec "Validation Rules for Shader Capabilities":
547// "Composite objects in the StorageBuffer, PhysicalStorageBuffer, Uniform, and
548// PushConstant Storage Classes must be explicitly laid out."
549bool Serializer::isInterfaceStructPtrType(Type type) const {
550 if (auto ptrType = dyn_cast<spirv::PointerType>(Val&: type)) {
551 switch (ptrType.getStorageClass()) {
552 case spirv::StorageClass::PhysicalStorageBuffer:
553 case spirv::StorageClass::PushConstant:
554 case spirv::StorageClass::StorageBuffer:
555 case spirv::StorageClass::Uniform:
556 return isa<spirv::StructType>(Val: ptrType.getPointeeType());
557 default:
558 break;
559 }
560 }
561 return false;
562}
563
564LogicalResult Serializer::processType(Location loc, Type type,
565 uint32_t &typeID) {
566 // Maintains a set of names for nested identified struct types. This is used
567 // to properly serialize recursive references.
568 SetVector<StringRef> serializationCtx;
569 return processTypeImpl(loc, type, typeID, serializationCtx);
570}
571
572LogicalResult
573Serializer::processTypeImpl(Location loc, Type type, uint32_t &typeID,
574 SetVector<StringRef> &serializationCtx) {
575
576 // Map unsigned integer types to singless integer types.
577 // This is needed otherwise the generated spirv assembly will contain
578 // twice a type declaration (like OpTypeInt 32 0) which is no permitted and
579 // such module fails validation. Indeed at MLIR level the two types are
580 // different and lookup in the cache below misses.
581 // Note: This conversion needs to happen here before the type is looked up in
582 // the cache.
583 if (type.isUnsignedInteger()) {
584 type = IntegerType::get(context: loc->getContext(), width: type.getIntOrFloatBitWidth(),
585 signedness: IntegerType::SignednessSemantics::Signless);
586 }
587
588 typeID = getTypeID(type);
589 if (typeID)
590 return success();
591
592 typeID = getNextID();
593 SmallVector<uint32_t, 4> operands;
594
595 operands.push_back(Elt: typeID);
596 auto typeEnum = spirv::Opcode::OpTypeVoid;
597 bool deferSerialization = false;
598
599 if ((isa<FunctionType>(Val: type) &&
600 succeeded(Result: prepareFunctionType(loc, type: cast<FunctionType>(Val&: type), typeEnum,
601 operands))) ||
602 (isa<GraphType>(Val: type) &&
603 succeeded(
604 Result: prepareGraphType(loc, type: cast<GraphType>(Val&: type), typeEnum, operands))) ||
605 succeeded(Result: prepareBasicType(loc, type, resultID: typeID, typeEnum, operands,
606 deferSerialization, serializationCtx))) {
607 if (deferSerialization)
608 return success();
609
610 typeIDMap[type] = typeID;
611
612 if (typeEnum == spirv::Opcode::OpTypeStruct)
613 encodeInstructionWithContinuationInto(binary&: typesGlobalValues, op: typeEnum,
614 operands);
615 else
616 encodeInstructionInto(binary&: typesGlobalValues, op: typeEnum, operands);
617
618 if (recursiveStructInfos.count(Val: type) != 0) {
619 // This recursive struct type is emitted already, now the OpTypePointer
620 // instructions referring to recursive references are emitted as well.
621 for (auto &ptrInfo : recursiveStructInfos[type]) {
622 // TODO: This might not work if more than 1 recursive reference is
623 // present in the struct.
624 SmallVector<uint32_t, 4> ptrOperands;
625 ptrOperands.push_back(Elt: ptrInfo.pointerTypeID);
626 ptrOperands.push_back(Elt: static_cast<uint32_t>(ptrInfo.storageClass));
627 ptrOperands.push_back(Elt: typeIDMap[type]);
628
629 encodeInstructionInto(binary&: typesGlobalValues, op: spirv::Opcode::OpTypePointer,
630 operands: ptrOperands);
631 }
632
633 recursiveStructInfos[type].clear();
634 }
635
636 return success();
637 }
638
639 return emitError(loc, message: "failed to process type: ") << type;
640}
641
642LogicalResult Serializer::prepareBasicType(
643 Location loc, Type type, uint32_t resultID, spirv::Opcode &typeEnum,
644 SmallVectorImpl<uint32_t> &operands, bool &deferSerialization,
645 SetVector<StringRef> &serializationCtx) {
646 deferSerialization = false;
647
648 if (isVoidType(type)) {
649 typeEnum = spirv::Opcode::OpTypeVoid;
650 return success();
651 }
652
653 if (auto intType = dyn_cast<IntegerType>(Val&: type)) {
654 if (intType.getWidth() == 1) {
655 typeEnum = spirv::Opcode::OpTypeBool;
656 return success();
657 }
658
659 typeEnum = spirv::Opcode::OpTypeInt;
660 operands.push_back(Elt: intType.getWidth());
661 // SPIR-V OpTypeInt "Signedness specifies whether there are signed semantics
662 // to preserve or validate.
663 // 0 indicates unsigned, or no signedness semantics
664 // 1 indicates signed semantics."
665 operands.push_back(Elt: intType.isSigned() ? 1 : 0);
666 return success();
667 }
668
669 if (auto floatType = dyn_cast<FloatType>(Val&: type)) {
670 typeEnum = spirv::Opcode::OpTypeFloat;
671 operands.push_back(Elt: floatType.getWidth());
672 if (floatType.isBF16()) {
673 operands.push_back(Elt: static_cast<uint32_t>(spirv::FPEncoding::BFloat16KHR));
674 }
675 if (floatType.isF8E4M3FN()) {
676 operands.push_back(
677 Elt: static_cast<uint32_t>(spirv::FPEncoding::Float8E4M3EXT));
678 }
679 if (floatType.isF8E5M2()) {
680 operands.push_back(
681 Elt: static_cast<uint32_t>(spirv::FPEncoding::Float8E5M2EXT));
682 }
683
684 return success();
685 }
686
687 if (auto vectorType = dyn_cast<VectorType>(Val&: type)) {
688 uint32_t elementTypeID = 0;
689 if (failed(Result: processTypeImpl(loc, type: vectorType.getElementType(), typeID&: elementTypeID,
690 serializationCtx))) {
691 return failure();
692 }
693 typeEnum = spirv::Opcode::OpTypeVector;
694 operands.push_back(Elt: elementTypeID);
695 operands.push_back(Elt: vectorType.getNumElements());
696 return success();
697 }
698
699 if (auto imageType = dyn_cast<spirv::ImageType>(Val&: type)) {
700 typeEnum = spirv::Opcode::OpTypeImage;
701 uint32_t sampledTypeID = 0;
702 if (failed(Result: processType(loc, type: imageType.getElementType(), typeID&: sampledTypeID)))
703 return failure();
704
705 llvm::append_values(C&: operands, Values&: sampledTypeID,
706 Values: static_cast<uint32_t>(imageType.getDim()),
707 Values: static_cast<uint32_t>(imageType.getDepthInfo()),
708 Values: static_cast<uint32_t>(imageType.getArrayedInfo()),
709 Values: static_cast<uint32_t>(imageType.getSamplingInfo()),
710 Values: static_cast<uint32_t>(imageType.getSamplerUseInfo()),
711 Values: static_cast<uint32_t>(imageType.getImageFormat()));
712 return success();
713 }
714
715 if (auto arrayType = dyn_cast<spirv::ArrayType>(Val&: type)) {
716 typeEnum = spirv::Opcode::OpTypeArray;
717 uint32_t elementTypeID = 0;
718 if (failed(Result: processTypeImpl(loc, type: arrayType.getElementType(), typeID&: elementTypeID,
719 serializationCtx))) {
720 return failure();
721 }
722 operands.push_back(Elt: elementTypeID);
723 if (auto elementCountID = prepareConstantInt(
724 loc, intAttr: mlirBuilder.getI32IntegerAttr(value: arrayType.getNumElements()))) {
725 operands.push_back(Elt: elementCountID);
726 }
727 return processTypeDecoration(loc, type: arrayType, resultID);
728 }
729
730 if (auto ptrType = dyn_cast<spirv::PointerType>(Val&: type)) {
731 uint32_t pointeeTypeID = 0;
732 spirv::StructType pointeeStruct =
733 dyn_cast<spirv::StructType>(Val: ptrType.getPointeeType());
734
735 if (pointeeStruct && pointeeStruct.isIdentified() &&
736 serializationCtx.count(key: pointeeStruct.getIdentifier()) != 0) {
737 // A recursive reference to an enclosing struct is found.
738 //
739 // 1. Prepare an OpTypeForwardPointer with resultID and the ptr storage
740 // class as operands.
741 SmallVector<uint32_t, 2> forwardPtrOperands;
742 forwardPtrOperands.push_back(Elt: resultID);
743 forwardPtrOperands.push_back(
744 Elt: static_cast<uint32_t>(ptrType.getStorageClass()));
745
746 encodeInstructionInto(binary&: typesGlobalValues,
747 op: spirv::Opcode::OpTypeForwardPointer,
748 operands: forwardPtrOperands);
749
750 // 2. Find the pointee (enclosing) struct.
751 auto structType = spirv::StructType::getIdentified(
752 context: module.getContext(), identifier: pointeeStruct.getIdentifier());
753
754 if (!structType)
755 return failure();
756
757 // 3. Mark the OpTypePointer that is supposed to be emitted by this call
758 // as deferred.
759 deferSerialization = true;
760
761 // 4. Record the info needed to emit the deferred OpTypePointer
762 // instruction when the enclosing struct is completely serialized.
763 recursiveStructInfos[structType].push_back(
764 Elt: {.pointerTypeID: resultID, .storageClass: ptrType.getStorageClass()});
765 } else {
766 if (failed(Result: processTypeImpl(loc, type: ptrType.getPointeeType(), typeID&: pointeeTypeID,
767 serializationCtx)))
768 return failure();
769 }
770
771 typeEnum = spirv::Opcode::OpTypePointer;
772 operands.push_back(Elt: static_cast<uint32_t>(ptrType.getStorageClass()));
773 operands.push_back(Elt: pointeeTypeID);
774
775 // TODO: Now struct decorations are supported this code may not be
776 // necessary. However, it is left to support backwards compatibility.
777 // Ideally, Block decorations should be inserted when converting to SPIR-V.
778 if (isInterfaceStructPtrType(type: ptrType)) {
779 auto structType = cast<spirv::StructType>(Val: ptrType.getPointeeType());
780 if (!structType.hasDecoration(decoration: spirv::Decoration::Block) &&
781 !structType.hasDecoration(decoration: spirv::Decoration::BufferBlock))
782 if (failed(Result: emitDecoration(target: getTypeID(type: pointeeStruct),
783 decoration: spirv::Decoration::Block)))
784 return emitError(loc, message: "cannot decorate ")
785 << pointeeStruct << " with Block decoration";
786 }
787
788 return success();
789 }
790
791 if (auto runtimeArrayType = dyn_cast<spirv::RuntimeArrayType>(Val&: type)) {
792 uint32_t elementTypeID = 0;
793 if (failed(Result: processTypeImpl(loc, type: runtimeArrayType.getElementType(),
794 typeID&: elementTypeID, serializationCtx))) {
795 return failure();
796 }
797 typeEnum = spirv::Opcode::OpTypeRuntimeArray;
798 operands.push_back(Elt: elementTypeID);
799 return processTypeDecoration(loc, type: runtimeArrayType, resultID);
800 }
801
802 if (isa<spirv::SamplerType>(Val: type)) {
803 typeEnum = spirv::Opcode::OpTypeSampler;
804 return success();
805 }
806
807 if (isa<spirv::NamedBarrierType>(Val: type)) {
808 typeEnum = spirv::Opcode::OpTypeNamedBarrier;
809 return success();
810 }
811
812 if (auto sampledImageType = dyn_cast<spirv::SampledImageType>(Val&: type)) {
813 typeEnum = spirv::Opcode::OpTypeSampledImage;
814 uint32_t imageTypeID = 0;
815 if (failed(
816 Result: processType(loc, type: sampledImageType.getImageType(), typeID&: imageTypeID))) {
817 return failure();
818 }
819 operands.push_back(Elt: imageTypeID);
820 return success();
821 }
822
823 if (auto structType = dyn_cast<spirv::StructType>(Val&: type)) {
824 if (structType.isIdentified()) {
825 if (failed(Result: processName(resultID, name: structType.getIdentifier())))
826 return failure();
827 serializationCtx.insert(X: structType.getIdentifier());
828 }
829
830 bool hasOffset = structType.hasOffset();
831 for (auto elementIndex :
832 llvm::seq<uint32_t>(Begin: 0, End: structType.getNumElements())) {
833 uint32_t elementTypeID = 0;
834 if (failed(Result: processTypeImpl(loc, type: structType.getElementType(elementIndex),
835 typeID&: elementTypeID, serializationCtx))) {
836 return failure();
837 }
838 operands.push_back(Elt: elementTypeID);
839 if (hasOffset) {
840 auto intType = IntegerType::get(context: structType.getContext(), width: 32);
841 // Decorate each struct member with an offset
842 spirv::StructType::MemberDecorationInfo offsetDecoration{
843 elementIndex, spirv::Decoration::Offset,
844 IntegerAttr::get(type: intType,
845 value: structType.getMemberOffset(elementIndex))};
846 if (failed(Result: processMemberDecoration(structID: resultID, memberDecoration: offsetDecoration))) {
847 return emitError(loc, message: "cannot decorate ")
848 << elementIndex << "-th member of " << structType
849 << " with its offset";
850 }
851 }
852 }
853 SmallVector<spirv::StructType::MemberDecorationInfo, 4> memberDecorations;
854 structType.getMemberDecorations(memberDecorations);
855
856 for (auto &memberDecoration : memberDecorations) {
857 if (failed(Result: processMemberDecoration(structID: resultID, memberDecoration))) {
858 return emitError(loc, message: "cannot decorate ")
859 << static_cast<uint32_t>(memberDecoration.memberIndex)
860 << "-th member of " << structType << " with "
861 << stringifyDecoration(memberDecoration.decoration);
862 }
863 }
864
865 SmallVector<spirv::StructType::StructDecorationInfo, 1> structDecorations;
866 structType.getStructDecorations(structDecorations);
867
868 for (spirv::StructType::StructDecorationInfo &structDecoration :
869 structDecorations) {
870 if (failed(Result: processDecorationAttr(loc, resultID,
871 decoration: structDecoration.decoration,
872 attr: structDecoration.decorationValue))) {
873 return emitError(loc, message: "cannot decorate struct ")
874 << structType << " with "
875 << stringifyDecoration(structDecoration.decoration);
876 }
877 }
878
879 typeEnum = spirv::Opcode::OpTypeStruct;
880
881 if (structType.isIdentified())
882 serializationCtx.remove(X: structType.getIdentifier());
883
884 return success();
885 }
886
887 if (auto cooperativeMatrixType =
888 dyn_cast<spirv::CooperativeMatrixType>(Val&: type)) {
889 uint32_t elementTypeID = 0;
890 if (failed(Result: processTypeImpl(loc, type: cooperativeMatrixType.getElementType(),
891 typeID&: elementTypeID, serializationCtx))) {
892 return failure();
893 }
894 typeEnum = spirv::Opcode::OpTypeCooperativeMatrixKHR;
895 auto getConstantOp = [&](uint32_t id) {
896 auto attr = IntegerAttr::get(type: IntegerType::get(context: type.getContext(), width: 32), value: id);
897 return prepareConstantInt(loc, intAttr: attr);
898 };
899 llvm::append_values(
900 C&: operands, Values&: elementTypeID,
901 Values: getConstantOp(static_cast<uint32_t>(cooperativeMatrixType.getScope())),
902 Values: getConstantOp(cooperativeMatrixType.getRows()),
903 Values: getConstantOp(cooperativeMatrixType.getColumns()),
904 Values: getConstantOp(static_cast<uint32_t>(cooperativeMatrixType.getUse())));
905 return success();
906 }
907
908 if (auto matrixType = dyn_cast<spirv::MatrixType>(Val&: type)) {
909 uint32_t elementTypeID = 0;
910 if (failed(Result: processTypeImpl(loc, type: matrixType.getColumnType(), typeID&: elementTypeID,
911 serializationCtx))) {
912 return failure();
913 }
914 typeEnum = spirv::Opcode::OpTypeMatrix;
915 llvm::append_values(C&: operands, Values&: elementTypeID, Values: matrixType.getNumColumns());
916 return success();
917 }
918
919 if (auto tensorArmType = dyn_cast<TensorArmType>(Val&: type)) {
920 uint32_t elementTypeID = 0;
921 uint32_t rank = 0;
922 uint32_t shapeID = 0;
923 uint32_t rankID = 0;
924 if (failed(Result: processTypeImpl(loc, type: tensorArmType.getElementType(),
925 typeID&: elementTypeID, serializationCtx))) {
926 return failure();
927 }
928 if (tensorArmType.hasRank()) {
929 ArrayRef<int64_t> dims = tensorArmType.getShape();
930 rank = dims.size();
931 rankID = prepareConstantInt(loc, intAttr: mlirBuilder.getI32IntegerAttr(value: rank));
932 if (rankID == 0) {
933 return failure();
934 }
935
936 bool shaped = llvm::all_of(Range&: dims, P: [](const auto &dim) { return dim > 0; });
937 if (rank > 0 && shaped) {
938 auto I32Type = IntegerType::get(context: type.getContext(), width: 32);
939 auto shapeType = ArrayType::get(elementType: I32Type, elementCount: rank);
940 if (rank == 1) {
941 SmallVector<uint64_t, 1> index(rank);
942 shapeID = prepareDenseElementsConstant(
943 loc, constType: shapeType,
944 valueAttr: mlirBuilder.getI32TensorAttr(values: SmallVector<int32_t>(dims)), dim: 0,
945 index);
946 } else {
947 shapeID = prepareArrayConstant(
948 loc, constType: shapeType,
949 attr: mlirBuilder.getI32ArrayAttr(values: SmallVector<int32_t>(dims)));
950 }
951 if (shapeID == 0) {
952 return failure();
953 }
954 }
955 }
956 typeEnum = spirv::Opcode::OpTypeTensorARM;
957 operands.push_back(Elt: elementTypeID);
958 if (rankID == 0)
959 return success();
960 operands.push_back(Elt: rankID);
961 if (shapeID == 0)
962 return success();
963 operands.push_back(Elt: shapeID);
964 return success();
965 }
966
967 // TODO: Handle other types.
968 return emitError(loc, message: "unhandled type in serialization: ") << type;
969}
970
971LogicalResult
972Serializer::prepareFunctionType(Location loc, FunctionType type,
973 spirv::Opcode &typeEnum,
974 SmallVectorImpl<uint32_t> &operands) {
975 typeEnum = spirv::Opcode::OpTypeFunction;
976 assert(type.getNumResults() <= 1 &&
977 "serialization supports only a single return value");
978 uint32_t resultID = 0;
979 if (failed(Result: processType(
980 loc, type: type.getNumResults() == 1 ? type.getResult(i: 0) : getVoidType(),
981 typeID&: resultID))) {
982 return failure();
983 }
984 operands.push_back(Elt: resultID);
985 for (auto &res : type.getInputs()) {
986 uint32_t argTypeID = 0;
987 if (failed(Result: processType(loc, type: res, typeID&: argTypeID))) {
988 return failure();
989 }
990 operands.push_back(Elt: argTypeID);
991 }
992 return success();
993}
994
995LogicalResult
996Serializer::prepareGraphType(Location loc, GraphType type,
997 spirv::Opcode &typeEnum,
998 SmallVectorImpl<uint32_t> &operands) {
999 typeEnum = spirv::Opcode::OpTypeGraphARM;
1000 assert(type.getNumResults() >= 1 &&
1001 "serialization requires at least a return value");
1002
1003 operands.push_back(Elt: type.getNumInputs());
1004
1005 for (Type argType : type.getInputs()) {
1006 uint32_t argTypeID = 0;
1007 if (failed(Result: processType(loc, type: argType, typeID&: argTypeID)))
1008 return failure();
1009 operands.push_back(Elt: argTypeID);
1010 }
1011
1012 for (Type resType : type.getResults()) {
1013 uint32_t resTypeID = 0;
1014 if (failed(Result: processType(loc, type: resType, typeID&: resTypeID)))
1015 return failure();
1016 operands.push_back(Elt: resTypeID);
1017 }
1018
1019 return success();
1020}
1021
1022//===----------------------------------------------------------------------===//
1023// Constant
1024//===----------------------------------------------------------------------===//
1025
1026uint32_t Serializer::prepareConstant(Location loc, Type constType,
1027 Attribute valueAttr) {
1028 if (auto id = prepareConstantScalar(loc, valueAttr)) {
1029 return id;
1030 }
1031
1032 // This is a composite literal. We need to handle each component separately
1033 // and then emit an OpConstantComposite for the whole.
1034
1035 if (auto id = getConstantID(value: valueAttr)) {
1036 return id;
1037 }
1038
1039 uint32_t typeID = 0;
1040 if (failed(Result: processType(loc, type: constType, typeID))) {
1041 return 0;
1042 }
1043
1044 uint32_t resultID = 0;
1045 if (auto attr = dyn_cast<DenseElementsAttr>(Val&: valueAttr)) {
1046 int rank = dyn_cast<ShapedType>(Val: attr.getType()).getRank();
1047 SmallVector<uint64_t, 4> index(rank);
1048 resultID = prepareDenseElementsConstant(loc, constType, valueAttr: attr,
1049 /*dim=*/0, index);
1050 } else if (auto arrayAttr = dyn_cast<ArrayAttr>(Val&: valueAttr)) {
1051 resultID = prepareArrayConstant(loc, constType, attr: arrayAttr);
1052 } else if (isa<spirv::NullAttr>(Val: valueAttr)) {
1053 resultID = getNextID();
1054 encodeInstructionInto(binary&: typesGlobalValues, op: spirv::Opcode::OpConstantNull,
1055 operands: {typeID, resultID});
1056 }
1057
1058 if (resultID == 0) {
1059 emitError(loc, message: "cannot serialize attribute: ") << valueAttr;
1060 return 0;
1061 }
1062
1063 constIDMap[valueAttr] = resultID;
1064 return resultID;
1065}
1066
1067uint32_t Serializer::prepareArrayConstant(Location loc, Type constType,
1068 ArrayAttr attr) {
1069 uint32_t typeID = 0;
1070 if (failed(Result: processType(loc, type: constType, typeID))) {
1071 return 0;
1072 }
1073
1074 uint32_t resultID = getNextID();
1075 SmallVector<uint32_t, 4> operands = {typeID, resultID};
1076 operands.reserve(N: attr.size() + 2);
1077 spirv::CompositeType compositeType = cast<spirv::CompositeType>(Val&: constType);
1078 for (auto [idx, elementAttr] : llvm::enumerate(First&: attr)) {
1079 if (uint32_t elementID = prepareConstant(
1080 loc, constType: compositeType.getElementType(idx), valueAttr: elementAttr)) {
1081 operands.push_back(Elt: elementID);
1082 } else {
1083 return 0;
1084 }
1085 }
1086 encodeInstructionWithContinuationInto(
1087 binary&: typesGlobalValues, op: spirv::Opcode::OpConstantComposite, operands);
1088
1089 return resultID;
1090}
1091
1092// TODO: Turn the below function into iterative function, instead of
1093// recursive function.
1094uint32_t
1095Serializer::prepareDenseElementsConstant(Location loc, Type constType,
1096 DenseElementsAttr valueAttr, int dim,
1097 MutableArrayRef<uint64_t> index) {
1098 auto shapedType = dyn_cast<ShapedType>(Val: valueAttr.getType());
1099 assert(dim <= shapedType.getRank());
1100 if (shapedType.getRank() == dim) {
1101 if (auto attr = dyn_cast<DenseIntElementsAttr>(Val&: valueAttr)) {
1102 return attr.getType().getElementType().isInteger(width: 1)
1103 ? prepareConstantBool(loc, boolAttr: attr.getValues<BoolAttr>()[index])
1104 : prepareConstantInt(loc,
1105 intAttr: attr.getValues<IntegerAttr>()[index]);
1106 }
1107 if (auto attr = dyn_cast<DenseFPElementsAttr>(Val&: valueAttr)) {
1108 return prepareConstantFp(loc, floatAttr: attr.getValues<FloatAttr>()[index]);
1109 }
1110 return 0;
1111 }
1112
1113 uint32_t typeID = 0;
1114 if (failed(Result: processType(loc, type: constType, typeID))) {
1115 return 0;
1116 }
1117
1118 int64_t numberOfConstituents = shapedType.getDimSize(idx: dim);
1119 uint32_t resultID = getNextID();
1120 SmallVector<uint32_t, 4> operands = {typeID, resultID};
1121 auto elementType = cast<spirv::CompositeType>(Val&: constType).getElementType(0);
1122 if (auto tensorArmType = dyn_cast<spirv::TensorArmType>(Val&: constType)) {
1123 ArrayRef<int64_t> innerShape = tensorArmType.getShape().drop_front();
1124 if (!innerShape.empty())
1125 elementType = spirv::TensorArmType::get(shape: innerShape, elementType);
1126 }
1127
1128 // "If the Result Type is a cooperative matrix type, then there must be only
1129 // one Constituent, with scalar type matching the cooperative matrix Component
1130 // Type, and all components of the matrix are initialized to that value."
1131 // (https://github.khronos.org/SPIRV-Registry/extensions/KHR/SPV_KHR_cooperative_matrix.html)
1132 if (isa<spirv::CooperativeMatrixType>(Val: constType)) {
1133 if (!valueAttr.isSplat()) {
1134 emitError(
1135 loc,
1136 message: "cannot serialize a non-splat value for a cooperative matrix type");
1137 return 0;
1138 }
1139 // numberOfConstituents is 1, so we only need one more elements in the
1140 // SmallVector, so the total is 3 (1 + 2).
1141 operands.reserve(N: 3);
1142 // We set dim directly to `shapedType.getRank()` so the recursive call
1143 // directly returns the scalar type.
1144 if (auto elementID = prepareDenseElementsConstant(
1145 loc, constType: elementType, valueAttr, /*dim=*/shapedType.getRank(), index)) {
1146 operands.push_back(Elt: elementID);
1147 } else {
1148 return 0;
1149 }
1150 } else if (isa<spirv::TensorArmType>(Val: constType) && isZeroValue(attr: valueAttr)) {
1151 encodeInstructionInto(binary&: typesGlobalValues, op: spirv::Opcode::OpConstantNull,
1152 operands: {typeID, resultID});
1153 return resultID;
1154 } else {
1155 operands.reserve(N: numberOfConstituents + 2);
1156 for (int i = 0; i < numberOfConstituents; ++i) {
1157 index[dim] = i;
1158 if (auto elementID = prepareDenseElementsConstant(
1159 loc, constType: elementType, valueAttr, dim: dim + 1, index)) {
1160 operands.push_back(Elt: elementID);
1161 } else {
1162 return 0;
1163 }
1164 }
1165 }
1166 encodeInstructionWithContinuationInto(
1167 binary&: typesGlobalValues, op: spirv::Opcode::OpConstantComposite, operands);
1168
1169 return resultID;
1170}
1171
1172uint32_t Serializer::prepareConstantScalar(Location loc, Attribute valueAttr,
1173 bool isSpec) {
1174 if (auto floatAttr = dyn_cast<FloatAttr>(Val&: valueAttr)) {
1175 return prepareConstantFp(loc, floatAttr, isSpec);
1176 }
1177 if (auto boolAttr = dyn_cast<BoolAttr>(Val&: valueAttr)) {
1178 return prepareConstantBool(loc, boolAttr, isSpec);
1179 }
1180 if (auto intAttr = dyn_cast<IntegerAttr>(Val&: valueAttr)) {
1181 return prepareConstantInt(loc, intAttr, isSpec);
1182 }
1183
1184 return 0;
1185}
1186
1187uint32_t Serializer::prepareConstantBool(Location loc, BoolAttr boolAttr,
1188 bool isSpec) {
1189 if (!isSpec) {
1190 // We can de-duplicate normal constants, but not specialization constants.
1191 if (auto id = getConstantID(value: boolAttr)) {
1192 return id;
1193 }
1194 }
1195
1196 // Process the type for this bool literal
1197 uint32_t typeID = 0;
1198 if (failed(Result: processType(loc, type: cast<IntegerAttr>(Val&: boolAttr).getType(), typeID))) {
1199 return 0;
1200 }
1201
1202 auto resultID = getNextID();
1203 auto opcode = boolAttr.getValue()
1204 ? (isSpec ? spirv::Opcode::OpSpecConstantTrue
1205 : spirv::Opcode::OpConstantTrue)
1206 : (isSpec ? spirv::Opcode::OpSpecConstantFalse
1207 : spirv::Opcode::OpConstantFalse);
1208 encodeInstructionInto(binary&: typesGlobalValues, op: opcode, operands: {typeID, resultID});
1209
1210 if (!isSpec) {
1211 constIDMap[boolAttr] = resultID;
1212 }
1213 return resultID;
1214}
1215
1216uint32_t Serializer::prepareConstantInt(Location loc, IntegerAttr intAttr,
1217 bool isSpec) {
1218 if (!isSpec) {
1219 // We can de-duplicate normal constants, but not specialization constants.
1220 if (auto id = getConstantID(value: intAttr)) {
1221 return id;
1222 }
1223 }
1224
1225 // Process the type for this integer literal
1226 uint32_t typeID = 0;
1227 if (failed(Result: processType(loc, type: intAttr.getType(), typeID))) {
1228 return 0;
1229 }
1230
1231 auto resultID = getNextID();
1232 APInt value = intAttr.getValue();
1233 unsigned bitwidth = value.getBitWidth();
1234 bool isSigned = intAttr.getType().isSignedInteger();
1235 auto opcode =
1236 isSpec ? spirv::Opcode::OpSpecConstant : spirv::Opcode::OpConstant;
1237
1238 switch (bitwidth) {
1239 // According to SPIR-V spec, "When the type's bit width is less than
1240 // 32-bits, the literal's value appears in the low-order bits of the word,
1241 // and the high-order bits must be 0 for a floating-point type, or 0 for an
1242 // integer type with Signedness of 0, or sign extended when Signedness
1243 // is 1."
1244 case 32:
1245 case 16:
1246 case 8: {
1247 uint32_t word = 0;
1248 if (isSigned) {
1249 word = static_cast<int32_t>(value.getSExtValue());
1250 } else {
1251 word = static_cast<uint32_t>(value.getZExtValue());
1252 }
1253 encodeInstructionInto(binary&: typesGlobalValues, op: opcode, operands: {typeID, resultID, word});
1254 } break;
1255 // According to SPIR-V spec: "When the type's bit width is larger than one
1256 // word, the literal’s low-order words appear first."
1257 case 64: {
1258 struct DoubleWord {
1259 uint32_t word1;
1260 uint32_t word2;
1261 } words;
1262 if (isSigned) {
1263 words = llvm::bit_cast<DoubleWord>(from: value.getSExtValue());
1264 } else {
1265 words = llvm::bit_cast<DoubleWord>(from: value.getZExtValue());
1266 }
1267 encodeInstructionInto(binary&: typesGlobalValues, op: opcode,
1268 operands: {typeID, resultID, words.word1, words.word2});
1269 } break;
1270 default: {
1271 std::string valueStr;
1272 llvm::raw_string_ostream rss(valueStr);
1273 value.print(OS&: rss, /*isSigned=*/false);
1274
1275 emitError(loc, message: "cannot serialize ")
1276 << bitwidth << "-bit integer literal: " << valueStr;
1277 return 0;
1278 }
1279 }
1280
1281 if (!isSpec) {
1282 constIDMap[intAttr] = resultID;
1283 }
1284 return resultID;
1285}
1286
1287uint32_t Serializer::prepareGraphConstantId(Location loc, Type graphConstType,
1288 IntegerAttr intAttr) {
1289 // De-duplicate graph constants.
1290 if (uint32_t id = getGraphConstantARMId(value: intAttr)) {
1291 return id;
1292 }
1293
1294 // Process the type for this graph constant.
1295 uint32_t typeID = 0;
1296 if (failed(Result: processType(loc, type: graphConstType, typeID))) {
1297 return 0;
1298 }
1299
1300 uint32_t resultID = getNextID();
1301 APInt value = intAttr.getValue();
1302 unsigned bitwidth = value.getBitWidth();
1303 if (bitwidth > 32) {
1304 emitError(loc, message: "Too wide attribute for OpGraphConstantARM: ")
1305 << bitwidth << " bits";
1306 return 0;
1307 }
1308 bool isSigned = value.isSignedIntN(N: bitwidth);
1309
1310 uint32_t word = 0;
1311 if (isSigned) {
1312 word = static_cast<int32_t>(value.getSExtValue());
1313 } else {
1314 word = static_cast<uint32_t>(value.getZExtValue());
1315 }
1316 encodeInstructionInto(binary&: typesGlobalValues, op: spirv::Opcode::OpGraphConstantARM,
1317 operands: {typeID, resultID, word});
1318 graphConstIDMap[intAttr] = resultID;
1319 return resultID;
1320}
1321
1322uint32_t Serializer::prepareConstantFp(Location loc, FloatAttr floatAttr,
1323 bool isSpec) {
1324 if (!isSpec) {
1325 // We can de-duplicate normal constants, but not specialization constants.
1326 if (auto id = getConstantID(value: floatAttr)) {
1327 return id;
1328 }
1329 }
1330
1331 // Process the type for this float literal
1332 uint32_t typeID = 0;
1333 if (failed(Result: processType(loc, type: floatAttr.getType(), typeID))) {
1334 return 0;
1335 }
1336
1337 auto resultID = getNextID();
1338 APFloat value = floatAttr.getValue();
1339 const llvm::fltSemantics *semantics = &value.getSemantics();
1340
1341 auto opcode =
1342 isSpec ? spirv::Opcode::OpSpecConstant : spirv::Opcode::OpConstant;
1343
1344 if (semantics == &APFloat::IEEEsingle()) {
1345 uint32_t word = llvm::bit_cast<uint32_t>(from: value.convertToFloat());
1346 encodeInstructionInto(binary&: typesGlobalValues, op: opcode, operands: {typeID, resultID, word});
1347 } else if (semantics == &APFloat::IEEEdouble()) {
1348 struct DoubleWord {
1349 uint32_t word1;
1350 uint32_t word2;
1351 } words = llvm::bit_cast<DoubleWord>(from: value.convertToDouble());
1352 encodeInstructionInto(binary&: typesGlobalValues, op: opcode,
1353 operands: {typeID, resultID, words.word1, words.word2});
1354 } else if (llvm::is_contained(Set: {&APFloat::IEEEhalf(), &APFloat::BFloat(),
1355 &APFloat::Float8E4M3FN(),
1356 &APFloat::Float8E5M2()},
1357 Element: semantics)) {
1358 uint32_t word =
1359 static_cast<uint32_t>(value.bitcastToAPInt().getZExtValue());
1360 encodeInstructionInto(binary&: typesGlobalValues, op: opcode, operands: {typeID, resultID, word});
1361 } else {
1362 std::string valueStr;
1363 llvm::raw_string_ostream rss(valueStr);
1364 value.print(rss);
1365
1366 emitError(loc, message: "cannot serialize ")
1367 << floatAttr.getType() << "-typed float literal: " << valueStr;
1368 return 0;
1369 }
1370
1371 if (!isSpec) {
1372 constIDMap[floatAttr] = resultID;
1373 }
1374 return resultID;
1375}
1376
1377// Returns type of attribute. In case of a TypedAttr this will simply return
1378// the type. But for an ArrayAttr which is untyped and can be multidimensional
1379// it creates the ArrayType recursively.
1380static Type getValueType(Attribute attr) {
1381 if (auto typedAttr = dyn_cast<TypedAttr>(Val&: attr)) {
1382 return typedAttr.getType();
1383 }
1384
1385 if (auto arrayAttr = dyn_cast<ArrayAttr>(Val&: attr)) {
1386 return spirv::ArrayType::get(elementType: getValueType(attr: arrayAttr[0]), elementCount: arrayAttr.size());
1387 }
1388
1389 return nullptr;
1390}
1391
1392uint32_t Serializer::prepareConstantCompositeReplicate(Location loc,
1393 Type resultType,
1394 Attribute valueAttr) {
1395 std::pair<Attribute, Type> valueTypePair{valueAttr, resultType};
1396 if (uint32_t id = getConstantCompositeReplicateID(valueTypePair)) {
1397 return id;
1398 }
1399
1400 uint32_t typeID = 0;
1401 if (failed(Result: processType(loc, type: resultType, typeID))) {
1402 return 0;
1403 }
1404
1405 Type valueType = getValueType(attr: valueAttr);
1406 if (!valueAttr)
1407 return 0;
1408
1409 auto compositeType = dyn_cast<CompositeType>(Val&: resultType);
1410 if (!compositeType)
1411 return 0;
1412 Type elementType = compositeType.getElementType(0);
1413
1414 uint32_t constandID;
1415 if (elementType == valueType) {
1416 constandID = prepareConstant(loc, constType: elementType, valueAttr);
1417 } else {
1418 constandID = prepareConstantCompositeReplicate(loc, resultType: elementType, valueAttr);
1419 }
1420
1421 uint32_t resultID = getNextID();
1422 if (dyn_cast<spirv::TensorArmType>(Val&: resultType) && isZeroValue(attr: valueAttr)) {
1423 encodeInstructionInto(binary&: typesGlobalValues, op: spirv::Opcode::OpConstantNull,
1424 operands: {typeID, resultID});
1425 } else {
1426 encodeInstructionInto(binary&: typesGlobalValues,
1427 op: spirv::Opcode::OpConstantCompositeReplicateEXT,
1428 operands: {typeID, resultID, constandID});
1429 }
1430
1431 constCompositeReplicateIDMap[valueTypePair] = resultID;
1432 return resultID;
1433}
1434
1435//===----------------------------------------------------------------------===//
1436// Control flow
1437//===----------------------------------------------------------------------===//
1438
1439uint32_t Serializer::getOrCreateBlockID(Block *block) {
1440 if (uint32_t id = getBlockID(block))
1441 return id;
1442 return blockIDMap[block] = getNextID();
1443}
1444
1445#ifndef NDEBUG
1446void Serializer::printBlock(Block *block, raw_ostream &os) {
1447 os << "block " << block << " (id = ";
1448 if (uint32_t id = getBlockID(block))
1449 os << id;
1450 else
1451 os << "unknown";
1452 os << ")\n";
1453}
1454#endif
1455
1456LogicalResult
1457Serializer::processBlock(Block *block, bool omitLabel,
1458 function_ref<LogicalResult()> emitMerge) {
1459 LLVM_DEBUG(llvm::dbgs() << "processing block " << block << ":\n");
1460 LLVM_DEBUG(block->print(llvm::dbgs()));
1461 LLVM_DEBUG(llvm::dbgs() << '\n');
1462 if (!omitLabel) {
1463 uint32_t blockID = getOrCreateBlockID(block);
1464 LLVM_DEBUG(printBlock(block, llvm::dbgs()));
1465
1466 // Emit OpLabel for this block.
1467 encodeInstructionInto(binary&: functionBody, op: spirv::Opcode::OpLabel, operands: {blockID});
1468 }
1469
1470 // Emit OpPhi instructions for block arguments, if any.
1471 if (failed(Result: emitPhiForBlockArguments(block)))
1472 return failure();
1473
1474 // If we need to emit merge instructions, it must happen in this block. Check
1475 // whether we have other structured control flow ops, which will be expanded
1476 // into multiple basic blocks. If that's the case, we need to emit the merge
1477 // right now and then create new blocks for further serialization of the ops
1478 // in this block.
1479 if (emitMerge &&
1480 llvm::any_of(Range&: block->getOperations(),
1481 P: llvm::IsaPred<spirv::LoopOp, spirv::SelectionOp>)) {
1482 if (failed(Result: emitMerge()))
1483 return failure();
1484 emitMerge = nullptr;
1485
1486 // Start a new block for further serialization.
1487 uint32_t blockID = getNextID();
1488 encodeInstructionInto(binary&: functionBody, op: spirv::Opcode::OpBranch, operands: {blockID});
1489 encodeInstructionInto(binary&: functionBody, op: spirv::Opcode::OpLabel, operands: {blockID});
1490 }
1491
1492 // Process each op in this block except the terminator.
1493 for (Operation &op : llvm::drop_end(RangeOrContainer&: *block)) {
1494 if (failed(Result: processOperation(op: &op)))
1495 return failure();
1496 }
1497
1498 // Process the terminator.
1499 if (emitMerge)
1500 if (failed(Result: emitMerge()))
1501 return failure();
1502 if (failed(Result: processOperation(op: &block->back())))
1503 return failure();
1504
1505 return success();
1506}
1507
1508LogicalResult Serializer::emitPhiForBlockArguments(Block *block) {
1509 // Nothing to do if this block has no arguments or it's the entry block, which
1510 // always has the same arguments as the function signature.
1511 if (block->args_empty() || block->isEntryBlock())
1512 return success();
1513
1514 LLVM_DEBUG(llvm::dbgs() << "emitting phi instructions..\n");
1515
1516 // If the block has arguments, we need to create SPIR-V OpPhi instructions.
1517 // A SPIR-V OpPhi instruction is of the syntax:
1518 // OpPhi | result type | result <id> | (value <id>, parent block <id>) pair
1519 // So we need to collect all predecessor blocks and the arguments they send
1520 // to this block.
1521 SmallVector<std::pair<Block *, OperandRange>, 4> predecessors;
1522 for (Block *mlirPredecessor : block->getPredecessors()) {
1523 auto *terminator = mlirPredecessor->getTerminator();
1524 LLVM_DEBUG(llvm::dbgs() << " mlir predecessor ");
1525 LLVM_DEBUG(printBlock(mlirPredecessor, llvm::dbgs()));
1526 LLVM_DEBUG(llvm::dbgs() << " terminator: " << *terminator << "\n");
1527 // The predecessor here is the immediate one according to MLIR's IR
1528 // structure. It does not directly map to the incoming parent block for the
1529 // OpPhi instructions at SPIR-V binary level. This is because structured
1530 // control flow ops are serialized to multiple SPIR-V blocks. If there is a
1531 // spirv.mlir.selection/spirv.mlir.loop op in the MLIR predecessor block,
1532 // the branch op jumping to the OpPhi's block then resides in the previous
1533 // structured control flow op's merge block.
1534 Block *spirvPredecessor = getPhiIncomingBlock(block: mlirPredecessor);
1535 LLVM_DEBUG(llvm::dbgs() << " spirv predecessor ");
1536 LLVM_DEBUG(printBlock(spirvPredecessor, llvm::dbgs()));
1537 if (auto branchOp = dyn_cast<spirv::BranchOp>(Val: terminator)) {
1538 predecessors.emplace_back(Args&: spirvPredecessor, Args: branchOp.getOperands());
1539 } else if (auto branchCondOp =
1540 dyn_cast<spirv::BranchConditionalOp>(Val: terminator)) {
1541 std::optional<OperandRange> blockOperands;
1542 if (branchCondOp.getTrueTarget() == block) {
1543 blockOperands = branchCondOp.getTrueTargetOperands();
1544 } else {
1545 assert(branchCondOp.getFalseTarget() == block);
1546 blockOperands = branchCondOp.getFalseTargetOperands();
1547 }
1548 assert(!blockOperands->empty() &&
1549 "expected non-empty block operand range");
1550 predecessors.emplace_back(Args&: spirvPredecessor, Args&: *blockOperands);
1551 } else if (auto switchOp = dyn_cast<spirv::SwitchOp>(Val: terminator)) {
1552 std::optional<OperandRange> blockOperands;
1553 if (block == switchOp.getDefaultTarget()) {
1554 blockOperands = switchOp.getDefaultOperands();
1555 } else {
1556 SuccessorRange targets = switchOp.getTargets();
1557 auto it = llvm::find(Range&: targets, Val: block);
1558 assert(it != targets.end());
1559 size_t index = std::distance(first: targets.begin(), last: it);
1560 blockOperands = switchOp.getTargetOperands(index);
1561 }
1562 assert(!blockOperands->empty() &&
1563 "expected non-empty block operand range");
1564 predecessors.emplace_back(Args&: spirvPredecessor, Args&: *blockOperands);
1565 } else {
1566 return terminator->emitError(message: "unimplemented terminator for Phi creation");
1567 }
1568 LLVM_DEBUG({
1569 llvm::dbgs() << " block arguments:\n";
1570 for (Value v : predecessors.back().second)
1571 llvm::dbgs() << " " << v << "\n";
1572 });
1573 }
1574
1575 // Then create OpPhi instruction for each of the block argument.
1576 for (auto argIndex : llvm::seq<unsigned>(Begin: 0, End: block->getNumArguments())) {
1577 BlockArgument arg = block->getArgument(i: argIndex);
1578
1579 // Get the type <id> and result <id> for this OpPhi instruction.
1580 uint32_t phiTypeID = 0;
1581 if (failed(Result: processType(loc: arg.getLoc(), type: arg.getType(), typeID&: phiTypeID)))
1582 return failure();
1583 uint32_t phiID = getNextID();
1584
1585 LLVM_DEBUG(llvm::dbgs() << "[phi] for block argument #" << argIndex << ' '
1586 << arg << " (id = " << phiID << ")\n");
1587
1588 // Prepare the (value <id>, parent block <id>) pairs.
1589 SmallVector<uint32_t, 8> phiArgs;
1590 phiArgs.push_back(Elt: phiTypeID);
1591 phiArgs.push_back(Elt: phiID);
1592
1593 for (auto predIndex : llvm::seq<unsigned>(Begin: 0, End: predecessors.size())) {
1594 Value value = predecessors[predIndex].second[argIndex];
1595 uint32_t predBlockId = getOrCreateBlockID(block: predecessors[predIndex].first);
1596 LLVM_DEBUG(llvm::dbgs() << "[phi] use predecessor (id = " << predBlockId
1597 << ") value " << value << ' ');
1598 // Each pair is a value <id> ...
1599 uint32_t valueId = getValueID(val: value);
1600 if (valueId == 0) {
1601 // The op generating this value hasn't been visited yet so we don't have
1602 // an <id> assigned yet. Record this to fix up later.
1603 LLVM_DEBUG(llvm::dbgs() << "(need to fix)\n");
1604 deferredPhiValues[value].push_back(Elt: functionBody.size() + 1 +
1605 phiArgs.size());
1606 } else {
1607 LLVM_DEBUG(llvm::dbgs() << "(id = " << valueId << ")\n");
1608 }
1609 phiArgs.push_back(Elt: valueId);
1610 // ... and a parent block <id>.
1611 phiArgs.push_back(Elt: predBlockId);
1612 }
1613
1614 encodeInstructionInto(binary&: functionBody, op: spirv::Opcode::OpPhi, operands: phiArgs);
1615 valueIDMap[arg] = phiID;
1616 }
1617
1618 return success();
1619}
1620
1621//===----------------------------------------------------------------------===//
1622// Operation
1623//===----------------------------------------------------------------------===//
1624
1625LogicalResult Serializer::encodeExtensionInstruction(
1626 Operation *op, StringRef extensionSetName, uint32_t extensionOpcode,
1627 ArrayRef<uint32_t> operands, SmallVectorImpl<uint32_t> &binary) {
1628 // Check if the extension has been imported.
1629 auto &setID = extendedInstSetIDMap[extensionSetName];
1630 if (!setID) {
1631 setID = getNextID();
1632 SmallVector<uint32_t, 16> importOperands;
1633 importOperands.push_back(Elt: setID);
1634 spirv::encodeStringLiteralInto(binary&: importOperands, literal: extensionSetName);
1635 encodeInstructionInto(binary&: extendedSets, op: spirv::Opcode::OpExtInstImport,
1636 operands: importOperands);
1637 }
1638
1639 // The first two operands are the result type <id> and result <id>. The set
1640 // <id> and the opcode need to be insert after this.
1641 if (operands.size() < 2) {
1642 return op->emitError(message: "extended instructions must have a result encoding");
1643 }
1644 SmallVector<uint32_t, 8> extInstOperands;
1645 extInstOperands.reserve(N: operands.size() + 2);
1646 extInstOperands.append(in_start: operands.begin(), in_end: std::next(x: operands.begin(), n: 2));
1647 extInstOperands.push_back(Elt: setID);
1648 extInstOperands.push_back(Elt: extensionOpcode);
1649 extInstOperands.append(in_start: std::next(x: operands.begin(), n: 2), in_end: operands.end());
1650 encodeInstructionInto(binary, op: spirv::Opcode::OpExtInst, operands: extInstOperands);
1651 return success();
1652}
1653
1654LogicalResult Serializer::encodeExtensionInstruction(
1655 Operation *op, StringRef extensionSetName, uint32_t extensionOpcode,
1656 ArrayRef<uint32_t> operands) {
1657 if (failed(Result: encodeExtensionInstruction(op, extensionSetName, extensionOpcode,
1658 operands, binary&: functionBody)))
1659 return failure();
1660
1661 if (extensionSetName == extTosa)
1662 updateTosaOpsMap(op);
1663
1664 return success();
1665}
1666
1667LogicalResult Serializer::processOperation(Operation *opInst) {
1668 LLVM_DEBUG(llvm::dbgs() << "[op] '" << opInst->getName() << "'\n");
1669
1670 // First dispatch the ops that do not directly mirror an instruction from
1671 // the SPIR-V spec.
1672 return TypeSwitch<Operation *, LogicalResult>(opInst)
1673 .Case(caseFn: [&](spirv::AddressOfOp op) { return processAddressOfOp(addressOfOp: op); })
1674 .Case(caseFn: [&](spirv::BranchOp op) { return processBranchOp(branchOp: op); })
1675 .Case(caseFn: [&](spirv::BranchConditionalOp op) {
1676 return processBranchConditionalOp(op);
1677 })
1678 .Case(caseFn: [&](spirv::ConstantOp op) { return processConstantOp(op); })
1679 .Case(caseFn: [&](spirv::CompositeConstructOp op) {
1680 return processCompositeConstructOp(op);
1681 })
1682 .Case(caseFn: [&](spirv::EXTConstantCompositeReplicateOp op) {
1683 return processConstantCompositeReplicateOp(op);
1684 })
1685 .Case(caseFn: [&](spirv::FuncOp op) { return processFuncOp(op); })
1686 .Case(caseFn: [&](spirv::GraphARMOp op) { return processGraphARMOp(op); })
1687 .Case(caseFn: [&](spirv::GraphEntryPointARMOp op) {
1688 return processGraphEntryPointARMOp(op);
1689 })
1690 .Case(caseFn: [&](spirv::GraphOutputsARMOp op) {
1691 return processGraphOutputsARMOp(op);
1692 })
1693 .Case(caseFn: [&](spirv::GlobalVariableOp op) {
1694 return processGlobalVariableOp(varOp: op);
1695 })
1696 .Case(caseFn: [&](spirv::GraphConstantARMOp op) {
1697 return processGraphConstantARMOp(op);
1698 })
1699 .Case(caseFn: [&](spirv::LoopOp op) { return processLoopOp(loopOp: op); })
1700 .Case(caseFn: [&](spirv::ReferenceOfOp op) { return processReferenceOfOp(referenceOfOp: op); })
1701 .Case(caseFn: [&](spirv::SelectionOp op) { return processSelectionOp(selectionOp: op); })
1702 .Case(caseFn: [&](spirv::SpecConstantOp op) { return processSpecConstantOp(op); })
1703 .Case(caseFn: [&](spirv::SpecConstantCompositeOp op) {
1704 return processSpecConstantCompositeOp(op);
1705 })
1706 .Case(caseFn: [&](spirv::EXTSpecConstantCompositeReplicateOp op) {
1707 return processSpecConstantCompositeReplicateOp(op);
1708 })
1709 .Case(caseFn: [&](spirv::SpecConstantOperationOp op) {
1710 return processSpecConstantOperationOp(op);
1711 })
1712 .Case(caseFn: [&](spirv::SwitchOp op) { return processSwitchOp(switchOp: op); })
1713 .Case(caseFn: [&](spirv::UndefOp op) { return processUndefOp(op); })
1714 .Case(caseFn: [&](spirv::VariableOp op) { return processVariableOp(op); })
1715
1716 // Then handle all the ops that directly mirror SPIR-V instructions with
1717 // auto-generated methods.
1718 .Default(
1719 defaultFn: [&](Operation *op) { return dispatchToAutogenSerialization(op); });
1720}
1721
1722LogicalResult
1723Serializer::processCompositeConstructOp(spirv::CompositeConstructOp op) {
1724 Location loc = op.getLoc();
1725
1726 uint32_t resultTypeID = 0;
1727 if (failed(Result: processType(loc, type: op.getType(), typeID&: resultTypeID)))
1728 return failure();
1729
1730 uint32_t resultID = getNextID();
1731 valueIDMap[op.getResult()] = resultID;
1732
1733 SmallVector<uint32_t, 8> operands;
1734 operands.reserve(N: 2 + op.getConstituents().size());
1735 operands.push_back(Elt: resultTypeID);
1736 operands.push_back(Elt: resultID);
1737 for (Value constituent : op.getConstituents()) {
1738 uint32_t id = getValueID(val: constituent);
1739 assert(id && "use before def!");
1740 operands.push_back(Elt: id);
1741 }
1742
1743 if (failed(Result: emitDebugLine(binary&: functionBody, loc)))
1744 return failure();
1745
1746 encodeInstructionWithContinuationInto(
1747 binary&: functionBody, op: spirv::Opcode::OpCompositeConstruct, operands);
1748
1749 for (auto attr : op->getDiscardableAttrDictionary().getValue()) {
1750 if (failed(Result: processDecoration(loc, resultID, attr)))
1751 return failure();
1752 }
1753
1754 return success();
1755}
1756
1757LogicalResult Serializer::processOpWithoutGrammarAttr(Operation *op,
1758 StringRef extInstSet,
1759 uint32_t opcode) {
1760 SmallVector<uint32_t, 4> operands;
1761 Location loc = op->getLoc();
1762
1763 uint32_t resultID = 0;
1764 if (op->getNumResults() != 0) {
1765 uint32_t resultTypeID = 0;
1766 if (failed(Result: processType(loc, type: op->getResult(idx: 0).getType(), typeID&: resultTypeID)))
1767 return failure();
1768 operands.push_back(Elt: resultTypeID);
1769
1770 resultID = getNextID();
1771 operands.push_back(Elt: resultID);
1772 valueIDMap[op->getResult(idx: 0)] = resultID;
1773 };
1774
1775 for (Value operand : op->getOperands())
1776 operands.push_back(Elt: getValueID(val: operand));
1777
1778 if (extInstSet != extTosa)
1779 // OpLine cannot be present in graphs
1780 if (failed(Result: emitDebugLine(binary&: functionBody, loc)))
1781 return failure();
1782
1783 if (extInstSet.empty()) {
1784 encodeInstructionInto(binary&: functionBody, op: static_cast<spirv::Opcode>(opcode),
1785 operands);
1786 } else {
1787 if (failed(Result: encodeExtensionInstruction(op, extensionSetName: extInstSet, extensionOpcode: opcode, operands)))
1788 return failure();
1789 }
1790
1791 if (op->getNumResults() != 0) {
1792 for (auto attr : op->getDiscardableAttrDictionary().getValue()) {
1793 if (failed(Result: processDecoration(loc, resultID, attr)))
1794 return failure();
1795 }
1796 }
1797
1798 return success();
1799}
1800
1801void Serializer::updateTosaOpsMap(Operation *op) {
1802 if (!options.emitDebugInfo)
1803 return;
1804
1805 if (auto graphOp = dyn_cast<spirv::GraphARMOp>(Val: op->getParentOp())) {
1806 if (uint32_t graphID = getFunctionID(fnName: graphOp.getName()))
1807 tosaOpsMap[graphID][op->getLoc()].insert(X: op);
1808 }
1809}
1810
1811LogicalResult Serializer::emitDecoration(uint32_t target,
1812 spirv::Decoration decoration,
1813 ArrayRef<uint32_t> params) {
1814 uint32_t wordCount = 3 + params.size();
1815 llvm::append_values(
1816 C&: decorations,
1817 Values: spirv::getPrefixedOpcode(wordCount, opcode: spirv::Opcode::OpDecorate), Values&: target,
1818 Values: static_cast<uint32_t>(decoration));
1819 llvm::append_range(C&: decorations, R&: params);
1820 return success();
1821}
1822
1823LogicalResult Serializer::emitDecorationId(uint32_t target,
1824 spirv::Decoration decoration,
1825 ArrayRef<uint32_t> operandIds) {
1826 uint32_t wordCount = 3 + operandIds.size();
1827 llvm::append_values(
1828 C&: decorations,
1829 Values: spirv::getPrefixedOpcode(wordCount, opcode: spirv::Opcode::OpDecorateId), Values&: target,
1830 Values: static_cast<uint32_t>(decoration));
1831 llvm::append_range(C&: decorations, R&: operandIds);
1832 return success();
1833}
1834
1835LogicalResult Serializer::emitDebugLine(SmallVectorImpl<uint32_t> &binary,
1836 Location loc) {
1837 if (!options.emitDebugInfo)
1838 return success();
1839
1840 if (lastProcessedWasMergeInst) {
1841 lastProcessedWasMergeInst = false;
1842 return success();
1843 }
1844
1845 auto fileLoc = dyn_cast<FileLineColLoc>(Val&: loc);
1846 if (fileLoc)
1847 encodeInstructionInto(binary, op: spirv::Opcode::OpLine,
1848 operands: {fileID, fileLoc.getLine(), fileLoc.getColumn()});
1849 return success();
1850}
1851} // namespace spirv
1852} // namespace mlir
1853