| 1 | //===- Serializer.cpp - MLIR SPIR-V Serializer ----------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file defines the MLIR SPIR-V module to SPIR-V binary serializer. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "Serializer.h" |
| 14 | |
| 15 | #include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h" |
| 16 | #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h" |
| 17 | #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h" |
| 18 | #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h" |
| 19 | #include "mlir/Dialect/SPIRV/IR/TargetAndABI.h" |
| 20 | #include "mlir/Target/SPIRV/SPIRVBinaryUtils.h" |
| 21 | #include "llvm/ADT/STLExtras.h" |
| 22 | #include "llvm/ADT/Sequence.h" |
| 23 | #include "llvm/ADT/StringExtras.h" |
| 24 | #include "llvm/ADT/TypeSwitch.h" |
| 25 | #include "llvm/ADT/bit.h" |
| 26 | #include "llvm/Support/Debug.h" |
| 27 | #include <cstdint> |
| 28 | #include <optional> |
| 29 | |
| 30 | #define DEBUG_TYPE "spirv-serialization" |
| 31 | |
| 32 | using namespace mlir; |
| 33 | |
| 34 | /// Returns the merge block if the given `op` is a structured control flow op. |
| 35 | /// Otherwise returns nullptr. |
| 36 | static Block *getStructuredControlFlowOpMergeBlock(Operation *op) { |
| 37 | if (auto selectionOp = dyn_cast<spirv::SelectionOp>(Val: op)) |
| 38 | return selectionOp.getMergeBlock(); |
| 39 | if (auto loopOp = dyn_cast<spirv::LoopOp>(Val: op)) |
| 40 | return loopOp.getMergeBlock(); |
| 41 | return nullptr; |
| 42 | } |
| 43 | |
| 44 | /// Given a predecessor `block` for a block with arguments, returns the block |
| 45 | /// that should be used as the parent block for SPIR-V OpPhi instructions |
| 46 | /// corresponding to the block arguments. |
| 47 | static Block *getPhiIncomingBlock(Block *block) { |
| 48 | // If the predecessor block in question is the entry block for a |
| 49 | // spirv.mlir.loop, we jump to this spirv.mlir.loop from its enclosing block. |
| 50 | if (block->isEntryBlock()) { |
| 51 | if (auto loopOp = dyn_cast<spirv::LoopOp>(Val: block->getParentOp())) { |
| 52 | // Then the incoming parent block for OpPhi should be the merge block of |
| 53 | // the structured control flow op before this loop. |
| 54 | Operation *op = loopOp.getOperation(); |
| 55 | while ((op = op->getPrevNode()) != nullptr) |
| 56 | if (Block *incomingBlock = getStructuredControlFlowOpMergeBlock(op)) |
| 57 | return incomingBlock; |
| 58 | // Or the enclosing block itself if no structured control flow ops |
| 59 | // exists before this loop. |
| 60 | return loopOp->getBlock(); |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | // Otherwise, we jump from the given predecessor block. Try to see if there is |
| 65 | // a structured control flow op inside it. |
| 66 | for (Operation &op : llvm::reverse(C&: block->getOperations())) { |
| 67 | if (Block *incomingBlock = getStructuredControlFlowOpMergeBlock(op: &op)) |
| 68 | return incomingBlock; |
| 69 | } |
| 70 | return block; |
| 71 | } |
| 72 | |
| 73 | static bool isZeroValue(Attribute attr) { |
| 74 | if (auto floatAttr = dyn_cast<FloatAttr>(Val&: attr)) { |
| 75 | return floatAttr.getValue().isZero(); |
| 76 | } |
| 77 | if (auto boolAttr = dyn_cast<BoolAttr>(Val&: attr)) { |
| 78 | return !boolAttr.getValue(); |
| 79 | } |
| 80 | if (auto intAttr = dyn_cast<IntegerAttr>(Val&: attr)) { |
| 81 | return intAttr.getValue().isZero(); |
| 82 | } |
| 83 | if (auto splatElemAttr = dyn_cast<SplatElementsAttr>(Val&: attr)) { |
| 84 | return isZeroValue(attr: splatElemAttr.getSplatValue<Attribute>()); |
| 85 | } |
| 86 | if (auto denseElemAttr = dyn_cast<DenseElementsAttr>(Val&: attr)) { |
| 87 | return all_of(Range: denseElemAttr.getValues<Attribute>(), P: isZeroValue); |
| 88 | } |
| 89 | return false; |
| 90 | } |
| 91 | |
| 92 | /// Move all functions declaration before functions definitions. In SPIR-V |
| 93 | /// "declarations" are functions without a body and "definitions" functions |
| 94 | /// with a body. This is stronger than necessary. It should be sufficient to |
| 95 | /// ensure any declarations precede their uses and not all definitions, however |
| 96 | /// this allows to avoid analysing every function in the module this way. |
| 97 | static void moveFuncDeclarationsToTop(spirv::ModuleOp moduleOp) { |
| 98 | Block::OpListType &ops = moduleOp.getBody()->getOperations(); |
| 99 | if (ops.empty()) |
| 100 | return; |
| 101 | Operation &firstOp = ops.front(); |
| 102 | for (Operation &op : llvm::drop_begin(RangeOrContainer&: ops)) |
| 103 | if (auto funcOp = dyn_cast<spirv::FuncOp>(Val&: op)) |
| 104 | if (funcOp.getBody().empty()) |
| 105 | funcOp->moveBefore(existingOp: &firstOp); |
| 106 | } |
| 107 | |
| 108 | namespace mlir { |
| 109 | namespace spirv { |
| 110 | |
| 111 | /// Encodes an SPIR-V instruction with the given `opcode` and `operands` into |
| 112 | /// the given `binary` vector. |
| 113 | void encodeInstructionInto(SmallVectorImpl<uint32_t> &binary, spirv::Opcode op, |
| 114 | ArrayRef<uint32_t> operands) { |
| 115 | uint32_t wordCount = 1 + operands.size(); |
| 116 | binary.push_back(Elt: spirv::getPrefixedOpcode(wordCount, opcode: op)); |
| 117 | binary.append(in_start: operands.begin(), in_end: operands.end()); |
| 118 | } |
| 119 | |
| 120 | Serializer::Serializer(spirv::ModuleOp module, |
| 121 | const SerializationOptions &options) |
| 122 | : module(module), mlirBuilder(module.getContext()), options(options) {} |
| 123 | |
| 124 | LogicalResult Serializer::serialize() { |
| 125 | LLVM_DEBUG(llvm::dbgs() << "+++ starting serialization +++\n" ); |
| 126 | |
| 127 | if (failed(Result: module.verifyInvariants())) |
| 128 | return failure(); |
| 129 | |
| 130 | // TODO: handle the other sections |
| 131 | processCapability(); |
| 132 | if (failed(Result: processExtension())) { |
| 133 | return failure(); |
| 134 | } |
| 135 | processMemoryModel(); |
| 136 | processDebugInfo(); |
| 137 | |
| 138 | moveFuncDeclarationsToTop(moduleOp: module); |
| 139 | |
| 140 | // Iterate over the module body to serialize it. Assumptions are that there is |
| 141 | // only one basic block in the moduleOp |
| 142 | for (auto &op : *module.getBody()) { |
| 143 | if (failed(Result: processOperation(op: &op))) { |
| 144 | return failure(); |
| 145 | } |
| 146 | } |
| 147 | |
| 148 | LLVM_DEBUG(llvm::dbgs() << "+++ completed serialization +++\n" ); |
| 149 | return success(); |
| 150 | } |
| 151 | |
| 152 | void Serializer::collect(SmallVectorImpl<uint32_t> &binary) { |
| 153 | auto moduleSize = spirv::kHeaderWordCount + capabilities.size() + |
| 154 | extensions.size() + extendedSets.size() + |
| 155 | memoryModel.size() + entryPoints.size() + |
| 156 | executionModes.size() + decorations.size() + |
| 157 | typesGlobalValues.size() + functions.size() + graphs.size(); |
| 158 | |
| 159 | binary.clear(); |
| 160 | binary.reserve(N: moduleSize); |
| 161 | |
| 162 | spirv::appendModuleHeader(header&: binary, version: module.getVceTriple()->getVersion(), |
| 163 | idBound: nextID); |
| 164 | binary.append(in_start: capabilities.begin(), in_end: capabilities.end()); |
| 165 | binary.append(in_start: extensions.begin(), in_end: extensions.end()); |
| 166 | binary.append(in_start: extendedSets.begin(), in_end: extendedSets.end()); |
| 167 | binary.append(in_start: memoryModel.begin(), in_end: memoryModel.end()); |
| 168 | binary.append(in_start: entryPoints.begin(), in_end: entryPoints.end()); |
| 169 | binary.append(in_start: executionModes.begin(), in_end: executionModes.end()); |
| 170 | binary.append(in_start: debug.begin(), in_end: debug.end()); |
| 171 | binary.append(in_start: names.begin(), in_end: names.end()); |
| 172 | binary.append(in_start: decorations.begin(), in_end: decorations.end()); |
| 173 | binary.append(in_start: typesGlobalValues.begin(), in_end: typesGlobalValues.end()); |
| 174 | binary.append(in_start: functions.begin(), in_end: functions.end()); |
| 175 | binary.append(in_start: graphs.begin(), in_end: graphs.end()); |
| 176 | binary.append(in_start: graphsDebugInfo.begin(), in_end: graphsDebugInfo.end()); |
| 177 | } |
| 178 | |
| 179 | #ifndef NDEBUG |
| 180 | void Serializer::printValueIDMap(raw_ostream &os) { |
| 181 | os << "\n= Value <id> Map =\n\n" ; |
| 182 | for (auto valueIDPair : valueIDMap) { |
| 183 | Value val = valueIDPair.first; |
| 184 | os << " " << val << " " |
| 185 | << "id = " << valueIDPair.second << ' '; |
| 186 | if (auto *op = val.getDefiningOp()) { |
| 187 | os << "from op '" << op->getName() << "'" ; |
| 188 | } else if (auto arg = dyn_cast<BlockArgument>(val)) { |
| 189 | Block *block = arg.getOwner(); |
| 190 | os << "from argument of block " << block << ' '; |
| 191 | os << " in op '" << block->getParentOp()->getName() << "'" ; |
| 192 | } |
| 193 | os << '\n'; |
| 194 | } |
| 195 | } |
| 196 | #endif |
| 197 | |
| 198 | //===----------------------------------------------------------------------===// |
| 199 | // Module structure |
| 200 | //===----------------------------------------------------------------------===// |
| 201 | |
| 202 | uint32_t Serializer::getOrCreateFunctionID(StringRef fnName) { |
| 203 | auto funcID = funcIDMap.lookup(Key: fnName); |
| 204 | if (!funcID) { |
| 205 | funcID = getNextID(); |
| 206 | funcIDMap[fnName] = funcID; |
| 207 | } |
| 208 | return funcID; |
| 209 | } |
| 210 | |
| 211 | void Serializer::processCapability() { |
| 212 | for (auto cap : module.getVceTriple()->getCapabilities()) |
| 213 | encodeInstructionInto(binary&: capabilities, op: spirv::Opcode::OpCapability, |
| 214 | operands: {static_cast<uint32_t>(cap)}); |
| 215 | } |
| 216 | |
| 217 | void Serializer::addLongCompositesCapability() { |
| 218 | if (longCompositesEmitted) |
| 219 | return; |
| 220 | longCompositesEmitted = true; |
| 221 | auto vceTriple = module.getVceTriple(); |
| 222 | if (!llvm::is_contained(Range: vceTriple->getCapabilities(), |
| 223 | Element: spirv::Capability::LongCompositesINTEL)) |
| 224 | encodeInstructionInto( |
| 225 | binary&: capabilities, op: spirv::Opcode::OpCapability, |
| 226 | operands: {static_cast<uint32_t>(spirv::Capability::LongCompositesINTEL)}); |
| 227 | if (!llvm::is_contained(Range: vceTriple->getExtensions(), |
| 228 | Element: spirv::Extension::SPV_INTEL_long_composites)) { |
| 229 | SmallVector<uint32_t, 8> extName; |
| 230 | spirv::encodeStringLiteralInto( |
| 231 | binary&: extName, |
| 232 | literal: spirv::stringifyExtension(spirv::Extension::SPV_INTEL_long_composites)); |
| 233 | encodeInstructionInto(binary&: extensions, op: spirv::Opcode::OpExtension, operands: extName); |
| 234 | } |
| 235 | } |
| 236 | |
| 237 | void Serializer::encodeInstructionWithContinuationInto( |
| 238 | SmallVectorImpl<uint32_t> &binary, spirv::Opcode op, |
| 239 | ArrayRef<uint32_t> operands) { |
| 240 | if (1 + operands.size() <= spirv::kMaxWordCount) { |
| 241 | encodeInstructionInto(binary, op, operands); |
| 242 | return; |
| 243 | } |
| 244 | |
| 245 | std::optional<spirv::Opcode> continuationOp = |
| 246 | spirv::getContinuationOpcode(parent: op); |
| 247 | assert(continuationOp && "op is not a splittable composite/struct opcode" ); |
| 248 | |
| 249 | const unsigned chunk = spirv::kMaxWordCount - 1; |
| 250 | encodeInstructionInto(binary, op, operands: operands.take_front(N: chunk)); |
| 251 | for (ArrayRef<uint32_t> rest = operands.drop_front(N: chunk); !rest.empty(); |
| 252 | rest = rest.drop_front(N: std::min<size_t>(a: rest.size(), b: chunk))) { |
| 253 | encodeInstructionInto(binary, op: *continuationOp, operands: rest.take_front(N: chunk)); |
| 254 | } |
| 255 | |
| 256 | addLongCompositesCapability(); |
| 257 | } |
| 258 | |
| 259 | void Serializer::processDebugInfo() { |
| 260 | if (!options.emitDebugInfo) |
| 261 | return; |
| 262 | auto fileLoc = dyn_cast<FileLineColLoc>(Val: module.getLoc()); |
| 263 | auto fileName = fileLoc ? fileLoc.getFilename().strref() : "<unknown>" ; |
| 264 | fileID = getNextID(); |
| 265 | SmallVector<uint32_t, 16> operands; |
| 266 | operands.push_back(Elt: fileID); |
| 267 | spirv::encodeStringLiteralInto(binary&: operands, literal: fileName); |
| 268 | encodeInstructionInto(binary&: debug, op: spirv::Opcode::OpString, operands); |
| 269 | // TODO: Encode more debug instructions. |
| 270 | } |
| 271 | |
| 272 | LogicalResult Serializer::processExtension() { |
| 273 | llvm::SmallVector<uint32_t, 16> extName; |
| 274 | llvm::SmallSet<Extension, 4> deducedExts( |
| 275 | llvm::from_range, module.getVceTriple()->getExtensions()); |
| 276 | auto nonSemanticInfoExt = spirv::Extension::SPV_KHR_non_semantic_info; |
| 277 | if (options.emitDebugInfo && !deducedExts.contains(V: nonSemanticInfoExt)) { |
| 278 | TargetEnvAttr targetEnvAttr = lookupTargetEnvOrDefault(op: module); |
| 279 | if (!is_contained(Range: targetEnvAttr.getExtensions(), Element: nonSemanticInfoExt)) |
| 280 | return module.emitError( |
| 281 | message: "SPV_KHR_non_semantic_info extension not available" ); |
| 282 | deducedExts.insert(V: nonSemanticInfoExt); |
| 283 | } |
| 284 | for (spirv::Extension ext : deducedExts) { |
| 285 | extName.clear(); |
| 286 | spirv::encodeStringLiteralInto(binary&: extName, literal: spirv::stringifyExtension(ext)); |
| 287 | encodeInstructionInto(binary&: extensions, op: spirv::Opcode::OpExtension, operands: extName); |
| 288 | } |
| 289 | return success(); |
| 290 | } |
| 291 | |
| 292 | void Serializer::processMemoryModel() { |
| 293 | auto mm = static_cast<uint32_t>(module.getMemoryModel()); |
| 294 | auto am = static_cast<uint32_t>(module.getAddressingModel()); |
| 295 | |
| 296 | encodeInstructionInto(binary&: memoryModel, op: spirv::Opcode::OpMemoryModel, operands: {am, mm}); |
| 297 | } |
| 298 | |
| 299 | static std::string getDecorationName(StringRef attrName) { |
| 300 | // convertToCamelFromSnakeCase will convert this to FpFastMathMode instead of |
| 301 | // expected FPFastMathMode. |
| 302 | if (attrName == "fp_fast_math_mode" ) |
| 303 | return "FPFastMathMode" ; |
| 304 | // similar here |
| 305 | if (attrName == "fp_rounding_mode" ) |
| 306 | return "FPRoundingMode" ; |
| 307 | // convertToCamelFromSnakeCase will not capitalize "INTEL". |
| 308 | if (attrName == "cache_control_load_intel" ) |
| 309 | return "CacheControlLoadINTEL" ; |
| 310 | if (attrName == "cache_control_store_intel" ) |
| 311 | return "CacheControlStoreINTEL" ; |
| 312 | |
| 313 | return llvm::convertToCamelFromSnakeCase(input: attrName, /*capitalizeFirst=*/true); |
| 314 | } |
| 315 | |
| 316 | template <typename AttrTy, typename EmitF> |
| 317 | static LogicalResult processDecorationList(Location loc, Decoration decoration, |
| 318 | Attribute attrList, |
| 319 | StringRef attrName, EmitF emitter) { |
| 320 | auto arrayAttr = dyn_cast<ArrayAttr>(Val&: attrList); |
| 321 | if (!arrayAttr) { |
| 322 | return emitError(loc, message: "expecting array attribute of " ) |
| 323 | << attrName << " for " << stringifyDecoration(decoration); |
| 324 | } |
| 325 | if (arrayAttr.empty()) { |
| 326 | return emitError(loc, message: "expecting non-empty array attribute of " ) |
| 327 | << attrName << " for " << stringifyDecoration(decoration); |
| 328 | } |
| 329 | for (Attribute attr : arrayAttr.getValue()) { |
| 330 | auto cacheControlAttr = dyn_cast<AttrTy>(attr); |
| 331 | if (!cacheControlAttr) { |
| 332 | return emitError(loc, message: "expecting array attribute of " ) |
| 333 | << attrName << " for " << stringifyDecoration(decoration); |
| 334 | } |
| 335 | // This named attribute encodes several decorations. Emit one per |
| 336 | // element in the array. |
| 337 | if (failed(emitter(cacheControlAttr))) |
| 338 | return failure(); |
| 339 | } |
| 340 | return success(); |
| 341 | } |
| 342 | |
| 343 | LogicalResult Serializer::processDecorationAttr(Location loc, uint32_t resultID, |
| 344 | Decoration decoration, |
| 345 | Attribute attr) { |
| 346 | SmallVector<uint32_t, 1> args; |
| 347 | switch (decoration) { |
| 348 | case spirv::Decoration::LinkageAttributes: { |
| 349 | // Get the value of the Linkage Attributes |
| 350 | // e.g., LinkageAttributes=["linkageName", linkageType]. |
| 351 | auto linkageAttr = dyn_cast<spirv::LinkageAttributesAttr>(Val&: attr); |
| 352 | auto linkageName = linkageAttr.getLinkageName(); |
| 353 | auto linkageType = linkageAttr.getLinkageType().getValue(); |
| 354 | // Encode the Linkage Name (string literal to uint32_t). |
| 355 | spirv::encodeStringLiteralInto(binary&: args, literal: linkageName); |
| 356 | // Encode LinkageType & Add the Linkagetype to the args. |
| 357 | args.push_back(Elt: static_cast<uint32_t>(linkageType)); |
| 358 | break; |
| 359 | } |
| 360 | case spirv::Decoration::FPFastMathMode: |
| 361 | if (auto intAttr = dyn_cast<FPFastMathModeAttr>(Val&: attr)) { |
| 362 | args.push_back(Elt: static_cast<uint32_t>(intAttr.getValue())); |
| 363 | break; |
| 364 | } |
| 365 | return emitError(loc, message: "expected FPFastMathModeAttr attribute for " ) |
| 366 | << stringifyDecoration(decoration); |
| 367 | case spirv::Decoration::FPRoundingMode: |
| 368 | if (auto intAttr = dyn_cast<FPRoundingModeAttr>(Val&: attr)) { |
| 369 | args.push_back(Elt: static_cast<uint32_t>(intAttr.getValue())); |
| 370 | break; |
| 371 | } |
| 372 | return emitError(loc, message: "expected FPRoundingModeAttr attribute for " ) |
| 373 | << stringifyDecoration(decoration); |
| 374 | case spirv::Decoration::FuncParamAttr: |
| 375 | return processDecorationList<FunctionParameterAttributeAttr>( |
| 376 | loc, decoration, attrList: attr, attrName: "FunctionParameterAttribute" , |
| 377 | emitter: [&](FunctionParameterAttributeAttr attr) { |
| 378 | return emitDecoration(target: resultID, decoration, |
| 379 | params: {static_cast<uint32_t>(attr.getValue())}); |
| 380 | }); |
| 381 | case spirv::Decoration::Alignment: |
| 382 | case spirv::Decoration::MaxByteOffset: |
| 383 | case spirv::Decoration::Binding: |
| 384 | case spirv::Decoration::DescriptorSet: |
| 385 | case spirv::Decoration::Location: |
| 386 | case spirv::Decoration::Index: |
| 387 | case spirv::Decoration::Offset: |
| 388 | case spirv::Decoration::XfbBuffer: |
| 389 | case spirv::Decoration::XfbStride: |
| 390 | if (auto intAttr = dyn_cast<IntegerAttr>(Val&: attr)) { |
| 391 | args.push_back(Elt: intAttr.getValue().getZExtValue()); |
| 392 | break; |
| 393 | } |
| 394 | return emitError(loc, message: "expected integer attribute for " ) |
| 395 | << stringifyDecoration(decoration); |
| 396 | case spirv::Decoration::BuiltIn: |
| 397 | if (auto strAttr = dyn_cast<StringAttr>(Val&: attr)) { |
| 398 | auto enumVal = spirv::symbolizeBuiltIn(strAttr.getValue()); |
| 399 | if (enumVal) { |
| 400 | args.push_back(Elt: static_cast<uint32_t>(*enumVal)); |
| 401 | break; |
| 402 | } |
| 403 | return emitError(loc, message: "invalid " ) |
| 404 | << stringifyDecoration(decoration) << " decoration attribute " |
| 405 | << strAttr.getValue(); |
| 406 | } |
| 407 | return emitError(loc, message: "expected string attribute for " ) |
| 408 | << stringifyDecoration(decoration); |
| 409 | case spirv::Decoration::Aliased: |
| 410 | case spirv::Decoration::AliasedPointer: |
| 411 | case spirv::Decoration::Flat: |
| 412 | case spirv::Decoration::NonReadable: |
| 413 | case spirv::Decoration::NonWritable: |
| 414 | case spirv::Decoration::NoPerspective: |
| 415 | case spirv::Decoration::NoSignedWrap: |
| 416 | case spirv::Decoration::NoUnsignedWrap: |
| 417 | case spirv::Decoration::RelaxedPrecision: |
| 418 | case spirv::Decoration::Restrict: |
| 419 | case spirv::Decoration::RestrictPointer: |
| 420 | case spirv::Decoration::NoContraction: |
| 421 | case spirv::Decoration::Constant: |
| 422 | case spirv::Decoration::Block: |
| 423 | case spirv::Decoration::BufferBlock: |
| 424 | case spirv::Decoration::CPacked: |
| 425 | case spirv::Decoration::Invariant: |
| 426 | case spirv::Decoration::Patch: |
| 427 | case spirv::Decoration::Coherent: |
| 428 | case spirv::Decoration::Volatile: |
| 429 | case spirv::Decoration::SaturatedConversion: |
| 430 | // For unit attributes and decoration attributes, the args list |
| 431 | // has no values so we do nothing. |
| 432 | if (isa<UnitAttr, DecorationAttr>(Val: attr)) |
| 433 | break; |
| 434 | return emitError(loc, |
| 435 | message: "expected unit attribute or decoration attribute for " ) |
| 436 | << stringifyDecoration(decoration); |
| 437 | case spirv::Decoration::CacheControlLoadINTEL: |
| 438 | return processDecorationList<CacheControlLoadINTELAttr>( |
| 439 | loc, decoration, attrList: attr, attrName: "CacheControlLoadINTEL" , |
| 440 | emitter: [&](CacheControlLoadINTELAttr attr) { |
| 441 | unsigned cacheLevel = attr.getCacheLevel(); |
| 442 | LoadCacheControl loadCacheControl = attr.getLoadCacheControl(); |
| 443 | return emitDecoration( |
| 444 | target: resultID, decoration, |
| 445 | params: {cacheLevel, static_cast<uint32_t>(loadCacheControl)}); |
| 446 | }); |
| 447 | case spirv::Decoration::CacheControlStoreINTEL: |
| 448 | return processDecorationList<CacheControlStoreINTELAttr>( |
| 449 | loc, decoration, attrList: attr, attrName: "CacheControlStoreINTEL" , |
| 450 | emitter: [&](CacheControlStoreINTELAttr attr) { |
| 451 | unsigned cacheLevel = attr.getCacheLevel(); |
| 452 | StoreCacheControl storeCacheControl = attr.getStoreCacheControl(); |
| 453 | return emitDecoration( |
| 454 | target: resultID, decoration, |
| 455 | params: {cacheLevel, static_cast<uint32_t>(storeCacheControl)}); |
| 456 | }); |
| 457 | case spirv::Decoration::AlignmentId: |
| 458 | case spirv::Decoration::MaxByteOffsetId: |
| 459 | case spirv::Decoration::CounterBuffer: { |
| 460 | auto symRef = dyn_cast<FlatSymbolRefAttr>(Val&: attr); |
| 461 | if (!symRef) |
| 462 | return emitError(loc, message: "expected symbol reference for " ) |
| 463 | << stringifyDecoration(decoration); |
| 464 | StringRef symName = symRef.getValue(); |
| 465 | uint32_t operandID = getVariableID(varName: symName); |
| 466 | if (!operandID) |
| 467 | operandID = getSpecConstID(constName: symName); |
| 468 | if (!operandID) |
| 469 | return emitError(loc, message: "could not find <id> for symbol '" ) |
| 470 | << symName << "' referenced by " |
| 471 | << stringifyDecoration(decoration); |
| 472 | return emitDecorationId(target: resultID, decoration, operandIds: {operandID}); |
| 473 | } |
| 474 | default: |
| 475 | return emitError(loc, message: "unhandled decoration " ) |
| 476 | << stringifyDecoration(decoration); |
| 477 | } |
| 478 | return emitDecoration(target: resultID, decoration, params: args); |
| 479 | } |
| 480 | |
| 481 | LogicalResult Serializer::processDecoration(Location loc, uint32_t resultID, |
| 482 | NamedAttribute attr) { |
| 483 | StringRef attrName = attr.getName().strref(); |
| 484 | std::string decorationName = getDecorationName(attrName); |
| 485 | std::optional<Decoration> decoration = |
| 486 | spirv::symbolizeDecoration(decorationName); |
| 487 | if (!decoration) { |
| 488 | return emitError( |
| 489 | loc, message: "non-argument attributes expected to have snake-case-ified " |
| 490 | "decoration name, unhandled attribute with name : " ) |
| 491 | << attrName; |
| 492 | } |
| 493 | return processDecorationAttr(loc, resultID, decoration: *decoration, attr: attr.getValue()); |
| 494 | } |
| 495 | |
| 496 | LogicalResult Serializer::processName(uint32_t resultID, StringRef name) { |
| 497 | assert(!name.empty() && "unexpected empty string for OpName" ); |
| 498 | if (!options.emitSymbolName) |
| 499 | return success(); |
| 500 | |
| 501 | SmallVector<uint32_t, 4> nameOperands; |
| 502 | nameOperands.push_back(Elt: resultID); |
| 503 | spirv::encodeStringLiteralInto(binary&: nameOperands, literal: name); |
| 504 | encodeInstructionInto(binary&: names, op: spirv::Opcode::OpName, operands: nameOperands); |
| 505 | return success(); |
| 506 | } |
| 507 | |
| 508 | template <> |
| 509 | LogicalResult Serializer::processTypeDecoration<spirv::ArrayType>( |
| 510 | Location loc, spirv::ArrayType type, uint32_t resultID) { |
| 511 | if (unsigned stride = type.getArrayStride()) { |
| 512 | // OpDecorate %arrayTypeSSA ArrayStride strideLiteral |
| 513 | return emitDecoration(target: resultID, decoration: spirv::Decoration::ArrayStride, params: {stride}); |
| 514 | } |
| 515 | return success(); |
| 516 | } |
| 517 | |
| 518 | template <> |
| 519 | LogicalResult Serializer::processTypeDecoration<spirv::RuntimeArrayType>( |
| 520 | Location loc, spirv::RuntimeArrayType type, uint32_t resultID) { |
| 521 | if (unsigned stride = type.getArrayStride()) { |
| 522 | // OpDecorate %arrayTypeSSA ArrayStride strideLiteral |
| 523 | return emitDecoration(target: resultID, decoration: spirv::Decoration::ArrayStride, params: {stride}); |
| 524 | } |
| 525 | return success(); |
| 526 | } |
| 527 | |
| 528 | LogicalResult Serializer::processMemberDecoration( |
| 529 | uint32_t structID, |
| 530 | const spirv::StructType::MemberDecorationInfo &memberDecoration) { |
| 531 | SmallVector<uint32_t, 4> args( |
| 532 | {structID, memberDecoration.memberIndex, |
| 533 | static_cast<uint32_t>(memberDecoration.decoration)}); |
| 534 | if (memberDecoration.hasValue()) { |
| 535 | args.push_back( |
| 536 | Elt: cast<IntegerAttr>(Val: memberDecoration.decorationValue).getInt()); |
| 537 | } |
| 538 | encodeInstructionInto(binary&: decorations, op: spirv::Opcode::OpMemberDecorate, operands: args); |
| 539 | return success(); |
| 540 | } |
| 541 | |
| 542 | //===----------------------------------------------------------------------===// |
| 543 | // Type |
| 544 | //===----------------------------------------------------------------------===// |
| 545 | |
| 546 | // According to the SPIR-V spec "Validation Rules for Shader Capabilities": |
| 547 | // "Composite objects in the StorageBuffer, PhysicalStorageBuffer, Uniform, and |
| 548 | // PushConstant Storage Classes must be explicitly laid out." |
| 549 | bool Serializer::isInterfaceStructPtrType(Type type) const { |
| 550 | if (auto ptrType = dyn_cast<spirv::PointerType>(Val&: type)) { |
| 551 | switch (ptrType.getStorageClass()) { |
| 552 | case spirv::StorageClass::PhysicalStorageBuffer: |
| 553 | case spirv::StorageClass::PushConstant: |
| 554 | case spirv::StorageClass::StorageBuffer: |
| 555 | case spirv::StorageClass::Uniform: |
| 556 | return isa<spirv::StructType>(Val: ptrType.getPointeeType()); |
| 557 | default: |
| 558 | break; |
| 559 | } |
| 560 | } |
| 561 | return false; |
| 562 | } |
| 563 | |
| 564 | LogicalResult Serializer::processType(Location loc, Type type, |
| 565 | uint32_t &typeID) { |
| 566 | // Maintains a set of names for nested identified struct types. This is used |
| 567 | // to properly serialize recursive references. |
| 568 | SetVector<StringRef> serializationCtx; |
| 569 | return processTypeImpl(loc, type, typeID, serializationCtx); |
| 570 | } |
| 571 | |
| 572 | LogicalResult |
| 573 | Serializer::processTypeImpl(Location loc, Type type, uint32_t &typeID, |
| 574 | SetVector<StringRef> &serializationCtx) { |
| 575 | |
| 576 | // Map unsigned integer types to singless integer types. |
| 577 | // This is needed otherwise the generated spirv assembly will contain |
| 578 | // twice a type declaration (like OpTypeInt 32 0) which is no permitted and |
| 579 | // such module fails validation. Indeed at MLIR level the two types are |
| 580 | // different and lookup in the cache below misses. |
| 581 | // Note: This conversion needs to happen here before the type is looked up in |
| 582 | // the cache. |
| 583 | if (type.isUnsignedInteger()) { |
| 584 | type = IntegerType::get(context: loc->getContext(), width: type.getIntOrFloatBitWidth(), |
| 585 | signedness: IntegerType::SignednessSemantics::Signless); |
| 586 | } |
| 587 | |
| 588 | typeID = getTypeID(type); |
| 589 | if (typeID) |
| 590 | return success(); |
| 591 | |
| 592 | typeID = getNextID(); |
| 593 | SmallVector<uint32_t, 4> operands; |
| 594 | |
| 595 | operands.push_back(Elt: typeID); |
| 596 | auto typeEnum = spirv::Opcode::OpTypeVoid; |
| 597 | bool deferSerialization = false; |
| 598 | |
| 599 | if ((isa<FunctionType>(Val: type) && |
| 600 | succeeded(Result: prepareFunctionType(loc, type: cast<FunctionType>(Val&: type), typeEnum, |
| 601 | operands))) || |
| 602 | (isa<GraphType>(Val: type) && |
| 603 | succeeded( |
| 604 | Result: prepareGraphType(loc, type: cast<GraphType>(Val&: type), typeEnum, operands))) || |
| 605 | succeeded(Result: prepareBasicType(loc, type, resultID: typeID, typeEnum, operands, |
| 606 | deferSerialization, serializationCtx))) { |
| 607 | if (deferSerialization) |
| 608 | return success(); |
| 609 | |
| 610 | typeIDMap[type] = typeID; |
| 611 | |
| 612 | if (typeEnum == spirv::Opcode::OpTypeStruct) |
| 613 | encodeInstructionWithContinuationInto(binary&: typesGlobalValues, op: typeEnum, |
| 614 | operands); |
| 615 | else |
| 616 | encodeInstructionInto(binary&: typesGlobalValues, op: typeEnum, operands); |
| 617 | |
| 618 | if (recursiveStructInfos.count(Val: type) != 0) { |
| 619 | // This recursive struct type is emitted already, now the OpTypePointer |
| 620 | // instructions referring to recursive references are emitted as well. |
| 621 | for (auto &ptrInfo : recursiveStructInfos[type]) { |
| 622 | // TODO: This might not work if more than 1 recursive reference is |
| 623 | // present in the struct. |
| 624 | SmallVector<uint32_t, 4> ptrOperands; |
| 625 | ptrOperands.push_back(Elt: ptrInfo.pointerTypeID); |
| 626 | ptrOperands.push_back(Elt: static_cast<uint32_t>(ptrInfo.storageClass)); |
| 627 | ptrOperands.push_back(Elt: typeIDMap[type]); |
| 628 | |
| 629 | encodeInstructionInto(binary&: typesGlobalValues, op: spirv::Opcode::OpTypePointer, |
| 630 | operands: ptrOperands); |
| 631 | } |
| 632 | |
| 633 | recursiveStructInfos[type].clear(); |
| 634 | } |
| 635 | |
| 636 | return success(); |
| 637 | } |
| 638 | |
| 639 | return emitError(loc, message: "failed to process type: " ) << type; |
| 640 | } |
| 641 | |
| 642 | LogicalResult Serializer::prepareBasicType( |
| 643 | Location loc, Type type, uint32_t resultID, spirv::Opcode &typeEnum, |
| 644 | SmallVectorImpl<uint32_t> &operands, bool &deferSerialization, |
| 645 | SetVector<StringRef> &serializationCtx) { |
| 646 | deferSerialization = false; |
| 647 | |
| 648 | if (isVoidType(type)) { |
| 649 | typeEnum = spirv::Opcode::OpTypeVoid; |
| 650 | return success(); |
| 651 | } |
| 652 | |
| 653 | if (auto intType = dyn_cast<IntegerType>(Val&: type)) { |
| 654 | if (intType.getWidth() == 1) { |
| 655 | typeEnum = spirv::Opcode::OpTypeBool; |
| 656 | return success(); |
| 657 | } |
| 658 | |
| 659 | typeEnum = spirv::Opcode::OpTypeInt; |
| 660 | operands.push_back(Elt: intType.getWidth()); |
| 661 | // SPIR-V OpTypeInt "Signedness specifies whether there are signed semantics |
| 662 | // to preserve or validate. |
| 663 | // 0 indicates unsigned, or no signedness semantics |
| 664 | // 1 indicates signed semantics." |
| 665 | operands.push_back(Elt: intType.isSigned() ? 1 : 0); |
| 666 | return success(); |
| 667 | } |
| 668 | |
| 669 | if (auto floatType = dyn_cast<FloatType>(Val&: type)) { |
| 670 | typeEnum = spirv::Opcode::OpTypeFloat; |
| 671 | operands.push_back(Elt: floatType.getWidth()); |
| 672 | if (floatType.isBF16()) { |
| 673 | operands.push_back(Elt: static_cast<uint32_t>(spirv::FPEncoding::BFloat16KHR)); |
| 674 | } |
| 675 | if (floatType.isF8E4M3FN()) { |
| 676 | operands.push_back( |
| 677 | Elt: static_cast<uint32_t>(spirv::FPEncoding::Float8E4M3EXT)); |
| 678 | } |
| 679 | if (floatType.isF8E5M2()) { |
| 680 | operands.push_back( |
| 681 | Elt: static_cast<uint32_t>(spirv::FPEncoding::Float8E5M2EXT)); |
| 682 | } |
| 683 | |
| 684 | return success(); |
| 685 | } |
| 686 | |
| 687 | if (auto vectorType = dyn_cast<VectorType>(Val&: type)) { |
| 688 | uint32_t elementTypeID = 0; |
| 689 | if (failed(Result: processTypeImpl(loc, type: vectorType.getElementType(), typeID&: elementTypeID, |
| 690 | serializationCtx))) { |
| 691 | return failure(); |
| 692 | } |
| 693 | typeEnum = spirv::Opcode::OpTypeVector; |
| 694 | operands.push_back(Elt: elementTypeID); |
| 695 | operands.push_back(Elt: vectorType.getNumElements()); |
| 696 | return success(); |
| 697 | } |
| 698 | |
| 699 | if (auto imageType = dyn_cast<spirv::ImageType>(Val&: type)) { |
| 700 | typeEnum = spirv::Opcode::OpTypeImage; |
| 701 | uint32_t sampledTypeID = 0; |
| 702 | if (failed(Result: processType(loc, type: imageType.getElementType(), typeID&: sampledTypeID))) |
| 703 | return failure(); |
| 704 | |
| 705 | llvm::append_values(C&: operands, Values&: sampledTypeID, |
| 706 | Values: static_cast<uint32_t>(imageType.getDim()), |
| 707 | Values: static_cast<uint32_t>(imageType.getDepthInfo()), |
| 708 | Values: static_cast<uint32_t>(imageType.getArrayedInfo()), |
| 709 | Values: static_cast<uint32_t>(imageType.getSamplingInfo()), |
| 710 | Values: static_cast<uint32_t>(imageType.getSamplerUseInfo()), |
| 711 | Values: static_cast<uint32_t>(imageType.getImageFormat())); |
| 712 | return success(); |
| 713 | } |
| 714 | |
| 715 | if (auto arrayType = dyn_cast<spirv::ArrayType>(Val&: type)) { |
| 716 | typeEnum = spirv::Opcode::OpTypeArray; |
| 717 | uint32_t elementTypeID = 0; |
| 718 | if (failed(Result: processTypeImpl(loc, type: arrayType.getElementType(), typeID&: elementTypeID, |
| 719 | serializationCtx))) { |
| 720 | return failure(); |
| 721 | } |
| 722 | operands.push_back(Elt: elementTypeID); |
| 723 | if (auto elementCountID = prepareConstantInt( |
| 724 | loc, intAttr: mlirBuilder.getI32IntegerAttr(value: arrayType.getNumElements()))) { |
| 725 | operands.push_back(Elt: elementCountID); |
| 726 | } |
| 727 | return processTypeDecoration(loc, type: arrayType, resultID); |
| 728 | } |
| 729 | |
| 730 | if (auto ptrType = dyn_cast<spirv::PointerType>(Val&: type)) { |
| 731 | uint32_t pointeeTypeID = 0; |
| 732 | spirv::StructType pointeeStruct = |
| 733 | dyn_cast<spirv::StructType>(Val: ptrType.getPointeeType()); |
| 734 | |
| 735 | if (pointeeStruct && pointeeStruct.isIdentified() && |
| 736 | serializationCtx.count(key: pointeeStruct.getIdentifier()) != 0) { |
| 737 | // A recursive reference to an enclosing struct is found. |
| 738 | // |
| 739 | // 1. Prepare an OpTypeForwardPointer with resultID and the ptr storage |
| 740 | // class as operands. |
| 741 | SmallVector<uint32_t, 2> forwardPtrOperands; |
| 742 | forwardPtrOperands.push_back(Elt: resultID); |
| 743 | forwardPtrOperands.push_back( |
| 744 | Elt: static_cast<uint32_t>(ptrType.getStorageClass())); |
| 745 | |
| 746 | encodeInstructionInto(binary&: typesGlobalValues, |
| 747 | op: spirv::Opcode::OpTypeForwardPointer, |
| 748 | operands: forwardPtrOperands); |
| 749 | |
| 750 | // 2. Find the pointee (enclosing) struct. |
| 751 | auto structType = spirv::StructType::getIdentified( |
| 752 | context: module.getContext(), identifier: pointeeStruct.getIdentifier()); |
| 753 | |
| 754 | if (!structType) |
| 755 | return failure(); |
| 756 | |
| 757 | // 3. Mark the OpTypePointer that is supposed to be emitted by this call |
| 758 | // as deferred. |
| 759 | deferSerialization = true; |
| 760 | |
| 761 | // 4. Record the info needed to emit the deferred OpTypePointer |
| 762 | // instruction when the enclosing struct is completely serialized. |
| 763 | recursiveStructInfos[structType].push_back( |
| 764 | Elt: {.pointerTypeID: resultID, .storageClass: ptrType.getStorageClass()}); |
| 765 | } else { |
| 766 | if (failed(Result: processTypeImpl(loc, type: ptrType.getPointeeType(), typeID&: pointeeTypeID, |
| 767 | serializationCtx))) |
| 768 | return failure(); |
| 769 | } |
| 770 | |
| 771 | typeEnum = spirv::Opcode::OpTypePointer; |
| 772 | operands.push_back(Elt: static_cast<uint32_t>(ptrType.getStorageClass())); |
| 773 | operands.push_back(Elt: pointeeTypeID); |
| 774 | |
| 775 | // TODO: Now struct decorations are supported this code may not be |
| 776 | // necessary. However, it is left to support backwards compatibility. |
| 777 | // Ideally, Block decorations should be inserted when converting to SPIR-V. |
| 778 | if (isInterfaceStructPtrType(type: ptrType)) { |
| 779 | auto structType = cast<spirv::StructType>(Val: ptrType.getPointeeType()); |
| 780 | if (!structType.hasDecoration(decoration: spirv::Decoration::Block) && |
| 781 | !structType.hasDecoration(decoration: spirv::Decoration::BufferBlock)) |
| 782 | if (failed(Result: emitDecoration(target: getTypeID(type: pointeeStruct), |
| 783 | decoration: spirv::Decoration::Block))) |
| 784 | return emitError(loc, message: "cannot decorate " ) |
| 785 | << pointeeStruct << " with Block decoration" ; |
| 786 | } |
| 787 | |
| 788 | return success(); |
| 789 | } |
| 790 | |
| 791 | if (auto runtimeArrayType = dyn_cast<spirv::RuntimeArrayType>(Val&: type)) { |
| 792 | uint32_t elementTypeID = 0; |
| 793 | if (failed(Result: processTypeImpl(loc, type: runtimeArrayType.getElementType(), |
| 794 | typeID&: elementTypeID, serializationCtx))) { |
| 795 | return failure(); |
| 796 | } |
| 797 | typeEnum = spirv::Opcode::OpTypeRuntimeArray; |
| 798 | operands.push_back(Elt: elementTypeID); |
| 799 | return processTypeDecoration(loc, type: runtimeArrayType, resultID); |
| 800 | } |
| 801 | |
| 802 | if (isa<spirv::SamplerType>(Val: type)) { |
| 803 | typeEnum = spirv::Opcode::OpTypeSampler; |
| 804 | return success(); |
| 805 | } |
| 806 | |
| 807 | if (isa<spirv::NamedBarrierType>(Val: type)) { |
| 808 | typeEnum = spirv::Opcode::OpTypeNamedBarrier; |
| 809 | return success(); |
| 810 | } |
| 811 | |
| 812 | if (auto sampledImageType = dyn_cast<spirv::SampledImageType>(Val&: type)) { |
| 813 | typeEnum = spirv::Opcode::OpTypeSampledImage; |
| 814 | uint32_t imageTypeID = 0; |
| 815 | if (failed( |
| 816 | Result: processType(loc, type: sampledImageType.getImageType(), typeID&: imageTypeID))) { |
| 817 | return failure(); |
| 818 | } |
| 819 | operands.push_back(Elt: imageTypeID); |
| 820 | return success(); |
| 821 | } |
| 822 | |
| 823 | if (auto structType = dyn_cast<spirv::StructType>(Val&: type)) { |
| 824 | if (structType.isIdentified()) { |
| 825 | if (failed(Result: processName(resultID, name: structType.getIdentifier()))) |
| 826 | return failure(); |
| 827 | serializationCtx.insert(X: structType.getIdentifier()); |
| 828 | } |
| 829 | |
| 830 | bool hasOffset = structType.hasOffset(); |
| 831 | for (auto elementIndex : |
| 832 | llvm::seq<uint32_t>(Begin: 0, End: structType.getNumElements())) { |
| 833 | uint32_t elementTypeID = 0; |
| 834 | if (failed(Result: processTypeImpl(loc, type: structType.getElementType(elementIndex), |
| 835 | typeID&: elementTypeID, serializationCtx))) { |
| 836 | return failure(); |
| 837 | } |
| 838 | operands.push_back(Elt: elementTypeID); |
| 839 | if (hasOffset) { |
| 840 | auto intType = IntegerType::get(context: structType.getContext(), width: 32); |
| 841 | // Decorate each struct member with an offset |
| 842 | spirv::StructType::MemberDecorationInfo offsetDecoration{ |
| 843 | elementIndex, spirv::Decoration::Offset, |
| 844 | IntegerAttr::get(type: intType, |
| 845 | value: structType.getMemberOffset(elementIndex))}; |
| 846 | if (failed(Result: processMemberDecoration(structID: resultID, memberDecoration: offsetDecoration))) { |
| 847 | return emitError(loc, message: "cannot decorate " ) |
| 848 | << elementIndex << "-th member of " << structType |
| 849 | << " with its offset" ; |
| 850 | } |
| 851 | } |
| 852 | } |
| 853 | SmallVector<spirv::StructType::MemberDecorationInfo, 4> memberDecorations; |
| 854 | structType.getMemberDecorations(memberDecorations); |
| 855 | |
| 856 | for (auto &memberDecoration : memberDecorations) { |
| 857 | if (failed(Result: processMemberDecoration(structID: resultID, memberDecoration))) { |
| 858 | return emitError(loc, message: "cannot decorate " ) |
| 859 | << static_cast<uint32_t>(memberDecoration.memberIndex) |
| 860 | << "-th member of " << structType << " with " |
| 861 | << stringifyDecoration(memberDecoration.decoration); |
| 862 | } |
| 863 | } |
| 864 | |
| 865 | SmallVector<spirv::StructType::StructDecorationInfo, 1> structDecorations; |
| 866 | structType.getStructDecorations(structDecorations); |
| 867 | |
| 868 | for (spirv::StructType::StructDecorationInfo &structDecoration : |
| 869 | structDecorations) { |
| 870 | if (failed(Result: processDecorationAttr(loc, resultID, |
| 871 | decoration: structDecoration.decoration, |
| 872 | attr: structDecoration.decorationValue))) { |
| 873 | return emitError(loc, message: "cannot decorate struct " ) |
| 874 | << structType << " with " |
| 875 | << stringifyDecoration(structDecoration.decoration); |
| 876 | } |
| 877 | } |
| 878 | |
| 879 | typeEnum = spirv::Opcode::OpTypeStruct; |
| 880 | |
| 881 | if (structType.isIdentified()) |
| 882 | serializationCtx.remove(X: structType.getIdentifier()); |
| 883 | |
| 884 | return success(); |
| 885 | } |
| 886 | |
| 887 | if (auto cooperativeMatrixType = |
| 888 | dyn_cast<spirv::CooperativeMatrixType>(Val&: type)) { |
| 889 | uint32_t elementTypeID = 0; |
| 890 | if (failed(Result: processTypeImpl(loc, type: cooperativeMatrixType.getElementType(), |
| 891 | typeID&: elementTypeID, serializationCtx))) { |
| 892 | return failure(); |
| 893 | } |
| 894 | typeEnum = spirv::Opcode::OpTypeCooperativeMatrixKHR; |
| 895 | auto getConstantOp = [&](uint32_t id) { |
| 896 | auto attr = IntegerAttr::get(type: IntegerType::get(context: type.getContext(), width: 32), value: id); |
| 897 | return prepareConstantInt(loc, intAttr: attr); |
| 898 | }; |
| 899 | llvm::append_values( |
| 900 | C&: operands, Values&: elementTypeID, |
| 901 | Values: getConstantOp(static_cast<uint32_t>(cooperativeMatrixType.getScope())), |
| 902 | Values: getConstantOp(cooperativeMatrixType.getRows()), |
| 903 | Values: getConstantOp(cooperativeMatrixType.getColumns()), |
| 904 | Values: getConstantOp(static_cast<uint32_t>(cooperativeMatrixType.getUse()))); |
| 905 | return success(); |
| 906 | } |
| 907 | |
| 908 | if (auto matrixType = dyn_cast<spirv::MatrixType>(Val&: type)) { |
| 909 | uint32_t elementTypeID = 0; |
| 910 | if (failed(Result: processTypeImpl(loc, type: matrixType.getColumnType(), typeID&: elementTypeID, |
| 911 | serializationCtx))) { |
| 912 | return failure(); |
| 913 | } |
| 914 | typeEnum = spirv::Opcode::OpTypeMatrix; |
| 915 | llvm::append_values(C&: operands, Values&: elementTypeID, Values: matrixType.getNumColumns()); |
| 916 | return success(); |
| 917 | } |
| 918 | |
| 919 | if (auto tensorArmType = dyn_cast<TensorArmType>(Val&: type)) { |
| 920 | uint32_t elementTypeID = 0; |
| 921 | uint32_t rank = 0; |
| 922 | uint32_t shapeID = 0; |
| 923 | uint32_t rankID = 0; |
| 924 | if (failed(Result: processTypeImpl(loc, type: tensorArmType.getElementType(), |
| 925 | typeID&: elementTypeID, serializationCtx))) { |
| 926 | return failure(); |
| 927 | } |
| 928 | if (tensorArmType.hasRank()) { |
| 929 | ArrayRef<int64_t> dims = tensorArmType.getShape(); |
| 930 | rank = dims.size(); |
| 931 | rankID = prepareConstantInt(loc, intAttr: mlirBuilder.getI32IntegerAttr(value: rank)); |
| 932 | if (rankID == 0) { |
| 933 | return failure(); |
| 934 | } |
| 935 | |
| 936 | bool shaped = llvm::all_of(Range&: dims, P: [](const auto &dim) { return dim > 0; }); |
| 937 | if (rank > 0 && shaped) { |
| 938 | auto I32Type = IntegerType::get(context: type.getContext(), width: 32); |
| 939 | auto shapeType = ArrayType::get(elementType: I32Type, elementCount: rank); |
| 940 | if (rank == 1) { |
| 941 | SmallVector<uint64_t, 1> index(rank); |
| 942 | shapeID = prepareDenseElementsConstant( |
| 943 | loc, constType: shapeType, |
| 944 | valueAttr: mlirBuilder.getI32TensorAttr(values: SmallVector<int32_t>(dims)), dim: 0, |
| 945 | index); |
| 946 | } else { |
| 947 | shapeID = prepareArrayConstant( |
| 948 | loc, constType: shapeType, |
| 949 | attr: mlirBuilder.getI32ArrayAttr(values: SmallVector<int32_t>(dims))); |
| 950 | } |
| 951 | if (shapeID == 0) { |
| 952 | return failure(); |
| 953 | } |
| 954 | } |
| 955 | } |
| 956 | typeEnum = spirv::Opcode::OpTypeTensorARM; |
| 957 | operands.push_back(Elt: elementTypeID); |
| 958 | if (rankID == 0) |
| 959 | return success(); |
| 960 | operands.push_back(Elt: rankID); |
| 961 | if (shapeID == 0) |
| 962 | return success(); |
| 963 | operands.push_back(Elt: shapeID); |
| 964 | return success(); |
| 965 | } |
| 966 | |
| 967 | // TODO: Handle other types. |
| 968 | return emitError(loc, message: "unhandled type in serialization: " ) << type; |
| 969 | } |
| 970 | |
| 971 | LogicalResult |
| 972 | Serializer::prepareFunctionType(Location loc, FunctionType type, |
| 973 | spirv::Opcode &typeEnum, |
| 974 | SmallVectorImpl<uint32_t> &operands) { |
| 975 | typeEnum = spirv::Opcode::OpTypeFunction; |
| 976 | assert(type.getNumResults() <= 1 && |
| 977 | "serialization supports only a single return value" ); |
| 978 | uint32_t resultID = 0; |
| 979 | if (failed(Result: processType( |
| 980 | loc, type: type.getNumResults() == 1 ? type.getResult(i: 0) : getVoidType(), |
| 981 | typeID&: resultID))) { |
| 982 | return failure(); |
| 983 | } |
| 984 | operands.push_back(Elt: resultID); |
| 985 | for (auto &res : type.getInputs()) { |
| 986 | uint32_t argTypeID = 0; |
| 987 | if (failed(Result: processType(loc, type: res, typeID&: argTypeID))) { |
| 988 | return failure(); |
| 989 | } |
| 990 | operands.push_back(Elt: argTypeID); |
| 991 | } |
| 992 | return success(); |
| 993 | } |
| 994 | |
| 995 | LogicalResult |
| 996 | Serializer::prepareGraphType(Location loc, GraphType type, |
| 997 | spirv::Opcode &typeEnum, |
| 998 | SmallVectorImpl<uint32_t> &operands) { |
| 999 | typeEnum = spirv::Opcode::OpTypeGraphARM; |
| 1000 | assert(type.getNumResults() >= 1 && |
| 1001 | "serialization requires at least a return value" ); |
| 1002 | |
| 1003 | operands.push_back(Elt: type.getNumInputs()); |
| 1004 | |
| 1005 | for (Type argType : type.getInputs()) { |
| 1006 | uint32_t argTypeID = 0; |
| 1007 | if (failed(Result: processType(loc, type: argType, typeID&: argTypeID))) |
| 1008 | return failure(); |
| 1009 | operands.push_back(Elt: argTypeID); |
| 1010 | } |
| 1011 | |
| 1012 | for (Type resType : type.getResults()) { |
| 1013 | uint32_t resTypeID = 0; |
| 1014 | if (failed(Result: processType(loc, type: resType, typeID&: resTypeID))) |
| 1015 | return failure(); |
| 1016 | operands.push_back(Elt: resTypeID); |
| 1017 | } |
| 1018 | |
| 1019 | return success(); |
| 1020 | } |
| 1021 | |
| 1022 | //===----------------------------------------------------------------------===// |
| 1023 | // Constant |
| 1024 | //===----------------------------------------------------------------------===// |
| 1025 | |
| 1026 | uint32_t Serializer::prepareConstant(Location loc, Type constType, |
| 1027 | Attribute valueAttr) { |
| 1028 | if (auto id = prepareConstantScalar(loc, valueAttr)) { |
| 1029 | return id; |
| 1030 | } |
| 1031 | |
| 1032 | // This is a composite literal. We need to handle each component separately |
| 1033 | // and then emit an OpConstantComposite for the whole. |
| 1034 | |
| 1035 | if (auto id = getConstantID(value: valueAttr)) { |
| 1036 | return id; |
| 1037 | } |
| 1038 | |
| 1039 | uint32_t typeID = 0; |
| 1040 | if (failed(Result: processType(loc, type: constType, typeID))) { |
| 1041 | return 0; |
| 1042 | } |
| 1043 | |
| 1044 | uint32_t resultID = 0; |
| 1045 | if (auto attr = dyn_cast<DenseElementsAttr>(Val&: valueAttr)) { |
| 1046 | int rank = dyn_cast<ShapedType>(Val: attr.getType()).getRank(); |
| 1047 | SmallVector<uint64_t, 4> index(rank); |
| 1048 | resultID = prepareDenseElementsConstant(loc, constType, valueAttr: attr, |
| 1049 | /*dim=*/0, index); |
| 1050 | } else if (auto arrayAttr = dyn_cast<ArrayAttr>(Val&: valueAttr)) { |
| 1051 | resultID = prepareArrayConstant(loc, constType, attr: arrayAttr); |
| 1052 | } else if (isa<spirv::NullAttr>(Val: valueAttr)) { |
| 1053 | resultID = getNextID(); |
| 1054 | encodeInstructionInto(binary&: typesGlobalValues, op: spirv::Opcode::OpConstantNull, |
| 1055 | operands: {typeID, resultID}); |
| 1056 | } |
| 1057 | |
| 1058 | if (resultID == 0) { |
| 1059 | emitError(loc, message: "cannot serialize attribute: " ) << valueAttr; |
| 1060 | return 0; |
| 1061 | } |
| 1062 | |
| 1063 | constIDMap[valueAttr] = resultID; |
| 1064 | return resultID; |
| 1065 | } |
| 1066 | |
| 1067 | uint32_t Serializer::prepareArrayConstant(Location loc, Type constType, |
| 1068 | ArrayAttr attr) { |
| 1069 | uint32_t typeID = 0; |
| 1070 | if (failed(Result: processType(loc, type: constType, typeID))) { |
| 1071 | return 0; |
| 1072 | } |
| 1073 | |
| 1074 | uint32_t resultID = getNextID(); |
| 1075 | SmallVector<uint32_t, 4> operands = {typeID, resultID}; |
| 1076 | operands.reserve(N: attr.size() + 2); |
| 1077 | spirv::CompositeType compositeType = cast<spirv::CompositeType>(Val&: constType); |
| 1078 | for (auto [idx, elementAttr] : llvm::enumerate(First&: attr)) { |
| 1079 | if (uint32_t elementID = prepareConstant( |
| 1080 | loc, constType: compositeType.getElementType(idx), valueAttr: elementAttr)) { |
| 1081 | operands.push_back(Elt: elementID); |
| 1082 | } else { |
| 1083 | return 0; |
| 1084 | } |
| 1085 | } |
| 1086 | encodeInstructionWithContinuationInto( |
| 1087 | binary&: typesGlobalValues, op: spirv::Opcode::OpConstantComposite, operands); |
| 1088 | |
| 1089 | return resultID; |
| 1090 | } |
| 1091 | |
| 1092 | // TODO: Turn the below function into iterative function, instead of |
| 1093 | // recursive function. |
| 1094 | uint32_t |
| 1095 | Serializer::prepareDenseElementsConstant(Location loc, Type constType, |
| 1096 | DenseElementsAttr valueAttr, int dim, |
| 1097 | MutableArrayRef<uint64_t> index) { |
| 1098 | auto shapedType = dyn_cast<ShapedType>(Val: valueAttr.getType()); |
| 1099 | assert(dim <= shapedType.getRank()); |
| 1100 | if (shapedType.getRank() == dim) { |
| 1101 | if (auto attr = dyn_cast<DenseIntElementsAttr>(Val&: valueAttr)) { |
| 1102 | return attr.getType().getElementType().isInteger(width: 1) |
| 1103 | ? prepareConstantBool(loc, boolAttr: attr.getValues<BoolAttr>()[index]) |
| 1104 | : prepareConstantInt(loc, |
| 1105 | intAttr: attr.getValues<IntegerAttr>()[index]); |
| 1106 | } |
| 1107 | if (auto attr = dyn_cast<DenseFPElementsAttr>(Val&: valueAttr)) { |
| 1108 | return prepareConstantFp(loc, floatAttr: attr.getValues<FloatAttr>()[index]); |
| 1109 | } |
| 1110 | return 0; |
| 1111 | } |
| 1112 | |
| 1113 | uint32_t typeID = 0; |
| 1114 | if (failed(Result: processType(loc, type: constType, typeID))) { |
| 1115 | return 0; |
| 1116 | } |
| 1117 | |
| 1118 | int64_t numberOfConstituents = shapedType.getDimSize(idx: dim); |
| 1119 | uint32_t resultID = getNextID(); |
| 1120 | SmallVector<uint32_t, 4> operands = {typeID, resultID}; |
| 1121 | auto elementType = cast<spirv::CompositeType>(Val&: constType).getElementType(0); |
| 1122 | if (auto tensorArmType = dyn_cast<spirv::TensorArmType>(Val&: constType)) { |
| 1123 | ArrayRef<int64_t> innerShape = tensorArmType.getShape().drop_front(); |
| 1124 | if (!innerShape.empty()) |
| 1125 | elementType = spirv::TensorArmType::get(shape: innerShape, elementType); |
| 1126 | } |
| 1127 | |
| 1128 | // "If the Result Type is a cooperative matrix type, then there must be only |
| 1129 | // one Constituent, with scalar type matching the cooperative matrix Component |
| 1130 | // Type, and all components of the matrix are initialized to that value." |
| 1131 | // (https://github.khronos.org/SPIRV-Registry/extensions/KHR/SPV_KHR_cooperative_matrix.html) |
| 1132 | if (isa<spirv::CooperativeMatrixType>(Val: constType)) { |
| 1133 | if (!valueAttr.isSplat()) { |
| 1134 | emitError( |
| 1135 | loc, |
| 1136 | message: "cannot serialize a non-splat value for a cooperative matrix type" ); |
| 1137 | return 0; |
| 1138 | } |
| 1139 | // numberOfConstituents is 1, so we only need one more elements in the |
| 1140 | // SmallVector, so the total is 3 (1 + 2). |
| 1141 | operands.reserve(N: 3); |
| 1142 | // We set dim directly to `shapedType.getRank()` so the recursive call |
| 1143 | // directly returns the scalar type. |
| 1144 | if (auto elementID = prepareDenseElementsConstant( |
| 1145 | loc, constType: elementType, valueAttr, /*dim=*/shapedType.getRank(), index)) { |
| 1146 | operands.push_back(Elt: elementID); |
| 1147 | } else { |
| 1148 | return 0; |
| 1149 | } |
| 1150 | } else if (isa<spirv::TensorArmType>(Val: constType) && isZeroValue(attr: valueAttr)) { |
| 1151 | encodeInstructionInto(binary&: typesGlobalValues, op: spirv::Opcode::OpConstantNull, |
| 1152 | operands: {typeID, resultID}); |
| 1153 | return resultID; |
| 1154 | } else { |
| 1155 | operands.reserve(N: numberOfConstituents + 2); |
| 1156 | for (int i = 0; i < numberOfConstituents; ++i) { |
| 1157 | index[dim] = i; |
| 1158 | if (auto elementID = prepareDenseElementsConstant( |
| 1159 | loc, constType: elementType, valueAttr, dim: dim + 1, index)) { |
| 1160 | operands.push_back(Elt: elementID); |
| 1161 | } else { |
| 1162 | return 0; |
| 1163 | } |
| 1164 | } |
| 1165 | } |
| 1166 | encodeInstructionWithContinuationInto( |
| 1167 | binary&: typesGlobalValues, op: spirv::Opcode::OpConstantComposite, operands); |
| 1168 | |
| 1169 | return resultID; |
| 1170 | } |
| 1171 | |
| 1172 | uint32_t Serializer::prepareConstantScalar(Location loc, Attribute valueAttr, |
| 1173 | bool isSpec) { |
| 1174 | if (auto floatAttr = dyn_cast<FloatAttr>(Val&: valueAttr)) { |
| 1175 | return prepareConstantFp(loc, floatAttr, isSpec); |
| 1176 | } |
| 1177 | if (auto boolAttr = dyn_cast<BoolAttr>(Val&: valueAttr)) { |
| 1178 | return prepareConstantBool(loc, boolAttr, isSpec); |
| 1179 | } |
| 1180 | if (auto intAttr = dyn_cast<IntegerAttr>(Val&: valueAttr)) { |
| 1181 | return prepareConstantInt(loc, intAttr, isSpec); |
| 1182 | } |
| 1183 | |
| 1184 | return 0; |
| 1185 | } |
| 1186 | |
| 1187 | uint32_t Serializer::prepareConstantBool(Location loc, BoolAttr boolAttr, |
| 1188 | bool isSpec) { |
| 1189 | if (!isSpec) { |
| 1190 | // We can de-duplicate normal constants, but not specialization constants. |
| 1191 | if (auto id = getConstantID(value: boolAttr)) { |
| 1192 | return id; |
| 1193 | } |
| 1194 | } |
| 1195 | |
| 1196 | // Process the type for this bool literal |
| 1197 | uint32_t typeID = 0; |
| 1198 | if (failed(Result: processType(loc, type: cast<IntegerAttr>(Val&: boolAttr).getType(), typeID))) { |
| 1199 | return 0; |
| 1200 | } |
| 1201 | |
| 1202 | auto resultID = getNextID(); |
| 1203 | auto opcode = boolAttr.getValue() |
| 1204 | ? (isSpec ? spirv::Opcode::OpSpecConstantTrue |
| 1205 | : spirv::Opcode::OpConstantTrue) |
| 1206 | : (isSpec ? spirv::Opcode::OpSpecConstantFalse |
| 1207 | : spirv::Opcode::OpConstantFalse); |
| 1208 | encodeInstructionInto(binary&: typesGlobalValues, op: opcode, operands: {typeID, resultID}); |
| 1209 | |
| 1210 | if (!isSpec) { |
| 1211 | constIDMap[boolAttr] = resultID; |
| 1212 | } |
| 1213 | return resultID; |
| 1214 | } |
| 1215 | |
| 1216 | uint32_t Serializer::prepareConstantInt(Location loc, IntegerAttr intAttr, |
| 1217 | bool isSpec) { |
| 1218 | if (!isSpec) { |
| 1219 | // We can de-duplicate normal constants, but not specialization constants. |
| 1220 | if (auto id = getConstantID(value: intAttr)) { |
| 1221 | return id; |
| 1222 | } |
| 1223 | } |
| 1224 | |
| 1225 | // Process the type for this integer literal |
| 1226 | uint32_t typeID = 0; |
| 1227 | if (failed(Result: processType(loc, type: intAttr.getType(), typeID))) { |
| 1228 | return 0; |
| 1229 | } |
| 1230 | |
| 1231 | auto resultID = getNextID(); |
| 1232 | APInt value = intAttr.getValue(); |
| 1233 | unsigned bitwidth = value.getBitWidth(); |
| 1234 | bool isSigned = intAttr.getType().isSignedInteger(); |
| 1235 | auto opcode = |
| 1236 | isSpec ? spirv::Opcode::OpSpecConstant : spirv::Opcode::OpConstant; |
| 1237 | |
| 1238 | switch (bitwidth) { |
| 1239 | // According to SPIR-V spec, "When the type's bit width is less than |
| 1240 | // 32-bits, the literal's value appears in the low-order bits of the word, |
| 1241 | // and the high-order bits must be 0 for a floating-point type, or 0 for an |
| 1242 | // integer type with Signedness of 0, or sign extended when Signedness |
| 1243 | // is 1." |
| 1244 | case 32: |
| 1245 | case 16: |
| 1246 | case 8: { |
| 1247 | uint32_t word = 0; |
| 1248 | if (isSigned) { |
| 1249 | word = static_cast<int32_t>(value.getSExtValue()); |
| 1250 | } else { |
| 1251 | word = static_cast<uint32_t>(value.getZExtValue()); |
| 1252 | } |
| 1253 | encodeInstructionInto(binary&: typesGlobalValues, op: opcode, operands: {typeID, resultID, word}); |
| 1254 | } break; |
| 1255 | // According to SPIR-V spec: "When the type's bit width is larger than one |
| 1256 | // word, the literal’s low-order words appear first." |
| 1257 | case 64: { |
| 1258 | struct DoubleWord { |
| 1259 | uint32_t word1; |
| 1260 | uint32_t word2; |
| 1261 | } words; |
| 1262 | if (isSigned) { |
| 1263 | words = llvm::bit_cast<DoubleWord>(from: value.getSExtValue()); |
| 1264 | } else { |
| 1265 | words = llvm::bit_cast<DoubleWord>(from: value.getZExtValue()); |
| 1266 | } |
| 1267 | encodeInstructionInto(binary&: typesGlobalValues, op: opcode, |
| 1268 | operands: {typeID, resultID, words.word1, words.word2}); |
| 1269 | } break; |
| 1270 | default: { |
| 1271 | std::string valueStr; |
| 1272 | llvm::raw_string_ostream (valueStr); |
| 1273 | value.print(OS&: rss, /*isSigned=*/false); |
| 1274 | |
| 1275 | emitError(loc, message: "cannot serialize " ) |
| 1276 | << bitwidth << "-bit integer literal: " << valueStr; |
| 1277 | return 0; |
| 1278 | } |
| 1279 | } |
| 1280 | |
| 1281 | if (!isSpec) { |
| 1282 | constIDMap[intAttr] = resultID; |
| 1283 | } |
| 1284 | return resultID; |
| 1285 | } |
| 1286 | |
| 1287 | uint32_t Serializer::prepareGraphConstantId(Location loc, Type graphConstType, |
| 1288 | IntegerAttr intAttr) { |
| 1289 | // De-duplicate graph constants. |
| 1290 | if (uint32_t id = getGraphConstantARMId(value: intAttr)) { |
| 1291 | return id; |
| 1292 | } |
| 1293 | |
| 1294 | // Process the type for this graph constant. |
| 1295 | uint32_t typeID = 0; |
| 1296 | if (failed(Result: processType(loc, type: graphConstType, typeID))) { |
| 1297 | return 0; |
| 1298 | } |
| 1299 | |
| 1300 | uint32_t resultID = getNextID(); |
| 1301 | APInt value = intAttr.getValue(); |
| 1302 | unsigned bitwidth = value.getBitWidth(); |
| 1303 | if (bitwidth > 32) { |
| 1304 | emitError(loc, message: "Too wide attribute for OpGraphConstantARM: " ) |
| 1305 | << bitwidth << " bits" ; |
| 1306 | return 0; |
| 1307 | } |
| 1308 | bool isSigned = value.isSignedIntN(N: bitwidth); |
| 1309 | |
| 1310 | uint32_t word = 0; |
| 1311 | if (isSigned) { |
| 1312 | word = static_cast<int32_t>(value.getSExtValue()); |
| 1313 | } else { |
| 1314 | word = static_cast<uint32_t>(value.getZExtValue()); |
| 1315 | } |
| 1316 | encodeInstructionInto(binary&: typesGlobalValues, op: spirv::Opcode::OpGraphConstantARM, |
| 1317 | operands: {typeID, resultID, word}); |
| 1318 | graphConstIDMap[intAttr] = resultID; |
| 1319 | return resultID; |
| 1320 | } |
| 1321 | |
| 1322 | uint32_t Serializer::prepareConstantFp(Location loc, FloatAttr floatAttr, |
| 1323 | bool isSpec) { |
| 1324 | if (!isSpec) { |
| 1325 | // We can de-duplicate normal constants, but not specialization constants. |
| 1326 | if (auto id = getConstantID(value: floatAttr)) { |
| 1327 | return id; |
| 1328 | } |
| 1329 | } |
| 1330 | |
| 1331 | // Process the type for this float literal |
| 1332 | uint32_t typeID = 0; |
| 1333 | if (failed(Result: processType(loc, type: floatAttr.getType(), typeID))) { |
| 1334 | return 0; |
| 1335 | } |
| 1336 | |
| 1337 | auto resultID = getNextID(); |
| 1338 | APFloat value = floatAttr.getValue(); |
| 1339 | const llvm::fltSemantics *semantics = &value.getSemantics(); |
| 1340 | |
| 1341 | auto opcode = |
| 1342 | isSpec ? spirv::Opcode::OpSpecConstant : spirv::Opcode::OpConstant; |
| 1343 | |
| 1344 | if (semantics == &APFloat::IEEEsingle()) { |
| 1345 | uint32_t word = llvm::bit_cast<uint32_t>(from: value.convertToFloat()); |
| 1346 | encodeInstructionInto(binary&: typesGlobalValues, op: opcode, operands: {typeID, resultID, word}); |
| 1347 | } else if (semantics == &APFloat::IEEEdouble()) { |
| 1348 | struct DoubleWord { |
| 1349 | uint32_t word1; |
| 1350 | uint32_t word2; |
| 1351 | } words = llvm::bit_cast<DoubleWord>(from: value.convertToDouble()); |
| 1352 | encodeInstructionInto(binary&: typesGlobalValues, op: opcode, |
| 1353 | operands: {typeID, resultID, words.word1, words.word2}); |
| 1354 | } else if (llvm::is_contained(Set: {&APFloat::IEEEhalf(), &APFloat::BFloat(), |
| 1355 | &APFloat::Float8E4M3FN(), |
| 1356 | &APFloat::Float8E5M2()}, |
| 1357 | Element: semantics)) { |
| 1358 | uint32_t word = |
| 1359 | static_cast<uint32_t>(value.bitcastToAPInt().getZExtValue()); |
| 1360 | encodeInstructionInto(binary&: typesGlobalValues, op: opcode, operands: {typeID, resultID, word}); |
| 1361 | } else { |
| 1362 | std::string valueStr; |
| 1363 | llvm::raw_string_ostream (valueStr); |
| 1364 | value.print(rss); |
| 1365 | |
| 1366 | emitError(loc, message: "cannot serialize " ) |
| 1367 | << floatAttr.getType() << "-typed float literal: " << valueStr; |
| 1368 | return 0; |
| 1369 | } |
| 1370 | |
| 1371 | if (!isSpec) { |
| 1372 | constIDMap[floatAttr] = resultID; |
| 1373 | } |
| 1374 | return resultID; |
| 1375 | } |
| 1376 | |
| 1377 | // Returns type of attribute. In case of a TypedAttr this will simply return |
| 1378 | // the type. But for an ArrayAttr which is untyped and can be multidimensional |
| 1379 | // it creates the ArrayType recursively. |
| 1380 | static Type getValueType(Attribute attr) { |
| 1381 | if (auto typedAttr = dyn_cast<TypedAttr>(Val&: attr)) { |
| 1382 | return typedAttr.getType(); |
| 1383 | } |
| 1384 | |
| 1385 | if (auto arrayAttr = dyn_cast<ArrayAttr>(Val&: attr)) { |
| 1386 | return spirv::ArrayType::get(elementType: getValueType(attr: arrayAttr[0]), elementCount: arrayAttr.size()); |
| 1387 | } |
| 1388 | |
| 1389 | return nullptr; |
| 1390 | } |
| 1391 | |
| 1392 | uint32_t Serializer::prepareConstantCompositeReplicate(Location loc, |
| 1393 | Type resultType, |
| 1394 | Attribute valueAttr) { |
| 1395 | std::pair<Attribute, Type> valueTypePair{valueAttr, resultType}; |
| 1396 | if (uint32_t id = getConstantCompositeReplicateID(valueTypePair)) { |
| 1397 | return id; |
| 1398 | } |
| 1399 | |
| 1400 | uint32_t typeID = 0; |
| 1401 | if (failed(Result: processType(loc, type: resultType, typeID))) { |
| 1402 | return 0; |
| 1403 | } |
| 1404 | |
| 1405 | Type valueType = getValueType(attr: valueAttr); |
| 1406 | if (!valueAttr) |
| 1407 | return 0; |
| 1408 | |
| 1409 | auto compositeType = dyn_cast<CompositeType>(Val&: resultType); |
| 1410 | if (!compositeType) |
| 1411 | return 0; |
| 1412 | Type elementType = compositeType.getElementType(0); |
| 1413 | |
| 1414 | uint32_t constandID; |
| 1415 | if (elementType == valueType) { |
| 1416 | constandID = prepareConstant(loc, constType: elementType, valueAttr); |
| 1417 | } else { |
| 1418 | constandID = prepareConstantCompositeReplicate(loc, resultType: elementType, valueAttr); |
| 1419 | } |
| 1420 | |
| 1421 | uint32_t resultID = getNextID(); |
| 1422 | if (dyn_cast<spirv::TensorArmType>(Val&: resultType) && isZeroValue(attr: valueAttr)) { |
| 1423 | encodeInstructionInto(binary&: typesGlobalValues, op: spirv::Opcode::OpConstantNull, |
| 1424 | operands: {typeID, resultID}); |
| 1425 | } else { |
| 1426 | encodeInstructionInto(binary&: typesGlobalValues, |
| 1427 | op: spirv::Opcode::OpConstantCompositeReplicateEXT, |
| 1428 | operands: {typeID, resultID, constandID}); |
| 1429 | } |
| 1430 | |
| 1431 | constCompositeReplicateIDMap[valueTypePair] = resultID; |
| 1432 | return resultID; |
| 1433 | } |
| 1434 | |
| 1435 | //===----------------------------------------------------------------------===// |
| 1436 | // Control flow |
| 1437 | //===----------------------------------------------------------------------===// |
| 1438 | |
| 1439 | uint32_t Serializer::getOrCreateBlockID(Block *block) { |
| 1440 | if (uint32_t id = getBlockID(block)) |
| 1441 | return id; |
| 1442 | return blockIDMap[block] = getNextID(); |
| 1443 | } |
| 1444 | |
| 1445 | #ifndef NDEBUG |
| 1446 | void Serializer::printBlock(Block *block, raw_ostream &os) { |
| 1447 | os << "block " << block << " (id = " ; |
| 1448 | if (uint32_t id = getBlockID(block)) |
| 1449 | os << id; |
| 1450 | else |
| 1451 | os << "unknown" ; |
| 1452 | os << ")\n" ; |
| 1453 | } |
| 1454 | #endif |
| 1455 | |
| 1456 | LogicalResult |
| 1457 | Serializer::processBlock(Block *block, bool omitLabel, |
| 1458 | function_ref<LogicalResult()> emitMerge) { |
| 1459 | LLVM_DEBUG(llvm::dbgs() << "processing block " << block << ":\n" ); |
| 1460 | LLVM_DEBUG(block->print(llvm::dbgs())); |
| 1461 | LLVM_DEBUG(llvm::dbgs() << '\n'); |
| 1462 | if (!omitLabel) { |
| 1463 | uint32_t blockID = getOrCreateBlockID(block); |
| 1464 | LLVM_DEBUG(printBlock(block, llvm::dbgs())); |
| 1465 | |
| 1466 | // Emit OpLabel for this block. |
| 1467 | encodeInstructionInto(binary&: functionBody, op: spirv::Opcode::OpLabel, operands: {blockID}); |
| 1468 | } |
| 1469 | |
| 1470 | // Emit OpPhi instructions for block arguments, if any. |
| 1471 | if (failed(Result: emitPhiForBlockArguments(block))) |
| 1472 | return failure(); |
| 1473 | |
| 1474 | // If we need to emit merge instructions, it must happen in this block. Check |
| 1475 | // whether we have other structured control flow ops, which will be expanded |
| 1476 | // into multiple basic blocks. If that's the case, we need to emit the merge |
| 1477 | // right now and then create new blocks for further serialization of the ops |
| 1478 | // in this block. |
| 1479 | if (emitMerge && |
| 1480 | llvm::any_of(Range&: block->getOperations(), |
| 1481 | P: llvm::IsaPred<spirv::LoopOp, spirv::SelectionOp>)) { |
| 1482 | if (failed(Result: emitMerge())) |
| 1483 | return failure(); |
| 1484 | emitMerge = nullptr; |
| 1485 | |
| 1486 | // Start a new block for further serialization. |
| 1487 | uint32_t blockID = getNextID(); |
| 1488 | encodeInstructionInto(binary&: functionBody, op: spirv::Opcode::OpBranch, operands: {blockID}); |
| 1489 | encodeInstructionInto(binary&: functionBody, op: spirv::Opcode::OpLabel, operands: {blockID}); |
| 1490 | } |
| 1491 | |
| 1492 | // Process each op in this block except the terminator. |
| 1493 | for (Operation &op : llvm::drop_end(RangeOrContainer&: *block)) { |
| 1494 | if (failed(Result: processOperation(op: &op))) |
| 1495 | return failure(); |
| 1496 | } |
| 1497 | |
| 1498 | // Process the terminator. |
| 1499 | if (emitMerge) |
| 1500 | if (failed(Result: emitMerge())) |
| 1501 | return failure(); |
| 1502 | if (failed(Result: processOperation(op: &block->back()))) |
| 1503 | return failure(); |
| 1504 | |
| 1505 | return success(); |
| 1506 | } |
| 1507 | |
| 1508 | LogicalResult Serializer::emitPhiForBlockArguments(Block *block) { |
| 1509 | // Nothing to do if this block has no arguments or it's the entry block, which |
| 1510 | // always has the same arguments as the function signature. |
| 1511 | if (block->args_empty() || block->isEntryBlock()) |
| 1512 | return success(); |
| 1513 | |
| 1514 | LLVM_DEBUG(llvm::dbgs() << "emitting phi instructions..\n" ); |
| 1515 | |
| 1516 | // If the block has arguments, we need to create SPIR-V OpPhi instructions. |
| 1517 | // A SPIR-V OpPhi instruction is of the syntax: |
| 1518 | // OpPhi | result type | result <id> | (value <id>, parent block <id>) pair |
| 1519 | // So we need to collect all predecessor blocks and the arguments they send |
| 1520 | // to this block. |
| 1521 | SmallVector<std::pair<Block *, OperandRange>, 4> predecessors; |
| 1522 | for (Block *mlirPredecessor : block->getPredecessors()) { |
| 1523 | auto *terminator = mlirPredecessor->getTerminator(); |
| 1524 | LLVM_DEBUG(llvm::dbgs() << " mlir predecessor " ); |
| 1525 | LLVM_DEBUG(printBlock(mlirPredecessor, llvm::dbgs())); |
| 1526 | LLVM_DEBUG(llvm::dbgs() << " terminator: " << *terminator << "\n" ); |
| 1527 | // The predecessor here is the immediate one according to MLIR's IR |
| 1528 | // structure. It does not directly map to the incoming parent block for the |
| 1529 | // OpPhi instructions at SPIR-V binary level. This is because structured |
| 1530 | // control flow ops are serialized to multiple SPIR-V blocks. If there is a |
| 1531 | // spirv.mlir.selection/spirv.mlir.loop op in the MLIR predecessor block, |
| 1532 | // the branch op jumping to the OpPhi's block then resides in the previous |
| 1533 | // structured control flow op's merge block. |
| 1534 | Block *spirvPredecessor = getPhiIncomingBlock(block: mlirPredecessor); |
| 1535 | LLVM_DEBUG(llvm::dbgs() << " spirv predecessor " ); |
| 1536 | LLVM_DEBUG(printBlock(spirvPredecessor, llvm::dbgs())); |
| 1537 | if (auto branchOp = dyn_cast<spirv::BranchOp>(Val: terminator)) { |
| 1538 | predecessors.emplace_back(Args&: spirvPredecessor, Args: branchOp.getOperands()); |
| 1539 | } else if (auto branchCondOp = |
| 1540 | dyn_cast<spirv::BranchConditionalOp>(Val: terminator)) { |
| 1541 | std::optional<OperandRange> blockOperands; |
| 1542 | if (branchCondOp.getTrueTarget() == block) { |
| 1543 | blockOperands = branchCondOp.getTrueTargetOperands(); |
| 1544 | } else { |
| 1545 | assert(branchCondOp.getFalseTarget() == block); |
| 1546 | blockOperands = branchCondOp.getFalseTargetOperands(); |
| 1547 | } |
| 1548 | assert(!blockOperands->empty() && |
| 1549 | "expected non-empty block operand range" ); |
| 1550 | predecessors.emplace_back(Args&: spirvPredecessor, Args&: *blockOperands); |
| 1551 | } else if (auto switchOp = dyn_cast<spirv::SwitchOp>(Val: terminator)) { |
| 1552 | std::optional<OperandRange> blockOperands; |
| 1553 | if (block == switchOp.getDefaultTarget()) { |
| 1554 | blockOperands = switchOp.getDefaultOperands(); |
| 1555 | } else { |
| 1556 | SuccessorRange targets = switchOp.getTargets(); |
| 1557 | auto it = llvm::find(Range&: targets, Val: block); |
| 1558 | assert(it != targets.end()); |
| 1559 | size_t index = std::distance(first: targets.begin(), last: it); |
| 1560 | blockOperands = switchOp.getTargetOperands(index); |
| 1561 | } |
| 1562 | assert(!blockOperands->empty() && |
| 1563 | "expected non-empty block operand range" ); |
| 1564 | predecessors.emplace_back(Args&: spirvPredecessor, Args&: *blockOperands); |
| 1565 | } else { |
| 1566 | return terminator->emitError(message: "unimplemented terminator for Phi creation" ); |
| 1567 | } |
| 1568 | LLVM_DEBUG({ |
| 1569 | llvm::dbgs() << " block arguments:\n" ; |
| 1570 | for (Value v : predecessors.back().second) |
| 1571 | llvm::dbgs() << " " << v << "\n" ; |
| 1572 | }); |
| 1573 | } |
| 1574 | |
| 1575 | // Then create OpPhi instruction for each of the block argument. |
| 1576 | for (auto argIndex : llvm::seq<unsigned>(Begin: 0, End: block->getNumArguments())) { |
| 1577 | BlockArgument arg = block->getArgument(i: argIndex); |
| 1578 | |
| 1579 | // Get the type <id> and result <id> for this OpPhi instruction. |
| 1580 | uint32_t phiTypeID = 0; |
| 1581 | if (failed(Result: processType(loc: arg.getLoc(), type: arg.getType(), typeID&: phiTypeID))) |
| 1582 | return failure(); |
| 1583 | uint32_t phiID = getNextID(); |
| 1584 | |
| 1585 | LLVM_DEBUG(llvm::dbgs() << "[phi] for block argument #" << argIndex << ' ' |
| 1586 | << arg << " (id = " << phiID << ")\n" ); |
| 1587 | |
| 1588 | // Prepare the (value <id>, parent block <id>) pairs. |
| 1589 | SmallVector<uint32_t, 8> phiArgs; |
| 1590 | phiArgs.push_back(Elt: phiTypeID); |
| 1591 | phiArgs.push_back(Elt: phiID); |
| 1592 | |
| 1593 | for (auto predIndex : llvm::seq<unsigned>(Begin: 0, End: predecessors.size())) { |
| 1594 | Value value = predecessors[predIndex].second[argIndex]; |
| 1595 | uint32_t predBlockId = getOrCreateBlockID(block: predecessors[predIndex].first); |
| 1596 | LLVM_DEBUG(llvm::dbgs() << "[phi] use predecessor (id = " << predBlockId |
| 1597 | << ") value " << value << ' '); |
| 1598 | // Each pair is a value <id> ... |
| 1599 | uint32_t valueId = getValueID(val: value); |
| 1600 | if (valueId == 0) { |
| 1601 | // The op generating this value hasn't been visited yet so we don't have |
| 1602 | // an <id> assigned yet. Record this to fix up later. |
| 1603 | LLVM_DEBUG(llvm::dbgs() << "(need to fix)\n" ); |
| 1604 | deferredPhiValues[value].push_back(Elt: functionBody.size() + 1 + |
| 1605 | phiArgs.size()); |
| 1606 | } else { |
| 1607 | LLVM_DEBUG(llvm::dbgs() << "(id = " << valueId << ")\n" ); |
| 1608 | } |
| 1609 | phiArgs.push_back(Elt: valueId); |
| 1610 | // ... and a parent block <id>. |
| 1611 | phiArgs.push_back(Elt: predBlockId); |
| 1612 | } |
| 1613 | |
| 1614 | encodeInstructionInto(binary&: functionBody, op: spirv::Opcode::OpPhi, operands: phiArgs); |
| 1615 | valueIDMap[arg] = phiID; |
| 1616 | } |
| 1617 | |
| 1618 | return success(); |
| 1619 | } |
| 1620 | |
| 1621 | //===----------------------------------------------------------------------===// |
| 1622 | // Operation |
| 1623 | //===----------------------------------------------------------------------===// |
| 1624 | |
| 1625 | LogicalResult Serializer::encodeExtensionInstruction( |
| 1626 | Operation *op, StringRef extensionSetName, uint32_t extensionOpcode, |
| 1627 | ArrayRef<uint32_t> operands, SmallVectorImpl<uint32_t> &binary) { |
| 1628 | // Check if the extension has been imported. |
| 1629 | auto &setID = extendedInstSetIDMap[extensionSetName]; |
| 1630 | if (!setID) { |
| 1631 | setID = getNextID(); |
| 1632 | SmallVector<uint32_t, 16> importOperands; |
| 1633 | importOperands.push_back(Elt: setID); |
| 1634 | spirv::encodeStringLiteralInto(binary&: importOperands, literal: extensionSetName); |
| 1635 | encodeInstructionInto(binary&: extendedSets, op: spirv::Opcode::OpExtInstImport, |
| 1636 | operands: importOperands); |
| 1637 | } |
| 1638 | |
| 1639 | // The first two operands are the result type <id> and result <id>. The set |
| 1640 | // <id> and the opcode need to be insert after this. |
| 1641 | if (operands.size() < 2) { |
| 1642 | return op->emitError(message: "extended instructions must have a result encoding" ); |
| 1643 | } |
| 1644 | SmallVector<uint32_t, 8> extInstOperands; |
| 1645 | extInstOperands.reserve(N: operands.size() + 2); |
| 1646 | extInstOperands.append(in_start: operands.begin(), in_end: std::next(x: operands.begin(), n: 2)); |
| 1647 | extInstOperands.push_back(Elt: setID); |
| 1648 | extInstOperands.push_back(Elt: extensionOpcode); |
| 1649 | extInstOperands.append(in_start: std::next(x: operands.begin(), n: 2), in_end: operands.end()); |
| 1650 | encodeInstructionInto(binary, op: spirv::Opcode::OpExtInst, operands: extInstOperands); |
| 1651 | return success(); |
| 1652 | } |
| 1653 | |
| 1654 | LogicalResult Serializer::encodeExtensionInstruction( |
| 1655 | Operation *op, StringRef extensionSetName, uint32_t extensionOpcode, |
| 1656 | ArrayRef<uint32_t> operands) { |
| 1657 | if (failed(Result: encodeExtensionInstruction(op, extensionSetName, extensionOpcode, |
| 1658 | operands, binary&: functionBody))) |
| 1659 | return failure(); |
| 1660 | |
| 1661 | if (extensionSetName == extTosa) |
| 1662 | updateTosaOpsMap(op); |
| 1663 | |
| 1664 | return success(); |
| 1665 | } |
| 1666 | |
| 1667 | LogicalResult Serializer::processOperation(Operation *opInst) { |
| 1668 | LLVM_DEBUG(llvm::dbgs() << "[op] '" << opInst->getName() << "'\n" ); |
| 1669 | |
| 1670 | // First dispatch the ops that do not directly mirror an instruction from |
| 1671 | // the SPIR-V spec. |
| 1672 | return TypeSwitch<Operation *, LogicalResult>(opInst) |
| 1673 | .Case(caseFn: [&](spirv::AddressOfOp op) { return processAddressOfOp(addressOfOp: op); }) |
| 1674 | .Case(caseFn: [&](spirv::BranchOp op) { return processBranchOp(branchOp: op); }) |
| 1675 | .Case(caseFn: [&](spirv::BranchConditionalOp op) { |
| 1676 | return processBranchConditionalOp(op); |
| 1677 | }) |
| 1678 | .Case(caseFn: [&](spirv::ConstantOp op) { return processConstantOp(op); }) |
| 1679 | .Case(caseFn: [&](spirv::CompositeConstructOp op) { |
| 1680 | return processCompositeConstructOp(op); |
| 1681 | }) |
| 1682 | .Case(caseFn: [&](spirv::EXTConstantCompositeReplicateOp op) { |
| 1683 | return processConstantCompositeReplicateOp(op); |
| 1684 | }) |
| 1685 | .Case(caseFn: [&](spirv::FuncOp op) { return processFuncOp(op); }) |
| 1686 | .Case(caseFn: [&](spirv::GraphARMOp op) { return processGraphARMOp(op); }) |
| 1687 | .Case(caseFn: [&](spirv::GraphEntryPointARMOp op) { |
| 1688 | return processGraphEntryPointARMOp(op); |
| 1689 | }) |
| 1690 | .Case(caseFn: [&](spirv::GraphOutputsARMOp op) { |
| 1691 | return processGraphOutputsARMOp(op); |
| 1692 | }) |
| 1693 | .Case(caseFn: [&](spirv::GlobalVariableOp op) { |
| 1694 | return processGlobalVariableOp(varOp: op); |
| 1695 | }) |
| 1696 | .Case(caseFn: [&](spirv::GraphConstantARMOp op) { |
| 1697 | return processGraphConstantARMOp(op); |
| 1698 | }) |
| 1699 | .Case(caseFn: [&](spirv::LoopOp op) { return processLoopOp(loopOp: op); }) |
| 1700 | .Case(caseFn: [&](spirv::ReferenceOfOp op) { return processReferenceOfOp(referenceOfOp: op); }) |
| 1701 | .Case(caseFn: [&](spirv::SelectionOp op) { return processSelectionOp(selectionOp: op); }) |
| 1702 | .Case(caseFn: [&](spirv::SpecConstantOp op) { return processSpecConstantOp(op); }) |
| 1703 | .Case(caseFn: [&](spirv::SpecConstantCompositeOp op) { |
| 1704 | return processSpecConstantCompositeOp(op); |
| 1705 | }) |
| 1706 | .Case(caseFn: [&](spirv::EXTSpecConstantCompositeReplicateOp op) { |
| 1707 | return processSpecConstantCompositeReplicateOp(op); |
| 1708 | }) |
| 1709 | .Case(caseFn: [&](spirv::SpecConstantOperationOp op) { |
| 1710 | return processSpecConstantOperationOp(op); |
| 1711 | }) |
| 1712 | .Case(caseFn: [&](spirv::SwitchOp op) { return processSwitchOp(switchOp: op); }) |
| 1713 | .Case(caseFn: [&](spirv::UndefOp op) { return processUndefOp(op); }) |
| 1714 | .Case(caseFn: [&](spirv::VariableOp op) { return processVariableOp(op); }) |
| 1715 | |
| 1716 | // Then handle all the ops that directly mirror SPIR-V instructions with |
| 1717 | // auto-generated methods. |
| 1718 | .Default( |
| 1719 | defaultFn: [&](Operation *op) { return dispatchToAutogenSerialization(op); }); |
| 1720 | } |
| 1721 | |
| 1722 | LogicalResult |
| 1723 | Serializer::processCompositeConstructOp(spirv::CompositeConstructOp op) { |
| 1724 | Location loc = op.getLoc(); |
| 1725 | |
| 1726 | uint32_t resultTypeID = 0; |
| 1727 | if (failed(Result: processType(loc, type: op.getType(), typeID&: resultTypeID))) |
| 1728 | return failure(); |
| 1729 | |
| 1730 | uint32_t resultID = getNextID(); |
| 1731 | valueIDMap[op.getResult()] = resultID; |
| 1732 | |
| 1733 | SmallVector<uint32_t, 8> operands; |
| 1734 | operands.reserve(N: 2 + op.getConstituents().size()); |
| 1735 | operands.push_back(Elt: resultTypeID); |
| 1736 | operands.push_back(Elt: resultID); |
| 1737 | for (Value constituent : op.getConstituents()) { |
| 1738 | uint32_t id = getValueID(val: constituent); |
| 1739 | assert(id && "use before def!" ); |
| 1740 | operands.push_back(Elt: id); |
| 1741 | } |
| 1742 | |
| 1743 | if (failed(Result: emitDebugLine(binary&: functionBody, loc))) |
| 1744 | return failure(); |
| 1745 | |
| 1746 | encodeInstructionWithContinuationInto( |
| 1747 | binary&: functionBody, op: spirv::Opcode::OpCompositeConstruct, operands); |
| 1748 | |
| 1749 | for (auto attr : op->getDiscardableAttrDictionary().getValue()) { |
| 1750 | if (failed(Result: processDecoration(loc, resultID, attr))) |
| 1751 | return failure(); |
| 1752 | } |
| 1753 | |
| 1754 | return success(); |
| 1755 | } |
| 1756 | |
| 1757 | LogicalResult Serializer::processOpWithoutGrammarAttr(Operation *op, |
| 1758 | StringRef extInstSet, |
| 1759 | uint32_t opcode) { |
| 1760 | SmallVector<uint32_t, 4> operands; |
| 1761 | Location loc = op->getLoc(); |
| 1762 | |
| 1763 | uint32_t resultID = 0; |
| 1764 | if (op->getNumResults() != 0) { |
| 1765 | uint32_t resultTypeID = 0; |
| 1766 | if (failed(Result: processType(loc, type: op->getResult(idx: 0).getType(), typeID&: resultTypeID))) |
| 1767 | return failure(); |
| 1768 | operands.push_back(Elt: resultTypeID); |
| 1769 | |
| 1770 | resultID = getNextID(); |
| 1771 | operands.push_back(Elt: resultID); |
| 1772 | valueIDMap[op->getResult(idx: 0)] = resultID; |
| 1773 | }; |
| 1774 | |
| 1775 | for (Value operand : op->getOperands()) |
| 1776 | operands.push_back(Elt: getValueID(val: operand)); |
| 1777 | |
| 1778 | if (extInstSet != extTosa) |
| 1779 | // OpLine cannot be present in graphs |
| 1780 | if (failed(Result: emitDebugLine(binary&: functionBody, loc))) |
| 1781 | return failure(); |
| 1782 | |
| 1783 | if (extInstSet.empty()) { |
| 1784 | encodeInstructionInto(binary&: functionBody, op: static_cast<spirv::Opcode>(opcode), |
| 1785 | operands); |
| 1786 | } else { |
| 1787 | if (failed(Result: encodeExtensionInstruction(op, extensionSetName: extInstSet, extensionOpcode: opcode, operands))) |
| 1788 | return failure(); |
| 1789 | } |
| 1790 | |
| 1791 | if (op->getNumResults() != 0) { |
| 1792 | for (auto attr : op->getDiscardableAttrDictionary().getValue()) { |
| 1793 | if (failed(Result: processDecoration(loc, resultID, attr))) |
| 1794 | return failure(); |
| 1795 | } |
| 1796 | } |
| 1797 | |
| 1798 | return success(); |
| 1799 | } |
| 1800 | |
| 1801 | void Serializer::updateTosaOpsMap(Operation *op) { |
| 1802 | if (!options.emitDebugInfo) |
| 1803 | return; |
| 1804 | |
| 1805 | if (auto graphOp = dyn_cast<spirv::GraphARMOp>(Val: op->getParentOp())) { |
| 1806 | if (uint32_t graphID = getFunctionID(fnName: graphOp.getName())) |
| 1807 | tosaOpsMap[graphID][op->getLoc()].insert(X: op); |
| 1808 | } |
| 1809 | } |
| 1810 | |
| 1811 | LogicalResult Serializer::emitDecoration(uint32_t target, |
| 1812 | spirv::Decoration decoration, |
| 1813 | ArrayRef<uint32_t> params) { |
| 1814 | uint32_t wordCount = 3 + params.size(); |
| 1815 | llvm::append_values( |
| 1816 | C&: decorations, |
| 1817 | Values: spirv::getPrefixedOpcode(wordCount, opcode: spirv::Opcode::OpDecorate), Values&: target, |
| 1818 | Values: static_cast<uint32_t>(decoration)); |
| 1819 | llvm::append_range(C&: decorations, R&: params); |
| 1820 | return success(); |
| 1821 | } |
| 1822 | |
| 1823 | LogicalResult Serializer::emitDecorationId(uint32_t target, |
| 1824 | spirv::Decoration decoration, |
| 1825 | ArrayRef<uint32_t> operandIds) { |
| 1826 | uint32_t wordCount = 3 + operandIds.size(); |
| 1827 | llvm::append_values( |
| 1828 | C&: decorations, |
| 1829 | Values: spirv::getPrefixedOpcode(wordCount, opcode: spirv::Opcode::OpDecorateId), Values&: target, |
| 1830 | Values: static_cast<uint32_t>(decoration)); |
| 1831 | llvm::append_range(C&: decorations, R&: operandIds); |
| 1832 | return success(); |
| 1833 | } |
| 1834 | |
| 1835 | LogicalResult Serializer::emitDebugLine(SmallVectorImpl<uint32_t> &binary, |
| 1836 | Location loc) { |
| 1837 | if (!options.emitDebugInfo) |
| 1838 | return success(); |
| 1839 | |
| 1840 | if (lastProcessedWasMergeInst) { |
| 1841 | lastProcessedWasMergeInst = false; |
| 1842 | return success(); |
| 1843 | } |
| 1844 | |
| 1845 | auto fileLoc = dyn_cast<FileLineColLoc>(Val&: loc); |
| 1846 | if (fileLoc) |
| 1847 | encodeInstructionInto(binary, op: spirv::Opcode::OpLine, |
| 1848 | operands: {fileID, fileLoc.getLine(), fileLoc.getColumn()}); |
| 1849 | return success(); |
| 1850 | } |
| 1851 | } // namespace spirv |
| 1852 | } // namespace mlir |
| 1853 | |