| 1 | //===- MemoryOps.cpp - MLIR SPIR-V Memory Ops ----------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Defines the memory operations in the SPIR-V dialect. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h" |
| 14 | #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h" |
| 15 | |
| 16 | #include "SPIRVOpUtils.h" |
| 17 | #include "SPIRVParsingUtils.h" |
| 18 | #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h" |
| 19 | #include "mlir/IR/Diagnostics.h" |
| 20 | |
| 21 | #include "llvm/ADT/StringExtras.h" |
| 22 | #include "llvm/Support/Casting.h" |
| 23 | |
| 24 | using namespace mlir::spirv::AttrNames; |
| 25 | |
| 26 | namespace mlir::spirv { |
| 27 | |
| 28 | /// Parses optional memory access (a.k.a. memory operand) attributes attached to |
| 29 | /// a memory access operand/pointer. Specifically, parses the following syntax: |
| 30 | /// (`[` memory-access `]`)? |
| 31 | /// where: |
| 32 | /// memory-access ::= `"None"` | `"Volatile"` | `"Aligned", ` |
| 33 | /// integer-literal | `"NonTemporal"` |
| 34 | template <typename MemoryOpTy> |
| 35 | ParseResult parseMemoryAccessAttributes(OpAsmParser &parser, |
| 36 | OperationState &state) { |
| 37 | // Parse an optional list of attributes staring with '[' |
| 38 | if (parser.parseOptionalLSquare()) { |
| 39 | // Nothing to do |
| 40 | return success(); |
| 41 | } |
| 42 | |
| 43 | spirv::MemoryAccess memoryAccessAttr; |
| 44 | StringAttr memoryAccessAttrName = |
| 45 | MemoryOpTy::getMemoryAccessAttrName(state.name); |
| 46 | if (spirv::parseEnumStrAttr<spirv::MemoryAccessAttr>( |
| 47 | value&: memoryAccessAttr, parser, state, attrName: memoryAccessAttrName)) |
| 48 | return failure(); |
| 49 | |
| 50 | if (spirv::bitEnumContainsAll(bits: memoryAccessAttr, |
| 51 | bit: spirv::MemoryAccess::Aligned)) { |
| 52 | // Parse integer attribute for alignment. |
| 53 | Attribute alignmentAttr; |
| 54 | StringAttr alignmentAttrName = MemoryOpTy::getAlignmentAttrName(state.name); |
| 55 | Type i32Type = parser.getBuilder().getIntegerType(width: 32); |
| 56 | if (parser.parseComma() || |
| 57 | parser.parseAttribute(result&: alignmentAttr, type: i32Type, attrName: alignmentAttrName, |
| 58 | attrs&: state.attributes)) { |
| 59 | return failure(); |
| 60 | } |
| 61 | } |
| 62 | return parser.parseRSquare(); |
| 63 | } |
| 64 | |
| 65 | // TODO Make sure to merge this and the previous function into one template |
| 66 | // parameterized by memory access attribute name and alignment. Doing so now |
| 67 | // results in VS2017 in producing an internal error (at the call site) that's |
| 68 | // not detailed enough to understand what is happening. |
| 69 | template <typename MemoryOpTy> |
| 70 | static ParseResult parseSourceMemoryAccessAttributes(OpAsmParser &parser, |
| 71 | OperationState &state) { |
| 72 | // Parse an optional list of attributes staring with '[' |
| 73 | if (parser.parseOptionalLSquare()) { |
| 74 | // Nothing to do |
| 75 | return success(); |
| 76 | } |
| 77 | |
| 78 | spirv::MemoryAccess memoryAccessAttr; |
| 79 | StringRef memoryAccessAttrName = |
| 80 | MemoryOpTy::getSourceMemoryAccessAttrName(state.name); |
| 81 | if (spirv::parseEnumStrAttr<spirv::MemoryAccessAttr>( |
| 82 | value&: memoryAccessAttr, parser, state, attrName: memoryAccessAttrName)) |
| 83 | return failure(); |
| 84 | |
| 85 | if (spirv::bitEnumContainsAll(bits: memoryAccessAttr, |
| 86 | bit: spirv::MemoryAccess::Aligned)) { |
| 87 | // Parse integer attribute for alignment. |
| 88 | Attribute alignmentAttr; |
| 89 | StringAttr alignmentAttrName = |
| 90 | MemoryOpTy::getSourceAlignmentAttrName(state.name); |
| 91 | Type i32Type = parser.getBuilder().getIntegerType(width: 32); |
| 92 | if (parser.parseComma() || |
| 93 | parser.parseAttribute(result&: alignmentAttr, type: i32Type, attrName: alignmentAttrName, |
| 94 | attrs&: state.attributes)) { |
| 95 | return failure(); |
| 96 | } |
| 97 | } |
| 98 | return parser.parseRSquare(); |
| 99 | } |
| 100 | |
| 101 | // TODO Make sure to merge this and the previous function into one template |
| 102 | // parameterized by memory access attribute name and alignment. Doing so now |
| 103 | // results in VS2017 in producing an internal error (at the call site) that's |
| 104 | // not detailed enough to understand what is happening. |
| 105 | template <typename MemoryOpTy> |
| 106 | static void printSourceMemoryAccessAttribute( |
| 107 | MemoryOpTy memoryOp, OpAsmPrinter &printer, |
| 108 | SmallVectorImpl<StringRef> &elidedAttrs, |
| 109 | std::optional<spirv::MemoryAccess> memoryAccessAtrrValue = std::nullopt, |
| 110 | std::optional<uint32_t> alignmentAttrValue = std::nullopt) { |
| 111 | |
| 112 | // Print optional memory access attribute. |
| 113 | if (auto memAccess = |
| 114 | (memoryAccessAtrrValue ? memoryAccessAtrrValue |
| 115 | : memoryOp.getSourceMemoryAccess())) { |
| 116 | elidedAttrs.push_back(Elt: memoryOp.getSourceMemoryAccessAttrName()); |
| 117 | |
| 118 | printer << ", [\"" << stringifyMemoryAccess(*memAccess) << "\"" ; |
| 119 | |
| 120 | if (spirv::bitEnumContainsAll(*memAccess, spirv::MemoryAccess::Aligned)) { |
| 121 | // Print integer alignment attribute. |
| 122 | if (auto alignment = |
| 123 | (alignmentAttrValue ? alignmentAttrValue |
| 124 | : memoryOp.getSourceAlignment())) { |
| 125 | elidedAttrs.push_back(Elt: memoryOp.getSourceAlignmentAttrName()); |
| 126 | printer << ", " << *alignment; |
| 127 | } |
| 128 | } |
| 129 | printer << "]" ; |
| 130 | } |
| 131 | elidedAttrs.push_back(Elt: spirv::attributeName<spirv::StorageClass>()); |
| 132 | } |
| 133 | |
| 134 | template <typename MemoryOpTy> |
| 135 | static void printMemoryAccessAttribute( |
| 136 | MemoryOpTy memoryOp, OpAsmPrinter &printer, |
| 137 | SmallVectorImpl<StringRef> &elidedAttrs, |
| 138 | std::optional<spirv::MemoryAccess> memoryAccessAtrrValue = std::nullopt, |
| 139 | std::optional<uint32_t> alignmentAttrValue = std::nullopt) { |
| 140 | // Print optional memory access attribute. |
| 141 | if (auto memAccess = (memoryAccessAtrrValue ? memoryAccessAtrrValue |
| 142 | : memoryOp.getMemoryAccess())) { |
| 143 | elidedAttrs.push_back(Elt: memoryOp.getMemoryAccessAttrName()); |
| 144 | |
| 145 | printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"" ; |
| 146 | |
| 147 | if (spirv::bitEnumContainsAll(*memAccess, spirv::MemoryAccess::Aligned)) { |
| 148 | // Print integer alignment attribute. |
| 149 | if (auto alignment = (alignmentAttrValue ? alignmentAttrValue |
| 150 | : memoryOp.getAlignment())) { |
| 151 | elidedAttrs.push_back(Elt: memoryOp.getAlignmentAttrName()); |
| 152 | printer << ", " << *alignment; |
| 153 | } |
| 154 | } |
| 155 | printer << "]" ; |
| 156 | } |
| 157 | elidedAttrs.push_back(Elt: spirv::attributeName<spirv::StorageClass>()); |
| 158 | } |
| 159 | |
| 160 | template <typename LoadStoreOpTy> |
| 161 | static LogicalResult verifyLoadStorePtrAndValTypes(LoadStoreOpTy op, Value ptr, |
| 162 | Value val) { |
| 163 | // ODS already checks ptr is spirv::PointerType. Just check that the pointee |
| 164 | // type of the pointer and the type of the value are the same |
| 165 | // |
| 166 | // TODO: Check that the value type satisfies restrictions of |
| 167 | // SPIR-V OpLoad/OpStore operations |
| 168 | if (val.getType() != |
| 169 | cast<spirv::PointerType>(Val: ptr.getType()).getPointeeType()) { |
| 170 | return op.emitOpError("mismatch in result type and pointer type" ); |
| 171 | } |
| 172 | return success(); |
| 173 | } |
| 174 | |
| 175 | namespace { |
| 176 | /// Whether the pointer a memory operands mask applies to is read through, |
| 177 | /// written through, or both. |
| 178 | enum class MemoryAccessKind { Read, Write, ReadWrite }; |
| 179 | } // namespace |
| 180 | |
| 181 | /// Verifies the memory operands mask `memAccessAttr` of `op` and its companion |
| 182 | /// alignment attribute `alignmentAttr`. `kind` tells how the pointer the mask |
| 183 | /// applies to is accessed. |
| 184 | static LogicalResult |
| 185 | verifyMemoryAccessAttribute(Operation *op, |
| 186 | spirv::MemoryAccessAttr memAccessAttr, |
| 187 | Attribute alignmentAttr, MemoryAccessKind kind) { |
| 188 | // ODS checks for attributes values. Just need to verify that if the |
| 189 | // memory-access attribute is Aligned, then the alignment attribute must be |
| 190 | // present. |
| 191 | if (!memAccessAttr) { |
| 192 | // Alignment attribute shouldn't be present if memory access attribute is |
| 193 | // not present. |
| 194 | if (alignmentAttr) { |
| 195 | return op->emitOpError( |
| 196 | message: "invalid alignment specification without aligned memory access " |
| 197 | "specification" ); |
| 198 | } |
| 199 | return success(); |
| 200 | } |
| 201 | |
| 202 | spirv::MemoryAccess memAccess = memAccessAttr.getValue(); |
| 203 | |
| 204 | // MakePointerAvailable applies to writes through the pointer. |
| 205 | if (kind == MemoryAccessKind::Read && |
| 206 | spirv::bitEnumContainsAll(bits: memAccess, |
| 207 | bit: spirv::MemoryAccess::MakePointerAvailable)) { |
| 208 | return op->emitOpError( |
| 209 | message: "not compatible with memory operand 'MakePointerAvailable'" ); |
| 210 | } |
| 211 | |
| 212 | // MakePointerVisible applies to reads through the pointer. |
| 213 | if (kind == MemoryAccessKind::Write && |
| 214 | spirv::bitEnumContainsAll(bits: memAccess, |
| 215 | bit: spirv::MemoryAccess::MakePointerVisible)) { |
| 216 | return op->emitOpError( |
| 217 | message: "not compatible with memory operand 'MakePointerVisible'" ); |
| 218 | } |
| 219 | |
| 220 | if (spirv::bitEnumContainsAny(bits: memAccess, |
| 221 | bit: spirv::MemoryAccess::MakePointerAvailable | |
| 222 | spirv::MemoryAccess::MakePointerVisible) && |
| 223 | !spirv::bitEnumContainsAll(bits: memAccess, |
| 224 | bit: spirv::MemoryAccess::NonPrivatePointer)) { |
| 225 | return op->emitOpError( |
| 226 | message: "memory operand 'MakePointerAvailable' or 'MakePointerVisible' " |
| 227 | "requires 'NonPrivatePointer' to also be specified" ); |
| 228 | } |
| 229 | |
| 230 | if (spirv::bitEnumContainsAll(bits: memAccess, bit: spirv::MemoryAccess::Aligned)) { |
| 231 | if (!alignmentAttr) { |
| 232 | return op->emitOpError(message: "missing alignment value" ); |
| 233 | } |
| 234 | } else { |
| 235 | if (alignmentAttr) { |
| 236 | return op->emitOpError( |
| 237 | message: "invalid alignment specification with non-aligned memory access " |
| 238 | "specification" ); |
| 239 | } |
| 240 | } |
| 241 | return success(); |
| 242 | } |
| 243 | |
| 244 | /// Verifies the default (non-Source) memory operands mask of `memoryOp`. |
| 245 | template <typename MemoryOpTy> |
| 246 | static LogicalResult verifyMemoryAccessAttribute(MemoryOpTy memoryOp, |
| 247 | MemoryAccessKind kind) { |
| 248 | return verifyMemoryAccessAttribute(memoryOp.getOperation(), |
| 249 | memoryOp.getMemoryAccessAttr(), |
| 250 | memoryOp.getAlignmentAttr(), kind); |
| 251 | } |
| 252 | |
| 253 | //===----------------------------------------------------------------------===// |
| 254 | // spirv.AccessChainOp |
| 255 | //===----------------------------------------------------------------------===// |
| 256 | |
| 257 | static Type getElementPtrType(Type type, ValueRange indices, Location baseLoc) { |
| 258 | auto ptrType = dyn_cast<spirv::PointerType>(Val&: type); |
| 259 | if (!ptrType) { |
| 260 | emitError(loc: baseLoc, message: "'spirv.AccessChain' op expected a pointer " |
| 261 | "to composite type, but provided " ) |
| 262 | << type; |
| 263 | return nullptr; |
| 264 | } |
| 265 | |
| 266 | auto resultType = ptrType.getPointeeType(); |
| 267 | auto resultStorageClass = ptrType.getStorageClass(); |
| 268 | int32_t index = 0; |
| 269 | |
| 270 | for (auto indexSSA : indices) { |
| 271 | auto cType = dyn_cast<spirv::CompositeType>(Val&: resultType); |
| 272 | if (!cType) { |
| 273 | emitError( |
| 274 | loc: baseLoc, |
| 275 | message: "'spirv.AccessChain' op cannot extract from non-composite type " ) |
| 276 | << resultType << " with index " << index; |
| 277 | return nullptr; |
| 278 | } |
| 279 | index = 0; |
| 280 | if (isa<spirv::StructType>(Val: resultType)) { |
| 281 | Operation *op = indexSSA.getDefiningOp(); |
| 282 | if (!op) { |
| 283 | emitError(loc: baseLoc, message: "'spirv.AccessChain' op index must be an " |
| 284 | "integer spirv.Constant to access " |
| 285 | "element of spirv.struct" ); |
| 286 | return nullptr; |
| 287 | } |
| 288 | |
| 289 | // TODO: this should be relaxed to allow |
| 290 | // integer literals of other bitwidths. |
| 291 | if (failed(Result: spirv::extractValueFromConstOp(op, value&: index))) { |
| 292 | emitError( |
| 293 | loc: baseLoc, |
| 294 | message: "'spirv.AccessChain' index must be an integer spirv.Constant to " |
| 295 | "access element of spirv.struct, but provided " ) |
| 296 | << op->getName(); |
| 297 | return nullptr; |
| 298 | } |
| 299 | if (index < 0 || static_cast<uint64_t>(index) >= cType.getNumElements()) { |
| 300 | emitError(loc: baseLoc, message: "'spirv.AccessChain' op index " ) |
| 301 | << index << " out of bounds for " << resultType; |
| 302 | return nullptr; |
| 303 | } |
| 304 | } |
| 305 | resultType = cType.getElementType(index); |
| 306 | } |
| 307 | return spirv::PointerType::get(pointeeType: resultType, storageClass: resultStorageClass); |
| 308 | } |
| 309 | |
| 310 | void AccessChainOp::build(OpBuilder &builder, OperationState &state, |
| 311 | Value basePtr, ValueRange indices) { |
| 312 | auto type = getElementPtrType(type: basePtr.getType(), indices, baseLoc: state.location); |
| 313 | assert(type && "Unable to deduce return type based on basePtr and indices" ); |
| 314 | build(odsBuilder&: builder, odsState&: state, component_ptr: type, base_ptr: basePtr, indices); |
| 315 | } |
| 316 | |
| 317 | template <typename Op> |
| 318 | static LogicalResult verifyAccessChain(Op accessChainOp, ValueRange indices) { |
| 319 | auto resultType = getElementPtrType(accessChainOp.getBasePtr().getType(), |
| 320 | indices, accessChainOp.getLoc()); |
| 321 | if (!resultType) |
| 322 | return failure(); |
| 323 | |
| 324 | auto providedResultType = |
| 325 | dyn_cast<spirv::PointerType>(accessChainOp.getType()); |
| 326 | if (!providedResultType) |
| 327 | return accessChainOp.emitOpError( |
| 328 | "result type must be a pointer, but provided" ) |
| 329 | << providedResultType; |
| 330 | |
| 331 | if (resultType != providedResultType) |
| 332 | return accessChainOp.emitOpError("invalid result type: expected " ) |
| 333 | << resultType << ", but provided " << providedResultType; |
| 334 | |
| 335 | return success(); |
| 336 | } |
| 337 | |
| 338 | LogicalResult AccessChainOp::verify() { |
| 339 | return verifyAccessChain(accessChainOp: *this, indices: getIndices()); |
| 340 | } |
| 341 | |
| 342 | //===----------------------------------------------------------------------===// |
| 343 | // spirv.InBoundsAccessChainOp |
| 344 | //===----------------------------------------------------------------------===// |
| 345 | |
| 346 | void InBoundsAccessChainOp::build(OpBuilder &builder, OperationState &state, |
| 347 | Value basePtr, ValueRange indices) { |
| 348 | Type type = getElementPtrType(type: basePtr.getType(), indices, baseLoc: state.location); |
| 349 | assert(type && "Unable to deduce return type based on basePtr and indices" ); |
| 350 | build(odsBuilder&: builder, odsState&: state, component_ptr: type, base_ptr: basePtr, indices); |
| 351 | } |
| 352 | |
| 353 | LogicalResult InBoundsAccessChainOp::verify() { |
| 354 | return verifyAccessChain(accessChainOp: *this, indices: getIndices()); |
| 355 | } |
| 356 | |
| 357 | //===----------------------------------------------------------------------===// |
| 358 | // spirv.LoadOp |
| 359 | //===----------------------------------------------------------------------===// |
| 360 | |
| 361 | void LoadOp::build(OpBuilder &builder, OperationState &state, Value basePtr, |
| 362 | MemoryAccessAttr memoryAccess, IntegerAttr alignment) { |
| 363 | auto ptrType = cast<spirv::PointerType>(Val: basePtr.getType()); |
| 364 | build(odsBuilder&: builder, odsState&: state, value: ptrType.getPointeeType(), ptr: basePtr, memory_access: memoryAccess, |
| 365 | alignment); |
| 366 | } |
| 367 | |
| 368 | ParseResult LoadOp::parse(OpAsmParser &parser, OperationState &result) { |
| 369 | // Parse the storage class specification |
| 370 | spirv::StorageClass storageClass; |
| 371 | OpAsmParser::UnresolvedOperand ptrInfo; |
| 372 | Type elementType; |
| 373 | if (parseEnumStrAttr(value&: storageClass, parser) || parser.parseOperand(result&: ptrInfo) || |
| 374 | parseMemoryAccessAttributes<LoadOp>(parser, state&: result) || |
| 375 | parser.parseOptionalAttrDict(result&: result.attributes) || parser.parseColon() || |
| 376 | parser.parseType(result&: elementType)) { |
| 377 | return failure(); |
| 378 | } |
| 379 | |
| 380 | auto ptrType = spirv::PointerType::get(pointeeType: elementType, storageClass); |
| 381 | if (parser.resolveOperand(operand: ptrInfo, type: ptrType, result&: result.operands)) { |
| 382 | return failure(); |
| 383 | } |
| 384 | |
| 385 | result.addTypes(newTypes: elementType); |
| 386 | return success(); |
| 387 | } |
| 388 | |
| 389 | void LoadOp::print(OpAsmPrinter &printer) { |
| 390 | SmallVector<StringRef, 4> elidedAttrs; |
| 391 | StringRef sc = stringifyStorageClass( |
| 392 | cast<spirv::PointerType>(Val: getPtr().getType()).getStorageClass()); |
| 393 | printer << " \"" << sc << "\" " << getPtr(); |
| 394 | |
| 395 | printMemoryAccessAttribute(memoryOp: *this, printer, elidedAttrs); |
| 396 | |
| 397 | printer.printOptionalAttrDict( |
| 398 | attrs: (*this)->getDiscardableAttrDictionary().getValue(), elidedAttrs); |
| 399 | printer << " : " << getType(); |
| 400 | } |
| 401 | |
| 402 | LogicalResult LoadOp::verify() { |
| 403 | // SPIR-V spec : "Result Type is the type of the loaded object. It must be a |
| 404 | // type with fixed size; i.e., it cannot be, nor include, any |
| 405 | // OpTypeRuntimeArray types." |
| 406 | if (failed(Result: verifyLoadStorePtrAndValTypes(op: *this, ptr: getPtr(), val: getValue()))) { |
| 407 | return failure(); |
| 408 | } |
| 409 | return verifyMemoryAccessAttribute(memoryOp: *this, kind: MemoryAccessKind::Read); |
| 410 | } |
| 411 | |
| 412 | //===----------------------------------------------------------------------===// |
| 413 | // spirv.StoreOp |
| 414 | //===----------------------------------------------------------------------===// |
| 415 | |
| 416 | ParseResult StoreOp::parse(OpAsmParser &parser, OperationState &result) { |
| 417 | // Parse the storage class specification |
| 418 | spirv::StorageClass storageClass; |
| 419 | SmallVector<OpAsmParser::UnresolvedOperand, 2> operandInfo; |
| 420 | auto loc = parser.getCurrentLocation(); |
| 421 | Type elementType; |
| 422 | if (parseEnumStrAttr(value&: storageClass, parser) || |
| 423 | parser.parseOperandList(result&: operandInfo, requiredOperandCount: 2) || |
| 424 | parseMemoryAccessAttributes<StoreOp>(parser, state&: result) || |
| 425 | parser.parseColon() || parser.parseType(result&: elementType)) { |
| 426 | return failure(); |
| 427 | } |
| 428 | |
| 429 | auto ptrType = spirv::PointerType::get(pointeeType: elementType, storageClass); |
| 430 | if (parser.resolveOperands(operands&: operandInfo, types: {ptrType, elementType}, loc, |
| 431 | result&: result.operands)) { |
| 432 | return failure(); |
| 433 | } |
| 434 | return success(); |
| 435 | } |
| 436 | |
| 437 | void StoreOp::print(OpAsmPrinter &printer) { |
| 438 | SmallVector<StringRef, 4> elidedAttrs; |
| 439 | StringRef sc = stringifyStorageClass( |
| 440 | cast<spirv::PointerType>(Val: getPtr().getType()).getStorageClass()); |
| 441 | printer << " \"" << sc << "\" " << getPtr() << ", " << getValue(); |
| 442 | |
| 443 | printMemoryAccessAttribute(memoryOp: *this, printer, elidedAttrs); |
| 444 | |
| 445 | printer << " : " << getValue().getType(); |
| 446 | printer.printOptionalAttrDict( |
| 447 | attrs: (*this)->getDiscardableAttrDictionary().getValue(), elidedAttrs); |
| 448 | } |
| 449 | |
| 450 | LogicalResult StoreOp::verify() { |
| 451 | // SPIR-V spec : "Pointer is the pointer to store through. Its type must be an |
| 452 | // OpTypePointer whose Type operand is the same as the type of Object." |
| 453 | if (failed(Result: verifyLoadStorePtrAndValTypes(op: *this, ptr: getPtr(), val: getValue()))) |
| 454 | return failure(); |
| 455 | return verifyMemoryAccessAttribute(memoryOp: *this, kind: MemoryAccessKind::Write); |
| 456 | } |
| 457 | |
| 458 | //===----------------------------------------------------------------------===// |
| 459 | // spirv.CopyMemory |
| 460 | //===----------------------------------------------------------------------===// |
| 461 | |
| 462 | void CopyMemoryOp::print(OpAsmPrinter &printer) { |
| 463 | printer << ' '; |
| 464 | |
| 465 | StringRef targetStorageClass = stringifyStorageClass( |
| 466 | cast<spirv::PointerType>(Val: getTarget().getType()).getStorageClass()); |
| 467 | printer << " \"" << targetStorageClass << "\" " << getTarget() << ", " ; |
| 468 | |
| 469 | StringRef sourceStorageClass = stringifyStorageClass( |
| 470 | cast<spirv::PointerType>(Val: getSource().getType()).getStorageClass()); |
| 471 | printer << " \"" << sourceStorageClass << "\" " << getSource(); |
| 472 | |
| 473 | SmallVector<StringRef, 4> elidedAttrs; |
| 474 | printMemoryAccessAttribute(memoryOp: *this, printer, elidedAttrs); |
| 475 | printSourceMemoryAccessAttribute(memoryOp: *this, printer, elidedAttrs, |
| 476 | memoryAccessAtrrValue: getSourceMemoryAccess(), |
| 477 | alignmentAttrValue: getSourceAlignment()); |
| 478 | |
| 479 | printer.printOptionalAttrDict( |
| 480 | attrs: (*this)->getDiscardableAttrDictionary().getValue(), elidedAttrs); |
| 481 | |
| 482 | Type pointeeType = |
| 483 | cast<spirv::PointerType>(Val: getTarget().getType()).getPointeeType(); |
| 484 | printer << " : " << pointeeType; |
| 485 | } |
| 486 | |
| 487 | ParseResult CopyMemoryOp::parse(OpAsmParser &parser, OperationState &result) { |
| 488 | spirv::StorageClass targetStorageClass; |
| 489 | OpAsmParser::UnresolvedOperand targetPtrInfo; |
| 490 | |
| 491 | spirv::StorageClass sourceStorageClass; |
| 492 | OpAsmParser::UnresolvedOperand sourcePtrInfo; |
| 493 | |
| 494 | Type elementType; |
| 495 | |
| 496 | if (parseEnumStrAttr(value&: targetStorageClass, parser) || |
| 497 | parser.parseOperand(result&: targetPtrInfo) || parser.parseComma() || |
| 498 | parseEnumStrAttr(value&: sourceStorageClass, parser) || |
| 499 | parser.parseOperand(result&: sourcePtrInfo) || |
| 500 | parseMemoryAccessAttributes<CopyMemoryOp>(parser, state&: result)) { |
| 501 | return failure(); |
| 502 | } |
| 503 | |
| 504 | if (!parser.parseOptionalComma()) { |
| 505 | // Parse 2nd memory access attributes. |
| 506 | if (parseSourceMemoryAccessAttributes<CopyMemoryOp>(parser, state&: result)) { |
| 507 | return failure(); |
| 508 | } |
| 509 | } |
| 510 | |
| 511 | if (parser.parseColon() || parser.parseType(result&: elementType)) |
| 512 | return failure(); |
| 513 | |
| 514 | if (parser.parseOptionalAttrDict(result&: result.attributes)) |
| 515 | return failure(); |
| 516 | |
| 517 | auto targetPtrType = spirv::PointerType::get(pointeeType: elementType, storageClass: targetStorageClass); |
| 518 | auto sourcePtrType = spirv::PointerType::get(pointeeType: elementType, storageClass: sourceStorageClass); |
| 519 | |
| 520 | if (parser.resolveOperand(operand: targetPtrInfo, type: targetPtrType, result&: result.operands) || |
| 521 | parser.resolveOperand(operand: sourcePtrInfo, type: sourcePtrType, result&: result.operands)) { |
| 522 | return failure(); |
| 523 | } |
| 524 | |
| 525 | return success(); |
| 526 | } |
| 527 | |
| 528 | LogicalResult CopyMemoryOp::verify() { |
| 529 | Type targetType = |
| 530 | cast<spirv::PointerType>(Val: getTarget().getType()).getPointeeType(); |
| 531 | |
| 532 | Type sourceType = |
| 533 | cast<spirv::PointerType>(Val: getSource().getType()).getPointeeType(); |
| 534 | |
| 535 | if (targetType != sourceType) |
| 536 | return emitOpError(message: "both operands must be pointers to the same type" ); |
| 537 | |
| 538 | // A lone mask applies to both operands. Only the first of two masks is |
| 539 | // Target-only. |
| 540 | MemoryAccessKind targetKind = getSourceMemoryAccess() |
| 541 | ? MemoryAccessKind::Write |
| 542 | : MemoryAccessKind::ReadWrite; |
| 543 | if (failed(Result: verifyMemoryAccessAttribute(memoryOp: *this, kind: targetKind))) |
| 544 | return failure(); |
| 545 | |
| 546 | return verifyMemoryAccessAttribute( |
| 547 | op: getOperation(), memAccessAttr: getSourceMemoryAccessAttr(), alignmentAttr: getSourceAlignmentAttr(), |
| 548 | kind: MemoryAccessKind::Read); |
| 549 | } |
| 550 | |
| 551 | //===----------------------------------------------------------------------===// |
| 552 | // spirv.InBoundsPtrAccessChainOp |
| 553 | //===----------------------------------------------------------------------===// |
| 554 | |
| 555 | void InBoundsPtrAccessChainOp::build(OpBuilder &builder, OperationState &state, |
| 556 | Value basePtr, Value element, |
| 557 | ValueRange indices) { |
| 558 | auto type = getElementPtrType(type: basePtr.getType(), indices, baseLoc: state.location); |
| 559 | assert(type && "Unable to deduce return type based on basePtr and indices" ); |
| 560 | build(odsBuilder&: builder, odsState&: state, result: type, base_ptr: basePtr, element, indices); |
| 561 | } |
| 562 | |
| 563 | LogicalResult InBoundsPtrAccessChainOp::verify() { |
| 564 | return verifyAccessChain(accessChainOp: *this, indices: getIndices()); |
| 565 | } |
| 566 | |
| 567 | //===----------------------------------------------------------------------===// |
| 568 | // spirv.PtrAccessChainOp |
| 569 | //===----------------------------------------------------------------------===// |
| 570 | |
| 571 | void PtrAccessChainOp::build(OpBuilder &builder, OperationState &state, |
| 572 | Value basePtr, Value element, ValueRange indices) { |
| 573 | auto type = getElementPtrType(type: basePtr.getType(), indices, baseLoc: state.location); |
| 574 | assert(type && "Unable to deduce return type based on basePtr and indices" ); |
| 575 | build(odsBuilder&: builder, odsState&: state, result: type, base_ptr: basePtr, element, indices); |
| 576 | } |
| 577 | |
| 578 | LogicalResult PtrAccessChainOp::verify() { |
| 579 | return verifyAccessChain(accessChainOp: *this, indices: getIndices()); |
| 580 | } |
| 581 | |
| 582 | //===----------------------------------------------------------------------===// |
| 583 | // spirv.Variable |
| 584 | //===----------------------------------------------------------------------===// |
| 585 | |
| 586 | ParseResult VariableOp::parse(OpAsmParser &parser, OperationState &result) { |
| 587 | // Parse optional initializer |
| 588 | std::optional<OpAsmParser::UnresolvedOperand> initInfo; |
| 589 | if (succeeded(Result: parser.parseOptionalKeyword(keyword: "init" ))) { |
| 590 | initInfo = OpAsmParser::UnresolvedOperand(); |
| 591 | if (parser.parseLParen() || parser.parseOperand(result&: *initInfo) || |
| 592 | parser.parseRParen()) |
| 593 | return failure(); |
| 594 | } |
| 595 | |
| 596 | if (parseVariableDecorations(parser, state&: result)) { |
| 597 | return failure(); |
| 598 | } |
| 599 | |
| 600 | // Parse result pointer type |
| 601 | Type type; |
| 602 | if (parser.parseColon()) |
| 603 | return failure(); |
| 604 | auto loc = parser.getCurrentLocation(); |
| 605 | if (parser.parseType(result&: type)) |
| 606 | return failure(); |
| 607 | |
| 608 | auto ptrType = dyn_cast<spirv::PointerType>(Val&: type); |
| 609 | if (!ptrType) |
| 610 | return parser.emitError(loc, message: "expected spirv.ptr type" ); |
| 611 | result.addTypes(newTypes: ptrType); |
| 612 | |
| 613 | // Resolve the initializer operand |
| 614 | if (initInfo) { |
| 615 | if (parser.resolveOperand(operand: *initInfo, type: ptrType.getPointeeType(), |
| 616 | result&: result.operands)) |
| 617 | return failure(); |
| 618 | } |
| 619 | |
| 620 | auto attr = parser.getBuilder().getAttr<spirv::StorageClassAttr>( |
| 621 | args: ptrType.getStorageClass()); |
| 622 | result.addAttribute(name: spirv::attributeName<spirv::StorageClass>(), attr); |
| 623 | |
| 624 | return success(); |
| 625 | } |
| 626 | |
| 627 | void VariableOp::print(OpAsmPrinter &printer) { |
| 628 | SmallVector<StringRef, 4> elidedAttrs{ |
| 629 | spirv::attributeName<spirv::StorageClass>()}; |
| 630 | // Print optional initializer |
| 631 | if (getNumOperands() != 0) |
| 632 | printer << " init(" << getInitializer() << ")" ; |
| 633 | |
| 634 | printVariableDecorations(op: *this, printer, elidedAttrs); |
| 635 | printer << " : " << getType(); |
| 636 | } |
| 637 | |
| 638 | LogicalResult VariableOp::verify() { |
| 639 | // SPIR-V spec: "Storage Class is the Storage Class of the memory holding the |
| 640 | // object. It cannot be Generic. It must be the same as the Storage Class |
| 641 | // operand of the Result Type." |
| 642 | if (getStorageClass() != spirv::StorageClass::Function) { |
| 643 | return emitOpError( |
| 644 | message: "can only be used to model function-level variables. Use " |
| 645 | "spirv.GlobalVariable for module-level variables." ); |
| 646 | } |
| 647 | |
| 648 | auto pointerType = cast<spirv::PointerType>(Val: getPointer().getType()); |
| 649 | if (getStorageClass() != pointerType.getStorageClass()) |
| 650 | return emitOpError( |
| 651 | message: "storage class must match result pointer's storage class" ); |
| 652 | |
| 653 | if (getNumOperands() != 0) { |
| 654 | // SPIR-V spec: "Initializer must be an <id> from a constant instruction or |
| 655 | // a global (module scope) OpVariable instruction". |
| 656 | auto *initOp = getOperand(i: 0).getDefiningOp(); |
| 657 | if (!initOp || !isa<spirv::ConstantOp, // for normal constant |
| 658 | spirv::ReferenceOfOp, // for spec constant |
| 659 | spirv::AddressOfOp>(Val: initOp)) |
| 660 | return emitOpError(message: "initializer must be the result of a " |
| 661 | "constant or spirv.GlobalVariable op" ); |
| 662 | } |
| 663 | |
| 664 | auto getDecorationAttr = [op = getOperation()](spirv::Decoration decoration) { |
| 665 | return op->getDiscardableAttr(name: spirv::getDecorationString(decoration)); |
| 666 | }; |
| 667 | |
| 668 | // TODO: generate these strings using ODS. |
| 669 | for (auto decoration : |
| 670 | {spirv::Decoration::DescriptorSet, spirv::Decoration::Binding, |
| 671 | spirv::Decoration::BuiltIn}) { |
| 672 | if (auto attr = getDecorationAttr(decoration)) |
| 673 | return emitOpError(message: "cannot have '" ) |
| 674 | << spirv::getDecorationString(decoration) |
| 675 | << "' attribute (only allowed in spirv.GlobalVariable)" ; |
| 676 | } |
| 677 | |
| 678 | if (failed(Result: verifyPhysicalStorageBufferDecorations(op: getOperation(), |
| 679 | pointeeType: getPointeeType()))) |
| 680 | return failure(); |
| 681 | |
| 682 | return success(); |
| 683 | } |
| 684 | |
| 685 | } // namespace mlir::spirv |
| 686 | |