| 1 | //===- MemRefToSPIRV.cpp - MemRef to SPIR-V Patterns ----------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements patterns to convert MemRef dialect to SPIR-V dialect. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 14 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 15 | #include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h" |
| 16 | #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h" |
| 17 | #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h" |
| 18 | #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h" |
| 19 | #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h" |
| 20 | #include "mlir/IR/BuiltinAttributes.h" |
| 21 | #include "mlir/IR/BuiltinTypes.h" |
| 22 | #include "mlir/IR/MLIRContext.h" |
| 23 | #include "mlir/IR/Visitors.h" |
| 24 | #include "mlir/Interfaces/FunctionInterfaces.h" |
| 25 | #include <cassert> |
| 26 | #include <limits> |
| 27 | #include <optional> |
| 28 | |
| 29 | #define DEBUG_TYPE "memref-to-spirv-pattern" |
| 30 | |
| 31 | using namespace mlir; |
| 32 | |
| 33 | //===----------------------------------------------------------------------===// |
| 34 | // Utility functions |
| 35 | //===----------------------------------------------------------------------===// |
| 36 | |
| 37 | /// Returns the offset of the value in `targetBits` representation. |
| 38 | /// |
| 39 | /// `srcIdx` is an index into a 1-D array with each element having `sourceBits`. |
| 40 | /// It's assumed to be non-negative. |
| 41 | /// |
| 42 | /// When accessing an element in the array treating as having elements of |
| 43 | /// `targetBits`, multiple values are loaded in the same time. The method |
| 44 | /// returns the offset where the `srcIdx` locates in the value. For example, if |
| 45 | /// `sourceBits` equals to 8 and `targetBits` equals to 32, the x-th element is |
| 46 | /// located at (x % 4) * 8. Because there are four elements in one i32, and one |
| 47 | /// element has 8 bits. |
| 48 | static Value getOffsetForBitwidth(Location loc, Value srcIdx, int sourceBits, |
| 49 | int targetBits, OpBuilder &builder) { |
| 50 | assert(targetBits % sourceBits == 0); |
| 51 | Type type = srcIdx.getType(); |
| 52 | IntegerAttr idxAttr = builder.getIntegerAttr(type, value: targetBits / sourceBits); |
| 53 | auto idx = builder.createOrFold<spirv::ConstantOp>(location: loc, args&: type, args&: idxAttr); |
| 54 | IntegerAttr srcBitsAttr = builder.getIntegerAttr(type, value: sourceBits); |
| 55 | auto srcBitsValue = |
| 56 | builder.createOrFold<spirv::ConstantOp>(location: loc, args&: type, args&: srcBitsAttr); |
| 57 | auto m = builder.createOrFold<spirv::UModOp>(location: loc, args&: srcIdx, args&: idx); |
| 58 | return builder.createOrFold<spirv::IMulOp>(location: loc, args&: type, args&: m, args&: srcBitsValue); |
| 59 | } |
| 60 | |
| 61 | /// Returns an adjusted spirv::AccessChainOp. Based on the |
| 62 | /// extension/capabilities, certain integer bitwidths `sourceBits` might not be |
| 63 | /// supported. During conversion if a memref of an unsupported type is used, |
| 64 | /// load/stores to this memref need to be modified to use a supported higher |
| 65 | /// bitwidth `targetBits` and extracting the required bits. For an accessing a |
| 66 | /// 1D array (spirv.array or spirv.rtarray), the last index is modified to load |
| 67 | /// the bits needed. The extraction of the actual bits needed are handled |
| 68 | /// separately. Note that this only works for a 1-D tensor. |
| 69 | static Value |
| 70 | adjustAccessChainForBitwidth(const SPIRVTypeConverter &typeConverter, |
| 71 | spirv::AccessChainOp op, int sourceBits, |
| 72 | int targetBits, OpBuilder &builder) { |
| 73 | assert(targetBits % sourceBits == 0); |
| 74 | const auto loc = op.getLoc(); |
| 75 | Value lastDim = op->getOperand(idx: op.getNumOperands() - 1); |
| 76 | Type type = lastDim.getType(); |
| 77 | IntegerAttr attr = builder.getIntegerAttr(type, value: targetBits / sourceBits); |
| 78 | auto idx = builder.createOrFold<spirv::ConstantOp>(location: loc, args&: type, args&: attr); |
| 79 | auto indices = llvm::to_vector<4>(Range: op.getIndices()); |
| 80 | // There are two elements if this is a 1-D tensor. |
| 81 | assert(indices.size() == 2); |
| 82 | indices.back() = builder.createOrFold<spirv::SDivOp>(location: loc, args&: lastDim, args&: idx); |
| 83 | Type t = typeConverter.convertType(t: op.getComponentPtr().getType()); |
| 84 | return spirv::AccessChainOp::create(builder, location: loc, component_ptr: t, base_ptr: op.getBasePtr(), |
| 85 | indices); |
| 86 | } |
| 87 | |
| 88 | /// Casts the given `srcBool` into an integer of `dstType`. |
| 89 | static Value castBoolToIntN(Location loc, Value srcBool, Type dstType, |
| 90 | OpBuilder &builder) { |
| 91 | assert(srcBool.getType().isInteger(1)); |
| 92 | if (dstType.isInteger(width: 1)) |
| 93 | return srcBool; |
| 94 | Value zero = spirv::ConstantOp::getZero(type: dstType, loc, builder); |
| 95 | Value one = spirv::ConstantOp::getOne(type: dstType, loc, builder); |
| 96 | return builder.createOrFold<spirv::SelectOp>(location: loc, args&: dstType, args&: srcBool, args&: one, |
| 97 | args&: zero); |
| 98 | } |
| 99 | |
| 100 | /// Returns the `targetBits`-bit value shifted by the given `offset`, and cast |
| 101 | /// to the type destination type, and masked. |
| 102 | static Value shiftValue(Location loc, Value value, Value offset, Value mask, |
| 103 | OpBuilder &builder) { |
| 104 | IntegerType dstType = cast<IntegerType>(Val: mask.getType()); |
| 105 | int targetBits = static_cast<int>(dstType.getWidth()); |
| 106 | int valueBits = value.getType().getIntOrFloatBitWidth(); |
| 107 | assert(valueBits <= targetBits); |
| 108 | |
| 109 | if (valueBits == 1) { |
| 110 | value = castBoolToIntN(loc, srcBool: value, dstType, builder); |
| 111 | } else { |
| 112 | if (valueBits < targetBits) { |
| 113 | value = spirv::UConvertOp::create( |
| 114 | builder, location: loc, result: builder.getIntegerType(width: targetBits), operand: value); |
| 115 | } |
| 116 | |
| 117 | value = builder.createOrFold<spirv::BitwiseAndOp>(location: loc, args&: value, args&: mask); |
| 118 | } |
| 119 | return builder.createOrFold<spirv::ShiftLeftLogicalOp>(location: loc, args: value.getType(), |
| 120 | args&: value, args&: offset); |
| 121 | } |
| 122 | |
| 123 | /// Returns true if the allocations of memref `type` generated from `allocOp` |
| 124 | /// can be lowered to SPIR-V. |
| 125 | static bool isAllocationSupported(Operation *allocOp, MemRefType type) { |
| 126 | // Currently only support static shape |
| 127 | if (!type.hasStaticShape()) |
| 128 | return false; |
| 129 | |
| 130 | if (isa<memref::AllocOp, memref::DeallocOp>(Val: allocOp)) { |
| 131 | auto sc = dyn_cast_or_null<spirv::StorageClassAttr>(Val: type.getMemorySpace()); |
| 132 | if (!sc || sc.getValue() != spirv::StorageClass::Workgroup) |
| 133 | return false; |
| 134 | } else if (isa<memref::AllocaOp>(Val: allocOp)) { |
| 135 | auto sc = dyn_cast_or_null<spirv::StorageClassAttr>(Val: type.getMemorySpace()); |
| 136 | if (!sc || sc.getValue() != spirv::StorageClass::Function) |
| 137 | return false; |
| 138 | // Function allocations of memref-compatible element types may be lowered |
| 139 | // to SPIRV pointers/arrays of the corresponding SPIRV element type. |
| 140 | if (isa<MemRefElementTypeInterface>(Val: type.getElementType())) |
| 141 | return true; |
| 142 | } else { |
| 143 | return false; |
| 144 | } |
| 145 | |
| 146 | // Support memref of int or float types, or their vector/complex types. |
| 147 | Type elementType = type.getElementType(); |
| 148 | if (auto vecType = dyn_cast<VectorType>(Val&: elementType)) |
| 149 | elementType = vecType.getElementType(); |
| 150 | if (auto compType = dyn_cast<ComplexType>(Val&: elementType)) |
| 151 | elementType = compType.getElementType(); |
| 152 | return elementType.isIntOrFloat(); |
| 153 | } |
| 154 | |
| 155 | /// Returns the scope to use for atomic operations use for emulating store |
| 156 | /// operations of unsupported integer bitwidths, based on the memref |
| 157 | /// type. Returns std::nullopt on failure. |
| 158 | static std::optional<spirv::Scope> getAtomicOpScope(MemRefType type) { |
| 159 | auto sc = dyn_cast_or_null<spirv::StorageClassAttr>(Val: type.getMemorySpace()); |
| 160 | switch (sc.getValue()) { |
| 161 | case spirv::StorageClass::StorageBuffer: |
| 162 | return spirv::Scope::Device; |
| 163 | case spirv::StorageClass::Workgroup: |
| 164 | return spirv::Scope::Workgroup; |
| 165 | default: |
| 166 | break; |
| 167 | } |
| 168 | return {}; |
| 169 | } |
| 170 | |
| 171 | /// Returns the MemorySemantics storage-class bit corresponding to `sc`. |
| 172 | /// Per SPIR-V spec section 3.32 (Memory Semantics) this bit must be OR'd |
| 173 | /// with the ordering bits (Acquire/Release/...) on atomic operations. |
| 174 | static spirv::MemorySemantics |
| 175 | getMemorySemanticsForStorageClass(spirv::StorageClass sc) { |
| 176 | switch (sc) { |
| 177 | case spirv::StorageClass::StorageBuffer: |
| 178 | case spirv::StorageClass::Uniform: |
| 179 | return spirv::MemorySemantics::UniformMemory; |
| 180 | case spirv::StorageClass::Workgroup: |
| 181 | return spirv::MemorySemantics::WorkgroupMemory; |
| 182 | case spirv::StorageClass::CrossWorkgroup: |
| 183 | return spirv::MemorySemantics::CrossWorkgroupMemory; |
| 184 | case spirv::StorageClass::AtomicCounter: |
| 185 | return spirv::MemorySemantics::AtomicCounterMemory; |
| 186 | case spirv::StorageClass::Image: |
| 187 | return spirv::MemorySemantics::ImageMemory; |
| 188 | default: |
| 189 | return spirv::MemorySemantics::None; |
| 190 | } |
| 191 | } |
| 192 | |
| 193 | /// Returns the AcquireRelease memory semantics OR'd with the storage-class |
| 194 | /// bit derived from the memory space of `type`. |
| 195 | static spirv::MemorySemantics getAtomicAcqRelMemorySemantics(MemRefType type) { |
| 196 | auto sc = cast<spirv::StorageClassAttr>(Val: type.getMemorySpace()).getValue(); |
| 197 | return spirv::MemorySemantics::AcquireRelease | |
| 198 | getMemorySemanticsForStorageClass(sc); |
| 199 | } |
| 200 | |
| 201 | /// Extracts the element type from a SPIR-V pointer type pointing to storage. |
| 202 | /// |
| 203 | /// For Kernel capability, the pointer points directly to the element type |
| 204 | /// (possibly wrapped in an array). For Vulkan, the pointer points to a struct |
| 205 | /// containing an array or runtime array, and we need to unwrap to get the |
| 206 | /// element type. |
| 207 | static Type |
| 208 | getElementTypeForStoragePointer(Type pointeeType, |
| 209 | const SPIRVTypeConverter &typeConverter) { |
| 210 | if (typeConverter.allows(capability: spirv::Capability::Kernel)) { |
| 211 | if (auto arrayType = dyn_cast<spirv::ArrayType>(Val&: pointeeType)) |
| 212 | return arrayType.getElementType(); |
| 213 | return pointeeType; |
| 214 | } |
| 215 | // For Vulkan we need to extract element from wrapping struct and array. |
| 216 | Type structElemType = cast<spirv::StructType>(Val&: pointeeType).getElementType(0); |
| 217 | if (auto arrayType = dyn_cast<spirv::ArrayType>(Val&: structElemType)) |
| 218 | return arrayType.getElementType(); |
| 219 | return cast<spirv::RuntimeArrayType>(Val&: structElemType).getElementType(); |
| 220 | } |
| 221 | |
| 222 | /// Casts the given `srcInt` into a boolean value. |
| 223 | static Value castIntNToBool(Location loc, Value srcInt, OpBuilder &builder) { |
| 224 | if (srcInt.getType().isInteger(width: 1)) |
| 225 | return srcInt; |
| 226 | |
| 227 | auto one = spirv::ConstantOp::getZero(type: srcInt.getType(), loc, builder); |
| 228 | return builder.createOrFold<spirv::INotEqualOp>(location: loc, args&: srcInt, args&: one); |
| 229 | } |
| 230 | |
| 231 | //===----------------------------------------------------------------------===// |
| 232 | // Operation conversion |
| 233 | //===----------------------------------------------------------------------===// |
| 234 | |
| 235 | // Note that DRR cannot be used for the patterns in this file: we may need to |
| 236 | // convert type along the way, which requires ConversionPattern. DRR generates |
| 237 | // normal RewritePattern. |
| 238 | |
| 239 | namespace { |
| 240 | |
| 241 | /// Converts memref.alloca to SPIR-V Function variables. |
| 242 | class AllocaOpPattern final : public OpConversionPattern<memref::AllocaOp> { |
| 243 | public: |
| 244 | using Base::Base; |
| 245 | |
| 246 | LogicalResult |
| 247 | matchAndRewrite(memref::AllocaOp allocaOp, OpAdaptor adaptor, |
| 248 | ConversionPatternRewriter &rewriter) const override; |
| 249 | }; |
| 250 | |
| 251 | /// Converts an allocation operation to SPIR-V. Currently only supports lowering |
| 252 | /// to Workgroup memory when the size is constant. Note that this pattern needs |
| 253 | /// to be applied in a pass that runs at least at spirv.module scope since it |
| 254 | /// wil ladd global variables into the spirv.module. |
| 255 | class AllocOpPattern final : public OpConversionPattern<memref::AllocOp> { |
| 256 | public: |
| 257 | using Base::Base; |
| 258 | |
| 259 | LogicalResult |
| 260 | matchAndRewrite(memref::AllocOp operation, OpAdaptor adaptor, |
| 261 | ConversionPatternRewriter &rewriter) const override; |
| 262 | }; |
| 263 | |
| 264 | /// Converts memref.automic_rmw operations to SPIR-V atomic operations. |
| 265 | class AtomicRMWOpPattern final |
| 266 | : public OpConversionPattern<memref::AtomicRMWOp> { |
| 267 | public: |
| 268 | using Base::Base; |
| 269 | |
| 270 | LogicalResult |
| 271 | matchAndRewrite(memref::AtomicRMWOp atomicOp, OpAdaptor adaptor, |
| 272 | ConversionPatternRewriter &rewriter) const override; |
| 273 | }; |
| 274 | |
| 275 | /// Removed a deallocation if it is a supported allocation. Currently only |
| 276 | /// removes deallocation if the memory space is workgroup memory. |
| 277 | class DeallocOpPattern final : public OpConversionPattern<memref::DeallocOp> { |
| 278 | public: |
| 279 | using Base::Base; |
| 280 | |
| 281 | LogicalResult |
| 282 | matchAndRewrite(memref::DeallocOp operation, OpAdaptor adaptor, |
| 283 | ConversionPatternRewriter &rewriter) const override; |
| 284 | }; |
| 285 | |
| 286 | /// Converts memref.load to spirv.Load + spirv.AccessChain on integers. |
| 287 | class IntLoadOpPattern final : public OpConversionPattern<memref::LoadOp> { |
| 288 | public: |
| 289 | using Base::Base; |
| 290 | |
| 291 | LogicalResult |
| 292 | matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor, |
| 293 | ConversionPatternRewriter &rewriter) const override; |
| 294 | }; |
| 295 | |
| 296 | /// Converts memref.load to spirv.Load + spirv.AccessChain. |
| 297 | class LoadOpPattern final : public OpConversionPattern<memref::LoadOp> { |
| 298 | public: |
| 299 | using Base::Base; |
| 300 | |
| 301 | LogicalResult |
| 302 | matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor, |
| 303 | ConversionPatternRewriter &rewriter) const override; |
| 304 | }; |
| 305 | |
| 306 | /// Converts memref.load to spirv.Image + spirv.ImageFetch |
| 307 | class ImageLoadOpPattern final : public OpConversionPattern<memref::LoadOp> { |
| 308 | public: |
| 309 | using Base::Base; |
| 310 | |
| 311 | LogicalResult |
| 312 | matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor, |
| 313 | ConversionPatternRewriter &rewriter) const override; |
| 314 | }; |
| 315 | |
| 316 | /// Converts memref.store to spirv.Store on integers. |
| 317 | class IntStoreOpPattern final : public OpConversionPattern<memref::StoreOp> { |
| 318 | public: |
| 319 | using Base::Base; |
| 320 | |
| 321 | LogicalResult |
| 322 | matchAndRewrite(memref::StoreOp storeOp, OpAdaptor adaptor, |
| 323 | ConversionPatternRewriter &rewriter) const override; |
| 324 | }; |
| 325 | |
| 326 | /// Converts memref.memory_space_cast to the appropriate spirv cast operations. |
| 327 | class MemorySpaceCastOpPattern final |
| 328 | : public OpConversionPattern<memref::MemorySpaceCastOp> { |
| 329 | public: |
| 330 | using Base::Base; |
| 331 | |
| 332 | LogicalResult |
| 333 | matchAndRewrite(memref::MemorySpaceCastOp addrCastOp, OpAdaptor adaptor, |
| 334 | ConversionPatternRewriter &rewriter) const override; |
| 335 | }; |
| 336 | |
| 337 | /// Converts memref.store to spirv.Store. |
| 338 | class StoreOpPattern final : public OpConversionPattern<memref::StoreOp> { |
| 339 | public: |
| 340 | using Base::Base; |
| 341 | |
| 342 | LogicalResult |
| 343 | matchAndRewrite(memref::StoreOp storeOp, OpAdaptor adaptor, |
| 344 | ConversionPatternRewriter &rewriter) const override; |
| 345 | }; |
| 346 | |
| 347 | /// Converts memref.copy to spirv.CopyMemory. |
| 348 | class CopyOpPattern final : public OpConversionPattern<memref::CopyOp> { |
| 349 | public: |
| 350 | using Base::Base; |
| 351 | |
| 352 | LogicalResult |
| 353 | matchAndRewrite(memref::CopyOp copyOp, OpAdaptor adaptor, |
| 354 | ConversionPatternRewriter &rewriter) const override; |
| 355 | }; |
| 356 | |
| 357 | class ReinterpretCastPattern final |
| 358 | : public OpConversionPattern<memref::ReinterpretCastOp> { |
| 359 | public: |
| 360 | using Base::Base; |
| 361 | |
| 362 | LogicalResult |
| 363 | matchAndRewrite(memref::ReinterpretCastOp op, OpAdaptor adaptor, |
| 364 | ConversionPatternRewriter &rewriter) const override; |
| 365 | }; |
| 366 | |
| 367 | class CastPattern final : public OpConversionPattern<memref::CastOp> { |
| 368 | public: |
| 369 | using Base::Base; |
| 370 | |
| 371 | LogicalResult |
| 372 | matchAndRewrite(memref::CastOp op, OpAdaptor adaptor, |
| 373 | ConversionPatternRewriter &rewriter) const override { |
| 374 | Value src = adaptor.getSource(); |
| 375 | Type srcType = src.getType(); |
| 376 | |
| 377 | const TypeConverter *converter = getTypeConverter(); |
| 378 | Type dstType = converter->convertType(t: op.getType()); |
| 379 | if (srcType != dstType) |
| 380 | return rewriter.notifyMatchFailure(op, reasonCallback: [&](Diagnostic &diag) { |
| 381 | diag << "types doesn't match: " << srcType << " and " << dstType; |
| 382 | }); |
| 383 | |
| 384 | rewriter.replaceOp(op, newValues: src); |
| 385 | return success(); |
| 386 | } |
| 387 | }; |
| 388 | |
| 389 | /// Converts memref.extract_aligned_pointer_as_index to spirv.ConvertPtrToU. |
| 390 | class final |
| 391 | : public OpConversionPattern<memref::ExtractAlignedPointerAsIndexOp> { |
| 392 | public: |
| 393 | using Base::Base; |
| 394 | |
| 395 | LogicalResult |
| 396 | matchAndRewrite(memref::ExtractAlignedPointerAsIndexOp , |
| 397 | OpAdaptor adaptor, |
| 398 | ConversionPatternRewriter &rewriter) const override; |
| 399 | }; |
| 400 | } // namespace |
| 401 | |
| 402 | //===----------------------------------------------------------------------===// |
| 403 | // AllocaOp |
| 404 | //===----------------------------------------------------------------------===// |
| 405 | |
| 406 | LogicalResult |
| 407 | AllocaOpPattern::matchAndRewrite(memref::AllocaOp allocaOp, OpAdaptor adaptor, |
| 408 | ConversionPatternRewriter &rewriter) const { |
| 409 | MemRefType allocType = allocaOp.getType(); |
| 410 | if (!isAllocationSupported(allocOp: allocaOp, type: allocType)) |
| 411 | return rewriter.notifyMatchFailure(arg&: allocaOp, msg: "unhandled allocation type" ); |
| 412 | |
| 413 | // Get the SPIR-V type for the allocation. |
| 414 | Type spirvType = getTypeConverter()->convertType(t: allocType); |
| 415 | if (!spirvType) |
| 416 | return rewriter.notifyMatchFailure(arg&: allocaOp, msg: "type conversion failed" ); |
| 417 | |
| 418 | auto function = allocaOp->getParentOfType<FunctionOpInterface>(); |
| 419 | if (!function) |
| 420 | return rewriter.notifyMatchFailure(arg&: allocaOp, |
| 421 | msg: "requires a containing function" ); |
| 422 | |
| 423 | // SPIR-V requires Function variables to be declared in the first block. |
| 424 | OpBuilder::InsertionGuard guard(rewriter); |
| 425 | Block &entryBlock = function->getRegion(index: 0).front(); |
| 426 | Block::iterator insertionPoint = entryBlock.begin(); |
| 427 | // Insert the variable after any existing ones to preserve ordering. |
| 428 | while (insertionPoint != entryBlock.end() && |
| 429 | isa<spirv::VariableOp>(Val: *insertionPoint)) |
| 430 | ++insertionPoint; |
| 431 | rewriter.setInsertionPoint(block: &entryBlock, insertPoint: insertionPoint); |
| 432 | Value variable = spirv::VariableOp::create( |
| 433 | builder&: rewriter, location: allocaOp.getLoc(), pointer: spirvType, storage_class: spirv::StorageClass::Function, |
| 434 | /*initializer=*/nullptr); |
| 435 | rewriter.replaceOp(op: allocaOp, newValues: variable); |
| 436 | return success(); |
| 437 | } |
| 438 | |
| 439 | //===----------------------------------------------------------------------===// |
| 440 | // AllocOp |
| 441 | //===----------------------------------------------------------------------===// |
| 442 | |
| 443 | LogicalResult |
| 444 | AllocOpPattern::matchAndRewrite(memref::AllocOp operation, OpAdaptor adaptor, |
| 445 | ConversionPatternRewriter &rewriter) const { |
| 446 | MemRefType allocType = operation.getType(); |
| 447 | if (!isAllocationSupported(allocOp: operation, type: allocType)) |
| 448 | return rewriter.notifyMatchFailure(arg&: operation, msg: "unhandled allocation type" ); |
| 449 | |
| 450 | // Get the SPIR-V type for the allocation. |
| 451 | Type spirvType = getTypeConverter()->convertType(t: allocType); |
| 452 | if (!spirvType) |
| 453 | return rewriter.notifyMatchFailure(arg&: operation, msg: "type conversion failed" ); |
| 454 | |
| 455 | // Insert spirv.GlobalVariable for this allocation. |
| 456 | Operation *parent = |
| 457 | SymbolTable::getNearestSymbolTable(from: operation->getParentOp()); |
| 458 | if (!parent) |
| 459 | return failure(); |
| 460 | Location loc = operation.getLoc(); |
| 461 | spirv::GlobalVariableOp varOp; |
| 462 | { |
| 463 | OpBuilder::InsertionGuard guard(rewriter); |
| 464 | Block &entryBlock = *parent->getRegion(index: 0).begin(); |
| 465 | rewriter.setInsertionPointToStart(&entryBlock); |
| 466 | auto varOps = entryBlock.getOps<spirv::GlobalVariableOp>(); |
| 467 | std::string varName = |
| 468 | std::string("__workgroup_mem__" ) + |
| 469 | std::to_string(val: std::distance(first: varOps.begin(), last: varOps.end())); |
| 470 | varOp = spirv::GlobalVariableOp::create(builder&: rewriter, location: loc, type: spirvType, sym_name: varName, |
| 471 | /*initializer=*/nullptr); |
| 472 | } |
| 473 | |
| 474 | // Get pointer to global variable at the current scope. |
| 475 | rewriter.replaceOpWithNewOp<spirv::AddressOfOp>(op: operation, args&: varOp); |
| 476 | return success(); |
| 477 | } |
| 478 | |
| 479 | //===----------------------------------------------------------------------===// |
| 480 | // AllocOp |
| 481 | //===----------------------------------------------------------------------===// |
| 482 | |
| 483 | LogicalResult |
| 484 | AtomicRMWOpPattern::matchAndRewrite(memref::AtomicRMWOp atomicOp, |
| 485 | OpAdaptor adaptor, |
| 486 | ConversionPatternRewriter &rewriter) const { |
| 487 | auto memrefType = cast<MemRefType>(Val: atomicOp.getMemref().getType()); |
| 488 | std::optional<spirv::Scope> scope = getAtomicOpScope(type: memrefType); |
| 489 | if (!scope) |
| 490 | return rewriter.notifyMatchFailure(arg&: atomicOp, |
| 491 | msg: "unsupported memref memory space" ); |
| 492 | |
| 493 | auto &typeConverter = *getTypeConverter<SPIRVTypeConverter>(); |
| 494 | Type resultType = typeConverter.convertType(t: atomicOp.getType()); |
| 495 | if (!resultType) |
| 496 | return rewriter.notifyMatchFailure(arg&: atomicOp, |
| 497 | msg: "failed to convert result type" ); |
| 498 | |
| 499 | auto loc = atomicOp.getLoc(); |
| 500 | Value ptr = |
| 501 | spirv::getElementPtr(typeConverter, baseType: memrefType, basePtr: adaptor.getMemref(), |
| 502 | indices: adaptor.getIndices(), loc, builder&: rewriter); |
| 503 | |
| 504 | if (!ptr) |
| 505 | return failure(); |
| 506 | |
| 507 | // Determine the source and destination bitwidths. The source is the original |
| 508 | // memref element type and the destination is the SPIR-V storage type (e.g., |
| 509 | // i32 for Vulkan). |
| 510 | int srcBits = memrefType.getElementType().getIntOrFloatBitWidth(); |
| 511 | auto pointerType = typeConverter.convertType<spirv::PointerType>(t: memrefType); |
| 512 | if (!pointerType) |
| 513 | return rewriter.notifyMatchFailure(arg&: atomicOp, |
| 514 | msg: "failed to convert memref type" ); |
| 515 | |
| 516 | Type pointeeType = pointerType.getPointeeType(); |
| 517 | Type storageElemType = |
| 518 | getElementTypeForStoragePointer(pointeeType, typeConverter); |
| 519 | if (!storageElemType || !storageElemType.isIntOrFloat()) |
| 520 | return rewriter.notifyMatchFailure( |
| 521 | arg&: atomicOp, msg: "failed to determine destination element type" ); |
| 522 | |
| 523 | int dstBits = static_cast<int>(storageElemType.getIntOrFloatBitWidth()); |
| 524 | assert(dstBits % srcBits == 0); |
| 525 | |
| 526 | spirv::MemorySemantics memSem = getAtomicAcqRelMemorySemantics(type: memrefType); |
| 527 | |
| 528 | // When the source and destination bitwidths match, emit the atomic operation |
| 529 | // directly. |
| 530 | if (srcBits == dstBits) { |
| 531 | #define ATOMIC_CASE(kind, spirvOp) \ |
| 532 | case arith::AtomicRMWKind::kind: \ |
| 533 | rewriter.replaceOpWithNewOp<spirv::spirvOp>( \ |
| 534 | atomicOp, resultType, ptr, *scope, memSem, adaptor.getValue()); \ |
| 535 | break |
| 536 | |
| 537 | switch (atomicOp.getKind()) { |
| 538 | ATOMIC_CASE(addf, EXTAtomicFAddOp); |
| 539 | ATOMIC_CASE(addi, AtomicIAddOp); |
| 540 | ATOMIC_CASE(maxs, AtomicSMaxOp); |
| 541 | ATOMIC_CASE(maxu, AtomicUMaxOp); |
| 542 | ATOMIC_CASE(mins, AtomicSMinOp); |
| 543 | ATOMIC_CASE(minu, AtomicUMinOp); |
| 544 | ATOMIC_CASE(ori, AtomicOrOp); |
| 545 | ATOMIC_CASE(andi, AtomicAndOp); |
| 546 | ATOMIC_CASE(xori, AtomicXorOp); |
| 547 | default: |
| 548 | return rewriter.notifyMatchFailure(arg&: atomicOp, msg: "unimplemented atomic kind" ); |
| 549 | } |
| 550 | |
| 551 | #undef ATOMIC_CASE |
| 552 | |
| 553 | return success(); |
| 554 | } |
| 555 | |
| 556 | // Sub-element-width atomic: the element type (e.g., i8) is narrower than the |
| 557 | // storage type (e.g., i32). We need to adjust the index and shift/mask the |
| 558 | // value to operate on the correct bits within the wider storage element. |
| 559 | // |
| 560 | // Only ori and andi can be emulated because they operate bitwise and don't |
| 561 | // carry across byte boundaries. Other kinds (addi, max, min) would require |
| 562 | // CAS loops. |
| 563 | if (atomicOp.getKind() != arith::AtomicRMWKind::ori && |
| 564 | atomicOp.getKind() != arith::AtomicRMWKind::andi) { |
| 565 | return rewriter.notifyMatchFailure( |
| 566 | arg&: atomicOp, |
| 567 | msg: "atomic op on sub-element-width types is only supported for ori/andi" ); |
| 568 | } |
| 569 | |
| 570 | // Bitcasting is currently unsupported for Kernel capability / |
| 571 | // spirv.PtrAccessChain. |
| 572 | if (typeConverter.allows(capability: spirv::Capability::Kernel)) |
| 573 | return rewriter.notifyMatchFailure( |
| 574 | arg&: atomicOp, |
| 575 | msg: "sub-element-width atomic ops unsupported with Kernel capability" ); |
| 576 | |
| 577 | auto dstType = cast<IntegerType>(Val&: storageElemType); |
| 578 | |
| 579 | auto accessChainOp = ptr.getDefiningOp<spirv::AccessChainOp>(); |
| 580 | if (!accessChainOp) |
| 581 | return failure(); |
| 582 | |
| 583 | // Compute the bit offset within the storage element and adjust the pointer |
| 584 | // to address the containing storage element. |
| 585 | assert(accessChainOp.getIndices().size() == 2); |
| 586 | Value lastDim = accessChainOp->getOperand(idx: accessChainOp.getNumOperands() - 1); |
| 587 | Value offset = getOffsetForBitwidth(loc, srcIdx: lastDim, sourceBits: srcBits, targetBits: dstBits, builder&: rewriter); |
| 588 | Value adjustedPtr = adjustAccessChainForBitwidth(typeConverter, op: accessChainOp, |
| 589 | sourceBits: srcBits, targetBits: dstBits, builder&: rewriter); |
| 590 | Value result; |
| 591 | switch (atomicOp.getKind()) { |
| 592 | case arith::AtomicRMWKind::ori: { |
| 593 | // OR only sets bits, so shifting the value to the target position and |
| 594 | // ORing with zeros in other positions preserves the unaffected bits. |
| 595 | Value elemMask = rewriter.createOrFold<spirv::ConstantOp>( |
| 596 | location: loc, args&: dstType, args: rewriter.getIntegerAttr(type: dstType, value: (1uLL << srcBits) - 1)); |
| 597 | Value storeVal = |
| 598 | shiftValue(loc, value: adaptor.getValue(), offset, mask: elemMask, builder&: rewriter); |
| 599 | result = spirv::AtomicOrOp::create(builder&: rewriter, location: loc, result: dstType, pointer: adjustedPtr, |
| 600 | memory_scope: *scope, semantics: memSem, value: storeVal); |
| 601 | break; |
| 602 | } |
| 603 | case arith::AtomicRMWKind::andi: { |
| 604 | // Build a mask that preserves all bits outside the target element |
| 605 | // and applies the operand mask to the target element. |
| 606 | // mask = (operand << offset) | ~(elemMask << offset) |
| 607 | Value elemMask = rewriter.createOrFold<spirv::ConstantOp>( |
| 608 | location: loc, args&: dstType, args: rewriter.getIntegerAttr(type: dstType, value: (1uLL << srcBits) - 1)); |
| 609 | Value storeVal = |
| 610 | shiftValue(loc, value: adaptor.getValue(), offset, mask: elemMask, builder&: rewriter); |
| 611 | Value shiftedElemMask = rewriter.createOrFold<spirv::ShiftLeftLogicalOp>( |
| 612 | location: loc, args&: dstType, args&: elemMask, args&: offset); |
| 613 | Value invertedElemMask = |
| 614 | rewriter.createOrFold<spirv::NotOp>(location: loc, args&: dstType, args&: shiftedElemMask); |
| 615 | Value mask = rewriter.createOrFold<spirv::BitwiseOrOp>(location: loc, args&: storeVal, |
| 616 | args&: invertedElemMask); |
| 617 | result = spirv::AtomicAndOp::create(builder&: rewriter, location: loc, result: dstType, pointer: adjustedPtr, |
| 618 | memory_scope: *scope, semantics: memSem, value: mask); |
| 619 | break; |
| 620 | } |
| 621 | default: |
| 622 | return rewriter.notifyMatchFailure(arg&: atomicOp, msg: "unimplemented atomic kind" ); |
| 623 | } |
| 624 | |
| 625 | // The atomic op returns the old value of the full storage element (e.g., |
| 626 | // i32). Extract the original sub-element value from the correct position. |
| 627 | result = rewriter.createOrFold<spirv::ShiftRightLogicalOp>(location: loc, args&: dstType, |
| 628 | args&: result, args&: offset); |
| 629 | Value mask = rewriter.createOrFold<spirv::ConstantOp>( |
| 630 | location: loc, args&: dstType, args: rewriter.getIntegerAttr(type: dstType, value: (1uLL << srcBits) - 1)); |
| 631 | result = |
| 632 | rewriter.createOrFold<spirv::BitwiseAndOp>(location: loc, args&: dstType, args&: result, args&: mask); |
| 633 | rewriter.replaceOp(op: atomicOp, newValues: result); |
| 634 | |
| 635 | return success(); |
| 636 | } |
| 637 | |
| 638 | //===----------------------------------------------------------------------===// |
| 639 | // DeallocOp |
| 640 | //===----------------------------------------------------------------------===// |
| 641 | |
| 642 | LogicalResult |
| 643 | DeallocOpPattern::matchAndRewrite(memref::DeallocOp operation, |
| 644 | OpAdaptor adaptor, |
| 645 | ConversionPatternRewriter &rewriter) const { |
| 646 | MemRefType deallocType = cast<MemRefType>(Val: operation.getMemref().getType()); |
| 647 | if (!isAllocationSupported(allocOp: operation, type: deallocType)) |
| 648 | return rewriter.notifyMatchFailure(arg&: operation, msg: "unhandled allocation type" ); |
| 649 | rewriter.eraseOp(op: operation); |
| 650 | return success(); |
| 651 | } |
| 652 | |
| 653 | //===----------------------------------------------------------------------===// |
| 654 | // LoadOp |
| 655 | //===----------------------------------------------------------------------===// |
| 656 | |
| 657 | struct MemoryRequirements { |
| 658 | spirv::MemoryAccessAttr memoryAccess; |
| 659 | IntegerAttr alignment; |
| 660 | }; |
| 661 | |
| 662 | /// Given an accessed SPIR-V pointer, calculates its alignment requirements, if |
| 663 | /// any. |
| 664 | static FailureOr<MemoryRequirements> |
| 665 | calculateMemoryRequirements(Value accessedPtr, bool isNontemporal, |
| 666 | uint64_t preferredAlignment) { |
| 667 | if (preferredAlignment >= std::numeric_limits<uint32_t>::max()) { |
| 668 | return failure(); |
| 669 | } |
| 670 | |
| 671 | MLIRContext *ctx = accessedPtr.getContext(); |
| 672 | |
| 673 | auto memoryAccess = spirv::MemoryAccess::None; |
| 674 | if (isNontemporal) { |
| 675 | memoryAccess = spirv::MemoryAccess::Nontemporal; |
| 676 | } |
| 677 | |
| 678 | auto ptrType = cast<spirv::PointerType>(Val: accessedPtr.getType()); |
| 679 | bool mayOmitAlignment = |
| 680 | !preferredAlignment && |
| 681 | ptrType.getStorageClass() != spirv::StorageClass::PhysicalStorageBuffer; |
| 682 | if (mayOmitAlignment) { |
| 683 | if (memoryAccess == spirv::MemoryAccess::None) { |
| 684 | return MemoryRequirements{.memoryAccess: spirv::MemoryAccessAttr{}, .alignment: IntegerAttr{}}; |
| 685 | } |
| 686 | return MemoryRequirements{.memoryAccess: spirv::MemoryAccessAttr::get(context: ctx, value: memoryAccess), |
| 687 | .alignment: IntegerAttr{}}; |
| 688 | } |
| 689 | |
| 690 | // PhysicalStorageBuffers require the `Aligned` attribute. |
| 691 | // Other storage types may show an `Aligned` attribute. |
| 692 | std::optional<int64_t> sizeInBytes; |
| 693 | Type rawPointeeType = ptrType.getPointeeType(); |
| 694 | if (auto scalarType = dyn_cast<spirv::ScalarType>(Val&: rawPointeeType)) { |
| 695 | // For scalar types, the alignment is determined by their size. |
| 696 | sizeInBytes = scalarType.getSizeInBytes(); |
| 697 | } else if (auto vecType = dyn_cast<VectorType>(Val&: rawPointeeType)) { |
| 698 | // For vector element types, the alignment should equal the total size of |
| 699 | // the vector. |
| 700 | if (auto scalarElem = |
| 701 | dyn_cast<spirv::ScalarType>(Val: vecType.getElementType())) { |
| 702 | if (auto elemSize = scalarElem.getSizeInBytes()) |
| 703 | sizeInBytes = *elemSize * vecType.getNumElements(); |
| 704 | } |
| 705 | } |
| 706 | |
| 707 | if (!sizeInBytes.has_value()) |
| 708 | return failure(); |
| 709 | |
| 710 | memoryAccess |= spirv::MemoryAccess::Aligned; |
| 711 | auto memAccessAttr = spirv::MemoryAccessAttr::get(context: ctx, value: memoryAccess); |
| 712 | auto alignmentValue = preferredAlignment ? preferredAlignment : *sizeInBytes; |
| 713 | auto alignment = IntegerAttr::get(type: IntegerType::get(context: ctx, width: 32), value: alignmentValue); |
| 714 | return MemoryRequirements{.memoryAccess: memAccessAttr, .alignment: alignment}; |
| 715 | } |
| 716 | |
| 717 | /// Given an accessed SPIR-V pointer and the original memref load/store |
| 718 | /// `memAccess` op, calculates the alignment requirements, if any. Takes into |
| 719 | /// account the alignment attributes applied to the load/store op. |
| 720 | template <class LoadOrStoreOp> |
| 721 | static FailureOr<MemoryRequirements> |
| 722 | calculateMemoryRequirements(Value accessedPtr, LoadOrStoreOp loadOrStoreOp) { |
| 723 | static_assert( |
| 724 | llvm::is_one_of<LoadOrStoreOp, memref::LoadOp, memref::StoreOp>::value, |
| 725 | "Must be called on either memref::LoadOp or memref::StoreOp" ); |
| 726 | |
| 727 | return calculateMemoryRequirements(accessedPtr, |
| 728 | loadOrStoreOp.getNontemporal(), |
| 729 | loadOrStoreOp.getAlignment().value_or(0)); |
| 730 | } |
| 731 | |
| 732 | LogicalResult |
| 733 | IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor, |
| 734 | ConversionPatternRewriter &rewriter) const { |
| 735 | auto loc = loadOp.getLoc(); |
| 736 | auto memrefType = cast<MemRefType>(Val: loadOp.getMemref().getType()); |
| 737 | if (!memrefType.getElementType().isSignlessInteger()) |
| 738 | return failure(); |
| 739 | |
| 740 | auto memorySpaceAttr = |
| 741 | dyn_cast_if_present<spirv::StorageClassAttr>(Val: memrefType.getMemorySpace()); |
| 742 | if (!memorySpaceAttr) |
| 743 | return rewriter.notifyMatchFailure( |
| 744 | arg&: loadOp, msg: "missing memory space SPIR-V storage class attribute" ); |
| 745 | |
| 746 | if (memorySpaceAttr.getValue() == spirv::StorageClass::Image) |
| 747 | return rewriter.notifyMatchFailure( |
| 748 | arg&: loadOp, |
| 749 | msg: "failed to lower memref in image storage class to storage buffer" ); |
| 750 | |
| 751 | const auto &typeConverter = *getTypeConverter<SPIRVTypeConverter>(); |
| 752 | Value accessChain = |
| 753 | spirv::getElementPtr(typeConverter, baseType: memrefType, basePtr: adaptor.getMemref(), |
| 754 | indices: adaptor.getIndices(), loc, builder&: rewriter); |
| 755 | |
| 756 | if (!accessChain) |
| 757 | return failure(); |
| 758 | |
| 759 | int srcBits = memrefType.getElementType().getIntOrFloatBitWidth(); |
| 760 | bool isBool = srcBits == 1; |
| 761 | if (isBool) |
| 762 | srcBits = typeConverter.getOptions().boolNumBits; |
| 763 | |
| 764 | auto pointerType = typeConverter.convertType<spirv::PointerType>(t: memrefType); |
| 765 | if (!pointerType) |
| 766 | return rewriter.notifyMatchFailure(arg&: loadOp, msg: "failed to convert memref type" ); |
| 767 | |
| 768 | Type pointeeType = pointerType.getPointeeType(); |
| 769 | Type dstType = getElementTypeForStoragePointer(pointeeType, typeConverter); |
| 770 | int dstBits = dstType.getIntOrFloatBitWidth(); |
| 771 | assert(dstBits % srcBits == 0); |
| 772 | |
| 773 | // If the rewritten load op has the same bit width, use the loading value |
| 774 | // directly. |
| 775 | if (srcBits == dstBits) { |
| 776 | auto memoryRequirements = calculateMemoryRequirements(accessedPtr: accessChain, loadOrStoreOp: loadOp); |
| 777 | if (failed(Result: memoryRequirements)) |
| 778 | return rewriter.notifyMatchFailure( |
| 779 | arg&: loadOp, msg: "failed to determine memory requirements" ); |
| 780 | |
| 781 | auto [memoryAccess, alignment] = *memoryRequirements; |
| 782 | Value loadVal = spirv::LoadOp::create(builder&: rewriter, location: loc, basePtr: accessChain, |
| 783 | memory_access: memoryAccess, alignment); |
| 784 | if (isBool) |
| 785 | loadVal = castIntNToBool(loc, srcInt: loadVal, builder&: rewriter); |
| 786 | rewriter.replaceOp(op: loadOp, newValues: loadVal); |
| 787 | return success(); |
| 788 | } |
| 789 | |
| 790 | // Bitcasting is currently unsupported for Kernel capability / |
| 791 | // spirv.PtrAccessChain. |
| 792 | if (typeConverter.allows(capability: spirv::Capability::Kernel)) |
| 793 | return failure(); |
| 794 | |
| 795 | auto accessChainOp = accessChain.getDefiningOp<spirv::AccessChainOp>(); |
| 796 | if (!accessChainOp) |
| 797 | return failure(); |
| 798 | |
| 799 | // Assume that getElementPtr() works linearizely. If it's a scalar, the method |
| 800 | // still returns a linearized accessing. If the accessing is not linearized, |
| 801 | // there will be offset issues. |
| 802 | assert(accessChainOp.getIndices().size() == 2); |
| 803 | Value adjustedPtr = adjustAccessChainForBitwidth(typeConverter, op: accessChainOp, |
| 804 | sourceBits: srcBits, targetBits: dstBits, builder&: rewriter); |
| 805 | auto memoryRequirements = calculateMemoryRequirements(accessedPtr: adjustedPtr, loadOrStoreOp: loadOp); |
| 806 | if (failed(Result: memoryRequirements)) |
| 807 | return rewriter.notifyMatchFailure( |
| 808 | arg&: loadOp, msg: "failed to determine memory requirements" ); |
| 809 | |
| 810 | auto [memoryAccess, alignment] = *memoryRequirements; |
| 811 | Value spvLoadOp = spirv::LoadOp::create(builder&: rewriter, location: loc, value: dstType, ptr: adjustedPtr, |
| 812 | memory_access: memoryAccess, alignment); |
| 813 | |
| 814 | // Shift the bits to the rightmost. |
| 815 | // ____XXXX________ -> ____________XXXX |
| 816 | Value lastDim = accessChainOp->getOperand(idx: accessChainOp.getNumOperands() - 1); |
| 817 | Value offset = getOffsetForBitwidth(loc, srcIdx: lastDim, sourceBits: srcBits, targetBits: dstBits, builder&: rewriter); |
| 818 | Value result = rewriter.createOrFold<spirv::ShiftRightArithmeticOp>( |
| 819 | location: loc, args: spvLoadOp.getType(), args&: spvLoadOp, args&: offset); |
| 820 | |
| 821 | // Apply the mask to extract corresponding bits. |
| 822 | Value mask = rewriter.createOrFold<spirv::ConstantOp>( |
| 823 | location: loc, args&: dstType, args: rewriter.getIntegerAttr(type: dstType, value: (1 << srcBits) - 1)); |
| 824 | result = |
| 825 | rewriter.createOrFold<spirv::BitwiseAndOp>(location: loc, args&: dstType, args&: result, args&: mask); |
| 826 | |
| 827 | // Apply sign extension on the loading value unconditionally. The signedness |
| 828 | // semantic is carried in the operator itself, we relies other pattern to |
| 829 | // handle the casting. |
| 830 | IntegerAttr shiftValueAttr = |
| 831 | rewriter.getIntegerAttr(type: dstType, value: dstBits - srcBits); |
| 832 | Value shiftValue = |
| 833 | rewriter.createOrFold<spirv::ConstantOp>(location: loc, args&: dstType, args&: shiftValueAttr); |
| 834 | result = rewriter.createOrFold<spirv::ShiftLeftLogicalOp>(location: loc, args&: dstType, |
| 835 | args&: result, args&: shiftValue); |
| 836 | result = rewriter.createOrFold<spirv::ShiftRightArithmeticOp>( |
| 837 | location: loc, args&: dstType, args&: result, args&: shiftValue); |
| 838 | |
| 839 | rewriter.replaceOp(op: loadOp, newValues: result); |
| 840 | |
| 841 | assert(accessChainOp.use_empty()); |
| 842 | rewriter.eraseOp(op: accessChainOp); |
| 843 | |
| 844 | return success(); |
| 845 | } |
| 846 | |
| 847 | LogicalResult |
| 848 | LoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor, |
| 849 | ConversionPatternRewriter &rewriter) const { |
| 850 | auto memrefType = cast<MemRefType>(Val: loadOp.getMemref().getType()); |
| 851 | if (memrefType.getElementType().isSignlessInteger()) |
| 852 | return failure(); |
| 853 | |
| 854 | auto memorySpaceAttr = |
| 855 | dyn_cast_if_present<spirv::StorageClassAttr>(Val: memrefType.getMemorySpace()); |
| 856 | if (!memorySpaceAttr) |
| 857 | return rewriter.notifyMatchFailure( |
| 858 | arg&: loadOp, msg: "missing memory space SPIR-V storage class attribute" ); |
| 859 | |
| 860 | if (memorySpaceAttr.getValue() == spirv::StorageClass::Image) |
| 861 | return rewriter.notifyMatchFailure( |
| 862 | arg&: loadOp, |
| 863 | msg: "failed to lower memref in image storage class to storage buffer" ); |
| 864 | |
| 865 | Value loadPtr = spirv::getElementPtr( |
| 866 | typeConverter: *getTypeConverter<SPIRVTypeConverter>(), baseType: memrefType, basePtr: adaptor.getMemref(), |
| 867 | indices: adaptor.getIndices(), loc: loadOp.getLoc(), builder&: rewriter); |
| 868 | |
| 869 | if (!loadPtr) |
| 870 | return failure(); |
| 871 | |
| 872 | auto memoryRequirements = calculateMemoryRequirements(accessedPtr: loadPtr, loadOrStoreOp: loadOp); |
| 873 | if (failed(Result: memoryRequirements)) |
| 874 | return rewriter.notifyMatchFailure( |
| 875 | arg&: loadOp, msg: "failed to determine memory requirements" ); |
| 876 | |
| 877 | auto [memoryAccess, alignment] = *memoryRequirements; |
| 878 | rewriter.replaceOpWithNewOp<spirv::LoadOp>(op: loadOp, args&: loadPtr, args&: memoryAccess, |
| 879 | args&: alignment); |
| 880 | return success(); |
| 881 | } |
| 882 | |
| 883 | template <typename OpAdaptor> |
| 884 | static FailureOr<SmallVector<Value>> |
| 885 | (memref::LoadOp loadOp, OpAdaptor adaptor, |
| 886 | ConversionPatternRewriter &rewriter) { |
| 887 | // At present we only support linear "tiling" as specified in Vulkan, this |
| 888 | // means that texels are assumed to be laid out in memory in a row-major |
| 889 | // order. This allows us to support any memref layout that is a permutation of |
| 890 | // the dimensions. Future work will pass an optional image layout to the |
| 891 | // rewrite pattern so that we can support optimized target specific tilings. |
| 892 | SmallVector<Value> indices = adaptor.getIndices(); |
| 893 | AffineMap map = loadOp.getMemRefType().getLayout().getAffineMap(); |
| 894 | if (!map.isPermutation()) |
| 895 | return rewriter.notifyMatchFailure( |
| 896 | arg&: loadOp, |
| 897 | msg: "Cannot lower memrefs with memory layout which is not a permutation" ); |
| 898 | |
| 899 | // The memrefs layout determines the dimension ordering so we need to follow |
| 900 | // the map to get the ordering of the dimensions/indices. |
| 901 | const unsigned dimCount = map.getNumDims(); |
| 902 | SmallVector<Value, 3> coords(dimCount); |
| 903 | for (unsigned dim = 0; dim < dimCount; ++dim) |
| 904 | coords[map.getDimPosition(idx: dim)] = indices[dim]; |
| 905 | |
| 906 | // We need to reverse the coordinates because the memref layout is slowest to |
| 907 | // fastest moving and the vector coordinates for the image op is fastest to |
| 908 | // slowest moving. |
| 909 | return llvm::to_vector(Range: llvm::reverse(C&: coords)); |
| 910 | } |
| 911 | |
| 912 | LogicalResult |
| 913 | ImageLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor, |
| 914 | ConversionPatternRewriter &rewriter) const { |
| 915 | auto memrefType = cast<MemRefType>(Val: loadOp.getMemref().getType()); |
| 916 | |
| 917 | auto memorySpaceAttr = |
| 918 | dyn_cast_if_present<spirv::StorageClassAttr>(Val: memrefType.getMemorySpace()); |
| 919 | if (!memorySpaceAttr) |
| 920 | return rewriter.notifyMatchFailure( |
| 921 | arg&: loadOp, msg: "missing memory space SPIR-V storage class attribute" ); |
| 922 | |
| 923 | if (memorySpaceAttr.getValue() != spirv::StorageClass::Image) |
| 924 | return rewriter.notifyMatchFailure( |
| 925 | arg&: loadOp, msg: "failed to lower memref in non-image storage class to image" ); |
| 926 | |
| 927 | Value loadPtr = adaptor.getMemref(); |
| 928 | auto memoryRequirements = calculateMemoryRequirements(accessedPtr: loadPtr, loadOrStoreOp: loadOp); |
| 929 | if (failed(Result: memoryRequirements)) |
| 930 | return rewriter.notifyMatchFailure( |
| 931 | arg&: loadOp, msg: "failed to determine memory requirements" ); |
| 932 | |
| 933 | const auto [memoryAccess, alignment] = *memoryRequirements; |
| 934 | |
| 935 | if (!loadOp.getMemRefType().hasRank()) |
| 936 | return rewriter.notifyMatchFailure( |
| 937 | arg&: loadOp, msg: "cannot lower unranked memrefs to SPIR-V images" ); |
| 938 | |
| 939 | // We currently only support lowering of scalar memref elements to texels in |
| 940 | // the R[16|32][f|i|ui] formats. Future work will enable lowering of vector |
| 941 | // elements to texels in richer formats. |
| 942 | if (!isa<spirv::ScalarType>(Val: loadOp.getMemRefType().getElementType())) |
| 943 | return rewriter.notifyMatchFailure( |
| 944 | arg&: loadOp, |
| 945 | msg: "cannot lower memrefs who's element type is not a SPIR-V scalar type" |
| 946 | "to SPIR-V images" ); |
| 947 | |
| 948 | // We currently only support sampled images since OpImageFetch does not work |
| 949 | // for plain images and the OpImageRead instruction needs to be materialized |
| 950 | // instead or texels need to be accessed via atomics through a texel pointer. |
| 951 | // Future work will generalize support to plain images. |
| 952 | auto convertedPointeeType = cast<spirv::PointerType>( |
| 953 | Val: getTypeConverter()->convertType(t: loadOp.getMemRefType())); |
| 954 | if (!isa<spirv::SampledImageType>(Val: convertedPointeeType.getPointeeType())) |
| 955 | return rewriter.notifyMatchFailure(arg&: loadOp, |
| 956 | msg: "cannot lower memrefs which do not " |
| 957 | "convert to SPIR-V sampled images" ); |
| 958 | |
| 959 | // Materialize the lowering. |
| 960 | Location loc = loadOp->getLoc(); |
| 961 | auto imageLoadOp = |
| 962 | spirv::LoadOp::create(builder&: rewriter, location: loc, basePtr: loadPtr, memory_access: memoryAccess, alignment); |
| 963 | // Extract the image from the sampled image. |
| 964 | auto imageOp = spirv::ImageOp::create(builder&: rewriter, location: loc, sampled_image: imageLoadOp); |
| 965 | |
| 966 | // Build a vector of coordinates or just a scalar index if we have a 1D image. |
| 967 | Value coords; |
| 968 | if (memrefType.getRank() == 1) { |
| 969 | coords = adaptor.getIndices()[0]; |
| 970 | } else { |
| 971 | FailureOr<SmallVector<Value>> maybeCoords = |
| 972 | extractLoadCoordsForComposite(loadOp, adaptor, rewriter); |
| 973 | if (failed(Result: maybeCoords)) |
| 974 | return failure(); |
| 975 | auto coordVectorType = VectorType::get(shape: {loadOp.getMemRefType().getRank()}, |
| 976 | elementType: adaptor.getIndices().getType()[0]); |
| 977 | coords = spirv::CompositeConstructOp::create(builder&: rewriter, location: loc, result: coordVectorType, |
| 978 | constituents: maybeCoords.value()); |
| 979 | } |
| 980 | |
| 981 | // Fetch the value out of the image. |
| 982 | auto resultVectorType = VectorType::get(shape: {4}, elementType: loadOp.getType()); |
| 983 | auto fetchOp = spirv::ImageFetchOp::create( |
| 984 | builder&: rewriter, location: loc, result: resultVectorType, image: imageOp, coordinate: coords, |
| 985 | image_operands: mlir::spirv::ImageOperandsAttr{}, operand_arguments: ValueRange{}); |
| 986 | |
| 987 | // Note that because OpImageFetch returns a rank 4 vector we need to extract |
| 988 | // the elements corresponding to the load which will since we only support the |
| 989 | // R[16|32][f|i|ui] formats will always be the R(red) 0th vector element. |
| 990 | auto = |
| 991 | spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: fetchOp, indices: 0); |
| 992 | |
| 993 | rewriter.replaceOp(op: loadOp, newOp: compositeExtractOp); |
| 994 | return success(); |
| 995 | } |
| 996 | |
| 997 | LogicalResult |
| 998 | IntStoreOpPattern::matchAndRewrite(memref::StoreOp storeOp, OpAdaptor adaptor, |
| 999 | ConversionPatternRewriter &rewriter) const { |
| 1000 | auto memrefType = cast<MemRefType>(Val: storeOp.getMemref().getType()); |
| 1001 | if (!memrefType.getElementType().isSignlessInteger()) |
| 1002 | return rewriter.notifyMatchFailure(arg&: storeOp, |
| 1003 | msg: "element type is not a signless int" ); |
| 1004 | |
| 1005 | auto loc = storeOp.getLoc(); |
| 1006 | auto &typeConverter = *getTypeConverter<SPIRVTypeConverter>(); |
| 1007 | Value accessChain = |
| 1008 | spirv::getElementPtr(typeConverter, baseType: memrefType, basePtr: adaptor.getMemref(), |
| 1009 | indices: adaptor.getIndices(), loc, builder&: rewriter); |
| 1010 | |
| 1011 | if (!accessChain) |
| 1012 | return rewriter.notifyMatchFailure( |
| 1013 | arg&: storeOp, msg: "failed to convert element pointer type" ); |
| 1014 | |
| 1015 | int srcBits = memrefType.getElementType().getIntOrFloatBitWidth(); |
| 1016 | |
| 1017 | bool isBool = srcBits == 1; |
| 1018 | if (isBool) |
| 1019 | srcBits = typeConverter.getOptions().boolNumBits; |
| 1020 | |
| 1021 | auto pointerType = typeConverter.convertType<spirv::PointerType>(t: memrefType); |
| 1022 | if (!pointerType) |
| 1023 | return rewriter.notifyMatchFailure(arg&: storeOp, |
| 1024 | msg: "failed to convert memref type" ); |
| 1025 | |
| 1026 | Type pointeeType = pointerType.getPointeeType(); |
| 1027 | auto dstType = dyn_cast<IntegerType>( |
| 1028 | Val: getElementTypeForStoragePointer(pointeeType, typeConverter)); |
| 1029 | if (!dstType) |
| 1030 | return rewriter.notifyMatchFailure( |
| 1031 | arg&: storeOp, msg: "failed to determine destination element type" ); |
| 1032 | |
| 1033 | int dstBits = static_cast<int>(dstType.getWidth()); |
| 1034 | assert(dstBits % srcBits == 0); |
| 1035 | |
| 1036 | if (srcBits == dstBits) { |
| 1037 | auto memoryRequirements = calculateMemoryRequirements(accessedPtr: accessChain, loadOrStoreOp: storeOp); |
| 1038 | if (failed(Result: memoryRequirements)) |
| 1039 | return rewriter.notifyMatchFailure( |
| 1040 | arg&: storeOp, msg: "failed to determine memory requirements" ); |
| 1041 | |
| 1042 | auto [memoryAccess, alignment] = *memoryRequirements; |
| 1043 | Value storeVal = adaptor.getValue(); |
| 1044 | if (isBool) |
| 1045 | storeVal = castBoolToIntN(loc, srcBool: storeVal, dstType, builder&: rewriter); |
| 1046 | rewriter.replaceOpWithNewOp<spirv::StoreOp>(op: storeOp, args&: accessChain, args&: storeVal, |
| 1047 | args&: memoryAccess, args&: alignment); |
| 1048 | return success(); |
| 1049 | } |
| 1050 | |
| 1051 | // Bitcasting is currently unsupported for Kernel capability / |
| 1052 | // spirv.PtrAccessChain. |
| 1053 | if (typeConverter.allows(capability: spirv::Capability::Kernel)) |
| 1054 | return failure(); |
| 1055 | |
| 1056 | auto accessChainOp = accessChain.getDefiningOp<spirv::AccessChainOp>(); |
| 1057 | if (!accessChainOp) |
| 1058 | return failure(); |
| 1059 | |
| 1060 | // Since there are multiple threads in the processing, the emulation will be |
| 1061 | // done with atomic operations. E.g., if the stored value is i8, rewrite the |
| 1062 | // StoreOp to: |
| 1063 | // 1) load a 32-bit integer |
| 1064 | // 2) clear 8 bits in the loaded value |
| 1065 | // 3) set 8 bits in the loaded value |
| 1066 | // 4) store 32-bit value back |
| 1067 | // |
| 1068 | // Step 2 is done with AtomicAnd, and step 3 is done with AtomicOr (of the |
| 1069 | // loaded 32-bit value and the shifted 8-bit store value) as another atomic |
| 1070 | // step. |
| 1071 | assert(accessChainOp.getIndices().size() == 2); |
| 1072 | Value lastDim = accessChainOp->getOperand(idx: accessChainOp.getNumOperands() - 1); |
| 1073 | Value offset = getOffsetForBitwidth(loc, srcIdx: lastDim, sourceBits: srcBits, targetBits: dstBits, builder&: rewriter); |
| 1074 | |
| 1075 | // Create a mask to clear the destination. E.g., if it is the second i8 in |
| 1076 | // i32, 0xFFFF00FF is created. |
| 1077 | Value mask = rewriter.createOrFold<spirv::ConstantOp>( |
| 1078 | location: loc, args&: dstType, args: rewriter.getIntegerAttr(type: dstType, value: (1 << srcBits) - 1)); |
| 1079 | Value clearBitsMask = rewriter.createOrFold<spirv::ShiftLeftLogicalOp>( |
| 1080 | location: loc, args&: dstType, args&: mask, args&: offset); |
| 1081 | clearBitsMask = |
| 1082 | rewriter.createOrFold<spirv::NotOp>(location: loc, args&: dstType, args&: clearBitsMask); |
| 1083 | |
| 1084 | Value storeVal = shiftValue(loc, value: adaptor.getValue(), offset, mask, builder&: rewriter); |
| 1085 | Value adjustedPtr = adjustAccessChainForBitwidth(typeConverter, op: accessChainOp, |
| 1086 | sourceBits: srcBits, targetBits: dstBits, builder&: rewriter); |
| 1087 | std::optional<spirv::Scope> scope = getAtomicOpScope(type: memrefType); |
| 1088 | if (!scope) |
| 1089 | return rewriter.notifyMatchFailure(arg&: storeOp, msg: "atomic scope not available" ); |
| 1090 | |
| 1091 | spirv::MemorySemantics memSem = getAtomicAcqRelMemorySemantics(type: memrefType); |
| 1092 | Value result = spirv::AtomicAndOp::create(builder&: rewriter, location: loc, result: dstType, pointer: adjustedPtr, |
| 1093 | memory_scope: *scope, semantics: memSem, value: clearBitsMask); |
| 1094 | result = spirv::AtomicOrOp::create(builder&: rewriter, location: loc, result: dstType, pointer: adjustedPtr, |
| 1095 | memory_scope: *scope, semantics: memSem, value: storeVal); |
| 1096 | |
| 1097 | // The AtomicOrOp has no side effect. Since it is already inserted, we can |
| 1098 | // just remove the original StoreOp. Note that rewriter.replaceOp() |
| 1099 | // doesn't work because it only accepts that the numbers of result are the |
| 1100 | // same. |
| 1101 | rewriter.eraseOp(op: storeOp); |
| 1102 | |
| 1103 | assert(accessChainOp.use_empty()); |
| 1104 | rewriter.eraseOp(op: accessChainOp); |
| 1105 | |
| 1106 | return success(); |
| 1107 | } |
| 1108 | |
| 1109 | //===----------------------------------------------------------------------===// |
| 1110 | // MemorySpaceCastOp |
| 1111 | //===----------------------------------------------------------------------===// |
| 1112 | |
| 1113 | LogicalResult MemorySpaceCastOpPattern::matchAndRewrite( |
| 1114 | memref::MemorySpaceCastOp addrCastOp, OpAdaptor adaptor, |
| 1115 | ConversionPatternRewriter &rewriter) const { |
| 1116 | Location loc = addrCastOp.getLoc(); |
| 1117 | auto &typeConverter = *getTypeConverter<SPIRVTypeConverter>(); |
| 1118 | if (!typeConverter.allows(capability: spirv::Capability::Kernel)) |
| 1119 | return rewriter.notifyMatchFailure( |
| 1120 | arg&: loc, msg: "address space casts require kernel capability" ); |
| 1121 | |
| 1122 | auto sourceType = dyn_cast<MemRefType>(Val: addrCastOp.getSource().getType()); |
| 1123 | if (!sourceType) |
| 1124 | return rewriter.notifyMatchFailure( |
| 1125 | arg&: loc, msg: "SPIR-V lowering requires ranked memref types" ); |
| 1126 | auto resultType = cast<MemRefType>(Val: addrCastOp.getResult().getType()); |
| 1127 | |
| 1128 | auto sourceStorageClassAttr = |
| 1129 | dyn_cast_or_null<spirv::StorageClassAttr>(Val: sourceType.getMemorySpace()); |
| 1130 | if (!sourceStorageClassAttr) |
| 1131 | return rewriter.notifyMatchFailure(loc, reasonCallback: [sourceType](Diagnostic &diag) { |
| 1132 | diag << "source address space " << sourceType.getMemorySpace() |
| 1133 | << " must be a SPIR-V storage class" ; |
| 1134 | }); |
| 1135 | auto resultStorageClassAttr = |
| 1136 | dyn_cast_or_null<spirv::StorageClassAttr>(Val: resultType.getMemorySpace()); |
| 1137 | if (!resultStorageClassAttr) |
| 1138 | return rewriter.notifyMatchFailure(loc, reasonCallback: [resultType](Diagnostic &diag) { |
| 1139 | diag << "result address space " << resultType.getMemorySpace() |
| 1140 | << " must be a SPIR-V storage class" ; |
| 1141 | }); |
| 1142 | |
| 1143 | spirv::StorageClass sourceSc = sourceStorageClassAttr.getValue(); |
| 1144 | spirv::StorageClass resultSc = resultStorageClassAttr.getValue(); |
| 1145 | |
| 1146 | Value result = adaptor.getSource(); |
| 1147 | Type resultPtrType = typeConverter.convertType(t: resultType); |
| 1148 | if (!resultPtrType) |
| 1149 | return rewriter.notifyMatchFailure(arg&: addrCastOp, |
| 1150 | msg: "failed to convert memref type" ); |
| 1151 | |
| 1152 | Type genericPtrType = resultPtrType; |
| 1153 | // SPIR-V doesn't have a general address space cast operation. Instead, it has |
| 1154 | // conversions to and from generic pointers. To implement the general case, |
| 1155 | // we use specific-to-generic conversions when the source class is not |
| 1156 | // generic. Then when the result storage class is not generic, we convert the |
| 1157 | // generic pointer (either the input on ar intermediate result) to that |
| 1158 | // class. This also means that we'll need the intermediate generic pointer |
| 1159 | // type if neither the source or destination have it. |
| 1160 | if (sourceSc != spirv::StorageClass::Generic && |
| 1161 | resultSc != spirv::StorageClass::Generic) { |
| 1162 | Type intermediateType = |
| 1163 | MemRefType::get(shape: sourceType.getShape(), elementType: sourceType.getElementType(), |
| 1164 | layout: sourceType.getLayout(), |
| 1165 | memorySpace: rewriter.getAttr<spirv::StorageClassAttr>( |
| 1166 | args: spirv::StorageClass::Generic)); |
| 1167 | genericPtrType = typeConverter.convertType(t: intermediateType); |
| 1168 | } |
| 1169 | if (sourceSc != spirv::StorageClass::Generic) { |
| 1170 | result = spirv::PtrCastToGenericOp::create(builder&: rewriter, location: loc, result: genericPtrType, |
| 1171 | pointer: result); |
| 1172 | } |
| 1173 | if (resultSc != spirv::StorageClass::Generic) { |
| 1174 | result = |
| 1175 | spirv::GenericCastToPtrOp::create(builder&: rewriter, location: loc, result: resultPtrType, pointer: result); |
| 1176 | } |
| 1177 | rewriter.replaceOp(op: addrCastOp, newValues: result); |
| 1178 | return success(); |
| 1179 | } |
| 1180 | |
| 1181 | LogicalResult |
| 1182 | StoreOpPattern::matchAndRewrite(memref::StoreOp storeOp, OpAdaptor adaptor, |
| 1183 | ConversionPatternRewriter &rewriter) const { |
| 1184 | auto memrefType = cast<MemRefType>(Val: storeOp.getMemref().getType()); |
| 1185 | if (memrefType.getElementType().isSignlessInteger()) |
| 1186 | return rewriter.notifyMatchFailure(arg&: storeOp, msg: "signless int" ); |
| 1187 | auto storePtr = spirv::getElementPtr( |
| 1188 | typeConverter: *getTypeConverter<SPIRVTypeConverter>(), baseType: memrefType, basePtr: adaptor.getMemref(), |
| 1189 | indices: adaptor.getIndices(), loc: storeOp.getLoc(), builder&: rewriter); |
| 1190 | |
| 1191 | if (!storePtr) |
| 1192 | return rewriter.notifyMatchFailure(arg&: storeOp, msg: "type conversion failed" ); |
| 1193 | |
| 1194 | auto memoryRequirements = calculateMemoryRequirements(accessedPtr: storePtr, loadOrStoreOp: storeOp); |
| 1195 | if (failed(Result: memoryRequirements)) |
| 1196 | return rewriter.notifyMatchFailure( |
| 1197 | arg&: storeOp, msg: "failed to determine memory requirements" ); |
| 1198 | |
| 1199 | auto [memoryAccess, alignment] = *memoryRequirements; |
| 1200 | rewriter.replaceOpWithNewOp<spirv::StoreOp>( |
| 1201 | op: storeOp, args&: storePtr, args: adaptor.getValue(), args&: memoryAccess, args&: alignment); |
| 1202 | return success(); |
| 1203 | } |
| 1204 | |
| 1205 | //===----------------------------------------------------------------------===// |
| 1206 | // CopyOp |
| 1207 | //===----------------------------------------------------------------------===// |
| 1208 | |
| 1209 | LogicalResult |
| 1210 | CopyOpPattern::matchAndRewrite(memref::CopyOp copyOp, OpAdaptor adaptor, |
| 1211 | ConversionPatternRewriter &rewriter) const { |
| 1212 | auto memrefType = cast<MemRefType>(Val: copyOp.getSource().getType()); |
| 1213 | if (!memrefType.hasStaticShape()) |
| 1214 | return rewriter.notifyMatchFailure(arg&: copyOp, msg: "unsupported dynamic shape" ); |
| 1215 | |
| 1216 | for (MemRefType type : |
| 1217 | {memrefType, cast<MemRefType>(Val: copyOp.getTarget().getType())}) { |
| 1218 | auto memorySpaceAttr = |
| 1219 | dyn_cast_if_present<spirv::StorageClassAttr>(Val: type.getMemorySpace()); |
| 1220 | if (memorySpaceAttr && |
| 1221 | memorySpaceAttr.getValue() == spirv::StorageClass::Image) |
| 1222 | return rewriter.notifyMatchFailure( |
| 1223 | arg&: copyOp, msg: "cannot lower memref.copy in image storage class" ); |
| 1224 | } |
| 1225 | |
| 1226 | // The converted operands are SPIR-V pointers to the source and target |
| 1227 | // storage. spirv.CopyMemory copies the whole pointed-to object, so it only |
| 1228 | // applies when both pointers point to the same fixed-size element type. |
| 1229 | Value source = adaptor.getSource(); |
| 1230 | Value target = adaptor.getTarget(); |
| 1231 | auto sourcePtrType = dyn_cast<spirv::PointerType>(Val: source.getType()); |
| 1232 | auto targetPtrType = dyn_cast<spirv::PointerType>(Val: target.getType()); |
| 1233 | if (!sourcePtrType || !targetPtrType) |
| 1234 | return rewriter.notifyMatchFailure(arg&: copyOp, msg: "failed to convert memref type" ); |
| 1235 | |
| 1236 | if (sourcePtrType.getPointeeType() != targetPtrType.getPointeeType()) |
| 1237 | return rewriter.notifyMatchFailure( |
| 1238 | arg&: copyOp, msg: "source and target pointee types do not match" ); |
| 1239 | |
| 1240 | rewriter.replaceOpWithNewOp<spirv::CopyMemoryOp>( |
| 1241 | op: copyOp, args&: target, args&: source, /*memory_access=*/args: spirv::MemoryAccessAttr{}, |
| 1242 | /*alignment=*/args: IntegerAttr{}, /*source_memory_access=*/ |
| 1243 | args: spirv::MemoryAccessAttr{}, /*source_alignment=*/args: IntegerAttr{}); |
| 1244 | return success(); |
| 1245 | } |
| 1246 | |
| 1247 | LogicalResult ReinterpretCastPattern::matchAndRewrite( |
| 1248 | memref::ReinterpretCastOp op, OpAdaptor adaptor, |
| 1249 | ConversionPatternRewriter &rewriter) const { |
| 1250 | Value src = adaptor.getSource(); |
| 1251 | auto srcType = dyn_cast<spirv::PointerType>(Val: src.getType()); |
| 1252 | |
| 1253 | if (!srcType) |
| 1254 | return rewriter.notifyMatchFailure(op, reasonCallback: [&](Diagnostic &diag) { |
| 1255 | diag << "invalid src type " << src.getType(); |
| 1256 | }); |
| 1257 | |
| 1258 | const TypeConverter *converter = getTypeConverter(); |
| 1259 | |
| 1260 | auto dstType = converter->convertType<spirv::PointerType>(t: op.getType()); |
| 1261 | if (dstType != srcType) |
| 1262 | return rewriter.notifyMatchFailure(op, reasonCallback: [&](Diagnostic &diag) { |
| 1263 | diag << "invalid dst type " << op.getType(); |
| 1264 | }); |
| 1265 | |
| 1266 | OpFoldResult offset = |
| 1267 | getMixedValues(staticValues: adaptor.getStaticOffsets(), dynamicValues: adaptor.getOffsets(), b&: rewriter) |
| 1268 | .front(); |
| 1269 | if (isZeroInteger(v: offset)) { |
| 1270 | rewriter.replaceOp(op, newValues: src); |
| 1271 | return success(); |
| 1272 | } |
| 1273 | |
| 1274 | Type intType = converter->convertType(t: rewriter.getIndexType()); |
| 1275 | if (!intType) |
| 1276 | return rewriter.notifyMatchFailure(arg&: op, msg: "failed to convert index type" ); |
| 1277 | |
| 1278 | Location loc = op.getLoc(); |
| 1279 | auto offsetValue = [&]() -> Value { |
| 1280 | if (auto val = dyn_cast<Value>(Val&: offset)) |
| 1281 | return val; |
| 1282 | |
| 1283 | int64_t attrVal = cast<IntegerAttr>(Val: cast<Attribute>(Val&: offset)).getInt(); |
| 1284 | Attribute attr = rewriter.getIntegerAttr(type: intType, value: attrVal); |
| 1285 | return rewriter.createOrFold<spirv::ConstantOp>(location: loc, args&: intType, args&: attr); |
| 1286 | }(); |
| 1287 | |
| 1288 | rewriter.replaceOpWithNewOp<spirv::InBoundsPtrAccessChainOp>( |
| 1289 | op, args&: src, args&: offsetValue, args: ValueRange()); |
| 1290 | return success(); |
| 1291 | } |
| 1292 | |
| 1293 | //===----------------------------------------------------------------------===// |
| 1294 | // ExtractAlignedPointerAsIndexOp |
| 1295 | //===----------------------------------------------------------------------===// |
| 1296 | |
| 1297 | LogicalResult ExtractAlignedPointerAsIndexOpPattern::matchAndRewrite( |
| 1298 | memref::ExtractAlignedPointerAsIndexOp , OpAdaptor adaptor, |
| 1299 | ConversionPatternRewriter &rewriter) const { |
| 1300 | auto &typeConverter = *getTypeConverter<SPIRVTypeConverter>(); |
| 1301 | Type indexType = typeConverter.getIndexType(); |
| 1302 | rewriter.replaceOpWithNewOp<spirv::ConvertPtrToUOp>(op: extractOp, args&: indexType, |
| 1303 | args: adaptor.getSource()); |
| 1304 | return success(); |
| 1305 | } |
| 1306 | |
| 1307 | //===----------------------------------------------------------------------===// |
| 1308 | // Pattern population |
| 1309 | //===----------------------------------------------------------------------===// |
| 1310 | |
| 1311 | namespace mlir { |
| 1312 | void populateMemRefToSPIRVPatterns(const SPIRVTypeConverter &typeConverter, |
| 1313 | RewritePatternSet &patterns) { |
| 1314 | patterns.add<AllocaOpPattern, AllocOpPattern, AtomicRMWOpPattern, |
| 1315 | CopyOpPattern, DeallocOpPattern, IntLoadOpPattern, |
| 1316 | ImageLoadOpPattern, IntStoreOpPattern, LoadOpPattern, |
| 1317 | MemorySpaceCastOpPattern, StoreOpPattern, ReinterpretCastPattern, |
| 1318 | CastPattern, ExtractAlignedPointerAsIndexOpPattern>( |
| 1319 | arg: typeConverter, args: patterns.getContext()); |
| 1320 | } |
| 1321 | } // namespace mlir |
| 1322 | |