| 1 | //===-- XeGPUToXeVM.cpp - XeGPU to XeVM dialect conversion ------*- C++ -*-===// |
| 2 | // |
| 3 | // This file is licensed under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/XeGPUToXeVM/XeGPUToXeVM.h" |
| 10 | #include "mlir/Dialect/LLVMIR/LLVMTypes.h" |
| 11 | #include "mlir/Dialect/LLVMIR/XeVMDialect.h" |
| 12 | |
| 13 | #include "mlir/Conversion/LLVMCommon/Pattern.h" |
| 14 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 15 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 16 | #include "mlir/Dialect/Index/IR/IndexDialect.h" |
| 17 | #include "mlir/Dialect/Index/IR/IndexOps.h" |
| 18 | #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h" |
| 19 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 20 | #include "mlir/Dialect/LLVMIR/LLVMConversionOps.h" |
| 21 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 22 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 23 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 24 | #include "mlir/Dialect/SCF/Transforms/Patterns.h" |
| 25 | #include "mlir/Dialect/Utils/IndexingUtils.h" |
| 26 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 27 | #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" |
| 28 | #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h" |
| 29 | #include "mlir/Dialect/XeGPU/IR/XeGPU.h" |
| 30 | #include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h" |
| 31 | #include "mlir/Dialect/XeGPU/uArch/uArchCommon.h" |
| 32 | #include "mlir/IR/IRMapping.h" |
| 33 | #include "mlir/Pass/Pass.h" |
| 34 | #include "mlir/Support/LLVM.h" |
| 35 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 36 | #include "llvm/ADT/STLExtras.h" |
| 37 | #include "llvm/ADT/SmallPtrSet.h" |
| 38 | #include "llvm/Support/FormatVariadic.h" |
| 39 | |
| 40 | #include "mlir/IR/BuiltinTypes.h" |
| 41 | #include "mlir/IR/Types.h" |
| 42 | |
| 43 | #include "llvm/ADT/TypeSwitch.h" |
| 44 | |
| 45 | #include <string> |
| 46 | |
| 47 | namespace mlir { |
| 48 | #define GEN_PASS_DEF_CONVERTXEGPUTOXEVMPASS |
| 49 | #include "mlir/Conversion/Passes.h.inc" |
| 50 | } // namespace mlir |
| 51 | |
| 52 | using namespace mlir; |
| 53 | |
| 54 | namespace { |
| 55 | // TODO: Below are uArch dependent values, should move away from hardcoding |
| 56 | static constexpr int32_t systolicDepth{8}; |
| 57 | static constexpr int32_t executionSize{16}; |
| 58 | |
| 59 | // Offsets to individual fields of the 8xi32 layout nd tensor descriptor. |
| 60 | enum class NdTdescOffset : uint32_t { |
| 61 | BasePtr = 0, // Base pointer (i64) |
| 62 | BaseShapeW = 2, // Base shape width (i32) |
| 63 | BaseShapeH = 3, // Base shape height (i32) |
| 64 | BasePitch = 4, // Base pitch/stride of dim rank-2 (i32) |
| 65 | LeadingStride0 = 5, // Row strides of the leading (batch) dims of a >2D |
| 66 | LeadingStride1 = 6, // descriptor (i32); added into offset_h by the load/store |
| 67 | LeadingStride2 = 7, // lowering. Left at 0 for 2D descriptors. |
| 68 | }; |
| 69 | |
| 70 | // Spare payload slots above, and the resulting max lowerable descriptor rank. |
| 71 | static constexpr int64_t maxNdTdescLeadingDims{3}; |
| 72 | static constexpr int64_t maxNdTdescRank{2 + maxNdTdescLeadingDims}; |
| 73 | |
| 74 | static int32_t getNumericXeVMAddrSpace(xegpu::MemorySpace xeGpuMemspace) { |
| 75 | switch (xeGpuMemspace) { |
| 76 | case xegpu::MemorySpace::Global: |
| 77 | return static_cast<int>(xevm::AddrSpace::GLOBAL); |
| 78 | case xegpu::MemorySpace::SLM: |
| 79 | return static_cast<int>(xevm::AddrSpace::SHARED); |
| 80 | } |
| 81 | llvm_unreachable("Unknown XeGPU memory space" ); |
| 82 | } |
| 83 | |
| 84 | /// Translates a memref memory space attribute into XeVM's numeric address |
| 85 | /// space, which follows the OpenCL/SPIR-V convention (0 = private, 1 = |
| 86 | /// global, 2 = constant, 3 = shared/local, 4 = generic). A null attribute, |
| 87 | /// meaning the memory space was left unspecified, maps to the default space |
| 88 | /// 0. Returns failure if `memSpace` is a representation this pass does not |
| 89 | /// know how to translate (e.g. a SPIR-V storage class or an arbitrary string |
| 90 | /// attribute), rather than assuming it is an `IntegerAttr` and asserting. |
| 91 | static FailureOr<unsigned> getNumericMemorySpace(Attribute memSpace) { |
| 92 | if (!memSpace) |
| 93 | return 0u; |
| 94 | if (auto intAttr = llvm::dyn_cast<IntegerAttr>(Val&: memSpace)) |
| 95 | return static_cast<unsigned>(intAttr.getInt()); |
| 96 | if (auto xevmSpace = llvm::dyn_cast<xevm::AddrSpaceAttr>(Val&: memSpace)) |
| 97 | return static_cast<unsigned>(xevmSpace.getValue()); |
| 98 | if (auto gpuSpace = llvm::dyn_cast<gpu::AddressSpaceAttr>(Val&: memSpace)) { |
| 99 | switch (gpuSpace.getValue()) { |
| 100 | case gpu::AddressSpace::Global: |
| 101 | return static_cast<unsigned>(xevm::AddrSpace::GLOBAL); |
| 102 | case gpu::AddressSpace::Workgroup: |
| 103 | return static_cast<unsigned>(xevm::AddrSpace::SHARED); |
| 104 | case gpu::AddressSpace::Private: |
| 105 | return static_cast<unsigned>(xevm::AddrSpace::PRIVATE); |
| 106 | case gpu::AddressSpace::Constant: |
| 107 | return static_cast<unsigned>(xevm::AddrSpace::CONSTANT); |
| 108 | } |
| 109 | llvm_unreachable("Unknown GPU address space" ); |
| 110 | } |
| 111 | return failure(); |
| 112 | } |
| 113 | |
| 114 | /// Checks if the given MemRefType refers to shared memory. |
| 115 | static bool isSharedMemRef(const MemRefType &memrefTy) { |
| 116 | FailureOr<unsigned> addrSpace = |
| 117 | getNumericMemorySpace(memSpace: memrefTy.getMemorySpace()); |
| 118 | return succeeded(Result: addrSpace) && |
| 119 | *addrSpace == static_cast<unsigned>(xevm::AddrSpace::SHARED); |
| 120 | } |
| 121 | |
| 122 | // Get same bitwidth flat vector type of new element type. |
| 123 | static VectorType encodeVectorTypeTo(VectorType currentVecType, |
| 124 | Type toElemType) { |
| 125 | auto elemType = currentVecType.getElementType(); |
| 126 | auto currentBitWidth = elemType.getIntOrFloatBitWidth(); |
| 127 | auto newBitWidth = toElemType.getIntOrFloatBitWidth(); |
| 128 | const int size = |
| 129 | currentVecType.getNumElements() * currentBitWidth / newBitWidth; |
| 130 | return VectorType::get(shape: size, elementType: toElemType); |
| 131 | } |
| 132 | |
| 133 | static xevm::LoadCacheControl |
| 134 | translateLoadXeGPUCacheHint(std::optional<xegpu::CachePolicy> L1hint, |
| 135 | std::optional<xegpu::CachePolicy> L3hint) { |
| 136 | // If no hints are provided, use the default cache control. |
| 137 | if (!L1hint && !L3hint) |
| 138 | return xevm::LoadCacheControl::USE_DEFAULT; |
| 139 | // If only one of the hints is provided, use the default for the other level. |
| 140 | auto L1hintVal = L1hint.value_or(u: xegpu::CachePolicy::CACHED); |
| 141 | auto L3hintVal = L3hint.value_or(u: xegpu::CachePolicy::CACHED); |
| 142 | switch (L1hintVal) { |
| 143 | case xegpu::CachePolicy::CACHED: |
| 144 | if (L3hintVal == xegpu::CachePolicy::CACHED) |
| 145 | return xevm::LoadCacheControl::L1C_L2UC_L3C; |
| 146 | else if (L3hintVal == xegpu::CachePolicy::UNCACHED) |
| 147 | return xevm::LoadCacheControl::L1C_L2UC_L3UC; |
| 148 | else |
| 149 | llvm_unreachable("Unsupported cache control." ); |
| 150 | case xegpu::CachePolicy::UNCACHED: |
| 151 | if (L3hintVal == xegpu::CachePolicy::CACHED) |
| 152 | return xevm::LoadCacheControl::L1UC_L2UC_L3C; |
| 153 | else if (L3hintVal == xegpu::CachePolicy::UNCACHED) |
| 154 | return xevm::LoadCacheControl::L1UC_L2UC_L3UC; |
| 155 | else |
| 156 | llvm_unreachable("Unsupported cache control." ); |
| 157 | case xegpu::CachePolicy::STREAMING: |
| 158 | if (L3hintVal == xegpu::CachePolicy::CACHED) |
| 159 | return xevm::LoadCacheControl::L1S_L2UC_L3C; |
| 160 | else if (L3hintVal == xegpu::CachePolicy::UNCACHED) |
| 161 | return xevm::LoadCacheControl::L1S_L2UC_L3UC; |
| 162 | else |
| 163 | llvm_unreachable("Unsupported cache control." ); |
| 164 | case xegpu::CachePolicy::READ_INVALIDATE: |
| 165 | return xevm::LoadCacheControl::INVALIDATE_READ; |
| 166 | default: |
| 167 | llvm_unreachable("Unsupported cache control." ); |
| 168 | } |
| 169 | } |
| 170 | |
| 171 | static xevm::StoreCacheControl |
| 172 | translateStoreXeGPUCacheHint(std::optional<xegpu::CachePolicy> L1hint, |
| 173 | std::optional<xegpu::CachePolicy> L3hint) { |
| 174 | // If no hints are provided, use the default cache control. |
| 175 | if (!L1hint && !L3hint) |
| 176 | return xevm::StoreCacheControl::USE_DEFAULT; |
| 177 | // If only one of the hints is provided, use the default for the other level. |
| 178 | auto L1hintVal = L1hint.value_or(u: xegpu::CachePolicy::UNCACHED); |
| 179 | auto L3hintVal = L3hint.value_or(u: xegpu::CachePolicy::WRITE_BACK); |
| 180 | switch (L1hintVal) { |
| 181 | case xegpu::CachePolicy::UNCACHED: |
| 182 | if (L3hintVal == xegpu::CachePolicy::UNCACHED) |
| 183 | return xevm::StoreCacheControl::L1UC_L2UC_L3UC; |
| 184 | else if (L3hintVal == xegpu::CachePolicy::WRITE_BACK) |
| 185 | return xevm::StoreCacheControl::L1UC_L2UC_L3WB; |
| 186 | else |
| 187 | llvm_unreachable("Unsupported cache control." ); |
| 188 | case xegpu::CachePolicy::STREAMING: |
| 189 | if (L3hintVal == xegpu::CachePolicy::UNCACHED) |
| 190 | return xevm::StoreCacheControl::L1S_L2UC_L3UC; |
| 191 | else if (L3hintVal == xegpu::CachePolicy::WRITE_BACK) |
| 192 | return xevm::StoreCacheControl::L1S_L2UC_L3WB; |
| 193 | else |
| 194 | llvm_unreachable("Unsupported cache control." ); |
| 195 | case xegpu::CachePolicy::WRITE_BACK: |
| 196 | if (L3hintVal == xegpu::CachePolicy::UNCACHED) |
| 197 | return xevm::StoreCacheControl::L1WB_L2UC_L3UC; |
| 198 | else if (L3hintVal == xegpu::CachePolicy::WRITE_BACK) |
| 199 | return xevm::StoreCacheControl::L1WB_L2UC_L3WB; |
| 200 | else |
| 201 | llvm_unreachable("Unsupported cache control." ); |
| 202 | case xegpu::CachePolicy::WRITE_THROUGH: |
| 203 | if (L3hintVal == xegpu::CachePolicy::UNCACHED) |
| 204 | return xevm::StoreCacheControl::L1WT_L2UC_L3UC; |
| 205 | else if (L3hintVal == xegpu::CachePolicy::WRITE_BACK) |
| 206 | return xevm::StoreCacheControl::L1WT_L2UC_L3WB; |
| 207 | else |
| 208 | llvm_unreachable("Unsupported cache control." ); |
| 209 | default: |
| 210 | llvm_unreachable("Unsupported cache control." ); |
| 211 | } |
| 212 | } |
| 213 | |
| 214 | // |
| 215 | // Note: |
| 216 | // Block operations for tile of sub byte element types are handled by |
| 217 | // emulating with larger element types. |
| 218 | // Tensor descriptor are keep intact and only ops consuming them are |
| 219 | // emulated |
| 220 | // |
| 221 | |
| 222 | // |
| 223 | // High-D (>2D) nd descriptors are lowered by viewing the source as a single |
| 224 | // flattened 2D plane, so the 2D-block surface covers every leading (batch) |
| 225 | // plane at once and a batch position becomes a row offset into it. The surface |
| 226 | // height is the row extent the leading dims reach: |
| 227 | // base_height = size[R-2] + sum_d (size[d] - 1) * (stride[d] / stride[R-2]) |
| 228 | // |
| 229 | // Leaving `base_ptr` at the true base means an out-of-range batch index lands |
| 230 | // past `base_height`, where the HW boundary check handles it, instead of aiming |
| 231 | // the surface at unmapped memory. Encoding the leading strides as row counts |
| 232 | // (`stride[d] / stride[R-2]`) also makes them dimensionless, so they survive |
| 233 | // element-type repacking (e.g. the f16 -> i32 transpose repack) without a unit |
| 234 | // conversion. |
| 235 | // |
| 236 | // Limitations of the flattened-plane view: |
| 237 | // 1. Each leading stride must be a whole number of rows. A source with gaps |
| 238 | // between planes (`stride[d] % stride[R-2] != 0`) is not lowered. This can |
| 239 | // only be checked when the strides are static; for dynamic strides the |
| 240 | // divisibility is assumed. |
| 241 | // 2. `base_height` grows with the leading extent, so a source with a large |
| 242 | // batch x head x sequence extent can exceed the HW 2D-block surface height. |
| 243 | // 3. Plane boundaries are invisible to the boundary check, which only knows |
| 244 | // the surface: a tile whose rows run past `size[R-2]` spills into the |
| 245 | // following rows of the surface -- the next plane, or the padding between |
| 246 | // planes -- rather than being clipped there as it would be on a per-plane |
| 247 | // surface. A load reads those rows instead of returning zeros, and a store |
| 248 | // overwrites them, so it can corrupt the next plane. This only matters |
| 249 | // when `size[R-2]` is not a multiple of the tile height. |
| 250 | // |
| 251 | |
| 252 | class CreateNdDescToXeVMPattern |
| 253 | : public OpConversionPattern<xegpu::CreateNdDescOp> { |
| 254 | using Base::Base; |
| 255 | |
| 256 | // Checks the surface width and pitch against the target's 2D block memory |
| 257 | // restrictions. Only these two are enforced: both are backed by the source's |
| 258 | // static shape and strides, whereas the base address alignment is a property |
| 259 | // of the allocation that is not visible here. The width and the pitch are |
| 260 | // checked independently, each only when it is static; a value known only at |
| 261 | // runtime is left alone, as guarding it inside the kernel would cost far more |
| 262 | // than it is worth. |
| 263 | // |
| 264 | // A violation is a dead end rather than a pattern that does not apply: the op |
| 265 | // has no other legalization path, so it is reported with an error that names |
| 266 | // the limit it misses. |
| 267 | LogicalResult checkMemoryRestrictions(xegpu::CreateNdDescOp op, |
| 268 | int64_t rank) const { |
| 269 | const auto *uArch = |
| 270 | xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(u: "" )); |
| 271 | // Without a known target chip there is no uArch to check against, so the |
| 272 | // descriptor must be accepted as-is. |
| 273 | if (!uArch) |
| 274 | return success(); |
| 275 | const xegpu::uArch::BlockIOMemoryRestrictions &limits = |
| 276 | uArch->getBlockIOMemoryRestrictions(); |
| 277 | |
| 278 | // Recover the width and the pitch as a number of elements. A memref source |
| 279 | // carries both in its type, and either one may be dynamic independently of |
| 280 | // the other, so they are queried separately instead of being dropped |
| 281 | // together when the source is not fully static. `getMixedSizes` / |
| 282 | // `getMixedStrides` are only consulted for an integer source, where the op |
| 283 | // itself holds the extents. |
| 284 | std::optional<int64_t> widthElems; |
| 285 | std::optional<int64_t> pitchElems; |
| 286 | if (auto memrefTy = dyn_cast<MemRefType>(Val: op.getSource().getType())) { |
| 287 | if (!ShapedType::isDynamic(dValue: memrefTy.getDimSize(idx: rank - 1))) |
| 288 | widthElems = memrefTy.getDimSize(idx: rank - 1); |
| 289 | // The source is known to be strided; the caller checked that already. |
| 290 | SmallVector<int64_t> staticStrides; |
| 291 | int64_t staticOffset; |
| 292 | if (succeeded( |
| 293 | Result: memrefTy.getStridesAndOffset(strides&: staticStrides, offset&: staticOffset)) && |
| 294 | !ShapedType::isDynamic(dValue: staticStrides[rank - 2])) |
| 295 | pitchElems = staticStrides[rank - 2]; |
| 296 | } else { |
| 297 | widthElems = getConstantIntValue(ofr: op.getMixedSizes()[rank - 1]); |
| 298 | pitchElems = getConstantIntValue(ofr: op.getMixedStrides()[rank - 2]); |
| 299 | } |
| 300 | |
| 301 | unsigned elemBitSize = |
| 302 | op.getType().getElementType().getIntOrFloatBitWidth(); |
| 303 | // The restrictions are expressed in bytes, but a sub-byte element type can |
| 304 | // leave the width or the pitch on a fraction of a byte, so the comparisons |
| 305 | // are done in bits. Such a size meets no byte alignment at all and is |
| 306 | // reported as a violation rather than waved through. |
| 307 | auto describe = [](int64_t bits) -> std::string { |
| 308 | if (bits % 8 == 0) |
| 309 | return llvm::formatv(Fmt: "{0} bytes" , Vals: bits / 8).str(); |
| 310 | return llvm::formatv(Fmt: "{0} bits" , Vals&: bits).str(); |
| 311 | }; |
| 312 | auto check = [&](std::optional<int64_t> elems, int64_t minBytes, |
| 313 | int64_t alignBytes, |
| 314 | llvm::StringRef what) -> LogicalResult { |
| 315 | if (!elems) |
| 316 | return success(); |
| 317 | int64_t bits = *elems * elemBitSize; |
| 318 | if (bits < minBytes * 8) |
| 319 | return op.emitError(message: llvm::formatv( |
| 320 | Fmt: "2D block surface {0} must be at least {1} bytes on this " |
| 321 | "target, got {2}." , |
| 322 | Vals&: what, Vals&: minBytes, Vals: describe(bits))); |
| 323 | if (alignBytes > 0 && bits % (alignBytes * 8) != 0) |
| 324 | return op.emitError(message: llvm::formatv( |
| 325 | Fmt: "2D block surface {0} must be a multiple of {1} bytes on this " |
| 326 | "target, got {2}." , |
| 327 | Vals&: what, Vals&: alignBytes, Vals: describe(bits))); |
| 328 | return success(); |
| 329 | }; |
| 330 | |
| 331 | if (failed(Result: check(widthElems, limits.minBaseWidthBytes, |
| 332 | limits.baseWidthAlignmentBytes, "width" ))) |
| 333 | return failure(); |
| 334 | // The pitch is the row stride of the innermost 2D tile. |
| 335 | if (failed(Result: check(pitchElems, limits.minBasePitchBytes, |
| 336 | limits.basePitchAlignmentBytes, "pitch" ))) |
| 337 | return failure(); |
| 338 | // Consecutive rows may not overlap, so the pitch has to span the full |
| 339 | // width. |
| 340 | if (widthElems && pitchElems && *pitchElems < *widthElems) |
| 341 | return op.emitError( |
| 342 | message: llvm::formatv(Fmt: "2D block surface pitch must be at least the surface " |
| 343 | "width, got a pitch of {0} for a surface {1} wide." , |
| 344 | Vals: describe(*pitchElems * elemBitSize), |
| 345 | Vals: describe(*widthElems * elemBitSize))); |
| 346 | return success(); |
| 347 | } |
| 348 | |
| 349 | LogicalResult |
| 350 | matchAndRewrite(xegpu::CreateNdDescOp op, |
| 351 | xegpu::CreateNdDescOp::Adaptor adaptor, |
| 352 | ConversionPatternRewriter &rewriter) const override { |
| 353 | auto loc = op.getLoc(); |
| 354 | auto source = op.getSource(); |
| 355 | |
| 356 | // Check all failure conditions before generating any IR, so nothing has to |
| 357 | // be rolled back. |
| 358 | int64_t rank = op.getType().getRank(); |
| 359 | int64_t sourceRank; |
| 360 | auto memrefTy = dyn_cast<MemRefType>(Val: source.getType()); |
| 361 | if (memrefTy) { |
| 362 | if (!memrefTy.isStrided()) |
| 363 | return rewriter.notifyMatchFailure(arg&: op, msg: "Expected strided Memref." ); |
| 364 | sourceRank = memrefTy.getRank(); |
| 365 | } else if (isa<IntegerType>(Val: source.getType())) { |
| 366 | sourceRank = op.getMixedSizes().size(); |
| 367 | } else { |
| 368 | return rewriter.notifyMatchFailure(arg&: op, |
| 369 | msg: "Expected ranked Memref or integer." ); |
| 370 | } |
| 371 | if (sourceRank != rank) |
| 372 | return rewriter.notifyMatchFailure( |
| 373 | arg&: op, msg: "Expected descriptor rank to match source rank; subview the " |
| 374 | "source down to the descriptor rank." ); |
| 375 | if (rank > maxNdTdescRank) |
| 376 | return rewriter.notifyMatchFailure( |
| 377 | arg&: op, msg: "Batched nd descriptor supports at most " + |
| 378 | std::to_string(val: maxNdTdescLeadingDims) + |
| 379 | " leading dims (rank <= " + std::to_string(val: maxNdTdescRank) + |
| 380 | ")." ); |
| 381 | // Limitation 1 above; dynamic strides are assumed to divide evenly. |
| 382 | if (rank > 2) { |
| 383 | SmallVector<std::optional<int64_t>> constStrides(rank, std::nullopt); |
| 384 | if (memrefTy) { |
| 385 | SmallVector<int64_t> staticStrides; |
| 386 | int64_t staticOffset; |
| 387 | if (succeeded( |
| 388 | Result: memrefTy.getStridesAndOffset(strides&: staticStrides, offset&: staticOffset))) |
| 389 | for (int64_t d = 0; d < rank; ++d) |
| 390 | if (!ShapedType::isDynamic(dValue: staticStrides[d])) |
| 391 | constStrides[d] = staticStrides[d]; |
| 392 | } else { |
| 393 | SmallVector<OpFoldResult> mixed = op.getMixedStrides(); |
| 394 | for (int64_t d = 0; d < rank; ++d) |
| 395 | constStrides[d] = getConstantIntValue(ofr: mixed[d]); |
| 396 | } |
| 397 | if (std::optional<int64_t> pitch = constStrides[rank - 2]) { |
| 398 | for (int64_t d = 0; d < rank - 2; ++d) { |
| 399 | std::optional<int64_t> leading = constStrides[d]; |
| 400 | if (leading && (*pitch == 0 || *leading % *pitch != 0)) |
| 401 | return rewriter.notifyMatchFailure( |
| 402 | arg&: op, msg: "Expected each leading (batch) stride to be a multiple of " |
| 403 | "the row stride; the source has gaps between planes." ); |
| 404 | } |
| 405 | } |
| 406 | } |
| 407 | // Enforce the target's 2D block memory restrictions. A rank-1 descriptor is |
| 408 | // just a base address and is not consumed by a 2D block op, so it is |
| 409 | // exempt. |
| 410 | if (rank >= 2 && failed(Result: checkMemoryRestrictions(op, rank))) |
| 411 | return failure(); |
| 412 | |
| 413 | Type payloadElemTy = rewriter.getI32Type(); |
| 414 | Type i64Ty = rewriter.getI64Type(); |
| 415 | |
| 416 | // Access the adaptor only after the failure checks, so a bail-out leaves no |
| 417 | // materialization cast behind. |
| 418 | Value baseAddr = adaptor.getSource(); |
| 419 | if (isa<IntegerType>(Val: source.getType()) && baseAddr.getType() != i64Ty) { |
| 420 | // Pointer type may be i32. Cast to i64 if needed. |
| 421 | baseAddr = arith::ExtUIOp::create(builder&: rewriter, location: loc, out: i64Ty, in: baseAddr); |
| 422 | } |
| 423 | // 1D tensor descriptor is just the base address. |
| 424 | if (rank == 1) { |
| 425 | rewriter.replaceOp(op, newValues: baseAddr); |
| 426 | return success(); |
| 427 | } |
| 428 | |
| 429 | SmallVector<OpFoldResult> mixedSizes; |
| 430 | SmallVector<OpFoldResult> mixedStrides; |
| 431 | if (memrefTy && !xegpu::hasStaticShapeAndStrides(type: memrefTy)) { |
| 432 | auto meta = |
| 433 | memref::ExtractStridedMetadataOp::create(builder&: rewriter, location: loc, source); |
| 434 | mixedSizes = meta.getConstifiedMixedSizes(); |
| 435 | mixedStrides = meta.getConstifiedMixedStrides(); |
| 436 | } else { |
| 437 | mixedSizes = op.getMixedSizes(); |
| 438 | mixedStrides = op.getMixedStrides(); |
| 439 | } |
| 440 | |
| 441 | // Op is lowered to a code sequence that populates payload. |
| 442 | // Payload is a 8xi32 vector. Offset to individual fields are defined in |
| 443 | // NdTdescOffset enum. |
| 444 | VectorType payloadTy = VectorType::get(shape: 8, elementType: payloadElemTy); |
| 445 | // 4xi64 view is used for inserting the base pointer. |
| 446 | VectorType payloadI64Ty = VectorType::get(shape: 4, elementType: i64Ty); |
| 447 | // Initialize payload to zero. |
| 448 | Value payload = arith::ConstantOp::create( |
| 449 | builder&: rewriter, location: loc, |
| 450 | value: DenseElementsAttr::get(type: payloadTy, values: IntegerAttr::get(type: payloadElemTy, value: 0))); |
| 451 | |
| 452 | // Utility for creating offset values from op fold result. |
| 453 | auto createOffset = [&](SmallVector<OpFoldResult> &ofrVec, |
| 454 | unsigned idx) -> Value { |
| 455 | Value val = getValueOrCreateConstantIntOp(b&: rewriter, loc, ofr: ofrVec[idx]); |
| 456 | val = getValueOrCreateCastToIndexLike(b&: rewriter, loc, targetType: payloadElemTy, value: val); |
| 457 | return val; |
| 458 | }; |
| 459 | // The descriptor's innermost 2 dims are the 2D tile (H, W). |
| 460 | Value baseShapeW = createOffset(mixedSizes, rank - 1); |
| 461 | // Pitch is the stride of dim rank-2 (the row stride of the 2D tile). |
| 462 | Value basePitch = createOffset(mixedStrides, rank - 2); |
| 463 | // Leading (batch) strides as a number of rows (`stride[d] / pitch`). |
| 464 | SmallVector<Value> leadingRowStrides; |
| 465 | for (int64_t d = 0; d < rank - 2; ++d) { |
| 466 | std::optional<int64_t> leading = getConstantIntValue(ofr: mixedStrides[d]); |
| 467 | std::optional<int64_t> pitch = |
| 468 | getConstantIntValue(ofr: mixedStrides[rank - 2]); |
| 469 | if (leading && pitch && *pitch != 0) |
| 470 | leadingRowStrides.push_back(Elt: arith::ConstantIntOp::create( |
| 471 | builder&: rewriter, location: loc, type: payloadElemTy, value: *leading / *pitch)); |
| 472 | else |
| 473 | leadingRowStrides.push_back(Elt: arith::DivUIOp::create( |
| 474 | builder&: rewriter, location: loc, lhs: createOffset(mixedStrides, d), rhs: basePitch)); |
| 475 | } |
| 476 | // Height of the flattened plane: every leading (batch) plane is stacked |
| 477 | // into the surface, so the boundary check covers an out-of-range batch. |
| 478 | // base_height = size[R-2] + sum_d (size[d] - 1) * leadingRowStride[d] |
| 479 | // For rank 2 this is just size[0]. |
| 480 | Value baseShapeH = createOffset(mixedSizes, rank - 2); |
| 481 | if (rank > 2) { |
| 482 | Value one = arith::ConstantIntOp::create(builder&: rewriter, location: loc, type: payloadElemTy, value: 1); |
| 483 | for (int64_t d = 0; d < rank - 2; ++d) { |
| 484 | Value planesBelow = rewriter.createOrFold<arith::SubIOp>( |
| 485 | location: loc, args: createOffset(mixedSizes, d), args&: one); |
| 486 | Value rows = rewriter.createOrFold<arith::MulIOp>(location: loc, args&: planesBelow, |
| 487 | args&: leadingRowStrides[d]); |
| 488 | baseShapeH = |
| 489 | rewriter.createOrFold<arith::AddIOp>(location: loc, args&: baseShapeH, args&: rows); |
| 490 | } |
| 491 | } |
| 492 | // Populate payload. |
| 493 | Value payLoadAsI64 = |
| 494 | vector::BitCastOp::create(builder&: rewriter, location: loc, result: payloadI64Ty, source: payload); |
| 495 | payLoadAsI64 = |
| 496 | vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: baseAddr, dest: payLoadAsI64, |
| 497 | position: static_cast<int>(NdTdescOffset::BasePtr)); |
| 498 | payload = vector::BitCastOp::create(builder&: rewriter, location: loc, result: payloadTy, source: payLoadAsI64); |
| 499 | payload = |
| 500 | vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: baseShapeW, dest: payload, |
| 501 | position: static_cast<int>(NdTdescOffset::BaseShapeW)); |
| 502 | payload = |
| 503 | vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: baseShapeH, dest: payload, |
| 504 | position: static_cast<int>(NdTdescOffset::BaseShapeH)); |
| 505 | payload = |
| 506 | vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: basePitch, dest: payload, |
| 507 | position: static_cast<int>(NdTdescOffset::BasePitch)); |
| 508 | // The leading (batch) row strides go into the spare payload slots; the |
| 509 | // load/store/prefetch lowering turns the batch offsets into a row offset |
| 510 | // with them. |
| 511 | for (int64_t d = 0; d < rank - 2; ++d) |
| 512 | payload = vector::InsertOp::create( |
| 513 | builder&: rewriter, location: loc, valueToStore: leadingRowStrides[d], dest: payload, |
| 514 | position: static_cast<int>(NdTdescOffset::LeadingStride0) + d); |
| 515 | rewriter.replaceOp(op, newValues: payload); |
| 516 | return success(); |
| 517 | } |
| 518 | }; |
| 519 | |
| 520 | template < |
| 521 | typename OpType, |
| 522 | typename = std::enable_if_t<llvm::is_one_of< |
| 523 | OpType, xegpu::LoadNdOp, xegpu::StoreNdOp, xegpu::PrefetchNdOp>::value>> |
| 524 | class LoadStorePrefetchNdToXeVMPattern : public OpConversionPattern<OpType> { |
| 525 | using OpConversionPattern<OpType>::OpConversionPattern; |
| 526 | LogicalResult |
| 527 | matchAndRewrite(OpType op, typename OpType::Adaptor adaptor, |
| 528 | ConversionPatternRewriter &rewriter) const override { |
| 529 | auto mixedOffsets = op.getMixedOffsets(); |
| 530 | int64_t opOffsetsSize = mixedOffsets.size(); |
| 531 | auto loc = op.getLoc(); |
| 532 | auto ctxt = rewriter.getContext(); |
| 533 | |
| 534 | auto tdesc = adaptor.getTensorDesc(); |
| 535 | auto tdescTy = op.getTensorDescType(); |
| 536 | auto tileRank = tdescTy.getRank(); |
| 537 | if (opOffsetsSize != tileRank) |
| 538 | return rewriter.notifyMatchFailure( |
| 539 | op, "Expected offset rank to match descriptor rank." ); |
| 540 | if (tileRank > 2 && llvm::any_of(tdescTy.getShape().drop_back(2), |
| 541 | [](int64_t d) { return d != 1; })) |
| 542 | return rewriter.notifyMatchFailure( |
| 543 | op, "Expected leading (batch) descriptor dims to be unit." ); |
| 544 | if (tileRank > maxNdTdescRank) |
| 545 | return rewriter.notifyMatchFailure( |
| 546 | op, "Expected descriptor rank <= " + std::to_string(val: maxNdTdescRank) + |
| 547 | "." ); |
| 548 | auto elemType = tdescTy.getElementType(); |
| 549 | auto elemBitSize = elemType.getIntOrFloatBitWidth(); |
| 550 | bool isSubByte = elemBitSize < 8; |
| 551 | uint64_t wScaleFactor = 1; |
| 552 | |
| 553 | if (!isSubByte && (elemBitSize % 8 != 0)) |
| 554 | return rewriter.notifyMatchFailure( |
| 555 | op, "Expected element type bit width to be multiple of 8." ); |
| 556 | auto tileW = tdescTy.getDimSize(tileRank - 1); |
| 557 | // For sub byte types, only 4bits are currently supported. |
| 558 | if (isSubByte) { |
| 559 | if (elemBitSize != 4) |
| 560 | return rewriter.notifyMatchFailure( |
| 561 | op, "Only sub byte types of 4bits are supported." ); |
| 562 | if (tileRank != 2) |
| 563 | return rewriter.notifyMatchFailure( |
| 564 | op, "Sub byte types are only supported for 2D tensor descriptors." ); |
| 565 | auto subByteFactor = 8 / elemBitSize; |
| 566 | auto tileH = tdescTy.getDimSize(0); |
| 567 | // Handle special case for packed load. |
| 568 | if constexpr (std::is_same_v<OpType, xegpu::LoadNdOp>) { |
| 569 | if (op.getPacked().value_or(false)) { |
| 570 | // packed load is implemented as packed loads of 8bit elements. |
| 571 | if (tileH == systolicDepth * 4 && |
| 572 | tileW == executionSize * subByteFactor) { |
| 573 | // Usage case for loading as Matrix B with pack request. |
| 574 | // source is assumed to pre-packed into 8bit elements |
| 575 | // Emulate with 8bit loads with pack request. |
| 576 | // scaled_tileW = executionSize |
| 577 | elemType = rewriter.getIntegerType(width: 8); |
| 578 | tileW = executionSize; |
| 579 | wScaleFactor = subByteFactor; |
| 580 | } |
| 581 | } |
| 582 | } |
| 583 | // If not handled by packed load case above, handle other cases. |
| 584 | if (wScaleFactor == 1) { |
| 585 | auto sub16BitFactor = subByteFactor * 2; |
| 586 | if (tileW == executionSize * sub16BitFactor) { |
| 587 | // Usage case for loading as Matrix A operand |
| 588 | // Emulate with 16bit loads/stores. |
| 589 | // scaled_tileW = executionSize |
| 590 | elemType = rewriter.getIntegerType(width: 16); |
| 591 | tileW = executionSize; |
| 592 | wScaleFactor = sub16BitFactor; |
| 593 | } else { |
| 594 | return rewriter.notifyMatchFailure( |
| 595 | op, "Unsupported tile shape for sub byte types." ); |
| 596 | } |
| 597 | } |
| 598 | // recompute element bit size for emulation. |
| 599 | elemBitSize = elemType.getIntOrFloatBitWidth(); |
| 600 | } |
| 601 | |
| 602 | // Get address space from tensor descriptor memory space. |
| 603 | auto ptrTypeLLVM = LLVM::LLVMPointerType::get( |
| 604 | context: ctxt, addressSpace: getNumericXeVMAddrSpace(tdescTy.getMemorySpace())); |
| 605 | if (tileRank >= 2) { |
| 606 | // Compute element byte size. |
| 607 | Value elemByteSize = arith::ConstantIntOp::create( |
| 608 | rewriter, loc, rewriter.getI32Type(), elemBitSize / 8); |
| 609 | VectorType payloadI64Ty = VectorType::get(shape: 4, elementType: rewriter.getI64Type()); |
| 610 | Value payLoadAsI64 = |
| 611 | vector::BitCastOp::create(rewriter, loc, payloadI64Ty, tdesc); |
| 612 | Value basePtr = |
| 613 | vector::ExtractOp::create(rewriter, loc, payLoadAsI64, |
| 614 | static_cast<int>(NdTdescOffset::BasePtr)); |
| 615 | Value baseShapeW = vector::ExtractOp::create( |
| 616 | rewriter, loc, tdesc, static_cast<int>(NdTdescOffset::BaseShapeW)); |
| 617 | Value baseShapeH = vector::ExtractOp::create( |
| 618 | rewriter, loc, tdesc, static_cast<int>(NdTdescOffset::BaseShapeH)); |
| 619 | Value basePitch = vector::ExtractOp::create( |
| 620 | rewriter, loc, tdesc, static_cast<int>(NdTdescOffset::BasePitch)); |
| 621 | |
| 622 | Value offsetW = getValueOrCreateConstantIntOp(rewriter, loc, |
| 623 | mixedOffsets[tileRank - 1]); |
| 624 | offsetW = getValueOrCreateCastToIndexLike(rewriter, loc, |
| 625 | rewriter.getI32Type(), offsetW); |
| 626 | Value offsetH = getValueOrCreateConstantIntOp(rewriter, loc, |
| 627 | mixedOffsets[tileRank - 2]); |
| 628 | offsetH = getValueOrCreateCastToIndexLike(rewriter, loc, |
| 629 | rewriter.getI32Type(), offsetH); |
| 630 | // Turn the leading (batch) offsets into a row offset into the flattened |
| 631 | // plane, using the row-unit batch strides encoded at create time: |
| 632 | // offsetH += sum_d offset[d] * leadingRowStride[d] |
| 633 | // The base pointer stays at the true base, so an out-of-range batch index |
| 634 | // is caught by the HW boundary check instead of moving the surface to |
| 635 | // unmapped memory. |
| 636 | for (int64_t d = 0; d < tileRank - 2; ++d) { |
| 637 | Value off = |
| 638 | getValueOrCreateConstantIntOp(rewriter, loc, mixedOffsets[d]); |
| 639 | off = getValueOrCreateCastToIndexLike(rewriter, loc, |
| 640 | rewriter.getI32Type(), off); |
| 641 | Value rowStride = vector::ExtractOp::create( |
| 642 | rewriter, loc, tdesc, |
| 643 | static_cast<int>(NdTdescOffset::LeadingStride0) + d); |
| 644 | Value term = arith::MulIOp::create(rewriter, loc, off, rowStride); |
| 645 | offsetH = arith::AddIOp::create(rewriter, loc, offsetH, term); |
| 646 | } |
| 647 | // Convert base pointer (i64) to LLVM pointer type. |
| 648 | Value basePtrLLVM = |
| 649 | LLVM::IntToPtrOp::create(rewriter, loc, ptrTypeLLVM, basePtr); |
| 650 | // FIXME: width or pitch is not the same as baseShapeW it should be the |
| 651 | // stride of the second to last dimension in row major layout. |
| 652 | // Compute width in bytes. |
| 653 | Value baseShapeWInBytes = |
| 654 | arith::MulIOp::create(rewriter, loc, baseShapeW, elemByteSize); |
| 655 | // Compute pitch in bytes. |
| 656 | Value basePitchBytes = |
| 657 | arith::MulIOp::create(rewriter, loc, basePitch, elemByteSize); |
| 658 | |
| 659 | if (wScaleFactor > 1) { |
| 660 | // Scale offsetW, baseShapeWInBytes for sub byte emulation. |
| 661 | // Note: tileW is already scaled above. |
| 662 | Value wScaleFactorValLog2 = arith::ConstantIntOp::create( |
| 663 | rewriter, loc, rewriter.getI32Type(), llvm::Log2_64(Value: wScaleFactor)); |
| 664 | baseShapeWInBytes = arith::ShRSIOp::create( |
| 665 | rewriter, loc, baseShapeWInBytes, wScaleFactorValLog2); |
| 666 | basePitchBytes = arith::ShRSIOp::create(rewriter, loc, basePitchBytes, |
| 667 | wScaleFactorValLog2); |
| 668 | offsetW = |
| 669 | arith::ShRSIOp::create(rewriter, loc, offsetW, wScaleFactorValLog2); |
| 670 | } |
| 671 | // Get tile height from the tensor descriptor type (second-to-last dim). |
| 672 | auto tileH = tdescTy.getDimSize(tileRank - 2); |
| 673 | // Get vblocks from the tensor descriptor type. |
| 674 | int32_t vblocks = tdescTy.getArrayLength(); |
| 675 | if constexpr (std::is_same_v<OpType, xegpu::StoreNdOp>) { |
| 676 | Value src = adaptor.getValue(); |
| 677 | // If store value is a scalar, get value from op instead of adaptor. |
| 678 | // Adaptor might have optimized away single element vector |
| 679 | if (src.getType().isIntOrFloat()) { |
| 680 | src = op.getValue(); |
| 681 | } |
| 682 | VectorType srcVecTy = dyn_cast<VectorType>(Val: src.getType()); |
| 683 | if (!srcVecTy) |
| 684 | return rewriter.notifyMatchFailure( |
| 685 | op, "Expected store value to be a vector type." ); |
| 686 | // Get flat vector type of integer type with matching element bit size. |
| 687 | VectorType newSrcVecTy = |
| 688 | encodeVectorTypeTo(srcVecTy, rewriter.getIntegerType(elemBitSize)); |
| 689 | if (srcVecTy != newSrcVecTy) |
| 690 | src = vector::BitCastOp::create(rewriter, loc, newSrcVecTy, src); |
| 691 | auto storeCacheControl = |
| 692 | translateStoreXeGPUCacheHint(op.getL1Hint(), op.getL3Hint()); |
| 693 | xevm::BlockStore2dOp::create( |
| 694 | rewriter, loc, basePtrLLVM, baseShapeWInBytes, baseShapeH, |
| 695 | basePitchBytes, offsetW, offsetH, elemBitSize, tileW, tileH, src, |
| 696 | xevm::StoreCacheControlAttr::get(context: ctxt, value: storeCacheControl)); |
| 697 | rewriter.eraseOp(op); |
| 698 | } else { |
| 699 | auto loadCacheControl = |
| 700 | translateLoadXeGPUCacheHint(op.getL1Hint(), op.getL3Hint()); |
| 701 | if constexpr (std::is_same_v<OpType, xegpu::PrefetchNdOp>) { |
| 702 | xevm::BlockPrefetch2dOp::create( |
| 703 | rewriter, loc, basePtrLLVM, baseShapeWInBytes, baseShapeH, |
| 704 | basePitchBytes, offsetW, offsetH, elemBitSize, tileW, tileH, |
| 705 | vblocks, xevm::LoadCacheControlAttr::get(context: ctxt, value: loadCacheControl)); |
| 706 | rewriter.eraseOp(op); |
| 707 | } else { |
| 708 | VectorType dstVecTy = cast<VectorType>(op.getValue().getType()); |
| 709 | bool vnni = op.getPacked().value_or(false); |
| 710 | auto transposeValue = op.getTranspose(); |
| 711 | bool transpose = |
| 712 | transposeValue.has_value() && transposeValue.value()[0] == 1; |
| 713 | // Handle special case of 32x16 and 8bit element load |
| 714 | // with no vnni, no transpose, no vblocks. |
| 715 | // For this special case, vnni and non vnni yields the same output |
| 716 | // and only the vnni variant is supported by HW. |
| 717 | // Check and set vnni of the special case. |
| 718 | if (elemBitSize == 8 && tileW == 16 && tileH == 32 && !vnni && |
| 719 | !transpose) { |
| 720 | vnni = true; |
| 721 | } |
| 722 | // Handle tranpose request on small element size |
| 723 | // Transpose needs to be requested on 32bit element type. |
| 724 | // offsetW and tileW needs to be adjusted to account for element type |
| 725 | // change. |
| 726 | if (transpose && elemBitSize < 32) { |
| 727 | int32_t scale = 32 / elemBitSize; |
| 728 | Value scaleLog2 = arith::ConstantIntOp::create( |
| 729 | rewriter, loc, rewriter.getI32Type(), llvm::Log2_64(Value: scale)); |
| 730 | offsetW = arith::ShRSIOp::create(rewriter, loc, offsetW, scaleLog2); |
| 731 | tileW = tileW * elemBitSize / 32; |
| 732 | elemBitSize = 32; |
| 733 | } |
| 734 | VectorType loadedTy = encodeVectorTypeTo( |
| 735 | dstVecTy, vnni ? rewriter.getI32Type() |
| 736 | : rewriter.getIntegerType(elemBitSize)); |
| 737 | |
| 738 | Value resultFlatVec = xevm::BlockLoad2dOp::create( |
| 739 | rewriter, loc, loadedTy, basePtrLLVM, baseShapeWInBytes, |
| 740 | baseShapeH, basePitchBytes, offsetW, offsetH, elemBitSize, tileW, |
| 741 | tileH, vblocks, transpose, vnni, |
| 742 | xevm::LoadCacheControlAttr::get(context: ctxt, value: loadCacheControl)); |
| 743 | resultFlatVec = vector::BitCastOp::create( |
| 744 | rewriter, loc, |
| 745 | encodeVectorTypeTo(currentVecType: loadedTy, toElemType: dstVecTy.getElementType()), |
| 746 | resultFlatVec); |
| 747 | rewriter.replaceOp(op, resultFlatVec); |
| 748 | } |
| 749 | } |
| 750 | } else { |
| 751 | // 1D tensor descriptor. |
| 752 | // `tdesc` represents base address as i64 |
| 753 | // Offset in number of elements, need to multiply by element byte size. |
| 754 | // Compute byte offset. |
| 755 | // byteOffset = offset * elementByteSize |
| 756 | Value offset = |
| 757 | getValueOrCreateConstantIntOp(rewriter, loc, mixedOffsets[0]); |
| 758 | offset = getValueOrCreateCastToIndexLike(rewriter, loc, |
| 759 | rewriter.getI64Type(), offset); |
| 760 | // Compute element byte size. |
| 761 | Value elemByteSize = arith::ConstantIntOp::create( |
| 762 | rewriter, loc, rewriter.getI64Type(), elemBitSize / 8); |
| 763 | Value byteOffset = |
| 764 | rewriter.createOrFold<arith::MulIOp>(loc, offset, elemByteSize); |
| 765 | // Final address = basePtr + byteOffset |
| 766 | Value finalAddrI64 = rewriter.createOrFold<arith::AddIOp>( |
| 767 | loc, tdesc, |
| 768 | getValueOrCreateCastToIndexLike(rewriter, loc, rewriter.getI64Type(), |
| 769 | byteOffset)); |
| 770 | // Convert base pointer (i64) to LLVM pointer type. |
| 771 | Value finalPtrLLVM = |
| 772 | LLVM::IntToPtrOp::create(rewriter, loc, ptrTypeLLVM, finalAddrI64); |
| 773 | if constexpr (std::is_same_v<OpType, xegpu::StoreNdOp>) { |
| 774 | Value src = adaptor.getValue(); |
| 775 | // If store value is a scalar, get value from op instead of adaptor. |
| 776 | // Adaptor might have optimized away single element vector |
| 777 | if (src.getType().isIntOrFloat()) { |
| 778 | src = op.getValue(); |
| 779 | } |
| 780 | VectorType srcVecTy = dyn_cast<VectorType>(Val: src.getType()); |
| 781 | if (!srcVecTy) |
| 782 | return rewriter.notifyMatchFailure( |
| 783 | op, "Expected store value to be a vector type." ); |
| 784 | // Get flat vector type of integer type with matching element bit size. |
| 785 | VectorType newSrcVecTy = |
| 786 | encodeVectorTypeTo(srcVecTy, rewriter.getIntegerType(elemBitSize)); |
| 787 | if (srcVecTy != newSrcVecTy) |
| 788 | src = vector::BitCastOp::create(rewriter, loc, newSrcVecTy, src); |
| 789 | auto storeCacheControl = |
| 790 | translateStoreXeGPUCacheHint(op.getL1Hint(), op.getL3Hint()); |
| 791 | rewriter.replaceOpWithNewOp<xevm::BlockStoreOp>( |
| 792 | op, finalPtrLLVM, src, |
| 793 | xevm::StoreCacheControlAttr::get(context: ctxt, value: storeCacheControl)); |
| 794 | } else if constexpr (std::is_same_v<OpType, xegpu::LoadNdOp>) { |
| 795 | auto loadCacheControl = |
| 796 | translateLoadXeGPUCacheHint(op.getL1Hint(), op.getL3Hint()); |
| 797 | VectorType resTy = cast<VectorType>(op.getValue().getType()); |
| 798 | VectorType loadedTy = |
| 799 | encodeVectorTypeTo(resTy, rewriter.getIntegerType(elemBitSize)); |
| 800 | Value load = xevm::BlockLoadOp::create( |
| 801 | rewriter, loc, loadedTy, finalPtrLLVM, |
| 802 | xevm::LoadCacheControlAttr::get(context: ctxt, value: loadCacheControl)); |
| 803 | if (loadedTy != resTy) |
| 804 | load = vector::BitCastOp::create(rewriter, loc, resTy, load); |
| 805 | rewriter.replaceOp(op, load); |
| 806 | } else { |
| 807 | return rewriter.notifyMatchFailure( |
| 808 | op, "Unsupported operation: xegpu.prefetch_nd with tensor " |
| 809 | "descriptor rank == 1" ); |
| 810 | } |
| 811 | } |
| 812 | return success(); |
| 813 | } |
| 814 | }; |
| 815 | |
| 816 | // Add a builder that creates |
| 817 | // offset * elemByteSize + baseAddr |
| 818 | static Value addOffsetToBaseAddr(ConversionPatternRewriter &rewriter, |
| 819 | Location loc, Value baseAddr, Value offset, |
| 820 | int64_t elemByteSize) { |
| 821 | Value byteSize = arith::ConstantIntOp::create( |
| 822 | builder&: rewriter, location: loc, type: baseAddr.getType(), value: elemByteSize); |
| 823 | Value byteOffset = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: offset, rhs: byteSize); |
| 824 | Value newAddr = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: baseAddr, rhs: byteOffset); |
| 825 | return newAddr; |
| 826 | } |
| 827 | |
| 828 | // Returns true when every element of `mask` carries the same bit, so gating a |
| 829 | // whole contiguous block on element 0 is equivalent. Splat constants, an |
| 830 | // all-ones or all-zeros `vector.constant_mask`, broadcasts of a scalar, and a |
| 831 | // `vector.from_elements` of one repeated value qualify. |
| 832 | // |
| 833 | // TODO: this recognizes a fixed list of producers. A general uniformity query |
| 834 | // on the vector dialect would cover more forms and would not need updating |
| 835 | // every time a new producer shows up here. |
| 836 | static bool isUniformMask(Value mask) { |
| 837 | if (!isa<VectorType>(Val: mask.getType())) |
| 838 | return true; |
| 839 | DenseElementsAttr splat; |
| 840 | if (matchPattern(value: mask, pattern: m_Constant(bind_value: &splat)) && splat.isSplat()) |
| 841 | return true; |
| 842 | if (auto constantMask = mask.getDefiningOp<vector::ConstantMaskOp>()) { |
| 843 | // All-ones and all-zeros are uniform. A zero dim size is only legal when |
| 844 | // every dim is zero, so those are the only two uniform cases. |
| 845 | return constantMask.isAllOnesMask() || |
| 846 | llvm::all_of(Range: constantMask.getMaskDimSizes(), |
| 847 | P: [](int64_t size) { return size == 0; }); |
| 848 | } |
| 849 | if (auto broadcast = mask.getDefiningOp<vector::BroadcastOp>()) |
| 850 | return !isa<VectorType>(Val: broadcast.getSource().getType()); |
| 851 | if (auto fromElements = mask.getDefiningOp<vector::FromElementsOp>()) |
| 852 | return llvm::all_equal(Range: fromElements.getElements()); |
| 853 | // Flattening a rank > 1 operand to rank 1 inserts a shape cast. It keeps the |
| 854 | // same elements, so it keeps uniformity. |
| 855 | if (auto shapeCast = mask.getDefiningOp<vector::ShapeCastOp>()) |
| 856 | return isUniformMask(mask: shapeCast.getSource()); |
| 857 | return false; |
| 858 | } |
| 859 | |
| 860 | template <typename OpType, |
| 861 | typename = std::enable_if_t<llvm::is_one_of< |
| 862 | OpType, xegpu::LoadGatherOp, xegpu::StoreScatterOp>::value>> |
| 863 | class LoadStoreToXeVMPattern : public OpConversionPattern<OpType> { |
| 864 | using OpConversionPattern<OpType>::OpConversionPattern; |
| 865 | LogicalResult |
| 866 | matchAndRewrite(OpType op, typename OpType::Adaptor adaptor, |
| 867 | ConversionPatternRewriter &rewriter) const override { |
| 868 | Value offset = adaptor.getOffsets(); |
| 869 | if (!offset) |
| 870 | return rewriter.notifyMatchFailure(op, "Expected offset to be provided." ); |
| 871 | auto loc = op.getLoc(); |
| 872 | auto ctxt = rewriter.getContext(); |
| 873 | Value basePtrI64; |
| 874 | // Load result or Store valye Type can be vector or scalar. |
| 875 | Type valOrResTy; |
| 876 | if constexpr (std::is_same_v<OpType, xegpu::LoadGatherOp>) |
| 877 | valOrResTy = |
| 878 | this->getTypeConverter()->convertType(op.getResult().getType()); |
| 879 | else |
| 880 | valOrResTy = adaptor.getValue().getType(); |
| 881 | VectorType valOrResVecTy = dyn_cast<VectorType>(Val&: valOrResTy); |
| 882 | bool hasScalarVal = !valOrResVecTy; |
| 883 | int64_t elemBitWidth = |
| 884 | hasScalarVal ? valOrResTy.getIntOrFloatBitWidth() |
| 885 | : valOrResVecTy.getElementType().getIntOrFloatBitWidth(); |
| 886 | // Element type must be multiple of 8 bits. |
| 887 | if (elemBitWidth % 8 != 0) |
| 888 | return rewriter.notifyMatchFailure( |
| 889 | op, "Expected element type bit width to be multiple of 8." ); |
| 890 | int64_t elemByteSize = elemBitWidth / 8; |
| 891 | // Default memory space is global. |
| 892 | LLVM::LLVMPointerType ptrTypeLLVM = LLVM::LLVMPointerType::get( |
| 893 | context: ctxt, addressSpace: getNumericXeVMAddrSpace(xeGpuMemspace: xegpu::MemorySpace::Global)); |
| 894 | // Base pointer can come from source (load) or dest (store). |
| 895 | // If they are memrefs, we use their memory space. |
| 896 | if constexpr (std::is_same_v<OpType, xegpu::LoadGatherOp>) { |
| 897 | basePtrI64 = adaptor.getSource(); |
| 898 | if (auto memRefTy = dyn_cast<MemRefType>(op.getSource().getType())) { |
| 899 | FailureOr<unsigned> addrSpace = |
| 900 | getNumericMemorySpace(memRefTy.getMemorySpace()); |
| 901 | if (failed(Result: addrSpace)) |
| 902 | return rewriter.notifyMatchFailure( |
| 903 | op, "Unsupported memref memory space attribute." ); |
| 904 | if (*addrSpace != 0) |
| 905 | ptrTypeLLVM = LLVM::LLVMPointerType::get(context: ctxt, addressSpace: *addrSpace); |
| 906 | } |
| 907 | } else { |
| 908 | basePtrI64 = adaptor.getDest(); |
| 909 | if (auto memRefTy = dyn_cast<MemRefType>(op.getDest().getType())) { |
| 910 | FailureOr<unsigned> addrSpace = |
| 911 | getNumericMemorySpace(memRefTy.getMemorySpace()); |
| 912 | if (failed(Result: addrSpace)) |
| 913 | return rewriter.notifyMatchFailure( |
| 914 | op, "Unsupported memref memory space attribute." ); |
| 915 | if (*addrSpace != 0) |
| 916 | ptrTypeLLVM = LLVM::LLVMPointerType::get(context: ctxt, addressSpace: *addrSpace); |
| 917 | } |
| 918 | } |
| 919 | // Base pointer is passed as i32 or i64 by adaptor, cast to i64 if needed. |
| 920 | if (basePtrI64.getType() != rewriter.getI64Type()) { |
| 921 | basePtrI64 = arith::ExtUIOp::create(rewriter, loc, rewriter.getI64Type(), |
| 922 | basePtrI64); |
| 923 | } |
| 924 | Value mask = adaptor.getMask(); |
| 925 | |
| 926 | // Coalesce a lane's multi-element access into one block access. |
| 927 | // |
| 928 | // Distribution gives every element its own offset and mask bit, so a lane |
| 929 | // that takes D neighbouring elements arrives with `vector<D>` offsets and |
| 930 | // mask. One block access needs one base offset and one mask bit, so take |
| 931 | // both from element 0. A non-uniform mask cannot be reduced to one bit, so |
| 932 | // it stays a vector and fails to match below. |
| 933 | auto origOffsetsTy = dyn_cast<VectorType>(op.getOffsets().getType()); |
| 934 | if (isa<VectorType>(Val: offset.getType()) && origOffsetsTy && valOrResVecTy && |
| 935 | origOffsetsTy.getNumElements() == valOrResVecTy.getNumElements() && |
| 936 | isUniformMask(op.getMask())) { |
| 937 | offset = vector::ExtractOp::create(rewriter, loc, offset, |
| 938 | ArrayRef<int64_t>{0}); |
| 939 | if (isa<VectorType>(Val: mask.getType())) |
| 940 | mask = vector::ExtractOp::create(rewriter, loc, mask, |
| 941 | ArrayRef<int64_t>{0}); |
| 942 | } |
| 943 | |
| 944 | if (dyn_cast<VectorType>(Val: offset.getType())) { |
| 945 | // Offset needs be scalar. Single element vector is converted to scalar |
| 946 | // by type converter. |
| 947 | return rewriter.notifyMatchFailure(op, "Expected offset to be a scalar." ); |
| 948 | } else { |
| 949 | // If offset is provided, we add them to the base pointer. |
| 950 | // Offset is in number of elements, we need to multiply by |
| 951 | // element byte size. |
| 952 | basePtrI64 = |
| 953 | addOffsetToBaseAddr(rewriter, loc, basePtrI64, offset, elemByteSize); |
| 954 | } |
| 955 | // Convert base pointer (i64) to LLVM pointer type. |
| 956 | Value basePtrLLVM = |
| 957 | LLVM::IntToPtrOp::create(rewriter, loc, ptrTypeLLVM, basePtrI64); |
| 958 | |
| 959 | Value maskForLane; |
| 960 | VectorType maskVecTy = dyn_cast<VectorType>(Val: mask.getType()); |
| 961 | if (maskVecTy) { |
| 962 | // Mask needs be scalar. Single element vector is converted to scalar by |
| 963 | // type converter. |
| 964 | return rewriter.notifyMatchFailure(op, "Expected mask to be a scalar." ); |
| 965 | } else |
| 966 | maskForLane = mask; |
| 967 | if constexpr (std::is_same_v<OpType, xegpu::LoadGatherOp>) { |
| 968 | scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, {valOrResTy}, |
| 969 | maskForLane, true, true); |
| 970 | // If mask is true,- then clause - load from memory and yield. |
| 971 | rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front()); |
| 972 | if (!hasScalarVal) |
| 973 | valOrResTy = VectorType::get(shape: {valOrResVecTy.getNumElements()}, |
| 974 | elementType: valOrResVecTy.getElementType()); |
| 975 | Value loaded = |
| 976 | LLVM::LoadOp::create(rewriter, loc, valOrResTy, basePtrLLVM); |
| 977 | // Set cache control attribute on the load operation. |
| 978 | loaded.getDefiningOp()->setDiscardableAttr( |
| 979 | "cache_control" , xevm::LoadCacheControlAttr::get( |
| 980 | context: ctxt, value: translateLoadXeGPUCacheHint( |
| 981 | op.getL1Hint(), op.getL3Hint()))); |
| 982 | scf::YieldOp::create(rewriter, loc, ValueRange{loaded}); |
| 983 | rewriter.setInsertionPointToStart(&ifOp.getElseRegion().front()); |
| 984 | // If mask is false - else clause -yield a vector of zeros. |
| 985 | auto eTy = hasScalarVal ? valOrResTy : valOrResVecTy.getElementType(); |
| 986 | TypedAttr eVal; |
| 987 | if (eTy.isFloat()) |
| 988 | eVal = FloatAttr::get(type: eTy, value: 0.0); |
| 989 | else |
| 990 | eVal = IntegerAttr::get(type: eTy, value: 0); |
| 991 | if (hasScalarVal) |
| 992 | loaded = arith::ConstantOp::create(rewriter, loc, eVal); |
| 993 | else |
| 994 | loaded = arith::ConstantOp::create( |
| 995 | rewriter, loc, DenseElementsAttr::get(type: valOrResVecTy, values: eVal)); |
| 996 | scf::YieldOp::create(rewriter, loc, ValueRange{loaded}); |
| 997 | rewriter.replaceOp(op, ifOp.getResult(i: 0)); |
| 998 | } else { |
| 999 | // If mask is true, perform the store. |
| 1000 | scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, maskForLane, false); |
| 1001 | auto body = ifOp.getBody(); |
| 1002 | rewriter.setInsertionPointToStart(body); |
| 1003 | auto storeOp = |
| 1004 | LLVM::StoreOp::create(rewriter, loc, adaptor.getValue(), basePtrLLVM); |
| 1005 | // Set cache control attribute on the store operation. |
| 1006 | storeOp.getOperation()->setDiscardableAttr( |
| 1007 | "cache_control" , xevm::StoreCacheControlAttr::get( |
| 1008 | context: ctxt, value: translateStoreXeGPUCacheHint( |
| 1009 | op.getL1Hint(), op.getL3Hint()))); |
| 1010 | rewriter.eraseOp(op); |
| 1011 | } |
| 1012 | return success(); |
| 1013 | } |
| 1014 | }; |
| 1015 | |
| 1016 | class CreateMemDescOpPattern final |
| 1017 | : public OpConversionPattern<xegpu::CreateMemDescOp> { |
| 1018 | public: |
| 1019 | using OpConversionPattern<xegpu::CreateMemDescOp>::OpConversionPattern; |
| 1020 | LogicalResult |
| 1021 | matchAndRewrite(xegpu::CreateMemDescOp op, OpAdaptor adaptor, |
| 1022 | ConversionPatternRewriter &rewriter) const override { |
| 1023 | |
| 1024 | rewriter.replaceOp(op, newValues: adaptor.getSource()); |
| 1025 | return success(); |
| 1026 | } |
| 1027 | }; |
| 1028 | |
| 1029 | /// Returns the shape of the largest piece of a matrix op payload that one LLVM |
| 1030 | /// access can move, as an access only moves consecutive elements. |
| 1031 | /// Examples: |
| 1032 | /// mem_desc<16x2xf32> (stride [2, 1], row-major) |
| 1033 | /// payload [1, 2] -> [1, 2] one access of 2 elements |
| 1034 | /// payload [2, 1] -> [1, 1] 2 accesses of 1 element |
| 1035 | /// mem_desc<16x2xf32, stride = [1, 16]> (column-major) |
| 1036 | /// payload [4, 1] -> [4, 1] one access of 4 elements |
| 1037 | /// payload [2, 2] -> [1, 1] 4 accesses of 1 element; each column is |
| 1038 | /// consecutive in memory but not in the |
| 1039 | /// row-major flattened vector |
| 1040 | /// |
| 1041 | /// TODO: On a blocked layout memory is only consecutive inside a block, so a |
| 1042 | /// piece that crosses a block boundary along the stride-1 dim is not |
| 1043 | /// contiguous. Blocks are not taken into account yet. |
| 1044 | static SmallVector<int64_t> |
| 1045 | getContiguousPayloadShape(ArrayRef<int64_t> payloadShape, size_t numOffsets, |
| 1046 | xegpu::MemDescType mdescTy) { |
| 1047 | SmallVector<int64_t> shape(payloadShape); |
| 1048 | int64_t rank = shape.size(); |
| 1049 | if (rank != static_cast<int64_t>(numOffsets)) |
| 1050 | return shape; |
| 1051 | |
| 1052 | int64_t fastestDim = 0; |
| 1053 | for (auto [dim, strideAttr] : |
| 1054 | llvm::enumerate(First: mdescTy.getStrideAttr().getValue())) { |
| 1055 | if (cast<IntegerAttr>(Val: strideAttr).getInt() == 1) { |
| 1056 | fastestDim = dim; |
| 1057 | break; |
| 1058 | } |
| 1059 | } |
| 1060 | |
| 1061 | // A slice that runs along fastestDim is only consecutive in the vector if |
| 1062 | // every dim to its right has size 1. |
| 1063 | bool spansFastestDim = |
| 1064 | llvm::all_of(Range: llvm::seq<int64_t>(Begin: fastestDim + 1, End: rank), |
| 1065 | P: [&](int64_t d) { return payloadShape[d] == 1; }); |
| 1066 | for (int64_t d = 0; d < rank; ++d) |
| 1067 | shape[d] = spansFastestDim && d == fastestDim ? payloadShape[d] : 1; |
| 1068 | |
| 1069 | return shape; |
| 1070 | } |
| 1071 | |
| 1072 | template <typename OpType, |
| 1073 | typename = std::enable_if_t<llvm::is_one_of< |
| 1074 | OpType, xegpu::LoadMatrixOp, xegpu::StoreMatrixOp>::value>> |
| 1075 | class LoadStoreMatrixToXeVMPattern : public OpConversionPattern<OpType> { |
| 1076 | using OpConversionPattern<OpType>::OpConversionPattern; |
| 1077 | |
| 1078 | /// Lowers `op` to one LLVM access per `sliceShape`-sized piece of its |
| 1079 | /// payload, each at its own address. `basePtrLLVM` already addresses the |
| 1080 | /// first piece. A payload that is one contiguous piece, which is the common |
| 1081 | /// case, gives a single access of the whole payload. |
| 1082 | LogicalResult emitContiguousAccesses( |
| 1083 | OpType op, typename OpType::Adaptor adaptor, |
| 1084 | ConversionPatternRewriter &rewriter, xegpu::MemDescType mdescTy, |
| 1085 | ArrayRef<OpFoldResult> offsets, ArrayRef<int64_t> payloadShape, |
| 1086 | ArrayRef<int64_t> sliceShape, Value baseAddr32, Value basePtrLLVM, |
| 1087 | LLVM::LLVMPointerType ptrTypeLLVM, int64_t elemByteSize) const { |
| 1088 | Location loc = op.getLoc(); |
| 1089 | int64_t rank = payloadShape.size(); |
| 1090 | SmallVector<int64_t> sliceGrid(rank); |
| 1091 | for (int64_t d = 0; d < rank; ++d) |
| 1092 | sliceGrid[d] = payloadShape[d] / sliceShape[d]; |
| 1093 | int64_t numSlices = computeProduct(basis: sliceGrid); |
| 1094 | int64_t sliceLen = computeProduct(basis: sliceShape); |
| 1095 | SmallVector<int64_t> payloadStrides = computeStrides(sizes: payloadShape); |
| 1096 | SmallVector<int64_t> gridStrides = computeStrides(sizes: sliceGrid); |
| 1097 | |
| 1098 | // The payload type LLVM load/store accepts: f8E8M0FNU and friends become |
| 1099 | // same-width integers, vector<1> becomes a scalar. Operands arrive |
| 1100 | // converted in the adaptor, results do not. |
| 1101 | Type payloadTy; |
| 1102 | if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp>) |
| 1103 | payloadTy = |
| 1104 | this->getTypeConverter()->convertType(op.getResult().getType()); |
| 1105 | else |
| 1106 | payloadTy = adaptor.getData().getType(); |
| 1107 | auto payloadVecTy = dyn_cast<VectorType>(Val&: payloadTy); |
| 1108 | |
| 1109 | Type sliceTy = payloadTy; |
| 1110 | if (numSlices != 1) { |
| 1111 | if (!payloadVecTy) |
| 1112 | return rewriter.notifyMatchFailure( |
| 1113 | op, "Expected a vector payload to slice." ); |
| 1114 | sliceTy = sliceLen == 1 ? payloadVecTy.getElementType() |
| 1115 | : VectorType::get(shape: {sliceLen}, |
| 1116 | elementType: payloadVecTy.getElementType()); |
| 1117 | } |
| 1118 | |
| 1119 | // Addresses the slice `delta` elements into the payload. |
| 1120 | auto slicePtr = [&](ArrayRef<int64_t> delta) -> Value { |
| 1121 | if (llvm::all_of(delta, [](int64_t d) { return d == 0; })) |
| 1122 | return basePtrLLVM; |
| 1123 | SmallVector<OpFoldResult> sliceOffsets; |
| 1124 | for (auto [base, d] : llvm::zip_equal(t&: offsets, u&: delta)) { |
| 1125 | if (d == 0) |
| 1126 | sliceOffsets.push_back(Elt: base); |
| 1127 | else if (auto attr = dyn_cast<Attribute>(Val: base)) |
| 1128 | sliceOffsets.push_back( |
| 1129 | Elt: rewriter.getIndexAttr(value: cast<IntegerAttr>(Val&: attr).getInt() + d)); |
| 1130 | else |
| 1131 | sliceOffsets.push_back( |
| 1132 | Elt: OpFoldResult(arith::AddIOp::create( |
| 1133 | builder&: rewriter, location: loc, lhs: cast<Value>(Val: base), |
| 1134 | rhs: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: d)) |
| 1135 | .getResult())); |
| 1136 | } |
| 1137 | Value sliceOffset = mdescTy.getLinearOffsets(builder&: rewriter, loc, offsets: sliceOffsets); |
| 1138 | sliceOffset = arith::IndexCastUIOp::create( |
| 1139 | builder&: rewriter, location: loc, out: rewriter.getI32Type(), in: sliceOffset); |
| 1140 | Value slicePtrI32 = addOffsetToBaseAddr(rewriter, loc, baseAddr: baseAddr32, |
| 1141 | offset: sliceOffset, elemByteSize); |
| 1142 | return LLVM::IntToPtrOp::create(builder&: rewriter, location: loc, resultType: ptrTypeLLVM, operands: slicePtrI32); |
| 1143 | }; |
| 1144 | |
| 1145 | Value result; |
| 1146 | if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp>) |
| 1147 | if (numSlices != 1) |
| 1148 | result = arith::ConstantOp::create(builder&: rewriter, location: loc, result: payloadVecTy, |
| 1149 | value: rewriter.getZeroAttr(type: payloadVecTy)); |
| 1150 | |
| 1151 | for (int64_t slice = 0; slice < numSlices; ++slice) { |
| 1152 | SmallVector<int64_t> gridCoords = delinearize(linearIndex: slice, strides: gridStrides); |
| 1153 | SmallVector<int64_t> delta(rank); |
| 1154 | for (int64_t d = 0; d < rank; ++d) |
| 1155 | delta[d] = gridCoords[d] * sliceShape[d]; |
| 1156 | int64_t payloadOffset = linearize(offsets: delta, basis: payloadStrides); |
| 1157 | Value ptr = slicePtr(delta); |
| 1158 | |
| 1159 | if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp>) { |
| 1160 | Value loaded = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: sliceTy, addr: ptr); |
| 1161 | if (numSlices == 1) |
| 1162 | result = loaded; |
| 1163 | else if (sliceLen == 1) |
| 1164 | result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: loaded, dest: result, |
| 1165 | position: ArrayRef<int64_t>{payloadOffset}); |
| 1166 | else |
| 1167 | result = vector::InsertStridedSliceOp::create( |
| 1168 | builder&: rewriter, location: loc, valueToStore: loaded, dest: result, offsets: ArrayRef<int64_t>{payloadOffset}, |
| 1169 | strides: ArrayRef<int64_t>{1}); |
| 1170 | } else { |
| 1171 | Value data = adaptor.getData(); |
| 1172 | if (numSlices != 1) { |
| 1173 | if (sliceLen == 1) |
| 1174 | data = vector::ExtractOp::create(builder&: rewriter, location: loc, source: data, |
| 1175 | position: ArrayRef<int64_t>{payloadOffset}); |
| 1176 | else |
| 1177 | data = vector::ExtractStridedSliceOp::create( |
| 1178 | builder&: rewriter, location: loc, source: data, offsets: ArrayRef<int64_t>{payloadOffset}, |
| 1179 | sizes: ArrayRef<int64_t>{sliceLen}, strides: ArrayRef<int64_t>{1}); |
| 1180 | } |
| 1181 | LLVM::StoreOp::create(builder&: rewriter, location: loc, value: data, addr: ptr); |
| 1182 | } |
| 1183 | } |
| 1184 | |
| 1185 | if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp>) |
| 1186 | rewriter.replaceOp(op, result); |
| 1187 | else |
| 1188 | rewriter.eraseOp(op); |
| 1189 | return success(); |
| 1190 | } |
| 1191 | LogicalResult |
| 1192 | matchAndRewrite(OpType op, typename OpType::Adaptor adaptor, |
| 1193 | ConversionPatternRewriter &rewriter) const override { |
| 1194 | |
| 1195 | SmallVector<OpFoldResult> offsets = op.getMixedOffsets(); |
| 1196 | if (offsets.empty()) |
| 1197 | return rewriter.notifyMatchFailure(op, "Expected offset to be provided." ); |
| 1198 | |
| 1199 | auto loc = op.getLoc(); |
| 1200 | auto ctxt = rewriter.getContext(); |
| 1201 | Value baseAddr32 = adaptor.getMemDesc(); |
| 1202 | Value mdescVal = op.getMemDesc(); |
| 1203 | // Load result or Store value Type can be vector or scalar. |
| 1204 | Type dataTy; |
| 1205 | if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp>) { |
| 1206 | Type resType = op.getResult().getType(); |
| 1207 | // Some transforms may leave unit dimension in the 2D vector, adaptors do |
| 1208 | // not catch it for results. |
| 1209 | if (auto vecType = dyn_cast<VectorType>(Val&: resType)) |
| 1210 | resType = VectorType::get(shape: {vecType.getNumElements()}, |
| 1211 | elementType: vecType.getElementType()); |
| 1212 | dataTy = resType; |
| 1213 | } else |
| 1214 | dataTy = adaptor.getData().getType(); |
| 1215 | VectorType valOrResVecTy = dyn_cast<VectorType>(Val&: dataTy); |
| 1216 | if (!valOrResVecTy) |
| 1217 | valOrResVecTy = VectorType::get(shape: 1, elementType: dataTy); |
| 1218 | |
| 1219 | int64_t elemBitWidth = |
| 1220 | valOrResVecTy.getElementType().getIntOrFloatBitWidth(); |
| 1221 | // Element type must be multiple of 8 bits. |
| 1222 | if (elemBitWidth % 8 != 0) |
| 1223 | return rewriter.notifyMatchFailure( |
| 1224 | op, "Expected element type bit width to be multiple of 8." ); |
| 1225 | int64_t elemByteSize = elemBitWidth / 8; |
| 1226 | |
| 1227 | // Default memory space is SLM. |
| 1228 | LLVM::LLVMPointerType ptrTypeLLVM = LLVM::LLVMPointerType::get( |
| 1229 | context: ctxt, addressSpace: getNumericXeVMAddrSpace(xeGpuMemspace: xegpu::MemorySpace::SLM)); |
| 1230 | |
| 1231 | auto mdescTy = cast<xegpu::MemDescType>(Val: mdescVal.getType()); |
| 1232 | |
| 1233 | Value linearOffset = mdescTy.getLinearOffsets(builder&: rewriter, loc, offsets); |
| 1234 | linearOffset = arith::IndexCastUIOp::create( |
| 1235 | rewriter, loc, rewriter.getI32Type(), linearOffset); |
| 1236 | Value basePtrI32 = addOffsetToBaseAddr(rewriter, loc, baseAddr32, |
| 1237 | linearOffset, elemByteSize); |
| 1238 | |
| 1239 | // convert base pointer (i32) to LLVM pointer type |
| 1240 | Value basePtrLLVM = |
| 1241 | LLVM::IntToPtrOp::create(rewriter, loc, ptrTypeLLVM, basePtrI32); |
| 1242 | |
| 1243 | if (op.getSubgroupBlockIoAttr()) { |
| 1244 | // if the attribute 'subgroup_block_io' is set to true, it lowers to |
| 1245 | // xevm.blockload |
| 1246 | |
| 1247 | Type intElemTy = rewriter.getIntegerType(width: elemBitWidth); |
| 1248 | VectorType intVecTy = |
| 1249 | VectorType::get(shape: valOrResVecTy.getShape(), elementType: intElemTy); |
| 1250 | |
| 1251 | if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp>) { |
| 1252 | Value loadOp = xevm::BlockLoadOp::create( |
| 1253 | rewriter, loc, intVecTy, basePtrLLVM, /*cache_control=*/nullptr); |
| 1254 | if (intVecTy != valOrResVecTy) { |
| 1255 | loadOp = |
| 1256 | vector::BitCastOp::create(rewriter, loc, valOrResVecTy, loadOp); |
| 1257 | } |
| 1258 | rewriter.replaceOp(op, loadOp); |
| 1259 | } else { |
| 1260 | Value dataToStore = adaptor.getData(); |
| 1261 | if (valOrResVecTy != intVecTy) { |
| 1262 | dataToStore = |
| 1263 | vector::BitCastOp::create(rewriter, loc, intVecTy, dataToStore); |
| 1264 | } |
| 1265 | xevm::BlockStoreOp::create(rewriter, loc, basePtrLLVM, dataToStore, |
| 1266 | nullptr); |
| 1267 | rewriter.eraseOp(op); |
| 1268 | } |
| 1269 | return success(); |
| 1270 | } |
| 1271 | |
| 1272 | if (valOrResVecTy.getNumElements() >= 1) { |
| 1273 | auto chipOpt = xegpu::getChipStr(op); |
| 1274 | if (!chipOpt || |
| 1275 | (*chipOpt != "pvc" && *chipOpt != "bmg" && *chipOpt != "cri" )) { |
| 1276 | // the lowering for chunk load only works for pvc, bmg or cri |
| 1277 | return rewriter.notifyMatchFailure( |
| 1278 | op, "The lowering is specific to pvc, bmg or cri." ); |
| 1279 | } |
| 1280 | } |
| 1281 | |
| 1282 | // The payload shape says where in the matrix each element lives, which the |
| 1283 | // adaptor has already linearized away. |
| 1284 | ArrayRef<int64_t> payloadShape; |
| 1285 | Type opPayloadTy; |
| 1286 | if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp>) |
| 1287 | opPayloadTy = op.getResult().getType(); |
| 1288 | else |
| 1289 | opPayloadTy = op.getData().getType(); |
| 1290 | if (auto opPayloadVecTy = dyn_cast<VectorType>(Val&: opPayloadTy)) |
| 1291 | payloadShape = opPayloadVecTy.getShape(); |
| 1292 | |
| 1293 | return emitContiguousAccesses( |
| 1294 | op, adaptor, rewriter, mdescTy, offsets, payloadShape, |
| 1295 | sliceShape: getContiguousPayloadShape(payloadShape, numOffsets: offsets.size(), mdescTy), |
| 1296 | baseAddr32, basePtrLLVM, ptrTypeLLVM, elemByteSize); |
| 1297 | } |
| 1298 | }; |
| 1299 | |
| 1300 | class PrefetchToXeVMPattern : public OpConversionPattern<xegpu::PrefetchOp> { |
| 1301 | using Base::Base; |
| 1302 | LogicalResult |
| 1303 | matchAndRewrite(xegpu::PrefetchOp op, xegpu::PrefetchOp::Adaptor adaptor, |
| 1304 | ConversionPatternRewriter &rewriter) const override { |
| 1305 | auto loc = op.getLoc(); |
| 1306 | auto ctxt = rewriter.getContext(); |
| 1307 | Value basePtrI64 = adaptor.getSource(); |
| 1308 | // Base pointer is passed as i32 or i64 by adaptor, cast to i64 if needed. |
| 1309 | if (basePtrI64.getType() != rewriter.getI64Type()) |
| 1310 | basePtrI64 = arith::ExtUIOp::create(builder&: rewriter, location: loc, out: rewriter.getI64Type(), |
| 1311 | in: basePtrI64); |
| 1312 | Value offsets = adaptor.getOffsets(); |
| 1313 | if (offsets) { |
| 1314 | VectorType offsetsVecTy = dyn_cast<VectorType>(Val: offsets.getType()); |
| 1315 | if (offsetsVecTy) { |
| 1316 | // Offset needs be scalar. |
| 1317 | return rewriter.notifyMatchFailure(arg&: op, |
| 1318 | msg: "Expected offsets to be a scalar." ); |
| 1319 | } else { |
| 1320 | int64_t elemBitWidth{0}; |
| 1321 | int64_t elemByteSize; |
| 1322 | // Element byte size can come from two sources: |
| 1323 | if (auto memRefTy = dyn_cast<MemRefType>(Val: op.getSourceType())) { |
| 1324 | // If memref is available, we use its element type to |
| 1325 | // determine element byte size. |
| 1326 | elemBitWidth = memRefTy.getElementType().getIntOrFloatBitWidth(); |
| 1327 | } else { |
| 1328 | // Otherwise, we use the provided offset byte alignment. |
| 1329 | elemByteSize = *op.getOffsetAlignByte(); |
| 1330 | } |
| 1331 | if (elemBitWidth != 0) { |
| 1332 | if (elemBitWidth % 8 != 0) |
| 1333 | return rewriter.notifyMatchFailure( |
| 1334 | arg&: op, msg: "Expected element type bit width to be multiple of 8." ); |
| 1335 | elemByteSize = elemBitWidth / 8; |
| 1336 | } |
| 1337 | basePtrI64 = addOffsetToBaseAddr(rewriter, loc, baseAddr: basePtrI64, offset: offsets, |
| 1338 | elemByteSize); |
| 1339 | } |
| 1340 | } |
| 1341 | // Default memory space is global. |
| 1342 | LLVM::LLVMPointerType ptrTypeLLVM = LLVM::LLVMPointerType::get( |
| 1343 | context: ctxt, addressSpace: getNumericXeVMAddrSpace(xeGpuMemspace: xegpu::MemorySpace::Global)); |
| 1344 | // If source is a memref, we use its memory space. |
| 1345 | if (auto memRefTy = dyn_cast<MemRefType>(Val: op.getSource().getType())) { |
| 1346 | FailureOr<unsigned> addrSpace = |
| 1347 | getNumericMemorySpace(memSpace: memRefTy.getMemorySpace()); |
| 1348 | if (failed(Result: addrSpace)) |
| 1349 | return rewriter.notifyMatchFailure( |
| 1350 | arg&: op, msg: "Unsupported memref memory space attribute." ); |
| 1351 | if (*addrSpace != 0) |
| 1352 | ptrTypeLLVM = LLVM::LLVMPointerType::get(context: ctxt, addressSpace: *addrSpace); |
| 1353 | } |
| 1354 | // Convert base pointer (i64) to LLVM pointer type. |
| 1355 | Value ptrLLVM = |
| 1356 | LLVM::IntToPtrOp::create(builder&: rewriter, location: loc, resultType: ptrTypeLLVM, operands: basePtrI64); |
| 1357 | // Create the prefetch op with cache control attribute. |
| 1358 | xevm::PrefetchOp::create( |
| 1359 | builder&: rewriter, location: loc, ptr: ptrLLVM, |
| 1360 | cache_control: xevm::LoadCacheControlAttr::get( |
| 1361 | context: ctxt, value: translateLoadXeGPUCacheHint(L1hint: op.getL1Hint(), L3hint: op.getL3Hint()))); |
| 1362 | rewriter.eraseOp(op); |
| 1363 | return success(); |
| 1364 | } |
| 1365 | }; |
| 1366 | |
| 1367 | class FenceToXeVMPattern : public OpConversionPattern<xegpu::FenceOp> { |
| 1368 | using Base::Base; |
| 1369 | LogicalResult |
| 1370 | matchAndRewrite(xegpu::FenceOp op, xegpu::FenceOp::Adaptor adaptor, |
| 1371 | ConversionPatternRewriter &rewriter) const override { |
| 1372 | auto loc = op.getLoc(); |
| 1373 | xevm::MemScope memScope{xevm::MemScope::WORKGROUP}; |
| 1374 | switch (op.getFenceScope()) { |
| 1375 | case xegpu::FenceScope::Workgroup: |
| 1376 | memScope = xevm::MemScope::WORKGROUP; |
| 1377 | break; |
| 1378 | case xegpu::FenceScope::GPU: |
| 1379 | memScope = xevm::MemScope::DEVICE; |
| 1380 | break; |
| 1381 | } |
| 1382 | xevm::AddrSpace addrSpace{xevm::AddrSpace::GLOBAL}; |
| 1383 | switch (op.getMemoryKind()) { |
| 1384 | case xegpu::MemorySpace::Global: |
| 1385 | addrSpace = xevm::AddrSpace::GLOBAL; |
| 1386 | break; |
| 1387 | case xegpu::MemorySpace::SLM: |
| 1388 | addrSpace = xevm::AddrSpace::SHARED; |
| 1389 | break; |
| 1390 | } |
| 1391 | xevm::MemfenceOp::create(builder&: rewriter, location: loc, scope: memScope, addrspace: addrSpace); |
| 1392 | rewriter.eraseOp(op); |
| 1393 | return success(); |
| 1394 | } |
| 1395 | }; |
| 1396 | |
| 1397 | static auto encodePrecision = [](Type type) -> xevm::ElemType { |
| 1398 | if (type.isBF16()) |
| 1399 | return xevm::ElemType::BF16; |
| 1400 | else if (type.isF16()) |
| 1401 | return xevm::ElemType::F16; |
| 1402 | else if (type.isTF32()) |
| 1403 | return xevm::ElemType::TF32; |
| 1404 | else if (type.isInteger(width: 8)) { |
| 1405 | if (type.isUnsignedInteger()) |
| 1406 | return xevm::ElemType::U8; |
| 1407 | return xevm::ElemType::S8; |
| 1408 | } else if (type.isF32()) |
| 1409 | return xevm::ElemType::F32; |
| 1410 | else if (type.isInteger(width: 32)) |
| 1411 | return xevm::ElemType::S32; |
| 1412 | else if (type.isF8E5M2()) |
| 1413 | return xevm::ElemType::BF8; |
| 1414 | else if (type.isF8E4M3FN()) |
| 1415 | return xevm::ElemType::F8; |
| 1416 | else if (mlir::isa<Float4E2M1FNType>(Val: type)) |
| 1417 | return xevm::ElemType::E2M1; |
| 1418 | llvm_unreachable("add more support for ElemType" ); |
| 1419 | }; |
| 1420 | |
| 1421 | static unsigned getNumOperandsPerDword(xevm::ElemType pTy) { |
| 1422 | switch (pTy) { |
| 1423 | case xevm::ElemType::TF32: |
| 1424 | return 1; |
| 1425 | case xevm::ElemType::BF16: |
| 1426 | case xevm::ElemType::F16: |
| 1427 | return 2; |
| 1428 | case xevm::ElemType::U8: |
| 1429 | case xevm::ElemType::S8: |
| 1430 | case xevm::ElemType::F8: |
| 1431 | case xevm::ElemType::BF8: |
| 1432 | return 4; |
| 1433 | case xevm::ElemType::E2M1: |
| 1434 | return 8; |
| 1435 | default: |
| 1436 | llvm_unreachable("unsupported xevm::ElemType" ); |
| 1437 | } |
| 1438 | } |
| 1439 | |
| 1440 | class DpasToXeVMPattern : public OpConversionPattern<xegpu::DpasOp> { |
| 1441 | using Base::Base; |
| 1442 | LogicalResult |
| 1443 | matchAndRewrite(xegpu::DpasOp op, xegpu::DpasOp::Adaptor adaptor, |
| 1444 | ConversionPatternRewriter &rewriter) const override { |
| 1445 | auto loc = op.getLoc(); |
| 1446 | auto ctxt = rewriter.getContext(); |
| 1447 | auto aTy = cast<VectorType>(Val: op.getLhs().getType()); |
| 1448 | auto bTy = cast<VectorType>(Val: op.getRhs().getType()); |
| 1449 | auto resultType = cast<VectorType>(Val: op.getResultType()); |
| 1450 | |
| 1451 | // get the correct dpasInst by getting info from chip |
| 1452 | auto chipStr = xegpu::getChipStr(op); |
| 1453 | if (!chipStr) |
| 1454 | return rewriter.notifyMatchFailure(arg&: op, msg: "cannot determine target chip" ); |
| 1455 | |
| 1456 | const auto *uArch = mlir::xegpu::uArch::getUArch(archName: *chipStr); |
| 1457 | if (!uArch) |
| 1458 | return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported target uArch" ); |
| 1459 | |
| 1460 | auto *dpasInst = const_cast<xegpu::uArch::SubgroupMatrixMultiplyAcc *>( |
| 1461 | llvm::dyn_cast_or_null<xegpu::uArch::SubgroupMatrixMultiplyAcc>( |
| 1462 | Val: uArch->getInstruction( |
| 1463 | instKind: xegpu::uArch::InstructionKind::SubgroupMatrixMultiplyAcc))); |
| 1464 | if (!dpasInst) |
| 1465 | return rewriter.notifyMatchFailure(arg&: op, |
| 1466 | msg: "DPAS not supported by target uArch" ); |
| 1467 | |
| 1468 | auto checkSupportedTypes = [&](VectorType vecTy, |
| 1469 | xegpu::uArch::MMAOpndKind kind) -> bool { |
| 1470 | auto supported = dpasInst->getSupportedTypes(context&: *ctxt, matrixType: kind); |
| 1471 | return llvm::find(Range&: supported, Val: vecTy.getElementType()) != supported.end(); |
| 1472 | }; |
| 1473 | |
| 1474 | if (!checkSupportedTypes(aTy, xegpu::uArch::MMAOpndKind::MatrixA)) |
| 1475 | return rewriter.notifyMatchFailure( |
| 1476 | arg&: op, msg: "A-matrix element type not supported by target uArch" ); |
| 1477 | if (!checkSupportedTypes(bTy, xegpu::uArch::MMAOpndKind::MatrixB)) |
| 1478 | return rewriter.notifyMatchFailure( |
| 1479 | arg&: op, msg: "B-matrix element type not supported by target uArch" ); |
| 1480 | // NOTE: Supported types for MatrixC and MatrixD are identical |
| 1481 | if (!checkSupportedTypes(resultType, xegpu::uArch::MMAOpndKind::MatrixD)) |
| 1482 | return rewriter.notifyMatchFailure( |
| 1483 | arg&: op, msg: "result/accumulator element type not supported by target uArch" ); |
| 1484 | |
| 1485 | xevm::ElemType precATy = encodePrecision(aTy.getElementType()); |
| 1486 | xevm::ElemType precBTy = encodePrecision(bTy.getElementType()); |
| 1487 | Value c = op.getAcc(); |
| 1488 | if (!c) { |
| 1489 | auto elementTy = resultType.getElementType(); |
| 1490 | Attribute initValueAttr; |
| 1491 | if (isa<FloatType>(Val: elementTy)) |
| 1492 | initValueAttr = FloatAttr::get(type: elementTy, value: 0.0); |
| 1493 | else |
| 1494 | initValueAttr = IntegerAttr::get(type: elementTy, value: 0); |
| 1495 | c = arith::ConstantOp::create( |
| 1496 | builder&: rewriter, location: loc, value: DenseElementsAttr::get(type: resultType, values: initValueAttr)); |
| 1497 | } |
| 1498 | |
| 1499 | Value aVec = op.getLhs(); |
| 1500 | Value bVec = op.getRhs(); |
| 1501 | auto cvecty = cast<VectorType>(Val: c.getType()); |
| 1502 | xevm::ElemType precCTy = encodePrecision(cvecty.getElementType()); |
| 1503 | xevm::ElemType precDTy = encodePrecision(resultType.getElementType()); |
| 1504 | VectorType cNty = |
| 1505 | VectorType::get(shape: cvecty.getNumElements(), elementType: cvecty.getElementType()); |
| 1506 | if (cvecty != cNty) |
| 1507 | c = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: cNty, source: c); |
| 1508 | Value dpasRes = xevm::MMAOp::create( |
| 1509 | builder&: rewriter, location: loc, d: cNty, a: aVec, b: bVec, c, |
| 1510 | shape: xevm::MMAShapeAttr::get(context: ctxt, m: cvecty.getNumElements(), n: executionSize, |
| 1511 | k: systolicDepth * |
| 1512 | getNumOperandsPerDword(pTy: precATy)), |
| 1513 | types: xevm::MMATypesAttr::get(context: ctxt, d: precDTy, a: precATy, b: precBTy, c: precCTy)); |
| 1514 | if (cvecty != cNty) |
| 1515 | dpasRes = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: resultType, source: dpasRes); |
| 1516 | rewriter.replaceOp(op, newValues: dpasRes); |
| 1517 | return success(); |
| 1518 | } |
| 1519 | }; |
| 1520 | |
| 1521 | static std::optional<LLVM::AtomicBinOp> |
| 1522 | matchSimpleAtomicOp(arith::AtomicRMWKind arithKind) { |
| 1523 | switch (arithKind) { |
| 1524 | case arith::AtomicRMWKind::addf: |
| 1525 | return LLVM::AtomicBinOp::fadd; |
| 1526 | case arith::AtomicRMWKind::addi: |
| 1527 | return LLVM::AtomicBinOp::add; |
| 1528 | case arith::AtomicRMWKind::assign: |
| 1529 | return LLVM::AtomicBinOp::xchg; |
| 1530 | case arith::AtomicRMWKind::maximumf: |
| 1531 | return LLVM::AtomicBinOp::fmax; |
| 1532 | case arith::AtomicRMWKind::maxs: |
| 1533 | return LLVM::AtomicBinOp::max; |
| 1534 | case arith::AtomicRMWKind::maxu: |
| 1535 | return LLVM::AtomicBinOp::umax; |
| 1536 | case arith::AtomicRMWKind::minimumf: |
| 1537 | return LLVM::AtomicBinOp::fmin; |
| 1538 | case arith::AtomicRMWKind::mins: |
| 1539 | return LLVM::AtomicBinOp::min; |
| 1540 | case arith::AtomicRMWKind::minu: |
| 1541 | return LLVM::AtomicBinOp::umin; |
| 1542 | case arith::AtomicRMWKind::ori: |
| 1543 | return LLVM::AtomicBinOp::_or; |
| 1544 | case arith::AtomicRMWKind::andi: |
| 1545 | return LLVM::AtomicBinOp::_and; |
| 1546 | default: |
| 1547 | return std::nullopt; |
| 1548 | } |
| 1549 | } |
| 1550 | |
| 1551 | class AtomicRMWToXeVMPattern : public OpConversionPattern<xegpu::AtomicRMWOp> { |
| 1552 | using Base::Base; |
| 1553 | LogicalResult |
| 1554 | matchAndRewrite(xegpu::AtomicRMWOp op, xegpu::AtomicRMWOp::Adaptor adaptor, |
| 1555 | ConversionPatternRewriter &rewriter) const override { |
| 1556 | auto loc = op.getLoc(); |
| 1557 | auto ctxt = rewriter.getContext(); |
| 1558 | auto tdesc = op.getTensorDesc().getType(); |
| 1559 | auto ptrTypeLLVM = LLVM::LLVMPointerType::get( |
| 1560 | context: ctxt, addressSpace: getNumericXeVMAddrSpace(xeGpuMemspace: tdesc.getMemorySpace())); |
| 1561 | Value basePtrI64 = arith::IndexCastOp::create( |
| 1562 | builder&: rewriter, location: loc, out: rewriter.getI64Type(), in: adaptor.getTensorDesc()); |
| 1563 | Value basePtrLLVM = |
| 1564 | LLVM::IntToPtrOp::create(builder&: rewriter, location: loc, resultType: ptrTypeLLVM, operands: basePtrI64); |
| 1565 | VectorType srcOrDstVecTy = cast<VectorType>(Val: op.getValue().getType()); |
| 1566 | VectorType srcOrDstFlatVecTy = VectorType::get( |
| 1567 | shape: srcOrDstVecTy.getNumElements(), elementType: srcOrDstVecTy.getElementType()); |
| 1568 | Value srcFlatVec = vector::ShapeCastOp::create( |
| 1569 | builder&: rewriter, location: loc, result: srcOrDstFlatVecTy, source: op.getValue()); |
| 1570 | auto atomicKind = matchSimpleAtomicOp(arithKind: op.getKind()); |
| 1571 | assert(atomicKind.has_value()); |
| 1572 | Value resVec = srcFlatVec; |
| 1573 | for (int i = 0; i < srcOrDstVecTy.getNumElements(); i++) { |
| 1574 | auto val = vector::ExtractOp::create(builder&: rewriter, location: loc, source: resVec, position: i); |
| 1575 | Value idx = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: rewriter.getI64Type(), |
| 1576 | value: rewriter.getI64IntegerAttr(value: i)); |
| 1577 | Value currPtr = |
| 1578 | LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrTypeLLVM, |
| 1579 | elementType: srcOrDstVecTy.getElementType(), basePtr: basePtrLLVM, indices: idx); |
| 1580 | Value newVal = |
| 1581 | LLVM::AtomicRMWOp::create(builder&: rewriter, location: loc, binOp: atomicKind.value(), ptr: currPtr, |
| 1582 | val, ordering: LLVM::AtomicOrdering::seq_cst); |
| 1583 | resVec = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: newVal, dest: resVec, position: i); |
| 1584 | } |
| 1585 | rewriter.replaceOp(op, newValues: resVec); |
| 1586 | return success(); |
| 1587 | } |
| 1588 | }; |
| 1589 | |
| 1590 | class DpasMxToXeVMPattern : public OpConversionPattern<xegpu::DpasMxOp> { |
| 1591 | using Base::Base; |
| 1592 | LogicalResult |
| 1593 | matchAndRewrite(xegpu::DpasMxOp op, xegpu::DpasMxOp::Adaptor adaptor, |
| 1594 | ConversionPatternRewriter &rewriter) const override { |
| 1595 | auto loc = op.getLoc(); |
| 1596 | auto ctxt = rewriter.getContext(); |
| 1597 | auto aTy = op.getA().getType(); |
| 1598 | auto bTy = op.getB().getType(); |
| 1599 | auto resVecTy = |
| 1600 | cast<VectorType>(Val: getTypeConverter()->convertType(t: op.getType())); |
| 1601 | |
| 1602 | auto chipStr = xegpu::getChipStr(op); |
| 1603 | if (!chipStr) |
| 1604 | return rewriter.notifyMatchFailure(arg&: op, msg: "cannot determine target chip" ); |
| 1605 | |
| 1606 | const auto *uArch = xegpu::uArch::getUArch(archName: *chipStr); |
| 1607 | if (!uArch) |
| 1608 | return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported target uArch" ); |
| 1609 | |
| 1610 | // TODO: Add supported shape check |
| 1611 | |
| 1612 | xevm::ElemType precATy = encodePrecision(aTy.getElementType()); |
| 1613 | xevm::ElemType precBTy = encodePrecision(bTy.getElementType()); |
| 1614 | Value c = adaptor.getAcc(); |
| 1615 | if (!c) { |
| 1616 | auto elementTy = resVecTy.getElementType(); |
| 1617 | Attribute initValueAttr; |
| 1618 | if (isa<FloatType>(Val: elementTy)) |
| 1619 | initValueAttr = FloatAttr::get(type: elementTy, value: 0.0); |
| 1620 | else |
| 1621 | initValueAttr = IntegerAttr::get(type: elementTy, value: 0); |
| 1622 | c = arith::ConstantOp::create( |
| 1623 | builder&: rewriter, location: loc, value: DenseElementsAttr::get(type: resVecTy, values: initValueAttr)); |
| 1624 | } |
| 1625 | |
| 1626 | Value aVec = adaptor.getA(); |
| 1627 | Value bVec = adaptor.getB(); |
| 1628 | auto aVecTy = cast<VectorType>(Val: aVec.getType()); |
| 1629 | auto bVecTy = cast<VectorType>(Val: bVec.getType()); |
| 1630 | if (aVecTy.getElementTypeBitWidth() == 4) |
| 1631 | aVec = vector::BitCastOp::create( |
| 1632 | builder&: rewriter, location: loc, |
| 1633 | result: VectorType::get(shape: aVecTy.getNumElements() / 2, elementType: rewriter.getI8Type()), |
| 1634 | source: aVec); |
| 1635 | if (bVecTy.getElementTypeBitWidth() == 4) |
| 1636 | bVec = vector::BitCastOp::create( |
| 1637 | builder&: rewriter, location: loc, |
| 1638 | result: VectorType::get(shape: bVecTy.getNumElements() / 2, elementType: rewriter.getI8Type()), |
| 1639 | source: bVec); |
| 1640 | auto cVecTy = cast<VectorType>(Val: c.getType()); |
| 1641 | xevm::ElemType precCTy = encodePrecision(cVecTy.getElementType()); |
| 1642 | xevm::ElemType precDTy = encodePrecision(resVecTy.getElementType()); |
| 1643 | Value scaleA = adaptor.getScaleA(); |
| 1644 | Value scaleB = adaptor.getScaleB(); |
| 1645 | Value dpasMxRes = xevm::MMAMxOp::create( |
| 1646 | builder&: rewriter, location: loc, d: resVecTy, a: aVec, b: bVec, scale_a: scaleA, scale_b: scaleB, c, |
| 1647 | shape: xevm::MMAShapeAttr::get(context: ctxt, m: cVecTy.getNumElements(), n: executionSize, |
| 1648 | k: systolicDepth * |
| 1649 | getNumOperandsPerDword(pTy: precATy)), |
| 1650 | types: xevm::MMATypesAttr::get(context: ctxt, d: precDTy, a: precATy, b: precBTy, c: precCTy)); |
| 1651 | rewriter.replaceOp(op, newValues: dpasMxRes); |
| 1652 | return success(); |
| 1653 | } |
| 1654 | }; |
| 1655 | |
| 1656 | //===----------------------------------------------------------------------===// |
| 1657 | // arith.extf / arith.truncf to xevm.extf / xevm.truncf |
| 1658 | //===----------------------------------------------------------------------===// |
| 1659 | // |
| 1660 | // Micro-scaling (MX) GEMM lowering breaks arith.scaling_extf/scaling_truncf |
| 1661 | // into plain arith.extf/arith.truncf whose narrow side uses one of the MX float |
| 1662 | // formats (f8E5M2, f8E4M3FN or f4E2M1FN). These narrow floats have no native |
| 1663 | // LLVM support, so the conversions are mapped onto the dedicated xevm.extf / |
| 1664 | // xevm.truncf ops which lower to hardware builtins. The f8E8M0FNU scale type is |
| 1665 | // intentionally not handled here: it is expanded into integer arithmetic by |
| 1666 | // arith-expand before this pass runs. |
| 1667 | |
| 1668 | // xevm.extf / xevm.truncf only convert between the MX narrow floats and |
| 1669 | // f16/bf16, and the underlying builtins operate on exactly 16 f16/bf16 values. |
| 1670 | static constexpr int64_t kXeVMExtfTruncfNumElems = 16; |
| 1671 | |
| 1672 | // Maps a narrow MX float element type to the matching xevm.extf source enum. |
| 1673 | static std::optional<xevm::ExtfSrcElemTypes> getExtfNarrowType(Type etype) { |
| 1674 | if (isa<Float8E5M2Type>(Val: etype)) |
| 1675 | return xevm::ExtfSrcElemTypes::BF8; |
| 1676 | if (isa<Float8E4M3FNType>(Val: etype)) |
| 1677 | return xevm::ExtfSrcElemTypes::F8; |
| 1678 | if (isa<Float4E2M1FNType>(Val: etype)) |
| 1679 | return xevm::ExtfSrcElemTypes::E2M1; |
| 1680 | return std::nullopt; |
| 1681 | } |
| 1682 | |
| 1683 | // Maps a narrow MX float element type to the matching xevm.truncf dest enum. |
| 1684 | static std::optional<xevm::TruncfDstElemTypes> getTruncfNarrowType(Type etype) { |
| 1685 | if (isa<Float8E5M2Type>(Val: etype)) |
| 1686 | return xevm::TruncfDstElemTypes::BF8; |
| 1687 | if (isa<Float8E4M3FNType>(Val: etype)) |
| 1688 | return xevm::TruncfDstElemTypes::F8; |
| 1689 | if (isa<Float4E2M1FNType>(Val: etype)) |
| 1690 | return xevm::TruncfDstElemTypes::E2M1; |
| 1691 | return std::nullopt; |
| 1692 | } |
| 1693 | |
| 1694 | // Returns true if `op` is an arith.extf that can be lowered to xevm.extf, i.e. |
| 1695 | // a rank-1 widening from an MX narrow float to a 16-element f16/bf16 vector. |
| 1696 | static bool isXeVMExtf(arith::ExtFOp op) { |
| 1697 | auto srcTy = dyn_cast<VectorType>(Val: op.getIn().getType()); |
| 1698 | auto dstTy = dyn_cast<VectorType>(Val: op.getType()); |
| 1699 | if (!srcTy || !dstTy || srcTy.getRank() != 1 || dstTy.getRank() != 1) |
| 1700 | return false; |
| 1701 | if (dstTy.getNumElements() != kXeVMExtfTruncfNumElems) |
| 1702 | return false; |
| 1703 | Type dstETy = dstTy.getElementType(); |
| 1704 | if (!dstETy.isF16() && !dstETy.isBF16()) |
| 1705 | return false; |
| 1706 | return getExtfNarrowType(etype: srcTy.getElementType()).has_value(); |
| 1707 | } |
| 1708 | |
| 1709 | // Returns true if `op` is an arith.truncf that can be lowered to xevm.truncf, |
| 1710 | // i.e. a rank-1 truncation from an f16/bf16 vector to an MX narrow float. The |
| 1711 | // source has to hold a whole number of the fixed-size groups xevm.truncf |
| 1712 | // converts at a time; wider vectors are converted in several steps. |
| 1713 | static bool isXeVMTruncf(arith::TruncFOp op) { |
| 1714 | auto srcTy = dyn_cast<VectorType>(Val: op.getIn().getType()); |
| 1715 | auto dstTy = dyn_cast<VectorType>(Val: op.getType()); |
| 1716 | if (!srcTy || !dstTy || srcTy.getRank() != 1 || dstTy.getRank() != 1) |
| 1717 | return false; |
| 1718 | int64_t numElems = srcTy.getNumElements(); |
| 1719 | if (numElems == 0 || numElems % kXeVMExtfTruncfNumElems != 0) |
| 1720 | return false; |
| 1721 | Type srcETy = srcTy.getElementType(); |
| 1722 | if (!srcETy.isF16() && !srcETy.isBF16()) |
| 1723 | return false; |
| 1724 | return getTruncfNarrowType(etype: dstTy.getElementType()).has_value(); |
| 1725 | } |
| 1726 | |
| 1727 | class ExtfToXeVMPattern : public OpConversionPattern<arith::ExtFOp> { |
| 1728 | using Base::Base; |
| 1729 | LogicalResult |
| 1730 | matchAndRewrite(arith::ExtFOp op, OpAdaptor adaptor, |
| 1731 | ConversionPatternRewriter &rewriter) const override { |
| 1732 | if (!isXeVMExtf(op)) |
| 1733 | return rewriter.notifyMatchFailure(arg&: op, msg: "not a xevm.extf compatible extf" ); |
| 1734 | Location loc = op.getLoc(); |
| 1735 | MLIRContext *ctx = op.getContext(); |
| 1736 | auto srcVecTy = cast<VectorType>(Val: op.getIn().getType()); |
| 1737 | auto dstVecTy = cast<VectorType>(Val: op.getType()); |
| 1738 | xevm::ExtfSrcElemTypes srcEnum = |
| 1739 | *getExtfNarrowType(etype: srcVecTy.getElementType()); |
| 1740 | xevm::ExtfDstElemTypes dstEnum = dstVecTy.getElementType().isF16() |
| 1741 | ? xevm::ExtfDstElemTypes::F16 |
| 1742 | : xevm::ExtfDstElemTypes::BF16; |
| 1743 | // The narrow float operand has already been type-converted to an integer |
| 1744 | // vector of the same bit width (i4 for fp4, i8 for fp8). xevm.extf takes |
| 1745 | // the values packed into an i8 vector, so re-pack fp4 (i4) operands. |
| 1746 | Value src = adaptor.getIn(); |
| 1747 | auto convSrcTy = cast<VectorType>(Val: src.getType()); |
| 1748 | if (convSrcTy.getElementTypeBitWidth() == 4) |
| 1749 | src = vector::BitCastOp::create( |
| 1750 | builder&: rewriter, location: loc, |
| 1751 | result: VectorType::get(shape: convSrcTy.getNumElements() / 2, elementType: rewriter.getI8Type()), |
| 1752 | source: src); |
| 1753 | Type resTy = getTypeConverter()->convertType(t: dstVecTy); |
| 1754 | Value res = xevm::ExtfOp::create( |
| 1755 | builder&: rewriter, location: loc, dst: resTy, src, src_etype: xevm::ExtfSrcElemTypeAttr::get(context: ctx, etype: srcEnum), |
| 1756 | dst_etype: xevm::ExtfDstElemTypeAttr::get(context: ctx, etype: dstEnum)); |
| 1757 | rewriter.replaceOp(op, newValues: res); |
| 1758 | return success(); |
| 1759 | } |
| 1760 | }; |
| 1761 | |
| 1762 | class TruncfToXeVMPattern : public OpConversionPattern<arith::TruncFOp> { |
| 1763 | using Base::Base; |
| 1764 | LogicalResult |
| 1765 | matchAndRewrite(arith::TruncFOp op, OpAdaptor adaptor, |
| 1766 | ConversionPatternRewriter &rewriter) const override { |
| 1767 | if (!isXeVMTruncf(op)) |
| 1768 | return rewriter.notifyMatchFailure(arg&: op, |
| 1769 | msg: "not a xevm.truncf compatible truncf" ); |
| 1770 | Location loc = op.getLoc(); |
| 1771 | MLIRContext *ctx = op.getContext(); |
| 1772 | auto srcVecTy = cast<VectorType>(Val: op.getIn().getType()); |
| 1773 | auto dstVecTy = cast<VectorType>(Val: op.getType()); |
| 1774 | xevm::TruncfSrcElemTypes srcEnum = srcVecTy.getElementType().isF16() |
| 1775 | ? xevm::TruncfSrcElemTypes::F16 |
| 1776 | : xevm::TruncfSrcElemTypes::BF16; |
| 1777 | xevm::TruncfDstElemTypes dstEnum = |
| 1778 | *getTruncfNarrowType(etype: dstVecTy.getElementType()); |
| 1779 | auto srcEnumAttr = xevm::TruncfSrcElemTypeAttr::get(context: ctx, etype: srcEnum); |
| 1780 | auto dstEnumAttr = xevm::TruncfDstElemTypeAttr::get(context: ctx, etype: dstEnum); |
| 1781 | |
| 1782 | // xevm.truncf lowers to instructions that convert a fixed number of |
| 1783 | // elements at a time, so a wider source is converted one group at a time |
| 1784 | // and the packed results are concatenated. Each group produces the narrow |
| 1785 | // floats packed into an i8 vector. |
| 1786 | int64_t numGroups = srcVecTy.getNumElements() / kXeVMExtfTruncfNumElems; |
| 1787 | int64_t groupBytes = |
| 1788 | kXeVMExtfTruncfNumElems * dstVecTy.getElementTypeBitWidth() / 8; |
| 1789 | Type groupTy = VectorType::get(shape: groupBytes, elementType: rewriter.getI8Type()); |
| 1790 | |
| 1791 | Value src = adaptor.getIn(); |
| 1792 | Value packed; |
| 1793 | if (numGroups == 1) { |
| 1794 | packed = xevm::TruncfOp::create(builder&: rewriter, location: loc, dst: groupTy, src, src_etype: srcEnumAttr, |
| 1795 | dst_etype: dstEnumAttr); |
| 1796 | } else { |
| 1797 | auto packedTy = |
| 1798 | VectorType::get(shape: groupBytes * numGroups, elementType: rewriter.getI8Type()); |
| 1799 | packed = arith::ConstantOp::create(builder&: rewriter, location: loc, result: packedTy, |
| 1800 | value: rewriter.getZeroAttr(type: packedTy)); |
| 1801 | for (int64_t group = 0; group < numGroups; group++) { |
| 1802 | Value slice = vector::ExtractStridedSliceOp::create( |
| 1803 | builder&: rewriter, location: loc, source: src, offsets: group * kXeVMExtfTruncfNumElems, |
| 1804 | sizes: kXeVMExtfTruncfNumElems, /*strides=*/1); |
| 1805 | Value converted = xevm::TruncfOp::create(builder&: rewriter, location: loc, dst: groupTy, src: slice, |
| 1806 | src_etype: srcEnumAttr, dst_etype: dstEnumAttr); |
| 1807 | packed = vector::InsertStridedSliceOp::create( |
| 1808 | builder&: rewriter, location: loc, valueToStore: converted, dest: packed, offsets: group * groupBytes, |
| 1809 | /*strides=*/1); |
| 1810 | } |
| 1811 | } |
| 1812 | // Re-shape to the type-converted result type (i4 vector for fp4). |
| 1813 | Type resTy = getTypeConverter()->convertType(t: dstVecTy); |
| 1814 | if (packed.getType() != resTy) |
| 1815 | packed = vector::BitCastOp::create(builder&: rewriter, location: loc, result: resTy, source: packed); |
| 1816 | rewriter.replaceOp(op, newValues: packed); |
| 1817 | return success(); |
| 1818 | } |
| 1819 | }; |
| 1820 | |
| 1821 | // Hoist the i4-to-i8 part of compatible extensions through a closed |
| 1822 | // data-movement graph and lane shuffle. Widening all the way to the extension's |
| 1823 | // result type could exceed the bitcast-shuffle lowering's 64-bit fragment |
| 1824 | // limit. Signed widening preserves the eventual signed interpretation of each |
| 1825 | // nibble; unsigned widening preserves its unsigned interpretation. |
| 1826 | template <typename ExtOp> |
| 1827 | class HoistI4ExtensionAboveLaneShuffle final : public OpRewritePattern<ExtOp> { |
| 1828 | public: |
| 1829 | using OpRewritePattern<ExtOp>::OpRewritePattern; |
| 1830 | |
| 1831 | LogicalResult matchAndRewrite(ExtOp op, |
| 1832 | PatternRewriter &rewriter) const override { |
| 1833 | auto srcTy = dyn_cast<VectorType>(op.getIn().getType()); |
| 1834 | auto dstTy = dyn_cast<VectorType>(op.getType()); |
| 1835 | if (!srcTy || !srcTy.getElementType().isInteger(4) || !dstTy || |
| 1836 | dstTy.getElementTypeBitWidth() < 8) |
| 1837 | return failure(); |
| 1838 | |
| 1839 | auto isMover = [](Operation *def) { |
| 1840 | return isa<vector::ShapeCastOp, vector::ExtractStridedSliceOp, |
| 1841 | vector::ExtractOp, vector::BroadcastOp, vector::TransposeOp>( |
| 1842 | Val: def); |
| 1843 | }; |
| 1844 | Value source = op.getIn(); |
| 1845 | while (Operation *def = source.getDefiningOp()) { |
| 1846 | if (isa<xegpu::LaneShuffleOp>(Val: def)) |
| 1847 | break; |
| 1848 | if (!isMover(def)) |
| 1849 | return failure(); |
| 1850 | source = def->getOperand(idx: 0); |
| 1851 | } |
| 1852 | auto shuffle = source.getDefiningOp<xegpu::LaneShuffleOp>(); |
| 1853 | if (!shuffle) |
| 1854 | return failure(); |
| 1855 | VectorType fragmentTy = shuffle.getSourceType(); |
| 1856 | int64_t numElements = fragmentTy.getNumElements(); |
| 1857 | if (!fragmentTy.getElementType().isInteger(width: 4) || numElements < 2 || |
| 1858 | numElements > 8 || !llvm::isPowerOf2_64(Value: numElements)) |
| 1859 | return failure(); |
| 1860 | |
| 1861 | // Validate every use before changing IR. Widening a shared fragment is |
| 1862 | // safe only when all paths end in extensions with the same signedness. |
| 1863 | SmallVector<Operation *> movers; |
| 1864 | SmallVector<ExtOp> extensions; |
| 1865 | SmallPtrSet<Operation *, 16> visited; |
| 1866 | auto visit = [&](auto &&visit, Value value) -> LogicalResult { |
| 1867 | for (Operation *user : value.getUsers()) { |
| 1868 | if (!visited.insert(Ptr: user).second) |
| 1869 | continue; |
| 1870 | if (auto ext = dyn_cast<ExtOp>(user)) { |
| 1871 | auto type = dyn_cast<VectorType>(ext.getType()); |
| 1872 | if (!type || type.getElementTypeBitWidth() < 8) |
| 1873 | return failure(); |
| 1874 | extensions.push_back(ext); |
| 1875 | continue; |
| 1876 | } |
| 1877 | if (!isMover(user) || user->getOperand(idx: 0) != value || |
| 1878 | user->getNumResults() != 1 || |
| 1879 | failed(visit(visit, user->getResult(idx: 0)))) |
| 1880 | return failure(); |
| 1881 | movers.push_back(Elt: user); |
| 1882 | } |
| 1883 | return success(); |
| 1884 | }; |
| 1885 | if (failed(visit(visit, shuffle.getResult()))) |
| 1886 | return failure(); |
| 1887 | |
| 1888 | rewriter.setInsertionPoint(shuffle); |
| 1889 | VectorType wideTy = fragmentTy.clone(elementType: rewriter.getI8Type()); |
| 1890 | Value wide = |
| 1891 | ExtOp::create(rewriter, op.getLoc(), wideTy, shuffle.getSource()); |
| 1892 | Value result = xegpu::LaneShuffleOp::create( |
| 1893 | builder&: rewriter, location: shuffle.getLoc(), result: wideTy, source: wide, mode: shuffle.getMode()); |
| 1894 | IRMapping mapping; |
| 1895 | mapping.map(from: shuffle.getResult(), to: result); |
| 1896 | for (Operation *mover : llvm::reverse(C&: movers)) { |
| 1897 | rewriter.setInsertionPoint(mover); |
| 1898 | Operation *clone = rewriter.clone(op&: *mover, mapper&: mapping); |
| 1899 | Type type = rewriter.getI8Type(); |
| 1900 | if (auto vectorTy = dyn_cast<VectorType>(Val: mover->getResult(idx: 0).getType())) |
| 1901 | type = vectorTy.clone(elementType: type); |
| 1902 | clone->getResult(idx: 0).setType(type); |
| 1903 | } |
| 1904 | for (ExtOp ext : extensions) { |
| 1905 | Value input = mapping.lookup(ext.getIn()); |
| 1906 | if (input.getType() == ext.getType()) |
| 1907 | rewriter.replaceOp(ext, input); |
| 1908 | else |
| 1909 | rewriter.modifyOpInPlace(ext, |
| 1910 | [&] { ext.getInMutable().assign(input); }); |
| 1911 | } |
| 1912 | for (Operation *mover : movers) |
| 1913 | rewriter.eraseOp(op: mover); |
| 1914 | rewriter.eraseOp(op: shuffle); |
| 1915 | return success(); |
| 1916 | } |
| 1917 | }; |
| 1918 | |
| 1919 | // Lowers `xegpu.lane_shuffle` to `xevm.bitcast_shuffle`. |
| 1920 | // |
| 1921 | // `xevm.bitcast_shuffle` concatenates the components of its operand across the |
| 1922 | // subgroup, the first component of every lane first, and then hands chunks the |
| 1923 | // size of a result component back out to the lanes in order. Numbering the |
| 1924 | // elements of a `vector<NxT>` fragment held by lane `i` of a subgroup of size |
| 1925 | // `S` by their logical position, that concatenation is exactly the `pack` mode |
| 1926 | // input numbering `j * S + i`. Taking the result as a single `N * width(T)` bit |
| 1927 | // scalar then hands lane `i` the logical positions `i * N .. i * N + N - 1`, |
| 1928 | // which is the `pack` mode output numbering. |
| 1929 | // |
| 1930 | // So `pack` is a vector-to-scalar `xevm.bitcast_shuffle` followed by a bitcast |
| 1931 | // back to the fragment type, and `unpack`, being its inverse, is a bitcast to |
| 1932 | // the scalar followed by a scalar-to-vector `xevm.bitcast_shuffle`. |
| 1933 | // |
| 1934 | // `xevm.bitcast_shuffle` only accepts the integer types `i8`, `i16`, `i32` and |
| 1935 | // `i64`, since it is bit-preserving and so does not depend on how the bits are |
| 1936 | // interpreted. A fragment of a floating point type is therefore bitcast to a |
| 1937 | // same-width integer vector on the way in and back on the way out. |
| 1938 | class LaneShuffleToXeVMPattern |
| 1939 | : public OpConversionPattern<xegpu::LaneShuffleOp> { |
| 1940 | using Base::Base; |
| 1941 | LogicalResult |
| 1942 | matchAndRewrite(xegpu::LaneShuffleOp op, OpAdaptor adaptor, |
| 1943 | ConversionPatternRewriter &rewriter) const override { |
| 1944 | auto vecTy = dyn_cast<VectorType>(Val: adaptor.getSource().getType()); |
| 1945 | if (!vecTy) |
| 1946 | return rewriter.notifyMatchFailure(arg&: op, msg: "Expected a vector fragment." ); |
| 1947 | // The shuffle redistributes whole bytes between the lanes, so sub-byte |
| 1948 | // element types, fp4 in particular, cannot be shuffled. Widths without a |
| 1949 | // matching integer type the op accepts are rejected for the same reason. |
| 1950 | unsigned elemBits = vecTy.getElementTypeBitWidth(); |
| 1951 | if (elemBits != 8 && elemBits != 16 && elemBits != 32 && elemBits != 64) |
| 1952 | return rewriter.notifyMatchFailure( |
| 1953 | arg&: op, msg: "Expected an element type of 8, 16, 32 or 64 bits." ); |
| 1954 | int64_t fragmentBits = vecTy.getNumElements() * elemBits; |
| 1955 | if (fragmentBits > 64 || !llvm::isPowerOf2_64(Value: fragmentBits)) |
| 1956 | return rewriter.notifyMatchFailure( |
| 1957 | arg&: op, msg: "Expected a fragment of 8, 16, 32 or 64 bits." ); |
| 1958 | |
| 1959 | Location loc = op.getLoc(); |
| 1960 | Type packedTy = rewriter.getIntegerType(width: fragmentBits); |
| 1961 | // The integer vector type the shuffle actually operates on. Equal to the |
| 1962 | // fragment type when that is already an integer vector. |
| 1963 | VectorType shuffleTy = |
| 1964 | VectorType::get(shape: vecTy.getShape(), elementType: rewriter.getIntegerType(width: elemBits)); |
| 1965 | |
| 1966 | Value res; |
| 1967 | if (op.getMode() == xegpu::LaneShuffleMode::Pack) { |
| 1968 | Value src = adaptor.getSource(); |
| 1969 | if (shuffleTy != vecTy) |
| 1970 | src = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: shuffleTy, arg: src); |
| 1971 | res = xevm::BitcastShuffleOp::create(builder&: rewriter, location: loc, res: packedTy, src); |
| 1972 | res = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: vecTy, arg: res); |
| 1973 | } else { |
| 1974 | Value packed = |
| 1975 | LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: packedTy, arg: adaptor.getSource()); |
| 1976 | res = xevm::BitcastShuffleOp::create(builder&: rewriter, location: loc, res: shuffleTy, src: packed); |
| 1977 | if (shuffleTy != vecTy) |
| 1978 | res = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: vecTy, arg: res); |
| 1979 | } |
| 1980 | rewriter.replaceOp(op, newValues: res); |
| 1981 | return success(); |
| 1982 | } |
| 1983 | }; |
| 1984 | |
| 1985 | //===----------------------------------------------------------------------===// |
| 1986 | // Pass Definition |
| 1987 | //===----------------------------------------------------------------------===// |
| 1988 | |
| 1989 | struct ConvertXeGPUToXeVMPass |
| 1990 | : public impl::ConvertXeGPUToXeVMPassBase<ConvertXeGPUToXeVMPass> { |
| 1991 | using Base::Base; |
| 1992 | |
| 1993 | void runOnOperation() override { |
| 1994 | MLIRContext *context = &getContext(); |
| 1995 | |
| 1996 | // Restrict greedy rewriting to casts and newly generated operations so |
| 1997 | // unrelated dead XeGPU ops still receive conversion diagnostics. |
| 1998 | GreedyRewriteConfig config; |
| 1999 | config.setStrictness(GreedyRewriteStrictness::ExistingAndNewOps); |
| 2000 | // Preserve extension consumers until hoisting has finished: narrow-type |
| 2001 | // legalization would otherwise obscure the chains this peephole matches. |
| 2002 | { |
| 2003 | RewritePatternSet patterns(context); |
| 2004 | patterns.add<HoistI4ExtensionAboveLaneShuffle<arith::ExtUIOp>, |
| 2005 | HoistI4ExtensionAboveLaneShuffle<arith::ExtSIOp>>(arg&: context); |
| 2006 | SmallVector<Operation *> extensions; |
| 2007 | getOperation()->walk(callback: [&](Operation *op) { |
| 2008 | if (isa<arith::ExtUIOp, arith::ExtSIOp>(Val: op)) |
| 2009 | extensions.push_back(Elt: op); |
| 2010 | }); |
| 2011 | if (failed( |
| 2012 | Result: applyOpPatternsGreedily(ops: extensions, patterns: std::move(patterns), config))) |
| 2013 | return signalPassFailure(); |
| 2014 | } |
| 2015 | // Expand the new i4-to-i8 casts into byte-level masks and shifts before |
| 2016 | // lowering. LLVM extensions on i4 vectors are not legal in the backend. |
| 2017 | { |
| 2018 | RewritePatternSet patterns(context); |
| 2019 | vector::populateVectorNarrowTypeRewritePatterns(patterns); |
| 2020 | vector::populateVectorInterleaveToShufflePatterns(patterns); |
| 2021 | vector::populateVectorDeinterleaveToShufflePatterns(patterns); |
| 2022 | SmallVector<Operation *> casts; |
| 2023 | getOperation()->walk(callback: [&](Operation *op) { |
| 2024 | if (isa<arith::ExtUIOp, arith::ExtSIOp, arith::TruncIOp, |
| 2025 | vector::BitCastOp>(Val: op)) |
| 2026 | casts.push_back(Elt: op); |
| 2027 | }); |
| 2028 | if (failed(Result: applyOpPatternsGreedily(ops: casts, patterns: std::move(patterns), config))) |
| 2029 | return signalPassFailure(); |
| 2030 | } |
| 2031 | |
| 2032 | // XeVM type converter is based on LLVM type converter with the |
| 2033 | // following customizations. |
| 2034 | // First, type conversion rules are added for xegpu custom types, |
| 2035 | // TensorDescType and MemDescType. |
| 2036 | // Second, MemRefType is lowered to single integer type |
| 2037 | // Third, VectorType of single element or 0D is converted to vector |
| 2038 | // element type. Otherwise, vector type is flatten to 1D. |
| 2039 | LowerToLLVMOptions options(context); |
| 2040 | options.overrideIndexBitwidth(bitwidth: this->use64bitIndex ? 64 : 32); |
| 2041 | LLVMTypeConverter typeConverter(context, options); |
| 2042 | |
| 2043 | Type xevmIndexType = typeConverter.convertType(t: IndexType::get(context)); |
| 2044 | Type i32Type = IntegerType::get(context, width: 32); |
| 2045 | typeConverter.addConversion(callback: [&](VectorType type) -> Type { |
| 2046 | auto elemType = typeConverter.convertType(t: type.getElementType()); |
| 2047 | // If the vector rank is 0 or has a single element, return the element |
| 2048 | unsigned rank = type.getRank(); |
| 2049 | if (rank == 0 || type.getNumElements() == 1) |
| 2050 | return elemType; |
| 2051 | // Otherwise, convert the vector to a flat vector type. |
| 2052 | int64_t sum = llvm::product_of(Range: type.getShape()); |
| 2053 | return VectorType::get(shape: sum, elementType: elemType); |
| 2054 | }); |
| 2055 | typeConverter.addConversion(callback: [&](xegpu::TensorDescType type) -> Type { |
| 2056 | if (type.getRank() == 1) |
| 2057 | return xevmIndexType; |
| 2058 | return VectorType::get(shape: 8, elementType: i32Type); |
| 2059 | }); |
| 2060 | // SLM access related type conversions. |
| 2061 | // TODO: LLVM DLTI provides clean way of representing different pointer size |
| 2062 | // based on address space. Currently pointer size of SLM access is hard |
| 2063 | // coded to 32bit. Update to use DLTI when switching overall XeGPU lowering |
| 2064 | // to use DLTI instead of use64bitIndex option used above. |
| 2065 | |
| 2066 | // Convert MemDescType into i32 for SLM |
| 2067 | typeConverter.addConversion( |
| 2068 | callback: [&](xegpu::MemDescType type) -> Type { return i32Type; }); |
| 2069 | |
| 2070 | typeConverter.addConversion(callback: [&](MemRefType type) -> Type { |
| 2071 | return isSharedMemRef(memrefTy: type) ? i32Type : xevmIndexType; |
| 2072 | }); |
| 2073 | |
| 2074 | // LLVM type converter puts unrealized casts for the following cases: |
| 2075 | // add materialization casts to handle them. |
| 2076 | |
| 2077 | // Materialization to convert memref to i64 or i32 depending on global/SLM |
| 2078 | // Applies only to target materialization. |
| 2079 | // Note: int type to memref materialization is not required as xegpu ops |
| 2080 | // currently do not produce memrefs as result. |
| 2081 | auto memrefToIntMaterializationCast = [](OpBuilder &builder, Type type, |
| 2082 | ValueRange inputs, |
| 2083 | Location loc) -> Value { |
| 2084 | if (inputs.size() != 1) |
| 2085 | return {}; |
| 2086 | auto input = inputs.front(); |
| 2087 | if (auto memrefTy = dyn_cast<MemRefType>(Val: input.getType())) { |
| 2088 | unsigned rank = memrefTy.getRank(); |
| 2089 | Type indexType = builder.getIndexType(); |
| 2090 | |
| 2091 | int64_t intOffsets; |
| 2092 | SmallVector<int64_t> intStrides; |
| 2093 | Value addr; |
| 2094 | Value offset; |
| 2095 | if (succeeded(Result: memrefTy.getStridesAndOffset(strides&: intStrides, offset&: intOffsets)) && |
| 2096 | ShapedType::isStatic(dValue: intOffsets)) { |
| 2097 | addr = memref::ExtractAlignedPointerAsIndexOp::create(builder, location: loc, |
| 2098 | source: input); |
| 2099 | offset = arith::ConstantOp::create(builder, location: loc, |
| 2100 | value: builder.getIndexAttr(value: intOffsets)); |
| 2101 | } else { |
| 2102 | |
| 2103 | // Result types: [base_memref, offset, stride0, stride1, ..., |
| 2104 | // strideN-1, size0, size1, ..., sizeN-1] |
| 2105 | SmallVector<Type> resultTypes{ |
| 2106 | MemRefType::get(shape: {}, elementType: memrefTy.getElementType(), |
| 2107 | layout: MemRefLayoutAttrInterface(), |
| 2108 | memorySpace: memrefTy.getMemorySpace()), |
| 2109 | indexType}; |
| 2110 | // strides + sizes |
| 2111 | resultTypes.append(NumInputs: 2 * rank, Elt: indexType); |
| 2112 | |
| 2113 | auto meta = memref::ExtractStridedMetadataOp::create( |
| 2114 | builder, location: loc, resultTypes, source: input); |
| 2115 | |
| 2116 | addr = memref::ExtractAlignedPointerAsIndexOp::create( |
| 2117 | builder, location: loc, source: meta.getBaseBuffer()); |
| 2118 | offset = meta.getOffset(); |
| 2119 | } |
| 2120 | |
| 2121 | auto addrCasted = |
| 2122 | arith::IndexCastUIOp::create(builder, location: loc, out: type, in: addr); |
| 2123 | auto offsetCasted = |
| 2124 | arith::IndexCastUIOp::create(builder, location: loc, out: type, in: offset); |
| 2125 | |
| 2126 | // Compute the final address: base address + byte offset. |
| 2127 | return xegpu::addElemOffsetInBytes(builder, loc, baseAddr: addrCasted, |
| 2128 | elemOffset: offsetCasted, memrefTy); |
| 2129 | } |
| 2130 | return {}; |
| 2131 | }; |
| 2132 | |
| 2133 | // Materialization to convert ui64 to i64 |
| 2134 | // Applies only to target materialization. |
| 2135 | // Note: i64 to ui64 materialization is not required as xegpu ops |
| 2136 | // currently do not produce ui64 as result. |
| 2137 | auto ui64ToI64MaterializationCast = [](OpBuilder &builder, Type type, |
| 2138 | ValueRange inputs, |
| 2139 | Location loc) -> Value { |
| 2140 | if (inputs.size() != 1) |
| 2141 | return {}; |
| 2142 | auto input = inputs.front(); |
| 2143 | if (input.getType() == builder.getIntegerType(width: 64, isSigned: false)) { |
| 2144 | Value cast = |
| 2145 | index::CastUOp::create(builder, location: loc, output: builder.getIndexType(), input) |
| 2146 | .getResult(); |
| 2147 | return arith::IndexCastUIOp::create(builder, location: loc, out: type, in: cast) |
| 2148 | .getResult(); |
| 2149 | } |
| 2150 | return {}; |
| 2151 | }; |
| 2152 | |
| 2153 | // Materialization to convert ui32 to i32 |
| 2154 | // Applies only to target materialization. |
| 2155 | // Note: i32 to ui32 materialization is not required as xegpu ops |
| 2156 | // currently do not produce ui32 as result. |
| 2157 | auto ui32ToI32MaterializationCast = [](OpBuilder &builder, Type type, |
| 2158 | ValueRange inputs, |
| 2159 | Location loc) -> Value { |
| 2160 | if (inputs.size() != 1) |
| 2161 | return {}; |
| 2162 | auto input = inputs.front(); |
| 2163 | if (input.getType() == builder.getIntegerType(width: 32, isSigned: false)) { |
| 2164 | Value cast = |
| 2165 | index::CastUOp::create(builder, location: loc, output: builder.getIndexType(), input) |
| 2166 | .getResult(); |
| 2167 | return arith::IndexCastUIOp::create(builder, location: loc, out: type, in: cast) |
| 2168 | .getResult(); |
| 2169 | } |
| 2170 | return {}; |
| 2171 | }; |
| 2172 | |
| 2173 | // Materialization to convert between vector types |
| 2174 | // - Add shape cast for different shapes |
| 2175 | // - Add bitcast for different element types |
| 2176 | // Applies to both source and target materialization. |
| 2177 | auto vectorToVectorMaterializationCast = [](OpBuilder &builder, Type type, |
| 2178 | ValueRange inputs, |
| 2179 | Location loc) -> Value { |
| 2180 | if (inputs.size() != 1) |
| 2181 | return {}; |
| 2182 | auto input = inputs.front(); |
| 2183 | if (auto vecTy = dyn_cast<VectorType>(Val: input.getType())) { |
| 2184 | if (auto targetVecTy = dyn_cast<VectorType>(Val&: type)) { |
| 2185 | Value cast = input; |
| 2186 | // If the target type has a different shape, add a shape cast |
| 2187 | // If the target type has a different element type, add a bitcast |
| 2188 | if (targetVecTy.getShape() != vecTy.getShape()) { |
| 2189 | cast = vector::ShapeCastOp::create( |
| 2190 | builder, location: loc, |
| 2191 | result: VectorType::get(shape: targetVecTy.getShape(), |
| 2192 | elementType: vecTy.getElementType()), |
| 2193 | source: cast) |
| 2194 | .getResult(); |
| 2195 | } |
| 2196 | if (targetVecTy.getElementType() != vecTy.getElementType()) { |
| 2197 | cast = vector::BitCastOp::create(builder, location: loc, result: targetVecTy, source: cast) |
| 2198 | .getResult(); |
| 2199 | } |
| 2200 | return cast; |
| 2201 | } |
| 2202 | } |
| 2203 | return {}; |
| 2204 | }; |
| 2205 | |
| 2206 | // Materialization to convert |
| 2207 | // - single element vector to single element of vector element type |
| 2208 | // Applies only to target materialization. |
| 2209 | auto vectorToSingleElementMaterializationCast = |
| 2210 | [](OpBuilder &builder, Type type, ValueRange inputs, |
| 2211 | Location loc) -> Value { |
| 2212 | if (inputs.size() != 1) |
| 2213 | return {}; |
| 2214 | auto input = inputs.front(); |
| 2215 | if (auto vecTy = dyn_cast<VectorType>(Val: input.getType())) { |
| 2216 | // Source needs to be single element vector |
| 2217 | auto rank = vecTy.getRank(); |
| 2218 | if (rank != 0 && vecTy.getNumElements() != 1) |
| 2219 | return {}; |
| 2220 | auto inElemTy = vecTy.getElementType(); |
| 2221 | // extract scalar |
| 2222 | Value cast = input; |
| 2223 | if (rank == 0) { |
| 2224 | cast = vector::ExtractOp::create(builder, location: loc, source: cast, position: {}).getResult(); |
| 2225 | } else { |
| 2226 | cast = vector::ExtractOp::create(builder, location: loc, source: cast, |
| 2227 | position: SmallVector<int64_t>(rank, 0)) |
| 2228 | .getResult(); |
| 2229 | } |
| 2230 | // Extracted element type may need conversion |
| 2231 | // Two cases |
| 2232 | // 1. Index type to integer type |
| 2233 | // 2. Other element type mismatch |
| 2234 | if (inElemTy.isIndex()) { |
| 2235 | cast = arith::IndexCastUIOp::create(builder, location: loc, out: type, in: cast) |
| 2236 | .getResult(); |
| 2237 | } else if (inElemTy != type) { |
| 2238 | cast = arith::BitcastOp::create(builder, location: loc, out: type, in: cast).getResult(); |
| 2239 | } |
| 2240 | return cast; |
| 2241 | } |
| 2242 | return {}; |
| 2243 | }; |
| 2244 | |
| 2245 | // Materialization to convert |
| 2246 | // - single element of vector element type to single element vector |
| 2247 | // If result type of original op is single element vector and lowered type |
| 2248 | // is scalar. This materialization cast creates a single element vector by |
| 2249 | // First convert element type if needed and then broadcast to single |
| 2250 | // element vector. |
| 2251 | // Applies only to source materialization. |
| 2252 | auto singleElementToVectorMaterializationCast = |
| 2253 | [](OpBuilder &builder, Type type, ValueRange inputs, |
| 2254 | Location loc) -> Value { |
| 2255 | if (inputs.size() != 1) |
| 2256 | return {}; |
| 2257 | auto input = inputs.front(); |
| 2258 | auto inTy = input.getType(); |
| 2259 | if (!inTy.isIntOrFloat()) |
| 2260 | return {}; |
| 2261 | // If the target type is a vector of rank 0 or single element vector |
| 2262 | // of element type matching input type, broadcast input to target type. |
| 2263 | if (auto vecTy = dyn_cast<VectorType>(Val&: type)) { |
| 2264 | if (vecTy.getRank() != 0 && vecTy.getNumElements() != 1) |
| 2265 | return {}; |
| 2266 | auto outElemTy = vecTy.getElementType(); |
| 2267 | Value cast = input; |
| 2268 | if (outElemTy.isIndex()) { |
| 2269 | cast = arith::IndexCastUIOp::create(builder, location: loc, |
| 2270 | out: builder.getIndexType(), in: cast) |
| 2271 | .getResult(); |
| 2272 | } else if (inTy != outElemTy) { |
| 2273 | cast = arith::BitcastOp::create(builder, location: loc, out: outElemTy, in: cast) |
| 2274 | .getResult(); |
| 2275 | } |
| 2276 | return vector::BroadcastOp::create(builder, location: loc, result: vecTy, source: cast) |
| 2277 | .getResult(); |
| 2278 | } |
| 2279 | return {}; |
| 2280 | }; |
| 2281 | typeConverter.addSourceMaterialization( |
| 2282 | callback&: singleElementToVectorMaterializationCast); |
| 2283 | typeConverter.addSourceMaterialization(callback&: vectorToVectorMaterializationCast); |
| 2284 | typeConverter.addTargetMaterialization(callback&: memrefToIntMaterializationCast); |
| 2285 | typeConverter.addTargetMaterialization(callback&: ui32ToI32MaterializationCast); |
| 2286 | typeConverter.addTargetMaterialization(callback&: ui64ToI64MaterializationCast); |
| 2287 | typeConverter.addTargetMaterialization( |
| 2288 | callback&: vectorToSingleElementMaterializationCast); |
| 2289 | typeConverter.addTargetMaterialization(callback&: vectorToVectorMaterializationCast); |
| 2290 | ConversionTarget target(*context); |
| 2291 | target.addLegalDialect<xevm::XeVMDialect, LLVM::LLVMDialect, |
| 2292 | vector::VectorDialect, arith::ArithDialect, |
| 2293 | memref::MemRefDialect, gpu::GPUDialect, |
| 2294 | index::IndexDialect>(); |
| 2295 | target.addIllegalDialect<xegpu::XeGPUDialect>(); |
| 2296 | // arith.extf/arith.truncf between MX narrow floats and f16/bf16 are routed |
| 2297 | // to xevm.extf/xevm.truncf; all other arith float casts stay legal. |
| 2298 | target.addDynamicallyLegalOp<arith::ExtFOp>( |
| 2299 | callback: [](arith::ExtFOp op) { return !isXeVMExtf(op); }); |
| 2300 | target.addDynamicallyLegalOp<arith::TruncFOp>( |
| 2301 | callback: [](arith::TruncFOp op) { return !isXeVMTruncf(op); }); |
| 2302 | |
| 2303 | RewritePatternSet patterns(context); |
| 2304 | populateXeGPUToXeVMConversionPatterns(typeConverter, patterns); |
| 2305 | scf::populateSCFStructuralTypeConversionsAndLegality(typeConverter, |
| 2306 | patterns, target); |
| 2307 | if (failed(Result: applyPartialConversion(op: getOperation(), target, |
| 2308 | patterns: std::move(patterns)))) |
| 2309 | signalPassFailure(); |
| 2310 | } |
| 2311 | }; |
| 2312 | } // namespace |
| 2313 | |
| 2314 | //===----------------------------------------------------------------------===// |
| 2315 | // Pattern Population |
| 2316 | //===----------------------------------------------------------------------===// |
| 2317 | void mlir::populateXeGPUToXeVMConversionPatterns( |
| 2318 | const LLVMTypeConverter &typeConverter, RewritePatternSet &patterns) { |
| 2319 | patterns.add<CreateNdDescToXeVMPattern, |
| 2320 | LoadStorePrefetchNdToXeVMPattern<xegpu::LoadNdOp>, |
| 2321 | LoadStorePrefetchNdToXeVMPattern<xegpu::StoreNdOp>, |
| 2322 | LoadStorePrefetchNdToXeVMPattern<xegpu::PrefetchNdOp>>( |
| 2323 | arg: typeConverter, args: patterns.getContext()); |
| 2324 | patterns.add<AtomicRMWToXeVMPattern, PrefetchToXeVMPattern, |
| 2325 | LoadStoreToXeVMPattern<xegpu::LoadGatherOp>, |
| 2326 | LoadStoreToXeVMPattern<xegpu::StoreScatterOp>>( |
| 2327 | arg: typeConverter, args: patterns.getContext()); |
| 2328 | patterns.add<LoadStoreMatrixToXeVMPattern<xegpu::LoadMatrixOp>, |
| 2329 | LoadStoreMatrixToXeVMPattern<xegpu::StoreMatrixOp>, |
| 2330 | CreateMemDescOpPattern>(arg: typeConverter, args: patterns.getContext()); |
| 2331 | patterns.add<FenceToXeVMPattern, DpasToXeVMPattern>(arg: typeConverter, |
| 2332 | args: patterns.getContext()); |
| 2333 | patterns.add<DpasMxToXeVMPattern>(arg: typeConverter, args: patterns.getContext()); |
| 2334 | patterns.add<ExtfToXeVMPattern, TruncfToXeVMPattern>(arg: typeConverter, |
| 2335 | args: patterns.getContext()); |
| 2336 | patterns.add<LaneShuffleToXeVMPattern>(arg: typeConverter, args: patterns.getContext()); |
| 2337 | } |
| 2338 | |