1//===-- XeGPUToXeVM.cpp - XeGPU to XeVM dialect conversion ------*- C++ -*-===//
2//
3// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/XeGPUToXeVM/XeGPUToXeVM.h"
10#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
11#include "mlir/Dialect/LLVMIR/XeVMDialect.h"
12
13#include "mlir/Conversion/LLVMCommon/Pattern.h"
14#include "mlir/Dialect/Arith/IR/Arith.h"
15#include "mlir/Dialect/GPU/IR/GPUDialect.h"
16#include "mlir/Dialect/Index/IR/IndexDialect.h"
17#include "mlir/Dialect/Index/IR/IndexOps.h"
18#include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
19#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
20#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
21#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
22#include "mlir/Dialect/MemRef/IR/MemRef.h"
23#include "mlir/Dialect/SCF/IR/SCF.h"
24#include "mlir/Dialect/SCF/Transforms/Patterns.h"
25#include "mlir/Dialect/Utils/IndexingUtils.h"
26#include "mlir/Dialect/Vector/IR/VectorOps.h"
27#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
28#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
29#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
30#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
31#include "mlir/Dialect/XeGPU/uArch/uArchCommon.h"
32#include "mlir/IR/IRMapping.h"
33#include "mlir/Pass/Pass.h"
34#include "mlir/Support/LLVM.h"
35#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
36#include "llvm/ADT/STLExtras.h"
37#include "llvm/ADT/SmallPtrSet.h"
38#include "llvm/Support/FormatVariadic.h"
39
40#include "mlir/IR/BuiltinTypes.h"
41#include "mlir/IR/Types.h"
42
43#include "llvm/ADT/TypeSwitch.h"
44
45#include <string>
46
47namespace mlir {
48#define GEN_PASS_DEF_CONVERTXEGPUTOXEVMPASS
49#include "mlir/Conversion/Passes.h.inc"
50} // namespace mlir
51
52using namespace mlir;
53
54namespace {
55// TODO: Below are uArch dependent values, should move away from hardcoding
56static constexpr int32_t systolicDepth{8};
57static constexpr int32_t executionSize{16};
58
59// Offsets to individual fields of the 8xi32 layout nd tensor descriptor.
60enum class NdTdescOffset : uint32_t {
61 BasePtr = 0, // Base pointer (i64)
62 BaseShapeW = 2, // Base shape width (i32)
63 BaseShapeH = 3, // Base shape height (i32)
64 BasePitch = 4, // Base pitch/stride of dim rank-2 (i32)
65 LeadingStride0 = 5, // Row strides of the leading (batch) dims of a >2D
66 LeadingStride1 = 6, // descriptor (i32); added into offset_h by the load/store
67 LeadingStride2 = 7, // lowering. Left at 0 for 2D descriptors.
68};
69
70// Spare payload slots above, and the resulting max lowerable descriptor rank.
71static constexpr int64_t maxNdTdescLeadingDims{3};
72static constexpr int64_t maxNdTdescRank{2 + maxNdTdescLeadingDims};
73
74static int32_t getNumericXeVMAddrSpace(xegpu::MemorySpace xeGpuMemspace) {
75 switch (xeGpuMemspace) {
76 case xegpu::MemorySpace::Global:
77 return static_cast<int>(xevm::AddrSpace::GLOBAL);
78 case xegpu::MemorySpace::SLM:
79 return static_cast<int>(xevm::AddrSpace::SHARED);
80 }
81 llvm_unreachable("Unknown XeGPU memory space");
82}
83
84/// Translates a memref memory space attribute into XeVM's numeric address
85/// space, which follows the OpenCL/SPIR-V convention (0 = private, 1 =
86/// global, 2 = constant, 3 = shared/local, 4 = generic). A null attribute,
87/// meaning the memory space was left unspecified, maps to the default space
88/// 0. Returns failure if `memSpace` is a representation this pass does not
89/// know how to translate (e.g. a SPIR-V storage class or an arbitrary string
90/// attribute), rather than assuming it is an `IntegerAttr` and asserting.
91static FailureOr<unsigned> getNumericMemorySpace(Attribute memSpace) {
92 if (!memSpace)
93 return 0u;
94 if (auto intAttr = llvm::dyn_cast<IntegerAttr>(Val&: memSpace))
95 return static_cast<unsigned>(intAttr.getInt());
96 if (auto xevmSpace = llvm::dyn_cast<xevm::AddrSpaceAttr>(Val&: memSpace))
97 return static_cast<unsigned>(xevmSpace.getValue());
98 if (auto gpuSpace = llvm::dyn_cast<gpu::AddressSpaceAttr>(Val&: memSpace)) {
99 switch (gpuSpace.getValue()) {
100 case gpu::AddressSpace::Global:
101 return static_cast<unsigned>(xevm::AddrSpace::GLOBAL);
102 case gpu::AddressSpace::Workgroup:
103 return static_cast<unsigned>(xevm::AddrSpace::SHARED);
104 case gpu::AddressSpace::Private:
105 return static_cast<unsigned>(xevm::AddrSpace::PRIVATE);
106 case gpu::AddressSpace::Constant:
107 return static_cast<unsigned>(xevm::AddrSpace::CONSTANT);
108 }
109 llvm_unreachable("Unknown GPU address space");
110 }
111 return failure();
112}
113
114/// Checks if the given MemRefType refers to shared memory.
115static bool isSharedMemRef(const MemRefType &memrefTy) {
116 FailureOr<unsigned> addrSpace =
117 getNumericMemorySpace(memSpace: memrefTy.getMemorySpace());
118 return succeeded(Result: addrSpace) &&
119 *addrSpace == static_cast<unsigned>(xevm::AddrSpace::SHARED);
120}
121
122// Get same bitwidth flat vector type of new element type.
123static VectorType encodeVectorTypeTo(VectorType currentVecType,
124 Type toElemType) {
125 auto elemType = currentVecType.getElementType();
126 auto currentBitWidth = elemType.getIntOrFloatBitWidth();
127 auto newBitWidth = toElemType.getIntOrFloatBitWidth();
128 const int size =
129 currentVecType.getNumElements() * currentBitWidth / newBitWidth;
130 return VectorType::get(shape: size, elementType: toElemType);
131}
132
133static xevm::LoadCacheControl
134translateLoadXeGPUCacheHint(std::optional<xegpu::CachePolicy> L1hint,
135 std::optional<xegpu::CachePolicy> L3hint) {
136 // If no hints are provided, use the default cache control.
137 if (!L1hint && !L3hint)
138 return xevm::LoadCacheControl::USE_DEFAULT;
139 // If only one of the hints is provided, use the default for the other level.
140 auto L1hintVal = L1hint.value_or(u: xegpu::CachePolicy::CACHED);
141 auto L3hintVal = L3hint.value_or(u: xegpu::CachePolicy::CACHED);
142 switch (L1hintVal) {
143 case xegpu::CachePolicy::CACHED:
144 if (L3hintVal == xegpu::CachePolicy::CACHED)
145 return xevm::LoadCacheControl::L1C_L2UC_L3C;
146 else if (L3hintVal == xegpu::CachePolicy::UNCACHED)
147 return xevm::LoadCacheControl::L1C_L2UC_L3UC;
148 else
149 llvm_unreachable("Unsupported cache control.");
150 case xegpu::CachePolicy::UNCACHED:
151 if (L3hintVal == xegpu::CachePolicy::CACHED)
152 return xevm::LoadCacheControl::L1UC_L2UC_L3C;
153 else if (L3hintVal == xegpu::CachePolicy::UNCACHED)
154 return xevm::LoadCacheControl::L1UC_L2UC_L3UC;
155 else
156 llvm_unreachable("Unsupported cache control.");
157 case xegpu::CachePolicy::STREAMING:
158 if (L3hintVal == xegpu::CachePolicy::CACHED)
159 return xevm::LoadCacheControl::L1S_L2UC_L3C;
160 else if (L3hintVal == xegpu::CachePolicy::UNCACHED)
161 return xevm::LoadCacheControl::L1S_L2UC_L3UC;
162 else
163 llvm_unreachable("Unsupported cache control.");
164 case xegpu::CachePolicy::READ_INVALIDATE:
165 return xevm::LoadCacheControl::INVALIDATE_READ;
166 default:
167 llvm_unreachable("Unsupported cache control.");
168 }
169}
170
171static xevm::StoreCacheControl
172translateStoreXeGPUCacheHint(std::optional<xegpu::CachePolicy> L1hint,
173 std::optional<xegpu::CachePolicy> L3hint) {
174 // If no hints are provided, use the default cache control.
175 if (!L1hint && !L3hint)
176 return xevm::StoreCacheControl::USE_DEFAULT;
177 // If only one of the hints is provided, use the default for the other level.
178 auto L1hintVal = L1hint.value_or(u: xegpu::CachePolicy::UNCACHED);
179 auto L3hintVal = L3hint.value_or(u: xegpu::CachePolicy::WRITE_BACK);
180 switch (L1hintVal) {
181 case xegpu::CachePolicy::UNCACHED:
182 if (L3hintVal == xegpu::CachePolicy::UNCACHED)
183 return xevm::StoreCacheControl::L1UC_L2UC_L3UC;
184 else if (L3hintVal == xegpu::CachePolicy::WRITE_BACK)
185 return xevm::StoreCacheControl::L1UC_L2UC_L3WB;
186 else
187 llvm_unreachable("Unsupported cache control.");
188 case xegpu::CachePolicy::STREAMING:
189 if (L3hintVal == xegpu::CachePolicy::UNCACHED)
190 return xevm::StoreCacheControl::L1S_L2UC_L3UC;
191 else if (L3hintVal == xegpu::CachePolicy::WRITE_BACK)
192 return xevm::StoreCacheControl::L1S_L2UC_L3WB;
193 else
194 llvm_unreachable("Unsupported cache control.");
195 case xegpu::CachePolicy::WRITE_BACK:
196 if (L3hintVal == xegpu::CachePolicy::UNCACHED)
197 return xevm::StoreCacheControl::L1WB_L2UC_L3UC;
198 else if (L3hintVal == xegpu::CachePolicy::WRITE_BACK)
199 return xevm::StoreCacheControl::L1WB_L2UC_L3WB;
200 else
201 llvm_unreachable("Unsupported cache control.");
202 case xegpu::CachePolicy::WRITE_THROUGH:
203 if (L3hintVal == xegpu::CachePolicy::UNCACHED)
204 return xevm::StoreCacheControl::L1WT_L2UC_L3UC;
205 else if (L3hintVal == xegpu::CachePolicy::WRITE_BACK)
206 return xevm::StoreCacheControl::L1WT_L2UC_L3WB;
207 else
208 llvm_unreachable("Unsupported cache control.");
209 default:
210 llvm_unreachable("Unsupported cache control.");
211 }
212}
213
214//
215// Note:
216// Block operations for tile of sub byte element types are handled by
217// emulating with larger element types.
218// Tensor descriptor are keep intact and only ops consuming them are
219// emulated
220//
221
222//
223// High-D (>2D) nd descriptors are lowered by viewing the source as a single
224// flattened 2D plane, so the 2D-block surface covers every leading (batch)
225// plane at once and a batch position becomes a row offset into it. The surface
226// height is the row extent the leading dims reach:
227// base_height = size[R-2] + sum_d (size[d] - 1) * (stride[d] / stride[R-2])
228//
229// Leaving `base_ptr` at the true base means an out-of-range batch index lands
230// past `base_height`, where the HW boundary check handles it, instead of aiming
231// the surface at unmapped memory. Encoding the leading strides as row counts
232// (`stride[d] / stride[R-2]`) also makes them dimensionless, so they survive
233// element-type repacking (e.g. the f16 -> i32 transpose repack) without a unit
234// conversion.
235//
236// Limitations of the flattened-plane view:
237// 1. Each leading stride must be a whole number of rows. A source with gaps
238// between planes (`stride[d] % stride[R-2] != 0`) is not lowered. This can
239// only be checked when the strides are static; for dynamic strides the
240// divisibility is assumed.
241// 2. `base_height` grows with the leading extent, so a source with a large
242// batch x head x sequence extent can exceed the HW 2D-block surface height.
243// 3. Plane boundaries are invisible to the boundary check, which only knows
244// the surface: a tile whose rows run past `size[R-2]` spills into the
245// following rows of the surface -- the next plane, or the padding between
246// planes -- rather than being clipped there as it would be on a per-plane
247// surface. A load reads those rows instead of returning zeros, and a store
248// overwrites them, so it can corrupt the next plane. This only matters
249// when `size[R-2]` is not a multiple of the tile height.
250//
251
252class CreateNdDescToXeVMPattern
253 : public OpConversionPattern<xegpu::CreateNdDescOp> {
254 using Base::Base;
255
256 // Checks the surface width and pitch against the target's 2D block memory
257 // restrictions. Only these two are enforced: both are backed by the source's
258 // static shape and strides, whereas the base address alignment is a property
259 // of the allocation that is not visible here. The width and the pitch are
260 // checked independently, each only when it is static; a value known only at
261 // runtime is left alone, as guarding it inside the kernel would cost far more
262 // than it is worth.
263 //
264 // A violation is a dead end rather than a pattern that does not apply: the op
265 // has no other legalization path, so it is reported with an error that names
266 // the limit it misses.
267 LogicalResult checkMemoryRestrictions(xegpu::CreateNdDescOp op,
268 int64_t rank) const {
269 const auto *uArch =
270 xegpu::uArch::getUArch(archName: xegpu::getChipStr(op).value_or(u: ""));
271 // Without a known target chip there is no uArch to check against, so the
272 // descriptor must be accepted as-is.
273 if (!uArch)
274 return success();
275 const xegpu::uArch::BlockIOMemoryRestrictions &limits =
276 uArch->getBlockIOMemoryRestrictions();
277
278 // Recover the width and the pitch as a number of elements. A memref source
279 // carries both in its type, and either one may be dynamic independently of
280 // the other, so they are queried separately instead of being dropped
281 // together when the source is not fully static. `getMixedSizes` /
282 // `getMixedStrides` are only consulted for an integer source, where the op
283 // itself holds the extents.
284 std::optional<int64_t> widthElems;
285 std::optional<int64_t> pitchElems;
286 if (auto memrefTy = dyn_cast<MemRefType>(Val: op.getSource().getType())) {
287 if (!ShapedType::isDynamic(dValue: memrefTy.getDimSize(idx: rank - 1)))
288 widthElems = memrefTy.getDimSize(idx: rank - 1);
289 // The source is known to be strided; the caller checked that already.
290 SmallVector<int64_t> staticStrides;
291 int64_t staticOffset;
292 if (succeeded(
293 Result: memrefTy.getStridesAndOffset(strides&: staticStrides, offset&: staticOffset)) &&
294 !ShapedType::isDynamic(dValue: staticStrides[rank - 2]))
295 pitchElems = staticStrides[rank - 2];
296 } else {
297 widthElems = getConstantIntValue(ofr: op.getMixedSizes()[rank - 1]);
298 pitchElems = getConstantIntValue(ofr: op.getMixedStrides()[rank - 2]);
299 }
300
301 unsigned elemBitSize =
302 op.getType().getElementType().getIntOrFloatBitWidth();
303 // The restrictions are expressed in bytes, but a sub-byte element type can
304 // leave the width or the pitch on a fraction of a byte, so the comparisons
305 // are done in bits. Such a size meets no byte alignment at all and is
306 // reported as a violation rather than waved through.
307 auto describe = [](int64_t bits) -> std::string {
308 if (bits % 8 == 0)
309 return llvm::formatv(Fmt: "{0} bytes", Vals: bits / 8).str();
310 return llvm::formatv(Fmt: "{0} bits", Vals&: bits).str();
311 };
312 auto check = [&](std::optional<int64_t> elems, int64_t minBytes,
313 int64_t alignBytes,
314 llvm::StringRef what) -> LogicalResult {
315 if (!elems)
316 return success();
317 int64_t bits = *elems * elemBitSize;
318 if (bits < minBytes * 8)
319 return op.emitError(message: llvm::formatv(
320 Fmt: "2D block surface {0} must be at least {1} bytes on this "
321 "target, got {2}.",
322 Vals&: what, Vals&: minBytes, Vals: describe(bits)));
323 if (alignBytes > 0 && bits % (alignBytes * 8) != 0)
324 return op.emitError(message: llvm::formatv(
325 Fmt: "2D block surface {0} must be a multiple of {1} bytes on this "
326 "target, got {2}.",
327 Vals&: what, Vals&: alignBytes, Vals: describe(bits)));
328 return success();
329 };
330
331 if (failed(Result: check(widthElems, limits.minBaseWidthBytes,
332 limits.baseWidthAlignmentBytes, "width")))
333 return failure();
334 // The pitch is the row stride of the innermost 2D tile.
335 if (failed(Result: check(pitchElems, limits.minBasePitchBytes,
336 limits.basePitchAlignmentBytes, "pitch")))
337 return failure();
338 // Consecutive rows may not overlap, so the pitch has to span the full
339 // width.
340 if (widthElems && pitchElems && *pitchElems < *widthElems)
341 return op.emitError(
342 message: llvm::formatv(Fmt: "2D block surface pitch must be at least the surface "
343 "width, got a pitch of {0} for a surface {1} wide.",
344 Vals: describe(*pitchElems * elemBitSize),
345 Vals: describe(*widthElems * elemBitSize)));
346 return success();
347 }
348
349 LogicalResult
350 matchAndRewrite(xegpu::CreateNdDescOp op,
351 xegpu::CreateNdDescOp::Adaptor adaptor,
352 ConversionPatternRewriter &rewriter) const override {
353 auto loc = op.getLoc();
354 auto source = op.getSource();
355
356 // Check all failure conditions before generating any IR, so nothing has to
357 // be rolled back.
358 int64_t rank = op.getType().getRank();
359 int64_t sourceRank;
360 auto memrefTy = dyn_cast<MemRefType>(Val: source.getType());
361 if (memrefTy) {
362 if (!memrefTy.isStrided())
363 return rewriter.notifyMatchFailure(arg&: op, msg: "Expected strided Memref.");
364 sourceRank = memrefTy.getRank();
365 } else if (isa<IntegerType>(Val: source.getType())) {
366 sourceRank = op.getMixedSizes().size();
367 } else {
368 return rewriter.notifyMatchFailure(arg&: op,
369 msg: "Expected ranked Memref or integer.");
370 }
371 if (sourceRank != rank)
372 return rewriter.notifyMatchFailure(
373 arg&: op, msg: "Expected descriptor rank to match source rank; subview the "
374 "source down to the descriptor rank.");
375 if (rank > maxNdTdescRank)
376 return rewriter.notifyMatchFailure(
377 arg&: op, msg: "Batched nd descriptor supports at most " +
378 std::to_string(val: maxNdTdescLeadingDims) +
379 " leading dims (rank <= " + std::to_string(val: maxNdTdescRank) +
380 ").");
381 // Limitation 1 above; dynamic strides are assumed to divide evenly.
382 if (rank > 2) {
383 SmallVector<std::optional<int64_t>> constStrides(rank, std::nullopt);
384 if (memrefTy) {
385 SmallVector<int64_t> staticStrides;
386 int64_t staticOffset;
387 if (succeeded(
388 Result: memrefTy.getStridesAndOffset(strides&: staticStrides, offset&: staticOffset)))
389 for (int64_t d = 0; d < rank; ++d)
390 if (!ShapedType::isDynamic(dValue: staticStrides[d]))
391 constStrides[d] = staticStrides[d];
392 } else {
393 SmallVector<OpFoldResult> mixed = op.getMixedStrides();
394 for (int64_t d = 0; d < rank; ++d)
395 constStrides[d] = getConstantIntValue(ofr: mixed[d]);
396 }
397 if (std::optional<int64_t> pitch = constStrides[rank - 2]) {
398 for (int64_t d = 0; d < rank - 2; ++d) {
399 std::optional<int64_t> leading = constStrides[d];
400 if (leading && (*pitch == 0 || *leading % *pitch != 0))
401 return rewriter.notifyMatchFailure(
402 arg&: op, msg: "Expected each leading (batch) stride to be a multiple of "
403 "the row stride; the source has gaps between planes.");
404 }
405 }
406 }
407 // Enforce the target's 2D block memory restrictions. A rank-1 descriptor is
408 // just a base address and is not consumed by a 2D block op, so it is
409 // exempt.
410 if (rank >= 2 && failed(Result: checkMemoryRestrictions(op, rank)))
411 return failure();
412
413 Type payloadElemTy = rewriter.getI32Type();
414 Type i64Ty = rewriter.getI64Type();
415
416 // Access the adaptor only after the failure checks, so a bail-out leaves no
417 // materialization cast behind.
418 Value baseAddr = adaptor.getSource();
419 if (isa<IntegerType>(Val: source.getType()) && baseAddr.getType() != i64Ty) {
420 // Pointer type may be i32. Cast to i64 if needed.
421 baseAddr = arith::ExtUIOp::create(builder&: rewriter, location: loc, out: i64Ty, in: baseAddr);
422 }
423 // 1D tensor descriptor is just the base address.
424 if (rank == 1) {
425 rewriter.replaceOp(op, newValues: baseAddr);
426 return success();
427 }
428
429 SmallVector<OpFoldResult> mixedSizes;
430 SmallVector<OpFoldResult> mixedStrides;
431 if (memrefTy && !xegpu::hasStaticShapeAndStrides(type: memrefTy)) {
432 auto meta =
433 memref::ExtractStridedMetadataOp::create(builder&: rewriter, location: loc, source);
434 mixedSizes = meta.getConstifiedMixedSizes();
435 mixedStrides = meta.getConstifiedMixedStrides();
436 } else {
437 mixedSizes = op.getMixedSizes();
438 mixedStrides = op.getMixedStrides();
439 }
440
441 // Op is lowered to a code sequence that populates payload.
442 // Payload is a 8xi32 vector. Offset to individual fields are defined in
443 // NdTdescOffset enum.
444 VectorType payloadTy = VectorType::get(shape: 8, elementType: payloadElemTy);
445 // 4xi64 view is used for inserting the base pointer.
446 VectorType payloadI64Ty = VectorType::get(shape: 4, elementType: i64Ty);
447 // Initialize payload to zero.
448 Value payload = arith::ConstantOp::create(
449 builder&: rewriter, location: loc,
450 value: DenseElementsAttr::get(type: payloadTy, values: IntegerAttr::get(type: payloadElemTy, value: 0)));
451
452 // Utility for creating offset values from op fold result.
453 auto createOffset = [&](SmallVector<OpFoldResult> &ofrVec,
454 unsigned idx) -> Value {
455 Value val = getValueOrCreateConstantIntOp(b&: rewriter, loc, ofr: ofrVec[idx]);
456 val = getValueOrCreateCastToIndexLike(b&: rewriter, loc, targetType: payloadElemTy, value: val);
457 return val;
458 };
459 // The descriptor's innermost 2 dims are the 2D tile (H, W).
460 Value baseShapeW = createOffset(mixedSizes, rank - 1);
461 // Pitch is the stride of dim rank-2 (the row stride of the 2D tile).
462 Value basePitch = createOffset(mixedStrides, rank - 2);
463 // Leading (batch) strides as a number of rows (`stride[d] / pitch`).
464 SmallVector<Value> leadingRowStrides;
465 for (int64_t d = 0; d < rank - 2; ++d) {
466 std::optional<int64_t> leading = getConstantIntValue(ofr: mixedStrides[d]);
467 std::optional<int64_t> pitch =
468 getConstantIntValue(ofr: mixedStrides[rank - 2]);
469 if (leading && pitch && *pitch != 0)
470 leadingRowStrides.push_back(Elt: arith::ConstantIntOp::create(
471 builder&: rewriter, location: loc, type: payloadElemTy, value: *leading / *pitch));
472 else
473 leadingRowStrides.push_back(Elt: arith::DivUIOp::create(
474 builder&: rewriter, location: loc, lhs: createOffset(mixedStrides, d), rhs: basePitch));
475 }
476 // Height of the flattened plane: every leading (batch) plane is stacked
477 // into the surface, so the boundary check covers an out-of-range batch.
478 // base_height = size[R-2] + sum_d (size[d] - 1) * leadingRowStride[d]
479 // For rank 2 this is just size[0].
480 Value baseShapeH = createOffset(mixedSizes, rank - 2);
481 if (rank > 2) {
482 Value one = arith::ConstantIntOp::create(builder&: rewriter, location: loc, type: payloadElemTy, value: 1);
483 for (int64_t d = 0; d < rank - 2; ++d) {
484 Value planesBelow = rewriter.createOrFold<arith::SubIOp>(
485 location: loc, args: createOffset(mixedSizes, d), args&: one);
486 Value rows = rewriter.createOrFold<arith::MulIOp>(location: loc, args&: planesBelow,
487 args&: leadingRowStrides[d]);
488 baseShapeH =
489 rewriter.createOrFold<arith::AddIOp>(location: loc, args&: baseShapeH, args&: rows);
490 }
491 }
492 // Populate payload.
493 Value payLoadAsI64 =
494 vector::BitCastOp::create(builder&: rewriter, location: loc, result: payloadI64Ty, source: payload);
495 payLoadAsI64 =
496 vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: baseAddr, dest: payLoadAsI64,
497 position: static_cast<int>(NdTdescOffset::BasePtr));
498 payload = vector::BitCastOp::create(builder&: rewriter, location: loc, result: payloadTy, source: payLoadAsI64);
499 payload =
500 vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: baseShapeW, dest: payload,
501 position: static_cast<int>(NdTdescOffset::BaseShapeW));
502 payload =
503 vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: baseShapeH, dest: payload,
504 position: static_cast<int>(NdTdescOffset::BaseShapeH));
505 payload =
506 vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: basePitch, dest: payload,
507 position: static_cast<int>(NdTdescOffset::BasePitch));
508 // The leading (batch) row strides go into the spare payload slots; the
509 // load/store/prefetch lowering turns the batch offsets into a row offset
510 // with them.
511 for (int64_t d = 0; d < rank - 2; ++d)
512 payload = vector::InsertOp::create(
513 builder&: rewriter, location: loc, valueToStore: leadingRowStrides[d], dest: payload,
514 position: static_cast<int>(NdTdescOffset::LeadingStride0) + d);
515 rewriter.replaceOp(op, newValues: payload);
516 return success();
517 }
518};
519
520template <
521 typename OpType,
522 typename = std::enable_if_t<llvm::is_one_of<
523 OpType, xegpu::LoadNdOp, xegpu::StoreNdOp, xegpu::PrefetchNdOp>::value>>
524class LoadStorePrefetchNdToXeVMPattern : public OpConversionPattern<OpType> {
525 using OpConversionPattern<OpType>::OpConversionPattern;
526 LogicalResult
527 matchAndRewrite(OpType op, typename OpType::Adaptor adaptor,
528 ConversionPatternRewriter &rewriter) const override {
529 auto mixedOffsets = op.getMixedOffsets();
530 int64_t opOffsetsSize = mixedOffsets.size();
531 auto loc = op.getLoc();
532 auto ctxt = rewriter.getContext();
533
534 auto tdesc = adaptor.getTensorDesc();
535 auto tdescTy = op.getTensorDescType();
536 auto tileRank = tdescTy.getRank();
537 if (opOffsetsSize != tileRank)
538 return rewriter.notifyMatchFailure(
539 op, "Expected offset rank to match descriptor rank.");
540 if (tileRank > 2 && llvm::any_of(tdescTy.getShape().drop_back(2),
541 [](int64_t d) { return d != 1; }))
542 return rewriter.notifyMatchFailure(
543 op, "Expected leading (batch) descriptor dims to be unit.");
544 if (tileRank > maxNdTdescRank)
545 return rewriter.notifyMatchFailure(
546 op, "Expected descriptor rank <= " + std::to_string(val: maxNdTdescRank) +
547 ".");
548 auto elemType = tdescTy.getElementType();
549 auto elemBitSize = elemType.getIntOrFloatBitWidth();
550 bool isSubByte = elemBitSize < 8;
551 uint64_t wScaleFactor = 1;
552
553 if (!isSubByte && (elemBitSize % 8 != 0))
554 return rewriter.notifyMatchFailure(
555 op, "Expected element type bit width to be multiple of 8.");
556 auto tileW = tdescTy.getDimSize(tileRank - 1);
557 // For sub byte types, only 4bits are currently supported.
558 if (isSubByte) {
559 if (elemBitSize != 4)
560 return rewriter.notifyMatchFailure(
561 op, "Only sub byte types of 4bits are supported.");
562 if (tileRank != 2)
563 return rewriter.notifyMatchFailure(
564 op, "Sub byte types are only supported for 2D tensor descriptors.");
565 auto subByteFactor = 8 / elemBitSize;
566 auto tileH = tdescTy.getDimSize(0);
567 // Handle special case for packed load.
568 if constexpr (std::is_same_v<OpType, xegpu::LoadNdOp>) {
569 if (op.getPacked().value_or(false)) {
570 // packed load is implemented as packed loads of 8bit elements.
571 if (tileH == systolicDepth * 4 &&
572 tileW == executionSize * subByteFactor) {
573 // Usage case for loading as Matrix B with pack request.
574 // source is assumed to pre-packed into 8bit elements
575 // Emulate with 8bit loads with pack request.
576 // scaled_tileW = executionSize
577 elemType = rewriter.getIntegerType(width: 8);
578 tileW = executionSize;
579 wScaleFactor = subByteFactor;
580 }
581 }
582 }
583 // If not handled by packed load case above, handle other cases.
584 if (wScaleFactor == 1) {
585 auto sub16BitFactor = subByteFactor * 2;
586 if (tileW == executionSize * sub16BitFactor) {
587 // Usage case for loading as Matrix A operand
588 // Emulate with 16bit loads/stores.
589 // scaled_tileW = executionSize
590 elemType = rewriter.getIntegerType(width: 16);
591 tileW = executionSize;
592 wScaleFactor = sub16BitFactor;
593 } else {
594 return rewriter.notifyMatchFailure(
595 op, "Unsupported tile shape for sub byte types.");
596 }
597 }
598 // recompute element bit size for emulation.
599 elemBitSize = elemType.getIntOrFloatBitWidth();
600 }
601
602 // Get address space from tensor descriptor memory space.
603 auto ptrTypeLLVM = LLVM::LLVMPointerType::get(
604 context: ctxt, addressSpace: getNumericXeVMAddrSpace(tdescTy.getMemorySpace()));
605 if (tileRank >= 2) {
606 // Compute element byte size.
607 Value elemByteSize = arith::ConstantIntOp::create(
608 rewriter, loc, rewriter.getI32Type(), elemBitSize / 8);
609 VectorType payloadI64Ty = VectorType::get(shape: 4, elementType: rewriter.getI64Type());
610 Value payLoadAsI64 =
611 vector::BitCastOp::create(rewriter, loc, payloadI64Ty, tdesc);
612 Value basePtr =
613 vector::ExtractOp::create(rewriter, loc, payLoadAsI64,
614 static_cast<int>(NdTdescOffset::BasePtr));
615 Value baseShapeW = vector::ExtractOp::create(
616 rewriter, loc, tdesc, static_cast<int>(NdTdescOffset::BaseShapeW));
617 Value baseShapeH = vector::ExtractOp::create(
618 rewriter, loc, tdesc, static_cast<int>(NdTdescOffset::BaseShapeH));
619 Value basePitch = vector::ExtractOp::create(
620 rewriter, loc, tdesc, static_cast<int>(NdTdescOffset::BasePitch));
621
622 Value offsetW = getValueOrCreateConstantIntOp(rewriter, loc,
623 mixedOffsets[tileRank - 1]);
624 offsetW = getValueOrCreateCastToIndexLike(rewriter, loc,
625 rewriter.getI32Type(), offsetW);
626 Value offsetH = getValueOrCreateConstantIntOp(rewriter, loc,
627 mixedOffsets[tileRank - 2]);
628 offsetH = getValueOrCreateCastToIndexLike(rewriter, loc,
629 rewriter.getI32Type(), offsetH);
630 // Turn the leading (batch) offsets into a row offset into the flattened
631 // plane, using the row-unit batch strides encoded at create time:
632 // offsetH += sum_d offset[d] * leadingRowStride[d]
633 // The base pointer stays at the true base, so an out-of-range batch index
634 // is caught by the HW boundary check instead of moving the surface to
635 // unmapped memory.
636 for (int64_t d = 0; d < tileRank - 2; ++d) {
637 Value off =
638 getValueOrCreateConstantIntOp(rewriter, loc, mixedOffsets[d]);
639 off = getValueOrCreateCastToIndexLike(rewriter, loc,
640 rewriter.getI32Type(), off);
641 Value rowStride = vector::ExtractOp::create(
642 rewriter, loc, tdesc,
643 static_cast<int>(NdTdescOffset::LeadingStride0) + d);
644 Value term = arith::MulIOp::create(rewriter, loc, off, rowStride);
645 offsetH = arith::AddIOp::create(rewriter, loc, offsetH, term);
646 }
647 // Convert base pointer (i64) to LLVM pointer type.
648 Value basePtrLLVM =
649 LLVM::IntToPtrOp::create(rewriter, loc, ptrTypeLLVM, basePtr);
650 // FIXME: width or pitch is not the same as baseShapeW it should be the
651 // stride of the second to last dimension in row major layout.
652 // Compute width in bytes.
653 Value baseShapeWInBytes =
654 arith::MulIOp::create(rewriter, loc, baseShapeW, elemByteSize);
655 // Compute pitch in bytes.
656 Value basePitchBytes =
657 arith::MulIOp::create(rewriter, loc, basePitch, elemByteSize);
658
659 if (wScaleFactor > 1) {
660 // Scale offsetW, baseShapeWInBytes for sub byte emulation.
661 // Note: tileW is already scaled above.
662 Value wScaleFactorValLog2 = arith::ConstantIntOp::create(
663 rewriter, loc, rewriter.getI32Type(), llvm::Log2_64(Value: wScaleFactor));
664 baseShapeWInBytes = arith::ShRSIOp::create(
665 rewriter, loc, baseShapeWInBytes, wScaleFactorValLog2);
666 basePitchBytes = arith::ShRSIOp::create(rewriter, loc, basePitchBytes,
667 wScaleFactorValLog2);
668 offsetW =
669 arith::ShRSIOp::create(rewriter, loc, offsetW, wScaleFactorValLog2);
670 }
671 // Get tile height from the tensor descriptor type (second-to-last dim).
672 auto tileH = tdescTy.getDimSize(tileRank - 2);
673 // Get vblocks from the tensor descriptor type.
674 int32_t vblocks = tdescTy.getArrayLength();
675 if constexpr (std::is_same_v<OpType, xegpu::StoreNdOp>) {
676 Value src = adaptor.getValue();
677 // If store value is a scalar, get value from op instead of adaptor.
678 // Adaptor might have optimized away single element vector
679 if (src.getType().isIntOrFloat()) {
680 src = op.getValue();
681 }
682 VectorType srcVecTy = dyn_cast<VectorType>(Val: src.getType());
683 if (!srcVecTy)
684 return rewriter.notifyMatchFailure(
685 op, "Expected store value to be a vector type.");
686 // Get flat vector type of integer type with matching element bit size.
687 VectorType newSrcVecTy =
688 encodeVectorTypeTo(srcVecTy, rewriter.getIntegerType(elemBitSize));
689 if (srcVecTy != newSrcVecTy)
690 src = vector::BitCastOp::create(rewriter, loc, newSrcVecTy, src);
691 auto storeCacheControl =
692 translateStoreXeGPUCacheHint(op.getL1Hint(), op.getL3Hint());
693 xevm::BlockStore2dOp::create(
694 rewriter, loc, basePtrLLVM, baseShapeWInBytes, baseShapeH,
695 basePitchBytes, offsetW, offsetH, elemBitSize, tileW, tileH, src,
696 xevm::StoreCacheControlAttr::get(context: ctxt, value: storeCacheControl));
697 rewriter.eraseOp(op);
698 } else {
699 auto loadCacheControl =
700 translateLoadXeGPUCacheHint(op.getL1Hint(), op.getL3Hint());
701 if constexpr (std::is_same_v<OpType, xegpu::PrefetchNdOp>) {
702 xevm::BlockPrefetch2dOp::create(
703 rewriter, loc, basePtrLLVM, baseShapeWInBytes, baseShapeH,
704 basePitchBytes, offsetW, offsetH, elemBitSize, tileW, tileH,
705 vblocks, xevm::LoadCacheControlAttr::get(context: ctxt, value: loadCacheControl));
706 rewriter.eraseOp(op);
707 } else {
708 VectorType dstVecTy = cast<VectorType>(op.getValue().getType());
709 bool vnni = op.getPacked().value_or(false);
710 auto transposeValue = op.getTranspose();
711 bool transpose =
712 transposeValue.has_value() && transposeValue.value()[0] == 1;
713 // Handle special case of 32x16 and 8bit element load
714 // with no vnni, no transpose, no vblocks.
715 // For this special case, vnni and non vnni yields the same output
716 // and only the vnni variant is supported by HW.
717 // Check and set vnni of the special case.
718 if (elemBitSize == 8 && tileW == 16 && tileH == 32 && !vnni &&
719 !transpose) {
720 vnni = true;
721 }
722 // Handle tranpose request on small element size
723 // Transpose needs to be requested on 32bit element type.
724 // offsetW and tileW needs to be adjusted to account for element type
725 // change.
726 if (transpose && elemBitSize < 32) {
727 int32_t scale = 32 / elemBitSize;
728 Value scaleLog2 = arith::ConstantIntOp::create(
729 rewriter, loc, rewriter.getI32Type(), llvm::Log2_64(Value: scale));
730 offsetW = arith::ShRSIOp::create(rewriter, loc, offsetW, scaleLog2);
731 tileW = tileW * elemBitSize / 32;
732 elemBitSize = 32;
733 }
734 VectorType loadedTy = encodeVectorTypeTo(
735 dstVecTy, vnni ? rewriter.getI32Type()
736 : rewriter.getIntegerType(elemBitSize));
737
738 Value resultFlatVec = xevm::BlockLoad2dOp::create(
739 rewriter, loc, loadedTy, basePtrLLVM, baseShapeWInBytes,
740 baseShapeH, basePitchBytes, offsetW, offsetH, elemBitSize, tileW,
741 tileH, vblocks, transpose, vnni,
742 xevm::LoadCacheControlAttr::get(context: ctxt, value: loadCacheControl));
743 resultFlatVec = vector::BitCastOp::create(
744 rewriter, loc,
745 encodeVectorTypeTo(currentVecType: loadedTy, toElemType: dstVecTy.getElementType()),
746 resultFlatVec);
747 rewriter.replaceOp(op, resultFlatVec);
748 }
749 }
750 } else {
751 // 1D tensor descriptor.
752 // `tdesc` represents base address as i64
753 // Offset in number of elements, need to multiply by element byte size.
754 // Compute byte offset.
755 // byteOffset = offset * elementByteSize
756 Value offset =
757 getValueOrCreateConstantIntOp(rewriter, loc, mixedOffsets[0]);
758 offset = getValueOrCreateCastToIndexLike(rewriter, loc,
759 rewriter.getI64Type(), offset);
760 // Compute element byte size.
761 Value elemByteSize = arith::ConstantIntOp::create(
762 rewriter, loc, rewriter.getI64Type(), elemBitSize / 8);
763 Value byteOffset =
764 rewriter.createOrFold<arith::MulIOp>(loc, offset, elemByteSize);
765 // Final address = basePtr + byteOffset
766 Value finalAddrI64 = rewriter.createOrFold<arith::AddIOp>(
767 loc, tdesc,
768 getValueOrCreateCastToIndexLike(rewriter, loc, rewriter.getI64Type(),
769 byteOffset));
770 // Convert base pointer (i64) to LLVM pointer type.
771 Value finalPtrLLVM =
772 LLVM::IntToPtrOp::create(rewriter, loc, ptrTypeLLVM, finalAddrI64);
773 if constexpr (std::is_same_v<OpType, xegpu::StoreNdOp>) {
774 Value src = adaptor.getValue();
775 // If store value is a scalar, get value from op instead of adaptor.
776 // Adaptor might have optimized away single element vector
777 if (src.getType().isIntOrFloat()) {
778 src = op.getValue();
779 }
780 VectorType srcVecTy = dyn_cast<VectorType>(Val: src.getType());
781 if (!srcVecTy)
782 return rewriter.notifyMatchFailure(
783 op, "Expected store value to be a vector type.");
784 // Get flat vector type of integer type with matching element bit size.
785 VectorType newSrcVecTy =
786 encodeVectorTypeTo(srcVecTy, rewriter.getIntegerType(elemBitSize));
787 if (srcVecTy != newSrcVecTy)
788 src = vector::BitCastOp::create(rewriter, loc, newSrcVecTy, src);
789 auto storeCacheControl =
790 translateStoreXeGPUCacheHint(op.getL1Hint(), op.getL3Hint());
791 rewriter.replaceOpWithNewOp<xevm::BlockStoreOp>(
792 op, finalPtrLLVM, src,
793 xevm::StoreCacheControlAttr::get(context: ctxt, value: storeCacheControl));
794 } else if constexpr (std::is_same_v<OpType, xegpu::LoadNdOp>) {
795 auto loadCacheControl =
796 translateLoadXeGPUCacheHint(op.getL1Hint(), op.getL3Hint());
797 VectorType resTy = cast<VectorType>(op.getValue().getType());
798 VectorType loadedTy =
799 encodeVectorTypeTo(resTy, rewriter.getIntegerType(elemBitSize));
800 Value load = xevm::BlockLoadOp::create(
801 rewriter, loc, loadedTy, finalPtrLLVM,
802 xevm::LoadCacheControlAttr::get(context: ctxt, value: loadCacheControl));
803 if (loadedTy != resTy)
804 load = vector::BitCastOp::create(rewriter, loc, resTy, load);
805 rewriter.replaceOp(op, load);
806 } else {
807 return rewriter.notifyMatchFailure(
808 op, "Unsupported operation: xegpu.prefetch_nd with tensor "
809 "descriptor rank == 1");
810 }
811 }
812 return success();
813 }
814};
815
816// Add a builder that creates
817// offset * elemByteSize + baseAddr
818static Value addOffsetToBaseAddr(ConversionPatternRewriter &rewriter,
819 Location loc, Value baseAddr, Value offset,
820 int64_t elemByteSize) {
821 Value byteSize = arith::ConstantIntOp::create(
822 builder&: rewriter, location: loc, type: baseAddr.getType(), value: elemByteSize);
823 Value byteOffset = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: offset, rhs: byteSize);
824 Value newAddr = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: baseAddr, rhs: byteOffset);
825 return newAddr;
826}
827
828// Returns true when every element of `mask` carries the same bit, so gating a
829// whole contiguous block on element 0 is equivalent. Splat constants, an
830// all-ones or all-zeros `vector.constant_mask`, broadcasts of a scalar, and a
831// `vector.from_elements` of one repeated value qualify.
832//
833// TODO: this recognizes a fixed list of producers. A general uniformity query
834// on the vector dialect would cover more forms and would not need updating
835// every time a new producer shows up here.
836static bool isUniformMask(Value mask) {
837 if (!isa<VectorType>(Val: mask.getType()))
838 return true;
839 DenseElementsAttr splat;
840 if (matchPattern(value: mask, pattern: m_Constant(bind_value: &splat)) && splat.isSplat())
841 return true;
842 if (auto constantMask = mask.getDefiningOp<vector::ConstantMaskOp>()) {
843 // All-ones and all-zeros are uniform. A zero dim size is only legal when
844 // every dim is zero, so those are the only two uniform cases.
845 return constantMask.isAllOnesMask() ||
846 llvm::all_of(Range: constantMask.getMaskDimSizes(),
847 P: [](int64_t size) { return size == 0; });
848 }
849 if (auto broadcast = mask.getDefiningOp<vector::BroadcastOp>())
850 return !isa<VectorType>(Val: broadcast.getSource().getType());
851 if (auto fromElements = mask.getDefiningOp<vector::FromElementsOp>())
852 return llvm::all_equal(Range: fromElements.getElements());
853 // Flattening a rank > 1 operand to rank 1 inserts a shape cast. It keeps the
854 // same elements, so it keeps uniformity.
855 if (auto shapeCast = mask.getDefiningOp<vector::ShapeCastOp>())
856 return isUniformMask(mask: shapeCast.getSource());
857 return false;
858}
859
860template <typename OpType,
861 typename = std::enable_if_t<llvm::is_one_of<
862 OpType, xegpu::LoadGatherOp, xegpu::StoreScatterOp>::value>>
863class LoadStoreToXeVMPattern : public OpConversionPattern<OpType> {
864 using OpConversionPattern<OpType>::OpConversionPattern;
865 LogicalResult
866 matchAndRewrite(OpType op, typename OpType::Adaptor adaptor,
867 ConversionPatternRewriter &rewriter) const override {
868 Value offset = adaptor.getOffsets();
869 if (!offset)
870 return rewriter.notifyMatchFailure(op, "Expected offset to be provided.");
871 auto loc = op.getLoc();
872 auto ctxt = rewriter.getContext();
873 Value basePtrI64;
874 // Load result or Store valye Type can be vector or scalar.
875 Type valOrResTy;
876 if constexpr (std::is_same_v<OpType, xegpu::LoadGatherOp>)
877 valOrResTy =
878 this->getTypeConverter()->convertType(op.getResult().getType());
879 else
880 valOrResTy = adaptor.getValue().getType();
881 VectorType valOrResVecTy = dyn_cast<VectorType>(Val&: valOrResTy);
882 bool hasScalarVal = !valOrResVecTy;
883 int64_t elemBitWidth =
884 hasScalarVal ? valOrResTy.getIntOrFloatBitWidth()
885 : valOrResVecTy.getElementType().getIntOrFloatBitWidth();
886 // Element type must be multiple of 8 bits.
887 if (elemBitWidth % 8 != 0)
888 return rewriter.notifyMatchFailure(
889 op, "Expected element type bit width to be multiple of 8.");
890 int64_t elemByteSize = elemBitWidth / 8;
891 // Default memory space is global.
892 LLVM::LLVMPointerType ptrTypeLLVM = LLVM::LLVMPointerType::get(
893 context: ctxt, addressSpace: getNumericXeVMAddrSpace(xeGpuMemspace: xegpu::MemorySpace::Global));
894 // Base pointer can come from source (load) or dest (store).
895 // If they are memrefs, we use their memory space.
896 if constexpr (std::is_same_v<OpType, xegpu::LoadGatherOp>) {
897 basePtrI64 = adaptor.getSource();
898 if (auto memRefTy = dyn_cast<MemRefType>(op.getSource().getType())) {
899 FailureOr<unsigned> addrSpace =
900 getNumericMemorySpace(memRefTy.getMemorySpace());
901 if (failed(Result: addrSpace))
902 return rewriter.notifyMatchFailure(
903 op, "Unsupported memref memory space attribute.");
904 if (*addrSpace != 0)
905 ptrTypeLLVM = LLVM::LLVMPointerType::get(context: ctxt, addressSpace: *addrSpace);
906 }
907 } else {
908 basePtrI64 = adaptor.getDest();
909 if (auto memRefTy = dyn_cast<MemRefType>(op.getDest().getType())) {
910 FailureOr<unsigned> addrSpace =
911 getNumericMemorySpace(memRefTy.getMemorySpace());
912 if (failed(Result: addrSpace))
913 return rewriter.notifyMatchFailure(
914 op, "Unsupported memref memory space attribute.");
915 if (*addrSpace != 0)
916 ptrTypeLLVM = LLVM::LLVMPointerType::get(context: ctxt, addressSpace: *addrSpace);
917 }
918 }
919 // Base pointer is passed as i32 or i64 by adaptor, cast to i64 if needed.
920 if (basePtrI64.getType() != rewriter.getI64Type()) {
921 basePtrI64 = arith::ExtUIOp::create(rewriter, loc, rewriter.getI64Type(),
922 basePtrI64);
923 }
924 Value mask = adaptor.getMask();
925
926 // Coalesce a lane's multi-element access into one block access.
927 //
928 // Distribution gives every element its own offset and mask bit, so a lane
929 // that takes D neighbouring elements arrives with `vector<D>` offsets and
930 // mask. One block access needs one base offset and one mask bit, so take
931 // both from element 0. A non-uniform mask cannot be reduced to one bit, so
932 // it stays a vector and fails to match below.
933 auto origOffsetsTy = dyn_cast<VectorType>(op.getOffsets().getType());
934 if (isa<VectorType>(Val: offset.getType()) && origOffsetsTy && valOrResVecTy &&
935 origOffsetsTy.getNumElements() == valOrResVecTy.getNumElements() &&
936 isUniformMask(op.getMask())) {
937 offset = vector::ExtractOp::create(rewriter, loc, offset,
938 ArrayRef<int64_t>{0});
939 if (isa<VectorType>(Val: mask.getType()))
940 mask = vector::ExtractOp::create(rewriter, loc, mask,
941 ArrayRef<int64_t>{0});
942 }
943
944 if (dyn_cast<VectorType>(Val: offset.getType())) {
945 // Offset needs be scalar. Single element vector is converted to scalar
946 // by type converter.
947 return rewriter.notifyMatchFailure(op, "Expected offset to be a scalar.");
948 } else {
949 // If offset is provided, we add them to the base pointer.
950 // Offset is in number of elements, we need to multiply by
951 // element byte size.
952 basePtrI64 =
953 addOffsetToBaseAddr(rewriter, loc, basePtrI64, offset, elemByteSize);
954 }
955 // Convert base pointer (i64) to LLVM pointer type.
956 Value basePtrLLVM =
957 LLVM::IntToPtrOp::create(rewriter, loc, ptrTypeLLVM, basePtrI64);
958
959 Value maskForLane;
960 VectorType maskVecTy = dyn_cast<VectorType>(Val: mask.getType());
961 if (maskVecTy) {
962 // Mask needs be scalar. Single element vector is converted to scalar by
963 // type converter.
964 return rewriter.notifyMatchFailure(op, "Expected mask to be a scalar.");
965 } else
966 maskForLane = mask;
967 if constexpr (std::is_same_v<OpType, xegpu::LoadGatherOp>) {
968 scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, {valOrResTy},
969 maskForLane, true, true);
970 // If mask is true,- then clause - load from memory and yield.
971 rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front());
972 if (!hasScalarVal)
973 valOrResTy = VectorType::get(shape: {valOrResVecTy.getNumElements()},
974 elementType: valOrResVecTy.getElementType());
975 Value loaded =
976 LLVM::LoadOp::create(rewriter, loc, valOrResTy, basePtrLLVM);
977 // Set cache control attribute on the load operation.
978 loaded.getDefiningOp()->setDiscardableAttr(
979 "cache_control", xevm::LoadCacheControlAttr::get(
980 context: ctxt, value: translateLoadXeGPUCacheHint(
981 op.getL1Hint(), op.getL3Hint())));
982 scf::YieldOp::create(rewriter, loc, ValueRange{loaded});
983 rewriter.setInsertionPointToStart(&ifOp.getElseRegion().front());
984 // If mask is false - else clause -yield a vector of zeros.
985 auto eTy = hasScalarVal ? valOrResTy : valOrResVecTy.getElementType();
986 TypedAttr eVal;
987 if (eTy.isFloat())
988 eVal = FloatAttr::get(type: eTy, value: 0.0);
989 else
990 eVal = IntegerAttr::get(type: eTy, value: 0);
991 if (hasScalarVal)
992 loaded = arith::ConstantOp::create(rewriter, loc, eVal);
993 else
994 loaded = arith::ConstantOp::create(
995 rewriter, loc, DenseElementsAttr::get(type: valOrResVecTy, values: eVal));
996 scf::YieldOp::create(rewriter, loc, ValueRange{loaded});
997 rewriter.replaceOp(op, ifOp.getResult(i: 0));
998 } else {
999 // If mask is true, perform the store.
1000 scf::IfOp ifOp = scf::IfOp::create(rewriter, loc, maskForLane, false);
1001 auto body = ifOp.getBody();
1002 rewriter.setInsertionPointToStart(body);
1003 auto storeOp =
1004 LLVM::StoreOp::create(rewriter, loc, adaptor.getValue(), basePtrLLVM);
1005 // Set cache control attribute on the store operation.
1006 storeOp.getOperation()->setDiscardableAttr(
1007 "cache_control", xevm::StoreCacheControlAttr::get(
1008 context: ctxt, value: translateStoreXeGPUCacheHint(
1009 op.getL1Hint(), op.getL3Hint())));
1010 rewriter.eraseOp(op);
1011 }
1012 return success();
1013 }
1014};
1015
1016class CreateMemDescOpPattern final
1017 : public OpConversionPattern<xegpu::CreateMemDescOp> {
1018public:
1019 using OpConversionPattern<xegpu::CreateMemDescOp>::OpConversionPattern;
1020 LogicalResult
1021 matchAndRewrite(xegpu::CreateMemDescOp op, OpAdaptor adaptor,
1022 ConversionPatternRewriter &rewriter) const override {
1023
1024 rewriter.replaceOp(op, newValues: adaptor.getSource());
1025 return success();
1026 }
1027};
1028
1029/// Returns the shape of the largest piece of a matrix op payload that one LLVM
1030/// access can move, as an access only moves consecutive elements.
1031/// Examples:
1032/// mem_desc<16x2xf32> (stride [2, 1], row-major)
1033/// payload [1, 2] -> [1, 2] one access of 2 elements
1034/// payload [2, 1] -> [1, 1] 2 accesses of 1 element
1035/// mem_desc<16x2xf32, stride = [1, 16]> (column-major)
1036/// payload [4, 1] -> [4, 1] one access of 4 elements
1037/// payload [2, 2] -> [1, 1] 4 accesses of 1 element; each column is
1038/// consecutive in memory but not in the
1039/// row-major flattened vector
1040///
1041/// TODO: On a blocked layout memory is only consecutive inside a block, so a
1042/// piece that crosses a block boundary along the stride-1 dim is not
1043/// contiguous. Blocks are not taken into account yet.
1044static SmallVector<int64_t>
1045getContiguousPayloadShape(ArrayRef<int64_t> payloadShape, size_t numOffsets,
1046 xegpu::MemDescType mdescTy) {
1047 SmallVector<int64_t> shape(payloadShape);
1048 int64_t rank = shape.size();
1049 if (rank != static_cast<int64_t>(numOffsets))
1050 return shape;
1051
1052 int64_t fastestDim = 0;
1053 for (auto [dim, strideAttr] :
1054 llvm::enumerate(First: mdescTy.getStrideAttr().getValue())) {
1055 if (cast<IntegerAttr>(Val: strideAttr).getInt() == 1) {
1056 fastestDim = dim;
1057 break;
1058 }
1059 }
1060
1061 // A slice that runs along fastestDim is only consecutive in the vector if
1062 // every dim to its right has size 1.
1063 bool spansFastestDim =
1064 llvm::all_of(Range: llvm::seq<int64_t>(Begin: fastestDim + 1, End: rank),
1065 P: [&](int64_t d) { return payloadShape[d] == 1; });
1066 for (int64_t d = 0; d < rank; ++d)
1067 shape[d] = spansFastestDim && d == fastestDim ? payloadShape[d] : 1;
1068
1069 return shape;
1070}
1071
1072template <typename OpType,
1073 typename = std::enable_if_t<llvm::is_one_of<
1074 OpType, xegpu::LoadMatrixOp, xegpu::StoreMatrixOp>::value>>
1075class LoadStoreMatrixToXeVMPattern : public OpConversionPattern<OpType> {
1076 using OpConversionPattern<OpType>::OpConversionPattern;
1077
1078 /// Lowers `op` to one LLVM access per `sliceShape`-sized piece of its
1079 /// payload, each at its own address. `basePtrLLVM` already addresses the
1080 /// first piece. A payload that is one contiguous piece, which is the common
1081 /// case, gives a single access of the whole payload.
1082 LogicalResult emitContiguousAccesses(
1083 OpType op, typename OpType::Adaptor adaptor,
1084 ConversionPatternRewriter &rewriter, xegpu::MemDescType mdescTy,
1085 ArrayRef<OpFoldResult> offsets, ArrayRef<int64_t> payloadShape,
1086 ArrayRef<int64_t> sliceShape, Value baseAddr32, Value basePtrLLVM,
1087 LLVM::LLVMPointerType ptrTypeLLVM, int64_t elemByteSize) const {
1088 Location loc = op.getLoc();
1089 int64_t rank = payloadShape.size();
1090 SmallVector<int64_t> sliceGrid(rank);
1091 for (int64_t d = 0; d < rank; ++d)
1092 sliceGrid[d] = payloadShape[d] / sliceShape[d];
1093 int64_t numSlices = computeProduct(basis: sliceGrid);
1094 int64_t sliceLen = computeProduct(basis: sliceShape);
1095 SmallVector<int64_t> payloadStrides = computeStrides(sizes: payloadShape);
1096 SmallVector<int64_t> gridStrides = computeStrides(sizes: sliceGrid);
1097
1098 // The payload type LLVM load/store accepts: f8E8M0FNU and friends become
1099 // same-width integers, vector<1> becomes a scalar. Operands arrive
1100 // converted in the adaptor, results do not.
1101 Type payloadTy;
1102 if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp>)
1103 payloadTy =
1104 this->getTypeConverter()->convertType(op.getResult().getType());
1105 else
1106 payloadTy = adaptor.getData().getType();
1107 auto payloadVecTy = dyn_cast<VectorType>(Val&: payloadTy);
1108
1109 Type sliceTy = payloadTy;
1110 if (numSlices != 1) {
1111 if (!payloadVecTy)
1112 return rewriter.notifyMatchFailure(
1113 op, "Expected a vector payload to slice.");
1114 sliceTy = sliceLen == 1 ? payloadVecTy.getElementType()
1115 : VectorType::get(shape: {sliceLen},
1116 elementType: payloadVecTy.getElementType());
1117 }
1118
1119 // Addresses the slice `delta` elements into the payload.
1120 auto slicePtr = [&](ArrayRef<int64_t> delta) -> Value {
1121 if (llvm::all_of(delta, [](int64_t d) { return d == 0; }))
1122 return basePtrLLVM;
1123 SmallVector<OpFoldResult> sliceOffsets;
1124 for (auto [base, d] : llvm::zip_equal(t&: offsets, u&: delta)) {
1125 if (d == 0)
1126 sliceOffsets.push_back(Elt: base);
1127 else if (auto attr = dyn_cast<Attribute>(Val: base))
1128 sliceOffsets.push_back(
1129 Elt: rewriter.getIndexAttr(value: cast<IntegerAttr>(Val&: attr).getInt() + d));
1130 else
1131 sliceOffsets.push_back(
1132 Elt: OpFoldResult(arith::AddIOp::create(
1133 builder&: rewriter, location: loc, lhs: cast<Value>(Val: base),
1134 rhs: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: d))
1135 .getResult()));
1136 }
1137 Value sliceOffset = mdescTy.getLinearOffsets(builder&: rewriter, loc, offsets: sliceOffsets);
1138 sliceOffset = arith::IndexCastUIOp::create(
1139 builder&: rewriter, location: loc, out: rewriter.getI32Type(), in: sliceOffset);
1140 Value slicePtrI32 = addOffsetToBaseAddr(rewriter, loc, baseAddr: baseAddr32,
1141 offset: sliceOffset, elemByteSize);
1142 return LLVM::IntToPtrOp::create(builder&: rewriter, location: loc, resultType: ptrTypeLLVM, operands: slicePtrI32);
1143 };
1144
1145 Value result;
1146 if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp>)
1147 if (numSlices != 1)
1148 result = arith::ConstantOp::create(builder&: rewriter, location: loc, result: payloadVecTy,
1149 value: rewriter.getZeroAttr(type: payloadVecTy));
1150
1151 for (int64_t slice = 0; slice < numSlices; ++slice) {
1152 SmallVector<int64_t> gridCoords = delinearize(linearIndex: slice, strides: gridStrides);
1153 SmallVector<int64_t> delta(rank);
1154 for (int64_t d = 0; d < rank; ++d)
1155 delta[d] = gridCoords[d] * sliceShape[d];
1156 int64_t payloadOffset = linearize(offsets: delta, basis: payloadStrides);
1157 Value ptr = slicePtr(delta);
1158
1159 if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp>) {
1160 Value loaded = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: sliceTy, addr: ptr);
1161 if (numSlices == 1)
1162 result = loaded;
1163 else if (sliceLen == 1)
1164 result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: loaded, dest: result,
1165 position: ArrayRef<int64_t>{payloadOffset});
1166 else
1167 result = vector::InsertStridedSliceOp::create(
1168 builder&: rewriter, location: loc, valueToStore: loaded, dest: result, offsets: ArrayRef<int64_t>{payloadOffset},
1169 strides: ArrayRef<int64_t>{1});
1170 } else {
1171 Value data = adaptor.getData();
1172 if (numSlices != 1) {
1173 if (sliceLen == 1)
1174 data = vector::ExtractOp::create(builder&: rewriter, location: loc, source: data,
1175 position: ArrayRef<int64_t>{payloadOffset});
1176 else
1177 data = vector::ExtractStridedSliceOp::create(
1178 builder&: rewriter, location: loc, source: data, offsets: ArrayRef<int64_t>{payloadOffset},
1179 sizes: ArrayRef<int64_t>{sliceLen}, strides: ArrayRef<int64_t>{1});
1180 }
1181 LLVM::StoreOp::create(builder&: rewriter, location: loc, value: data, addr: ptr);
1182 }
1183 }
1184
1185 if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp>)
1186 rewriter.replaceOp(op, result);
1187 else
1188 rewriter.eraseOp(op);
1189 return success();
1190 }
1191 LogicalResult
1192 matchAndRewrite(OpType op, typename OpType::Adaptor adaptor,
1193 ConversionPatternRewriter &rewriter) const override {
1194
1195 SmallVector<OpFoldResult> offsets = op.getMixedOffsets();
1196 if (offsets.empty())
1197 return rewriter.notifyMatchFailure(op, "Expected offset to be provided.");
1198
1199 auto loc = op.getLoc();
1200 auto ctxt = rewriter.getContext();
1201 Value baseAddr32 = adaptor.getMemDesc();
1202 Value mdescVal = op.getMemDesc();
1203 // Load result or Store value Type can be vector or scalar.
1204 Type dataTy;
1205 if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp>) {
1206 Type resType = op.getResult().getType();
1207 // Some transforms may leave unit dimension in the 2D vector, adaptors do
1208 // not catch it for results.
1209 if (auto vecType = dyn_cast<VectorType>(Val&: resType))
1210 resType = VectorType::get(shape: {vecType.getNumElements()},
1211 elementType: vecType.getElementType());
1212 dataTy = resType;
1213 } else
1214 dataTy = adaptor.getData().getType();
1215 VectorType valOrResVecTy = dyn_cast<VectorType>(Val&: dataTy);
1216 if (!valOrResVecTy)
1217 valOrResVecTy = VectorType::get(shape: 1, elementType: dataTy);
1218
1219 int64_t elemBitWidth =
1220 valOrResVecTy.getElementType().getIntOrFloatBitWidth();
1221 // Element type must be multiple of 8 bits.
1222 if (elemBitWidth % 8 != 0)
1223 return rewriter.notifyMatchFailure(
1224 op, "Expected element type bit width to be multiple of 8.");
1225 int64_t elemByteSize = elemBitWidth / 8;
1226
1227 // Default memory space is SLM.
1228 LLVM::LLVMPointerType ptrTypeLLVM = LLVM::LLVMPointerType::get(
1229 context: ctxt, addressSpace: getNumericXeVMAddrSpace(xeGpuMemspace: xegpu::MemorySpace::SLM));
1230
1231 auto mdescTy = cast<xegpu::MemDescType>(Val: mdescVal.getType());
1232
1233 Value linearOffset = mdescTy.getLinearOffsets(builder&: rewriter, loc, offsets);
1234 linearOffset = arith::IndexCastUIOp::create(
1235 rewriter, loc, rewriter.getI32Type(), linearOffset);
1236 Value basePtrI32 = addOffsetToBaseAddr(rewriter, loc, baseAddr32,
1237 linearOffset, elemByteSize);
1238
1239 // convert base pointer (i32) to LLVM pointer type
1240 Value basePtrLLVM =
1241 LLVM::IntToPtrOp::create(rewriter, loc, ptrTypeLLVM, basePtrI32);
1242
1243 if (op.getSubgroupBlockIoAttr()) {
1244 // if the attribute 'subgroup_block_io' is set to true, it lowers to
1245 // xevm.blockload
1246
1247 Type intElemTy = rewriter.getIntegerType(width: elemBitWidth);
1248 VectorType intVecTy =
1249 VectorType::get(shape: valOrResVecTy.getShape(), elementType: intElemTy);
1250
1251 if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp>) {
1252 Value loadOp = xevm::BlockLoadOp::create(
1253 rewriter, loc, intVecTy, basePtrLLVM, /*cache_control=*/nullptr);
1254 if (intVecTy != valOrResVecTy) {
1255 loadOp =
1256 vector::BitCastOp::create(rewriter, loc, valOrResVecTy, loadOp);
1257 }
1258 rewriter.replaceOp(op, loadOp);
1259 } else {
1260 Value dataToStore = adaptor.getData();
1261 if (valOrResVecTy != intVecTy) {
1262 dataToStore =
1263 vector::BitCastOp::create(rewriter, loc, intVecTy, dataToStore);
1264 }
1265 xevm::BlockStoreOp::create(rewriter, loc, basePtrLLVM, dataToStore,
1266 nullptr);
1267 rewriter.eraseOp(op);
1268 }
1269 return success();
1270 }
1271
1272 if (valOrResVecTy.getNumElements() >= 1) {
1273 auto chipOpt = xegpu::getChipStr(op);
1274 if (!chipOpt ||
1275 (*chipOpt != "pvc" && *chipOpt != "bmg" && *chipOpt != "cri")) {
1276 // the lowering for chunk load only works for pvc, bmg or cri
1277 return rewriter.notifyMatchFailure(
1278 op, "The lowering is specific to pvc, bmg or cri.");
1279 }
1280 }
1281
1282 // The payload shape says where in the matrix each element lives, which the
1283 // adaptor has already linearized away.
1284 ArrayRef<int64_t> payloadShape;
1285 Type opPayloadTy;
1286 if constexpr (std::is_same_v<OpType, xegpu::LoadMatrixOp>)
1287 opPayloadTy = op.getResult().getType();
1288 else
1289 opPayloadTy = op.getData().getType();
1290 if (auto opPayloadVecTy = dyn_cast<VectorType>(Val&: opPayloadTy))
1291 payloadShape = opPayloadVecTy.getShape();
1292
1293 return emitContiguousAccesses(
1294 op, adaptor, rewriter, mdescTy, offsets, payloadShape,
1295 sliceShape: getContiguousPayloadShape(payloadShape, numOffsets: offsets.size(), mdescTy),
1296 baseAddr32, basePtrLLVM, ptrTypeLLVM, elemByteSize);
1297 }
1298};
1299
1300class PrefetchToXeVMPattern : public OpConversionPattern<xegpu::PrefetchOp> {
1301 using Base::Base;
1302 LogicalResult
1303 matchAndRewrite(xegpu::PrefetchOp op, xegpu::PrefetchOp::Adaptor adaptor,
1304 ConversionPatternRewriter &rewriter) const override {
1305 auto loc = op.getLoc();
1306 auto ctxt = rewriter.getContext();
1307 Value basePtrI64 = adaptor.getSource();
1308 // Base pointer is passed as i32 or i64 by adaptor, cast to i64 if needed.
1309 if (basePtrI64.getType() != rewriter.getI64Type())
1310 basePtrI64 = arith::ExtUIOp::create(builder&: rewriter, location: loc, out: rewriter.getI64Type(),
1311 in: basePtrI64);
1312 Value offsets = adaptor.getOffsets();
1313 if (offsets) {
1314 VectorType offsetsVecTy = dyn_cast<VectorType>(Val: offsets.getType());
1315 if (offsetsVecTy) {
1316 // Offset needs be scalar.
1317 return rewriter.notifyMatchFailure(arg&: op,
1318 msg: "Expected offsets to be a scalar.");
1319 } else {
1320 int64_t elemBitWidth{0};
1321 int64_t elemByteSize;
1322 // Element byte size can come from two sources:
1323 if (auto memRefTy = dyn_cast<MemRefType>(Val: op.getSourceType())) {
1324 // If memref is available, we use its element type to
1325 // determine element byte size.
1326 elemBitWidth = memRefTy.getElementType().getIntOrFloatBitWidth();
1327 } else {
1328 // Otherwise, we use the provided offset byte alignment.
1329 elemByteSize = *op.getOffsetAlignByte();
1330 }
1331 if (elemBitWidth != 0) {
1332 if (elemBitWidth % 8 != 0)
1333 return rewriter.notifyMatchFailure(
1334 arg&: op, msg: "Expected element type bit width to be multiple of 8.");
1335 elemByteSize = elemBitWidth / 8;
1336 }
1337 basePtrI64 = addOffsetToBaseAddr(rewriter, loc, baseAddr: basePtrI64, offset: offsets,
1338 elemByteSize);
1339 }
1340 }
1341 // Default memory space is global.
1342 LLVM::LLVMPointerType ptrTypeLLVM = LLVM::LLVMPointerType::get(
1343 context: ctxt, addressSpace: getNumericXeVMAddrSpace(xeGpuMemspace: xegpu::MemorySpace::Global));
1344 // If source is a memref, we use its memory space.
1345 if (auto memRefTy = dyn_cast<MemRefType>(Val: op.getSource().getType())) {
1346 FailureOr<unsigned> addrSpace =
1347 getNumericMemorySpace(memSpace: memRefTy.getMemorySpace());
1348 if (failed(Result: addrSpace))
1349 return rewriter.notifyMatchFailure(
1350 arg&: op, msg: "Unsupported memref memory space attribute.");
1351 if (*addrSpace != 0)
1352 ptrTypeLLVM = LLVM::LLVMPointerType::get(context: ctxt, addressSpace: *addrSpace);
1353 }
1354 // Convert base pointer (i64) to LLVM pointer type.
1355 Value ptrLLVM =
1356 LLVM::IntToPtrOp::create(builder&: rewriter, location: loc, resultType: ptrTypeLLVM, operands: basePtrI64);
1357 // Create the prefetch op with cache control attribute.
1358 xevm::PrefetchOp::create(
1359 builder&: rewriter, location: loc, ptr: ptrLLVM,
1360 cache_control: xevm::LoadCacheControlAttr::get(
1361 context: ctxt, value: translateLoadXeGPUCacheHint(L1hint: op.getL1Hint(), L3hint: op.getL3Hint())));
1362 rewriter.eraseOp(op);
1363 return success();
1364 }
1365};
1366
1367class FenceToXeVMPattern : public OpConversionPattern<xegpu::FenceOp> {
1368 using Base::Base;
1369 LogicalResult
1370 matchAndRewrite(xegpu::FenceOp op, xegpu::FenceOp::Adaptor adaptor,
1371 ConversionPatternRewriter &rewriter) const override {
1372 auto loc = op.getLoc();
1373 xevm::MemScope memScope{xevm::MemScope::WORKGROUP};
1374 switch (op.getFenceScope()) {
1375 case xegpu::FenceScope::Workgroup:
1376 memScope = xevm::MemScope::WORKGROUP;
1377 break;
1378 case xegpu::FenceScope::GPU:
1379 memScope = xevm::MemScope::DEVICE;
1380 break;
1381 }
1382 xevm::AddrSpace addrSpace{xevm::AddrSpace::GLOBAL};
1383 switch (op.getMemoryKind()) {
1384 case xegpu::MemorySpace::Global:
1385 addrSpace = xevm::AddrSpace::GLOBAL;
1386 break;
1387 case xegpu::MemorySpace::SLM:
1388 addrSpace = xevm::AddrSpace::SHARED;
1389 break;
1390 }
1391 xevm::MemfenceOp::create(builder&: rewriter, location: loc, scope: memScope, addrspace: addrSpace);
1392 rewriter.eraseOp(op);
1393 return success();
1394 }
1395};
1396
1397static auto encodePrecision = [](Type type) -> xevm::ElemType {
1398 if (type.isBF16())
1399 return xevm::ElemType::BF16;
1400 else if (type.isF16())
1401 return xevm::ElemType::F16;
1402 else if (type.isTF32())
1403 return xevm::ElemType::TF32;
1404 else if (type.isInteger(width: 8)) {
1405 if (type.isUnsignedInteger())
1406 return xevm::ElemType::U8;
1407 return xevm::ElemType::S8;
1408 } else if (type.isF32())
1409 return xevm::ElemType::F32;
1410 else if (type.isInteger(width: 32))
1411 return xevm::ElemType::S32;
1412 else if (type.isF8E5M2())
1413 return xevm::ElemType::BF8;
1414 else if (type.isF8E4M3FN())
1415 return xevm::ElemType::F8;
1416 else if (mlir::isa<Float4E2M1FNType>(Val: type))
1417 return xevm::ElemType::E2M1;
1418 llvm_unreachable("add more support for ElemType");
1419};
1420
1421static unsigned getNumOperandsPerDword(xevm::ElemType pTy) {
1422 switch (pTy) {
1423 case xevm::ElemType::TF32:
1424 return 1;
1425 case xevm::ElemType::BF16:
1426 case xevm::ElemType::F16:
1427 return 2;
1428 case xevm::ElemType::U8:
1429 case xevm::ElemType::S8:
1430 case xevm::ElemType::F8:
1431 case xevm::ElemType::BF8:
1432 return 4;
1433 case xevm::ElemType::E2M1:
1434 return 8;
1435 default:
1436 llvm_unreachable("unsupported xevm::ElemType");
1437 }
1438}
1439
1440class DpasToXeVMPattern : public OpConversionPattern<xegpu::DpasOp> {
1441 using Base::Base;
1442 LogicalResult
1443 matchAndRewrite(xegpu::DpasOp op, xegpu::DpasOp::Adaptor adaptor,
1444 ConversionPatternRewriter &rewriter) const override {
1445 auto loc = op.getLoc();
1446 auto ctxt = rewriter.getContext();
1447 auto aTy = cast<VectorType>(Val: op.getLhs().getType());
1448 auto bTy = cast<VectorType>(Val: op.getRhs().getType());
1449 auto resultType = cast<VectorType>(Val: op.getResultType());
1450
1451 // get the correct dpasInst by getting info from chip
1452 auto chipStr = xegpu::getChipStr(op);
1453 if (!chipStr)
1454 return rewriter.notifyMatchFailure(arg&: op, msg: "cannot determine target chip");
1455
1456 const auto *uArch = mlir::xegpu::uArch::getUArch(archName: *chipStr);
1457 if (!uArch)
1458 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported target uArch");
1459
1460 auto *dpasInst = const_cast<xegpu::uArch::SubgroupMatrixMultiplyAcc *>(
1461 llvm::dyn_cast_or_null<xegpu::uArch::SubgroupMatrixMultiplyAcc>(
1462 Val: uArch->getInstruction(
1463 instKind: xegpu::uArch::InstructionKind::SubgroupMatrixMultiplyAcc)));
1464 if (!dpasInst)
1465 return rewriter.notifyMatchFailure(arg&: op,
1466 msg: "DPAS not supported by target uArch");
1467
1468 auto checkSupportedTypes = [&](VectorType vecTy,
1469 xegpu::uArch::MMAOpndKind kind) -> bool {
1470 auto supported = dpasInst->getSupportedTypes(context&: *ctxt, matrixType: kind);
1471 return llvm::find(Range&: supported, Val: vecTy.getElementType()) != supported.end();
1472 };
1473
1474 if (!checkSupportedTypes(aTy, xegpu::uArch::MMAOpndKind::MatrixA))
1475 return rewriter.notifyMatchFailure(
1476 arg&: op, msg: "A-matrix element type not supported by target uArch");
1477 if (!checkSupportedTypes(bTy, xegpu::uArch::MMAOpndKind::MatrixB))
1478 return rewriter.notifyMatchFailure(
1479 arg&: op, msg: "B-matrix element type not supported by target uArch");
1480 // NOTE: Supported types for MatrixC and MatrixD are identical
1481 if (!checkSupportedTypes(resultType, xegpu::uArch::MMAOpndKind::MatrixD))
1482 return rewriter.notifyMatchFailure(
1483 arg&: op, msg: "result/accumulator element type not supported by target uArch");
1484
1485 xevm::ElemType precATy = encodePrecision(aTy.getElementType());
1486 xevm::ElemType precBTy = encodePrecision(bTy.getElementType());
1487 Value c = op.getAcc();
1488 if (!c) {
1489 auto elementTy = resultType.getElementType();
1490 Attribute initValueAttr;
1491 if (isa<FloatType>(Val: elementTy))
1492 initValueAttr = FloatAttr::get(type: elementTy, value: 0.0);
1493 else
1494 initValueAttr = IntegerAttr::get(type: elementTy, value: 0);
1495 c = arith::ConstantOp::create(
1496 builder&: rewriter, location: loc, value: DenseElementsAttr::get(type: resultType, values: initValueAttr));
1497 }
1498
1499 Value aVec = op.getLhs();
1500 Value bVec = op.getRhs();
1501 auto cvecty = cast<VectorType>(Val: c.getType());
1502 xevm::ElemType precCTy = encodePrecision(cvecty.getElementType());
1503 xevm::ElemType precDTy = encodePrecision(resultType.getElementType());
1504 VectorType cNty =
1505 VectorType::get(shape: cvecty.getNumElements(), elementType: cvecty.getElementType());
1506 if (cvecty != cNty)
1507 c = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: cNty, source: c);
1508 Value dpasRes = xevm::MMAOp::create(
1509 builder&: rewriter, location: loc, d: cNty, a: aVec, b: bVec, c,
1510 shape: xevm::MMAShapeAttr::get(context: ctxt, m: cvecty.getNumElements(), n: executionSize,
1511 k: systolicDepth *
1512 getNumOperandsPerDword(pTy: precATy)),
1513 types: xevm::MMATypesAttr::get(context: ctxt, d: precDTy, a: precATy, b: precBTy, c: precCTy));
1514 if (cvecty != cNty)
1515 dpasRes = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: resultType, source: dpasRes);
1516 rewriter.replaceOp(op, newValues: dpasRes);
1517 return success();
1518 }
1519};
1520
1521static std::optional<LLVM::AtomicBinOp>
1522matchSimpleAtomicOp(arith::AtomicRMWKind arithKind) {
1523 switch (arithKind) {
1524 case arith::AtomicRMWKind::addf:
1525 return LLVM::AtomicBinOp::fadd;
1526 case arith::AtomicRMWKind::addi:
1527 return LLVM::AtomicBinOp::add;
1528 case arith::AtomicRMWKind::assign:
1529 return LLVM::AtomicBinOp::xchg;
1530 case arith::AtomicRMWKind::maximumf:
1531 return LLVM::AtomicBinOp::fmax;
1532 case arith::AtomicRMWKind::maxs:
1533 return LLVM::AtomicBinOp::max;
1534 case arith::AtomicRMWKind::maxu:
1535 return LLVM::AtomicBinOp::umax;
1536 case arith::AtomicRMWKind::minimumf:
1537 return LLVM::AtomicBinOp::fmin;
1538 case arith::AtomicRMWKind::mins:
1539 return LLVM::AtomicBinOp::min;
1540 case arith::AtomicRMWKind::minu:
1541 return LLVM::AtomicBinOp::umin;
1542 case arith::AtomicRMWKind::ori:
1543 return LLVM::AtomicBinOp::_or;
1544 case arith::AtomicRMWKind::andi:
1545 return LLVM::AtomicBinOp::_and;
1546 default:
1547 return std::nullopt;
1548 }
1549}
1550
1551class AtomicRMWToXeVMPattern : public OpConversionPattern<xegpu::AtomicRMWOp> {
1552 using Base::Base;
1553 LogicalResult
1554 matchAndRewrite(xegpu::AtomicRMWOp op, xegpu::AtomicRMWOp::Adaptor adaptor,
1555 ConversionPatternRewriter &rewriter) const override {
1556 auto loc = op.getLoc();
1557 auto ctxt = rewriter.getContext();
1558 auto tdesc = op.getTensorDesc().getType();
1559 auto ptrTypeLLVM = LLVM::LLVMPointerType::get(
1560 context: ctxt, addressSpace: getNumericXeVMAddrSpace(xeGpuMemspace: tdesc.getMemorySpace()));
1561 Value basePtrI64 = arith::IndexCastOp::create(
1562 builder&: rewriter, location: loc, out: rewriter.getI64Type(), in: adaptor.getTensorDesc());
1563 Value basePtrLLVM =
1564 LLVM::IntToPtrOp::create(builder&: rewriter, location: loc, resultType: ptrTypeLLVM, operands: basePtrI64);
1565 VectorType srcOrDstVecTy = cast<VectorType>(Val: op.getValue().getType());
1566 VectorType srcOrDstFlatVecTy = VectorType::get(
1567 shape: srcOrDstVecTy.getNumElements(), elementType: srcOrDstVecTy.getElementType());
1568 Value srcFlatVec = vector::ShapeCastOp::create(
1569 builder&: rewriter, location: loc, result: srcOrDstFlatVecTy, source: op.getValue());
1570 auto atomicKind = matchSimpleAtomicOp(arithKind: op.getKind());
1571 assert(atomicKind.has_value());
1572 Value resVec = srcFlatVec;
1573 for (int i = 0; i < srcOrDstVecTy.getNumElements(); i++) {
1574 auto val = vector::ExtractOp::create(builder&: rewriter, location: loc, source: resVec, position: i);
1575 Value idx = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: rewriter.getI64Type(),
1576 value: rewriter.getI64IntegerAttr(value: i));
1577 Value currPtr =
1578 LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrTypeLLVM,
1579 elementType: srcOrDstVecTy.getElementType(), basePtr: basePtrLLVM, indices: idx);
1580 Value newVal =
1581 LLVM::AtomicRMWOp::create(builder&: rewriter, location: loc, binOp: atomicKind.value(), ptr: currPtr,
1582 val, ordering: LLVM::AtomicOrdering::seq_cst);
1583 resVec = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: newVal, dest: resVec, position: i);
1584 }
1585 rewriter.replaceOp(op, newValues: resVec);
1586 return success();
1587 }
1588};
1589
1590class DpasMxToXeVMPattern : public OpConversionPattern<xegpu::DpasMxOp> {
1591 using Base::Base;
1592 LogicalResult
1593 matchAndRewrite(xegpu::DpasMxOp op, xegpu::DpasMxOp::Adaptor adaptor,
1594 ConversionPatternRewriter &rewriter) const override {
1595 auto loc = op.getLoc();
1596 auto ctxt = rewriter.getContext();
1597 auto aTy = op.getA().getType();
1598 auto bTy = op.getB().getType();
1599 auto resVecTy =
1600 cast<VectorType>(Val: getTypeConverter()->convertType(t: op.getType()));
1601
1602 auto chipStr = xegpu::getChipStr(op);
1603 if (!chipStr)
1604 return rewriter.notifyMatchFailure(arg&: op, msg: "cannot determine target chip");
1605
1606 const auto *uArch = xegpu::uArch::getUArch(archName: *chipStr);
1607 if (!uArch)
1608 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported target uArch");
1609
1610 // TODO: Add supported shape check
1611
1612 xevm::ElemType precATy = encodePrecision(aTy.getElementType());
1613 xevm::ElemType precBTy = encodePrecision(bTy.getElementType());
1614 Value c = adaptor.getAcc();
1615 if (!c) {
1616 auto elementTy = resVecTy.getElementType();
1617 Attribute initValueAttr;
1618 if (isa<FloatType>(Val: elementTy))
1619 initValueAttr = FloatAttr::get(type: elementTy, value: 0.0);
1620 else
1621 initValueAttr = IntegerAttr::get(type: elementTy, value: 0);
1622 c = arith::ConstantOp::create(
1623 builder&: rewriter, location: loc, value: DenseElementsAttr::get(type: resVecTy, values: initValueAttr));
1624 }
1625
1626 Value aVec = adaptor.getA();
1627 Value bVec = adaptor.getB();
1628 auto aVecTy = cast<VectorType>(Val: aVec.getType());
1629 auto bVecTy = cast<VectorType>(Val: bVec.getType());
1630 if (aVecTy.getElementTypeBitWidth() == 4)
1631 aVec = vector::BitCastOp::create(
1632 builder&: rewriter, location: loc,
1633 result: VectorType::get(shape: aVecTy.getNumElements() / 2, elementType: rewriter.getI8Type()),
1634 source: aVec);
1635 if (bVecTy.getElementTypeBitWidth() == 4)
1636 bVec = vector::BitCastOp::create(
1637 builder&: rewriter, location: loc,
1638 result: VectorType::get(shape: bVecTy.getNumElements() / 2, elementType: rewriter.getI8Type()),
1639 source: bVec);
1640 auto cVecTy = cast<VectorType>(Val: c.getType());
1641 xevm::ElemType precCTy = encodePrecision(cVecTy.getElementType());
1642 xevm::ElemType precDTy = encodePrecision(resVecTy.getElementType());
1643 Value scaleA = adaptor.getScaleA();
1644 Value scaleB = adaptor.getScaleB();
1645 Value dpasMxRes = xevm::MMAMxOp::create(
1646 builder&: rewriter, location: loc, d: resVecTy, a: aVec, b: bVec, scale_a: scaleA, scale_b: scaleB, c,
1647 shape: xevm::MMAShapeAttr::get(context: ctxt, m: cVecTy.getNumElements(), n: executionSize,
1648 k: systolicDepth *
1649 getNumOperandsPerDword(pTy: precATy)),
1650 types: xevm::MMATypesAttr::get(context: ctxt, d: precDTy, a: precATy, b: precBTy, c: precCTy));
1651 rewriter.replaceOp(op, newValues: dpasMxRes);
1652 return success();
1653 }
1654};
1655
1656//===----------------------------------------------------------------------===//
1657// arith.extf / arith.truncf to xevm.extf / xevm.truncf
1658//===----------------------------------------------------------------------===//
1659//
1660// Micro-scaling (MX) GEMM lowering breaks arith.scaling_extf/scaling_truncf
1661// into plain arith.extf/arith.truncf whose narrow side uses one of the MX float
1662// formats (f8E5M2, f8E4M3FN or f4E2M1FN). These narrow floats have no native
1663// LLVM support, so the conversions are mapped onto the dedicated xevm.extf /
1664// xevm.truncf ops which lower to hardware builtins. The f8E8M0FNU scale type is
1665// intentionally not handled here: it is expanded into integer arithmetic by
1666// arith-expand before this pass runs.
1667
1668// xevm.extf / xevm.truncf only convert between the MX narrow floats and
1669// f16/bf16, and the underlying builtins operate on exactly 16 f16/bf16 values.
1670static constexpr int64_t kXeVMExtfTruncfNumElems = 16;
1671
1672// Maps a narrow MX float element type to the matching xevm.extf source enum.
1673static std::optional<xevm::ExtfSrcElemTypes> getExtfNarrowType(Type etype) {
1674 if (isa<Float8E5M2Type>(Val: etype))
1675 return xevm::ExtfSrcElemTypes::BF8;
1676 if (isa<Float8E4M3FNType>(Val: etype))
1677 return xevm::ExtfSrcElemTypes::F8;
1678 if (isa<Float4E2M1FNType>(Val: etype))
1679 return xevm::ExtfSrcElemTypes::E2M1;
1680 return std::nullopt;
1681}
1682
1683// Maps a narrow MX float element type to the matching xevm.truncf dest enum.
1684static std::optional<xevm::TruncfDstElemTypes> getTruncfNarrowType(Type etype) {
1685 if (isa<Float8E5M2Type>(Val: etype))
1686 return xevm::TruncfDstElemTypes::BF8;
1687 if (isa<Float8E4M3FNType>(Val: etype))
1688 return xevm::TruncfDstElemTypes::F8;
1689 if (isa<Float4E2M1FNType>(Val: etype))
1690 return xevm::TruncfDstElemTypes::E2M1;
1691 return std::nullopt;
1692}
1693
1694// Returns true if `op` is an arith.extf that can be lowered to xevm.extf, i.e.
1695// a rank-1 widening from an MX narrow float to a 16-element f16/bf16 vector.
1696static bool isXeVMExtf(arith::ExtFOp op) {
1697 auto srcTy = dyn_cast<VectorType>(Val: op.getIn().getType());
1698 auto dstTy = dyn_cast<VectorType>(Val: op.getType());
1699 if (!srcTy || !dstTy || srcTy.getRank() != 1 || dstTy.getRank() != 1)
1700 return false;
1701 if (dstTy.getNumElements() != kXeVMExtfTruncfNumElems)
1702 return false;
1703 Type dstETy = dstTy.getElementType();
1704 if (!dstETy.isF16() && !dstETy.isBF16())
1705 return false;
1706 return getExtfNarrowType(etype: srcTy.getElementType()).has_value();
1707}
1708
1709// Returns true if `op` is an arith.truncf that can be lowered to xevm.truncf,
1710// i.e. a rank-1 truncation from an f16/bf16 vector to an MX narrow float. The
1711// source has to hold a whole number of the fixed-size groups xevm.truncf
1712// converts at a time; wider vectors are converted in several steps.
1713static bool isXeVMTruncf(arith::TruncFOp op) {
1714 auto srcTy = dyn_cast<VectorType>(Val: op.getIn().getType());
1715 auto dstTy = dyn_cast<VectorType>(Val: op.getType());
1716 if (!srcTy || !dstTy || srcTy.getRank() != 1 || dstTy.getRank() != 1)
1717 return false;
1718 int64_t numElems = srcTy.getNumElements();
1719 if (numElems == 0 || numElems % kXeVMExtfTruncfNumElems != 0)
1720 return false;
1721 Type srcETy = srcTy.getElementType();
1722 if (!srcETy.isF16() && !srcETy.isBF16())
1723 return false;
1724 return getTruncfNarrowType(etype: dstTy.getElementType()).has_value();
1725}
1726
1727class ExtfToXeVMPattern : public OpConversionPattern<arith::ExtFOp> {
1728 using Base::Base;
1729 LogicalResult
1730 matchAndRewrite(arith::ExtFOp op, OpAdaptor adaptor,
1731 ConversionPatternRewriter &rewriter) const override {
1732 if (!isXeVMExtf(op))
1733 return rewriter.notifyMatchFailure(arg&: op, msg: "not a xevm.extf compatible extf");
1734 Location loc = op.getLoc();
1735 MLIRContext *ctx = op.getContext();
1736 auto srcVecTy = cast<VectorType>(Val: op.getIn().getType());
1737 auto dstVecTy = cast<VectorType>(Val: op.getType());
1738 xevm::ExtfSrcElemTypes srcEnum =
1739 *getExtfNarrowType(etype: srcVecTy.getElementType());
1740 xevm::ExtfDstElemTypes dstEnum = dstVecTy.getElementType().isF16()
1741 ? xevm::ExtfDstElemTypes::F16
1742 : xevm::ExtfDstElemTypes::BF16;
1743 // The narrow float operand has already been type-converted to an integer
1744 // vector of the same bit width (i4 for fp4, i8 for fp8). xevm.extf takes
1745 // the values packed into an i8 vector, so re-pack fp4 (i4) operands.
1746 Value src = adaptor.getIn();
1747 auto convSrcTy = cast<VectorType>(Val: src.getType());
1748 if (convSrcTy.getElementTypeBitWidth() == 4)
1749 src = vector::BitCastOp::create(
1750 builder&: rewriter, location: loc,
1751 result: VectorType::get(shape: convSrcTy.getNumElements() / 2, elementType: rewriter.getI8Type()),
1752 source: src);
1753 Type resTy = getTypeConverter()->convertType(t: dstVecTy);
1754 Value res = xevm::ExtfOp::create(
1755 builder&: rewriter, location: loc, dst: resTy, src, src_etype: xevm::ExtfSrcElemTypeAttr::get(context: ctx, etype: srcEnum),
1756 dst_etype: xevm::ExtfDstElemTypeAttr::get(context: ctx, etype: dstEnum));
1757 rewriter.replaceOp(op, newValues: res);
1758 return success();
1759 }
1760};
1761
1762class TruncfToXeVMPattern : public OpConversionPattern<arith::TruncFOp> {
1763 using Base::Base;
1764 LogicalResult
1765 matchAndRewrite(arith::TruncFOp op, OpAdaptor adaptor,
1766 ConversionPatternRewriter &rewriter) const override {
1767 if (!isXeVMTruncf(op))
1768 return rewriter.notifyMatchFailure(arg&: op,
1769 msg: "not a xevm.truncf compatible truncf");
1770 Location loc = op.getLoc();
1771 MLIRContext *ctx = op.getContext();
1772 auto srcVecTy = cast<VectorType>(Val: op.getIn().getType());
1773 auto dstVecTy = cast<VectorType>(Val: op.getType());
1774 xevm::TruncfSrcElemTypes srcEnum = srcVecTy.getElementType().isF16()
1775 ? xevm::TruncfSrcElemTypes::F16
1776 : xevm::TruncfSrcElemTypes::BF16;
1777 xevm::TruncfDstElemTypes dstEnum =
1778 *getTruncfNarrowType(etype: dstVecTy.getElementType());
1779 auto srcEnumAttr = xevm::TruncfSrcElemTypeAttr::get(context: ctx, etype: srcEnum);
1780 auto dstEnumAttr = xevm::TruncfDstElemTypeAttr::get(context: ctx, etype: dstEnum);
1781
1782 // xevm.truncf lowers to instructions that convert a fixed number of
1783 // elements at a time, so a wider source is converted one group at a time
1784 // and the packed results are concatenated. Each group produces the narrow
1785 // floats packed into an i8 vector.
1786 int64_t numGroups = srcVecTy.getNumElements() / kXeVMExtfTruncfNumElems;
1787 int64_t groupBytes =
1788 kXeVMExtfTruncfNumElems * dstVecTy.getElementTypeBitWidth() / 8;
1789 Type groupTy = VectorType::get(shape: groupBytes, elementType: rewriter.getI8Type());
1790
1791 Value src = adaptor.getIn();
1792 Value packed;
1793 if (numGroups == 1) {
1794 packed = xevm::TruncfOp::create(builder&: rewriter, location: loc, dst: groupTy, src, src_etype: srcEnumAttr,
1795 dst_etype: dstEnumAttr);
1796 } else {
1797 auto packedTy =
1798 VectorType::get(shape: groupBytes * numGroups, elementType: rewriter.getI8Type());
1799 packed = arith::ConstantOp::create(builder&: rewriter, location: loc, result: packedTy,
1800 value: rewriter.getZeroAttr(type: packedTy));
1801 for (int64_t group = 0; group < numGroups; group++) {
1802 Value slice = vector::ExtractStridedSliceOp::create(
1803 builder&: rewriter, location: loc, source: src, offsets: group * kXeVMExtfTruncfNumElems,
1804 sizes: kXeVMExtfTruncfNumElems, /*strides=*/1);
1805 Value converted = xevm::TruncfOp::create(builder&: rewriter, location: loc, dst: groupTy, src: slice,
1806 src_etype: srcEnumAttr, dst_etype: dstEnumAttr);
1807 packed = vector::InsertStridedSliceOp::create(
1808 builder&: rewriter, location: loc, valueToStore: converted, dest: packed, offsets: group * groupBytes,
1809 /*strides=*/1);
1810 }
1811 }
1812 // Re-shape to the type-converted result type (i4 vector for fp4).
1813 Type resTy = getTypeConverter()->convertType(t: dstVecTy);
1814 if (packed.getType() != resTy)
1815 packed = vector::BitCastOp::create(builder&: rewriter, location: loc, result: resTy, source: packed);
1816 rewriter.replaceOp(op, newValues: packed);
1817 return success();
1818 }
1819};
1820
1821// Hoist the i4-to-i8 part of compatible extensions through a closed
1822// data-movement graph and lane shuffle. Widening all the way to the extension's
1823// result type could exceed the bitcast-shuffle lowering's 64-bit fragment
1824// limit. Signed widening preserves the eventual signed interpretation of each
1825// nibble; unsigned widening preserves its unsigned interpretation.
1826template <typename ExtOp>
1827class HoistI4ExtensionAboveLaneShuffle final : public OpRewritePattern<ExtOp> {
1828public:
1829 using OpRewritePattern<ExtOp>::OpRewritePattern;
1830
1831 LogicalResult matchAndRewrite(ExtOp op,
1832 PatternRewriter &rewriter) const override {
1833 auto srcTy = dyn_cast<VectorType>(op.getIn().getType());
1834 auto dstTy = dyn_cast<VectorType>(op.getType());
1835 if (!srcTy || !srcTy.getElementType().isInteger(4) || !dstTy ||
1836 dstTy.getElementTypeBitWidth() < 8)
1837 return failure();
1838
1839 auto isMover = [](Operation *def) {
1840 return isa<vector::ShapeCastOp, vector::ExtractStridedSliceOp,
1841 vector::ExtractOp, vector::BroadcastOp, vector::TransposeOp>(
1842 Val: def);
1843 };
1844 Value source = op.getIn();
1845 while (Operation *def = source.getDefiningOp()) {
1846 if (isa<xegpu::LaneShuffleOp>(Val: def))
1847 break;
1848 if (!isMover(def))
1849 return failure();
1850 source = def->getOperand(idx: 0);
1851 }
1852 auto shuffle = source.getDefiningOp<xegpu::LaneShuffleOp>();
1853 if (!shuffle)
1854 return failure();
1855 VectorType fragmentTy = shuffle.getSourceType();
1856 int64_t numElements = fragmentTy.getNumElements();
1857 if (!fragmentTy.getElementType().isInteger(width: 4) || numElements < 2 ||
1858 numElements > 8 || !llvm::isPowerOf2_64(Value: numElements))
1859 return failure();
1860
1861 // Validate every use before changing IR. Widening a shared fragment is
1862 // safe only when all paths end in extensions with the same signedness.
1863 SmallVector<Operation *> movers;
1864 SmallVector<ExtOp> extensions;
1865 SmallPtrSet<Operation *, 16> visited;
1866 auto visit = [&](auto &&visit, Value value) -> LogicalResult {
1867 for (Operation *user : value.getUsers()) {
1868 if (!visited.insert(Ptr: user).second)
1869 continue;
1870 if (auto ext = dyn_cast<ExtOp>(user)) {
1871 auto type = dyn_cast<VectorType>(ext.getType());
1872 if (!type || type.getElementTypeBitWidth() < 8)
1873 return failure();
1874 extensions.push_back(ext);
1875 continue;
1876 }
1877 if (!isMover(user) || user->getOperand(idx: 0) != value ||
1878 user->getNumResults() != 1 ||
1879 failed(visit(visit, user->getResult(idx: 0))))
1880 return failure();
1881 movers.push_back(Elt: user);
1882 }
1883 return success();
1884 };
1885 if (failed(visit(visit, shuffle.getResult())))
1886 return failure();
1887
1888 rewriter.setInsertionPoint(shuffle);
1889 VectorType wideTy = fragmentTy.clone(elementType: rewriter.getI8Type());
1890 Value wide =
1891 ExtOp::create(rewriter, op.getLoc(), wideTy, shuffle.getSource());
1892 Value result = xegpu::LaneShuffleOp::create(
1893 builder&: rewriter, location: shuffle.getLoc(), result: wideTy, source: wide, mode: shuffle.getMode());
1894 IRMapping mapping;
1895 mapping.map(from: shuffle.getResult(), to: result);
1896 for (Operation *mover : llvm::reverse(C&: movers)) {
1897 rewriter.setInsertionPoint(mover);
1898 Operation *clone = rewriter.clone(op&: *mover, mapper&: mapping);
1899 Type type = rewriter.getI8Type();
1900 if (auto vectorTy = dyn_cast<VectorType>(Val: mover->getResult(idx: 0).getType()))
1901 type = vectorTy.clone(elementType: type);
1902 clone->getResult(idx: 0).setType(type);
1903 }
1904 for (ExtOp ext : extensions) {
1905 Value input = mapping.lookup(ext.getIn());
1906 if (input.getType() == ext.getType())
1907 rewriter.replaceOp(ext, input);
1908 else
1909 rewriter.modifyOpInPlace(ext,
1910 [&] { ext.getInMutable().assign(input); });
1911 }
1912 for (Operation *mover : movers)
1913 rewriter.eraseOp(op: mover);
1914 rewriter.eraseOp(op: shuffle);
1915 return success();
1916 }
1917};
1918
1919// Lowers `xegpu.lane_shuffle` to `xevm.bitcast_shuffle`.
1920//
1921// `xevm.bitcast_shuffle` concatenates the components of its operand across the
1922// subgroup, the first component of every lane first, and then hands chunks the
1923// size of a result component back out to the lanes in order. Numbering the
1924// elements of a `vector<NxT>` fragment held by lane `i` of a subgroup of size
1925// `S` by their logical position, that concatenation is exactly the `pack` mode
1926// input numbering `j * S + i`. Taking the result as a single `N * width(T)` bit
1927// scalar then hands lane `i` the logical positions `i * N .. i * N + N - 1`,
1928// which is the `pack` mode output numbering.
1929//
1930// So `pack` is a vector-to-scalar `xevm.bitcast_shuffle` followed by a bitcast
1931// back to the fragment type, and `unpack`, being its inverse, is a bitcast to
1932// the scalar followed by a scalar-to-vector `xevm.bitcast_shuffle`.
1933//
1934// `xevm.bitcast_shuffle` only accepts the integer types `i8`, `i16`, `i32` and
1935// `i64`, since it is bit-preserving and so does not depend on how the bits are
1936// interpreted. A fragment of a floating point type is therefore bitcast to a
1937// same-width integer vector on the way in and back on the way out.
1938class LaneShuffleToXeVMPattern
1939 : public OpConversionPattern<xegpu::LaneShuffleOp> {
1940 using Base::Base;
1941 LogicalResult
1942 matchAndRewrite(xegpu::LaneShuffleOp op, OpAdaptor adaptor,
1943 ConversionPatternRewriter &rewriter) const override {
1944 auto vecTy = dyn_cast<VectorType>(Val: adaptor.getSource().getType());
1945 if (!vecTy)
1946 return rewriter.notifyMatchFailure(arg&: op, msg: "Expected a vector fragment.");
1947 // The shuffle redistributes whole bytes between the lanes, so sub-byte
1948 // element types, fp4 in particular, cannot be shuffled. Widths without a
1949 // matching integer type the op accepts are rejected for the same reason.
1950 unsigned elemBits = vecTy.getElementTypeBitWidth();
1951 if (elemBits != 8 && elemBits != 16 && elemBits != 32 && elemBits != 64)
1952 return rewriter.notifyMatchFailure(
1953 arg&: op, msg: "Expected an element type of 8, 16, 32 or 64 bits.");
1954 int64_t fragmentBits = vecTy.getNumElements() * elemBits;
1955 if (fragmentBits > 64 || !llvm::isPowerOf2_64(Value: fragmentBits))
1956 return rewriter.notifyMatchFailure(
1957 arg&: op, msg: "Expected a fragment of 8, 16, 32 or 64 bits.");
1958
1959 Location loc = op.getLoc();
1960 Type packedTy = rewriter.getIntegerType(width: fragmentBits);
1961 // The integer vector type the shuffle actually operates on. Equal to the
1962 // fragment type when that is already an integer vector.
1963 VectorType shuffleTy =
1964 VectorType::get(shape: vecTy.getShape(), elementType: rewriter.getIntegerType(width: elemBits));
1965
1966 Value res;
1967 if (op.getMode() == xegpu::LaneShuffleMode::Pack) {
1968 Value src = adaptor.getSource();
1969 if (shuffleTy != vecTy)
1970 src = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: shuffleTy, arg: src);
1971 res = xevm::BitcastShuffleOp::create(builder&: rewriter, location: loc, res: packedTy, src);
1972 res = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: vecTy, arg: res);
1973 } else {
1974 Value packed =
1975 LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: packedTy, arg: adaptor.getSource());
1976 res = xevm::BitcastShuffleOp::create(builder&: rewriter, location: loc, res: shuffleTy, src: packed);
1977 if (shuffleTy != vecTy)
1978 res = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: vecTy, arg: res);
1979 }
1980 rewriter.replaceOp(op, newValues: res);
1981 return success();
1982 }
1983};
1984
1985//===----------------------------------------------------------------------===//
1986// Pass Definition
1987//===----------------------------------------------------------------------===//
1988
1989struct ConvertXeGPUToXeVMPass
1990 : public impl::ConvertXeGPUToXeVMPassBase<ConvertXeGPUToXeVMPass> {
1991 using Base::Base;
1992
1993 void runOnOperation() override {
1994 MLIRContext *context = &getContext();
1995
1996 // Restrict greedy rewriting to casts and newly generated operations so
1997 // unrelated dead XeGPU ops still receive conversion diagnostics.
1998 GreedyRewriteConfig config;
1999 config.setStrictness(GreedyRewriteStrictness::ExistingAndNewOps);
2000 // Preserve extension consumers until hoisting has finished: narrow-type
2001 // legalization would otherwise obscure the chains this peephole matches.
2002 {
2003 RewritePatternSet patterns(context);
2004 patterns.add<HoistI4ExtensionAboveLaneShuffle<arith::ExtUIOp>,
2005 HoistI4ExtensionAboveLaneShuffle<arith::ExtSIOp>>(arg&: context);
2006 SmallVector<Operation *> extensions;
2007 getOperation()->walk(callback: [&](Operation *op) {
2008 if (isa<arith::ExtUIOp, arith::ExtSIOp>(Val: op))
2009 extensions.push_back(Elt: op);
2010 });
2011 if (failed(
2012 Result: applyOpPatternsGreedily(ops: extensions, patterns: std::move(patterns), config)))
2013 return signalPassFailure();
2014 }
2015 // Expand the new i4-to-i8 casts into byte-level masks and shifts before
2016 // lowering. LLVM extensions on i4 vectors are not legal in the backend.
2017 {
2018 RewritePatternSet patterns(context);
2019 vector::populateVectorNarrowTypeRewritePatterns(patterns);
2020 vector::populateVectorInterleaveToShufflePatterns(patterns);
2021 vector::populateVectorDeinterleaveToShufflePatterns(patterns);
2022 SmallVector<Operation *> casts;
2023 getOperation()->walk(callback: [&](Operation *op) {
2024 if (isa<arith::ExtUIOp, arith::ExtSIOp, arith::TruncIOp,
2025 vector::BitCastOp>(Val: op))
2026 casts.push_back(Elt: op);
2027 });
2028 if (failed(Result: applyOpPatternsGreedily(ops: casts, patterns: std::move(patterns), config)))
2029 return signalPassFailure();
2030 }
2031
2032 // XeVM type converter is based on LLVM type converter with the
2033 // following customizations.
2034 // First, type conversion rules are added for xegpu custom types,
2035 // TensorDescType and MemDescType.
2036 // Second, MemRefType is lowered to single integer type
2037 // Third, VectorType of single element or 0D is converted to vector
2038 // element type. Otherwise, vector type is flatten to 1D.
2039 LowerToLLVMOptions options(context);
2040 options.overrideIndexBitwidth(bitwidth: this->use64bitIndex ? 64 : 32);
2041 LLVMTypeConverter typeConverter(context, options);
2042
2043 Type xevmIndexType = typeConverter.convertType(t: IndexType::get(context));
2044 Type i32Type = IntegerType::get(context, width: 32);
2045 typeConverter.addConversion(callback: [&](VectorType type) -> Type {
2046 auto elemType = typeConverter.convertType(t: type.getElementType());
2047 // If the vector rank is 0 or has a single element, return the element
2048 unsigned rank = type.getRank();
2049 if (rank == 0 || type.getNumElements() == 1)
2050 return elemType;
2051 // Otherwise, convert the vector to a flat vector type.
2052 int64_t sum = llvm::product_of(Range: type.getShape());
2053 return VectorType::get(shape: sum, elementType: elemType);
2054 });
2055 typeConverter.addConversion(callback: [&](xegpu::TensorDescType type) -> Type {
2056 if (type.getRank() == 1)
2057 return xevmIndexType;
2058 return VectorType::get(shape: 8, elementType: i32Type);
2059 });
2060 // SLM access related type conversions.
2061 // TODO: LLVM DLTI provides clean way of representing different pointer size
2062 // based on address space. Currently pointer size of SLM access is hard
2063 // coded to 32bit. Update to use DLTI when switching overall XeGPU lowering
2064 // to use DLTI instead of use64bitIndex option used above.
2065
2066 // Convert MemDescType into i32 for SLM
2067 typeConverter.addConversion(
2068 callback: [&](xegpu::MemDescType type) -> Type { return i32Type; });
2069
2070 typeConverter.addConversion(callback: [&](MemRefType type) -> Type {
2071 return isSharedMemRef(memrefTy: type) ? i32Type : xevmIndexType;
2072 });
2073
2074 // LLVM type converter puts unrealized casts for the following cases:
2075 // add materialization casts to handle them.
2076
2077 // Materialization to convert memref to i64 or i32 depending on global/SLM
2078 // Applies only to target materialization.
2079 // Note: int type to memref materialization is not required as xegpu ops
2080 // currently do not produce memrefs as result.
2081 auto memrefToIntMaterializationCast = [](OpBuilder &builder, Type type,
2082 ValueRange inputs,
2083 Location loc) -> Value {
2084 if (inputs.size() != 1)
2085 return {};
2086 auto input = inputs.front();
2087 if (auto memrefTy = dyn_cast<MemRefType>(Val: input.getType())) {
2088 unsigned rank = memrefTy.getRank();
2089 Type indexType = builder.getIndexType();
2090
2091 int64_t intOffsets;
2092 SmallVector<int64_t> intStrides;
2093 Value addr;
2094 Value offset;
2095 if (succeeded(Result: memrefTy.getStridesAndOffset(strides&: intStrides, offset&: intOffsets)) &&
2096 ShapedType::isStatic(dValue: intOffsets)) {
2097 addr = memref::ExtractAlignedPointerAsIndexOp::create(builder, location: loc,
2098 source: input);
2099 offset = arith::ConstantOp::create(builder, location: loc,
2100 value: builder.getIndexAttr(value: intOffsets));
2101 } else {
2102
2103 // Result types: [base_memref, offset, stride0, stride1, ...,
2104 // strideN-1, size0, size1, ..., sizeN-1]
2105 SmallVector<Type> resultTypes{
2106 MemRefType::get(shape: {}, elementType: memrefTy.getElementType(),
2107 layout: MemRefLayoutAttrInterface(),
2108 memorySpace: memrefTy.getMemorySpace()),
2109 indexType};
2110 // strides + sizes
2111 resultTypes.append(NumInputs: 2 * rank, Elt: indexType);
2112
2113 auto meta = memref::ExtractStridedMetadataOp::create(
2114 builder, location: loc, resultTypes, source: input);
2115
2116 addr = memref::ExtractAlignedPointerAsIndexOp::create(
2117 builder, location: loc, source: meta.getBaseBuffer());
2118 offset = meta.getOffset();
2119 }
2120
2121 auto addrCasted =
2122 arith::IndexCastUIOp::create(builder, location: loc, out: type, in: addr);
2123 auto offsetCasted =
2124 arith::IndexCastUIOp::create(builder, location: loc, out: type, in: offset);
2125
2126 // Compute the final address: base address + byte offset.
2127 return xegpu::addElemOffsetInBytes(builder, loc, baseAddr: addrCasted,
2128 elemOffset: offsetCasted, memrefTy);
2129 }
2130 return {};
2131 };
2132
2133 // Materialization to convert ui64 to i64
2134 // Applies only to target materialization.
2135 // Note: i64 to ui64 materialization is not required as xegpu ops
2136 // currently do not produce ui64 as result.
2137 auto ui64ToI64MaterializationCast = [](OpBuilder &builder, Type type,
2138 ValueRange inputs,
2139 Location loc) -> Value {
2140 if (inputs.size() != 1)
2141 return {};
2142 auto input = inputs.front();
2143 if (input.getType() == builder.getIntegerType(width: 64, isSigned: false)) {
2144 Value cast =
2145 index::CastUOp::create(builder, location: loc, output: builder.getIndexType(), input)
2146 .getResult();
2147 return arith::IndexCastUIOp::create(builder, location: loc, out: type, in: cast)
2148 .getResult();
2149 }
2150 return {};
2151 };
2152
2153 // Materialization to convert ui32 to i32
2154 // Applies only to target materialization.
2155 // Note: i32 to ui32 materialization is not required as xegpu ops
2156 // currently do not produce ui32 as result.
2157 auto ui32ToI32MaterializationCast = [](OpBuilder &builder, Type type,
2158 ValueRange inputs,
2159 Location loc) -> Value {
2160 if (inputs.size() != 1)
2161 return {};
2162 auto input = inputs.front();
2163 if (input.getType() == builder.getIntegerType(width: 32, isSigned: false)) {
2164 Value cast =
2165 index::CastUOp::create(builder, location: loc, output: builder.getIndexType(), input)
2166 .getResult();
2167 return arith::IndexCastUIOp::create(builder, location: loc, out: type, in: cast)
2168 .getResult();
2169 }
2170 return {};
2171 };
2172
2173 // Materialization to convert between vector types
2174 // - Add shape cast for different shapes
2175 // - Add bitcast for different element types
2176 // Applies to both source and target materialization.
2177 auto vectorToVectorMaterializationCast = [](OpBuilder &builder, Type type,
2178 ValueRange inputs,
2179 Location loc) -> Value {
2180 if (inputs.size() != 1)
2181 return {};
2182 auto input = inputs.front();
2183 if (auto vecTy = dyn_cast<VectorType>(Val: input.getType())) {
2184 if (auto targetVecTy = dyn_cast<VectorType>(Val&: type)) {
2185 Value cast = input;
2186 // If the target type has a different shape, add a shape cast
2187 // If the target type has a different element type, add a bitcast
2188 if (targetVecTy.getShape() != vecTy.getShape()) {
2189 cast = vector::ShapeCastOp::create(
2190 builder, location: loc,
2191 result: VectorType::get(shape: targetVecTy.getShape(),
2192 elementType: vecTy.getElementType()),
2193 source: cast)
2194 .getResult();
2195 }
2196 if (targetVecTy.getElementType() != vecTy.getElementType()) {
2197 cast = vector::BitCastOp::create(builder, location: loc, result: targetVecTy, source: cast)
2198 .getResult();
2199 }
2200 return cast;
2201 }
2202 }
2203 return {};
2204 };
2205
2206 // Materialization to convert
2207 // - single element vector to single element of vector element type
2208 // Applies only to target materialization.
2209 auto vectorToSingleElementMaterializationCast =
2210 [](OpBuilder &builder, Type type, ValueRange inputs,
2211 Location loc) -> Value {
2212 if (inputs.size() != 1)
2213 return {};
2214 auto input = inputs.front();
2215 if (auto vecTy = dyn_cast<VectorType>(Val: input.getType())) {
2216 // Source needs to be single element vector
2217 auto rank = vecTy.getRank();
2218 if (rank != 0 && vecTy.getNumElements() != 1)
2219 return {};
2220 auto inElemTy = vecTy.getElementType();
2221 // extract scalar
2222 Value cast = input;
2223 if (rank == 0) {
2224 cast = vector::ExtractOp::create(builder, location: loc, source: cast, position: {}).getResult();
2225 } else {
2226 cast = vector::ExtractOp::create(builder, location: loc, source: cast,
2227 position: SmallVector<int64_t>(rank, 0))
2228 .getResult();
2229 }
2230 // Extracted element type may need conversion
2231 // Two cases
2232 // 1. Index type to integer type
2233 // 2. Other element type mismatch
2234 if (inElemTy.isIndex()) {
2235 cast = arith::IndexCastUIOp::create(builder, location: loc, out: type, in: cast)
2236 .getResult();
2237 } else if (inElemTy != type) {
2238 cast = arith::BitcastOp::create(builder, location: loc, out: type, in: cast).getResult();
2239 }
2240 return cast;
2241 }
2242 return {};
2243 };
2244
2245 // Materialization to convert
2246 // - single element of vector element type to single element vector
2247 // If result type of original op is single element vector and lowered type
2248 // is scalar. This materialization cast creates a single element vector by
2249 // First convert element type if needed and then broadcast to single
2250 // element vector.
2251 // Applies only to source materialization.
2252 auto singleElementToVectorMaterializationCast =
2253 [](OpBuilder &builder, Type type, ValueRange inputs,
2254 Location loc) -> Value {
2255 if (inputs.size() != 1)
2256 return {};
2257 auto input = inputs.front();
2258 auto inTy = input.getType();
2259 if (!inTy.isIntOrFloat())
2260 return {};
2261 // If the target type is a vector of rank 0 or single element vector
2262 // of element type matching input type, broadcast input to target type.
2263 if (auto vecTy = dyn_cast<VectorType>(Val&: type)) {
2264 if (vecTy.getRank() != 0 && vecTy.getNumElements() != 1)
2265 return {};
2266 auto outElemTy = vecTy.getElementType();
2267 Value cast = input;
2268 if (outElemTy.isIndex()) {
2269 cast = arith::IndexCastUIOp::create(builder, location: loc,
2270 out: builder.getIndexType(), in: cast)
2271 .getResult();
2272 } else if (inTy != outElemTy) {
2273 cast = arith::BitcastOp::create(builder, location: loc, out: outElemTy, in: cast)
2274 .getResult();
2275 }
2276 return vector::BroadcastOp::create(builder, location: loc, result: vecTy, source: cast)
2277 .getResult();
2278 }
2279 return {};
2280 };
2281 typeConverter.addSourceMaterialization(
2282 callback&: singleElementToVectorMaterializationCast);
2283 typeConverter.addSourceMaterialization(callback&: vectorToVectorMaterializationCast);
2284 typeConverter.addTargetMaterialization(callback&: memrefToIntMaterializationCast);
2285 typeConverter.addTargetMaterialization(callback&: ui32ToI32MaterializationCast);
2286 typeConverter.addTargetMaterialization(callback&: ui64ToI64MaterializationCast);
2287 typeConverter.addTargetMaterialization(
2288 callback&: vectorToSingleElementMaterializationCast);
2289 typeConverter.addTargetMaterialization(callback&: vectorToVectorMaterializationCast);
2290 ConversionTarget target(*context);
2291 target.addLegalDialect<xevm::XeVMDialect, LLVM::LLVMDialect,
2292 vector::VectorDialect, arith::ArithDialect,
2293 memref::MemRefDialect, gpu::GPUDialect,
2294 index::IndexDialect>();
2295 target.addIllegalDialect<xegpu::XeGPUDialect>();
2296 // arith.extf/arith.truncf between MX narrow floats and f16/bf16 are routed
2297 // to xevm.extf/xevm.truncf; all other arith float casts stay legal.
2298 target.addDynamicallyLegalOp<arith::ExtFOp>(
2299 callback: [](arith::ExtFOp op) { return !isXeVMExtf(op); });
2300 target.addDynamicallyLegalOp<arith::TruncFOp>(
2301 callback: [](arith::TruncFOp op) { return !isXeVMTruncf(op); });
2302
2303 RewritePatternSet patterns(context);
2304 populateXeGPUToXeVMConversionPatterns(typeConverter, patterns);
2305 scf::populateSCFStructuralTypeConversionsAndLegality(typeConverter,
2306 patterns, target);
2307 if (failed(Result: applyPartialConversion(op: getOperation(), target,
2308 patterns: std::move(patterns))))
2309 signalPassFailure();
2310 }
2311};
2312} // namespace
2313
2314//===----------------------------------------------------------------------===//
2315// Pattern Population
2316//===----------------------------------------------------------------------===//
2317void mlir::populateXeGPUToXeVMConversionPatterns(
2318 const LLVMTypeConverter &typeConverter, RewritePatternSet &patterns) {
2319 patterns.add<CreateNdDescToXeVMPattern,
2320 LoadStorePrefetchNdToXeVMPattern<xegpu::LoadNdOp>,
2321 LoadStorePrefetchNdToXeVMPattern<xegpu::StoreNdOp>,
2322 LoadStorePrefetchNdToXeVMPattern<xegpu::PrefetchNdOp>>(
2323 arg: typeConverter, args: patterns.getContext());
2324 patterns.add<AtomicRMWToXeVMPattern, PrefetchToXeVMPattern,
2325 LoadStoreToXeVMPattern<xegpu::LoadGatherOp>,
2326 LoadStoreToXeVMPattern<xegpu::StoreScatterOp>>(
2327 arg: typeConverter, args: patterns.getContext());
2328 patterns.add<LoadStoreMatrixToXeVMPattern<xegpu::LoadMatrixOp>,
2329 LoadStoreMatrixToXeVMPattern<xegpu::StoreMatrixOp>,
2330 CreateMemDescOpPattern>(arg: typeConverter, args: patterns.getContext());
2331 patterns.add<FenceToXeVMPattern, DpasToXeVMPattern>(arg: typeConverter,
2332 args: patterns.getContext());
2333 patterns.add<DpasMxToXeVMPattern>(arg: typeConverter, args: patterns.getContext());
2334 patterns.add<ExtfToXeVMPattern, TruncfToXeVMPattern>(arg: typeConverter,
2335 args: patterns.getContext());
2336 patterns.add<LaneShuffleToXeVMPattern>(arg: typeConverter, args: patterns.getContext());
2337}
2338