1//===-- XeVMToLLVM.cpp - XeVM to LLVM dialect conversion --------*- C++ -*-===//
2//
3// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/XeVMToLLVM/XeVMToLLVM.h"
10
11#include "mlir/Conversion/LLVMCommon/Pattern.h"
12#include "mlir/Dialect/GPU/IR/GPUDialect.h"
13#include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
14#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
15#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
16#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMGeneralIntrinsicOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
20#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
21#include "mlir/Dialect/LLVMIR/LLVMVectorOps.h"
22#include "mlir/Dialect/LLVMIR/XeVMDialect.h"
23#include "mlir/Pass/Pass.h"
24#include "mlir/Support/LLVM.h"
25#include "llvm/ADT/ArrayRef.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/Support/FormatVariadic.h"
28#include "llvm/Support/MathExtras.h"
29
30#include "mlir/IR/BuiltinTypes.h"
31#include "mlir/IR/Matchers.h"
32#include "mlir/IR/Types.h"
33#include "mlir/Interfaces/SideEffectInterfaces.h"
34#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
35
36#include "llvm/ADT/TypeSwitch.h"
37
38namespace mlir {
39#define GEN_PASS_DEF_CONVERTXEVMTOLLVMPASS
40#include "mlir/Conversion/Passes.h.inc"
41} // namespace mlir
42
43using namespace mlir;
44using namespace xevm;
45
46namespace {
47
48struct LLVMFuncAttributeOptions {
49 bool isConvergent = false;
50 bool isNoUnwind = false;
51 bool isWillReturn = false;
52 LLVM::MemoryEffectsAttr memEffectsAttr{};
53};
54static constexpr LLVMFuncAttributeOptions noUnwindAttrs = {
55 .isConvergent: false, .isNoUnwind: true, .isWillReturn: false, .memEffectsAttr: {}};
56static constexpr LLVMFuncAttributeOptions noUnwindWillReturnAttrs = {
57 .isConvergent: false, .isNoUnwind: true, .isWillReturn: true, .memEffectsAttr: {}};
58static constexpr LLVMFuncAttributeOptions convergentNoUnwindWillReturnAttrs = {
59 .isConvergent: true, .isNoUnwind: true, .isWillReturn: true, .memEffectsAttr: {}};
60
61std::string getTypeMangling(Type ty, bool isUnsigned = false) {
62 return TypeSwitch<Type, std::string>(ty)
63 .Case(caseFn: [isUnsigned](VectorType ty) -> std::string {
64 return "Dv" + std::to_string(val: ty.getNumElements()) + "_" +
65 getTypeMangling(ty: ty.getElementType(), isUnsigned);
66 })
67 .Case(caseFn: [](Float16Type) -> std::string { return "Dh"; })
68 .Case(caseFn: [](Float32Type) -> std::string { return "f"; })
69 .Case(caseFn: [](Float64Type) -> std::string { return "d"; })
70 .Case(caseFn: [isUnsigned](IntegerType ty) -> std::string {
71 switch (ty.getWidth()) {
72 case 8:
73 return isUnsigned ? "h" : "c";
74 case 16:
75 return isUnsigned ? "t" : "s";
76 case 32:
77 return isUnsigned ? "j" : "i";
78 case 64:
79 return isUnsigned ? "m" : "l";
80 default:
81 llvm_unreachable("unhandled integer type");
82 }
83 })
84 .DefaultUnreachable(message: "unhandled type for mangling");
85}
86
87std::string mangle(StringRef baseName, ArrayRef<Type> types,
88 ArrayRef<bool> isUnsigned = {}) {
89 assert((isUnsigned.empty() || isUnsigned.size() == types.size()) &&
90 "Signedness info doesn't match");
91 std::string s;
92 llvm::raw_string_ostream os(s);
93 llvm::SmallDenseMap<Type, unsigned> substitutions;
94 os << "_Z" << baseName.size() << baseName;
95 for (auto [idx, type] : llvm::enumerate(First&: types)) {
96 auto it = substitutions.find(Val: type);
97 if (it != substitutions.end()) {
98 os << "S";
99 // First substitution is `S_`, second is `S0_`, and so on.
100 if (unsigned firstIdx = it->getSecond(); firstIdx > 0)
101 os << firstIdx - 1;
102 os << "_";
103 } else {
104 if (!type.isIntOrFloat())
105 substitutions[type] = substitutions.size();
106 os << getTypeMangling(ty: type, isUnsigned: isUnsigned.empty() ? false : isUnsigned[idx]);
107 }
108 }
109 return os.str();
110}
111
112// Returns the mangling of `ty` used to name an overloaded `llvm.genx.GenISA.*`
113// intrinsic: `i32`, `v8i16`, ... Note that this is IGC's own scheme for its
114// intrinsics, not the Itanium mangling used for the SPIR-V friendly and OCL
115// builtins that `mangle` above produces.
116std::string getGenISATypeMangling(Type ty) {
117 return TypeSwitch<Type, std::string>(ty)
118 .Case(caseFn: [](VectorType ty) -> std::string {
119 return "v" + std::to_string(val: ty.getNumElements()) +
120 getGenISATypeMangling(ty: ty.getElementType());
121 })
122 .Case(caseFn: [](IntegerType ty) -> std::string {
123 return "i" + std::to_string(val: ty.getWidth());
124 })
125 .DefaultUnreachable(message: "unhandled type for GenISA mangling");
126}
127
128std::string builtinElemType(ElemType elemType) {
129 switch (elemType) {
130 case ElemType::BF8:
131 return "bf8";
132 case ElemType::F8:
133 return "hf8";
134 case ElemType::BF16:
135 return "bf";
136 case ElemType::F16:
137 return "hf";
138 case ElemType::F32:
139 return "f";
140 default:
141 return stringifyElemType(elemType).str();
142 }
143}
144
145static int32_t getL1CacheControl(LoadCacheControl cc) {
146 int32_t control = 0;
147 switch (cc) {
148 case LoadCacheControl::USE_DEFAULT:
149 control = -1;
150 break;
151 case LoadCacheControl::L1C_L2UC_L3UC:
152 case LoadCacheControl::L1C_L2UC_L3C:
153 case LoadCacheControl::L1C_L2C_L3UC:
154 case LoadCacheControl::L1C_L2C_L3C:
155 control = 1;
156 break;
157 case LoadCacheControl::L1S_L2UC_L3UC:
158 case LoadCacheControl::L1S_L2UC_L3C:
159 case LoadCacheControl::L1S_L2C_L3UC:
160 case LoadCacheControl::L1S_L2C_L3C:
161 control = 2;
162 break;
163 case LoadCacheControl::INVALIDATE_READ:
164 control = 3;
165 break;
166 default:
167 break;
168 }
169 return control;
170}
171
172static int32_t getL1CacheControl(StoreCacheControl cc) {
173 int32_t control = 0;
174 switch (cc) {
175 case StoreCacheControl::USE_DEFAULT:
176 control = -1;
177 break;
178 case StoreCacheControl::L1WT_L2UC_L3UC:
179 case StoreCacheControl::L1WT_L2UC_L3WB:
180 case StoreCacheControl::L1WT_L2WB_L3UC:
181 case StoreCacheControl::L1WT_L2WB_L3WB:
182 control = 1;
183 break;
184 case StoreCacheControl::L1WB_L2UC_L3UC:
185 case StoreCacheControl::L1WB_L2WB_L3UC:
186 case StoreCacheControl::L1WB_L2UC_L3WB:
187 control = 2;
188 break;
189 case StoreCacheControl::L1S_L2UC_L3UC:
190 case StoreCacheControl::L1S_L2UC_L3WB:
191 case StoreCacheControl::L1S_L2WB_L3UC:
192 case StoreCacheControl::L1S_L2WB_L3WB:
193 control = 3;
194 break;
195 default:
196 break;
197 }
198 return control;
199}
200
201static int32_t getL3CacheControl(LoadCacheControl cc) {
202 int32_t control = 0;
203 switch (cc) {
204 case LoadCacheControl::USE_DEFAULT:
205 control = -1;
206 break;
207 case LoadCacheControl::L1UC_L2UC_L3C:
208 case LoadCacheControl::L1UC_L2C_L3C:
209 case LoadCacheControl::L1C_L2UC_L3C:
210 case LoadCacheControl::L1C_L2C_L3C:
211 case LoadCacheControl::L1S_L2UC_L3C:
212 case LoadCacheControl::L1S_L2C_L3C:
213 control = 1;
214 break;
215 case LoadCacheControl::INVALIDATE_READ:
216 control = 3;
217 break;
218 default:
219 break;
220 }
221 return control;
222}
223
224static int32_t getL3CacheControl(StoreCacheControl cc) {
225 int32_t control = 0;
226 switch (cc) {
227 case StoreCacheControl::USE_DEFAULT:
228 control = -1;
229 break;
230 case StoreCacheControl::L1UC_L2UC_L3WB:
231 case StoreCacheControl::L1UC_L2WB_L3WB:
232 case StoreCacheControl::L1WT_L2UC_L3WB:
233 case StoreCacheControl::L1WT_L2WB_L3WB:
234 case StoreCacheControl::L1S_L2UC_L3WB:
235 case StoreCacheControl::L1S_L2WB_L3WB:
236 case StoreCacheControl::L1WB_L2UC_L3WB:
237 control = 2;
238 break;
239 default:
240 break;
241 }
242 return control;
243}
244
245static std::optional<LoadCacheControl> getCacheControl(PrefetchOp op) {
246 return op.getCacheControl();
247}
248
249static std::optional<LoadCacheControl> getCacheControl(BlockLoad2dOp op) {
250 return op.getCacheControl();
251}
252
253static std::optional<LoadCacheControl> getCacheControl(BlockLoadOp op) {
254 return op.getCacheControl();
255}
256
257static std::optional<LoadCacheControl> getCacheControl(BlockPrefetch2dOp op) {
258 return op.getCacheControl();
259}
260
261static std::optional<StoreCacheControl> getCacheControl(BlockStore2dOp op) {
262 return op.getCacheControl();
263}
264
265static std::optional<StoreCacheControl> getCacheControl(BlockStoreOp op) {
266 return op.getCacheControl();
267}
268
269static std::optional<LoadCacheControl> getCacheControl(LLVM::LoadOp op) {
270 if (op->hasDiscardableAttr(name: "cache_control")) {
271 auto attr = op->getDiscardableAttrOfType<xevm::LoadCacheControlAttr>(
272 name: "cache_control");
273 if (!attr)
274 return std::nullopt;
275 return std::optional<LoadCacheControl>(attr.getValue());
276 }
277 return std::nullopt;
278}
279
280static std::optional<StoreCacheControl> getCacheControl(LLVM::StoreOp op) {
281 if (op->hasDiscardableAttr(name: "cache_control")) {
282 auto attr = op->getDiscardableAttrOfType<xevm::StoreCacheControlAttr>(
283 name: "cache_control");
284 if (!attr)
285 return std::nullopt;
286 return std::optional<StoreCacheControl>(attr.getValue());
287 }
288 return std::nullopt;
289}
290
291template <typename OpType>
292int32_t getL1CacheControl(OpType op) {
293 return getL1CacheControl(*getCacheControl(op));
294}
295
296template <typename OpType>
297int32_t getL3CacheControl(OpType op) {
298 return getL3CacheControl(*getCacheControl(op));
299}
300
301template <typename OpType>
302static std::optional<ArrayAttr>
303getCacheControlMetadata(ConversionPatternRewriter &rewriter, OpType op) {
304 if (!getCacheControl(op))
305 return {};
306
307 constexpr int32_t decorationCacheControlArity{3};
308 constexpr int32_t loadCacheControlKey{6442};
309 constexpr int32_t storeCacheControlKey{6443};
310 constexpr bool isLoad = std::is_same_v<OpType, BlockLoad2dOp> ||
311 std::is_same_v<OpType, BlockPrefetch2dOp> ||
312 std::is_same_v<OpType, LLVM::LoadOp> ||
313 std::is_same_v<OpType, BlockLoadOp> ||
314 std::is_same_v<OpType, PrefetchOp>;
315
316 // If the cache control is USE_DEFAULT, then we don’t emit any metadata.
317 // Assert that if one of the L1 or L3 cache control values is USE_DEFAULT
318 // (represented as -1), then both must be USE_DEFAULT; otherwise there is a
319 // bug.
320 assert(((getL1CacheControl<OpType>(op) == -1) ==
321 (getL3CacheControl<OpType>(op) == -1)) &&
322 "If one of L1 or L3 cache control is USE_DEFAULT, both must be "
323 "USE_DEFAULT");
324
325 if (getL1CacheControl<OpType>(op) == -1 &&
326 getL3CacheControl<OpType>(op) == -1)
327 return {};
328 const int32_t controlKey{isLoad ? loadCacheControlKey : storeCacheControlKey};
329 SmallVector<int32_t, decorationCacheControlArity> decorationsL1{
330 controlKey, 0, getL1CacheControl<OpType>(op)};
331 SmallVector<int32_t, decorationCacheControlArity> decorationsL3{
332 controlKey, 1, getL3CacheControl<OpType>(op)};
333 auto arrayAttrL1 = rewriter.getI32ArrayAttr(values: decorationsL1);
334 auto arrayAttrL3 = rewriter.getI32ArrayAttr(values: decorationsL3);
335
336 SmallVector<Attribute, 2> combinedAttrs = {arrayAttrL1, arrayAttrL3};
337 return rewriter.getArrayAttr(value: combinedAttrs);
338}
339
340//===----------------------------------------------------------------------===//
341// Cache control annotation utilities
342//
343// Instead of attaching cache control as MLIR attributes and handling them
344// during LLVM translation, we directly emit llvm.intr.ptr.annotation op in
345// MLIR.
346//===----------------------------------------------------------------------===//
347
348/// Build one cache-control payload string per attribute.
349///
350/// Each Attribute is expected to be an ArrayAttr of 3 IntegerAttr values:
351/// [SPIR-V decoration token, cache level, cache control value]
352///
353/// A single entry produces a string like: {6442:"0,1"}
354/// where the quote characters (0x22) will appear as \22 in LLVM IR textual
355/// form.
356static SmallVector<std::string>
357buildCacheControlPayloads(ArrayRef<Attribute> attrs) {
358 SmallVector<std::string> payloads;
359 llvm::StringMap<bool> seen;
360
361 for (auto arr : llvm::make_isa_range<ArrayAttr>(Range&: attrs)) {
362 auto vals = arr.getValue();
363 assert(vals.size() == 3 &&
364 "Expected exactly 3 integer values (Token, CacheLevel, "
365 "ControlValue) in cache control attribute.");
366
367 auto tokenAttr = dyn_cast<IntegerAttr>(Val: vals[0]);
368 auto secondAttr = dyn_cast<IntegerAttr>(Val: vals[1]);
369 auto thirdAttr = dyn_cast<IntegerAttr>(Val: vals[2]);
370
371 if (!tokenAttr || !secondAttr || !thirdAttr)
372 continue;
373
374 // Produce: {SPIR-V decoration token:"L1 cache control,L3 cache control"}
375 // The quote char (0x22) is embedded literally; LLVM IR prints it as \22.
376 std::string entry =
377 llvm::formatv(Fmt: "{{{0}:\"{1},{2}\"}", Vals: tokenAttr.getValue().getZExtValue(),
378 Vals: secondAttr.getValue().getZExtValue(),
379 Vals: thirdAttr.getValue().getZExtValue());
380
381 // Deduplicate identical annotations.
382 if (!seen.insert(KV: {entry, true}).second)
383 continue;
384
385 payloads.push_back(Elt: std::move(entry));
386 }
387 return payloads;
388}
389/// Counter for generating unique global variable names.
390static std::atomic<uint64_t> globalNameCounter{0};
391
392/// Get or create a global metadata string and return a !llvm.ptr<1> value
393/// pointing to it. The AddressOfOp is created at the current rewriter
394/// insertion point; the GlobalOp is created at the module start.
395static Value createMetadataStringPtr(ConversionPatternRewriter &rewriter,
396 Operation *moduleOp, Location loc,
397 StringRef value, StringRef nameHint) {
398 // Build null-terminated string.
399 std::string strWithNull = value.str();
400 strWithNull.push_back(c: '\0');
401 StringRef strRef(strWithNull.data(), strWithNull.size());
402
403 auto as1PtrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: 1);
404
405 // Search for an existing global with the same content.
406 for (auto &op : moduleOp->getRegion(index: 0).front()) {
407 if (auto existingGlobal = dyn_cast<LLVM::GlobalOp>(Val: &op)) {
408 if (!existingGlobal.getSection() ||
409 *existingGlobal.getSection() != "llvm.metadata")
410 continue;
411 if (auto strAttr =
412 dyn_cast_or_null<StringAttr>(Val: existingGlobal.getValueOrNull())) {
413 if (strAttr.getValue() == strRef) {
414 return LLVM::AddressOfOp::create(builder&: rewriter, location: loc, res: as1PtrTy,
415 global_name: existingGlobal.getSymName());
416 }
417 }
418 }
419 }
420
421 // Create new global at module start.
422 auto i8Type = rewriter.getI8Type();
423 auto arrayType = LLVM::LLVMArrayType::get(elementType: i8Type, numElements: strWithNull.size());
424 std::string globalName =
425 llvm::formatv(Fmt: "{0}.{1}", Vals&: nameHint,
426 Vals: globalNameCounter.fetch_add(i: 1, m: std::memory_order_relaxed))
427 .str();
428
429 {
430 OpBuilder::InsertionGuard guard(rewriter);
431 rewriter.setInsertionPointToStart(&moduleOp->getRegion(index: 0).front());
432
433 auto globalOp =
434 LLVM::GlobalOp::create(builder&: rewriter, location: loc, type: arrayType,
435 /*isConstant=*/true, linkage: LLVM::Linkage::Private,
436 name: globalName, value: rewriter.getStringAttr(bytes: strRef));
437 globalOp.setSection(StringRef("llvm.metadata"));
438 globalOp.setUnnamedAddr(LLVM::UnnamedAddr::Global);
439 globalOp.setAlignment(1);
440 globalOp.setAddrSpace(1);
441 }
442 // InsertionGuard restores the original insertion point here.
443
444 return LLVM::AddressOfOp::create(builder&: rewriter, location: loc, res: as1PtrTy, global_name: globalName);
445}
446
447/// Annotate a pointer value with cache control metadata by emitting chained
448/// `llvm.intr.ptr.annotation` ops (LLVM::PtrAnnotation).
449///
450/// This is the MLIR-level equivalent of handleDecorationCacheControl() from
451/// the LLVM translation layer. For each cache control attribute, it emits:
452///
453/// %ann = llvm.intr.ptr.annotation %ptr, @".str.cachecontrol.N",
454/// @".str.file.N", 0, null : !llvm.ptr<AS>
455///
456/// Multiple annotations are chained: the result of each annotation op is
457/// fed as the pointer input to the next one.
458///
459/// \param rewriter The pattern rewriter.
460/// \param loc Source location for created ops.
461/// \param ptr The pointer value to annotate.
462/// \param cacheControls The cache control ArrayAttr (from
463/// getCacheControlMetadata).
464/// \param moduleOp The enclosing module (for creating globals).
465/// \returns The annotated pointer value (or the original ptr if no
466/// annotations).
467static Value annotatePtrWithCacheControl(ConversionPatternRewriter &rewriter,
468 Location loc, Value ptr,
469 ArrayAttr cacheControls,
470 Operation *moduleOp) {
471 SmallVector<std::string> payloads =
472 buildCacheControlPayloads(attrs: cacheControls.getValue());
473 if (payloads.empty())
474 return ptr;
475
476 auto ptrType = cast<LLVM::LLVMPointerType>(Val: ptr.getType());
477 auto as1PtrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: 1);
478 auto i32Ty = rewriter.getI32Type();
479
480 // Create shared constants for all annotations on this pointer.
481 Value fileStr =
482 createMetadataStringPtr(rewriter, moduleOp, loc, value: "", nameHint: ".str.file");
483 Value lineVal = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: 0);
484 Value nullAS1 = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: as1PtrTy);
485
486 // Chain: each annotation takes the result of the previous one as its
487 // pointer operand.
488 Value curPtr = ptr;
489 for (const std::string &payload : payloads) {
490 Value annStr = createMetadataStringPtr(rewriter, moduleOp, loc, value: payload,
491 nameHint: ".str.cachecontrol");
492 auto annOp = LLVM::PtrAnnotation::create(builder&: rewriter, location: loc, res: ptrType, ptr: curPtr,
493 annotation: annStr, fileName: fileStr, line: lineVal, attr: nullAS1);
494 curPtr = annOp.getResult();
495 }
496
497 return curPtr;
498}
499
500/// Helper to apply cache control annotation on a pointer operand of a call.
501/// Replaces the pointer argument of the call with an annotated version.
502///
503/// For operations that produce a call (like block load/store/prefetch), the
504/// pointer is typically the first argument. This function:
505/// 1. Builds the annotation chain on the pointer.
506/// 2. Replaces the pointer operand in the provided args list.
507///
508/// \param rewriter The pattern rewriter.
509/// \param loc Source location.
510/// \param ptr The original pointer value (first arg to the call).
511/// \param cacheControls The cache control metadata.
512/// \param moduleOp The enclosing module.
513/// \param args The argument list (modified in place: args[ptrIdx] is
514/// replaced).
515/// \param ptrIdx Index of the pointer in the args list (default 0).
516template <typename OpType>
517static void
518applyCacheControlAnnotation(ConversionPatternRewriter &rewriter, Location loc,
519 OpType op, SmallVectorImpl<Value> &args,
520 Operation *moduleOp, unsigned ptrIdx = 0) {
521 std::optional<ArrayAttr> optCacheControls =
522 getCacheControlMetadata(rewriter, op);
523 if (!optCacheControls)
524 return;
525
526 Value annotatedPtr = annotatePtrWithCacheControl(rewriter, loc, ptr: args[ptrIdx],
527 cacheControls: *optCacheControls, moduleOp);
528 args[ptrIdx] = annotatedPtr;
529}
530
531//===----------------------------------------------------------------------===//
532// End cache control annotation utilities
533//===----------------------------------------------------------------------===//
534
535static LLVM::CallOp createDeviceFunctionCall(
536 ConversionPatternRewriter &rewriter, StringRef funcName, Type retType,
537 ArrayRef<Type> argTypes, ArrayRef<Value> args,
538 mlir::ArrayRef<std::pair<unsigned, mlir::StringRef>> paramAttrs,
539 LLVMFuncAttributeOptions funcAttributeOptions, Operation *op) {
540 auto *moduleOp = op->getParentWithTrait<OpTrait::SymbolTable>();
541 assert(moduleOp && "Expecting module");
542 Location loc = op->getLoc();
543
544 auto funcOpRes =
545 LLVM::lookupOrCreateFn(b&: rewriter, moduleOp, name: funcName, paramTypes: argTypes, resultType: retType);
546 assert(!failed(funcOpRes));
547 LLVM::LLVMFuncOp funcOp = funcOpRes.value();
548 funcOp.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
549 funcOp.setConvergent(funcAttributeOptions.isConvergent);
550 funcOp.setNoUnwind(funcAttributeOptions.isNoUnwind);
551 funcOp.setWillReturn(funcAttributeOptions.isWillReturn);
552
553 if (funcAttributeOptions.memEffectsAttr)
554 funcOp.setMemoryEffectsAttr(funcAttributeOptions.memEffectsAttr);
555
556 for (auto [idx, attrName] : paramAttrs)
557 funcOp.setArgAttr(index: idx, name: attrName, value: rewriter.getUnitAttr());
558
559 auto callOp = LLVM::CallOp::create(builder&: rewriter, location: loc, func: funcOp, args);
560 SmallVector<NamedAttribute> discardableAttrs;
561 auto copyAttr = [&](StringAttr name, Attribute attr) {
562 if (callOp->getInherentAttr(name).has_value())
563 callOp->setInherentAttr(name, value: attr);
564 else
565 discardableAttrs.emplace_back(Args&: name, Args&: attr);
566 };
567 for (NamedAttribute attr : funcOp->getDiscardableAttrDictionary())
568 copyAttr(attr.getName(), attr.getValue());
569 funcOp->getName().walkInherentAttrs(
570 op: funcOp, visitor: [&](StringRef name, Attribute &attr) {
571 copyAttr(rewriter.getStringAttr(bytes: name), attr);
572 });
573 callOp->setDiscardableAttrs(discardableAttrs);
574
575 return callOp;
576}
577
578static unsigned getNumOperandsPerDword(xevm::ElemType pTy) {
579 switch (pTy) {
580 case xevm::ElemType::F32:
581 case xevm::ElemType::TF32:
582 return 1;
583 case xevm::ElemType::BF16:
584 case xevm::ElemType::F16:
585 return 2;
586 case xevm::ElemType::U8:
587 case xevm::ElemType::S8:
588 case xevm::ElemType::BF8:
589 case xevm::ElemType::F8:
590 return 4;
591 case xevm::ElemType::E2M1:
592 case xevm::ElemType::U4:
593 case xevm::ElemType::S4:
594 return 8;
595 default:
596 llvm_unreachable("unsupported xevm::ElemType");
597 }
598}
599
600class MMAToOCLPattern : public OpConversionPattern<xevm::MMAOp> {
601 using Base::Base;
602 LogicalResult
603 matchAndRewrite(xevm::MMAOp op, xevm::MMAOp::Adaptor adaptor,
604 ConversionPatternRewriter &rewriter) const override {
605 if (!op.getC()) {
606 return rewriter.notifyMatchFailure(arg&: op, msg: "OCL requires C operand");
607 }
608 auto precisionA = op.getTypes().getA();
609 auto precisionB = op.getTypes().getB();
610 auto precisionC = op.getTypes().getC();
611 auto precisionD = op.getTypes().getD();
612 if (precisionC != precisionD) {
613 return rewriter.notifyMatchFailure(arg&: op, msg: "type of C and D need to match");
614 }
615 if (precisionC != xevm::ElemType::S32 &&
616 precisionC != xevm::ElemType::F32 &&
617 precisionC != xevm::ElemType::F16 &&
618 precisionC != xevm::ElemType::BF16) {
619 return rewriter.notifyMatchFailure(
620 arg&: op, msg: "type of C and D must be S32, F32, F16 or BF16");
621 }
622 if (precisionA == xevm::ElemType::S32 ||
623 precisionA == xevm::ElemType::F32) {
624 return rewriter.notifyMatchFailure(arg&: op, msg: "type of A cannot be S32 or F32");
625 }
626 if (precisionB == xevm::ElemType::S32 ||
627 precisionB == xevm::ElemType::F32) {
628 return rewriter.notifyMatchFailure(arg&: op, msg: "type of B cannot be S32 or F32");
629 }
630 constexpr uint32_t bitWidthPackedA{16};
631 constexpr uint32_t bitWidthPackedB{32};
632 auto loc = op.getLoc();
633
634 auto castIfNeeded = [&](Value val, Type packedType) -> Value {
635 VectorType origTy = cast<VectorType>(Val: val.getType());
636 const uint32_t vecBitSize =
637 origTy.getNumElements() *
638 origTy.getElementType().getIntOrFloatBitWidth();
639 VectorType newTy = VectorType::get(
640 shape: vecBitSize / packedType.getIntOrFloatBitWidth(), elementType: packedType);
641 if (origTy != newTy)
642 val = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: newTy, arg: val);
643 return val;
644 };
645
646 Value a = op.getA();
647 Type packedAType = (op.getTypes().getA() == xevm::ElemType::TF32)
648 ? cast<Type>(Val: rewriter.getF32Type())
649 : rewriter.getIntegerType(width: bitWidthPackedA);
650 a = castIfNeeded(a, packedAType);
651
652 Value b = op.getB();
653 Type packedBType = (op.getTypes().getB() == xevm::ElemType::TF32)
654 ? cast<Type>(Val: rewriter.getF32Type())
655 : rewriter.getIntegerType(width: bitWidthPackedB);
656 b = castIfNeeded(b, packedBType);
657
658 Value c = op.getC();
659 VectorType cOrigTy = cast<VectorType>(Val: c.getType());
660 VectorType resOrigTy = cast<VectorType>(Val: op->getResultTypes()[0]);
661 assert(cOrigTy == resOrigTy && "Accumulator and result type mismatch");
662 // OCL builtins encode bfloat16 as int16
663 VectorType cTy =
664 cOrigTy.getElementType().isBF16()
665 ? VectorType::get(shape: cOrigTy.getShape(), elementType: rewriter.getIntegerType(width: 16))
666 : cOrigTy;
667 VectorType resTy = cTy;
668 if (cOrigTy != cTy)
669 c = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: cTy, arg: c);
670
671 constexpr int32_t systolicDepth{8};
672 std::string fnName =
673 llvm::formatv(Fmt: "intel_sub_group_{0}_{1}_matrix_mad_k{2}",
674 Vals: stringifyElemType(op.getTypes().getA()).str(),
675 Vals: stringifyElemType(op.getTypes().getB()).str(),
676 Vals: systolicDepth *
677 getNumOperandsPerDword(pTy: op.getTypes().getA()))
678 .str();
679 SmallVector<Type> argTypes{a.getType(), b.getType(), cTy};
680 fnName = mangle(baseName: fnName, types: argTypes);
681 SmallVector<Value> args{a, b, c};
682
683 auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>(
684 /*other=*/args: LLVM::ModRefInfo::NoModRef,
685 /*argMem=*/args: LLVM::ModRefInfo::NoModRef,
686 /*inaccessibleMem=*/args: LLVM::ModRefInfo::NoModRef,
687 /*errnoMem=*/args: LLVM::ModRefInfo::NoModRef,
688 /*targetMem0=*/args: LLVM::ModRefInfo::NoModRef,
689 /*targetMem1=*/args: LLVM::ModRefInfo::NoModRef);
690 auto funcAttrs = convergentNoUnwindWillReturnAttrs;
691 funcAttrs.memEffectsAttr = memAttr;
692 Value result =
693 createDeviceFunctionCall(rewriter, funcName: fnName, retType: resTy, argTypes, args, paramAttrs: {},
694 funcAttributeOptions: funcAttrs, op: op.getOperation())
695 ->getResult(idx: 0);
696
697 if (resOrigTy != resTy)
698 result = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: resOrigTy, arg: result);
699
700 rewriter.replaceOp(op, newValues: result);
701 return success();
702 }
703};
704
705class PrefetchToOCLPattern : public OpConversionPattern<PrefetchOp> {
706 using Base::Base;
707 LogicalResult
708 matchAndRewrite(PrefetchOp op, PrefetchOp::Adaptor adaptor,
709 ConversionPatternRewriter &rewriter) const override {
710 auto loc = op.getLoc();
711 auto *moduleOp = op->getParentWithTrait<OpTrait::SymbolTable>();
712
713 const std::string fnName{"_Z8prefetchPU3AS1Kcm"};
714 Value one =
715 LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI64Type(), value: 1);
716 SmallVector<Value> args{op.getPtr(), one};
717
718 // Annotate pointer with cache control before passing to the call.
719 applyCacheControlAnnotation(rewriter, loc, op, args, moduleOp,
720 /*ptrIdx=*/0);
721
722 SmallVector<Type> argTypes;
723 for (auto arg : args)
724 argTypes.push_back(Elt: arg.getType());
725 auto funcAttr = noUnwindAttrs;
726 auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>(
727 /*other=*/args: LLVM::ModRefInfo::NoModRef,
728 /*argMem=*/args: LLVM::ModRefInfo::Ref,
729 /*inaccessibleMem=*/args: LLVM::ModRefInfo::NoModRef,
730 /*errnoMem=*/args: LLVM::ModRefInfo::NoModRef,
731 /*targetMem0=*/args: LLVM::ModRefInfo::NoModRef,
732 /*targetMem1=*/args: LLVM::ModRefInfo::NoModRef);
733 funcAttr.memEffectsAttr = memAttr;
734
735 createDeviceFunctionCall(rewriter, funcName: fnName,
736 retType: LLVM::LLVMVoidType::get(ctx: rewriter.getContext()),
737 argTypes, args, paramAttrs: {}, funcAttributeOptions: funcAttr, op: op.getOperation());
738 rewriter.eraseOp(op);
739 return success();
740 }
741};
742
743class MemfenceToOCLPattern : public OpConversionPattern<MemfenceOp> {
744 using Base::Base;
745 LogicalResult
746 matchAndRewrite(MemfenceOp op, MemfenceOp::Adaptor adaptor,
747 ConversionPatternRewriter &rewriter) const override {
748 auto loc = op.getLoc();
749 const std::string fnName{"atomic_work_item_fence"};
750 int memScope, addrSpace;
751 switch (op.getAddrspace()) {
752 case xevm::AddrSpace::SHARED:
753 addrSpace = 1; // CLK_LOCAL_MEM_FENCE
754 break;
755 case xevm::AddrSpace::GLOBAL:
756 addrSpace = 2; // CLK_GLOBAL_MEM_FENCE
757 break;
758 default:
759 // GENERIC is not supported in OpenCL
760 return rewriter.notifyMatchFailure(
761 arg&: op, msg: "Fence only supports global and shared address spaces.");
762 }
763 switch (op.getScope()) {
764 case xevm::MemScope::WORKGROUP:
765 memScope = 1;
766 break;
767 case xevm::MemScope::DEVICE:
768 memScope = 2;
769 break;
770 default:
771 // CLUSTER and SYSTEM are not supported in OpenCL
772 return rewriter.notifyMatchFailure(
773 arg&: op, msg: "Fence only supports workgroup and device memory scopes.");
774 }
775 Type i32Type = rewriter.getI32Type();
776 Value acqRel = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Type, value: 4);
777 Value memScopeConst =
778 LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Type, value: memScope);
779 Value addrSpaceConst =
780 LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Type, value: addrSpace);
781 SmallVector<Value> args{addrSpaceConst, acqRel, memScopeConst};
782 SmallVector<Type> argTypes{3, i32Type};
783 createDeviceFunctionCall(rewriter, funcName: mangle(baseName: fnName, types: argTypes),
784 retType: LLVM::LLVMVoidType::get(ctx: rewriter.getContext()),
785 argTypes, args, paramAttrs: {}, funcAttributeOptions: noUnwindAttrs,
786 op: op.getOperation());
787 rewriter.eraseOp(op);
788 return success();
789 }
790};
791template <typename OpType>
792class LoadStorePrefetchToOCLPattern : public OpConversionPattern<OpType> {
793 using OpConversionPattern<OpType>::OpConversionPattern;
794 LogicalResult
795 matchAndRewrite(OpType op, typename OpType::Adaptor adaptor,
796 ConversionPatternRewriter &rewriter) const override {
797 constexpr bool isLoad = std::is_same_v<OpType, BlockLoad2dOp>;
798 constexpr bool isPrefetch = std::is_same_v<OpType, BlockPrefetch2dOp>;
799
800 auto loc = op.getLoc();
801 auto *moduleOp = op->template getParentWithTrait<OpTrait::SymbolTable>();
802 VectorType vecType;
803 bool packReg = false;
804 bool transpose = false;
805 if constexpr (isLoad) {
806 vecType = op.getRes().getType();
807 packReg = op.getPackRegister();
808 transpose = op.getTranspose();
809 } else if constexpr (!isPrefetch) {
810 vecType = op.getStoredVal().getType();
811 }
812
813 auto i32Type = rewriter.getI32Type();
814 Value byteCoord =
815 LLVM::UndefOp::create(rewriter, loc, VectorType::get(shape: 2, elementType: i32Type));
816 Value zero = LLVM::ConstantOp::create(rewriter, loc, i32Type, 0);
817 Value one = LLVM::ConstantOp::create(rewriter, loc, i32Type, 1);
818 byteCoord = LLVM::InsertElementOp::create(
819 rewriter, loc, VectorType::get(shape: 2, elementType: i32Type), byteCoord, op.getX(), zero);
820 byteCoord = LLVM::InsertElementOp::create(
821 rewriter, loc, VectorType::get(shape: 2, elementType: i32Type), byteCoord, op.getY(), one);
822 SmallVector<Value> args{op.getPtr(), op.getBaseWidth(), op.getBaseHeight(),
823 op.getBasePitch(), byteCoord};
824
825 // Annotate pointer (args[0]) with cache control before the call.
826 applyCacheControlAnnotation(rewriter, loc, op, args, moduleOp,
827 /*ptrIdx=*/0);
828
829 SmallVector<Type> retTypes;
830 Value spvLoadDstPtr;
831 std::string funcName{"intel_sub_group_2d_block_"};
832 std::string bitWidthId;
833 LLVMFuncAttributeOptions funcAttr{noUnwindWillReturnAttrs};
834 SmallVector<std::pair<unsigned, StringRef>, 4> paramAttrs;
835 if constexpr (isPrefetch) { // Prefetch
836 funcName += "prefetch";
837 paramAttrs = {std::make_pair(x: 0, y: LLVM::LLVMDialect::getNonNullAttrName())};
838 auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>(
839 /*other=*/args: LLVM::ModRefInfo::NoModRef,
840 /*argMem=*/args: LLVM::ModRefInfo::Ref,
841 /*inaccessibleMem=*/args: LLVM::ModRefInfo::NoModRef,
842 /*errnoMem=*/args: LLVM::ModRefInfo::NoModRef,
843 /*targetMem0=*/args: LLVM::ModRefInfo::NoModRef,
844 /*targetMem1=*/args: LLVM::ModRefInfo::NoModRef);
845 funcAttr = noUnwindAttrs;
846 funcAttr.memEffectsAttr = memAttr;
847 } else {
848 auto vecElemType = vecType.getElementType();
849 auto vecElemBitWidth = vecElemType.getIntOrFloatBitWidth();
850 auto vecNumElems = vecType.getNumElements();
851 // OpenCL Intel 2D block load has a special case
852 // when element bit size is 8 and tile width is 32, which is twice
853 // the subgroup size, loaded element is packed as i16.
854 // To reflect this, element bit size is updated to 16 and
855 // vector length is reduced by half.
856 if (op.getElemSizeInBits() == 8 && op.getTileWidth() == 32) {
857 vecElemBitWidth = 16;
858 vecElemType = rewriter.getI16Type();
859 vecNumElems = vecNumElems / 2;
860 }
861 Value numElems =
862 LLVM::ConstantOp::create(rewriter, loc, i32Type, vecNumElems);
863 auto dstOrSrcPtr = LLVM::AllocaOp::create(
864 rewriter, loc, LLVM::LLVMPointerType::get(context: rewriter.getContext()),
865 vecElemType, numElems);
866 args.push_back(Elt: dstOrSrcPtr);
867 if constexpr (isLoad) { // Load
868 funcName += "read";
869 bitWidthId = getTypeMangling(ty: vecElemType, /*isUnsigned=*/true);
870 if (packReg)
871 funcName += "_transform";
872 else if (transpose)
873 funcName += "_transpose";
874 spvLoadDstPtr = dstOrSrcPtr;
875 retTypes.push_back(Elt: vecType);
876 paramAttrs = {
877 std::make_pair(x: 0, y: LLVM::LLVMDialect::getNonNullAttrName()),
878 std::make_pair(x: 0, y: LLVM::LLVMDialect::getReadonlyAttrName()),
879 std::make_pair(x: 5, y: LLVM::LLVMDialect::getNonNullAttrName()),
880 std::make_pair(x: 5, y: LLVM::LLVMDialect::getWriteOnlyAttrName()),
881 };
882 } else { // Store
883 funcName += "write";
884 bitWidthId = (vecElemBitWidth == 32)
885 ? "j"
886 : ((vecElemBitWidth == 16) ? "t" : "h");
887 LLVM::StoreOp::create(rewriter, loc, op.getStoredVal(), dstOrSrcPtr);
888 paramAttrs = {
889 std::make_pair(x: 0, y: LLVM::LLVMDialect::getNonNullAttrName()),
890 std::make_pair(x: 0, y: LLVM::LLVMDialect::getWriteOnlyAttrName()),
891 std::make_pair(x: 5, y: LLVM::LLVMDialect::getNonNullAttrName()),
892 std::make_pair(x: 5, y: LLVM::LLVMDialect::getReadonlyAttrName()),
893 };
894 }
895 }
896
897 funcName =
898 llvm::formatv("{0}_{1}b_{2}r{3}x{4}c", funcName, op.getElemSizeInBits(),
899 op.getTileHeight(), op.getTileWidth(), op.getVBlocks())
900 .str();
901 std::string prefetchCode("");
902 if (!isPrefetch)
903 prefetchCode += "P";
904 funcName = llvm::formatv(Fmt: "_Z{0}{1}PU3AS1viiiDv2_i{2}{3}", Vals: funcName.size(),
905 Vals&: funcName, Vals&: prefetchCode, Vals&: bitWidthId)
906 .str();
907 SmallVector<Type> argTypes;
908 for (auto arg : args) {
909 argTypes.push_back(Elt: arg.getType());
910 }
911 createDeviceFunctionCall(
912 rewriter, funcName, LLVM::LLVMVoidType::get(ctx: rewriter.getContext()),
913 argTypes, args, paramAttrs, funcAttr, op.getOperation());
914
915 if constexpr (isLoad)
916 rewriter.replaceOp(
917 op, LLVM::LoadOp::create(rewriter, loc, vecType, spvLoadDstPtr));
918 else
919 rewriter.eraseOp(op);
920 return success();
921 }
922};
923
924template <typename OpType>
925class BlockLoadStore1DToOCLPattern : public OpConversionPattern<OpType> {
926 using OpConversionPattern<OpType>::OpConversionPattern;
927 LogicalResult
928 matchAndRewrite(OpType op, typename OpType::Adaptor adaptor,
929 ConversionPatternRewriter &rewriter) const override {
930 constexpr bool isStore = std::is_same_v<OpType, xevm::BlockStoreOp>;
931 auto loc = op.getLoc();
932 auto *moduleOp = op->template getParentWithTrait<OpTrait::SymbolTable>();
933
934 // Get OpenCL function name
935 // https://registry.khronos.org/OpenCL/extensions/
936 // intel/cl_intel_subgroup_local_block_io.html
937 std::string funcName{"intel_sub_group_block_"};
938 // Value or Result type can be vector or scalar
939 Type valOrResTy;
940 if constexpr (isStore) {
941 funcName += "write_u";
942 valOrResTy = op.getVal().getType();
943 } else {
944 funcName += "read_u";
945 valOrResTy = op.getType();
946 }
947 // Get element type of the vector/scalar
948 VectorType vecTy = dyn_cast<VectorType>(Val&: valOrResTy);
949 Type elemType = vecTy ? vecTy.getElementType() : valOrResTy;
950 funcName += getTypeMangling(ty: elemType);
951 if (vecTy)
952 funcName += std::to_string(val: vecTy.getNumElements());
953 SmallVector<Type, 2> argTypes{};
954 // XeVM BlockLoad/StoreOp always use signless integer types
955 // but OpenCL builtins expect unsigned types
956 // use unsigned types for mangling
957 SmallVector<bool, 2> isUnsigned{};
958 // arg0: pointer to the src/dst address
959 // arg1 - only if store : vector to store
960 // Prepare arguments
961 SmallVector<Value, 2> args{};
962 args.push_back(Elt: op.getPtr());
963 argTypes.push_back(Elt: op.getPtr().getType());
964 isUnsigned.push_back(Elt: true);
965
966 // Annotate pointer (args[0]) with cache control.
967 applyCacheControlAnnotation(rewriter, loc, op, args, moduleOp,
968 /*ptrIdx=*/0);
969 // Update argTypes[0] in case the pointer type changed (it shouldn't
970 // change type, but the value is now the annotated pointer).
971 argTypes[0] = args[0].getType();
972
973 Type retType;
974 if constexpr (isStore) {
975 args.push_back(Elt: op.getVal());
976 argTypes.push_back(Elt: op.getVal().getType());
977 isUnsigned.push_back(Elt: true);
978 retType = LLVM::LLVMVoidType::get(ctx: rewriter.getContext());
979 } else {
980 retType = valOrResTy;
981 }
982 funcName = std::string("_Z") + std::to_string(val: funcName.size()) + funcName +
983 "PU3AS" +
984 std::to_string(op.getPtr().getType().getAddressSpace());
985 funcName += getTypeMangling(ty: elemType, /*isUnsigned=*/true);
986 if constexpr (isStore)
987 funcName += getTypeMangling(ty: valOrResTy, /*isUnsigned=*/true);
988 LLVMFuncAttributeOptions funcAttr{noUnwindWillReturnAttrs};
989
990 LLVM::CallOp call =
991 createDeviceFunctionCall(rewriter, funcName, retType, argTypes, args,
992 {}, funcAttr, op.getOperation());
993
994 if constexpr (isStore)
995 rewriter.eraseOp(op);
996 else
997 rewriter.replaceOp(op, call->getResult(idx: 0));
998 return success();
999 }
1000};
1001
1002template <typename OpType>
1003class LLVMLoadStoreToOCLPattern : public OpConversionPattern<OpType> {
1004 using OpConversionPattern<OpType>::OpConversionPattern;
1005 LogicalResult
1006 matchAndRewrite(OpType op, typename OpType::Adaptor adaptor,
1007 ConversionPatternRewriter &rewriter) const override {
1008 if (!op->hasDiscardableAttr("cache_control"))
1009 return failure();
1010
1011 auto *moduleOp = op->template getParentWithTrait<OpTrait::SymbolTable>();
1012 std::optional<ArrayAttr> optCacheControls =
1013 getCacheControlMetadata(rewriter, op);
1014 if (!optCacheControls) {
1015 rewriter.modifyOpInPlace(
1016 op, [&]() { op->removeDiscardableAttr("cache_control"); });
1017 return success();
1018 }
1019
1020 // Determine which operand is the pointer.
1021 constexpr bool isStore = std::is_same_v<OpType, LLVM::StoreOp>;
1022 unsigned ptrIdx = isStore ? 1 : 0;
1023 Value ptr = op->getOperand(ptrIdx);
1024
1025 // Emit annotation intrinsic calls on the pointer.
1026 Value annotatedPtr = annotatePtrWithCacheControl(
1027 rewriter, op->getLoc(), ptr, *optCacheControls, moduleOp);
1028
1029 // Replace the pointer operand with the annotated one.
1030 rewriter.modifyOpInPlace(op, [&]() {
1031 op->setOperand(ptrIdx, annotatedPtr);
1032 op->removeDiscardableAttr("cache_control");
1033 });
1034 return success();
1035 }
1036};
1037
1038//===----------------------------------------------------------------------===//
1039// GPU index id operations
1040//===----------------------------------------------------------------------===//
1041/*
1042// Launch Config ops
1043// dimidx - x, y, z - is fixed to i32
1044// return type is set by XeVM type converter
1045// get_local_id
1046xevm::WorkitemIdXOp;
1047xevm::WorkitemIdYOp;
1048xevm::WorkitemIdZOp;
1049// get_local_size
1050xevm::WorkgroupDimXOp;
1051xevm::WorkgroupDimYOp;
1052xevm::WorkgroupDimZOp;
1053// get_group_id
1054xevm::WorkgroupIdXOp;
1055xevm::WorkgroupIdYOp;
1056xevm::WorkgroupIdZOp;
1057// get_num_groups
1058xevm::GridDimXOp;
1059xevm::GridDimYOp;
1060xevm::GridDimZOp;
1061// get_global_id : to be added if needed
1062*/
1063
1064// Helpers to get the OpenCL function name and dimension argument for each op.
1065static std::pair<StringRef, int64_t> getConfig(xevm::WorkitemIdXOp) {
1066 return {"get_local_id", 0};
1067}
1068static std::pair<StringRef, int64_t> getConfig(xevm::WorkitemIdYOp) {
1069 return {"get_local_id", 1};
1070}
1071static std::pair<StringRef, int64_t> getConfig(xevm::WorkitemIdZOp) {
1072 return {"get_local_id", 2};
1073}
1074static std::pair<StringRef, int64_t> getConfig(xevm::WorkgroupDimXOp) {
1075 return {"get_local_size", 0};
1076}
1077static std::pair<StringRef, int64_t> getConfig(xevm::WorkgroupDimYOp) {
1078 return {"get_local_size", 1};
1079}
1080static std::pair<StringRef, int64_t> getConfig(xevm::WorkgroupDimZOp) {
1081 return {"get_local_size", 2};
1082}
1083static std::pair<StringRef, int64_t> getConfig(xevm::WorkgroupIdXOp) {
1084 return {"get_group_id", 0};
1085}
1086static std::pair<StringRef, int64_t> getConfig(xevm::WorkgroupIdYOp) {
1087 return {"get_group_id", 1};
1088}
1089static std::pair<StringRef, int64_t> getConfig(xevm::WorkgroupIdZOp) {
1090 return {"get_group_id", 2};
1091}
1092static std::pair<StringRef, int64_t> getConfig(xevm::GridDimXOp) {
1093 return {"get_num_groups", 0};
1094}
1095static std::pair<StringRef, int64_t> getConfig(xevm::GridDimYOp) {
1096 return {"get_num_groups", 1};
1097}
1098static std::pair<StringRef, int64_t> getConfig(xevm::GridDimZOp) {
1099 return {"get_num_groups", 2};
1100}
1101/// Replace `xevm.*` with an `llvm.call` to the corresponding OpenCL func with
1102/// a constant argument for the dimension - x, y or z.
1103template <typename OpType>
1104class LaunchConfigOpToOCLPattern : public OpConversionPattern<OpType> {
1105 using OpConversionPattern<OpType>::OpConversionPattern;
1106 LogicalResult
1107 matchAndRewrite(OpType op, typename OpType::Adaptor adaptor,
1108 ConversionPatternRewriter &rewriter) const override {
1109 Location loc = op->getLoc();
1110 auto [baseName, dim] = getConfig(op);
1111 Type dimTy = rewriter.getI32Type();
1112 Value dimVal = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: dimTy,
1113 value: static_cast<int64_t>(dim));
1114 std::string func = mangle(baseName, {dimTy}, {true});
1115 Type resTy = op.getType();
1116 auto call =
1117 createDeviceFunctionCall(rewriter, func, resTy, {dimTy}, {dimVal}, {},
1118 noUnwindWillReturnAttrs, op.getOperation());
1119 constexpr auto noModRef = LLVM::ModRefInfo::NoModRef;
1120 auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>(
1121 /*other=*/args: noModRef,
1122 /*argMem=*/args: noModRef, /*inaccessibleMem=*/args: noModRef,
1123 /*errnoMem=*/args: noModRef,
1124 /*targetMem0=*/args: noModRef,
1125 /*targetMem1=*/args: noModRef);
1126 call.setMemoryEffectsAttr(memAttr);
1127 rewriter.replaceOp(op, call);
1128 return success();
1129 }
1130};
1131
1132/*
1133// Subgroup ops
1134// get_sub_group_local_id
1135xevm::LaneIdOp;
1136// get_sub_group_id
1137xevm::SubgroupIdOp;
1138// get_sub_group_size
1139xevm::SubgroupSizeOp;
1140// get_num_sub_groups : to be added if needed
1141*/
1142
1143// Helpers to get the OpenCL function name for each op.
1144static StringRef getConfig(xevm::LaneIdOp) { return "get_sub_group_local_id"; }
1145static StringRef getConfig(xevm::SubgroupIdOp) { return "get_sub_group_id"; }
1146static StringRef getConfig(xevm::SubgroupSizeOp) {
1147 return "get_sub_group_size";
1148}
1149template <typename OpType>
1150class SubgroupOpWorkitemOpToOCLPattern : public OpConversionPattern<OpType> {
1151 using OpConversionPattern<OpType>::OpConversionPattern;
1152 LogicalResult
1153 matchAndRewrite(OpType op, typename OpType::Adaptor adaptor,
1154 ConversionPatternRewriter &rewriter) const override {
1155 std::string func = mangle(getConfig(op).str(), {});
1156 Type resTy = op.getType();
1157 auto call =
1158 createDeviceFunctionCall(rewriter, func, resTy, {}, {}, {},
1159 noUnwindWillReturnAttrs, op.getOperation());
1160 constexpr auto noModRef = LLVM::ModRefInfo::NoModRef;
1161 auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>(
1162 /*other=*/args: noModRef,
1163 /*argMem=*/args: noModRef, /*inaccessibleMem=*/args: noModRef,
1164 /*errnoMem=*/args: noModRef,
1165 /*targetMem0=*/args: noModRef,
1166 /*targetMem1=*/args: noModRef);
1167 call.setMemoryEffectsAttr(memAttr);
1168 rewriter.replaceOp(op, call);
1169 return success();
1170 }
1171};
1172
1173/// SPIR-V, and so the OpenCL builtins the float conversions call into, only
1174/// provides vector types of 2, 3, 4, 8 and 16 elements.
1175static bool isSupportedSPIRVVectorLength(int64_t numElements) {
1176 return llvm::is_contained(Set: {2, 3, 4, 8, 16}, Element: numElements);
1177}
1178
1179/// Bitcasts `val` to `ty` unless it already has that type.
1180static Value castIfNeeded(ConversionPatternRewriter &rewriter, Location loc,
1181 Type ty, Value val) {
1182 if (val.getType() == ty)
1183 return val;
1184 return LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: ty, arg: val);
1185}
1186
1187/// Selects `numElements` leading elements of the vector `val`. Used to drop the
1188/// padding the 3 element case needs, as the hardware conversions work on whole
1189/// pairs of elements.
1190static Value takeLeadingElements(ConversionPatternRewriter &rewriter,
1191 Location loc, Value val, int64_t numElements) {
1192 auto vecTy = cast<VectorType>(Val: val.getType());
1193 if (vecTy.getNumElements() == numElements)
1194 return val;
1195 SmallVector<int32_t> mask =
1196 llvm::to_vector(Range: llvm::seq<int32_t>(Begin: 0, End: static_cast<int32_t>(numElements)));
1197 return LLVM::ShuffleVectorOp::create(builder&: rewriter, location: loc, v1: val, v2: val, mask);
1198}
1199
1200//
1201// Note: TruncfToOCLPattern and ExtfToOCLPattern does not lower to OpenCL API
1202// calls as there are not official ones yet. They are lowered directly to Intel
1203// graphics compiler built in functions.
1204// See
1205// https://github.com/intel/intel-graphics-compiler/tree/master/IGC/BiFModule/Implementation/SPV_INTEL_fp_conversions
1206// for builtin function usage. The folder contains implementation of
1207// experimental SPIR-V extension for truncf and extf using builtin functions.
1208// TODO: Move to OpenCL API call once they are available.
1209//
1210
1211class TruncfToOCLPattern : public OpConversionPattern<TruncfOp> {
1212 using Base::Base;
1213 LogicalResult
1214 matchAndRewrite(TruncfOp op, TruncfOp::Adaptor adaptor,
1215 ConversionPatternRewriter &rewriter) const override {
1216 // Supported source and result types are resticted for now.
1217 auto srcEtype = op.getSrcEtype().getEtype();
1218 auto dstEtype = op.getDstEtype().getEtype();
1219 // The conversions are provided as OpenCL builtins, one per vector length,
1220 // so only the SPIR-V vector lengths can be lowered. A wider conversion has
1221 // to be split into several ops before reaching this pattern.
1222 //
1223 // Scalar case is not supported until usage case become clear.
1224 auto vecSrcTy = dyn_cast<VectorType>(Val: op.getSrc().getType());
1225 if (!vecSrcTy) {
1226 return rewriter.notifyMatchFailure(arg&: op, msg: "Scalar src is not supported.");
1227 }
1228 int64_t numElements = vecSrcTy.getNumElements();
1229 if (!isSupportedSPIRVVectorLength(numElements))
1230 return rewriter.notifyMatchFailure(
1231 arg&: op, msg: "src vector length must be 2, 3, 4, 8 or 16");
1232 // The destination is scalar only where the packed values fit in one byte,
1233 // which SPIR-V spells as a scalar rather than a one element vector.
1234 Type dstTy = op.getDst().getType();
1235 Location loc = op.getLoc();
1236 Value src = op.getSrc();
1237 auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>(
1238 /*other=*/args: LLVM::ModRefInfo::NoModRef,
1239 /*argMem=*/args: LLVM::ModRefInfo::NoModRef,
1240 /*inaccessibleMem=*/args: LLVM::ModRefInfo::NoModRef,
1241 /*errnoMem=*/args: LLVM::ModRefInfo::NoModRef,
1242 /*targetMem0=*/args: LLVM::ModRefInfo::NoModRef,
1243 /*targetMem1=*/args: LLVM::ModRefInfo::NoModRef);
1244 auto funcAttrs = convergentNoUnwindWillReturnAttrs;
1245 funcAttrs.memEffectsAttr = memAttr;
1246
1247 // Handle the case where dst type is fp4 first.
1248 if (dstEtype == TruncfDstElemTypes::E2M1) {
1249 // `__builtin_IB_dnscl_{hf16,bf16}(uint a, uint b, convert_to, mode)`
1250 // takes two dwords, each holding two source elements, and packs each pair
1251 // into one byte of the result dword. `mode` picks which bytes of that
1252 // dword are written: mode 0 writes bytes 0 and 2, mode 2 writes bytes 1
1253 // and 3. Two calls with complementary modes therefore OR together into
1254 // one fully packed dword covering eight source elements.
1255 //
1256 // A pair of elements is the conversion granularity, so an odd length is
1257 // padded up and the spare nibble left undefined.
1258 constexpr int kDnsclConvertToE2M1 = 1;
1259 constexpr int kDnsclModeBytes02 = 0;
1260 constexpr int kDnsclModeBytes13 = 2;
1261 // One dword lane per element pair, and one result byte per lane. The op
1262 // verifier has already checked that the destination is exactly that wide.
1263 int64_t numLanes = llvm::divideCeil(Numerator: numElements, Denominator: 2);
1264
1265 Type i32Ty = rewriter.getI32Type();
1266 Type i8Ty = rewriter.getI8Type();
1267 // Pad an odd length up to a whole number of pairs, then view the source
1268 // as dword lanes.
1269 Value padded = src;
1270 if (numElements != numLanes * 2) {
1271 SmallVector<int32_t> mask = llvm::to_vector(
1272 Range: llvm::seq<int32_t>(Begin: 0, End: static_cast<int32_t>(numElements)));
1273 // The padding element is never read back, so any valid index will do.
1274 mask.append(NumInputs: static_cast<size_t>(numLanes * 2 - numElements), Elt: 0);
1275 padded = LLVM::ShuffleVectorOp::create(builder&: rewriter, location: loc, v1: src, v2: src, mask);
1276 }
1277 // A single lane is passed as a bare i32 rather than a one element vector,
1278 // which SPIR-V has no type for.
1279 Value laneVec;
1280 if (numLanes > 1)
1281 laneVec = LLVM::BitcastOp::create(
1282 builder&: rewriter, location: loc, res: VectorType::get(shape: numLanes, elementType: i32Ty), arg: padded);
1283 else
1284 laneVec = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: i32Ty, arg: padded);
1285 auto getLane = [&](int64_t idx) -> Value {
1286 if (numLanes == 1)
1287 return laneVec;
1288 Value pos =
1289 LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: idx);
1290 return LLVM::ExtractElementOp::create(builder&: rewriter, location: loc, vector: laneVec, position: pos)
1291 ->getResult(idx: 0);
1292 };
1293
1294 std::string fnName = "__builtin_IB_dnscl_";
1295 fnName += (srcEtype == TruncfSrcElemTypes::F16) ? "hf16" : "bf16";
1296 Value convertTo =
1297 LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: kDnsclConvertToE2M1);
1298 auto genDnscl = [&](Value lo, Value hi, int mode) -> Value {
1299 Value modeVal = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: mode);
1300 SmallVector<Type> argTypes{lo.getType(), hi.getType(),
1301 convertTo.getType(), modeVal.getType()};
1302 SmallVector<Value> args{lo, hi, convertTo, modeVal};
1303 return createDeviceFunctionCall(rewriter, funcName: fnName, retType: i32Ty, argTypes, args,
1304 paramAttrs: {}, funcAttributeOptions: funcAttrs, op: op.getOperation())
1305 ->getResult(idx: 0);
1306 };
1307
1308 Value result;
1309 if (numLanes <= 2) {
1310 // Fewer than four lanes cannot fill a dword, so a single call is made
1311 // and the written bytes, 0 and 2, are compacted afterwards.
1312 Value lo = getLane(0);
1313 Value hi =
1314 numLanes == 2
1315 ? getLane(1)
1316 : LLVM::UndefOp::create(builder&: rewriter, location: loc, res: i32Ty)->getResult(idx: 0);
1317 Value dword = genDnscl(lo, hi, kDnsclModeBytes02);
1318 if (numLanes == 1) {
1319 // A single byte, so the low one, is all that is kept.
1320 result = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i8Ty, arg: dword);
1321 } else {
1322 Value bytes = LLVM::BitcastOp::create(
1323 builder&: rewriter, location: loc, res: VectorType::get(shape: 4, elementType: i8Ty), arg: dword);
1324 result = LLVM::ShuffleVectorOp::create(builder&: rewriter, location: loc, v1: bytes, v2: bytes,
1325 mask: ArrayRef<int32_t>{0, 2});
1326 }
1327 } else {
1328 // Four lanes, eight source elements, per fully packed dword.
1329 SmallVector<Value> dwords;
1330 for (int64_t base = 0; base < numLanes; base += 4) {
1331 // Each lane is bound to a name first: `getLane` builds ops, and the
1332 // order of evaluation within an argument list is unspecified, which
1333 // would otherwise leave the order of the emitted ops up to the host
1334 // compiler.
1335 Value lane0 = getLane(base);
1336 Value lane2 = getLane(base + 2);
1337 Value even = genDnscl(lane0, lane2, kDnsclModeBytes02);
1338 Value lane1 = getLane(base + 1);
1339 Value lane3 = getLane(base + 3);
1340 Value odd = genDnscl(lane1, lane3, kDnsclModeBytes13);
1341 dwords.push_back(Elt: LLVM::OrOp::create(builder&: rewriter, location: loc, lhs: even, rhs: odd));
1342 }
1343 if (dwords.size() == 1) {
1344 result = dwords.front();
1345 } else {
1346 Type packedTy = VectorType::get(shape: dwords.size(), elementType: i32Ty);
1347 result = LLVM::UndefOp::create(builder&: rewriter, location: loc, res: packedTy);
1348 for (auto [idx, dword] : llvm::enumerate(First&: dwords)) {
1349 Value pos = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: idx);
1350 result =
1351 LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: result, value: dword, position: pos)
1352 ->getResult(idx: 0);
1353 }
1354 }
1355 }
1356 rewriter.replaceOp(op, newValues: castIfNeeded(rewriter, loc, ty: dstTy, val: result));
1357 return success();
1358 }
1359
1360 // Handle the case where dst type is fp8, via the SPV_INTEL_fp_conversions
1361 // wrappers in IGC's BiFModule:
1362 // charN __builtin_spirv_Convert{FP16,BF16}To{E5M2,E4M3}EXT(halfN/shortN)
1363 //
1364 // These replace the __builtin_IB_* names used before, two of which did not
1365 // exist: IGC spells E4M3 as `e4m3` in the conversion builtins, not `hf8`.
1366 // createDeviceFunctionCall emits names unmangled, so the missing symbol
1367 // linked silently and the call returned undef. The wrappers are
1368 // `overloadable`, so mangle() takes the length from the argument type and a
1369 // mismatch fails to link instead. The bf16 forms take the bits as shortN
1370 // and route through f32 and f16 internally, replacing the explicit
1371 // bftof/convert_half round trip.
1372 //
1373 // `src_etype` is what says how to read the source, so the bf16 bitcast is
1374 // driven by it, and is only the step the wrapper's shortN signature needs.
1375 // An operand whose element type disagrees with `src_etype` is a case
1376 // `TruncfOp::verify` accepts today but should not: mangle() has no bf16
1377 // case, so it reaches the unreachable in getTypeMangling. That verifier
1378 // rule belongs with the op, not here.
1379 bool srcIsBF16 = srcEtype == TruncfSrcElemTypes::BF16;
1380 if (srcIsBF16)
1381 src = LLVM::BitcastOp::create(
1382 builder&: rewriter, location: loc,
1383 res: VectorType::get(shape: vecSrcTy.getShape(), elementType: rewriter.getI16Type()), arg: src);
1384 StringRef convertName;
1385 if (dstEtype == TruncfDstElemTypes::BF8) { // Float8E5M2Type
1386 convertName = srcIsBF16 ? "__builtin_spirv_ConvertBF16ToE5M2EXT"
1387 : "__builtin_spirv_ConvertFP16ToE5M2EXT";
1388 } else if (dstEtype == TruncfDstElemTypes::F8) { // Float8E4M3FNType
1389 convertName = srcIsBF16 ? "__builtin_spirv_ConvertBF16ToE4M3EXT"
1390 : "__builtin_spirv_ConvertFP16ToE4M3EXT";
1391 } else {
1392 return rewriter.notifyMatchFailure(
1393 arg&: op, msg: "Unsupported src, dst element type pair.");
1394 }
1395 SmallVector<Type> argTypes{src.getType()};
1396 SmallVector<Value> args{src};
1397 Value result = createDeviceFunctionCall(
1398 rewriter, funcName: mangle(baseName: convertName, types: argTypes), retType: dstTy, argTypes,
1399 args, paramAttrs: {}, funcAttributeOptions: funcAttrs, op: op.getOperation())
1400 ->getResult(idx: 0);
1401 rewriter.replaceOp(op, newValues: result);
1402 return success();
1403 }
1404};
1405
1406class ExtfToOCLPattern : public OpConversionPattern<ExtfOp> {
1407 using Base::Base;
1408 LogicalResult
1409 matchAndRewrite(ExtfOp op, ExtfOp::Adaptor adaptor,
1410 ConversionPatternRewriter &rewriter) const override {
1411 // `xevm.extf` is the inverse of `xevm.truncf`. Supported source and result
1412 // types are restricted for now, mirroring the truncf lowering.
1413 auto srcEtype = op.getSrcEtype().getEtype();
1414 auto dstEtype = op.getDstEtype().getEtype();
1415 // The source is scalar only where the packed values fit in one byte, which
1416 // SPIR-V spells as a scalar rather than a one element vector.
1417 Type srcTy = op.getSrc().getType();
1418 // Scalar dst is not supported until usage case become clear.
1419 auto vecDstTy = dyn_cast<VectorType>(Val: op.getDst().getType());
1420 if (!vecDstTy)
1421 return rewriter.notifyMatchFailure(arg&: op, msg: "Scalar dst is not supported.");
1422 // As for truncf, one builtin exists per SPIR-V vector length.
1423 int64_t numElements = vecDstTy.getNumElements();
1424 if (!isSupportedSPIRVVectorLength(numElements))
1425 return rewriter.notifyMatchFailure(
1426 arg&: op, msg: "dst vector length must be 2, 3, 4, 8 or 16");
1427 Location loc = op.getLoc();
1428 Value src = op.getSrc();
1429 auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>(
1430 /*other=*/args: LLVM::ModRefInfo::NoModRef,
1431 /*argMem=*/args: LLVM::ModRefInfo::NoModRef,
1432 /*inaccessibleMem=*/args: LLVM::ModRefInfo::NoModRef,
1433 /*errnoMem=*/args: LLVM::ModRefInfo::NoModRef,
1434 /*targetMem0=*/args: LLVM::ModRefInfo::NoModRef,
1435 /*targetMem1=*/args: LLVM::ModRefInfo::NoModRef);
1436 auto funcAttrs = convergentNoUnwindWillReturnAttrs;
1437 funcAttrs.memEffectsAttr = memAttr;
1438
1439 // Handle the case where src type is fp4 (e2m1) first.
1440 if (srcEtype == ExtfSrcElemTypes::E2M1) {
1441 // Two fp4 values are packed per source byte, and one builtin exists per
1442 // source byte count:
1443 // uint16 __builtin_IB_shfl_idx4_lut(int lut_index)
1444 // uint __builtin_IB_shfl_idx4_to_fp16_packed(uint16 lut, char src)
1445 // uintN __builtin_IB_shfl_idx4_to_fp16_N_packed(uint16 lut, charN src)
1446 // Each returns one dword, holding two f16/bf16 values, per source byte.
1447 // The lookup table selects the target format:
1448 // 7 = e2m1 -> f16, 5 = e2m1 -> bf16.
1449 //
1450 // A byte is the conversion granularity, so an odd length reads one spare
1451 // value that is dropped afterwards. The op verifier has already checked
1452 // that the source is exactly as wide as those bytes.
1453 int64_t numBytes = llvm::divideCeil(Numerator: numElements, Denominator: 2);
1454 constexpr int kLutE2M1ToF16 = 7;
1455 constexpr int kLutE2M1ToBF16 = 5;
1456 int lutIndex =
1457 (dstEtype == ExtfDstElemTypes::F16) ? kLutE2M1ToF16 : kLutE2M1ToBF16;
1458 Value lutIdx = LLVM::ConstantOp::create(builder&: rewriter, location: loc,
1459 type: rewriter.getI32Type(), value: lutIndex);
1460 Type lutTy = VectorType::get(shape: 16, elementType: rewriter.getI32Type());
1461 Value lut =
1462 createDeviceFunctionCall(rewriter, funcName: "__builtin_IB_shfl_idx4_lut",
1463 retType: lutTy, argTypes: {lutIdx.getType()}, args: {lutIdx}, paramAttrs: {},
1464 funcAttributeOptions: funcAttrs, op: op.getOperation())
1465 ->getResult(idx: 0);
1466 // A single byte is passed as a bare i8, and one dword returned as a bare
1467 // i32, rather than as one element vectors SPIR-V has no type for.
1468 Type i8Ty = rewriter.getI8Type();
1469 Type i32Ty = rewriter.getI32Type();
1470 std::string fnName = "__builtin_IB_shfl_idx4_to_fp16_";
1471 Type argTy, packedResTy;
1472 if (numBytes == 1) {
1473 argTy = i8Ty;
1474 packedResTy = i32Ty;
1475 } else {
1476 fnName += std::to_string(val: numBytes) + "_";
1477 argTy = VectorType::get(shape: numBytes, elementType: i8Ty);
1478 packedResTy = VectorType::get(shape: numBytes, elementType: i32Ty);
1479 }
1480 fnName += "packed";
1481 SmallVector<Type> convArgTypes{lut.getType(), argTy};
1482 SmallVector<Value> convArgs{lut, castIfNeeded(rewriter, loc, ty: argTy, val: src)};
1483 Value result =
1484 createDeviceFunctionCall(rewriter, funcName: fnName, retType: packedResTy, argTypes: convArgTypes,
1485 args: convArgs, paramAttrs: {}, funcAttributeOptions: funcAttrs, op: op.getOperation())
1486 ->getResult(idx: 0);
1487 // The builtin returns the f16/bf16 bits packed as i32, bitcast to the
1488 // f16/bf16 dst type and drop the padding an odd length produced.
1489 Type wideTy = VectorType::get(shape: numBytes * 2, elementType: vecDstTy.getElementType());
1490 result = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: wideTy, arg: result);
1491 result = takeLeadingElements(rewriter, loc, val: result, numElements);
1492 rewriter.replaceOp(op, newValues: result);
1493 return success();
1494 }
1495
1496 // Handle the case where src type is fp8 (bf8/hf8). One fp8 value per source
1497 // byte, so source and destination lengths match.
1498 auto vecSrcTy = dyn_cast<VectorType>(Val&: srcTy);
1499 if (!vecSrcTy || vecSrcTy.getNumElements() != numElements)
1500 return rewriter.notifyMatchFailure(
1501 arg&: op, msg: "fp8 src and dst must have the same number of elements");
1502
1503 // The matching wrappers, for the reasons given in the truncf lowering:
1504 // halfN/shortN __builtin_spirv_Convert{E5M2,E4M3}To{FP16,BF16}EXT(charN)
1505 //
1506 // `__builtin_IB_hf8tohf_N`, used before, never existed either. A bf16
1507 // destination returns the bits as shortN, replacing the
1508 // convert_float/ftobf round trip.
1509 bool dstIsBF16 = dstEtype == ExtfDstElemTypes::BF16;
1510 StringRef convertName;
1511 if (srcEtype == ExtfSrcElemTypes::BF8) { // Float8E5M2Type
1512 convertName = dstIsBF16 ? "__builtin_spirv_ConvertE5M2ToBF16EXT"
1513 : "__builtin_spirv_ConvertE5M2ToFP16EXT";
1514 } else { // Float8E4M3FNType
1515 convertName = dstIsBF16 ? "__builtin_spirv_ConvertE4M3ToBF16EXT"
1516 : "__builtin_spirv_ConvertE4M3ToFP16EXT";
1517 }
1518 Type resElemTy = dstIsBF16 ? cast<Type>(Val: rewriter.getI16Type())
1519 : cast<Type>(Val: rewriter.getF16Type());
1520 Type resTy = VectorType::get(shape: vecSrcTy.getShape(), elementType: resElemTy);
1521 SmallVector<Type> argTypes{src.getType()};
1522 SmallVector<Value> args{src};
1523 Value result = createDeviceFunctionCall(
1524 rewriter, funcName: mangle(baseName: convertName, types: argTypes), retType: resTy, argTypes,
1525 args, paramAttrs: {}, funcAttributeOptions: funcAttrs, op: op.getOperation())
1526 ->getResult(idx: 0);
1527 // The bf16 wrapper returns the bits as i16, so cast to the bf16 result
1528 // type. An f16 result needs nothing: the wrapper returns it.
1529 rewriter.replaceOp(op, newValues: castIfNeeded(rewriter, loc, ty: vecDstTy, val: result));
1530 return success();
1531 }
1532};
1533
1534class MMAMxToOCLPattern : public OpConversionPattern<MMAMxOp> {
1535 using Base::Base;
1536 LogicalResult
1537 matchAndRewrite(MMAMxOp op, MMAMxOp::Adaptor adaptor,
1538 ConversionPatternRewriter &rewriter) const override {
1539 if (!op.getC()) {
1540 return rewriter.notifyMatchFailure(arg&: op, msg: "OCL requires C operand");
1541 }
1542 auto precisionC = op.getTypes().getC();
1543 auto precisionD = op.getTypes().getD();
1544 if (precisionC != precisionD) {
1545 return rewriter.notifyMatchFailure(arg&: op, msg: "type of C and D need to match");
1546 }
1547
1548 constexpr uint32_t bitWidthPackedA{16};
1549 constexpr uint32_t bitWidthPackedB{32};
1550 auto loc = op.getLoc();
1551
1552 auto castIfNeeded = [&](Value val, Type packedType) -> Value {
1553 VectorType origTy = cast<VectorType>(Val: val.getType());
1554 const uint32_t vecBitSize =
1555 origTy.getNumElements() *
1556 origTy.getElementType().getIntOrFloatBitWidth();
1557 VectorType newTy = VectorType::get(
1558 shape: vecBitSize / packedType.getIntOrFloatBitWidth(), elementType: packedType);
1559 if (origTy != newTy)
1560 val = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: newTy, arg: val);
1561 return val;
1562 };
1563
1564 Value a = op.getA();
1565 Type packedAType = (op.getTypes().getA() == xevm::ElemType::TF32)
1566 ? cast<Type>(Val: rewriter.getF32Type())
1567 : rewriter.getIntegerType(width: bitWidthPackedA);
1568 a = castIfNeeded(a, packedAType);
1569
1570 Value b = op.getB();
1571 Type packedBType = (op.getTypes().getB() == xevm::ElemType::TF32)
1572 ? cast<Type>(Val: rewriter.getF32Type())
1573 : rewriter.getIntegerType(width: bitWidthPackedB);
1574 b = castIfNeeded(b, packedBType);
1575
1576 Value c = op.getC();
1577 VectorType cOrigTy = cast<VectorType>(Val: c.getType());
1578 VectorType resOrigTy = cast<VectorType>(Val: op->getResultTypes()[0]);
1579 assert(cOrigTy == resOrigTy && "Accumulator and result type mismatch");
1580 // OCL builtins encode bfloat16 as int16
1581 VectorType cTy =
1582 cOrigTy.getElementType().isBF16()
1583 ? VectorType::get(shape: cOrigTy.getShape(), elementType: rewriter.getIntegerType(width: 16))
1584 : cOrigTy;
1585 VectorType resTy = cTy;
1586 if (cOrigTy != cTy)
1587 c = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: cTy, arg: c);
1588
1589 std::string fnName =
1590 llvm::formatv(Fmt: "__builtin_IB_sub_group16_bdpas_{0}_{1}_{2}_{3}_8_8",
1591 Vals: builtinElemType(elemType: op.getTypes().getD()),
1592 Vals: builtinElemType(elemType: op.getTypes().getC()),
1593 Vals: builtinElemType(elemType: op.getTypes().getA()),
1594 Vals: builtinElemType(elemType: op.getTypes().getB()))
1595 .str();
1596 auto scaleA = op.getScaleA();
1597 auto scaleB = op.getScaleB();
1598 SmallVector<Type> argTypes{cTy, a.getType(), b.getType(), scaleA.getType(),
1599 scaleB.getType()};
1600 SmallVector<Value> args{c, a, b, scaleA, scaleB};
1601
1602 auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>(
1603 /*other=*/args: LLVM::ModRefInfo::NoModRef,
1604 /*argMem=*/args: LLVM::ModRefInfo::NoModRef,
1605 /*inaccessibleMem=*/args: LLVM::ModRefInfo::NoModRef,
1606 /*errnoMem=*/args: LLVM::ModRefInfo::NoModRef,
1607 /*targetMem0=*/args: LLVM::ModRefInfo::NoModRef,
1608 /*targetMem1=*/args: LLVM::ModRefInfo::NoModRef);
1609 auto funcAttrs = convergentNoUnwindWillReturnAttrs;
1610 funcAttrs.memEffectsAttr = memAttr;
1611 Value result =
1612 createDeviceFunctionCall(rewriter, funcName: fnName, retType: resTy, argTypes, args, paramAttrs: {},
1613 funcAttributeOptions: funcAttrs, op: op.getOperation())
1614 ->getResult(idx: 0);
1615
1616 if (resOrigTy != resTy)
1617 result = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: resOrigTy, arg: result);
1618
1619 rewriter.replaceOp(op, newValues: result);
1620 return success();
1621 }
1622};
1623
1624// Lowers `xevm.bitcast_shuffle` to a call to the IGC intrinsic
1625// `llvm.genx.GenISA.SubgroupBitcastShuffle`, which is overloaded on both the
1626// result and the operand type. E.g. a `vector<4xi8>` -> `vector<2xi16>` shuffle
1627// becomes a call to
1628// `llvm.genx.GenISA.SubgroupBitcastShuffle.v2i16.v4i8`.
1629//
1630// Only integer types reach here: the op accepts nothing else, so a producer
1631// holding floating point data bitcasts it to a same-width integer beforehand.
1632class BitcastShuffleToGenISAPattern
1633 : public OpConversionPattern<BitcastShuffleOp> {
1634 using Base::Base;
1635 LogicalResult
1636 matchAndRewrite(BitcastShuffleOp op, BitcastShuffleOp::Adaptor adaptor,
1637 ConversionPatternRewriter &rewriter) const override {
1638 Type srcTy = op.getSrc().getType();
1639 Type resTy = op.getRes().getType();
1640
1641 std::string fnName = "llvm.genx.GenISA.SubgroupBitcastShuffle." +
1642 getGenISATypeMangling(ty: resTy) + "." +
1643 getGenISATypeMangling(ty: srcTy);
1644
1645 Value result = createDeviceFunctionCall(
1646 rewriter, funcName: fnName, retType: resTy, argTypes: {srcTy}, args: {adaptor.getSrc()}, paramAttrs: {},
1647 funcAttributeOptions: convergentNoUnwindWillReturnAttrs, op: op.getOperation())
1648 ->getResult(idx: 0);
1649
1650 rewriter.replaceOp(op, newValues: result);
1651 return success();
1652 }
1653};
1654
1655class AllocaToGlobalPattern : public OpConversionPattern<LLVM::AllocaOp> {
1656 using Base::Base;
1657 LogicalResult
1658 matchAndRewrite(LLVM::AllocaOp op, LLVM::AllocaOp::Adaptor adaptor,
1659 ConversionPatternRewriter &rewriter) const override {
1660 auto ptrType = cast<LLVM::LLVMPointerType>(Val: op.getType());
1661 auto addrSpace = ptrType.getAddressSpace();
1662 if (addrSpace != 3)
1663 return failure();
1664 auto symTable = op->getParentWithTrait<OpTrait::SymbolTable>();
1665 if (!symTable)
1666 return failure();
1667 Block *moduleBody;
1668 if (ModuleOp mod = dyn_cast<ModuleOp>(Val&: *symTable)) {
1669 moduleBody = mod.getBody();
1670 } else if (gpu::GPUModuleOp gpuMod =
1671 dyn_cast<gpu::GPUModuleOp>(Val&: *symTable)) {
1672 moduleBody = gpuMod.getBody();
1673 } else {
1674 return failure();
1675 }
1676 auto val = op.getArraySize();
1677 APInt cst;
1678 if (!matchPattern(value: val, pattern: m_ConstantInt(bind_value: &cst)))
1679 return failure();
1680 auto loc = op.getLoc();
1681 auto globalType = LLVM::LLVMArrayType::get(
1682 context: rewriter.getContext(), elementType: op.getElemType(), numElements: cst.getZExtValue());
1683 LLVM::GlobalOp globalVar;
1684 {
1685 OpBuilder::InsertionGuard guard(rewriter);
1686 rewriter.setInsertionPointToStart(moduleBody);
1687 auto alignment = op.getAlignment();
1688 globalVar = LLVM::GlobalOp::create(
1689 builder&: rewriter, location: loc, type: globalType, /*isConstant=*/false,
1690 /*linkage=*/LLVM::Linkage::Internal,
1691 /*name=*/std::string("__global_alloca_") +
1692 std::to_string(val: getNextGlobalIdx()),
1693 /*value=*/Attribute(),
1694 /*alignment=*/alignment ? *alignment : 0, /*addrSpace=*/addrSpace);
1695 }
1696 rewriter.replaceOpWithNewOp<LLVM::AddressOfOp>(op, args&: globalVar);
1697 return success();
1698 }
1699
1700private:
1701 static unsigned getNextGlobalIdx() {
1702 static unsigned globalIdx = 0;
1703 return globalIdx++;
1704 }
1705};
1706
1707// Checks if shufflevector is used as a way to extract a contiguous slice
1708// from a vector.
1709// - source vector V2 is either the same as V1, or a poison/undef value. In
1710// both cases the mask can only meaningfully address elements of V1, which
1711// the mask checks below enforce.
1712// - mask size is not greater than the source vector size
1713// - mask values represent a sequence of consecutive increasing numbers
1714// that stay in bounds of the source vector when used for indexing.
1715static bool isExtractingContiguousSlice(LLVM::ShuffleVectorOp op) {
1716 if (op.getV1() != op.getV2() &&
1717 !isa_and_present<LLVM::PoisonOp, LLVM::UndefOp>(
1718 Val: op.getV2().getDefiningOp()))
1719 return false;
1720 auto maskAttr = op.getMask();
1721 int64_t maskSize = static_cast<int64_t>(maskAttr.size());
1722 int64_t sourceSize = op.getV1().getType().getNumElements();
1723 if (maskSize > sourceSize)
1724 return false;
1725 int64_t firstIndex = maskAttr[0];
1726 if (firstIndex < 0 || firstIndex >= sourceSize)
1727 return false;
1728 for (int64_t i = 1; i < maskSize; ++i) {
1729 int64_t index = maskAttr[i];
1730 if (index != firstIndex + i)
1731 return false;
1732 if (index >= sourceSize)
1733 return false;
1734 }
1735 return true;
1736}
1737
1738// Clones `op` with new operands and result types, preserving its attributes
1739// and properties (e.g. an `icmp` predicate or fastmath flags). Same idiom used
1740// by the Vector dialect's unroll patterns.
1741static Operation *cloneOpWithOperandsAndTypes(RewriterBase &rewriter,
1742 Location loc, Operation *op,
1743 ArrayRef<Value> operands,
1744 ArrayRef<Type> resultTypes) {
1745 OperationState state(loc, op->getName().getStringRef(), operands, resultTypes,
1746 op->getRawDictionaryAttrs().getValue());
1747 state.propertiesAttr = op->getPropertiesAsAttribute();
1748 return rewriter.create(state);
1749}
1750
1751// Input vector of a shuffle vector op extracting a contiguous slice is an
1752// illegal vector in SPIRV kernel if the vector size is > 16 elements.
1753// To legalize this case, keep applying the following transformations until no
1754// more match:
1755// 1. keep hoisting the shuffle vector op past unary element-wise operations
1756// start with fpext, fptrunc and bitcast for now.
1757// 2. merge with another shuffle vector op
1758// 3. merge with load as a smaller load
1759// 4. hoist past n-ary element-wise operations (e.g. fdiv, select, icmp) by
1760// slicing every same-length vector operand with the same mask and cloning
1761// the op at the narrow width. Only done when the source is wider than 16
1762// elements.
1763class HandleVectorExtractPattern
1764 : public OpRewritePattern<LLVM::ShuffleVectorOp> {
1765 using OpRewritePattern<LLVM::ShuffleVectorOp>::OpRewritePattern;
1766
1767 void initialize() { setHasBoundedRewriteRecursion(); }
1768
1769 LogicalResult matchAndRewrite(LLVM::ShuffleVectorOp op,
1770 PatternRewriter &rewriter) const override {
1771
1772 if (!isExtractingContiguousSlice(op))
1773 return failure();
1774
1775 auto mask = op.getMask();
1776 auto loc = op.getLoc();
1777 auto ty = op.getType();
1778 // Check source operand to determine rewrite pattern.
1779 auto src = op.getV1();
1780 // 1. Hoist past unary element-wise operations
1781 if (auto srcOp = src.getDefiningOp()) {
1782 if (isa<LLVM::FPExtOp>(Val: srcOp) || isa<LLVM::FPTruncOp>(Val: srcOp)) {
1783 Value srcInput = srcOp->getOperand(idx: 0);
1784 // Create new shuffle vector op with unary input as source.
1785 auto srcVecTy = dyn_cast<VectorType>(Val: srcInput.getType());
1786 if (!srcVecTy)
1787 return failure();
1788 auto newShuffleVecTy =
1789 VectorType::get(shape: mask.size(), elementType: srcVecTy.getElementType());
1790 auto newShuffle = LLVM::ShuffleVectorOp::create(
1791 builder&: rewriter, location: loc, res: newShuffleVecTy, v1: srcInput, v2: srcInput, mask);
1792 // Create new unary op with new shuffle as input.
1793 Value newUnaryOp;
1794 if (isa<LLVM::FPExtOp>(Val: srcOp)) {
1795 newUnaryOp = LLVM::FPExtOp::create(builder&: rewriter, location: loc, res: ty, arg: newShuffle);
1796 } else {
1797 newUnaryOp = LLVM::FPTruncOp::create(builder&: rewriter, location: loc, res: ty, arg: newShuffle);
1798 }
1799 rewriter.replaceOp(op, newValues: newUnaryOp);
1800 } else if (isa<LLVM::BitcastOp>(Val: srcOp)) {
1801 Value srcInput = srcOp->getOperand(idx: 0);
1802 // Create new shuffle vector op with unary input as source. A bitcast
1803 // from a scalar has no slice to rewrite in terms of.
1804 auto srcInputVecTy = dyn_cast<VectorType>(Val: srcInput.getType());
1805 auto srcResVecTy = dyn_cast<VectorType>(Val: srcOp->getResult(idx: 0).getType());
1806 if (!srcInputVecTy || !srcResVecTy)
1807 return failure();
1808 auto srcInputSize = srcInputVecTy.getNumElements();
1809 auto srcResSize = srcResVecTy.getNumElements();
1810 auto maskSize = static_cast<int32_t>(mask.size());
1811 if (srcInputSize > srcResSize) {
1812 return failure();
1813 }
1814 if (srcResSize % srcInputSize != 0) {
1815 return failure();
1816 }
1817 auto maskScale = srcResSize / srcInputSize;
1818 // Storage for the rescaled mask. `mask` is an ArrayRef that gets
1819 // rebound to this buffer below, so it has to outlive the `if` that
1820 // fills it - otherwise the uses after the `if` read a destroyed
1821 // SmallVector.
1822 SmallVector<int32_t> newMask;
1823 if (maskScale != 1) {
1824 // The slice has to start at, and cover, whole source elements to be
1825 // expressible in terms of the bitcast source.
1826 if (mask[0] % maskScale != 0 || maskSize % maskScale != 0) {
1827 return failure();
1828 }
1829 // Create a new mask that maps to the source vector
1830 int32_t newMaskSize = maskSize / maskScale;
1831 int32_t maskStart = mask[0] / maskScale;
1832 for (int32_t i = 0; i < newMaskSize; ++i) {
1833 newMask.push_back(Elt: maskStart + i);
1834 }
1835 mask = newMask;
1836 }
1837 auto newShuffleVecTy = VectorType::get(
1838 shape: static_cast<int64_t>(mask.size()), elementType: srcInputVecTy.getElementType());
1839 auto newShuffle = LLVM::ShuffleVectorOp::create(
1840 builder&: rewriter, location: loc, res: newShuffleVecTy, v1: srcInput, v2: srcInput, mask);
1841 // Create new unary op with new shuffle as input.
1842 auto newBitcast =
1843 LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: ty, arg: newShuffle);
1844 rewriter.replaceOp(op, newOp: newBitcast);
1845 } else if (isa<LLVM::ShuffleVectorOp>(Val: srcOp)) {
1846 // 2. Merge with source shuffle vector op if, the source op is
1847 // also extracting a contigous slice and create a new
1848 // shuffle vector op directly from the source of
1849 // the first shuffle.
1850 auto srcShuffle = cast<LLVM::ShuffleVectorOp>(Val: srcOp);
1851 if (!isExtractingContiguousSlice(op: srcShuffle))
1852 return failure();
1853 auto srcMask = srcShuffle.getMask();
1854 SmallVector<int32_t> combinedMask;
1855 for (auto index : mask) {
1856 combinedMask.push_back(Elt: srcMask[index]);
1857 }
1858 auto newShuffle = LLVM::ShuffleVectorOp::create(
1859 builder&: rewriter, location: loc, res: ty, v1: srcShuffle.getV1(), v2: srcShuffle.getV1(),
1860 mask: DenseI32ArrayAttr::get(context: rewriter.getContext(), content: combinedMask));
1861 rewriter.replaceOp(op, newOp: newShuffle);
1862 } else if (isa<LLVM::LoadOp>(Val: srcOp)) {
1863 // 3. Merge with load as a smaller load
1864 auto loadOp = cast<LLVM::LoadOp>(Val: srcOp);
1865 auto loadPtr = loadOp.getAddr();
1866 auto loadAddrSpace = loadPtr.getType().getAddressSpace();
1867 if (loadAddrSpace != 0)
1868 return failure();
1869 auto loadTy = dyn_cast<VectorType>(Val: loadOp.getType());
1870 if (!loadTy)
1871 return failure();
1872 auto elemTy = loadTy.getElementType();
1873 auto firstIndex = mask[0];
1874 auto newVecTy = VectorType::get(shape: mask.size(), elementType: elemTy);
1875 // GEPOp is needed if first index is not zero
1876 if (firstIndex) {
1877 auto newPtr = LLVM::GEPOp::create(
1878 builder&: rewriter, location: loc,
1879 resultType: LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: loadAddrSpace),
1880 elementType: elemTy, basePtr: loadPtr, indices: ArrayRef<LLVM::GEPArg>{firstIndex});
1881 auto newLoad = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: newVecTy, addr: newPtr);
1882 rewriter.replaceOp(op, newOp: newLoad);
1883 } else {
1884 auto newLoad = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: newVecTy, addr: loadPtr);
1885 rewriter.replaceOp(op, newOp: newLoad);
1886 }
1887 } else if (isMemoryEffectFree(op: srcOp) && srcOp->getNumResults() == 1 &&
1888 srcOp->getNumOperands() >= 1 &&
1889 // Only to legalize. A source of 16 or fewer elements is
1890 // already legal, and splitting it would put a narrow copy of
1891 // the op next to every slice user.
1892 cast<VectorType>(Val: src.getType()).getNumElements() > 16 &&
1893 llvm::all_of(Range: srcOp->getOperands(), P: [&](Value operand) {
1894 auto operandTy = dyn_cast<VectorType>(Val: operand.getType());
1895 auto srcTy = cast<VectorType>(Val: src.getType());
1896 return operandTy && operandTy.getRank() == 1 &&
1897 operandTy.getNumElements() == srcTy.getNumElements();
1898 })) {
1899 // 4. Hoist past an n-ary element-wise op: slice each same-length vector
1900 // operand with the same contiguous mask and clone the op narrow.
1901 // The freshly created operand shuffles keep the greedy driver going,
1902 // hoisting them further up (or merging via case 2 / into loads via
1903 // case 3) until they reach operands that are already legal width.
1904 SmallVector<Value> newOperands;
1905 newOperands.reserve(N: srcOp->getNumOperands());
1906 for (Value operand : srcOp->getOperands()) {
1907 auto operandTy = cast<VectorType>(Val: operand.getType());
1908 auto sliceTy =
1909 VectorType::get(shape: mask.size(), elementType: operandTy.getElementType());
1910 newOperands.push_back(Elt: LLVM::ShuffleVectorOp::create(
1911 builder&: rewriter, location: loc, res: sliceTy, v1: operand, v2: operand, mask));
1912 }
1913 Operation *newOp =
1914 cloneOpWithOperandsAndTypes(rewriter, loc, op: srcOp, operands: newOperands, resultTypes: ty);
1915 rewriter.replaceOp(op, newValues: newOp->getResult(idx: 0));
1916 } else {
1917 return failure();
1918 }
1919 } else {
1920 // No defining op (e.g. function argument): nothing to hoist/merge.
1921 return failure();
1922 }
1923 return success();
1924 }
1925};
1926
1927//===----------------------------------------------------------------------===//
1928// Pass Definition
1929//===----------------------------------------------------------------------===//
1930
1931struct ConvertXeVMToLLVMPass
1932 : public impl::ConvertXeVMToLLVMPassBase<ConvertXeVMToLLVMPass> {
1933 using Base::Base;
1934
1935 void getDependentDialects(DialectRegistry &registry) const override {
1936 registry.insert<LLVM::LLVMDialect, XeVMDialect>();
1937 }
1938
1939 void runOnOperation() override {
1940 ConversionTarget target(getContext());
1941 RewritePatternSet patterns(&getContext());
1942 populateXeVMToLLVMConversionPatterns(target, patterns);
1943 if (failed(Result: applyPartialConversion(op: getOperation(), target,
1944 patterns: std::move(patterns))))
1945 signalPassFailure();
1946
1947 // Apply in-dialect lowerings to handle illegal vectors
1948 {
1949 RewritePatternSet vectorPatterns(&getContext());
1950 vectorPatterns.add<HandleVectorExtractPattern>(arg: &getContext());
1951 GreedyRewriteConfig config{};
1952 // folding can remove ops with temporary attributes used to
1953 // represent LLVM metadata, so disable it here.
1954 // Effectively just this single pattern is applied without any
1955 // op folding patterns from dialects.
1956 config.enableFolding(enable: false);
1957 // config.setMaxIterations(GreedyRewriteConfig::kNoLimit);
1958 // config.setMaxNumRewrites(GreedyRewriteConfig::kNoLimit);
1959 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(vectorPatterns),
1960 config);
1961 }
1962 }
1963};
1964} // namespace
1965
1966//===----------------------------------------------------------------------===//
1967// Pattern Population
1968//===----------------------------------------------------------------------===//
1969
1970void ::mlir::populateXeVMToLLVMConversionPatterns(ConversionTarget &target,
1971 RewritePatternSet &patterns) {
1972 // some LLVM operations need to be converted.
1973 target.addDynamicallyLegalDialect<LLVM::LLVMDialect>(callback: [](Operation *op) {
1974 // llvm alloca op with addrspace 3 for OpenCL (Workgroup) is not handled
1975 // properly by SPIRV backend. It needs to be rewritten as a sequence with
1976 // llvm global.
1977 if (isa<LLVM::AllocaOp>(Val: op)) {
1978 LLVM::AllocaOp aOp = cast<LLVM::AllocaOp>(Val: op);
1979 LLVM::LLVMPointerType pTy = cast<LLVM::LLVMPointerType>(Val: aOp.getType());
1980 auto addrSpace = pTy.getAddressSpace();
1981 return addrSpace != 3;
1982 }
1983 // cache_control attribute should be converted.
1984 return !op->hasDiscardableAttr(name: "cache_control");
1985 });
1986 target.addIllegalDialect<XeVMDialect>();
1987 patterns.add<LoadStorePrefetchToOCLPattern<BlockLoad2dOp>,
1988 LoadStorePrefetchToOCLPattern<BlockStore2dOp>,
1989 LoadStorePrefetchToOCLPattern<BlockPrefetch2dOp>,
1990 MMAToOCLPattern, MemfenceToOCLPattern, PrefetchToOCLPattern,
1991 LLVMLoadStoreToOCLPattern<LLVM::LoadOp>,
1992 LLVMLoadStoreToOCLPattern<LLVM::StoreOp>,
1993 BlockLoadStore1DToOCLPattern<BlockLoadOp>,
1994 BlockLoadStore1DToOCLPattern<BlockStoreOp>,
1995 LaunchConfigOpToOCLPattern<WorkitemIdXOp>,
1996 LaunchConfigOpToOCLPattern<WorkitemIdYOp>,
1997 LaunchConfigOpToOCLPattern<WorkitemIdZOp>,
1998 LaunchConfigOpToOCLPattern<WorkgroupDimXOp>,
1999 LaunchConfigOpToOCLPattern<WorkgroupDimYOp>,
2000 LaunchConfigOpToOCLPattern<WorkgroupDimZOp>,
2001 LaunchConfigOpToOCLPattern<WorkgroupIdXOp>,
2002 LaunchConfigOpToOCLPattern<WorkgroupIdYOp>,
2003 LaunchConfigOpToOCLPattern<WorkgroupIdZOp>,
2004 LaunchConfigOpToOCLPattern<GridDimXOp>,
2005 LaunchConfigOpToOCLPattern<GridDimYOp>,
2006 LaunchConfigOpToOCLPattern<GridDimZOp>,
2007 SubgroupOpWorkitemOpToOCLPattern<LaneIdOp>,
2008 SubgroupOpWorkitemOpToOCLPattern<SubgroupIdOp>,
2009 SubgroupOpWorkitemOpToOCLPattern<SubgroupSizeOp>,
2010 TruncfToOCLPattern, ExtfToOCLPattern, MMAMxToOCLPattern,
2011 BitcastShuffleToGenISAPattern, AllocaToGlobalPattern>(
2012 arg: patterns.getContext());
2013}
2014