| 1 | //===-- XeVMToLLVM.cpp - XeVM to LLVM dialect conversion --------*- C++ -*-===// |
| 2 | // |
| 3 | // This file is licensed under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/XeVMToLLVM/XeVMToLLVM.h" |
| 10 | |
| 11 | #include "mlir/Conversion/LLVMCommon/Pattern.h" |
| 12 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 13 | #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h" |
| 14 | #include "mlir/Dialect/LLVMIR/LLVMArithOps.h" |
| 15 | #include "mlir/Dialect/LLVMIR/LLVMCallOps.h" |
| 16 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 17 | #include "mlir/Dialect/LLVMIR/LLVMConversionOps.h" |
| 18 | #include "mlir/Dialect/LLVMIR/LLVMGeneralIntrinsicOps.h" |
| 19 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 20 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 21 | #include "mlir/Dialect/LLVMIR/LLVMVectorOps.h" |
| 22 | #include "mlir/Dialect/LLVMIR/XeVMDialect.h" |
| 23 | #include "mlir/Pass/Pass.h" |
| 24 | #include "mlir/Support/LLVM.h" |
| 25 | #include "llvm/ADT/ArrayRef.h" |
| 26 | #include "llvm/ADT/STLExtras.h" |
| 27 | #include "llvm/Support/FormatVariadic.h" |
| 28 | #include "llvm/Support/MathExtras.h" |
| 29 | |
| 30 | #include "mlir/IR/BuiltinTypes.h" |
| 31 | #include "mlir/IR/Matchers.h" |
| 32 | #include "mlir/IR/Types.h" |
| 33 | #include "mlir/Interfaces/SideEffectInterfaces.h" |
| 34 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 35 | |
| 36 | #include "llvm/ADT/TypeSwitch.h" |
| 37 | |
| 38 | namespace mlir { |
| 39 | #define GEN_PASS_DEF_CONVERTXEVMTOLLVMPASS |
| 40 | #include "mlir/Conversion/Passes.h.inc" |
| 41 | } // namespace mlir |
| 42 | |
| 43 | using namespace mlir; |
| 44 | using namespace xevm; |
| 45 | |
| 46 | namespace { |
| 47 | |
| 48 | struct LLVMFuncAttributeOptions { |
| 49 | bool isConvergent = false; |
| 50 | bool isNoUnwind = false; |
| 51 | bool isWillReturn = false; |
| 52 | LLVM::MemoryEffectsAttr memEffectsAttr{}; |
| 53 | }; |
| 54 | static constexpr LLVMFuncAttributeOptions noUnwindAttrs = { |
| 55 | .isConvergent: false, .isNoUnwind: true, .isWillReturn: false, .memEffectsAttr: {}}; |
| 56 | static constexpr LLVMFuncAttributeOptions noUnwindWillReturnAttrs = { |
| 57 | .isConvergent: false, .isNoUnwind: true, .isWillReturn: true, .memEffectsAttr: {}}; |
| 58 | static constexpr LLVMFuncAttributeOptions convergentNoUnwindWillReturnAttrs = { |
| 59 | .isConvergent: true, .isNoUnwind: true, .isWillReturn: true, .memEffectsAttr: {}}; |
| 60 | |
| 61 | std::string getTypeMangling(Type ty, bool isUnsigned = false) { |
| 62 | return TypeSwitch<Type, std::string>(ty) |
| 63 | .Case(caseFn: [isUnsigned](VectorType ty) -> std::string { |
| 64 | return "Dv" + std::to_string(val: ty.getNumElements()) + "_" + |
| 65 | getTypeMangling(ty: ty.getElementType(), isUnsigned); |
| 66 | }) |
| 67 | .Case(caseFn: [](Float16Type) -> std::string { return "Dh" ; }) |
| 68 | .Case(caseFn: [](Float32Type) -> std::string { return "f" ; }) |
| 69 | .Case(caseFn: [](Float64Type) -> std::string { return "d" ; }) |
| 70 | .Case(caseFn: [isUnsigned](IntegerType ty) -> std::string { |
| 71 | switch (ty.getWidth()) { |
| 72 | case 8: |
| 73 | return isUnsigned ? "h" : "c" ; |
| 74 | case 16: |
| 75 | return isUnsigned ? "t" : "s" ; |
| 76 | case 32: |
| 77 | return isUnsigned ? "j" : "i" ; |
| 78 | case 64: |
| 79 | return isUnsigned ? "m" : "l" ; |
| 80 | default: |
| 81 | llvm_unreachable("unhandled integer type" ); |
| 82 | } |
| 83 | }) |
| 84 | .DefaultUnreachable(message: "unhandled type for mangling" ); |
| 85 | } |
| 86 | |
| 87 | std::string mangle(StringRef baseName, ArrayRef<Type> types, |
| 88 | ArrayRef<bool> isUnsigned = {}) { |
| 89 | assert((isUnsigned.empty() || isUnsigned.size() == types.size()) && |
| 90 | "Signedness info doesn't match" ); |
| 91 | std::string s; |
| 92 | llvm::raw_string_ostream os(s); |
| 93 | llvm::SmallDenseMap<Type, unsigned> substitutions; |
| 94 | os << "_Z" << baseName.size() << baseName; |
| 95 | for (auto [idx, type] : llvm::enumerate(First&: types)) { |
| 96 | auto it = substitutions.find(Val: type); |
| 97 | if (it != substitutions.end()) { |
| 98 | os << "S" ; |
| 99 | // First substitution is `S_`, second is `S0_`, and so on. |
| 100 | if (unsigned firstIdx = it->getSecond(); firstIdx > 0) |
| 101 | os << firstIdx - 1; |
| 102 | os << "_" ; |
| 103 | } else { |
| 104 | if (!type.isIntOrFloat()) |
| 105 | substitutions[type] = substitutions.size(); |
| 106 | os << getTypeMangling(ty: type, isUnsigned: isUnsigned.empty() ? false : isUnsigned[idx]); |
| 107 | } |
| 108 | } |
| 109 | return os.str(); |
| 110 | } |
| 111 | |
| 112 | // Returns the mangling of `ty` used to name an overloaded `llvm.genx.GenISA.*` |
| 113 | // intrinsic: `i32`, `v8i16`, ... Note that this is IGC's own scheme for its |
| 114 | // intrinsics, not the Itanium mangling used for the SPIR-V friendly and OCL |
| 115 | // builtins that `mangle` above produces. |
| 116 | std::string getGenISATypeMangling(Type ty) { |
| 117 | return TypeSwitch<Type, std::string>(ty) |
| 118 | .Case(caseFn: [](VectorType ty) -> std::string { |
| 119 | return "v" + std::to_string(val: ty.getNumElements()) + |
| 120 | getGenISATypeMangling(ty: ty.getElementType()); |
| 121 | }) |
| 122 | .Case(caseFn: [](IntegerType ty) -> std::string { |
| 123 | return "i" + std::to_string(val: ty.getWidth()); |
| 124 | }) |
| 125 | .DefaultUnreachable(message: "unhandled type for GenISA mangling" ); |
| 126 | } |
| 127 | |
| 128 | std::string builtinElemType(ElemType elemType) { |
| 129 | switch (elemType) { |
| 130 | case ElemType::BF8: |
| 131 | return "bf8" ; |
| 132 | case ElemType::F8: |
| 133 | return "hf8" ; |
| 134 | case ElemType::BF16: |
| 135 | return "bf" ; |
| 136 | case ElemType::F16: |
| 137 | return "hf" ; |
| 138 | case ElemType::F32: |
| 139 | return "f" ; |
| 140 | default: |
| 141 | return stringifyElemType(elemType).str(); |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | static int32_t getL1CacheControl(LoadCacheControl cc) { |
| 146 | int32_t control = 0; |
| 147 | switch (cc) { |
| 148 | case LoadCacheControl::USE_DEFAULT: |
| 149 | control = -1; |
| 150 | break; |
| 151 | case LoadCacheControl::L1C_L2UC_L3UC: |
| 152 | case LoadCacheControl::L1C_L2UC_L3C: |
| 153 | case LoadCacheControl::L1C_L2C_L3UC: |
| 154 | case LoadCacheControl::L1C_L2C_L3C: |
| 155 | control = 1; |
| 156 | break; |
| 157 | case LoadCacheControl::L1S_L2UC_L3UC: |
| 158 | case LoadCacheControl::L1S_L2UC_L3C: |
| 159 | case LoadCacheControl::L1S_L2C_L3UC: |
| 160 | case LoadCacheControl::L1S_L2C_L3C: |
| 161 | control = 2; |
| 162 | break; |
| 163 | case LoadCacheControl::INVALIDATE_READ: |
| 164 | control = 3; |
| 165 | break; |
| 166 | default: |
| 167 | break; |
| 168 | } |
| 169 | return control; |
| 170 | } |
| 171 | |
| 172 | static int32_t getL1CacheControl(StoreCacheControl cc) { |
| 173 | int32_t control = 0; |
| 174 | switch (cc) { |
| 175 | case StoreCacheControl::USE_DEFAULT: |
| 176 | control = -1; |
| 177 | break; |
| 178 | case StoreCacheControl::L1WT_L2UC_L3UC: |
| 179 | case StoreCacheControl::L1WT_L2UC_L3WB: |
| 180 | case StoreCacheControl::L1WT_L2WB_L3UC: |
| 181 | case StoreCacheControl::L1WT_L2WB_L3WB: |
| 182 | control = 1; |
| 183 | break; |
| 184 | case StoreCacheControl::L1WB_L2UC_L3UC: |
| 185 | case StoreCacheControl::L1WB_L2WB_L3UC: |
| 186 | case StoreCacheControl::L1WB_L2UC_L3WB: |
| 187 | control = 2; |
| 188 | break; |
| 189 | case StoreCacheControl::L1S_L2UC_L3UC: |
| 190 | case StoreCacheControl::L1S_L2UC_L3WB: |
| 191 | case StoreCacheControl::L1S_L2WB_L3UC: |
| 192 | case StoreCacheControl::L1S_L2WB_L3WB: |
| 193 | control = 3; |
| 194 | break; |
| 195 | default: |
| 196 | break; |
| 197 | } |
| 198 | return control; |
| 199 | } |
| 200 | |
| 201 | static int32_t getL3CacheControl(LoadCacheControl cc) { |
| 202 | int32_t control = 0; |
| 203 | switch (cc) { |
| 204 | case LoadCacheControl::USE_DEFAULT: |
| 205 | control = -1; |
| 206 | break; |
| 207 | case LoadCacheControl::L1UC_L2UC_L3C: |
| 208 | case LoadCacheControl::L1UC_L2C_L3C: |
| 209 | case LoadCacheControl::L1C_L2UC_L3C: |
| 210 | case LoadCacheControl::L1C_L2C_L3C: |
| 211 | case LoadCacheControl::L1S_L2UC_L3C: |
| 212 | case LoadCacheControl::L1S_L2C_L3C: |
| 213 | control = 1; |
| 214 | break; |
| 215 | case LoadCacheControl::INVALIDATE_READ: |
| 216 | control = 3; |
| 217 | break; |
| 218 | default: |
| 219 | break; |
| 220 | } |
| 221 | return control; |
| 222 | } |
| 223 | |
| 224 | static int32_t getL3CacheControl(StoreCacheControl cc) { |
| 225 | int32_t control = 0; |
| 226 | switch (cc) { |
| 227 | case StoreCacheControl::USE_DEFAULT: |
| 228 | control = -1; |
| 229 | break; |
| 230 | case StoreCacheControl::L1UC_L2UC_L3WB: |
| 231 | case StoreCacheControl::L1UC_L2WB_L3WB: |
| 232 | case StoreCacheControl::L1WT_L2UC_L3WB: |
| 233 | case StoreCacheControl::L1WT_L2WB_L3WB: |
| 234 | case StoreCacheControl::L1S_L2UC_L3WB: |
| 235 | case StoreCacheControl::L1S_L2WB_L3WB: |
| 236 | case StoreCacheControl::L1WB_L2UC_L3WB: |
| 237 | control = 2; |
| 238 | break; |
| 239 | default: |
| 240 | break; |
| 241 | } |
| 242 | return control; |
| 243 | } |
| 244 | |
| 245 | static std::optional<LoadCacheControl> getCacheControl(PrefetchOp op) { |
| 246 | return op.getCacheControl(); |
| 247 | } |
| 248 | |
| 249 | static std::optional<LoadCacheControl> getCacheControl(BlockLoad2dOp op) { |
| 250 | return op.getCacheControl(); |
| 251 | } |
| 252 | |
| 253 | static std::optional<LoadCacheControl> getCacheControl(BlockLoadOp op) { |
| 254 | return op.getCacheControl(); |
| 255 | } |
| 256 | |
| 257 | static std::optional<LoadCacheControl> getCacheControl(BlockPrefetch2dOp op) { |
| 258 | return op.getCacheControl(); |
| 259 | } |
| 260 | |
| 261 | static std::optional<StoreCacheControl> getCacheControl(BlockStore2dOp op) { |
| 262 | return op.getCacheControl(); |
| 263 | } |
| 264 | |
| 265 | static std::optional<StoreCacheControl> getCacheControl(BlockStoreOp op) { |
| 266 | return op.getCacheControl(); |
| 267 | } |
| 268 | |
| 269 | static std::optional<LoadCacheControl> getCacheControl(LLVM::LoadOp op) { |
| 270 | if (op->hasDiscardableAttr(name: "cache_control" )) { |
| 271 | auto attr = op->getDiscardableAttrOfType<xevm::LoadCacheControlAttr>( |
| 272 | name: "cache_control" ); |
| 273 | if (!attr) |
| 274 | return std::nullopt; |
| 275 | return std::optional<LoadCacheControl>(attr.getValue()); |
| 276 | } |
| 277 | return std::nullopt; |
| 278 | } |
| 279 | |
| 280 | static std::optional<StoreCacheControl> getCacheControl(LLVM::StoreOp op) { |
| 281 | if (op->hasDiscardableAttr(name: "cache_control" )) { |
| 282 | auto attr = op->getDiscardableAttrOfType<xevm::StoreCacheControlAttr>( |
| 283 | name: "cache_control" ); |
| 284 | if (!attr) |
| 285 | return std::nullopt; |
| 286 | return std::optional<StoreCacheControl>(attr.getValue()); |
| 287 | } |
| 288 | return std::nullopt; |
| 289 | } |
| 290 | |
| 291 | template <typename OpType> |
| 292 | int32_t getL1CacheControl(OpType op) { |
| 293 | return getL1CacheControl(*getCacheControl(op)); |
| 294 | } |
| 295 | |
| 296 | template <typename OpType> |
| 297 | int32_t getL3CacheControl(OpType op) { |
| 298 | return getL3CacheControl(*getCacheControl(op)); |
| 299 | } |
| 300 | |
| 301 | template <typename OpType> |
| 302 | static std::optional<ArrayAttr> |
| 303 | getCacheControlMetadata(ConversionPatternRewriter &rewriter, OpType op) { |
| 304 | if (!getCacheControl(op)) |
| 305 | return {}; |
| 306 | |
| 307 | constexpr int32_t decorationCacheControlArity{3}; |
| 308 | constexpr int32_t loadCacheControlKey{6442}; |
| 309 | constexpr int32_t storeCacheControlKey{6443}; |
| 310 | constexpr bool isLoad = std::is_same_v<OpType, BlockLoad2dOp> || |
| 311 | std::is_same_v<OpType, BlockPrefetch2dOp> || |
| 312 | std::is_same_v<OpType, LLVM::LoadOp> || |
| 313 | std::is_same_v<OpType, BlockLoadOp> || |
| 314 | std::is_same_v<OpType, PrefetchOp>; |
| 315 | |
| 316 | // If the cache control is USE_DEFAULT, then we don’t emit any metadata. |
| 317 | // Assert that if one of the L1 or L3 cache control values is USE_DEFAULT |
| 318 | // (represented as -1), then both must be USE_DEFAULT; otherwise there is a |
| 319 | // bug. |
| 320 | assert(((getL1CacheControl<OpType>(op) == -1) == |
| 321 | (getL3CacheControl<OpType>(op) == -1)) && |
| 322 | "If one of L1 or L3 cache control is USE_DEFAULT, both must be " |
| 323 | "USE_DEFAULT" ); |
| 324 | |
| 325 | if (getL1CacheControl<OpType>(op) == -1 && |
| 326 | getL3CacheControl<OpType>(op) == -1) |
| 327 | return {}; |
| 328 | const int32_t controlKey{isLoad ? loadCacheControlKey : storeCacheControlKey}; |
| 329 | SmallVector<int32_t, decorationCacheControlArity> decorationsL1{ |
| 330 | controlKey, 0, getL1CacheControl<OpType>(op)}; |
| 331 | SmallVector<int32_t, decorationCacheControlArity> decorationsL3{ |
| 332 | controlKey, 1, getL3CacheControl<OpType>(op)}; |
| 333 | auto arrayAttrL1 = rewriter.getI32ArrayAttr(values: decorationsL1); |
| 334 | auto arrayAttrL3 = rewriter.getI32ArrayAttr(values: decorationsL3); |
| 335 | |
| 336 | SmallVector<Attribute, 2> combinedAttrs = {arrayAttrL1, arrayAttrL3}; |
| 337 | return rewriter.getArrayAttr(value: combinedAttrs); |
| 338 | } |
| 339 | |
| 340 | //===----------------------------------------------------------------------===// |
| 341 | // Cache control annotation utilities |
| 342 | // |
| 343 | // Instead of attaching cache control as MLIR attributes and handling them |
| 344 | // during LLVM translation, we directly emit llvm.intr.ptr.annotation op in |
| 345 | // MLIR. |
| 346 | //===----------------------------------------------------------------------===// |
| 347 | |
| 348 | /// Build one cache-control payload string per attribute. |
| 349 | /// |
| 350 | /// Each Attribute is expected to be an ArrayAttr of 3 IntegerAttr values: |
| 351 | /// [SPIR-V decoration token, cache level, cache control value] |
| 352 | /// |
| 353 | /// A single entry produces a string like: {6442:"0,1"} |
| 354 | /// where the quote characters (0x22) will appear as \22 in LLVM IR textual |
| 355 | /// form. |
| 356 | static SmallVector<std::string> |
| 357 | buildCacheControlPayloads(ArrayRef<Attribute> attrs) { |
| 358 | SmallVector<std::string> payloads; |
| 359 | llvm::StringMap<bool> seen; |
| 360 | |
| 361 | for (auto arr : llvm::make_isa_range<ArrayAttr>(Range&: attrs)) { |
| 362 | auto vals = arr.getValue(); |
| 363 | assert(vals.size() == 3 && |
| 364 | "Expected exactly 3 integer values (Token, CacheLevel, " |
| 365 | "ControlValue) in cache control attribute." ); |
| 366 | |
| 367 | auto tokenAttr = dyn_cast<IntegerAttr>(Val: vals[0]); |
| 368 | auto secondAttr = dyn_cast<IntegerAttr>(Val: vals[1]); |
| 369 | auto thirdAttr = dyn_cast<IntegerAttr>(Val: vals[2]); |
| 370 | |
| 371 | if (!tokenAttr || !secondAttr || !thirdAttr) |
| 372 | continue; |
| 373 | |
| 374 | // Produce: {SPIR-V decoration token:"L1 cache control,L3 cache control"} |
| 375 | // The quote char (0x22) is embedded literally; LLVM IR prints it as \22. |
| 376 | std::string entry = |
| 377 | llvm::formatv(Fmt: "{{{0}:\"{1},{2}\"}" , Vals: tokenAttr.getValue().getZExtValue(), |
| 378 | Vals: secondAttr.getValue().getZExtValue(), |
| 379 | Vals: thirdAttr.getValue().getZExtValue()); |
| 380 | |
| 381 | // Deduplicate identical annotations. |
| 382 | if (!seen.insert(KV: {entry, true}).second) |
| 383 | continue; |
| 384 | |
| 385 | payloads.push_back(Elt: std::move(entry)); |
| 386 | } |
| 387 | return payloads; |
| 388 | } |
| 389 | /// Counter for generating unique global variable names. |
| 390 | static std::atomic<uint64_t> globalNameCounter{0}; |
| 391 | |
| 392 | /// Get or create a global metadata string and return a !llvm.ptr<1> value |
| 393 | /// pointing to it. The AddressOfOp is created at the current rewriter |
| 394 | /// insertion point; the GlobalOp is created at the module start. |
| 395 | static Value createMetadataStringPtr(ConversionPatternRewriter &rewriter, |
| 396 | Operation *moduleOp, Location loc, |
| 397 | StringRef value, StringRef nameHint) { |
| 398 | // Build null-terminated string. |
| 399 | std::string strWithNull = value.str(); |
| 400 | strWithNull.push_back(c: '\0'); |
| 401 | StringRef strRef(strWithNull.data(), strWithNull.size()); |
| 402 | |
| 403 | auto as1PtrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: 1); |
| 404 | |
| 405 | // Search for an existing global with the same content. |
| 406 | for (auto &op : moduleOp->getRegion(index: 0).front()) { |
| 407 | if (auto existingGlobal = dyn_cast<LLVM::GlobalOp>(Val: &op)) { |
| 408 | if (!existingGlobal.getSection() || |
| 409 | *existingGlobal.getSection() != "llvm.metadata" ) |
| 410 | continue; |
| 411 | if (auto strAttr = |
| 412 | dyn_cast_or_null<StringAttr>(Val: existingGlobal.getValueOrNull())) { |
| 413 | if (strAttr.getValue() == strRef) { |
| 414 | return LLVM::AddressOfOp::create(builder&: rewriter, location: loc, res: as1PtrTy, |
| 415 | global_name: existingGlobal.getSymName()); |
| 416 | } |
| 417 | } |
| 418 | } |
| 419 | } |
| 420 | |
| 421 | // Create new global at module start. |
| 422 | auto i8Type = rewriter.getI8Type(); |
| 423 | auto arrayType = LLVM::LLVMArrayType::get(elementType: i8Type, numElements: strWithNull.size()); |
| 424 | std::string globalName = |
| 425 | llvm::formatv(Fmt: "{0}.{1}" , Vals&: nameHint, |
| 426 | Vals: globalNameCounter.fetch_add(i: 1, m: std::memory_order_relaxed)) |
| 427 | .str(); |
| 428 | |
| 429 | { |
| 430 | OpBuilder::InsertionGuard guard(rewriter); |
| 431 | rewriter.setInsertionPointToStart(&moduleOp->getRegion(index: 0).front()); |
| 432 | |
| 433 | auto globalOp = |
| 434 | LLVM::GlobalOp::create(builder&: rewriter, location: loc, type: arrayType, |
| 435 | /*isConstant=*/true, linkage: LLVM::Linkage::Private, |
| 436 | name: globalName, value: rewriter.getStringAttr(bytes: strRef)); |
| 437 | globalOp.setSection(StringRef("llvm.metadata" )); |
| 438 | globalOp.setUnnamedAddr(LLVM::UnnamedAddr::Global); |
| 439 | globalOp.setAlignment(1); |
| 440 | globalOp.setAddrSpace(1); |
| 441 | } |
| 442 | // InsertionGuard restores the original insertion point here. |
| 443 | |
| 444 | return LLVM::AddressOfOp::create(builder&: rewriter, location: loc, res: as1PtrTy, global_name: globalName); |
| 445 | } |
| 446 | |
| 447 | /// Annotate a pointer value with cache control metadata by emitting chained |
| 448 | /// `llvm.intr.ptr.annotation` ops (LLVM::PtrAnnotation). |
| 449 | /// |
| 450 | /// This is the MLIR-level equivalent of handleDecorationCacheControl() from |
| 451 | /// the LLVM translation layer. For each cache control attribute, it emits: |
| 452 | /// |
| 453 | /// %ann = llvm.intr.ptr.annotation %ptr, @".str.cachecontrol.N", |
| 454 | /// @".str.file.N", 0, null : !llvm.ptr<AS> |
| 455 | /// |
| 456 | /// Multiple annotations are chained: the result of each annotation op is |
| 457 | /// fed as the pointer input to the next one. |
| 458 | /// |
| 459 | /// \param rewriter The pattern rewriter. |
| 460 | /// \param loc Source location for created ops. |
| 461 | /// \param ptr The pointer value to annotate. |
| 462 | /// \param cacheControls The cache control ArrayAttr (from |
| 463 | /// getCacheControlMetadata). |
| 464 | /// \param moduleOp The enclosing module (for creating globals). |
| 465 | /// \returns The annotated pointer value (or the original ptr if no |
| 466 | /// annotations). |
| 467 | static Value annotatePtrWithCacheControl(ConversionPatternRewriter &rewriter, |
| 468 | Location loc, Value ptr, |
| 469 | ArrayAttr cacheControls, |
| 470 | Operation *moduleOp) { |
| 471 | SmallVector<std::string> payloads = |
| 472 | buildCacheControlPayloads(attrs: cacheControls.getValue()); |
| 473 | if (payloads.empty()) |
| 474 | return ptr; |
| 475 | |
| 476 | auto ptrType = cast<LLVM::LLVMPointerType>(Val: ptr.getType()); |
| 477 | auto as1PtrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: 1); |
| 478 | auto i32Ty = rewriter.getI32Type(); |
| 479 | |
| 480 | // Create shared constants for all annotations on this pointer. |
| 481 | Value fileStr = |
| 482 | createMetadataStringPtr(rewriter, moduleOp, loc, value: "" , nameHint: ".str.file" ); |
| 483 | Value lineVal = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: 0); |
| 484 | Value nullAS1 = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: as1PtrTy); |
| 485 | |
| 486 | // Chain: each annotation takes the result of the previous one as its |
| 487 | // pointer operand. |
| 488 | Value curPtr = ptr; |
| 489 | for (const std::string &payload : payloads) { |
| 490 | Value annStr = createMetadataStringPtr(rewriter, moduleOp, loc, value: payload, |
| 491 | nameHint: ".str.cachecontrol" ); |
| 492 | auto annOp = LLVM::PtrAnnotation::create(builder&: rewriter, location: loc, res: ptrType, ptr: curPtr, |
| 493 | annotation: annStr, fileName: fileStr, line: lineVal, attr: nullAS1); |
| 494 | curPtr = annOp.getResult(); |
| 495 | } |
| 496 | |
| 497 | return curPtr; |
| 498 | } |
| 499 | |
| 500 | /// Helper to apply cache control annotation on a pointer operand of a call. |
| 501 | /// Replaces the pointer argument of the call with an annotated version. |
| 502 | /// |
| 503 | /// For operations that produce a call (like block load/store/prefetch), the |
| 504 | /// pointer is typically the first argument. This function: |
| 505 | /// 1. Builds the annotation chain on the pointer. |
| 506 | /// 2. Replaces the pointer operand in the provided args list. |
| 507 | /// |
| 508 | /// \param rewriter The pattern rewriter. |
| 509 | /// \param loc Source location. |
| 510 | /// \param ptr The original pointer value (first arg to the call). |
| 511 | /// \param cacheControls The cache control metadata. |
| 512 | /// \param moduleOp The enclosing module. |
| 513 | /// \param args The argument list (modified in place: args[ptrIdx] is |
| 514 | /// replaced). |
| 515 | /// \param ptrIdx Index of the pointer in the args list (default 0). |
| 516 | template <typename OpType> |
| 517 | static void |
| 518 | applyCacheControlAnnotation(ConversionPatternRewriter &rewriter, Location loc, |
| 519 | OpType op, SmallVectorImpl<Value> &args, |
| 520 | Operation *moduleOp, unsigned ptrIdx = 0) { |
| 521 | std::optional<ArrayAttr> optCacheControls = |
| 522 | getCacheControlMetadata(rewriter, op); |
| 523 | if (!optCacheControls) |
| 524 | return; |
| 525 | |
| 526 | Value annotatedPtr = annotatePtrWithCacheControl(rewriter, loc, ptr: args[ptrIdx], |
| 527 | cacheControls: *optCacheControls, moduleOp); |
| 528 | args[ptrIdx] = annotatedPtr; |
| 529 | } |
| 530 | |
| 531 | //===----------------------------------------------------------------------===// |
| 532 | // End cache control annotation utilities |
| 533 | //===----------------------------------------------------------------------===// |
| 534 | |
| 535 | static LLVM::CallOp createDeviceFunctionCall( |
| 536 | ConversionPatternRewriter &rewriter, StringRef funcName, Type retType, |
| 537 | ArrayRef<Type> argTypes, ArrayRef<Value> args, |
| 538 | mlir::ArrayRef<std::pair<unsigned, mlir::StringRef>> paramAttrs, |
| 539 | LLVMFuncAttributeOptions funcAttributeOptions, Operation *op) { |
| 540 | auto *moduleOp = op->getParentWithTrait<OpTrait::SymbolTable>(); |
| 541 | assert(moduleOp && "Expecting module" ); |
| 542 | Location loc = op->getLoc(); |
| 543 | |
| 544 | auto funcOpRes = |
| 545 | LLVM::lookupOrCreateFn(b&: rewriter, moduleOp, name: funcName, paramTypes: argTypes, resultType: retType); |
| 546 | assert(!failed(funcOpRes)); |
| 547 | LLVM::LLVMFuncOp funcOp = funcOpRes.value(); |
| 548 | funcOp.setCConv(LLVM::cconv::CConv::SPIR_FUNC); |
| 549 | funcOp.setConvergent(funcAttributeOptions.isConvergent); |
| 550 | funcOp.setNoUnwind(funcAttributeOptions.isNoUnwind); |
| 551 | funcOp.setWillReturn(funcAttributeOptions.isWillReturn); |
| 552 | |
| 553 | if (funcAttributeOptions.memEffectsAttr) |
| 554 | funcOp.setMemoryEffectsAttr(funcAttributeOptions.memEffectsAttr); |
| 555 | |
| 556 | for (auto [idx, attrName] : paramAttrs) |
| 557 | funcOp.setArgAttr(index: idx, name: attrName, value: rewriter.getUnitAttr()); |
| 558 | |
| 559 | auto callOp = LLVM::CallOp::create(builder&: rewriter, location: loc, func: funcOp, args); |
| 560 | SmallVector<NamedAttribute> discardableAttrs; |
| 561 | auto copyAttr = [&](StringAttr name, Attribute attr) { |
| 562 | if (callOp->getInherentAttr(name).has_value()) |
| 563 | callOp->setInherentAttr(name, value: attr); |
| 564 | else |
| 565 | discardableAttrs.emplace_back(Args&: name, Args&: attr); |
| 566 | }; |
| 567 | for (NamedAttribute attr : funcOp->getDiscardableAttrDictionary()) |
| 568 | copyAttr(attr.getName(), attr.getValue()); |
| 569 | funcOp->getName().walkInherentAttrs( |
| 570 | op: funcOp, visitor: [&](StringRef name, Attribute &attr) { |
| 571 | copyAttr(rewriter.getStringAttr(bytes: name), attr); |
| 572 | }); |
| 573 | callOp->setDiscardableAttrs(discardableAttrs); |
| 574 | |
| 575 | return callOp; |
| 576 | } |
| 577 | |
| 578 | static unsigned getNumOperandsPerDword(xevm::ElemType pTy) { |
| 579 | switch (pTy) { |
| 580 | case xevm::ElemType::F32: |
| 581 | case xevm::ElemType::TF32: |
| 582 | return 1; |
| 583 | case xevm::ElemType::BF16: |
| 584 | case xevm::ElemType::F16: |
| 585 | return 2; |
| 586 | case xevm::ElemType::U8: |
| 587 | case xevm::ElemType::S8: |
| 588 | case xevm::ElemType::BF8: |
| 589 | case xevm::ElemType::F8: |
| 590 | return 4; |
| 591 | case xevm::ElemType::E2M1: |
| 592 | case xevm::ElemType::U4: |
| 593 | case xevm::ElemType::S4: |
| 594 | return 8; |
| 595 | default: |
| 596 | llvm_unreachable("unsupported xevm::ElemType" ); |
| 597 | } |
| 598 | } |
| 599 | |
| 600 | class MMAToOCLPattern : public OpConversionPattern<xevm::MMAOp> { |
| 601 | using Base::Base; |
| 602 | LogicalResult |
| 603 | matchAndRewrite(xevm::MMAOp op, xevm::MMAOp::Adaptor adaptor, |
| 604 | ConversionPatternRewriter &rewriter) const override { |
| 605 | if (!op.getC()) { |
| 606 | return rewriter.notifyMatchFailure(arg&: op, msg: "OCL requires C operand" ); |
| 607 | } |
| 608 | auto precisionA = op.getTypes().getA(); |
| 609 | auto precisionB = op.getTypes().getB(); |
| 610 | auto precisionC = op.getTypes().getC(); |
| 611 | auto precisionD = op.getTypes().getD(); |
| 612 | if (precisionC != precisionD) { |
| 613 | return rewriter.notifyMatchFailure(arg&: op, msg: "type of C and D need to match" ); |
| 614 | } |
| 615 | if (precisionC != xevm::ElemType::S32 && |
| 616 | precisionC != xevm::ElemType::F32 && |
| 617 | precisionC != xevm::ElemType::F16 && |
| 618 | precisionC != xevm::ElemType::BF16) { |
| 619 | return rewriter.notifyMatchFailure( |
| 620 | arg&: op, msg: "type of C and D must be S32, F32, F16 or BF16" ); |
| 621 | } |
| 622 | if (precisionA == xevm::ElemType::S32 || |
| 623 | precisionA == xevm::ElemType::F32) { |
| 624 | return rewriter.notifyMatchFailure(arg&: op, msg: "type of A cannot be S32 or F32" ); |
| 625 | } |
| 626 | if (precisionB == xevm::ElemType::S32 || |
| 627 | precisionB == xevm::ElemType::F32) { |
| 628 | return rewriter.notifyMatchFailure(arg&: op, msg: "type of B cannot be S32 or F32" ); |
| 629 | } |
| 630 | constexpr uint32_t bitWidthPackedA{16}; |
| 631 | constexpr uint32_t bitWidthPackedB{32}; |
| 632 | auto loc = op.getLoc(); |
| 633 | |
| 634 | auto castIfNeeded = [&](Value val, Type packedType) -> Value { |
| 635 | VectorType origTy = cast<VectorType>(Val: val.getType()); |
| 636 | const uint32_t vecBitSize = |
| 637 | origTy.getNumElements() * |
| 638 | origTy.getElementType().getIntOrFloatBitWidth(); |
| 639 | VectorType newTy = VectorType::get( |
| 640 | shape: vecBitSize / packedType.getIntOrFloatBitWidth(), elementType: packedType); |
| 641 | if (origTy != newTy) |
| 642 | val = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: newTy, arg: val); |
| 643 | return val; |
| 644 | }; |
| 645 | |
| 646 | Value a = op.getA(); |
| 647 | Type packedAType = (op.getTypes().getA() == xevm::ElemType::TF32) |
| 648 | ? cast<Type>(Val: rewriter.getF32Type()) |
| 649 | : rewriter.getIntegerType(width: bitWidthPackedA); |
| 650 | a = castIfNeeded(a, packedAType); |
| 651 | |
| 652 | Value b = op.getB(); |
| 653 | Type packedBType = (op.getTypes().getB() == xevm::ElemType::TF32) |
| 654 | ? cast<Type>(Val: rewriter.getF32Type()) |
| 655 | : rewriter.getIntegerType(width: bitWidthPackedB); |
| 656 | b = castIfNeeded(b, packedBType); |
| 657 | |
| 658 | Value c = op.getC(); |
| 659 | VectorType cOrigTy = cast<VectorType>(Val: c.getType()); |
| 660 | VectorType resOrigTy = cast<VectorType>(Val: op->getResultTypes()[0]); |
| 661 | assert(cOrigTy == resOrigTy && "Accumulator and result type mismatch" ); |
| 662 | // OCL builtins encode bfloat16 as int16 |
| 663 | VectorType cTy = |
| 664 | cOrigTy.getElementType().isBF16() |
| 665 | ? VectorType::get(shape: cOrigTy.getShape(), elementType: rewriter.getIntegerType(width: 16)) |
| 666 | : cOrigTy; |
| 667 | VectorType resTy = cTy; |
| 668 | if (cOrigTy != cTy) |
| 669 | c = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: cTy, arg: c); |
| 670 | |
| 671 | constexpr int32_t systolicDepth{8}; |
| 672 | std::string fnName = |
| 673 | llvm::formatv(Fmt: "intel_sub_group_{0}_{1}_matrix_mad_k{2}" , |
| 674 | Vals: stringifyElemType(op.getTypes().getA()).str(), |
| 675 | Vals: stringifyElemType(op.getTypes().getB()).str(), |
| 676 | Vals: systolicDepth * |
| 677 | getNumOperandsPerDword(pTy: op.getTypes().getA())) |
| 678 | .str(); |
| 679 | SmallVector<Type> argTypes{a.getType(), b.getType(), cTy}; |
| 680 | fnName = mangle(baseName: fnName, types: argTypes); |
| 681 | SmallVector<Value> args{a, b, c}; |
| 682 | |
| 683 | auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>( |
| 684 | /*other=*/args: LLVM::ModRefInfo::NoModRef, |
| 685 | /*argMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 686 | /*inaccessibleMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 687 | /*errnoMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 688 | /*targetMem0=*/args: LLVM::ModRefInfo::NoModRef, |
| 689 | /*targetMem1=*/args: LLVM::ModRefInfo::NoModRef); |
| 690 | auto funcAttrs = convergentNoUnwindWillReturnAttrs; |
| 691 | funcAttrs.memEffectsAttr = memAttr; |
| 692 | Value result = |
| 693 | createDeviceFunctionCall(rewriter, funcName: fnName, retType: resTy, argTypes, args, paramAttrs: {}, |
| 694 | funcAttributeOptions: funcAttrs, op: op.getOperation()) |
| 695 | ->getResult(idx: 0); |
| 696 | |
| 697 | if (resOrigTy != resTy) |
| 698 | result = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: resOrigTy, arg: result); |
| 699 | |
| 700 | rewriter.replaceOp(op, newValues: result); |
| 701 | return success(); |
| 702 | } |
| 703 | }; |
| 704 | |
| 705 | class PrefetchToOCLPattern : public OpConversionPattern<PrefetchOp> { |
| 706 | using Base::Base; |
| 707 | LogicalResult |
| 708 | matchAndRewrite(PrefetchOp op, PrefetchOp::Adaptor adaptor, |
| 709 | ConversionPatternRewriter &rewriter) const override { |
| 710 | auto loc = op.getLoc(); |
| 711 | auto *moduleOp = op->getParentWithTrait<OpTrait::SymbolTable>(); |
| 712 | |
| 713 | const std::string fnName{"_Z8prefetchPU3AS1Kcm" }; |
| 714 | Value one = |
| 715 | LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI64Type(), value: 1); |
| 716 | SmallVector<Value> args{op.getPtr(), one}; |
| 717 | |
| 718 | // Annotate pointer with cache control before passing to the call. |
| 719 | applyCacheControlAnnotation(rewriter, loc, op, args, moduleOp, |
| 720 | /*ptrIdx=*/0); |
| 721 | |
| 722 | SmallVector<Type> argTypes; |
| 723 | for (auto arg : args) |
| 724 | argTypes.push_back(Elt: arg.getType()); |
| 725 | auto funcAttr = noUnwindAttrs; |
| 726 | auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>( |
| 727 | /*other=*/args: LLVM::ModRefInfo::NoModRef, |
| 728 | /*argMem=*/args: LLVM::ModRefInfo::Ref, |
| 729 | /*inaccessibleMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 730 | /*errnoMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 731 | /*targetMem0=*/args: LLVM::ModRefInfo::NoModRef, |
| 732 | /*targetMem1=*/args: LLVM::ModRefInfo::NoModRef); |
| 733 | funcAttr.memEffectsAttr = memAttr; |
| 734 | |
| 735 | createDeviceFunctionCall(rewriter, funcName: fnName, |
| 736 | retType: LLVM::LLVMVoidType::get(ctx: rewriter.getContext()), |
| 737 | argTypes, args, paramAttrs: {}, funcAttributeOptions: funcAttr, op: op.getOperation()); |
| 738 | rewriter.eraseOp(op); |
| 739 | return success(); |
| 740 | } |
| 741 | }; |
| 742 | |
| 743 | class MemfenceToOCLPattern : public OpConversionPattern<MemfenceOp> { |
| 744 | using Base::Base; |
| 745 | LogicalResult |
| 746 | matchAndRewrite(MemfenceOp op, MemfenceOp::Adaptor adaptor, |
| 747 | ConversionPatternRewriter &rewriter) const override { |
| 748 | auto loc = op.getLoc(); |
| 749 | const std::string fnName{"atomic_work_item_fence" }; |
| 750 | int memScope, addrSpace; |
| 751 | switch (op.getAddrspace()) { |
| 752 | case xevm::AddrSpace::SHARED: |
| 753 | addrSpace = 1; // CLK_LOCAL_MEM_FENCE |
| 754 | break; |
| 755 | case xevm::AddrSpace::GLOBAL: |
| 756 | addrSpace = 2; // CLK_GLOBAL_MEM_FENCE |
| 757 | break; |
| 758 | default: |
| 759 | // GENERIC is not supported in OpenCL |
| 760 | return rewriter.notifyMatchFailure( |
| 761 | arg&: op, msg: "Fence only supports global and shared address spaces." ); |
| 762 | } |
| 763 | switch (op.getScope()) { |
| 764 | case xevm::MemScope::WORKGROUP: |
| 765 | memScope = 1; |
| 766 | break; |
| 767 | case xevm::MemScope::DEVICE: |
| 768 | memScope = 2; |
| 769 | break; |
| 770 | default: |
| 771 | // CLUSTER and SYSTEM are not supported in OpenCL |
| 772 | return rewriter.notifyMatchFailure( |
| 773 | arg&: op, msg: "Fence only supports workgroup and device memory scopes." ); |
| 774 | } |
| 775 | Type i32Type = rewriter.getI32Type(); |
| 776 | Value acqRel = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Type, value: 4); |
| 777 | Value memScopeConst = |
| 778 | LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Type, value: memScope); |
| 779 | Value addrSpaceConst = |
| 780 | LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Type, value: addrSpace); |
| 781 | SmallVector<Value> args{addrSpaceConst, acqRel, memScopeConst}; |
| 782 | SmallVector<Type> argTypes{3, i32Type}; |
| 783 | createDeviceFunctionCall(rewriter, funcName: mangle(baseName: fnName, types: argTypes), |
| 784 | retType: LLVM::LLVMVoidType::get(ctx: rewriter.getContext()), |
| 785 | argTypes, args, paramAttrs: {}, funcAttributeOptions: noUnwindAttrs, |
| 786 | op: op.getOperation()); |
| 787 | rewriter.eraseOp(op); |
| 788 | return success(); |
| 789 | } |
| 790 | }; |
| 791 | template <typename OpType> |
| 792 | class LoadStorePrefetchToOCLPattern : public OpConversionPattern<OpType> { |
| 793 | using OpConversionPattern<OpType>::OpConversionPattern; |
| 794 | LogicalResult |
| 795 | matchAndRewrite(OpType op, typename OpType::Adaptor adaptor, |
| 796 | ConversionPatternRewriter &rewriter) const override { |
| 797 | constexpr bool isLoad = std::is_same_v<OpType, BlockLoad2dOp>; |
| 798 | constexpr bool isPrefetch = std::is_same_v<OpType, BlockPrefetch2dOp>; |
| 799 | |
| 800 | auto loc = op.getLoc(); |
| 801 | auto *moduleOp = op->template getParentWithTrait<OpTrait::SymbolTable>(); |
| 802 | VectorType vecType; |
| 803 | bool packReg = false; |
| 804 | bool transpose = false; |
| 805 | if constexpr (isLoad) { |
| 806 | vecType = op.getRes().getType(); |
| 807 | packReg = op.getPackRegister(); |
| 808 | transpose = op.getTranspose(); |
| 809 | } else if constexpr (!isPrefetch) { |
| 810 | vecType = op.getStoredVal().getType(); |
| 811 | } |
| 812 | |
| 813 | auto i32Type = rewriter.getI32Type(); |
| 814 | Value byteCoord = |
| 815 | LLVM::UndefOp::create(rewriter, loc, VectorType::get(shape: 2, elementType: i32Type)); |
| 816 | Value zero = LLVM::ConstantOp::create(rewriter, loc, i32Type, 0); |
| 817 | Value one = LLVM::ConstantOp::create(rewriter, loc, i32Type, 1); |
| 818 | byteCoord = LLVM::InsertElementOp::create( |
| 819 | rewriter, loc, VectorType::get(shape: 2, elementType: i32Type), byteCoord, op.getX(), zero); |
| 820 | byteCoord = LLVM::InsertElementOp::create( |
| 821 | rewriter, loc, VectorType::get(shape: 2, elementType: i32Type), byteCoord, op.getY(), one); |
| 822 | SmallVector<Value> args{op.getPtr(), op.getBaseWidth(), op.getBaseHeight(), |
| 823 | op.getBasePitch(), byteCoord}; |
| 824 | |
| 825 | // Annotate pointer (args[0]) with cache control before the call. |
| 826 | applyCacheControlAnnotation(rewriter, loc, op, args, moduleOp, |
| 827 | /*ptrIdx=*/0); |
| 828 | |
| 829 | SmallVector<Type> retTypes; |
| 830 | Value spvLoadDstPtr; |
| 831 | std::string funcName{"intel_sub_group_2d_block_" }; |
| 832 | std::string bitWidthId; |
| 833 | LLVMFuncAttributeOptions funcAttr{noUnwindWillReturnAttrs}; |
| 834 | SmallVector<std::pair<unsigned, StringRef>, 4> paramAttrs; |
| 835 | if constexpr (isPrefetch) { // Prefetch |
| 836 | funcName += "prefetch" ; |
| 837 | paramAttrs = {std::make_pair(x: 0, y: LLVM::LLVMDialect::getNonNullAttrName())}; |
| 838 | auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>( |
| 839 | /*other=*/args: LLVM::ModRefInfo::NoModRef, |
| 840 | /*argMem=*/args: LLVM::ModRefInfo::Ref, |
| 841 | /*inaccessibleMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 842 | /*errnoMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 843 | /*targetMem0=*/args: LLVM::ModRefInfo::NoModRef, |
| 844 | /*targetMem1=*/args: LLVM::ModRefInfo::NoModRef); |
| 845 | funcAttr = noUnwindAttrs; |
| 846 | funcAttr.memEffectsAttr = memAttr; |
| 847 | } else { |
| 848 | auto vecElemType = vecType.getElementType(); |
| 849 | auto vecElemBitWidth = vecElemType.getIntOrFloatBitWidth(); |
| 850 | auto vecNumElems = vecType.getNumElements(); |
| 851 | // OpenCL Intel 2D block load has a special case |
| 852 | // when element bit size is 8 and tile width is 32, which is twice |
| 853 | // the subgroup size, loaded element is packed as i16. |
| 854 | // To reflect this, element bit size is updated to 16 and |
| 855 | // vector length is reduced by half. |
| 856 | if (op.getElemSizeInBits() == 8 && op.getTileWidth() == 32) { |
| 857 | vecElemBitWidth = 16; |
| 858 | vecElemType = rewriter.getI16Type(); |
| 859 | vecNumElems = vecNumElems / 2; |
| 860 | } |
| 861 | Value numElems = |
| 862 | LLVM::ConstantOp::create(rewriter, loc, i32Type, vecNumElems); |
| 863 | auto dstOrSrcPtr = LLVM::AllocaOp::create( |
| 864 | rewriter, loc, LLVM::LLVMPointerType::get(context: rewriter.getContext()), |
| 865 | vecElemType, numElems); |
| 866 | args.push_back(Elt: dstOrSrcPtr); |
| 867 | if constexpr (isLoad) { // Load |
| 868 | funcName += "read" ; |
| 869 | bitWidthId = getTypeMangling(ty: vecElemType, /*isUnsigned=*/true); |
| 870 | if (packReg) |
| 871 | funcName += "_transform" ; |
| 872 | else if (transpose) |
| 873 | funcName += "_transpose" ; |
| 874 | spvLoadDstPtr = dstOrSrcPtr; |
| 875 | retTypes.push_back(Elt: vecType); |
| 876 | paramAttrs = { |
| 877 | std::make_pair(x: 0, y: LLVM::LLVMDialect::getNonNullAttrName()), |
| 878 | std::make_pair(x: 0, y: LLVM::LLVMDialect::getReadonlyAttrName()), |
| 879 | std::make_pair(x: 5, y: LLVM::LLVMDialect::getNonNullAttrName()), |
| 880 | std::make_pair(x: 5, y: LLVM::LLVMDialect::getWriteOnlyAttrName()), |
| 881 | }; |
| 882 | } else { // Store |
| 883 | funcName += "write" ; |
| 884 | bitWidthId = (vecElemBitWidth == 32) |
| 885 | ? "j" |
| 886 | : ((vecElemBitWidth == 16) ? "t" : "h" ); |
| 887 | LLVM::StoreOp::create(rewriter, loc, op.getStoredVal(), dstOrSrcPtr); |
| 888 | paramAttrs = { |
| 889 | std::make_pair(x: 0, y: LLVM::LLVMDialect::getNonNullAttrName()), |
| 890 | std::make_pair(x: 0, y: LLVM::LLVMDialect::getWriteOnlyAttrName()), |
| 891 | std::make_pair(x: 5, y: LLVM::LLVMDialect::getNonNullAttrName()), |
| 892 | std::make_pair(x: 5, y: LLVM::LLVMDialect::getReadonlyAttrName()), |
| 893 | }; |
| 894 | } |
| 895 | } |
| 896 | |
| 897 | funcName = |
| 898 | llvm::formatv("{0}_{1}b_{2}r{3}x{4}c" , funcName, op.getElemSizeInBits(), |
| 899 | op.getTileHeight(), op.getTileWidth(), op.getVBlocks()) |
| 900 | .str(); |
| 901 | std::string prefetchCode("" ); |
| 902 | if (!isPrefetch) |
| 903 | prefetchCode += "P" ; |
| 904 | funcName = llvm::formatv(Fmt: "_Z{0}{1}PU3AS1viiiDv2_i{2}{3}" , Vals: funcName.size(), |
| 905 | Vals&: funcName, Vals&: prefetchCode, Vals&: bitWidthId) |
| 906 | .str(); |
| 907 | SmallVector<Type> argTypes; |
| 908 | for (auto arg : args) { |
| 909 | argTypes.push_back(Elt: arg.getType()); |
| 910 | } |
| 911 | createDeviceFunctionCall( |
| 912 | rewriter, funcName, LLVM::LLVMVoidType::get(ctx: rewriter.getContext()), |
| 913 | argTypes, args, paramAttrs, funcAttr, op.getOperation()); |
| 914 | |
| 915 | if constexpr (isLoad) |
| 916 | rewriter.replaceOp( |
| 917 | op, LLVM::LoadOp::create(rewriter, loc, vecType, spvLoadDstPtr)); |
| 918 | else |
| 919 | rewriter.eraseOp(op); |
| 920 | return success(); |
| 921 | } |
| 922 | }; |
| 923 | |
| 924 | template <typename OpType> |
| 925 | class BlockLoadStore1DToOCLPattern : public OpConversionPattern<OpType> { |
| 926 | using OpConversionPattern<OpType>::OpConversionPattern; |
| 927 | LogicalResult |
| 928 | matchAndRewrite(OpType op, typename OpType::Adaptor adaptor, |
| 929 | ConversionPatternRewriter &rewriter) const override { |
| 930 | constexpr bool isStore = std::is_same_v<OpType, xevm::BlockStoreOp>; |
| 931 | auto loc = op.getLoc(); |
| 932 | auto *moduleOp = op->template getParentWithTrait<OpTrait::SymbolTable>(); |
| 933 | |
| 934 | // Get OpenCL function name |
| 935 | // https://registry.khronos.org/OpenCL/extensions/ |
| 936 | // intel/cl_intel_subgroup_local_block_io.html |
| 937 | std::string funcName{"intel_sub_group_block_" }; |
| 938 | // Value or Result type can be vector or scalar |
| 939 | Type valOrResTy; |
| 940 | if constexpr (isStore) { |
| 941 | funcName += "write_u" ; |
| 942 | valOrResTy = op.getVal().getType(); |
| 943 | } else { |
| 944 | funcName += "read_u" ; |
| 945 | valOrResTy = op.getType(); |
| 946 | } |
| 947 | // Get element type of the vector/scalar |
| 948 | VectorType vecTy = dyn_cast<VectorType>(Val&: valOrResTy); |
| 949 | Type elemType = vecTy ? vecTy.getElementType() : valOrResTy; |
| 950 | funcName += getTypeMangling(ty: elemType); |
| 951 | if (vecTy) |
| 952 | funcName += std::to_string(val: vecTy.getNumElements()); |
| 953 | SmallVector<Type, 2> argTypes{}; |
| 954 | // XeVM BlockLoad/StoreOp always use signless integer types |
| 955 | // but OpenCL builtins expect unsigned types |
| 956 | // use unsigned types for mangling |
| 957 | SmallVector<bool, 2> isUnsigned{}; |
| 958 | // arg0: pointer to the src/dst address |
| 959 | // arg1 - only if store : vector to store |
| 960 | // Prepare arguments |
| 961 | SmallVector<Value, 2> args{}; |
| 962 | args.push_back(Elt: op.getPtr()); |
| 963 | argTypes.push_back(Elt: op.getPtr().getType()); |
| 964 | isUnsigned.push_back(Elt: true); |
| 965 | |
| 966 | // Annotate pointer (args[0]) with cache control. |
| 967 | applyCacheControlAnnotation(rewriter, loc, op, args, moduleOp, |
| 968 | /*ptrIdx=*/0); |
| 969 | // Update argTypes[0] in case the pointer type changed (it shouldn't |
| 970 | // change type, but the value is now the annotated pointer). |
| 971 | argTypes[0] = args[0].getType(); |
| 972 | |
| 973 | Type retType; |
| 974 | if constexpr (isStore) { |
| 975 | args.push_back(Elt: op.getVal()); |
| 976 | argTypes.push_back(Elt: op.getVal().getType()); |
| 977 | isUnsigned.push_back(Elt: true); |
| 978 | retType = LLVM::LLVMVoidType::get(ctx: rewriter.getContext()); |
| 979 | } else { |
| 980 | retType = valOrResTy; |
| 981 | } |
| 982 | funcName = std::string("_Z" ) + std::to_string(val: funcName.size()) + funcName + |
| 983 | "PU3AS" + |
| 984 | std::to_string(op.getPtr().getType().getAddressSpace()); |
| 985 | funcName += getTypeMangling(ty: elemType, /*isUnsigned=*/true); |
| 986 | if constexpr (isStore) |
| 987 | funcName += getTypeMangling(ty: valOrResTy, /*isUnsigned=*/true); |
| 988 | LLVMFuncAttributeOptions funcAttr{noUnwindWillReturnAttrs}; |
| 989 | |
| 990 | LLVM::CallOp call = |
| 991 | createDeviceFunctionCall(rewriter, funcName, retType, argTypes, args, |
| 992 | {}, funcAttr, op.getOperation()); |
| 993 | |
| 994 | if constexpr (isStore) |
| 995 | rewriter.eraseOp(op); |
| 996 | else |
| 997 | rewriter.replaceOp(op, call->getResult(idx: 0)); |
| 998 | return success(); |
| 999 | } |
| 1000 | }; |
| 1001 | |
| 1002 | template <typename OpType> |
| 1003 | class LLVMLoadStoreToOCLPattern : public OpConversionPattern<OpType> { |
| 1004 | using OpConversionPattern<OpType>::OpConversionPattern; |
| 1005 | LogicalResult |
| 1006 | matchAndRewrite(OpType op, typename OpType::Adaptor adaptor, |
| 1007 | ConversionPatternRewriter &rewriter) const override { |
| 1008 | if (!op->hasDiscardableAttr("cache_control" )) |
| 1009 | return failure(); |
| 1010 | |
| 1011 | auto *moduleOp = op->template getParentWithTrait<OpTrait::SymbolTable>(); |
| 1012 | std::optional<ArrayAttr> optCacheControls = |
| 1013 | getCacheControlMetadata(rewriter, op); |
| 1014 | if (!optCacheControls) { |
| 1015 | rewriter.modifyOpInPlace( |
| 1016 | op, [&]() { op->removeDiscardableAttr("cache_control" ); }); |
| 1017 | return success(); |
| 1018 | } |
| 1019 | |
| 1020 | // Determine which operand is the pointer. |
| 1021 | constexpr bool isStore = std::is_same_v<OpType, LLVM::StoreOp>; |
| 1022 | unsigned ptrIdx = isStore ? 1 : 0; |
| 1023 | Value ptr = op->getOperand(ptrIdx); |
| 1024 | |
| 1025 | // Emit annotation intrinsic calls on the pointer. |
| 1026 | Value annotatedPtr = annotatePtrWithCacheControl( |
| 1027 | rewriter, op->getLoc(), ptr, *optCacheControls, moduleOp); |
| 1028 | |
| 1029 | // Replace the pointer operand with the annotated one. |
| 1030 | rewriter.modifyOpInPlace(op, [&]() { |
| 1031 | op->setOperand(ptrIdx, annotatedPtr); |
| 1032 | op->removeDiscardableAttr("cache_control" ); |
| 1033 | }); |
| 1034 | return success(); |
| 1035 | } |
| 1036 | }; |
| 1037 | |
| 1038 | //===----------------------------------------------------------------------===// |
| 1039 | // GPU index id operations |
| 1040 | //===----------------------------------------------------------------------===// |
| 1041 | /* |
| 1042 | // Launch Config ops |
| 1043 | // dimidx - x, y, z - is fixed to i32 |
| 1044 | // return type is set by XeVM type converter |
| 1045 | // get_local_id |
| 1046 | xevm::WorkitemIdXOp; |
| 1047 | xevm::WorkitemIdYOp; |
| 1048 | xevm::WorkitemIdZOp; |
| 1049 | // get_local_size |
| 1050 | xevm::WorkgroupDimXOp; |
| 1051 | xevm::WorkgroupDimYOp; |
| 1052 | xevm::WorkgroupDimZOp; |
| 1053 | // get_group_id |
| 1054 | xevm::WorkgroupIdXOp; |
| 1055 | xevm::WorkgroupIdYOp; |
| 1056 | xevm::WorkgroupIdZOp; |
| 1057 | // get_num_groups |
| 1058 | xevm::GridDimXOp; |
| 1059 | xevm::GridDimYOp; |
| 1060 | xevm::GridDimZOp; |
| 1061 | // get_global_id : to be added if needed |
| 1062 | */ |
| 1063 | |
| 1064 | // Helpers to get the OpenCL function name and dimension argument for each op. |
| 1065 | static std::pair<StringRef, int64_t> getConfig(xevm::WorkitemIdXOp) { |
| 1066 | return {"get_local_id" , 0}; |
| 1067 | } |
| 1068 | static std::pair<StringRef, int64_t> getConfig(xevm::WorkitemIdYOp) { |
| 1069 | return {"get_local_id" , 1}; |
| 1070 | } |
| 1071 | static std::pair<StringRef, int64_t> getConfig(xevm::WorkitemIdZOp) { |
| 1072 | return {"get_local_id" , 2}; |
| 1073 | } |
| 1074 | static std::pair<StringRef, int64_t> getConfig(xevm::WorkgroupDimXOp) { |
| 1075 | return {"get_local_size" , 0}; |
| 1076 | } |
| 1077 | static std::pair<StringRef, int64_t> getConfig(xevm::WorkgroupDimYOp) { |
| 1078 | return {"get_local_size" , 1}; |
| 1079 | } |
| 1080 | static std::pair<StringRef, int64_t> getConfig(xevm::WorkgroupDimZOp) { |
| 1081 | return {"get_local_size" , 2}; |
| 1082 | } |
| 1083 | static std::pair<StringRef, int64_t> getConfig(xevm::WorkgroupIdXOp) { |
| 1084 | return {"get_group_id" , 0}; |
| 1085 | } |
| 1086 | static std::pair<StringRef, int64_t> getConfig(xevm::WorkgroupIdYOp) { |
| 1087 | return {"get_group_id" , 1}; |
| 1088 | } |
| 1089 | static std::pair<StringRef, int64_t> getConfig(xevm::WorkgroupIdZOp) { |
| 1090 | return {"get_group_id" , 2}; |
| 1091 | } |
| 1092 | static std::pair<StringRef, int64_t> getConfig(xevm::GridDimXOp) { |
| 1093 | return {"get_num_groups" , 0}; |
| 1094 | } |
| 1095 | static std::pair<StringRef, int64_t> getConfig(xevm::GridDimYOp) { |
| 1096 | return {"get_num_groups" , 1}; |
| 1097 | } |
| 1098 | static std::pair<StringRef, int64_t> getConfig(xevm::GridDimZOp) { |
| 1099 | return {"get_num_groups" , 2}; |
| 1100 | } |
| 1101 | /// Replace `xevm.*` with an `llvm.call` to the corresponding OpenCL func with |
| 1102 | /// a constant argument for the dimension - x, y or z. |
| 1103 | template <typename OpType> |
| 1104 | class LaunchConfigOpToOCLPattern : public OpConversionPattern<OpType> { |
| 1105 | using OpConversionPattern<OpType>::OpConversionPattern; |
| 1106 | LogicalResult |
| 1107 | matchAndRewrite(OpType op, typename OpType::Adaptor adaptor, |
| 1108 | ConversionPatternRewriter &rewriter) const override { |
| 1109 | Location loc = op->getLoc(); |
| 1110 | auto [baseName, dim] = getConfig(op); |
| 1111 | Type dimTy = rewriter.getI32Type(); |
| 1112 | Value dimVal = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: dimTy, |
| 1113 | value: static_cast<int64_t>(dim)); |
| 1114 | std::string func = mangle(baseName, {dimTy}, {true}); |
| 1115 | Type resTy = op.getType(); |
| 1116 | auto call = |
| 1117 | createDeviceFunctionCall(rewriter, func, resTy, {dimTy}, {dimVal}, {}, |
| 1118 | noUnwindWillReturnAttrs, op.getOperation()); |
| 1119 | constexpr auto noModRef = LLVM::ModRefInfo::NoModRef; |
| 1120 | auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>( |
| 1121 | /*other=*/args: noModRef, |
| 1122 | /*argMem=*/args: noModRef, /*inaccessibleMem=*/args: noModRef, |
| 1123 | /*errnoMem=*/args: noModRef, |
| 1124 | /*targetMem0=*/args: noModRef, |
| 1125 | /*targetMem1=*/args: noModRef); |
| 1126 | call.setMemoryEffectsAttr(memAttr); |
| 1127 | rewriter.replaceOp(op, call); |
| 1128 | return success(); |
| 1129 | } |
| 1130 | }; |
| 1131 | |
| 1132 | /* |
| 1133 | // Subgroup ops |
| 1134 | // get_sub_group_local_id |
| 1135 | xevm::LaneIdOp; |
| 1136 | // get_sub_group_id |
| 1137 | xevm::SubgroupIdOp; |
| 1138 | // get_sub_group_size |
| 1139 | xevm::SubgroupSizeOp; |
| 1140 | // get_num_sub_groups : to be added if needed |
| 1141 | */ |
| 1142 | |
| 1143 | // Helpers to get the OpenCL function name for each op. |
| 1144 | static StringRef getConfig(xevm::LaneIdOp) { return "get_sub_group_local_id" ; } |
| 1145 | static StringRef getConfig(xevm::SubgroupIdOp) { return "get_sub_group_id" ; } |
| 1146 | static StringRef getConfig(xevm::SubgroupSizeOp) { |
| 1147 | return "get_sub_group_size" ; |
| 1148 | } |
| 1149 | template <typename OpType> |
| 1150 | class SubgroupOpWorkitemOpToOCLPattern : public OpConversionPattern<OpType> { |
| 1151 | using OpConversionPattern<OpType>::OpConversionPattern; |
| 1152 | LogicalResult |
| 1153 | matchAndRewrite(OpType op, typename OpType::Adaptor adaptor, |
| 1154 | ConversionPatternRewriter &rewriter) const override { |
| 1155 | std::string func = mangle(getConfig(op).str(), {}); |
| 1156 | Type resTy = op.getType(); |
| 1157 | auto call = |
| 1158 | createDeviceFunctionCall(rewriter, func, resTy, {}, {}, {}, |
| 1159 | noUnwindWillReturnAttrs, op.getOperation()); |
| 1160 | constexpr auto noModRef = LLVM::ModRefInfo::NoModRef; |
| 1161 | auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>( |
| 1162 | /*other=*/args: noModRef, |
| 1163 | /*argMem=*/args: noModRef, /*inaccessibleMem=*/args: noModRef, |
| 1164 | /*errnoMem=*/args: noModRef, |
| 1165 | /*targetMem0=*/args: noModRef, |
| 1166 | /*targetMem1=*/args: noModRef); |
| 1167 | call.setMemoryEffectsAttr(memAttr); |
| 1168 | rewriter.replaceOp(op, call); |
| 1169 | return success(); |
| 1170 | } |
| 1171 | }; |
| 1172 | |
| 1173 | /// SPIR-V, and so the OpenCL builtins the float conversions call into, only |
| 1174 | /// provides vector types of 2, 3, 4, 8 and 16 elements. |
| 1175 | static bool isSupportedSPIRVVectorLength(int64_t numElements) { |
| 1176 | return llvm::is_contained(Set: {2, 3, 4, 8, 16}, Element: numElements); |
| 1177 | } |
| 1178 | |
| 1179 | /// Bitcasts `val` to `ty` unless it already has that type. |
| 1180 | static Value castIfNeeded(ConversionPatternRewriter &rewriter, Location loc, |
| 1181 | Type ty, Value val) { |
| 1182 | if (val.getType() == ty) |
| 1183 | return val; |
| 1184 | return LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: ty, arg: val); |
| 1185 | } |
| 1186 | |
| 1187 | /// Selects `numElements` leading elements of the vector `val`. Used to drop the |
| 1188 | /// padding the 3 element case needs, as the hardware conversions work on whole |
| 1189 | /// pairs of elements. |
| 1190 | static Value takeLeadingElements(ConversionPatternRewriter &rewriter, |
| 1191 | Location loc, Value val, int64_t numElements) { |
| 1192 | auto vecTy = cast<VectorType>(Val: val.getType()); |
| 1193 | if (vecTy.getNumElements() == numElements) |
| 1194 | return val; |
| 1195 | SmallVector<int32_t> mask = |
| 1196 | llvm::to_vector(Range: llvm::seq<int32_t>(Begin: 0, End: static_cast<int32_t>(numElements))); |
| 1197 | return LLVM::ShuffleVectorOp::create(builder&: rewriter, location: loc, v1: val, v2: val, mask); |
| 1198 | } |
| 1199 | |
| 1200 | // |
| 1201 | // Note: TruncfToOCLPattern and ExtfToOCLPattern does not lower to OpenCL API |
| 1202 | // calls as there are not official ones yet. They are lowered directly to Intel |
| 1203 | // graphics compiler built in functions. |
| 1204 | // See |
| 1205 | // https://github.com/intel/intel-graphics-compiler/tree/master/IGC/BiFModule/Implementation/SPV_INTEL_fp_conversions |
| 1206 | // for builtin function usage. The folder contains implementation of |
| 1207 | // experimental SPIR-V extension for truncf and extf using builtin functions. |
| 1208 | // TODO: Move to OpenCL API call once they are available. |
| 1209 | // |
| 1210 | |
| 1211 | class TruncfToOCLPattern : public OpConversionPattern<TruncfOp> { |
| 1212 | using Base::Base; |
| 1213 | LogicalResult |
| 1214 | matchAndRewrite(TruncfOp op, TruncfOp::Adaptor adaptor, |
| 1215 | ConversionPatternRewriter &rewriter) const override { |
| 1216 | // Supported source and result types are resticted for now. |
| 1217 | auto srcEtype = op.getSrcEtype().getEtype(); |
| 1218 | auto dstEtype = op.getDstEtype().getEtype(); |
| 1219 | // The conversions are provided as OpenCL builtins, one per vector length, |
| 1220 | // so only the SPIR-V vector lengths can be lowered. A wider conversion has |
| 1221 | // to be split into several ops before reaching this pattern. |
| 1222 | // |
| 1223 | // Scalar case is not supported until usage case become clear. |
| 1224 | auto vecSrcTy = dyn_cast<VectorType>(Val: op.getSrc().getType()); |
| 1225 | if (!vecSrcTy) { |
| 1226 | return rewriter.notifyMatchFailure(arg&: op, msg: "Scalar src is not supported." ); |
| 1227 | } |
| 1228 | int64_t numElements = vecSrcTy.getNumElements(); |
| 1229 | if (!isSupportedSPIRVVectorLength(numElements)) |
| 1230 | return rewriter.notifyMatchFailure( |
| 1231 | arg&: op, msg: "src vector length must be 2, 3, 4, 8 or 16" ); |
| 1232 | // The destination is scalar only where the packed values fit in one byte, |
| 1233 | // which SPIR-V spells as a scalar rather than a one element vector. |
| 1234 | Type dstTy = op.getDst().getType(); |
| 1235 | Location loc = op.getLoc(); |
| 1236 | Value src = op.getSrc(); |
| 1237 | auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>( |
| 1238 | /*other=*/args: LLVM::ModRefInfo::NoModRef, |
| 1239 | /*argMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 1240 | /*inaccessibleMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 1241 | /*errnoMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 1242 | /*targetMem0=*/args: LLVM::ModRefInfo::NoModRef, |
| 1243 | /*targetMem1=*/args: LLVM::ModRefInfo::NoModRef); |
| 1244 | auto funcAttrs = convergentNoUnwindWillReturnAttrs; |
| 1245 | funcAttrs.memEffectsAttr = memAttr; |
| 1246 | |
| 1247 | // Handle the case where dst type is fp4 first. |
| 1248 | if (dstEtype == TruncfDstElemTypes::E2M1) { |
| 1249 | // `__builtin_IB_dnscl_{hf16,bf16}(uint a, uint b, convert_to, mode)` |
| 1250 | // takes two dwords, each holding two source elements, and packs each pair |
| 1251 | // into one byte of the result dword. `mode` picks which bytes of that |
| 1252 | // dword are written: mode 0 writes bytes 0 and 2, mode 2 writes bytes 1 |
| 1253 | // and 3. Two calls with complementary modes therefore OR together into |
| 1254 | // one fully packed dword covering eight source elements. |
| 1255 | // |
| 1256 | // A pair of elements is the conversion granularity, so an odd length is |
| 1257 | // padded up and the spare nibble left undefined. |
| 1258 | constexpr int kDnsclConvertToE2M1 = 1; |
| 1259 | constexpr int kDnsclModeBytes02 = 0; |
| 1260 | constexpr int kDnsclModeBytes13 = 2; |
| 1261 | // One dword lane per element pair, and one result byte per lane. The op |
| 1262 | // verifier has already checked that the destination is exactly that wide. |
| 1263 | int64_t numLanes = llvm::divideCeil(Numerator: numElements, Denominator: 2); |
| 1264 | |
| 1265 | Type i32Ty = rewriter.getI32Type(); |
| 1266 | Type i8Ty = rewriter.getI8Type(); |
| 1267 | // Pad an odd length up to a whole number of pairs, then view the source |
| 1268 | // as dword lanes. |
| 1269 | Value padded = src; |
| 1270 | if (numElements != numLanes * 2) { |
| 1271 | SmallVector<int32_t> mask = llvm::to_vector( |
| 1272 | Range: llvm::seq<int32_t>(Begin: 0, End: static_cast<int32_t>(numElements))); |
| 1273 | // The padding element is never read back, so any valid index will do. |
| 1274 | mask.append(NumInputs: static_cast<size_t>(numLanes * 2 - numElements), Elt: 0); |
| 1275 | padded = LLVM::ShuffleVectorOp::create(builder&: rewriter, location: loc, v1: src, v2: src, mask); |
| 1276 | } |
| 1277 | // A single lane is passed as a bare i32 rather than a one element vector, |
| 1278 | // which SPIR-V has no type for. |
| 1279 | Value laneVec; |
| 1280 | if (numLanes > 1) |
| 1281 | laneVec = LLVM::BitcastOp::create( |
| 1282 | builder&: rewriter, location: loc, res: VectorType::get(shape: numLanes, elementType: i32Ty), arg: padded); |
| 1283 | else |
| 1284 | laneVec = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: i32Ty, arg: padded); |
| 1285 | auto getLane = [&](int64_t idx) -> Value { |
| 1286 | if (numLanes == 1) |
| 1287 | return laneVec; |
| 1288 | Value pos = |
| 1289 | LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: idx); |
| 1290 | return LLVM::ExtractElementOp::create(builder&: rewriter, location: loc, vector: laneVec, position: pos) |
| 1291 | ->getResult(idx: 0); |
| 1292 | }; |
| 1293 | |
| 1294 | std::string fnName = "__builtin_IB_dnscl_" ; |
| 1295 | fnName += (srcEtype == TruncfSrcElemTypes::F16) ? "hf16" : "bf16" ; |
| 1296 | Value convertTo = |
| 1297 | LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: kDnsclConvertToE2M1); |
| 1298 | auto genDnscl = [&](Value lo, Value hi, int mode) -> Value { |
| 1299 | Value modeVal = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: mode); |
| 1300 | SmallVector<Type> argTypes{lo.getType(), hi.getType(), |
| 1301 | convertTo.getType(), modeVal.getType()}; |
| 1302 | SmallVector<Value> args{lo, hi, convertTo, modeVal}; |
| 1303 | return createDeviceFunctionCall(rewriter, funcName: fnName, retType: i32Ty, argTypes, args, |
| 1304 | paramAttrs: {}, funcAttributeOptions: funcAttrs, op: op.getOperation()) |
| 1305 | ->getResult(idx: 0); |
| 1306 | }; |
| 1307 | |
| 1308 | Value result; |
| 1309 | if (numLanes <= 2) { |
| 1310 | // Fewer than four lanes cannot fill a dword, so a single call is made |
| 1311 | // and the written bytes, 0 and 2, are compacted afterwards. |
| 1312 | Value lo = getLane(0); |
| 1313 | Value hi = |
| 1314 | numLanes == 2 |
| 1315 | ? getLane(1) |
| 1316 | : LLVM::UndefOp::create(builder&: rewriter, location: loc, res: i32Ty)->getResult(idx: 0); |
| 1317 | Value dword = genDnscl(lo, hi, kDnsclModeBytes02); |
| 1318 | if (numLanes == 1) { |
| 1319 | // A single byte, so the low one, is all that is kept. |
| 1320 | result = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i8Ty, arg: dword); |
| 1321 | } else { |
| 1322 | Value bytes = LLVM::BitcastOp::create( |
| 1323 | builder&: rewriter, location: loc, res: VectorType::get(shape: 4, elementType: i8Ty), arg: dword); |
| 1324 | result = LLVM::ShuffleVectorOp::create(builder&: rewriter, location: loc, v1: bytes, v2: bytes, |
| 1325 | mask: ArrayRef<int32_t>{0, 2}); |
| 1326 | } |
| 1327 | } else { |
| 1328 | // Four lanes, eight source elements, per fully packed dword. |
| 1329 | SmallVector<Value> dwords; |
| 1330 | for (int64_t base = 0; base < numLanes; base += 4) { |
| 1331 | // Each lane is bound to a name first: `getLane` builds ops, and the |
| 1332 | // order of evaluation within an argument list is unspecified, which |
| 1333 | // would otherwise leave the order of the emitted ops up to the host |
| 1334 | // compiler. |
| 1335 | Value lane0 = getLane(base); |
| 1336 | Value lane2 = getLane(base + 2); |
| 1337 | Value even = genDnscl(lane0, lane2, kDnsclModeBytes02); |
| 1338 | Value lane1 = getLane(base + 1); |
| 1339 | Value lane3 = getLane(base + 3); |
| 1340 | Value odd = genDnscl(lane1, lane3, kDnsclModeBytes13); |
| 1341 | dwords.push_back(Elt: LLVM::OrOp::create(builder&: rewriter, location: loc, lhs: even, rhs: odd)); |
| 1342 | } |
| 1343 | if (dwords.size() == 1) { |
| 1344 | result = dwords.front(); |
| 1345 | } else { |
| 1346 | Type packedTy = VectorType::get(shape: dwords.size(), elementType: i32Ty); |
| 1347 | result = LLVM::UndefOp::create(builder&: rewriter, location: loc, res: packedTy); |
| 1348 | for (auto [idx, dword] : llvm::enumerate(First&: dwords)) { |
| 1349 | Value pos = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: idx); |
| 1350 | result = |
| 1351 | LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: result, value: dword, position: pos) |
| 1352 | ->getResult(idx: 0); |
| 1353 | } |
| 1354 | } |
| 1355 | } |
| 1356 | rewriter.replaceOp(op, newValues: castIfNeeded(rewriter, loc, ty: dstTy, val: result)); |
| 1357 | return success(); |
| 1358 | } |
| 1359 | |
| 1360 | // Handle the case where dst type is fp8, via the SPV_INTEL_fp_conversions |
| 1361 | // wrappers in IGC's BiFModule: |
| 1362 | // charN __builtin_spirv_Convert{FP16,BF16}To{E5M2,E4M3}EXT(halfN/shortN) |
| 1363 | // |
| 1364 | // These replace the __builtin_IB_* names used before, two of which did not |
| 1365 | // exist: IGC spells E4M3 as `e4m3` in the conversion builtins, not `hf8`. |
| 1366 | // createDeviceFunctionCall emits names unmangled, so the missing symbol |
| 1367 | // linked silently and the call returned undef. The wrappers are |
| 1368 | // `overloadable`, so mangle() takes the length from the argument type and a |
| 1369 | // mismatch fails to link instead. The bf16 forms take the bits as shortN |
| 1370 | // and route through f32 and f16 internally, replacing the explicit |
| 1371 | // bftof/convert_half round trip. |
| 1372 | // |
| 1373 | // `src_etype` is what says how to read the source, so the bf16 bitcast is |
| 1374 | // driven by it, and is only the step the wrapper's shortN signature needs. |
| 1375 | // An operand whose element type disagrees with `src_etype` is a case |
| 1376 | // `TruncfOp::verify` accepts today but should not: mangle() has no bf16 |
| 1377 | // case, so it reaches the unreachable in getTypeMangling. That verifier |
| 1378 | // rule belongs with the op, not here. |
| 1379 | bool srcIsBF16 = srcEtype == TruncfSrcElemTypes::BF16; |
| 1380 | if (srcIsBF16) |
| 1381 | src = LLVM::BitcastOp::create( |
| 1382 | builder&: rewriter, location: loc, |
| 1383 | res: VectorType::get(shape: vecSrcTy.getShape(), elementType: rewriter.getI16Type()), arg: src); |
| 1384 | StringRef convertName; |
| 1385 | if (dstEtype == TruncfDstElemTypes::BF8) { // Float8E5M2Type |
| 1386 | convertName = srcIsBF16 ? "__builtin_spirv_ConvertBF16ToE5M2EXT" |
| 1387 | : "__builtin_spirv_ConvertFP16ToE5M2EXT" ; |
| 1388 | } else if (dstEtype == TruncfDstElemTypes::F8) { // Float8E4M3FNType |
| 1389 | convertName = srcIsBF16 ? "__builtin_spirv_ConvertBF16ToE4M3EXT" |
| 1390 | : "__builtin_spirv_ConvertFP16ToE4M3EXT" ; |
| 1391 | } else { |
| 1392 | return rewriter.notifyMatchFailure( |
| 1393 | arg&: op, msg: "Unsupported src, dst element type pair." ); |
| 1394 | } |
| 1395 | SmallVector<Type> argTypes{src.getType()}; |
| 1396 | SmallVector<Value> args{src}; |
| 1397 | Value result = createDeviceFunctionCall( |
| 1398 | rewriter, funcName: mangle(baseName: convertName, types: argTypes), retType: dstTy, argTypes, |
| 1399 | args, paramAttrs: {}, funcAttributeOptions: funcAttrs, op: op.getOperation()) |
| 1400 | ->getResult(idx: 0); |
| 1401 | rewriter.replaceOp(op, newValues: result); |
| 1402 | return success(); |
| 1403 | } |
| 1404 | }; |
| 1405 | |
| 1406 | class ExtfToOCLPattern : public OpConversionPattern<ExtfOp> { |
| 1407 | using Base::Base; |
| 1408 | LogicalResult |
| 1409 | matchAndRewrite(ExtfOp op, ExtfOp::Adaptor adaptor, |
| 1410 | ConversionPatternRewriter &rewriter) const override { |
| 1411 | // `xevm.extf` is the inverse of `xevm.truncf`. Supported source and result |
| 1412 | // types are restricted for now, mirroring the truncf lowering. |
| 1413 | auto srcEtype = op.getSrcEtype().getEtype(); |
| 1414 | auto dstEtype = op.getDstEtype().getEtype(); |
| 1415 | // The source is scalar only where the packed values fit in one byte, which |
| 1416 | // SPIR-V spells as a scalar rather than a one element vector. |
| 1417 | Type srcTy = op.getSrc().getType(); |
| 1418 | // Scalar dst is not supported until usage case become clear. |
| 1419 | auto vecDstTy = dyn_cast<VectorType>(Val: op.getDst().getType()); |
| 1420 | if (!vecDstTy) |
| 1421 | return rewriter.notifyMatchFailure(arg&: op, msg: "Scalar dst is not supported." ); |
| 1422 | // As for truncf, one builtin exists per SPIR-V vector length. |
| 1423 | int64_t numElements = vecDstTy.getNumElements(); |
| 1424 | if (!isSupportedSPIRVVectorLength(numElements)) |
| 1425 | return rewriter.notifyMatchFailure( |
| 1426 | arg&: op, msg: "dst vector length must be 2, 3, 4, 8 or 16" ); |
| 1427 | Location loc = op.getLoc(); |
| 1428 | Value src = op.getSrc(); |
| 1429 | auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>( |
| 1430 | /*other=*/args: LLVM::ModRefInfo::NoModRef, |
| 1431 | /*argMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 1432 | /*inaccessibleMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 1433 | /*errnoMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 1434 | /*targetMem0=*/args: LLVM::ModRefInfo::NoModRef, |
| 1435 | /*targetMem1=*/args: LLVM::ModRefInfo::NoModRef); |
| 1436 | auto funcAttrs = convergentNoUnwindWillReturnAttrs; |
| 1437 | funcAttrs.memEffectsAttr = memAttr; |
| 1438 | |
| 1439 | // Handle the case where src type is fp4 (e2m1) first. |
| 1440 | if (srcEtype == ExtfSrcElemTypes::E2M1) { |
| 1441 | // Two fp4 values are packed per source byte, and one builtin exists per |
| 1442 | // source byte count: |
| 1443 | // uint16 __builtin_IB_shfl_idx4_lut(int lut_index) |
| 1444 | // uint __builtin_IB_shfl_idx4_to_fp16_packed(uint16 lut, char src) |
| 1445 | // uintN __builtin_IB_shfl_idx4_to_fp16_N_packed(uint16 lut, charN src) |
| 1446 | // Each returns one dword, holding two f16/bf16 values, per source byte. |
| 1447 | // The lookup table selects the target format: |
| 1448 | // 7 = e2m1 -> f16, 5 = e2m1 -> bf16. |
| 1449 | // |
| 1450 | // A byte is the conversion granularity, so an odd length reads one spare |
| 1451 | // value that is dropped afterwards. The op verifier has already checked |
| 1452 | // that the source is exactly as wide as those bytes. |
| 1453 | int64_t numBytes = llvm::divideCeil(Numerator: numElements, Denominator: 2); |
| 1454 | constexpr int kLutE2M1ToF16 = 7; |
| 1455 | constexpr int kLutE2M1ToBF16 = 5; |
| 1456 | int lutIndex = |
| 1457 | (dstEtype == ExtfDstElemTypes::F16) ? kLutE2M1ToF16 : kLutE2M1ToBF16; |
| 1458 | Value lutIdx = LLVM::ConstantOp::create(builder&: rewriter, location: loc, |
| 1459 | type: rewriter.getI32Type(), value: lutIndex); |
| 1460 | Type lutTy = VectorType::get(shape: 16, elementType: rewriter.getI32Type()); |
| 1461 | Value lut = |
| 1462 | createDeviceFunctionCall(rewriter, funcName: "__builtin_IB_shfl_idx4_lut" , |
| 1463 | retType: lutTy, argTypes: {lutIdx.getType()}, args: {lutIdx}, paramAttrs: {}, |
| 1464 | funcAttributeOptions: funcAttrs, op: op.getOperation()) |
| 1465 | ->getResult(idx: 0); |
| 1466 | // A single byte is passed as a bare i8, and one dword returned as a bare |
| 1467 | // i32, rather than as one element vectors SPIR-V has no type for. |
| 1468 | Type i8Ty = rewriter.getI8Type(); |
| 1469 | Type i32Ty = rewriter.getI32Type(); |
| 1470 | std::string fnName = "__builtin_IB_shfl_idx4_to_fp16_" ; |
| 1471 | Type argTy, packedResTy; |
| 1472 | if (numBytes == 1) { |
| 1473 | argTy = i8Ty; |
| 1474 | packedResTy = i32Ty; |
| 1475 | } else { |
| 1476 | fnName += std::to_string(val: numBytes) + "_" ; |
| 1477 | argTy = VectorType::get(shape: numBytes, elementType: i8Ty); |
| 1478 | packedResTy = VectorType::get(shape: numBytes, elementType: i32Ty); |
| 1479 | } |
| 1480 | fnName += "packed" ; |
| 1481 | SmallVector<Type> convArgTypes{lut.getType(), argTy}; |
| 1482 | SmallVector<Value> convArgs{lut, castIfNeeded(rewriter, loc, ty: argTy, val: src)}; |
| 1483 | Value result = |
| 1484 | createDeviceFunctionCall(rewriter, funcName: fnName, retType: packedResTy, argTypes: convArgTypes, |
| 1485 | args: convArgs, paramAttrs: {}, funcAttributeOptions: funcAttrs, op: op.getOperation()) |
| 1486 | ->getResult(idx: 0); |
| 1487 | // The builtin returns the f16/bf16 bits packed as i32, bitcast to the |
| 1488 | // f16/bf16 dst type and drop the padding an odd length produced. |
| 1489 | Type wideTy = VectorType::get(shape: numBytes * 2, elementType: vecDstTy.getElementType()); |
| 1490 | result = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: wideTy, arg: result); |
| 1491 | result = takeLeadingElements(rewriter, loc, val: result, numElements); |
| 1492 | rewriter.replaceOp(op, newValues: result); |
| 1493 | return success(); |
| 1494 | } |
| 1495 | |
| 1496 | // Handle the case where src type is fp8 (bf8/hf8). One fp8 value per source |
| 1497 | // byte, so source and destination lengths match. |
| 1498 | auto vecSrcTy = dyn_cast<VectorType>(Val&: srcTy); |
| 1499 | if (!vecSrcTy || vecSrcTy.getNumElements() != numElements) |
| 1500 | return rewriter.notifyMatchFailure( |
| 1501 | arg&: op, msg: "fp8 src and dst must have the same number of elements" ); |
| 1502 | |
| 1503 | // The matching wrappers, for the reasons given in the truncf lowering: |
| 1504 | // halfN/shortN __builtin_spirv_Convert{E5M2,E4M3}To{FP16,BF16}EXT(charN) |
| 1505 | // |
| 1506 | // `__builtin_IB_hf8tohf_N`, used before, never existed either. A bf16 |
| 1507 | // destination returns the bits as shortN, replacing the |
| 1508 | // convert_float/ftobf round trip. |
| 1509 | bool dstIsBF16 = dstEtype == ExtfDstElemTypes::BF16; |
| 1510 | StringRef convertName; |
| 1511 | if (srcEtype == ExtfSrcElemTypes::BF8) { // Float8E5M2Type |
| 1512 | convertName = dstIsBF16 ? "__builtin_spirv_ConvertE5M2ToBF16EXT" |
| 1513 | : "__builtin_spirv_ConvertE5M2ToFP16EXT" ; |
| 1514 | } else { // Float8E4M3FNType |
| 1515 | convertName = dstIsBF16 ? "__builtin_spirv_ConvertE4M3ToBF16EXT" |
| 1516 | : "__builtin_spirv_ConvertE4M3ToFP16EXT" ; |
| 1517 | } |
| 1518 | Type resElemTy = dstIsBF16 ? cast<Type>(Val: rewriter.getI16Type()) |
| 1519 | : cast<Type>(Val: rewriter.getF16Type()); |
| 1520 | Type resTy = VectorType::get(shape: vecSrcTy.getShape(), elementType: resElemTy); |
| 1521 | SmallVector<Type> argTypes{src.getType()}; |
| 1522 | SmallVector<Value> args{src}; |
| 1523 | Value result = createDeviceFunctionCall( |
| 1524 | rewriter, funcName: mangle(baseName: convertName, types: argTypes), retType: resTy, argTypes, |
| 1525 | args, paramAttrs: {}, funcAttributeOptions: funcAttrs, op: op.getOperation()) |
| 1526 | ->getResult(idx: 0); |
| 1527 | // The bf16 wrapper returns the bits as i16, so cast to the bf16 result |
| 1528 | // type. An f16 result needs nothing: the wrapper returns it. |
| 1529 | rewriter.replaceOp(op, newValues: castIfNeeded(rewriter, loc, ty: vecDstTy, val: result)); |
| 1530 | return success(); |
| 1531 | } |
| 1532 | }; |
| 1533 | |
| 1534 | class MMAMxToOCLPattern : public OpConversionPattern<MMAMxOp> { |
| 1535 | using Base::Base; |
| 1536 | LogicalResult |
| 1537 | matchAndRewrite(MMAMxOp op, MMAMxOp::Adaptor adaptor, |
| 1538 | ConversionPatternRewriter &rewriter) const override { |
| 1539 | if (!op.getC()) { |
| 1540 | return rewriter.notifyMatchFailure(arg&: op, msg: "OCL requires C operand" ); |
| 1541 | } |
| 1542 | auto precisionC = op.getTypes().getC(); |
| 1543 | auto precisionD = op.getTypes().getD(); |
| 1544 | if (precisionC != precisionD) { |
| 1545 | return rewriter.notifyMatchFailure(arg&: op, msg: "type of C and D need to match" ); |
| 1546 | } |
| 1547 | |
| 1548 | constexpr uint32_t bitWidthPackedA{16}; |
| 1549 | constexpr uint32_t bitWidthPackedB{32}; |
| 1550 | auto loc = op.getLoc(); |
| 1551 | |
| 1552 | auto castIfNeeded = [&](Value val, Type packedType) -> Value { |
| 1553 | VectorType origTy = cast<VectorType>(Val: val.getType()); |
| 1554 | const uint32_t vecBitSize = |
| 1555 | origTy.getNumElements() * |
| 1556 | origTy.getElementType().getIntOrFloatBitWidth(); |
| 1557 | VectorType newTy = VectorType::get( |
| 1558 | shape: vecBitSize / packedType.getIntOrFloatBitWidth(), elementType: packedType); |
| 1559 | if (origTy != newTy) |
| 1560 | val = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: newTy, arg: val); |
| 1561 | return val; |
| 1562 | }; |
| 1563 | |
| 1564 | Value a = op.getA(); |
| 1565 | Type packedAType = (op.getTypes().getA() == xevm::ElemType::TF32) |
| 1566 | ? cast<Type>(Val: rewriter.getF32Type()) |
| 1567 | : rewriter.getIntegerType(width: bitWidthPackedA); |
| 1568 | a = castIfNeeded(a, packedAType); |
| 1569 | |
| 1570 | Value b = op.getB(); |
| 1571 | Type packedBType = (op.getTypes().getB() == xevm::ElemType::TF32) |
| 1572 | ? cast<Type>(Val: rewriter.getF32Type()) |
| 1573 | : rewriter.getIntegerType(width: bitWidthPackedB); |
| 1574 | b = castIfNeeded(b, packedBType); |
| 1575 | |
| 1576 | Value c = op.getC(); |
| 1577 | VectorType cOrigTy = cast<VectorType>(Val: c.getType()); |
| 1578 | VectorType resOrigTy = cast<VectorType>(Val: op->getResultTypes()[0]); |
| 1579 | assert(cOrigTy == resOrigTy && "Accumulator and result type mismatch" ); |
| 1580 | // OCL builtins encode bfloat16 as int16 |
| 1581 | VectorType cTy = |
| 1582 | cOrigTy.getElementType().isBF16() |
| 1583 | ? VectorType::get(shape: cOrigTy.getShape(), elementType: rewriter.getIntegerType(width: 16)) |
| 1584 | : cOrigTy; |
| 1585 | VectorType resTy = cTy; |
| 1586 | if (cOrigTy != cTy) |
| 1587 | c = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: cTy, arg: c); |
| 1588 | |
| 1589 | std::string fnName = |
| 1590 | llvm::formatv(Fmt: "__builtin_IB_sub_group16_bdpas_{0}_{1}_{2}_{3}_8_8" , |
| 1591 | Vals: builtinElemType(elemType: op.getTypes().getD()), |
| 1592 | Vals: builtinElemType(elemType: op.getTypes().getC()), |
| 1593 | Vals: builtinElemType(elemType: op.getTypes().getA()), |
| 1594 | Vals: builtinElemType(elemType: op.getTypes().getB())) |
| 1595 | .str(); |
| 1596 | auto scaleA = op.getScaleA(); |
| 1597 | auto scaleB = op.getScaleB(); |
| 1598 | SmallVector<Type> argTypes{cTy, a.getType(), b.getType(), scaleA.getType(), |
| 1599 | scaleB.getType()}; |
| 1600 | SmallVector<Value> args{c, a, b, scaleA, scaleB}; |
| 1601 | |
| 1602 | auto memAttr = rewriter.getAttr<LLVM::MemoryEffectsAttr>( |
| 1603 | /*other=*/args: LLVM::ModRefInfo::NoModRef, |
| 1604 | /*argMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 1605 | /*inaccessibleMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 1606 | /*errnoMem=*/args: LLVM::ModRefInfo::NoModRef, |
| 1607 | /*targetMem0=*/args: LLVM::ModRefInfo::NoModRef, |
| 1608 | /*targetMem1=*/args: LLVM::ModRefInfo::NoModRef); |
| 1609 | auto funcAttrs = convergentNoUnwindWillReturnAttrs; |
| 1610 | funcAttrs.memEffectsAttr = memAttr; |
| 1611 | Value result = |
| 1612 | createDeviceFunctionCall(rewriter, funcName: fnName, retType: resTy, argTypes, args, paramAttrs: {}, |
| 1613 | funcAttributeOptions: funcAttrs, op: op.getOperation()) |
| 1614 | ->getResult(idx: 0); |
| 1615 | |
| 1616 | if (resOrigTy != resTy) |
| 1617 | result = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: resOrigTy, arg: result); |
| 1618 | |
| 1619 | rewriter.replaceOp(op, newValues: result); |
| 1620 | return success(); |
| 1621 | } |
| 1622 | }; |
| 1623 | |
| 1624 | // Lowers `xevm.bitcast_shuffle` to a call to the IGC intrinsic |
| 1625 | // `llvm.genx.GenISA.SubgroupBitcastShuffle`, which is overloaded on both the |
| 1626 | // result and the operand type. E.g. a `vector<4xi8>` -> `vector<2xi16>` shuffle |
| 1627 | // becomes a call to |
| 1628 | // `llvm.genx.GenISA.SubgroupBitcastShuffle.v2i16.v4i8`. |
| 1629 | // |
| 1630 | // Only integer types reach here: the op accepts nothing else, so a producer |
| 1631 | // holding floating point data bitcasts it to a same-width integer beforehand. |
| 1632 | class BitcastShuffleToGenISAPattern |
| 1633 | : public OpConversionPattern<BitcastShuffleOp> { |
| 1634 | using Base::Base; |
| 1635 | LogicalResult |
| 1636 | matchAndRewrite(BitcastShuffleOp op, BitcastShuffleOp::Adaptor adaptor, |
| 1637 | ConversionPatternRewriter &rewriter) const override { |
| 1638 | Type srcTy = op.getSrc().getType(); |
| 1639 | Type resTy = op.getRes().getType(); |
| 1640 | |
| 1641 | std::string fnName = "llvm.genx.GenISA.SubgroupBitcastShuffle." + |
| 1642 | getGenISATypeMangling(ty: resTy) + "." + |
| 1643 | getGenISATypeMangling(ty: srcTy); |
| 1644 | |
| 1645 | Value result = createDeviceFunctionCall( |
| 1646 | rewriter, funcName: fnName, retType: resTy, argTypes: {srcTy}, args: {adaptor.getSrc()}, paramAttrs: {}, |
| 1647 | funcAttributeOptions: convergentNoUnwindWillReturnAttrs, op: op.getOperation()) |
| 1648 | ->getResult(idx: 0); |
| 1649 | |
| 1650 | rewriter.replaceOp(op, newValues: result); |
| 1651 | return success(); |
| 1652 | } |
| 1653 | }; |
| 1654 | |
| 1655 | class AllocaToGlobalPattern : public OpConversionPattern<LLVM::AllocaOp> { |
| 1656 | using Base::Base; |
| 1657 | LogicalResult |
| 1658 | matchAndRewrite(LLVM::AllocaOp op, LLVM::AllocaOp::Adaptor adaptor, |
| 1659 | ConversionPatternRewriter &rewriter) const override { |
| 1660 | auto ptrType = cast<LLVM::LLVMPointerType>(Val: op.getType()); |
| 1661 | auto addrSpace = ptrType.getAddressSpace(); |
| 1662 | if (addrSpace != 3) |
| 1663 | return failure(); |
| 1664 | auto symTable = op->getParentWithTrait<OpTrait::SymbolTable>(); |
| 1665 | if (!symTable) |
| 1666 | return failure(); |
| 1667 | Block *moduleBody; |
| 1668 | if (ModuleOp mod = dyn_cast<ModuleOp>(Val&: *symTable)) { |
| 1669 | moduleBody = mod.getBody(); |
| 1670 | } else if (gpu::GPUModuleOp gpuMod = |
| 1671 | dyn_cast<gpu::GPUModuleOp>(Val&: *symTable)) { |
| 1672 | moduleBody = gpuMod.getBody(); |
| 1673 | } else { |
| 1674 | return failure(); |
| 1675 | } |
| 1676 | auto val = op.getArraySize(); |
| 1677 | APInt cst; |
| 1678 | if (!matchPattern(value: val, pattern: m_ConstantInt(bind_value: &cst))) |
| 1679 | return failure(); |
| 1680 | auto loc = op.getLoc(); |
| 1681 | auto globalType = LLVM::LLVMArrayType::get( |
| 1682 | context: rewriter.getContext(), elementType: op.getElemType(), numElements: cst.getZExtValue()); |
| 1683 | LLVM::GlobalOp globalVar; |
| 1684 | { |
| 1685 | OpBuilder::InsertionGuard guard(rewriter); |
| 1686 | rewriter.setInsertionPointToStart(moduleBody); |
| 1687 | auto alignment = op.getAlignment(); |
| 1688 | globalVar = LLVM::GlobalOp::create( |
| 1689 | builder&: rewriter, location: loc, type: globalType, /*isConstant=*/false, |
| 1690 | /*linkage=*/LLVM::Linkage::Internal, |
| 1691 | /*name=*/std::string("__global_alloca_" ) + |
| 1692 | std::to_string(val: getNextGlobalIdx()), |
| 1693 | /*value=*/Attribute(), |
| 1694 | /*alignment=*/alignment ? *alignment : 0, /*addrSpace=*/addrSpace); |
| 1695 | } |
| 1696 | rewriter.replaceOpWithNewOp<LLVM::AddressOfOp>(op, args&: globalVar); |
| 1697 | return success(); |
| 1698 | } |
| 1699 | |
| 1700 | private: |
| 1701 | static unsigned getNextGlobalIdx() { |
| 1702 | static unsigned globalIdx = 0; |
| 1703 | return globalIdx++; |
| 1704 | } |
| 1705 | }; |
| 1706 | |
| 1707 | // Checks if shufflevector is used as a way to extract a contiguous slice |
| 1708 | // from a vector. |
| 1709 | // - source vector V2 is either the same as V1, or a poison/undef value. In |
| 1710 | // both cases the mask can only meaningfully address elements of V1, which |
| 1711 | // the mask checks below enforce. |
| 1712 | // - mask size is not greater than the source vector size |
| 1713 | // - mask values represent a sequence of consecutive increasing numbers |
| 1714 | // that stay in bounds of the source vector when used for indexing. |
| 1715 | static bool (LLVM::ShuffleVectorOp op) { |
| 1716 | if (op.getV1() != op.getV2() && |
| 1717 | !isa_and_present<LLVM::PoisonOp, LLVM::UndefOp>( |
| 1718 | Val: op.getV2().getDefiningOp())) |
| 1719 | return false; |
| 1720 | auto maskAttr = op.getMask(); |
| 1721 | int64_t maskSize = static_cast<int64_t>(maskAttr.size()); |
| 1722 | int64_t sourceSize = op.getV1().getType().getNumElements(); |
| 1723 | if (maskSize > sourceSize) |
| 1724 | return false; |
| 1725 | int64_t firstIndex = maskAttr[0]; |
| 1726 | if (firstIndex < 0 || firstIndex >= sourceSize) |
| 1727 | return false; |
| 1728 | for (int64_t i = 1; i < maskSize; ++i) { |
| 1729 | int64_t index = maskAttr[i]; |
| 1730 | if (index != firstIndex + i) |
| 1731 | return false; |
| 1732 | if (index >= sourceSize) |
| 1733 | return false; |
| 1734 | } |
| 1735 | return true; |
| 1736 | } |
| 1737 | |
| 1738 | // Clones `op` with new operands and result types, preserving its attributes |
| 1739 | // and properties (e.g. an `icmp` predicate or fastmath flags). Same idiom used |
| 1740 | // by the Vector dialect's unroll patterns. |
| 1741 | static Operation *cloneOpWithOperandsAndTypes(RewriterBase &rewriter, |
| 1742 | Location loc, Operation *op, |
| 1743 | ArrayRef<Value> operands, |
| 1744 | ArrayRef<Type> resultTypes) { |
| 1745 | OperationState state(loc, op->getName().getStringRef(), operands, resultTypes, |
| 1746 | op->getRawDictionaryAttrs().getValue()); |
| 1747 | state.propertiesAttr = op->getPropertiesAsAttribute(); |
| 1748 | return rewriter.create(state); |
| 1749 | } |
| 1750 | |
| 1751 | // Input vector of a shuffle vector op extracting a contiguous slice is an |
| 1752 | // illegal vector in SPIRV kernel if the vector size is > 16 elements. |
| 1753 | // To legalize this case, keep applying the following transformations until no |
| 1754 | // more match: |
| 1755 | // 1. keep hoisting the shuffle vector op past unary element-wise operations |
| 1756 | // start with fpext, fptrunc and bitcast for now. |
| 1757 | // 2. merge with another shuffle vector op |
| 1758 | // 3. merge with load as a smaller load |
| 1759 | // 4. hoist past n-ary element-wise operations (e.g. fdiv, select, icmp) by |
| 1760 | // slicing every same-length vector operand with the same mask and cloning |
| 1761 | // the op at the narrow width. Only done when the source is wider than 16 |
| 1762 | // elements. |
| 1763 | class HandleVectorExtractPattern |
| 1764 | : public OpRewritePattern<LLVM::ShuffleVectorOp> { |
| 1765 | using OpRewritePattern<LLVM::ShuffleVectorOp>::OpRewritePattern; |
| 1766 | |
| 1767 | void initialize() { setHasBoundedRewriteRecursion(); } |
| 1768 | |
| 1769 | LogicalResult matchAndRewrite(LLVM::ShuffleVectorOp op, |
| 1770 | PatternRewriter &rewriter) const override { |
| 1771 | |
| 1772 | if (!isExtractingContiguousSlice(op)) |
| 1773 | return failure(); |
| 1774 | |
| 1775 | auto mask = op.getMask(); |
| 1776 | auto loc = op.getLoc(); |
| 1777 | auto ty = op.getType(); |
| 1778 | // Check source operand to determine rewrite pattern. |
| 1779 | auto src = op.getV1(); |
| 1780 | // 1. Hoist past unary element-wise operations |
| 1781 | if (auto srcOp = src.getDefiningOp()) { |
| 1782 | if (isa<LLVM::FPExtOp>(Val: srcOp) || isa<LLVM::FPTruncOp>(Val: srcOp)) { |
| 1783 | Value srcInput = srcOp->getOperand(idx: 0); |
| 1784 | // Create new shuffle vector op with unary input as source. |
| 1785 | auto srcVecTy = dyn_cast<VectorType>(Val: srcInput.getType()); |
| 1786 | if (!srcVecTy) |
| 1787 | return failure(); |
| 1788 | auto newShuffleVecTy = |
| 1789 | VectorType::get(shape: mask.size(), elementType: srcVecTy.getElementType()); |
| 1790 | auto newShuffle = LLVM::ShuffleVectorOp::create( |
| 1791 | builder&: rewriter, location: loc, res: newShuffleVecTy, v1: srcInput, v2: srcInput, mask); |
| 1792 | // Create new unary op with new shuffle as input. |
| 1793 | Value newUnaryOp; |
| 1794 | if (isa<LLVM::FPExtOp>(Val: srcOp)) { |
| 1795 | newUnaryOp = LLVM::FPExtOp::create(builder&: rewriter, location: loc, res: ty, arg: newShuffle); |
| 1796 | } else { |
| 1797 | newUnaryOp = LLVM::FPTruncOp::create(builder&: rewriter, location: loc, res: ty, arg: newShuffle); |
| 1798 | } |
| 1799 | rewriter.replaceOp(op, newValues: newUnaryOp); |
| 1800 | } else if (isa<LLVM::BitcastOp>(Val: srcOp)) { |
| 1801 | Value srcInput = srcOp->getOperand(idx: 0); |
| 1802 | // Create new shuffle vector op with unary input as source. A bitcast |
| 1803 | // from a scalar has no slice to rewrite in terms of. |
| 1804 | auto srcInputVecTy = dyn_cast<VectorType>(Val: srcInput.getType()); |
| 1805 | auto srcResVecTy = dyn_cast<VectorType>(Val: srcOp->getResult(idx: 0).getType()); |
| 1806 | if (!srcInputVecTy || !srcResVecTy) |
| 1807 | return failure(); |
| 1808 | auto srcInputSize = srcInputVecTy.getNumElements(); |
| 1809 | auto srcResSize = srcResVecTy.getNumElements(); |
| 1810 | auto maskSize = static_cast<int32_t>(mask.size()); |
| 1811 | if (srcInputSize > srcResSize) { |
| 1812 | return failure(); |
| 1813 | } |
| 1814 | if (srcResSize % srcInputSize != 0) { |
| 1815 | return failure(); |
| 1816 | } |
| 1817 | auto maskScale = srcResSize / srcInputSize; |
| 1818 | // Storage for the rescaled mask. `mask` is an ArrayRef that gets |
| 1819 | // rebound to this buffer below, so it has to outlive the `if` that |
| 1820 | // fills it - otherwise the uses after the `if` read a destroyed |
| 1821 | // SmallVector. |
| 1822 | SmallVector<int32_t> newMask; |
| 1823 | if (maskScale != 1) { |
| 1824 | // The slice has to start at, and cover, whole source elements to be |
| 1825 | // expressible in terms of the bitcast source. |
| 1826 | if (mask[0] % maskScale != 0 || maskSize % maskScale != 0) { |
| 1827 | return failure(); |
| 1828 | } |
| 1829 | // Create a new mask that maps to the source vector |
| 1830 | int32_t newMaskSize = maskSize / maskScale; |
| 1831 | int32_t maskStart = mask[0] / maskScale; |
| 1832 | for (int32_t i = 0; i < newMaskSize; ++i) { |
| 1833 | newMask.push_back(Elt: maskStart + i); |
| 1834 | } |
| 1835 | mask = newMask; |
| 1836 | } |
| 1837 | auto newShuffleVecTy = VectorType::get( |
| 1838 | shape: static_cast<int64_t>(mask.size()), elementType: srcInputVecTy.getElementType()); |
| 1839 | auto newShuffle = LLVM::ShuffleVectorOp::create( |
| 1840 | builder&: rewriter, location: loc, res: newShuffleVecTy, v1: srcInput, v2: srcInput, mask); |
| 1841 | // Create new unary op with new shuffle as input. |
| 1842 | auto newBitcast = |
| 1843 | LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: ty, arg: newShuffle); |
| 1844 | rewriter.replaceOp(op, newOp: newBitcast); |
| 1845 | } else if (isa<LLVM::ShuffleVectorOp>(Val: srcOp)) { |
| 1846 | // 2. Merge with source shuffle vector op if, the source op is |
| 1847 | // also extracting a contigous slice and create a new |
| 1848 | // shuffle vector op directly from the source of |
| 1849 | // the first shuffle. |
| 1850 | auto srcShuffle = cast<LLVM::ShuffleVectorOp>(Val: srcOp); |
| 1851 | if (!isExtractingContiguousSlice(op: srcShuffle)) |
| 1852 | return failure(); |
| 1853 | auto srcMask = srcShuffle.getMask(); |
| 1854 | SmallVector<int32_t> combinedMask; |
| 1855 | for (auto index : mask) { |
| 1856 | combinedMask.push_back(Elt: srcMask[index]); |
| 1857 | } |
| 1858 | auto newShuffle = LLVM::ShuffleVectorOp::create( |
| 1859 | builder&: rewriter, location: loc, res: ty, v1: srcShuffle.getV1(), v2: srcShuffle.getV1(), |
| 1860 | mask: DenseI32ArrayAttr::get(context: rewriter.getContext(), content: combinedMask)); |
| 1861 | rewriter.replaceOp(op, newOp: newShuffle); |
| 1862 | } else if (isa<LLVM::LoadOp>(Val: srcOp)) { |
| 1863 | // 3. Merge with load as a smaller load |
| 1864 | auto loadOp = cast<LLVM::LoadOp>(Val: srcOp); |
| 1865 | auto loadPtr = loadOp.getAddr(); |
| 1866 | auto loadAddrSpace = loadPtr.getType().getAddressSpace(); |
| 1867 | if (loadAddrSpace != 0) |
| 1868 | return failure(); |
| 1869 | auto loadTy = dyn_cast<VectorType>(Val: loadOp.getType()); |
| 1870 | if (!loadTy) |
| 1871 | return failure(); |
| 1872 | auto elemTy = loadTy.getElementType(); |
| 1873 | auto firstIndex = mask[0]; |
| 1874 | auto newVecTy = VectorType::get(shape: mask.size(), elementType: elemTy); |
| 1875 | // GEPOp is needed if first index is not zero |
| 1876 | if (firstIndex) { |
| 1877 | auto newPtr = LLVM::GEPOp::create( |
| 1878 | builder&: rewriter, location: loc, |
| 1879 | resultType: LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: loadAddrSpace), |
| 1880 | elementType: elemTy, basePtr: loadPtr, indices: ArrayRef<LLVM::GEPArg>{firstIndex}); |
| 1881 | auto newLoad = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: newVecTy, addr: newPtr); |
| 1882 | rewriter.replaceOp(op, newOp: newLoad); |
| 1883 | } else { |
| 1884 | auto newLoad = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: newVecTy, addr: loadPtr); |
| 1885 | rewriter.replaceOp(op, newOp: newLoad); |
| 1886 | } |
| 1887 | } else if (isMemoryEffectFree(op: srcOp) && srcOp->getNumResults() == 1 && |
| 1888 | srcOp->getNumOperands() >= 1 && |
| 1889 | // Only to legalize. A source of 16 or fewer elements is |
| 1890 | // already legal, and splitting it would put a narrow copy of |
| 1891 | // the op next to every slice user. |
| 1892 | cast<VectorType>(Val: src.getType()).getNumElements() > 16 && |
| 1893 | llvm::all_of(Range: srcOp->getOperands(), P: [&](Value operand) { |
| 1894 | auto operandTy = dyn_cast<VectorType>(Val: operand.getType()); |
| 1895 | auto srcTy = cast<VectorType>(Val: src.getType()); |
| 1896 | return operandTy && operandTy.getRank() == 1 && |
| 1897 | operandTy.getNumElements() == srcTy.getNumElements(); |
| 1898 | })) { |
| 1899 | // 4. Hoist past an n-ary element-wise op: slice each same-length vector |
| 1900 | // operand with the same contiguous mask and clone the op narrow. |
| 1901 | // The freshly created operand shuffles keep the greedy driver going, |
| 1902 | // hoisting them further up (or merging via case 2 / into loads via |
| 1903 | // case 3) until they reach operands that are already legal width. |
| 1904 | SmallVector<Value> newOperands; |
| 1905 | newOperands.reserve(N: srcOp->getNumOperands()); |
| 1906 | for (Value operand : srcOp->getOperands()) { |
| 1907 | auto operandTy = cast<VectorType>(Val: operand.getType()); |
| 1908 | auto sliceTy = |
| 1909 | VectorType::get(shape: mask.size(), elementType: operandTy.getElementType()); |
| 1910 | newOperands.push_back(Elt: LLVM::ShuffleVectorOp::create( |
| 1911 | builder&: rewriter, location: loc, res: sliceTy, v1: operand, v2: operand, mask)); |
| 1912 | } |
| 1913 | Operation *newOp = |
| 1914 | cloneOpWithOperandsAndTypes(rewriter, loc, op: srcOp, operands: newOperands, resultTypes: ty); |
| 1915 | rewriter.replaceOp(op, newValues: newOp->getResult(idx: 0)); |
| 1916 | } else { |
| 1917 | return failure(); |
| 1918 | } |
| 1919 | } else { |
| 1920 | // No defining op (e.g. function argument): nothing to hoist/merge. |
| 1921 | return failure(); |
| 1922 | } |
| 1923 | return success(); |
| 1924 | } |
| 1925 | }; |
| 1926 | |
| 1927 | //===----------------------------------------------------------------------===// |
| 1928 | // Pass Definition |
| 1929 | //===----------------------------------------------------------------------===// |
| 1930 | |
| 1931 | struct ConvertXeVMToLLVMPass |
| 1932 | : public impl::ConvertXeVMToLLVMPassBase<ConvertXeVMToLLVMPass> { |
| 1933 | using Base::Base; |
| 1934 | |
| 1935 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 1936 | registry.insert<LLVM::LLVMDialect, XeVMDialect>(); |
| 1937 | } |
| 1938 | |
| 1939 | void runOnOperation() override { |
| 1940 | ConversionTarget target(getContext()); |
| 1941 | RewritePatternSet patterns(&getContext()); |
| 1942 | populateXeVMToLLVMConversionPatterns(target, patterns); |
| 1943 | if (failed(Result: applyPartialConversion(op: getOperation(), target, |
| 1944 | patterns: std::move(patterns)))) |
| 1945 | signalPassFailure(); |
| 1946 | |
| 1947 | // Apply in-dialect lowerings to handle illegal vectors |
| 1948 | { |
| 1949 | RewritePatternSet vectorPatterns(&getContext()); |
| 1950 | vectorPatterns.add<HandleVectorExtractPattern>(arg: &getContext()); |
| 1951 | GreedyRewriteConfig config{}; |
| 1952 | // folding can remove ops with temporary attributes used to |
| 1953 | // represent LLVM metadata, so disable it here. |
| 1954 | // Effectively just this single pattern is applied without any |
| 1955 | // op folding patterns from dialects. |
| 1956 | config.enableFolding(enable: false); |
| 1957 | // config.setMaxIterations(GreedyRewriteConfig::kNoLimit); |
| 1958 | // config.setMaxNumRewrites(GreedyRewriteConfig::kNoLimit); |
| 1959 | (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(vectorPatterns), |
| 1960 | config); |
| 1961 | } |
| 1962 | } |
| 1963 | }; |
| 1964 | } // namespace |
| 1965 | |
| 1966 | //===----------------------------------------------------------------------===// |
| 1967 | // Pattern Population |
| 1968 | //===----------------------------------------------------------------------===// |
| 1969 | |
| 1970 | void ::mlir::populateXeVMToLLVMConversionPatterns(ConversionTarget &target, |
| 1971 | RewritePatternSet &patterns) { |
| 1972 | // some LLVM operations need to be converted. |
| 1973 | target.addDynamicallyLegalDialect<LLVM::LLVMDialect>(callback: [](Operation *op) { |
| 1974 | // llvm alloca op with addrspace 3 for OpenCL (Workgroup) is not handled |
| 1975 | // properly by SPIRV backend. It needs to be rewritten as a sequence with |
| 1976 | // llvm global. |
| 1977 | if (isa<LLVM::AllocaOp>(Val: op)) { |
| 1978 | LLVM::AllocaOp aOp = cast<LLVM::AllocaOp>(Val: op); |
| 1979 | LLVM::LLVMPointerType pTy = cast<LLVM::LLVMPointerType>(Val: aOp.getType()); |
| 1980 | auto addrSpace = pTy.getAddressSpace(); |
| 1981 | return addrSpace != 3; |
| 1982 | } |
| 1983 | // cache_control attribute should be converted. |
| 1984 | return !op->hasDiscardableAttr(name: "cache_control" ); |
| 1985 | }); |
| 1986 | target.addIllegalDialect<XeVMDialect>(); |
| 1987 | patterns.add<LoadStorePrefetchToOCLPattern<BlockLoad2dOp>, |
| 1988 | LoadStorePrefetchToOCLPattern<BlockStore2dOp>, |
| 1989 | LoadStorePrefetchToOCLPattern<BlockPrefetch2dOp>, |
| 1990 | MMAToOCLPattern, MemfenceToOCLPattern, PrefetchToOCLPattern, |
| 1991 | LLVMLoadStoreToOCLPattern<LLVM::LoadOp>, |
| 1992 | LLVMLoadStoreToOCLPattern<LLVM::StoreOp>, |
| 1993 | BlockLoadStore1DToOCLPattern<BlockLoadOp>, |
| 1994 | BlockLoadStore1DToOCLPattern<BlockStoreOp>, |
| 1995 | LaunchConfigOpToOCLPattern<WorkitemIdXOp>, |
| 1996 | LaunchConfigOpToOCLPattern<WorkitemIdYOp>, |
| 1997 | LaunchConfigOpToOCLPattern<WorkitemIdZOp>, |
| 1998 | LaunchConfigOpToOCLPattern<WorkgroupDimXOp>, |
| 1999 | LaunchConfigOpToOCLPattern<WorkgroupDimYOp>, |
| 2000 | LaunchConfigOpToOCLPattern<WorkgroupDimZOp>, |
| 2001 | LaunchConfigOpToOCLPattern<WorkgroupIdXOp>, |
| 2002 | LaunchConfigOpToOCLPattern<WorkgroupIdYOp>, |
| 2003 | LaunchConfigOpToOCLPattern<WorkgroupIdZOp>, |
| 2004 | LaunchConfigOpToOCLPattern<GridDimXOp>, |
| 2005 | LaunchConfigOpToOCLPattern<GridDimYOp>, |
| 2006 | LaunchConfigOpToOCLPattern<GridDimZOp>, |
| 2007 | SubgroupOpWorkitemOpToOCLPattern<LaneIdOp>, |
| 2008 | SubgroupOpWorkitemOpToOCLPattern<SubgroupIdOp>, |
| 2009 | SubgroupOpWorkitemOpToOCLPattern<SubgroupSizeOp>, |
| 2010 | TruncfToOCLPattern, ExtfToOCLPattern, MMAMxToOCLPattern, |
| 2011 | BitcastShuffleToGenISAPattern, AllocaToGlobalPattern>( |
| 2012 | arg: patterns.getContext()); |
| 2013 | } |
| 2014 | |