1//===- AMDGPUToROCDL.cpp - AMDGPU to ROCDL dialect conversion -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h"
10
11#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
12#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
13#include "mlir/Conversion/LLVMCommon/Pattern.h"
14#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
15#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
16#include "mlir/Dialect/AMDGPU/IR/AMDGPUEnums.h"
17#include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
20#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
21#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
22#include "mlir/Dialect/LLVMIR/LLVMIntegerIntrinsicOps.h"
23#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
24#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
25#include "mlir/Dialect/LLVMIR/LLVMVectorOps.h"
26#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
27#include "mlir/IR/Attributes.h"
28#include "mlir/IR/BuiltinAttributes.h"
29#include "mlir/IR/BuiltinTypes.h"
30#include "mlir/IR/Matchers.h"
31#include "mlir/IR/TypeUtilities.h"
32#include "mlir/Pass/Pass.h"
33
34#include "../LLVMCommon/MemRefDescriptor.h"
35
36#include "llvm/ADT/STLExtras.h"
37#include "llvm/ADT/TypeSwitch.h"
38#include "llvm/Support/AMDGPUAddrSpace.h"
39#include "llvm/Support/Casting.h"
40#include "llvm/Support/ErrorHandling.h"
41#include "llvm/Support/MathExtras.h"
42#include <cstdint>
43#include <optional>
44
45namespace mlir {
46#define GEN_PASS_DEF_CONVERTAMDGPUTOROCDLPASS
47#include "mlir/Conversion/Passes.h.inc"
48} // namespace mlir
49
50using namespace mlir;
51using namespace mlir::amdgpu;
52
53/// Zero-extend or truncate the unsigned number `val` to `width` bits.
54static Value convertUnsignedToInt(ConversionPatternRewriter &rewriter,
55 Location loc, Value val, unsigned width) {
56 IntegerType destTy = rewriter.getIntegerType(width);
57 // Force check that `val` is of int type.
58 auto valTy = cast<IntegerType>(Val: val.getType());
59 if (destTy == valTy)
60 return val;
61 return valTy.getWidth() > width
62 ? Value(LLVM::TruncOp::create(builder&: rewriter, location: loc, res: destTy, arg: val))
63 : Value(LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: destTy, arg: val));
64}
65
66/// Convert an unsigned number `val` to i32.
67static Value convertUnsignedToI32(ConversionPatternRewriter &rewriter,
68 Location loc, Value val) {
69 return convertUnsignedToInt(rewriter, loc, val, width: 32);
70}
71
72static Value createI32Constant(ConversionPatternRewriter &rewriter,
73 Location loc, int32_t value) {
74 return LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI32Type(), value);
75}
76
77/// Convert an unsigned number `val` to i64.
78static Value convertUnsignedToI64(ConversionPatternRewriter &rewriter,
79 Location loc, Value val) {
80 return convertUnsignedToInt(rewriter, loc, val, width: 64);
81}
82
83static Value createI64Constant(ConversionPatternRewriter &rewriter,
84 Location loc, int64_t value) {
85 return LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI64Type(), value);
86}
87
88/// Returns the linear index used to access an element in the memref.
89static Value getLinearIndexI32(ConversionPatternRewriter &rewriter,
90 Location loc, MemRefDescriptor &memRefDescriptor,
91 ValueRange indices, ArrayRef<int64_t> strides) {
92 IntegerType i32 = rewriter.getI32Type();
93 Value index;
94 for (auto [i, increment, stride] : llvm::enumerate(First&: indices, Rest&: strides)) {
95 if (stride != 1) { // Skip if stride is 1.
96 Value strideValue =
97 ShapedType::isDynamic(dValue: stride)
98 ? convertUnsignedToI32(rewriter, loc,
99 val: memRefDescriptor.stride(builder&: rewriter, loc, pos: i))
100 : LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32, value: stride);
101 increment = LLVM::MulOp::create(builder&: rewriter, location: loc, lhs: increment, rhs: strideValue);
102 }
103 index = index ? LLVM::AddOp::create(builder&: rewriter, location: loc, lhs: index, rhs: increment)
104 : increment;
105 }
106 return index ? index : createI32Constant(rewriter, loc, value: 0);
107}
108
109/// Compute the contents of the `num_records` field for a given memref
110/// descriptor - that is, the number of bytes that's one element past the
111/// greatest possible valid index into the memref.
112static Value getNumRecords(ConversionPatternRewriter &rewriter, Location loc,
113 MemRefType memrefType,
114 MemRefDescriptor &memrefDescriptor,
115 ArrayRef<int64_t> strides, int64_t elementByteWidth,
116 unsigned numRecordsWidth, bool boundsCheck) {
117 if (numRecordsWidth > 32 && !boundsCheck)
118 return createI64Constant(rewriter, loc, value: llvm::maxUIntN(N: numRecordsWidth));
119 if (memrefType.hasStaticShape() &&
120 !llvm::any_of(Range&: strides, P: ShapedType::isDynamic)) {
121 int64_t size = memrefType.getRank() == 0 ? 1 : 0;
122 ArrayRef<int64_t> shape = memrefType.getShape();
123 for (uint32_t i = 0, e = memrefType.getRank(); i < e; ++i)
124 size = std::max(a: shape[i] * strides[i], b: size);
125 size = size * elementByteWidth;
126 return createI64Constant(rewriter, loc, value: size);
127 }
128 Value maxIndex;
129 for (uint32_t i = 0, e = memrefType.getRank(); i < e; ++i) {
130 Value size = memrefDescriptor.size(builder&: rewriter, loc, pos: i);
131 Value stride = memrefDescriptor.stride(builder&: rewriter, loc, pos: i);
132 Value maxThisDim = LLVM::MulOp::create(builder&: rewriter, location: loc, lhs: size, rhs: stride);
133 maxIndex = maxIndex
134 ? LLVM::UMaxOp::create(builder&: rewriter, location: loc, a: maxIndex, b: maxThisDim)
135 : maxThisDim;
136 }
137 Value maxIndexI64 = convertUnsignedToI64(rewriter, loc, val: maxIndex);
138 Value byteWidthConst = createI64Constant(rewriter, loc, value: elementByteWidth);
139 return LLVM::MulOp::create(builder&: rewriter, location: loc, lhs: maxIndexI64, rhs: byteWidthConst);
140}
141
142static Value makeBufferRsrc(ConversionPatternRewriter &rewriter, Location loc,
143 Value basePointer, Value numRecords,
144 bool boundsCheck, const ROCDL::TargetInfo &target,
145 unsigned numRecordsWidth,
146 Value cacheSwizzleStride = nullptr,
147 unsigned addressSpace = 8) {
148 // The stride value is generally 0. However, on MI-300 and onward, you can
149 // enable a cache swizzling mode by setting bit 14 of the stride field
150 // and setting that stride to a cache stride.
151 Type i16 = rewriter.getI16Type();
152 Value stride;
153 if (target.has(feature: llvm::AMDGPU::FEAT_GFX940_INSTS) && cacheSwizzleStride) {
154 Value cacheStrideZext =
155 LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: i16, arg: cacheSwizzleStride);
156 Value swizzleBit = LLVM::ConstantOp::create(
157 builder&: rewriter, location: loc, res: i16, value: rewriter.getI16IntegerAttr(value: 1 << 14));
158 stride = LLVM::OrOp::create(builder&: rewriter, location: loc, lhs: cacheStrideZext, rhs: swizzleBit,
159 /*isDisjoint=*/true);
160 } else {
161 stride = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: i16,
162 value: rewriter.getI16IntegerAttr(value: 0));
163 }
164
165 uint32_t flags = 0;
166 if (target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS)) {
167 // Flag word:
168 // bit 0: swizzle
169 // bit 1: 0 means (total_offset + payload > numRecords)
170 // 1 means ((total_offset + payload >) numRecords) || ((offset +
171 // payload) > stride) only applied when swizzle_enable = 0. keep at
172 // zero.
173 // whether oob is done depends on numRecords.
174 // bits 2-3: Type (must be 0)
175 } else {
176 // Get the number of elements.
177 // Flag word:
178 // bits 0-11: dst sel, ignored by these intrinsics
179 // bits 12-14: data format (ignored, must be nonzero, 7=float)
180 // bits 15-18: data format (ignored, must be nonzero, 4=32bit)
181 // bit 19: In nested heap (0 here)
182 // bit 20: Behavior on unmap (0 means "return 0 / ignore")
183 // bits 21-22: Index stride for swizzles (N/A)
184 // bit 23: Add thread ID (0)
185 // bit 24: Reserved to 1 (RDNA) or 0 (CDNA)
186 // bits 25-26: Reserved (0)
187 // bit 27: Buffer is non-volatile (CDNA only)
188 // bits 28-29: Out of bounds select (0 = structured, 1 = check index, 2 =
189 // none, 3 = either swizzles or testing against offset field) RDNA only
190 // bits 30-31: Type (must be 0)
191 flags |= (7 << 12) | (4 << 15);
192 if (target.has(feature: llvm::AMDGPU::FEAT_GFX10_INSTS)) {
193 flags |= (1 << 24);
194 uint32_t oob = boundsCheck ? 3 : 2;
195 flags |= (oob << 28);
196 }
197 }
198 Value flagsConst = createI32Constant(rewriter, loc, value: flags);
199 numRecords = convertUnsignedToInt(rewriter, loc, val: numRecords, width: numRecordsWidth);
200 Type rsrcType =
201 LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace);
202 Value resource = rewriter.createOrFold<ROCDL::MakeBufferRsrcOp>(
203 location: loc, args&: rsrcType, args&: basePointer, args&: stride, args&: numRecords, args&: flagsConst);
204 return resource;
205}
206
207namespace {
208struct FatRawBufferCastLowering
209 : public ConvertOpToLLVMPattern<FatRawBufferCastOp> {
210 FatRawBufferCastLowering(const LLVMTypeConverter &converter,
211 const ROCDL::TargetInfo &target)
212 : ConvertOpToLLVMPattern<FatRawBufferCastOp>(converter), target(target) {}
213
214 ROCDL::TargetInfo target;
215
216 LogicalResult
217 matchAndRewrite(FatRawBufferCastOp op, FatRawBufferCastOpAdaptor adaptor,
218 ConversionPatternRewriter &rewriter) const override {
219 Location loc = op.getLoc();
220 Value memRef = adaptor.getSource();
221 Value unconvertedMemref = op.getSource();
222 MemRefType memrefType = cast<MemRefType>(Val: unconvertedMemref.getType());
223 MemRefDescriptor descriptor(memRef);
224
225 DataLayout dataLayout = DataLayout::closest(op);
226 int64_t elementByteWidth =
227 dataLayout.getTypeSizeInBits(t: memrefType.getElementType()) / 8;
228
229 int64_t unusedOffset = 0;
230 SmallVector<int64_t, 5> strideVals;
231 if (failed(Result: memrefType.getStridesAndOffset(strides&: strideVals, offset&: unusedOffset)))
232 return op.emitOpError(message: "Can't lower non-stride-offset memrefs");
233
234 std::optional<unsigned> numRecordsWidth =
235 target.getBufferResourceNumRecordsWidth();
236 if (!numRecordsWidth)
237 return op.emitOpError(
238 message: "buffer resource num_records width is unknown for this target");
239
240 Value numRecords = adaptor.getValidBytes();
241 if (!numRecords)
242 numRecords = getNumRecords(rewriter, loc, memrefType, memrefDescriptor&: descriptor,
243 strides: strideVals, elementByteWidth, numRecordsWidth: *numRecordsWidth,
244 boundsCheck: adaptor.getBoundsCheck());
245
246 Value basePointer =
247 adaptor.getResetOffset()
248 ? descriptor.bufferPtr(builder&: rewriter, loc, converter: *getTypeConverter(),
249 type: memrefType)
250 : descriptor.alignedPtr(builder&: rewriter, loc);
251
252 Value offset =
253 adaptor.getResetOffset()
254 ? createIndexAttrConstant(builder&: rewriter, loc, resultType: getIndexType(), value: 0)
255 : descriptor.offset(builder&: rewriter, loc);
256
257 bool hasSizes = memrefType.getRank() > 0;
258 // No need to unpack() and pack() all the individual sizes and strides,
259 // so we'll just extract the arrays.
260 Value sizes = hasSizes
261 ? LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: descriptor,
262 position: kSizePosInMemRefDescriptor)
263 : Value{};
264 Value strides =
265 hasSizes ? LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: descriptor,
266 position: kStridePosInMemRefDescriptor)
267 : Value{};
268
269 Value fatPtr = makeBufferRsrc(
270 rewriter, loc, basePointer, numRecords, boundsCheck: adaptor.getBoundsCheck(),
271 target, numRecordsWidth: *numRecordsWidth, cacheSwizzleStride: adaptor.getCacheSwizzleStride(),
272 /*addressSpace=*/7);
273
274 Value result = MemRefDescriptor::poison(
275 builder&: rewriter, loc,
276 descriptorType: getTypeConverter()->convertType(t: op.getResult().getType()));
277 SmallVector<int64_t> pos{kAllocatedPtrPosInMemRefDescriptor};
278 result = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: result, value: fatPtr, position: pos);
279 result = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: result, value: fatPtr,
280 position: kAlignedPtrPosInMemRefDescriptor);
281 result = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: result, value: offset,
282 position: kOffsetPosInMemRefDescriptor);
283 if (hasSizes) {
284 result = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: result, value: sizes,
285 position: kSizePosInMemRefDescriptor);
286 result = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: result, value: strides,
287 position: kStridePosInMemRefDescriptor);
288 }
289 rewriter.replaceOp(op, newValues: result);
290 return success();
291 }
292};
293
294/// Define lowering patterns for raw buffer ops
295template <typename GpuOp, typename Intrinsic>
296struct RawBufferOpLowering : public ConvertOpToLLVMPattern<GpuOp> {
297 RawBufferOpLowering(const LLVMTypeConverter &converter,
298 const ROCDL::TargetInfo &target)
299 : ConvertOpToLLVMPattern<GpuOp>(converter), target(target) {}
300
301 ROCDL::TargetInfo target;
302 static constexpr uint32_t maxVectorOpWidth = 128;
303
304 LogicalResult
305 matchAndRewrite(GpuOp gpuOp, typename GpuOp::Adaptor adaptor,
306 ConversionPatternRewriter &rewriter) const override {
307 Location loc = gpuOp.getLoc();
308 Value memref = adaptor.getMemref();
309 Value unconvertedMemref = gpuOp.getMemref();
310 MemRefType memrefType = cast<MemRefType>(Val: unconvertedMemref.getType());
311
312 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX9_INSTS))
313 return gpuOp.emitOpError("raw buffer ops require GCN or higher");
314
315 Value storeData = adaptor.getODSOperands(0)[0];
316 if (storeData == memref) // no write component to this op
317 storeData = Value();
318 Type wantedDataType;
319 if (storeData)
320 wantedDataType = storeData.getType();
321 else
322 wantedDataType = gpuOp.getODSResults(0)[0].getType();
323
324 Value atomicCmpData = Value();
325 // Operand index 1 of a load is the indices, trying to read them can crash.
326 if (storeData) {
327 Value maybeCmpData = adaptor.getODSOperands(1)[0];
328 if (maybeCmpData != memref)
329 atomicCmpData = maybeCmpData;
330 }
331
332 Type llvmWantedDataType = this->typeConverter->convertType(wantedDataType);
333
334 Type i32 = rewriter.getI32Type();
335
336 // Get the type size in bytes.
337 DataLayout dataLayout = DataLayout::closest(op: gpuOp);
338 int64_t elementByteWidth =
339 dataLayout.getTypeSizeInBits(t: memrefType.getElementType()) / 8;
340 Value byteWidthConst = createI32Constant(rewriter, loc, value: elementByteWidth);
341
342 // If we want to load a vector<NxT> with total size <= 32
343 // bits, use a scalar load and bitcast it. Similarly, if bitsize(T) < 32
344 // and the total load size is >= 32, use a vector load of N / (bitsize(T) /
345 // 32) x i32 and bitcast. Also, the CAS intrinsic requires integer operands,
346 // so bitcast any floats to integers.
347 Type llvmBufferValType = llvmWantedDataType;
348 if (atomicCmpData) {
349 if (auto floatType = dyn_cast<FloatType>(Val&: wantedDataType))
350 llvmBufferValType = this->getTypeConverter()->convertType(
351 rewriter.getIntegerType(width: floatType.getWidth()));
352 }
353 if (auto dataVector = dyn_cast<VectorType>(Val&: wantedDataType)) {
354 uint32_t vecLen = dataVector.getNumElements();
355 uint32_t elemBits =
356 dataLayout.getTypeSizeInBits(t: dataVector.getElementType());
357 uint32_t totalBits = elemBits * vecLen;
358 bool usePackedFp16 =
359 isa_and_present<RawBufferAtomicFaddOp>(*gpuOp) && vecLen == 2;
360 if (totalBits > maxVectorOpWidth)
361 return gpuOp.emitOpError(
362 "Total width of loads or stores must be no more than " +
363 Twine(maxVectorOpWidth) + " bits, but we call for " +
364 Twine(totalBits) +
365 " bits. This should've been caught in validation");
366 if (!usePackedFp16 && elemBits < 32) {
367 if (totalBits > 32) {
368 if (totalBits % 32 != 0)
369 return gpuOp.emitOpError("Load or store of more than 32-bits that "
370 "doesn't fit into words. Can't happen\n");
371 llvmBufferValType = this->typeConverter->convertType(
372 VectorType::get(shape: totalBits / 32, elementType: i32));
373 } else {
374 llvmBufferValType = this->typeConverter->convertType(
375 rewriter.getIntegerType(width: totalBits));
376 }
377 }
378 }
379 if (auto vecType = dyn_cast<VectorType>(Val&: llvmBufferValType)) {
380 // Buffer intrinsics doesn't support 1-element vectors, cast them to
381 // scalars.
382 if (vecType.getNumElements() == 1)
383 llvmBufferValType = vecType.getElementType();
384 }
385
386 SmallVector<Value, 6> args;
387 if (storeData) {
388 if (llvmBufferValType != llvmWantedDataType) {
389 Value castForStore = LLVM::BitcastOp::create(
390 builder&: rewriter, location: loc, res: llvmBufferValType, arg: storeData);
391 args.push_back(Elt: castForStore);
392 } else {
393 args.push_back(Elt: storeData);
394 }
395 }
396
397 if (atomicCmpData) {
398 if (llvmBufferValType != llvmWantedDataType) {
399 Value castForCmp = LLVM::BitcastOp::create(
400 builder&: rewriter, location: loc, res: llvmBufferValType, arg: atomicCmpData);
401 args.push_back(Elt: castForCmp);
402 } else {
403 args.push_back(Elt: atomicCmpData);
404 }
405 }
406
407 // Construct buffer descriptor from memref, attributes
408 int64_t offset = 0;
409 SmallVector<int64_t, 5> strides;
410 if (failed(Result: memrefType.getStridesAndOffset(strides, offset)))
411 return gpuOp.emitOpError("Can't lower non-stride-offset memrefs");
412
413 MemRefDescriptor memrefDescriptor(memref);
414
415 Value ptr = memrefDescriptor.bufferPtr(
416 builder&: rewriter, loc, converter: *this->getTypeConverter(), type: memrefType);
417 std::optional<unsigned> numRecordsWidth =
418 target.getBufferResourceNumRecordsWidth();
419 if (!numRecordsWidth)
420 return gpuOp.emitOpError(
421 "buffer resource num_records width is unknown for this target");
422
423 Value numRecords = getNumRecords(
424 rewriter, loc, memrefType, memrefDescriptor, strides, elementByteWidth,
425 *numRecordsWidth, adaptor.getBoundsCheck());
426 Value resource =
427 makeBufferRsrc(rewriter, loc, ptr, numRecords, adaptor.getBoundsCheck(),
428 target, *numRecordsWidth);
429 args.push_back(Elt: resource);
430
431 // Indexing (voffset)
432 Value voffset = getLinearIndexI32(rewriter, loc, memrefDescriptor,
433 adaptor.getIndices(), strides);
434 if (std::optional<int32_t> indexOffset = adaptor.getIndexOffset();
435 indexOffset && *indexOffset > 0) {
436 Value extraOffsetConst = createI32Constant(rewriter, loc, value: *indexOffset);
437 voffset = voffset ? LLVM::AddOp::create(builder&: rewriter, location: loc, lhs: voffset,
438 rhs: extraOffsetConst)
439 : extraOffsetConst;
440 }
441 voffset = LLVM::MulOp::create(builder&: rewriter, location: loc, lhs: voffset, rhs: byteWidthConst);
442 args.push_back(Elt: voffset);
443
444 // SGPR offset.
445 Value sgprOffset = adaptor.getSgprOffset();
446 if (!sgprOffset)
447 sgprOffset = createI32Constant(rewriter, loc, value: 0);
448 sgprOffset = LLVM::MulOp::create(builder&: rewriter, location: loc, lhs: sgprOffset, rhs: byteWidthConst);
449 args.push_back(Elt: sgprOffset);
450
451 llvm::SmallVector<Type, 1> resultTypes(gpuOp->getNumResults(),
452 llvmBufferValType);
453 typename Intrinsic::Properties properties{};
454 properties.aux = rewriter.getI32IntegerAttr(value: 0);
455 Operation *lowered =
456 Intrinsic::create(rewriter, loc, resultTypes, args, properties);
457 if (lowered->getNumResults() == 1) {
458 Value replacement = lowered->getResult(idx: 0);
459 if (llvmBufferValType != llvmWantedDataType) {
460 replacement = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: llvmWantedDataType,
461 arg: replacement);
462 }
463 rewriter.replaceOp(gpuOp, replacement);
464 } else {
465 rewriter.eraseOp(op: gpuOp);
466 }
467 return success();
468 }
469};
470
471// TODO: AMDGPU backend already have all this bitpacking logic, we should move
472// it to some common place.
473/// Vmcnt, Expcnt and Lgkmcnt are decoded as follows:
474/// Vmcnt = Waitcnt[3:0] (pre-gfx9)
475/// Vmcnt = Waitcnt[15:14,3:0] (gfx9,10)
476/// Vmcnt = Waitcnt[15:10] (gfx11)
477/// Expcnt = Waitcnt[6:4] (pre-gfx11)
478/// Expcnt = Waitcnt[2:0] (gfx11)
479/// Lgkmcnt = Waitcnt[11:8] (pre-gfx10)
480/// Lgkmcnt = Waitcnt[13:8] (gfx10)
481/// Lgkmcnt = Waitcnt[9:4] (gfx11)
482static FailureOr<unsigned> encodeWaitcnt(const ROCDL::TargetInfo &target,
483 unsigned vmcnt, unsigned expcnt,
484 unsigned lgkmcnt) {
485 if (target.isUnknown())
486 return failure();
487 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX9_INSTS)) {
488 vmcnt = std::min(a: 15u, b: vmcnt);
489 expcnt = std::min(a: 7u, b: expcnt);
490 lgkmcnt = std::min(a: 15u, b: lgkmcnt);
491 return vmcnt | (expcnt << 4) | (lgkmcnt << 8);
492 }
493 if (target.isGeneration(major: 9)) {
494 vmcnt = std::min(a: 63u, b: vmcnt);
495 expcnt = std::min(a: 7u, b: expcnt);
496 lgkmcnt = std::min(a: 15u, b: lgkmcnt);
497 unsigned lowBits = vmcnt & 0xF;
498 unsigned highBits = (vmcnt >> 4) << 14;
499 unsigned otherCnts = (expcnt << 4) | (lgkmcnt << 8);
500 return lowBits | highBits | otherCnts;
501 }
502 if (target.isGeneration(major: 10)) {
503 vmcnt = std::min(a: 63u, b: vmcnt);
504 expcnt = std::min(a: 7u, b: expcnt);
505 lgkmcnt = std::min(a: 63u, b: lgkmcnt);
506 unsigned lowBits = vmcnt & 0xF;
507 unsigned highBits = (vmcnt >> 4) << 14;
508 unsigned otherCnts = (expcnt << 4) | (lgkmcnt << 8);
509 return lowBits | highBits | otherCnts;
510 }
511 if (target.isGeneration(major: 11)) {
512 vmcnt = std::min(a: 63u, b: vmcnt);
513 expcnt = std::min(a: 7u, b: expcnt);
514 lgkmcnt = std::min(a: 63u, b: lgkmcnt);
515 return (vmcnt << 10) | expcnt | (lgkmcnt << 4);
516 }
517 return failure();
518}
519
520struct MemoryCounterWaitOpLowering
521 : public ConvertOpToLLVMPattern<MemoryCounterWaitOp> {
522 MemoryCounterWaitOpLowering(const LLVMTypeConverter &converter,
523 const ROCDL::TargetInfo &target)
524 : ConvertOpToLLVMPattern<MemoryCounterWaitOp>(converter), target(target) {
525 }
526
527 ROCDL::TargetInfo target;
528
529 LogicalResult
530 matchAndRewrite(MemoryCounterWaitOp op, OpAdaptor adaptor,
531 ConversionPatternRewriter &rewriter) const override {
532 if (target.has(feature: llvm::AMDGPU::FEAT_GFX12_INSTS)) {
533 Location loc = op.getLoc();
534 if (std::optional<int> ds = adaptor.getDs())
535 ROCDL::WaitDscntOp::create(builder&: rewriter, location: loc, count: *ds);
536
537 if (std::optional<int> load = adaptor.getLoad())
538 ROCDL::WaitLoadcntOp::create(builder&: rewriter, location: loc, count: *load);
539
540 if (std::optional<int> store = adaptor.getStore())
541 ROCDL::WaitStorecntOp::create(builder&: rewriter, location: loc, count: *store);
542
543 if (std::optional<int> exp = adaptor.getExp())
544 ROCDL::WaitExpcntOp::create(builder&: rewriter, location: loc, count: *exp);
545
546 if (std::optional<int> tensor = adaptor.getTensor())
547 ROCDL::WaitTensorcntOp::create(builder&: rewriter, location: loc, count: *tensor);
548
549 rewriter.eraseOp(op);
550 return success();
551 }
552
553 if (adaptor.getTensor())
554 return op.emitOpError(message: "unsupported chipset");
555
556 auto getVal = [](Attribute attr) -> unsigned {
557 if (attr)
558 return cast<IntegerAttr>(Val&: attr).getInt();
559
560 // This value will be clamped to the maximum value for the chipset.
561 return 1024;
562 };
563 unsigned ds = getVal(adaptor.getDsAttr());
564 unsigned exp = getVal(adaptor.getExpAttr());
565
566 unsigned vmcnt = 1024;
567 Attribute load = adaptor.getLoadAttr();
568 Attribute store = adaptor.getStoreAttr();
569 if (load && store) {
570 vmcnt = getVal(load) + getVal(store);
571 } else if (load) {
572 vmcnt = getVal(load);
573 } else if (store) {
574 vmcnt = getVal(store);
575 }
576
577 FailureOr<unsigned> waitcnt = encodeWaitcnt(target, vmcnt, expcnt: exp, lgkmcnt: ds);
578 if (failed(Result: waitcnt))
579 return op.emitOpError(message: "unsupported chipset");
580
581 rewriter.replaceOpWithNewOp<ROCDL::SWaitcntOp>(op, args&: *waitcnt);
582 return success();
583 }
584};
585
586struct LDSBarrierOpLowering : public ConvertOpToLLVMPattern<LDSBarrierOp> {
587 LDSBarrierOpLowering(const LLVMTypeConverter &converter,
588 const ROCDL::TargetInfo &target)
589 : ConvertOpToLLVMPattern<LDSBarrierOp>(converter), target(target) {}
590
591 ROCDL::TargetInfo target;
592
593 LogicalResult
594 matchAndRewrite(LDSBarrierOp op, LDSBarrierOp::Adaptor adaptor,
595 ConversionPatternRewriter &rewriter) const override {
596 Location loc = op.getLoc();
597 bool hasSplitBarriers = target.has(feature: llvm::AMDGPU::FEAT_GFX12_INSTS);
598 // Inline assembly is only needed for cases where the hardware doesn't use
599 // split barriers and doesn't have FeatureBackOffBarrier (this is mainly
600 // early gfx9). In that case, we use inline assembly to bypass the
601 // conservative insertion of global memory waits at barriers, since we care
602 // more about performance than having debug watches work correctly.
603 bool requiresInlineAsm =
604 !hasSplitBarriers && !target.has(feature: llvm::AMDGPU::FEAT_BACK_OFF_BARRIER);
605
606 Attribute mmra =
607 rewriter.getAttr<LLVM::MMRATagAttr>(args: "amdgpu-synchronize-as", args: "local");
608 // Note: while there *is* a workgroup-one-as scope, this, when combined with
609 // the MMRA, will lead to the fence having no effect. This is because the
610 // codepaths for an atomic load or store will observe that a
611 // one-address-space atomic to LDS requires no synchronization because
612 // operations on LDS are totally ordered with respect to each other, and so
613 // will not emit the correct waitcnt operations that these fences are
614 // intended to produce. Therefore, we use a broader type of fence and rely
615 // on the MMRA to relax it to the semantics we want.
616 StringRef scope = "workgroup";
617
618 auto relFence = LLVM::FenceOp::create(builder&: rewriter, location: loc,
619 ordering: LLVM::AtomicOrdering::release, syncscope: scope);
620 relFence->setDiscardableAttr(name: LLVM::LLVMDialect::getMmraAttrName(), value: mmra);
621 if (requiresInlineAsm) {
622 auto asmDialectAttr = LLVM::AsmDialectAttr::get(context: rewriter.getContext(),
623 val: LLVM::AsmDialect::AD_ATT);
624 const char *asmStr = ";;;WARNING: BREAKS DEBUG WATCHES\ns_barrier";
625 const char *constraints = "";
626 LLVM::InlineAsmOp::create(
627 builder&: rewriter, location: loc,
628 /*resultTypes=*/TypeRange(), /*operands=*/ValueRange(),
629 /*asm_string=*/asmStr, constraints, /*has_side_effects=*/true,
630 /*is_align_stack=*/false, tail_call_kind: LLVM::TailCallKind::None,
631 /*convergent=*/false,
632 /*asm_dialect=*/asmDialectAttr,
633 /*operand_attrs=*/ArrayAttr());
634 } else if (!hasSplitBarriers) {
635 ROCDL::SBarrierOp::create(builder&: rewriter, location: loc);
636 } else {
637 ROCDL::BarrierSignalOp::create(builder&: rewriter, location: loc, id: -1);
638 ROCDL::BarrierWaitOp::create(builder&: rewriter, location: loc, id: -1);
639 }
640
641 auto acqFence = LLVM::FenceOp::create(builder&: rewriter, location: loc,
642 ordering: LLVM::AtomicOrdering::acquire, syncscope: scope);
643 acqFence->setDiscardableAttr(name: LLVM::LLVMDialect::getMmraAttrName(), value: mmra);
644 rewriter.replaceOp(op, newOp: acqFence);
645 return success();
646 }
647};
648
649struct SchedBarrierOpLowering : public ConvertOpToLLVMPattern<SchedBarrierOp> {
650 SchedBarrierOpLowering(const LLVMTypeConverter &converter,
651 const ROCDL::TargetInfo &target)
652 : ConvertOpToLLVMPattern<SchedBarrierOp>(converter), target(target) {}
653
654 ROCDL::TargetInfo target;
655
656 LogicalResult
657 matchAndRewrite(SchedBarrierOp op, SchedBarrierOp::Adaptor adaptor,
658 ConversionPatternRewriter &rewriter) const override {
659 rewriter.replaceOpWithNewOp<ROCDL::SchedBarrier>(op, args: op.getOptsAttr());
660 return success();
661 }
662};
663
664} // namespace
665
666/// Pack small float vector operands (fp4/fp6/fp8/bf16) into the format
667/// expected by scaled matrix multiply intrinsics (MFMA/WMMA).
668///
669/// Specifically:
670/// 1. If the element type is bfloat16, bitcast it to i16 unless rocdl intrinsic
671/// allows bf16. Newer MFMAs support bf16 types on operand, check
672/// IntrinsicsAMDGPU.td file for reference.
673/// 2. If instead we have a more than 64-bit quantity, use a <N / 4 x i32>
674/// instead, which is what the f8f6f4 intrinsics use.
675/// 3. If `input` is a vector of N <= 8 bytes, bitcast it to a (N * 8)-bit
676/// integer.
677///
678/// Note that the type of `input` has already been LLVM type converted:
679/// therefore 8-bit and smaller floats are represented as their corresponding
680/// `iN` integers.
681static Value packSmallFloatVectorOperand(ConversionPatternRewriter &rewriter,
682 Location loc, Value input,
683 bool allowBf16 = true) {
684 Type inputType = input.getType();
685 if (auto vectorType = dyn_cast<VectorType>(Val&: inputType)) {
686 if (vectorType.getElementType().isBF16() && !allowBf16)
687 return LLVM::BitcastOp::create(
688 builder&: rewriter, location: loc, res: vectorType.clone(elementType: rewriter.getI16Type()), arg: input);
689 if (vectorType.getElementType().isInteger(width: 8) &&
690 vectorType.getNumElements() <= 8)
691 return LLVM::BitcastOp::create(
692 builder&: rewriter, location: loc,
693 res: rewriter.getIntegerType(width: vectorType.getNumElements() * 8), arg: input);
694 if (isa<IntegerType>(Val: vectorType.getElementType()) &&
695 vectorType.getElementTypeBitWidth() <= 8) {
696 int64_t numWords = llvm::divideCeil(
697 Numerator: vectorType.getNumElements() * vectorType.getElementTypeBitWidth(),
698 Denominator: 32);
699 return LLVM::BitcastOp::create(
700 builder&: rewriter, location: loc, res: VectorType::get(shape: numWords, elementType: rewriter.getI32Type()),
701 arg: input);
702 }
703 }
704 return input;
705}
706
707/// Converts packed vector operands to the expected ROCDL types.
708static Value convertPackedVectorOperand(ConversionPatternRewriter &rewriter,
709 Location loc, Value input,
710 bool allowBf16 = true) {
711 Type inputType = input.getType();
712 auto vectorType = cast<VectorType>(Val&: inputType);
713 // bf16 -> i16 when not allowed (pre-gfx950).
714 if (vectorType.getElementType().isBF16() && !allowBf16)
715 return LLVM::BitcastOp::create(
716 builder&: rewriter, location: loc, res: vectorType.clone(elementType: rewriter.getI16Type()), arg: input);
717 // i8/fp8 vectors -> vector<Nxi32>.
718 if (isa<IntegerType>(Val: vectorType.getElementType()) &&
719 vectorType.getElementTypeBitWidth() <= 8) {
720 int64_t numWords = llvm::divideCeil(
721 Numerator: vectorType.getNumElements() * vectorType.getElementTypeBitWidth(), Denominator: 32);
722 Type castType = (numWords > 1)
723 ? Type{VectorType::get(shape: numWords, elementType: rewriter.getI32Type())}
724 : rewriter.getI32Type();
725 return LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: castType, arg: input);
726 }
727 return input;
728}
729
730/// Converts the scaled MFMA/WMMA operands, `scalesA` and `scalesB`, from MLIR
731/// AMDGPU dialect convention to ROCDL and LLVM AMDGPU intrinsics convention.
732///
733/// Specifically:
734/// 1. If `input` is a i8 value, zero extend it to i32
735/// 2. If `input` is a vector of length 4 or 8 and type i8, cast it to i32
736///
737/// Note that the type of `input` has already been LLVM type converted:
738/// therefore 8-bit and smaller floats are represented as their corresponding
739/// `iN` integers.
740static Value castScaleOperand(ConversionPatternRewriter &rewriter, Location loc,
741 Value input) {
742 return TypeSwitch<Type, Value>(input.getType())
743 .Case(caseFn: [&](IntegerType) {
744 // Handle scalar i8: zero extend to i32.
745 return LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: rewriter.getI32Type(),
746 arg: input);
747 })
748 .Case(caseFn: [&](VectorType vectorType) {
749 // Handle vector<4xi8> -> i32 or vector<8xi8> -> i64.
750 int64_t numElements = vectorType.getNumElements();
751 assert((numElements == 4 || numElements == 8) &&
752 "scale operand must be a vector of length 4 or 8");
753 IntegerType outputType =
754 (numElements == 4) ? rewriter.getI32Type() : rewriter.getI64Type();
755 return LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: outputType, arg: input);
756 })
757 .DefaultUnreachable(message: "unexpected input type for scale operand");
758}
759
760/// Maps f8 scale element types to WMMA scale format codes.
761static std::optional<ROCDL::WMMAMatrixScaleFormat>
762getWmmaScaleFormat(Type elemType) {
763 return TypeSwitch<Type, std::optional<ROCDL::WMMAMatrixScaleFormat>>(elemType)
764 .Case(caseFn: [](Float8E8M0FNUType) { return ROCDL::WMMAMatrixScaleFormat::e8; })
765 .Case(caseFn: [](Float8E4M3FNType) { return ROCDL::WMMAMatrixScaleFormat::e4m3; })
766 .Default(std::nullopt);
767}
768
769/// Determines the ROCDL intrinsic name for scaled WMMA based on dimensions
770/// and scale block size (16 or 32).
771static std::optional<StringRef>
772getScaledWmmaIntrinsicName(int64_t m, int64_t n, int64_t k, bool isScale16) {
773 if (m == 16 && n == 16 && k == 128)
774 return isScale16
775 ? ROCDL::wmma_scale16_f32_16x16x128_f8f6f4::getOperationName()
776 : ROCDL::wmma_scale_f32_16x16x128_f8f6f4::getOperationName();
777
778 if (m == 32 && n == 16 && k == 128)
779 return isScale16 ? ROCDL::wmma_scale16_f32_32x16x128_f4::getOperationName()
780 : ROCDL::wmma_scale_f32_32x16x128_f4::getOperationName();
781
782 return std::nullopt;
783}
784
785/// Push an input operand. If it is a float type, nothing to do. If it is
786/// an integer type, then we need to also push its signdness (1 for signed, 0
787/// for unsigned) and we need to pack the input 16xi8 vector into a 4xi32
788/// vector (or the 8xi8 vector into a 2xi32 one for gfx12+).
789/// We also need to convert bfloat inputs to i16 to account for the bfloat
790/// intrinsics having been defined before the AMD backend supported bfloat. We
791/// similarly need to pack 8-bit float types into integers as if they were i8
792/// (which they are for the backend's purposes).
793static void wmmaPushInputOperand(
794 ConversionPatternRewriter &rewriter, Location loc,
795 const TypeConverter *typeConverter, bool isUnsigned, Value llvmInput,
796 Value mlirInput, SmallVectorImpl<Value> &operands,
797 SmallVectorImpl<NamedAttribute> &attrs, StringRef attrName) {
798 Type inputType = llvmInput.getType();
799 auto vectorType = dyn_cast<VectorType>(Val&: inputType);
800 if (!vectorType) {
801 operands.push_back(Elt: llvmInput);
802 return;
803 }
804 Type elemType = vectorType.getElementType();
805 if (elemType.getIntOrFloatBitWidth() > 8) {
806 operands.push_back(Elt: llvmInput);
807 return;
808 }
809
810 // We need to check the type of the input before conversion to properly test
811 // for int8. This is because, in LLVM, fp8 type is converted to int8, so the
812 // fp8/int8 information is lost during the conversion process.
813 auto mlirInputType = cast<VectorType>(Val: mlirInput.getType());
814 bool isInputInteger = mlirInputType.getElementType().isInteger();
815 if (isInputInteger) {
816 // if element type is 8-bit signed or unsigned, ignore the isUnsigned flag
817 bool localIsUnsigned = isUnsigned;
818 if (elemType.isUnsignedInteger()) {
819 localIsUnsigned = true;
820 } else if (elemType.isSignedInteger()) {
821 localIsUnsigned = false;
822 }
823 attrs.push_back(
824 Elt: NamedAttribute(attrName, rewriter.getBoolAttr(value: !localIsUnsigned)));
825 }
826
827 int64_t numBits =
828 vectorType.getNumElements() * elemType.getIntOrFloatBitWidth();
829 Type i32 = rewriter.getI32Type();
830 Type intrinsicInType = numBits <= 32
831 ? (Type)rewriter.getIntegerType(width: numBits)
832 : (Type)VectorType::get(shape: numBits / 32, elementType: i32);
833 auto llvmIntrinsicInType = typeConverter->convertType(t: intrinsicInType);
834 Value castInput = rewriter.createOrFold<LLVM::BitcastOp>(
835 location: loc, args&: llvmIntrinsicInType, args&: llvmInput);
836 // The wave64-mode 16x16x16 intrinsics that take 4-bit integers only need
837 // (256 / 64) * 4 = 16 bits of input (on gfx12+) but take i32 arguments.
838 // Add in the zeros here.
839 if (numBits < 32)
840 castInput = LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: i32, arg: castInput);
841 operands.push_back(Elt: castInput);
842}
843
844/// Push the output operand. For many cases this is only pushing the output in
845/// the operand list. But when we have f16 -> f16 or bf16 -> bf16 intrinsics,
846/// since the same numbers of VGPRs is used, we need to decide if to store the
847/// result in the upper 16 bits of the VGPRs or in the lower part. To store the
848/// result in the lower 16 bits, set subwordOffset to 1, otherwise result will
849/// be stored it in the upper part. The subwordOffset must not be set for gfx12,
850/// as the instructions have been changed to return fewer registers instead.
851static void wmmaPushOutputOperand(ConversionPatternRewriter &rewriter,
852 Location loc,
853 const TypeConverter *typeConverter,
854 Value output, int32_t subwordOffset,
855 bool clamp, SmallVectorImpl<Value> &operands,
856 SmallVectorImpl<NamedAttribute> &attrs) {
857 Type inputType = output.getType();
858 auto vectorType = dyn_cast<VectorType>(Val&: inputType);
859 Type elemType = vectorType.getElementType();
860 operands.push_back(Elt: output);
861 if (elemType.isF16() || elemType.isBF16() || elemType.isInteger(width: 16)) {
862 attrs.push_back(
863 Elt: NamedAttribute("opsel", rewriter.getBoolAttr(value: subwordOffset)));
864 } else if (elemType.isInteger(width: 32)) {
865 attrs.push_back(Elt: NamedAttribute("clamp", rewriter.getBoolAttr(value: clamp)));
866 }
867}
868
869/// Return true if `type` is the E5M2 variant of an 8-bit float that is
870/// supported by the `_bf8` instructions on `target`.
871static bool typeIsExpectedBf8ForTarget(const ROCDL::TargetInfo &target,
872 Type type) {
873 return (target.hasFnuzFp8() && isa<Float8E5M2FNUZType>(Val: type)) ||
874 (target.hasOcpFp8() && isa<Float8E5M2Type>(Val: type));
875}
876
877/// Return true if `type` is the E4M3FN variant of an 8-bit float that is
878/// supported by the `_fp8` instructions on `target`.
879static bool typeIsExpectedFp8ForTarget(const ROCDL::TargetInfo &target,
880 Type type) {
881 return (target.hasFnuzFp8() && isa<Float8E4M3FNUZType>(Val: type)) ||
882 (target.hasOcpFp8() && isa<Float8E4M3FNType>(Val: type));
883}
884
885/// Return the `rocdl` intrinsic corresponding to a MFMA operation `mfma`
886/// if one exists. This includes checking to ensure the intrinsic is supported
887/// on the architecture you are compiling for.
888static std::optional<StringRef>
889mfmaOpToIntrinsic(MFMAOp mfma, const ROCDL::TargetInfo &target) {
890 uint32_t m = mfma.getM(), n = mfma.getN(), k = mfma.getK(),
891 b = mfma.getBlocks();
892 Type sourceElem = getElementTypeOrSelf(type: mfma.getSourceA().getType());
893 Type destElem = getElementTypeOrSelf(type: mfma.getDestC().getType());
894
895 if (sourceElem.isF32() && destElem.isF32()) {
896 if (mfma.getReducePrecision() &&
897 target.has(feature: llvm::AMDGPU::FEAT_XF32_INSTS)) {
898 if (m == 32 && n == 32 && k == 4 && b == 1)
899 return ROCDL::mfma_f32_32x32x4_xf32::getOperationName();
900 if (m == 16 && n == 16 && k == 8 && b == 1)
901 return ROCDL::mfma_f32_16x16x8_xf32::getOperationName();
902 }
903 if (m == 32 && n == 32 && k == 1 && b == 2)
904 return ROCDL::mfma_f32_32x32x1f32::getOperationName();
905 if (m == 16 && n == 16 && k == 1 && b == 4)
906 return ROCDL::mfma_f32_16x16x1f32::getOperationName();
907 if (m == 4 && n == 4 && k == 1 && b == 16)
908 return ROCDL::mfma_f32_4x4x1f32::getOperationName();
909 if (m == 32 && n == 32 && k == 2 && b == 1)
910 return ROCDL::mfma_f32_32x32x2f32::getOperationName();
911 if (m == 16 && n == 16 && k == 4 && b == 1)
912 return ROCDL::mfma_f32_16x16x4f32::getOperationName();
913 }
914
915 if (sourceElem.isF16() && destElem.isF32()) {
916 if (target.has(feature: llvm::AMDGPU::FEAT_GFX950_INSTS)) {
917 if (m == 32 && n == 32 && k == 16 && b == 1)
918 return ROCDL::mfma_f32_32x32x16_f16::getOperationName();
919 if (m == 16 && n == 16 && k == 32 && b == 1)
920 return ROCDL::mfma_f32_16x16x32_f16::getOperationName();
921 }
922 if (m == 32 && n == 32 && k == 4 && b == 2)
923 return ROCDL::mfma_f32_32x32x4f16::getOperationName();
924 if (m == 16 && n == 16 && k == 4 && b == 4)
925 return ROCDL::mfma_f32_16x16x4f16::getOperationName();
926 if (m == 4 && n == 4 && k == 4 && b == 16)
927 return ROCDL::mfma_f32_4x4x4f16::getOperationName();
928 if (m == 32 && n == 32 && k == 8 && b == 1)
929 return ROCDL::mfma_f32_32x32x8f16::getOperationName();
930 if (m == 16 && n == 16 && k == 16 && b == 1)
931 return ROCDL::mfma_f32_16x16x16f16::getOperationName();
932 }
933
934 if (sourceElem.isBF16() && destElem.isF32()) {
935 if (target.has(feature: llvm::AMDGPU::FEAT_GFX950_INSTS)) {
936 if (m == 32 && n == 32 && k == 16 && b == 1)
937 return ROCDL::mfma_f32_32x32x16_bf16::getOperationName();
938 if (m == 16 && n == 16 && k == 32 && b == 1)
939 return ROCDL::mfma_f32_16x16x32_bf16::getOperationName();
940 }
941 if (target.has(feature: llvm::AMDGPU::FEAT_GFX90A_INSTS)) {
942 if (m == 32 && n == 32 && k == 4 && b == 2)
943 return ROCDL::mfma_f32_32x32x4bf16_1k::getOperationName();
944 if (m == 16 && n == 16 && k == 4 && b == 4)
945 return ROCDL::mfma_f32_16x16x4bf16_1k::getOperationName();
946 if (m == 4 && n == 4 && k == 4 && b == 16)
947 return ROCDL::mfma_f32_4x4x4bf16_1k::getOperationName();
948 if (m == 32 && n == 32 && k == 8 && b == 1)
949 return ROCDL::mfma_f32_32x32x8bf16_1k::getOperationName();
950 if (m == 16 && n == 16 && k == 16 && b == 1)
951 return ROCDL::mfma_f32_16x16x16bf16_1k::getOperationName();
952 }
953 if (m == 32 && n == 32 && k == 2 && b == 2)
954 return ROCDL::mfma_f32_32x32x2bf16::getOperationName();
955 if (m == 16 && n == 16 && k == 2 && b == 4)
956 return ROCDL::mfma_f32_16x16x2bf16::getOperationName();
957 if (m == 4 && n == 4 && k == 2 && b == 16)
958 return ROCDL::mfma_f32_4x4x2bf16::getOperationName();
959 if (m == 32 && n == 32 && k == 4 && b == 1)
960 return ROCDL::mfma_f32_32x32x4bf16::getOperationName();
961 if (m == 16 && n == 16 && k == 8 && b == 1)
962 return ROCDL::mfma_f32_16x16x8bf16::getOperationName();
963 }
964
965 if (sourceElem.isInteger(width: 8) && destElem.isInteger(width: 32)) {
966 if (target.has(feature: llvm::AMDGPU::FEAT_GFX950_INSTS)) {
967 if (m == 32 && n == 32 && k == 32 && b == 1)
968 return ROCDL::mfma_i32_32x32x32_i8::getOperationName();
969 if (m == 16 && n == 16 && k == 64 && b == 1)
970 return ROCDL::mfma_i32_16x16x64_i8::getOperationName();
971 }
972 if (m == 32 && n == 32 && k == 4 && b == 2)
973 return ROCDL::mfma_i32_32x32x4i8::getOperationName();
974 if (m == 16 && n == 16 && k == 4 && b == 4)
975 return ROCDL::mfma_i32_16x16x4i8::getOperationName();
976 if (m == 4 && n == 4 && k == 4 && b == 16)
977 return ROCDL::mfma_i32_4x4x4i8::getOperationName();
978 if (m == 32 && n == 32 && k == 8 && b == 1)
979 return ROCDL::mfma_i32_32x32x8i8::getOperationName();
980 if (m == 16 && n == 16 && k == 16 && b == 1)
981 return ROCDL::mfma_i32_16x16x16i8::getOperationName();
982 if (m == 32 && n == 32 && k == 16 && b == 1 &&
983 target.has(feature: llvm::AMDGPU::FEAT_GFX940_INSTS))
984 return ROCDL::mfma_i32_32x32x16_i8::getOperationName();
985 if (m == 16 && n == 16 && k == 32 && b == 1 &&
986 target.has(feature: llvm::AMDGPU::FEAT_GFX940_INSTS))
987 return ROCDL::mfma_i32_16x16x32_i8::getOperationName();
988 }
989
990 if (sourceElem.isF64() && destElem.isF64() &&
991 target.has(feature: llvm::AMDGPU::FEAT_GFX90A_INSTS)) {
992 if (m == 16 && n == 16 && k == 4 && b == 1)
993 return ROCDL::mfma_f64_16x16x4f64::getOperationName();
994 if (m == 4 && n == 4 && k == 4 && b == 4)
995 return ROCDL::mfma_f64_4x4x4f64::getOperationName();
996 }
997
998 if (destElem.isF32() && typeIsExpectedBf8ForTarget(target, type: sourceElem)) {
999 // Known to be correct because there are no scalar f8 instructions and
1000 // because a length mismatch will have been caught by the verifier.
1001 Type sourceBElem =
1002 cast<VectorType>(Val: mfma.getSourceB().getType()).getElementType();
1003 if (m == 16 && n == 16 && k == 32 && b == 1) {
1004 if (typeIsExpectedBf8ForTarget(target, type: sourceBElem))
1005 return ROCDL::mfma_f32_16x16x32_bf8_bf8::getOperationName();
1006 if (typeIsExpectedFp8ForTarget(target, type: sourceBElem))
1007 return ROCDL::mfma_f32_16x16x32_bf8_fp8::getOperationName();
1008 }
1009 if (m == 32 && n == 32 && k == 16 && b == 1) {
1010 if (typeIsExpectedBf8ForTarget(target, type: sourceBElem))
1011 return ROCDL::mfma_f32_32x32x16_bf8_bf8::getOperationName();
1012 if (typeIsExpectedFp8ForTarget(target, type: sourceBElem))
1013 return ROCDL::mfma_f32_32x32x16_bf8_fp8::getOperationName();
1014 }
1015 }
1016
1017 if (destElem.isF32() && typeIsExpectedFp8ForTarget(target, type: sourceElem)) {
1018 Type sourceBElem =
1019 cast<VectorType>(Val: mfma.getSourceB().getType()).getElementType();
1020 if (m == 16 && n == 16 && k == 32 && b == 1) {
1021 if (typeIsExpectedBf8ForTarget(target, type: sourceBElem))
1022 return ROCDL::mfma_f32_16x16x32_fp8_bf8::getOperationName();
1023 if (typeIsExpectedFp8ForTarget(target, type: sourceBElem))
1024 return ROCDL::mfma_f32_16x16x32_fp8_fp8::getOperationName();
1025 }
1026 if (m == 32 && n == 32 && k == 16 && b == 1) {
1027 if (typeIsExpectedBf8ForTarget(target, type: sourceBElem))
1028 return ROCDL::mfma_f32_32x32x16_fp8_bf8::getOperationName();
1029 if (typeIsExpectedFp8ForTarget(target, type: sourceBElem))
1030 return ROCDL::mfma_f32_32x32x16_fp8_fp8::getOperationName();
1031 }
1032 }
1033
1034 return std::nullopt;
1035}
1036
1037static std::optional<ROCDL::MatrixFormat>
1038smallFloatTypeToMatrixFormat(Type mlirElemType) {
1039 return llvm::TypeSwitch<Type, std::optional<ROCDL::MatrixFormat>>(
1040 mlirElemType)
1041 .Case(caseFn: [](Float8E4M3FNType) { return ROCDL::MatrixFormat::fp8_e4m3; })
1042 .Case(caseFn: [](Float8E5M2Type) { return ROCDL::MatrixFormat::fp8_e5m2; })
1043 .Case(caseFn: [](Float6E2M3FNType) { return ROCDL::MatrixFormat::fp6_e2m3; })
1044 .Case(caseFn: [](Float6E3M2FNType) { return ROCDL::MatrixFormat::fp6_e3m2; })
1045 .Case(caseFn: [](Float4E2M1FNType) { return ROCDL::MatrixFormat::fp4_e2m1; })
1046 .Default(std::nullopt);
1047}
1048
1049/// If there is a scaled MFMA instruction for the input element types `aType`
1050/// and `bType`, output type `destType`, problem size M, N, K, and B (number of
1051/// blocks) on the given `chipset`, return a tuple consisting of the
1052/// OperationName of the intrinsic and the type codes that need to be passed to
1053/// that intrinsic. Note that this is also used to implement some un-scaled
1054/// MFMAs, since the compiler represents the ordinary instruction as a "scaled"
1055/// MFMA with a scale of 0.
1056using ScaledMFMAIntrinsic =
1057 std::tuple<StringRef, ROCDL::MatrixFormat, ROCDL::MatrixFormat>;
1058
1059static std::optional<ScaledMFMAIntrinsic>
1060mfmaOpToScaledIntrinsic(Type aType, Type bType, Type destType, uint32_t m,
1061 uint32_t n, uint32_t k, uint32_t b,
1062 const ROCDL::TargetInfo &target) {
1063 aType = getElementTypeOrSelf(type: aType);
1064 bType = getElementTypeOrSelf(type: bType);
1065 destType = getElementTypeOrSelf(type: destType);
1066
1067 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX950_INSTS))
1068 return std::nullopt;
1069 if (!isa<Float32Type>(Val: destType))
1070 return std::nullopt;
1071
1072 std::optional<ROCDL::MatrixFormat> aTypeCode =
1073 smallFloatTypeToMatrixFormat(mlirElemType: aType);
1074 std::optional<ROCDL::MatrixFormat> bTypeCode =
1075 smallFloatTypeToMatrixFormat(mlirElemType: bType);
1076 if (!aTypeCode || !bTypeCode)
1077 return std::nullopt;
1078
1079 if (m == 32 && n == 32 && k == 64 && b == 1)
1080 return std::tuple{ROCDL::mfma_scale_f32_32x32x64_f8f6f4::getOperationName(),
1081 *aTypeCode, *bTypeCode};
1082 if (m == 16 && n == 16 && k == 128 && b == 1)
1083 return std::tuple{
1084 ROCDL::mfma_scale_f32_16x16x128_f8f6f4::getOperationName(), *aTypeCode,
1085 *bTypeCode};
1086
1087 return std::nullopt;
1088}
1089
1090static std::optional<ScaledMFMAIntrinsic>
1091mfmaOpToScaledIntrinsic(MFMAOp mfma, const ROCDL::TargetInfo &target) {
1092 return mfmaOpToScaledIntrinsic(
1093 aType: mfma.getSourceA().getType(), bType: mfma.getSourceB().getType(),
1094 destType: mfma.getDestC().getType(), m: mfma.getM(), n: mfma.getN(), k: mfma.getK(),
1095 b: mfma.getBlocks(), target);
1096}
1097
1098static std::optional<ScaledMFMAIntrinsic>
1099mfmaOpToScaledIntrinsic(ScaledMFMAOp smfma, const ROCDL::TargetInfo &target) {
1100 return mfmaOpToScaledIntrinsic(aType: smfma.getSourceA().getType(),
1101 bType: smfma.getSourceB().getType(),
1102 destType: smfma.getDestC().getType(), m: smfma.getM(),
1103 n: smfma.getN(), k: smfma.getK(), b: 1u, target);
1104}
1105
1106/// Returns the `rocdl` intrinsic corresponding to a WMMA operation `wmma`
1107/// for RDNA3/4 architectures.
1108static std::optional<StringRef>
1109wmmaOpToIntrinsicRDNA(Type elemSourceType, Type elemBSourceType,
1110 Type elemDestType, uint32_t k, bool isRDNA3) {
1111 using fp8 = Float8E4M3FNType;
1112 using bf8 = Float8E5M2Type;
1113
1114 // Handle k == 16 for RDNA3/4.
1115 if (k == 16) {
1116 // Common patterns for RDNA3 and RDNA4.
1117 if (elemSourceType.isF16() && elemDestType.isF32())
1118 return ROCDL::wmma_f32_16x16x16_f16::getOperationName();
1119 if (elemSourceType.isBF16() && elemDestType.isF32())
1120 return ROCDL::wmma_f32_16x16x16_bf16::getOperationName();
1121 if (elemSourceType.isF16() && elemDestType.isF16())
1122 return ROCDL::wmma_f16_16x16x16_f16::getOperationName();
1123 if (elemSourceType.isBF16() && elemDestType.isBF16())
1124 return ROCDL::wmma_bf16_16x16x16_bf16::getOperationName();
1125 if (elemSourceType.isInteger(width: 8) && elemDestType.isInteger(width: 32))
1126 return ROCDL::wmma_i32_16x16x16_iu8::getOperationName();
1127
1128 // RDNA3 specific patterns.
1129 if (isRDNA3) {
1130 if (elemSourceType.isInteger(width: 4) && elemDestType.isInteger(width: 32))
1131 return ROCDL::wmma_i32_16x16x16_iu4::getOperationName();
1132 return std::nullopt;
1133 }
1134
1135 // RDNA4 specific patterns (fp8/bf8).
1136 if (isa<fp8>(Val: elemSourceType) && isa<fp8>(Val: elemBSourceType) &&
1137 elemDestType.isF32())
1138 return ROCDL::wmma_f32_16x16x16_fp8_fp8::getOperationName();
1139 if (isa<fp8>(Val: elemSourceType) && isa<bf8>(Val: elemBSourceType) &&
1140 elemDestType.isF32())
1141 return ROCDL::wmma_f32_16x16x16_fp8_bf8::getOperationName();
1142 if (isa<bf8>(Val: elemSourceType) && isa<bf8>(Val: elemBSourceType) &&
1143 elemDestType.isF32())
1144 return ROCDL::wmma_f32_16x16x16_bf8_bf8::getOperationName();
1145 if (isa<bf8>(Val: elemSourceType) && isa<fp8>(Val: elemBSourceType) &&
1146 elemDestType.isF32())
1147 return ROCDL::wmma_f32_16x16x16_bf8_fp8::getOperationName();
1148 if (elemSourceType.isInteger(width: 4) && elemDestType.isInteger(width: 32))
1149 return ROCDL::wmma_i32_16x16x16_iu4::getOperationName();
1150
1151 return std::nullopt;
1152 }
1153
1154 // Handle k == 32 for RDNA4.
1155 if (k == 32 && !isRDNA3) {
1156 if (elemSourceType.isInteger(width: 4) && elemDestType.isInteger(width: 32))
1157 return ROCDL::wmma_i32_16x16x32_iu4::getOperationName();
1158 }
1159
1160 return std::nullopt;
1161}
1162
1163/// Return the `rocdl` intrinsic corresponding to a WMMA operation `wmma`
1164/// for the gfx1250 architecture.
1165static std::optional<StringRef> wmmaOpToIntrinsicGfx1250(Type elemSourceType,
1166 Type elemBSourceType,
1167 Type elemDestType,
1168 uint32_t k) {
1169 using fp8 = Float8E4M3FNType;
1170 using bf8 = Float8E5M2Type;
1171
1172 if (k == 4) {
1173 if (elemSourceType.isF32() && elemDestType.isF32())
1174 return ROCDL::wmma_f32_16x16x4_f32::getOperationName();
1175
1176 return std::nullopt;
1177 }
1178
1179 if (k == 32) {
1180 if (elemSourceType.isF16() && elemDestType.isF32())
1181 return ROCDL::wmma_f32_16x16x32_f16::getOperationName();
1182 if (elemSourceType.isBF16() && elemDestType.isF32())
1183 return ROCDL::wmma_f32_16x16x32_bf16::getOperationName();
1184 if (elemSourceType.isF16() && elemDestType.isF16())
1185 return ROCDL::wmma_f16_16x16x32_f16::getOperationName();
1186 if (elemSourceType.isBF16() && elemDestType.isBF16())
1187 return ROCDL::wmma_bf16_16x16x32_bf16::getOperationName();
1188
1189 return std::nullopt;
1190 }
1191
1192 if (k == 64) {
1193 if (isa<fp8>(Val: elemSourceType) && isa<fp8>(Val: elemBSourceType)) {
1194 if (elemDestType.isF32())
1195 return ROCDL::wmma_f32_16x16x64_fp8_fp8::getOperationName();
1196 if (elemDestType.isF16())
1197 return ROCDL::wmma_f16_16x16x64_fp8_fp8::getOperationName();
1198 }
1199 if (isa<fp8>(Val: elemSourceType) && isa<bf8>(Val: elemBSourceType)) {
1200 if (elemDestType.isF32())
1201 return ROCDL::wmma_f32_16x16x64_fp8_bf8::getOperationName();
1202 if (elemDestType.isF16())
1203 return ROCDL::wmma_f16_16x16x64_fp8_bf8::getOperationName();
1204 }
1205 if (isa<bf8>(Val: elemSourceType) && isa<bf8>(Val: elemBSourceType)) {
1206 if (elemDestType.isF32())
1207 return ROCDL::wmma_f32_16x16x64_bf8_bf8::getOperationName();
1208 if (elemDestType.isF16())
1209 return ROCDL::wmma_f16_16x16x64_bf8_bf8::getOperationName();
1210 }
1211 if (isa<bf8>(Val: elemSourceType) && isa<fp8>(Val: elemBSourceType)) {
1212 if (elemDestType.isF32())
1213 return ROCDL::wmma_f32_16x16x64_bf8_fp8::getOperationName();
1214 if (elemDestType.isF16())
1215 return ROCDL::wmma_f16_16x16x64_bf8_fp8::getOperationName();
1216 }
1217 if (elemSourceType.isInteger(width: 8) && elemDestType.isInteger(width: 32))
1218 return ROCDL::wmma_i32_16x16x64_iu8::getOperationName();
1219
1220 return std::nullopt;
1221 }
1222
1223 if (k == 128) {
1224 if (isa<fp8>(Val: elemSourceType) && isa<fp8>(Val: elemBSourceType)) {
1225 if (elemDestType.isF32())
1226 return ROCDL::wmma_f32_16x16x128_fp8_fp8::getOperationName();
1227 if (elemDestType.isF16())
1228 return ROCDL::wmma_f16_16x16x128_fp8_fp8::getOperationName();
1229 }
1230 if (isa<fp8>(Val: elemSourceType) && isa<bf8>(Val: elemBSourceType)) {
1231 if (elemDestType.isF32())
1232 return ROCDL::wmma_f32_16x16x128_fp8_bf8::getOperationName();
1233 if (elemDestType.isF16())
1234 return ROCDL::wmma_f16_16x16x128_fp8_bf8::getOperationName();
1235 }
1236 if (isa<bf8>(Val: elemSourceType) && isa<bf8>(Val: elemBSourceType)) {
1237 if (elemDestType.isF32())
1238 return ROCDL::wmma_f32_16x16x128_bf8_bf8::getOperationName();
1239 if (elemDestType.isF16())
1240 return ROCDL::wmma_f16_16x16x128_bf8_bf8::getOperationName();
1241 }
1242 if (isa<bf8>(Val: elemSourceType) && isa<fp8>(Val: elemBSourceType)) {
1243 if (elemDestType.isF32())
1244 return ROCDL::wmma_f32_16x16x128_bf8_fp8::getOperationName();
1245 if (elemDestType.isF16())
1246 return ROCDL::wmma_f16_16x16x128_bf8_fp8::getOperationName();
1247 }
1248
1249 return std::nullopt;
1250 }
1251
1252 return std::nullopt;
1253}
1254
1255/// Returns the `rocdl` intrinsic corresponding to a SparseMFMA (smfmac)
1256/// operation if one exists. This includes checking to ensure the intrinsic is
1257/// supported on the architecture you are compiling for.
1258static std::optional<StringRef>
1259smfmacOpToIntrinsic(SparseMFMAOp op, const ROCDL::TargetInfo &target) {
1260 bool isGfx950 = target.has(feature: llvm::AMDGPU::FEAT_GFX950_INSTS);
1261 auto isFp8 = [&](Type t) { return typeIsExpectedFp8ForTarget(target, type: t); };
1262 auto isBf8 = [&](Type t) { return typeIsExpectedBf8ForTarget(target, type: t); };
1263
1264 uint32_t m = op.getM(), n = op.getN(), k = op.getK();
1265 Type sourceAElem = getElementTypeOrSelf(type: op.getSourceA().getType());
1266 Type sourceBElem = getElementTypeOrSelf(type: op.getSourceB().getType());
1267 Type destElem = getElementTypeOrSelf(type: op.getDestC().getType());
1268
1269 if (m == 16 && n == 16 && k == 32) {
1270 if (sourceAElem.isF16() && sourceBElem.isF16() && destElem.isF32())
1271 return ROCDL::smfmac_f32_16x16x32_f16::getOperationName();
1272 if (sourceAElem.isBF16() && sourceBElem.isBF16() && destElem.isF32())
1273 return ROCDL::smfmac_f32_16x16x32_bf16::getOperationName();
1274 }
1275
1276 if (m == 16 && n == 16 && k == 64) {
1277 if (isGfx950) {
1278 if (sourceAElem.isF16() && sourceBElem.isF16() && destElem.isF32())
1279 return ROCDL::smfmac_f32_16x16x64_f16::getOperationName();
1280 if (sourceAElem.isBF16() && sourceBElem.isBF16() && destElem.isF32())
1281 return ROCDL::smfmac_f32_16x16x64_bf16::getOperationName();
1282 }
1283 if (sourceAElem.isInteger(width: 8) && sourceBElem.isInteger(width: 8) &&
1284 destElem.isInteger(width: 32))
1285 return ROCDL::smfmac_i32_16x16x64_i8::getOperationName();
1286 if (isFp8(sourceAElem) && isFp8(sourceBElem) && destElem.isF32())
1287 return ROCDL::smfmac_f32_16x16x64_fp8_fp8::getOperationName();
1288 if (isFp8(sourceAElem) && isBf8(sourceBElem) && destElem.isF32())
1289 return ROCDL::smfmac_f32_16x16x64_fp8_bf8::getOperationName();
1290 if (isBf8(sourceAElem) && isFp8(sourceBElem) && destElem.isF32())
1291 return ROCDL::smfmac_f32_16x16x64_bf8_fp8::getOperationName();
1292 if (isBf8(sourceAElem) && isBf8(sourceBElem) && destElem.isF32())
1293 return ROCDL::smfmac_f32_16x16x64_bf8_bf8::getOperationName();
1294 }
1295
1296 if (m == 16 && n == 16 && k == 128 && isGfx950) {
1297 if (sourceAElem.isInteger(width: 8) && sourceBElem.isInteger(width: 8) &&
1298 destElem.isInteger(width: 32))
1299 return ROCDL::smfmac_i32_16x16x128_i8::getOperationName();
1300 if (isFp8(sourceAElem) && isFp8(sourceBElem) && destElem.isF32())
1301 return ROCDL::smfmac_f32_16x16x128_fp8_fp8::getOperationName();
1302 if (isFp8(sourceAElem) && isBf8(sourceBElem) && destElem.isF32())
1303 return ROCDL::smfmac_f32_16x16x128_fp8_bf8::getOperationName();
1304 if (isBf8(sourceAElem) && isFp8(sourceBElem) && destElem.isF32())
1305 return ROCDL::smfmac_f32_16x16x128_bf8_fp8::getOperationName();
1306 if (isBf8(sourceAElem) && isBf8(sourceBElem) && destElem.isF32())
1307 return ROCDL::smfmac_f32_16x16x128_bf8_bf8::getOperationName();
1308 }
1309
1310 if (m == 32 && n == 32 && k == 16) {
1311 if (sourceAElem.isF16() && sourceBElem.isF16() && destElem.isF32())
1312 return ROCDL::smfmac_f32_32x32x16_f16::getOperationName();
1313 if (sourceAElem.isBF16() && sourceBElem.isBF16() && destElem.isF32())
1314 return ROCDL::smfmac_f32_32x32x16_bf16::getOperationName();
1315 }
1316
1317 if (m == 32 && n == 32 && k == 32) {
1318 if (isGfx950) {
1319 if (sourceAElem.isF16() && sourceBElem.isF16() && destElem.isF32())
1320 return ROCDL::smfmac_f32_32x32x32_f16::getOperationName();
1321 if (sourceAElem.isBF16() && sourceBElem.isBF16() && destElem.isF32())
1322 return ROCDL::smfmac_f32_32x32x32_bf16::getOperationName();
1323 }
1324 if (sourceAElem.isInteger(width: 8) && sourceBElem.isInteger(width: 8) &&
1325 destElem.isInteger(width: 32))
1326 return ROCDL::smfmac_i32_32x32x32_i8::getOperationName();
1327 if (isFp8(sourceAElem) && isFp8(sourceBElem) && destElem.isF32())
1328 return ROCDL::smfmac_f32_32x32x32_fp8_fp8::getOperationName();
1329 if (isFp8(sourceAElem) && isBf8(sourceBElem) && destElem.isF32())
1330 return ROCDL::smfmac_f32_32x32x32_fp8_bf8::getOperationName();
1331 if (isBf8(sourceAElem) && isFp8(sourceBElem) && destElem.isF32())
1332 return ROCDL::smfmac_f32_32x32x32_bf8_fp8::getOperationName();
1333 if (isBf8(sourceAElem) && isBf8(sourceBElem) && destElem.isF32())
1334 return ROCDL::smfmac_f32_32x32x32_bf8_bf8::getOperationName();
1335 }
1336
1337 if (m == 32 && n == 32 && k == 64 && isGfx950) {
1338 if (sourceAElem.isInteger(width: 8) && sourceBElem.isInteger(width: 8) &&
1339 destElem.isInteger(width: 32))
1340 return ROCDL::smfmac_i32_32x32x64_i8::getOperationName();
1341 if (isFp8(sourceAElem) && isFp8(sourceBElem) && destElem.isF32())
1342 return ROCDL::smfmac_f32_32x32x64_fp8_fp8::getOperationName();
1343 if (isFp8(sourceAElem) && isBf8(sourceBElem) && destElem.isF32())
1344 return ROCDL::smfmac_f32_32x32x64_fp8_bf8::getOperationName();
1345 if (isBf8(sourceAElem) && isFp8(sourceBElem) && destElem.isF32())
1346 return ROCDL::smfmac_f32_32x32x64_bf8_fp8::getOperationName();
1347 if (isBf8(sourceAElem) && isBf8(sourceBElem) && destElem.isF32())
1348 return ROCDL::smfmac_f32_32x32x64_bf8_bf8::getOperationName();
1349 }
1350
1351 return std::nullopt;
1352}
1353
1354/// Returns the `rocdl` intrinsic corresponding to a WMMA operation `wmma`
1355/// if one exists. This includes checking to ensure the intrinsic is supported
1356/// on the architecture you are compiling for.
1357static std::optional<StringRef>
1358wmmaOpToIntrinsic(WMMAOp wmma, const ROCDL::TargetInfo &target) {
1359 auto sourceVectorType = cast<VectorType>(Val: wmma.getSourceA().getType());
1360 auto sourceBVectorType = cast<VectorType>(Val: wmma.getSourceB().getType());
1361 auto destVectorType = cast<VectorType>(Val: wmma.getDestC().getType());
1362 Type elemSourceType = sourceVectorType.getElementType();
1363 Type elemBSourceType = sourceBVectorType.getElementType();
1364 Type elemDestType = destVectorType.getElementType();
1365
1366 const uint32_t k = wmma.getK();
1367 const bool isRDNA3 = target.isGeneration(major: 11);
1368 const bool isRDNA4 =
1369 target.isGeneration(major: 12) && !target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS);
1370
1371 // Handle RDNA3 and RDNA4.
1372 if (isRDNA3 || isRDNA4)
1373 return wmmaOpToIntrinsicRDNA(elemSourceType, elemBSourceType, elemDestType,
1374 k, isRDNA3);
1375
1376 // Handle gfx1250.
1377 if (target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
1378 return wmmaOpToIntrinsicGfx1250(elemSourceType, elemBSourceType,
1379 elemDestType, k);
1380
1381 return std::nullopt;
1382}
1383
1384/// Returns the `rocdl` intrinsic corresponding to a SparseWMMA operation
1385/// `swmmac` if one exists. This includes checking to ensure the intrinsic is
1386/// supported on the architecture you are compiling for.
1387struct SparseWMMAOpInfo {
1388 StringRef name;
1389 bool useSign;
1390 bool useReuse;
1391 bool useClamp;
1392};
1393
1394static std::optional<SparseWMMAOpInfo>
1395sparseWMMAOpToIntrinsic(SparseWMMAOp swmmac, const ROCDL::TargetInfo &target) {
1396 Type sourceAElem = getElementTypeOrSelf(type: swmmac.getSourceA().getType());
1397 Type sourceBElem = getElementTypeOrSelf(type: swmmac.getSourceB().getType());
1398 Type destElem = getElementTypeOrSelf(type: swmmac.getDestC().getType());
1399
1400 uint32_t m = swmmac.getM(), n = swmmac.getN(), k = swmmac.getK();
1401
1402 if ((m != 16) || (n != 16))
1403 return std::nullopt;
1404
1405 const bool isRDNA4 =
1406 target.isGeneration(major: 12) && !target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS);
1407 if (isRDNA4) {
1408 if (k == 32) {
1409 if (destElem.isF32() && sourceAElem.isF16() && sourceBElem.isF16())
1410 return SparseWMMAOpInfo{
1411 .name: ROCDL::swmmac_f32_16x16x32_f16::getOperationName(), .useSign: false, .useReuse: false,
1412 .useClamp: false};
1413 if (destElem.isF32() && sourceAElem.isBF16() && sourceBElem.isBF16())
1414 return SparseWMMAOpInfo{
1415 .name: ROCDL::swmmac_f32_16x16x32_bf16::getOperationName(), .useSign: false, .useReuse: false,
1416 .useClamp: false};
1417 if (destElem.isF16() && sourceAElem.isF16() && sourceBElem.isF16())
1418 return SparseWMMAOpInfo{
1419 .name: ROCDL::swmmac_f16_16x16x32_f16::getOperationName(), .useSign: false, .useReuse: false,
1420 .useClamp: false};
1421 if (destElem.isBF16() && sourceAElem.isBF16() && sourceBElem.isBF16())
1422 return SparseWMMAOpInfo{
1423 .name: ROCDL::swmmac_bf16_16x16x32_bf16::getOperationName(), .useSign: false, .useReuse: false,
1424 .useClamp: false};
1425 if (destElem.isInteger(width: 32) && sourceAElem.isInteger(width: 8) &&
1426 sourceBElem.isInteger(width: 8))
1427 return SparseWMMAOpInfo{
1428 .name: ROCDL::swmmac_i32_16x16x32_iu8::getOperationName(), .useSign: true, .useReuse: false,
1429 .useClamp: true};
1430 if (destElem.isInteger(width: 32) && sourceAElem.isInteger(width: 4) &&
1431 sourceBElem.isInteger(width: 4))
1432 return SparseWMMAOpInfo{
1433 .name: ROCDL::swmmac_i32_16x16x32_iu4::getOperationName(), .useSign: true, .useReuse: false,
1434 .useClamp: true};
1435 if (destElem.isF32() && sourceAElem.isF8E4M3FN() &&
1436 sourceBElem.isF8E4M3FN())
1437 return SparseWMMAOpInfo{
1438 .name: ROCDL::swmmac_f32_16x16x32_fp8_fp8::getOperationName(), .useSign: false,
1439 .useReuse: false, .useClamp: false};
1440 if (destElem.isF32() && sourceAElem.isF8E4M3FN() &&
1441 sourceBElem.isF8E5M2())
1442 return SparseWMMAOpInfo{
1443 .name: ROCDL::swmmac_f32_16x16x32_fp8_bf8::getOperationName(), .useSign: false,
1444 .useReuse: false, .useClamp: false};
1445 if (destElem.isF32() && sourceAElem.isF8E5M2() &&
1446 sourceBElem.isF8E4M3FN())
1447 return SparseWMMAOpInfo{
1448 .name: ROCDL::swmmac_f32_16x16x32_bf8_fp8::getOperationName(), .useSign: false,
1449 .useReuse: false, .useClamp: false};
1450 if (destElem.isF32() && sourceAElem.isF8E5M2() && sourceBElem.isF8E5M2())
1451 return SparseWMMAOpInfo{
1452 .name: ROCDL::swmmac_f32_16x16x32_bf8_bf8::getOperationName(), .useSign: false,
1453 .useReuse: false, .useClamp: false};
1454 }
1455 if (k == 64) {
1456 if (destElem.isInteger(width: 32) && sourceAElem.isInteger(width: 4) &&
1457 sourceBElem.isInteger(width: 4))
1458 return SparseWMMAOpInfo{
1459 .name: ROCDL::swmmac_i32_16x16x64_iu4::getOperationName(), .useSign: true, .useReuse: false,
1460 .useClamp: true};
1461 }
1462 }
1463
1464 const bool isGFX1250 = target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS);
1465 const bool isWavesize64 = swmmac.getWave64();
1466 if (isGFX1250 && !isWavesize64) {
1467 if (k == 64) {
1468 if (destElem.isF32() && sourceAElem.isF16() && sourceBElem.isF16())
1469 return SparseWMMAOpInfo{
1470 .name: ROCDL::swmmac_f32_16x16x64_f16::getOperationName(), .useSign: true, .useReuse: true,
1471 .useClamp: false};
1472 if (destElem.isF32() && sourceAElem.isBF16() && sourceBElem.isBF16())
1473 return SparseWMMAOpInfo{
1474 .name: ROCDL::swmmac_f32_16x16x64_bf16::getOperationName(), .useSign: true, .useReuse: true,
1475 .useClamp: false};
1476 if (destElem.isF16() && sourceAElem.isF16() && sourceBElem.isF16())
1477 return SparseWMMAOpInfo{
1478 .name: ROCDL::swmmac_f16_16x16x64_f16::getOperationName(), .useSign: true, .useReuse: true,
1479 .useClamp: false};
1480 if (destElem.isBF16() && sourceAElem.isBF16() && sourceBElem.isBF16())
1481 return SparseWMMAOpInfo{
1482 .name: ROCDL::swmmac_bf16_16x16x64_bf16::getOperationName(), .useSign: true, .useReuse: true,
1483 .useClamp: false};
1484 }
1485 if (k == 128) {
1486 if (destElem.isF32() && sourceAElem.isF8E4M3FN() &&
1487 sourceBElem.isF8E4M3FN())
1488 return SparseWMMAOpInfo{
1489 .name: ROCDL::swmmac_f32_16x16x128_fp8_fp8::getOperationName(), .useSign: false,
1490 .useReuse: true, .useClamp: false};
1491 if (destElem.isF32() && sourceAElem.isF8E4M3FN() &&
1492 sourceBElem.isF8E5M2())
1493 return SparseWMMAOpInfo{
1494 .name: ROCDL::swmmac_f32_16x16x128_fp8_bf8::getOperationName(), .useSign: false,
1495 .useReuse: true, .useClamp: false};
1496 if (destElem.isF32() && sourceAElem.isF8E5M2() &&
1497 sourceBElem.isF8E4M3FN())
1498 return SparseWMMAOpInfo{
1499 .name: ROCDL::swmmac_f32_16x16x128_bf8_fp8::getOperationName(), .useSign: false,
1500 .useReuse: true, .useClamp: false};
1501 if (destElem.isF32() && sourceAElem.isF8E5M2() && sourceBElem.isF8E5M2())
1502 return SparseWMMAOpInfo{
1503 .name: ROCDL::swmmac_f32_16x16x128_bf8_bf8::getOperationName(), .useSign: false,
1504 .useReuse: true, .useClamp: false};
1505 if (destElem.isF16() && sourceAElem.isF8E4M3FN() &&
1506 sourceBElem.isF8E4M3FN())
1507 return SparseWMMAOpInfo{
1508 .name: ROCDL::swmmac_f16_16x16x128_fp8_fp8::getOperationName(), .useSign: false,
1509 .useReuse: true, .useClamp: false};
1510 if (destElem.isF16() && sourceAElem.isF8E4M3FN() &&
1511 sourceBElem.isF8E5M2())
1512 return SparseWMMAOpInfo{
1513 .name: ROCDL::swmmac_f16_16x16x128_fp8_bf8::getOperationName(), .useSign: false,
1514 .useReuse: true, .useClamp: false};
1515 if (destElem.isF16() && sourceAElem.isF8E5M2() &&
1516 sourceBElem.isF8E4M3FN())
1517 return SparseWMMAOpInfo{
1518 .name: ROCDL::swmmac_f16_16x16x128_bf8_fp8::getOperationName(), .useSign: false,
1519 .useReuse: true, .useClamp: false};
1520 if (destElem.isF16() && sourceAElem.isF8E5M2() && sourceBElem.isF8E5M2())
1521 return SparseWMMAOpInfo{
1522 .name: ROCDL::swmmac_f16_16x16x128_bf8_bf8::getOperationName(), .useSign: false,
1523 .useReuse: true, .useClamp: false};
1524 if (destElem.isF16() && sourceAElem.isInteger(width: 8) &&
1525 sourceBElem.isInteger(width: 8))
1526 return SparseWMMAOpInfo{
1527 .name: ROCDL::swmmac_f16_16x16x128_bf8_bf8::getOperationName(), .useSign: false,
1528 .useReuse: true, .useClamp: false};
1529 if (destElem.isInteger(width: 32) && sourceAElem.isInteger(width: 8) &&
1530 sourceBElem.isInteger(width: 8))
1531 return SparseWMMAOpInfo{
1532 .name: ROCDL::swmmac_i32_16x16x128_iu8::getOperationName(), .useSign: true, .useReuse: true,
1533 .useClamp: true};
1534 }
1535 }
1536
1537 return std::nullopt;
1538}
1539
1540namespace {
1541struct MFMAOpLowering : public ConvertOpToLLVMPattern<MFMAOp> {
1542 MFMAOpLowering(const LLVMTypeConverter &converter,
1543 const ROCDL::TargetInfo &target)
1544 : ConvertOpToLLVMPattern<MFMAOp>(converter), target(target) {}
1545
1546 ROCDL::TargetInfo target;
1547
1548 LogicalResult
1549 matchAndRewrite(MFMAOp op, MFMAOpAdaptor adaptor,
1550 ConversionPatternRewriter &rewriter) const override {
1551 Location loc = op.getLoc();
1552 Type destElem = getElementTypeOrSelf(type: op.getDestD().getType());
1553 Type outType = typeConverter->convertType(t: op.getDestD().getType());
1554 Type intrinsicOutType = outType;
1555 if (auto outVecType = dyn_cast<VectorType>(Val&: outType))
1556 if (outVecType.getElementType().isBF16())
1557 intrinsicOutType = outVecType.clone(elementType: rewriter.getI16Type());
1558
1559 if (!target.has(feature: llvm::AMDGPU::FEAT_MAI_INSTS))
1560 return op->emitOpError(message: "MFMA only supported on gfx908+");
1561 uint32_t getBlgpField = static_cast<uint32_t>(op.getBlgp());
1562 if (op.getNegateA() || op.getNegateB() || op.getNegateC()) {
1563 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX940_INSTS))
1564 return op.emitOpError(message: "negation unsupported on older than gfx942");
1565 getBlgpField |=
1566 op.getNegateA() | (op.getNegateB() << 1) | (op.getNegateC() << 2);
1567 }
1568 std::optional<StringRef> maybeIntrinsic = mfmaOpToIntrinsic(mfma: op, target);
1569 std::optional<ScaledMFMAIntrinsic> maybeScaledIntrinsic =
1570 mfmaOpToScaledIntrinsic(mfma: op, target);
1571 if (!maybeIntrinsic.has_value() && !maybeScaledIntrinsic.has_value())
1572 return op.emitOpError(message: "no intrinsic matching MFMA size on given chipset");
1573
1574 bool isScaled =
1575 !maybeIntrinsic.has_value() && maybeScaledIntrinsic.has_value();
1576 if (isScaled &&
1577 (adaptor.getAbid() > 0 || getBlgpField > 0 || op.getCbsz() > 0)) {
1578 return op.emitOpError(
1579 message: "non-default abid, blgp, and cbsz aren't supported on MFMAs that can "
1580 "be scaled as those fields are used for type information");
1581 }
1582
1583 StringRef intrinsicName =
1584 isScaled ? std::get<0>(t&: *maybeScaledIntrinsic) : *maybeIntrinsic;
1585 // Determine if we can use bf16 in the intrinsic. Newer MFMAs in gfx950+
1586 // allows bf16 as the input. For reference check IntrinsicsAMDGPU.td file.
1587 bool allowBf16 = [&]() {
1588 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX950_INSTS))
1589 return false;
1590 if (isScaled)
1591 return true;
1592 return intrinsicName.contains(Other: "16x16x32.bf16") ||
1593 intrinsicName.contains(Other: "32x32x16.bf16");
1594 }();
1595 OperationState loweredOp(loc, intrinsicName);
1596 loweredOp.addTypes(newTypes: intrinsicOutType);
1597 loweredOp.addOperands(newOperands: {packSmallFloatVectorOperand(
1598 rewriter, loc, input: adaptor.getSourceA(), allowBf16),
1599 packSmallFloatVectorOperand(
1600 rewriter, loc, input: adaptor.getSourceB(), allowBf16),
1601 adaptor.getDestC()});
1602 if (isScaled) {
1603 Value zero = createI32Constant(rewriter, loc, value: 0);
1604 auto [_scaledName, aTypeCode, bTypeCode] = *maybeScaledIntrinsic;
1605 loweredOp.addOperands(newOperands: {/*scale A=*/zero, /*scale B=*/zero});
1606 loweredOp.addAttributes(
1607 newAttributes: {{"cbsz",
1608 ROCDL::MatrixFormatAttr::get(context: rewriter.getContext(), value: aTypeCode)},
1609 {"blgp",
1610 ROCDL::MatrixFormatAttr::get(context: rewriter.getContext(), value: bTypeCode)},
1611 {"opselA", rewriter.getI32IntegerAttr(value: 0)},
1612 {"opselB", rewriter.getI32IntegerAttr(value: 0)}});
1613 } else {
1614 Attribute blgpAttr =
1615 destElem.isF64()
1616 ? Attribute(ROCDL::MFMANegModifierAttr::get(
1617 context: rewriter.getContext(),
1618 value: static_cast<ROCDL::MFMANegModifier>(getBlgpField)))
1619 : Attribute(ROCDL::MFMAPermBAttr::get(
1620 context: rewriter.getContext(),
1621 value: static_cast<ROCDL::MFMAPermB>(getBlgpField)));
1622 loweredOp.addAttributes(
1623 newAttributes: {{"cbsz", rewriter.getI32IntegerAttr(value: op.getCbsz())},
1624 {"abid", rewriter.getI32IntegerAttr(value: op.getAbid())},
1625 {"blgp", blgpAttr}});
1626 };
1627 Value lowered = rewriter.create(state: loweredOp)->getResult(idx: 0);
1628 if (outType != intrinsicOutType)
1629 lowered = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: outType, arg: lowered);
1630 rewriter.replaceOp(op, newValues: lowered);
1631 return success();
1632 }
1633};
1634
1635struct ScaledMFMAOpLowering : public ConvertOpToLLVMPattern<ScaledMFMAOp> {
1636 ScaledMFMAOpLowering(const LLVMTypeConverter &converter,
1637 const ROCDL::TargetInfo &target)
1638 : ConvertOpToLLVMPattern(converter), target(target) {}
1639
1640 ROCDL::TargetInfo target;
1641
1642 LogicalResult
1643 matchAndRewrite(ScaledMFMAOp op, ScaledMFMAOpAdaptor adaptor,
1644 ConversionPatternRewriter &rewriter) const override {
1645 Location loc = op.getLoc();
1646 Type intrinsicOutType = typeConverter->convertType(t: op.getDestD().getType());
1647
1648 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX950_INSTS))
1649 return op->emitOpError(message: "scaled MFMA only supported on gfx908+");
1650 std::optional<ScaledMFMAIntrinsic> maybeScaledIntrinsic =
1651 mfmaOpToScaledIntrinsic(smfma: op, target);
1652 if (!maybeScaledIntrinsic.has_value())
1653 return op.emitOpError(
1654 message: "no intrinsic matching scaled MFMA size on given chipset");
1655
1656 auto [intrinsicName, aTypeCode, bTypeCode] = *maybeScaledIntrinsic;
1657 OperationState loweredOp(loc, intrinsicName);
1658 loweredOp.addTypes(newTypes: intrinsicOutType);
1659 loweredOp.addOperands(
1660 newOperands: {packSmallFloatVectorOperand(rewriter, loc, input: adaptor.getSourceA()),
1661 packSmallFloatVectorOperand(rewriter, loc, input: adaptor.getSourceB()),
1662 adaptor.getDestC()});
1663 loweredOp.addOperands(
1664 newOperands: {/*scales A*/
1665 castScaleOperand(rewriter, loc, input: adaptor.getScalesA()),
1666 /*scales B*/
1667 castScaleOperand(rewriter, loc, input: adaptor.getScalesB())});
1668 loweredOp.addAttributes(
1669 newAttributes: {{"cbsz",
1670 ROCDL::MatrixFormatAttr::get(context: rewriter.getContext(), value: aTypeCode)},
1671 {"blgp",
1672 ROCDL::MatrixFormatAttr::get(context: rewriter.getContext(), value: bTypeCode)},
1673 {"opselA", rewriter.getI32IntegerAttr(value: adaptor.getScalesIdxA())},
1674 {"opselB", rewriter.getI32IntegerAttr(value: adaptor.getScalesIdxB())}});
1675
1676 Value lowered = rewriter.create(state: loweredOp)->getResult(idx: 0);
1677 rewriter.replaceOp(op, newValues: lowered);
1678 return success();
1679 }
1680};
1681
1682struct SparseMFMAOpLowering : public ConvertOpToLLVMPattern<SparseMFMAOp> {
1683 SparseMFMAOpLowering(const LLVMTypeConverter &converter,
1684 const ROCDL::TargetInfo &target)
1685 : ConvertOpToLLVMPattern<SparseMFMAOp>(converter), target(target) {}
1686
1687 ROCDL::TargetInfo target;
1688
1689 LogicalResult
1690 matchAndRewrite(SparseMFMAOp op, SparseMFMAOpAdaptor adaptor,
1691 ConversionPatternRewriter &rewriter) const override {
1692 Location loc = op.getLoc();
1693 auto outType =
1694 typeConverter->convertType<VectorType>(t: op.getDestC().getType());
1695 if (!outType)
1696 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
1697
1698 // smfmac is supported on gfx942 and gfx950.
1699 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX940_INSTS))
1700 return op->emitOpError(message: "sparse MFMA (smfmac) only supported on gfx942+");
1701
1702 std::optional<StringRef> maybeIntrinsic = smfmacOpToIntrinsic(op, target);
1703 if (!maybeIntrinsic.has_value())
1704 return op.emitOpError(
1705 message: "no intrinsic matching sparse MFMA on the given chipset");
1706 bool isGfx942BF16 =
1707 (*maybeIntrinsic ==
1708 ROCDL::smfmac_f32_16x16x32_bf16::getOperationName() ||
1709 *maybeIntrinsic ==
1710 ROCDL::smfmac_f32_32x32x16_bf16::getOperationName());
1711 bool isGfx950 =
1712 (target.has(feature: llvm::AMDGPU::FEAT_GFX950_INSTS)) && !isGfx942BF16;
1713
1714 Value a = convertPackedVectorOperand(rewriter, loc, input: adaptor.getSourceA(),
1715 allowBf16: isGfx950);
1716 Value b = convertPackedVectorOperand(rewriter, loc, input: adaptor.getSourceB(),
1717 allowBf16: isGfx950);
1718 Value c = adaptor.getDestC();
1719
1720 // Bitcast sparse indices from vector<4xi8> or vector<2xi16> to i32.
1721 // gfx950 8-bit variants already carry the index as i32; skip the bitcast.
1722 Value sparseIdx = adaptor.getSparseIdx();
1723 Type i32Type = rewriter.getI32Type();
1724 if (sparseIdx.getType() != i32Type)
1725 sparseIdx = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: i32Type, arg: sparseIdx);
1726
1727 OperationState loweredOp(loc, maybeIntrinsic.value());
1728 loweredOp.addTypes(newTypes: outType);
1729 loweredOp.addOperands(newOperands: {a, b, c, sparseIdx});
1730 loweredOp.addAttributes(
1731 newAttributes: {{"cbsz", rewriter.getI32IntegerAttr(value: op.getCbsz())},
1732 {"abid", rewriter.getI32IntegerAttr(value: op.getAbid())}});
1733 Value lowered = rewriter.create(state: loweredOp)->getResult(idx: 0);
1734 rewriter.replaceOp(op, newValues: lowered);
1735 return success();
1736 }
1737};
1738
1739struct WMMAOpLowering : public ConvertOpToLLVMPattern<WMMAOp> {
1740 WMMAOpLowering(const LLVMTypeConverter &converter,
1741 const ROCDL::TargetInfo &target)
1742 : ConvertOpToLLVMPattern<WMMAOp>(converter), target(target) {}
1743
1744 ROCDL::TargetInfo target;
1745
1746 LogicalResult
1747 matchAndRewrite(WMMAOp op, WMMAOpAdaptor adaptor,
1748 ConversionPatternRewriter &rewriter) const override {
1749 Location loc = op.getLoc();
1750 auto outType =
1751 typeConverter->convertType<VectorType>(t: op.getDestD().getType());
1752 if (!outType)
1753 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
1754
1755 if (!target.isGeneration(major: 11) && !target.isGeneration(major: 12))
1756 return op->emitOpError(message: "WMMA only supported on gfx11 and gfx12");
1757
1758 bool isGFX1250 = target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS);
1759
1760 // The WMMA operations represent vectors of bf16s as vectors of i16s
1761 // (except on gfx1250), so we need to bitcast bfloats to i16 and then
1762 // bitcast them back.
1763 auto aType = cast<VectorType>(Val: adaptor.getSourceA().getType());
1764 auto bType = cast<VectorType>(Val: adaptor.getSourceB().getType());
1765 auto destCType = cast<VectorType>(Val: adaptor.getDestC().getType());
1766 bool castAToI16 = aType.getElementType().isBF16() && !isGFX1250;
1767 bool castBToI16 = bType.getElementType().isBF16() && !isGFX1250;
1768 bool castDestCToI16 = destCType.getElementType().isBF16() && !isGFX1250;
1769 bool castOutToI16 = outType.getElementType().isBF16() && !isGFX1250;
1770 VectorType rawOutType = outType;
1771 if (castOutToI16)
1772 rawOutType = outType.clone(elementType: rewriter.getI16Type());
1773 Value a = adaptor.getSourceA();
1774 if (castAToI16)
1775 a = LLVM::BitcastOp::create(builder&: rewriter, location: loc,
1776 res: aType.clone(elementType: rewriter.getI16Type()), arg: a);
1777 Value b = adaptor.getSourceB();
1778 if (castBToI16)
1779 b = LLVM::BitcastOp::create(builder&: rewriter, location: loc,
1780 res: bType.clone(elementType: rewriter.getI16Type()), arg: b);
1781 Value destC = adaptor.getDestC();
1782 if (castDestCToI16)
1783 destC = LLVM::BitcastOp::create(
1784 builder&: rewriter, location: loc, res: destCType.clone(elementType: rewriter.getI16Type()), arg: destC);
1785
1786 std::optional<StringRef> maybeIntrinsic = wmmaOpToIntrinsic(wmma: op, target);
1787
1788 if (!maybeIntrinsic.has_value())
1789 return op.emitOpError(message: "no intrinsic matching WMMA on the given chipset");
1790
1791 if (target.has(feature: llvm::AMDGPU::FEAT_GFX12_INSTS) &&
1792 op.getSubwordOffset() != 0)
1793 return op.emitOpError(message: "subwordOffset not supported on gfx12+");
1794
1795 SmallVector<Value, 4> operands;
1796 SmallVector<NamedAttribute, 4> attrs;
1797 wmmaPushInputOperand(rewriter, loc, typeConverter, isUnsigned: op.getUnsignedA(), llvmInput: a,
1798 mlirInput: op.getSourceA(), operands, attrs, attrName: "signA");
1799 wmmaPushInputOperand(rewriter, loc, typeConverter, isUnsigned: op.getUnsignedB(), llvmInput: b,
1800 mlirInput: op.getSourceB(), operands, attrs, attrName: "signB");
1801 wmmaPushOutputOperand(rewriter, loc, typeConverter, output: destC,
1802 subwordOffset: op.getSubwordOffset(), clamp: op.getClamp(), operands,
1803 attrs);
1804
1805 OperationState loweredOp(loc, *maybeIntrinsic);
1806 loweredOp.addTypes(newTypes: rawOutType);
1807 loweredOp.addOperands(newOperands: operands);
1808 loweredOp.addAttributes(newAttributes: attrs);
1809 Operation *lowered = rewriter.create(state: loweredOp);
1810
1811 Operation *maybeCastBack = lowered;
1812 if (rawOutType != outType)
1813 maybeCastBack = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: outType,
1814 arg: lowered->getResult(idx: 0));
1815 rewriter.replaceOp(op, newValues: maybeCastBack->getResults());
1816
1817 return success();
1818 }
1819};
1820
1821enum class DotFamily {
1822 /// ROCDL_Dot_IntrOp: single `clamp` attribute.
1823 Clamp,
1824 /// ROCDL_Dot_NoClamp_IntrOp: no attributes.
1825 NoClamp,
1826 /// ROCDL_Sudot_IntrOp: `signA`, `signB`, and `clamp` attributes.
1827 Sudot,
1828};
1829
1830static std::optional<std::pair<StringRef, DotFamily>>
1831dotOpToIntrinsic(DotOp op, const ROCDL::TargetInfo &target) {
1832 Type aElem = cast<VectorType>(Val: op.getSourceA().getType()).getElementType();
1833 Type bElem = cast<VectorType>(Val: op.getSourceB().getType()).getElementType();
1834 Type dest = op.getDestC().getType();
1835 bool uA = op.getUnsignedA();
1836 bool uB = op.getUnsignedB();
1837
1838 // f16 x f16 -> f32 / f16.
1839 if (aElem.isF16() && bElem.isF16()) {
1840 if (dest.isF32() && target.has(feature: llvm::AMDGPU::FEAT_DOT10_INSTS))
1841 return {{ROCDL::fdot2::getOperationName(), DotFamily::Clamp}};
1842 if (dest.isF16() && target.has(feature: llvm::AMDGPU::FEAT_DOT9_INSTS))
1843 return {{ROCDL::fdot2_f16_f16::getOperationName(), DotFamily::NoClamp}};
1844 return std::nullopt;
1845 }
1846
1847 // bf16 x bf16 -> f32 / bf16.
1848 if (aElem.isBF16() && bElem.isBF16()) {
1849 if (dest.isF32() && target.has(feature: llvm::AMDGPU::FEAT_DOT12_INSTS))
1850 return {{ROCDL::fdot2_f32_bf16::getOperationName(), DotFamily::Clamp}};
1851 if (dest.isBF16() && target.has(feature: llvm::AMDGPU::FEAT_DOT9_INSTS))
1852 return {{ROCDL::fdot2_bf16_bf16::getOperationName(), DotFamily::NoClamp}};
1853 return std::nullopt;
1854 }
1855
1856 // Integer sources -> i32.
1857 if (isa<IntegerType>(Val: aElem) && isa<IntegerType>(Val: bElem) &&
1858 dest.isInteger(width: 32)) {
1859 bool mixedSign = (uA != uB);
1860 unsigned elemWidth = aElem.getIntOrFloatBitWidth();
1861
1862 if (mixedSign) {
1863 if (!target.has(feature: llvm::AMDGPU::FEAT_DOT8_INSTS))
1864 return std::nullopt;
1865 StringRef name;
1866 switch (elemWidth) {
1867 case 8:
1868 name = ROCDL::sudot4::getOperationName();
1869 break;
1870 case 4:
1871 name = ROCDL::sudot8::getOperationName();
1872 break;
1873 default:
1874 return std::nullopt;
1875 }
1876 return {{name, DotFamily::Sudot}};
1877 }
1878
1879 StringRef name;
1880 bool supported = false;
1881 switch (elemWidth) {
1882 case 16:
1883 supported = target.has(feature: llvm::AMDGPU::FEAT_DOT2_INSTS);
1884 name = uA ? ROCDL::udot2::getOperationName()
1885 : ROCDL::sdot2::getOperationName();
1886 break;
1887 case 8:
1888 supported = uA ? target.has(feature: llvm::AMDGPU::FEAT_DOT7_INSTS)
1889 : target.has(feature: llvm::AMDGPU::FEAT_DOT1_INSTS) ||
1890 target.has(feature: llvm::AMDGPU::FEAT_DOT8_INSTS);
1891 name = uA ? ROCDL::udot4::getOperationName()
1892 : ROCDL::sdot4::getOperationName();
1893 break;
1894 case 4:
1895 supported = uA ? target.has(feature: llvm::AMDGPU::FEAT_DOT7_INSTS)
1896 : target.has(feature: llvm::AMDGPU::FEAT_DOT1_INSTS) ||
1897 target.has(feature: llvm::AMDGPU::FEAT_DOT8_INSTS);
1898 name = uA ? ROCDL::udot8::getOperationName()
1899 : ROCDL::sdot8::getOperationName();
1900 break;
1901 default:
1902 return std::nullopt;
1903 }
1904 if (!supported)
1905 return std::nullopt;
1906 return {{name, DotFamily::Clamp}};
1907 }
1908
1909 // fp8/bf8 x fp8/bf8 -> f32.
1910 bool aIsFp8 = isa<Float8E4M3FNType>(Val: aElem);
1911 bool aIsBf8 = isa<Float8E5M2Type>(Val: aElem);
1912 bool bIsFp8 = isa<Float8E4M3FNType>(Val: bElem);
1913 bool bIsBf8 = isa<Float8E5M2Type>(Val: bElem);
1914 if ((aIsFp8 || aIsBf8) && (bIsFp8 || bIsBf8) && dest.isF32()) {
1915 if (!target.has(feature: llvm::AMDGPU::FEAT_DOT11_INSTS))
1916 return std::nullopt;
1917 StringRef name;
1918 if (aIsFp8 && bIsFp8)
1919 name = ROCDL::dot4_f32_fp8_fp8::getOperationName();
1920 else if (aIsFp8 && bIsBf8)
1921 name = ROCDL::dot4_f32_fp8_bf8::getOperationName();
1922 else if (aIsBf8 && bIsFp8)
1923 name = ROCDL::dot4_f32_bf8_fp8::getOperationName();
1924 else
1925 name = ROCDL::dot4_f32_bf8_bf8::getOperationName();
1926 return {{name, DotFamily::NoClamp}};
1927 }
1928
1929 return std::nullopt;
1930}
1931
1932struct DotOpLowering : public ConvertOpToLLVMPattern<DotOp> {
1933 DotOpLowering(const LLVMTypeConverter &converter,
1934 const ROCDL::TargetInfo &target)
1935 : ConvertOpToLLVMPattern<DotOp>(converter), target(target) {}
1936
1937 ROCDL::TargetInfo target;
1938
1939 LogicalResult
1940 matchAndRewrite(DotOp op, DotOpAdaptor adaptor,
1941 ConversionPatternRewriter &rewriter) const override {
1942 Location loc = op.getLoc();
1943
1944 std::optional<std::pair<StringRef, DotFamily>> maybeIntrinsic =
1945 dotOpToIntrinsic(op, target);
1946 if (!maybeIntrinsic)
1947 return op.emitOpError(message: "no intrinsic matching dot on the given chipset: ")
1948 << op.getSourceA().getType() << " * " << op.getSourceB().getType()
1949 << " + " << op.getDestC().getType();
1950
1951 auto [intrinsicName, family] = maybeIntrinsic.value();
1952
1953 Value a = convertPackedVectorOperand(rewriter, loc, input: adaptor.getSourceA());
1954 Value b = convertPackedVectorOperand(rewriter, loc, input: adaptor.getSourceB());
1955 Value c = adaptor.getDestC();
1956
1957 SmallVector<NamedAttribute, 3> attrs;
1958 if (family == DotFamily::Sudot) {
1959 attrs.push_back(Elt: rewriter.getNamedAttr(
1960 name: "signA", val: rewriter.getBoolAttr(value: !op.getUnsignedA())));
1961 attrs.push_back(Elt: rewriter.getNamedAttr(
1962 name: "signB", val: rewriter.getBoolAttr(value: !op.getUnsignedB())));
1963 }
1964
1965 if (family != DotFamily::NoClamp && op.getClamp())
1966 attrs.push_back(
1967 Elt: rewriter.getNamedAttr(name: "clamp", val: rewriter.getBoolAttr(value: true)));
1968
1969 Type resultType = typeConverter->convertType(t: op.getDestD().getType());
1970
1971 OperationState loweredOp(loc, intrinsicName);
1972 loweredOp.addTypes(newTypes: resultType);
1973 loweredOp.addOperands(newOperands: {a, b, c});
1974 loweredOp.addAttributes(newAttributes: attrs);
1975 Operation *lowered = rewriter.create(state: loweredOp);
1976 rewriter.replaceOp(op, newValues: lowered->getResults());
1977 return success();
1978 }
1979};
1980
1981struct SparseWMMAOpLowering : public ConvertOpToLLVMPattern<SparseWMMAOp> {
1982 SparseWMMAOpLowering(const LLVMTypeConverter &converter,
1983 const ROCDL::TargetInfo &target)
1984 : ConvertOpToLLVMPattern<SparseWMMAOp>(converter), target(target) {}
1985
1986 ROCDL::TargetInfo target;
1987
1988 LogicalResult
1989 matchAndRewrite(SparseWMMAOp op, SparseWMMAOpAdaptor adaptor,
1990 ConversionPatternRewriter &rewriter) const override {
1991 Location loc = op.getLoc();
1992 auto outType =
1993 typeConverter->convertType<VectorType>(t: op.getDestD().getType());
1994 if (!outType)
1995 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
1996
1997 std::optional<SparseWMMAOpInfo> maybeIntrinsic =
1998 sparseWMMAOpToIntrinsic(swmmac: op, target);
1999
2000 if (!maybeIntrinsic.has_value())
2001 return op.emitOpError(
2002 message: "no intrinsic matching Sparse WMMA on the given chipset");
2003 SparseWMMAOpInfo intrinsic = maybeIntrinsic.value();
2004
2005 SmallVector<NamedAttribute> attrs;
2006
2007 if ((op.getUnsignedA() || op.getUnsignedB()) && !intrinsic.useSign)
2008 return op->emitOpError(message: "intrinsic doesn't support unsign");
2009 if (intrinsic.useSign) {
2010 if (auto attr = op.getUnsignedAAttr())
2011 attrs.push_back(Elt: {"signA", attr});
2012 if (auto attr = op.getUnsignedBAttr())
2013 attrs.push_back(Elt: {"signB", attr});
2014 }
2015
2016 if ((op.getReuseA() || op.getReuseB()) && !intrinsic.useReuse)
2017 return op->emitOpError(message: "intrinsic doesn't support reuse");
2018 if (intrinsic.useReuse) {
2019 if (auto attr = op.getReuseAAttr())
2020 attrs.push_back(Elt: {"reuseA", attr});
2021 if (auto attr = op.getReuseBAttr())
2022 attrs.push_back(Elt: {"reuseB", attr});
2023 }
2024
2025 if (op.getClamp() && !intrinsic.useClamp)
2026 return op->emitOpError(message: "intrinsic doesn't support clamp");
2027 if (intrinsic.useClamp && op.getClampAttr())
2028 attrs.push_back(Elt: {"clamp", op.getClampAttr()});
2029
2030 const bool isGFX1250orHigher = target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS);
2031 Value a = convertPackedVectorOperand(rewriter, loc, input: adaptor.getSourceA(),
2032 allowBf16: isGFX1250orHigher);
2033 Value b = convertPackedVectorOperand(rewriter, loc, input: adaptor.getSourceB(),
2034 allowBf16: isGFX1250orHigher);
2035 Value c = adaptor.getDestC();
2036 VectorType rawOutType = outType;
2037 if (!isGFX1250orHigher) {
2038 c = convertPackedVectorOperand(rewriter, loc, input: adaptor.getDestC(), allowBf16: false);
2039 rawOutType = cast<VectorType>(Val: c.getType());
2040 }
2041
2042 // Bitcast sparse indices from vector<4xi8> to i32.
2043 Value sparseIdx = LLVM::BitcastOp::create(
2044 builder&: rewriter, location: loc, res: rewriter.getI32Type(), arg: adaptor.getSparseIdx());
2045
2046 OperationState loweredOp(loc, intrinsic.name);
2047 loweredOp.addTypes(newTypes: rawOutType);
2048 loweredOp.addOperands(newOperands: {a, b, c, sparseIdx});
2049 loweredOp.addAttributes(newAttributes: attrs);
2050 Operation *lowered = rewriter.create(state: loweredOp);
2051
2052 Operation *maybeCastBack = lowered;
2053 if (rawOutType != outType)
2054 maybeCastBack = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: outType,
2055 arg: lowered->getResult(idx: 0));
2056 rewriter.replaceOp(op, newValues: maybeCastBack->getResults());
2057
2058 return success();
2059 }
2060};
2061
2062struct ScaledWMMAOpLowering : public ConvertOpToLLVMPattern<ScaledWMMAOp> {
2063 ScaledWMMAOpLowering(const LLVMTypeConverter &converter,
2064 const ROCDL::TargetInfo &target)
2065 : ConvertOpToLLVMPattern<ScaledWMMAOp>(converter), target(target) {}
2066
2067 ROCDL::TargetInfo target;
2068
2069 LogicalResult
2070 matchAndRewrite(ScaledWMMAOp op, ScaledWMMAOpAdaptor adaptor,
2071 ConversionPatternRewriter &rewriter) const override {
2072 Location loc = op.getLoc();
2073 auto outType =
2074 typeConverter->convertType<VectorType>(t: op.getDestD().getType());
2075 if (!outType)
2076 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
2077
2078 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
2079 return op->emitOpError(message: "WMMA scale only supported on gfx1250+");
2080
2081 int64_t m = op.getM();
2082 int64_t n = op.getN();
2083 int64_t k = op.getK();
2084
2085 Type aElemType = getElementTypeOrSelf(type: op.getSourceA().getType());
2086 Type bElemType = getElementTypeOrSelf(type: op.getSourceB().getType());
2087
2088 std::optional<ROCDL::MatrixFormat> aFmtCode =
2089 smallFloatTypeToMatrixFormat(mlirElemType: aElemType);
2090 std::optional<ROCDL::MatrixFormat> bFmtCode =
2091 smallFloatTypeToMatrixFormat(mlirElemType: bElemType);
2092
2093 if (!aFmtCode || !bFmtCode)
2094 return op.emitOpError(message: "unsupported element types for scaled_wmma");
2095
2096 // Get scale vector types and determine variant (scale vs scale16).
2097 auto scaleAVecType = cast<VectorType>(Val: op.getScaleA().getType());
2098 auto scaleBVecType = cast<VectorType>(Val: op.getScaleB().getType());
2099
2100 if (scaleAVecType.getNumElements() != scaleBVecType.getNumElements())
2101 return op.emitOpError(message: "scaleA and scaleB must have equal vector length");
2102
2103 // Extract scale format from element types.
2104 Type scaleAElemType = scaleAVecType.getElementType();
2105 Type scaleBElemType = scaleBVecType.getElementType();
2106
2107 std::optional<ROCDL::WMMAMatrixScaleFormat> scaleAFmt =
2108 getWmmaScaleFormat(elemType: scaleAElemType);
2109 std::optional<ROCDL::WMMAMatrixScaleFormat> scaleBFmt =
2110 getWmmaScaleFormat(elemType: scaleBElemType);
2111
2112 if (!scaleAFmt || !scaleBFmt)
2113 return op.emitOpError(message: "unsupported scale element types");
2114
2115 // Determine which intrinsic to use based on dimensions.
2116 bool isScale16 = (scaleAVecType.getNumElements() == 8);
2117 std::optional<StringRef> intrinsicName =
2118 getScaledWmmaIntrinsicName(m, n, k, isScale16);
2119 if (!intrinsicName)
2120 return op.emitOpError(message: "unsupported scaled_wmma dimensions: ")
2121 << m << "x" << n << "x" << k;
2122
2123 SmallVector<NamedAttribute, 8> attrs;
2124
2125 // The f4 variant does not have fmtA and fmtB attributes.
2126 bool is32x16 = (m == 32 && n == 16 && k == 128);
2127 if (!is32x16) {
2128 attrs.emplace_back(Args: "fmtA", Args: ROCDL::MatrixFormatAttr::get(
2129 context: rewriter.getContext(), value: *aFmtCode));
2130 attrs.emplace_back(Args: "fmtB", Args: ROCDL::MatrixFormatAttr::get(
2131 context: rewriter.getContext(), value: *bFmtCode));
2132 }
2133
2134 // modC uses default value of 0.
2135 attrs.emplace_back(
2136 Args: "modC", Args: ROCDL::WMMACModifierAttr::get(context: rewriter.getContext(),
2137 value: ROCDL::WMMACModifier::none));
2138
2139 // Scale attributes. Convert user-facing firstScaleLane (0 or 16) to the
2140 // half of the wave that is being selected (0 or 1).
2141 attrs.emplace_back(Args: "scaleAType", Args: ROCDL::WMMAMatrixScaleAttr::get(
2142 context: rewriter.getContext(),
2143 value: static_cast<ROCDL::WMMAMatrixScale>(
2144 op.getAFirstScaleLane() / 16)));
2145 attrs.emplace_back(Args: "fmtScaleA", Args: ROCDL::WMMAMatrixScaleFormatAttr::get(
2146 context: rewriter.getContext(), value: *scaleAFmt));
2147 attrs.emplace_back(Args: "scaleBType", Args: ROCDL::WMMAMatrixScaleAttr::get(
2148 context: rewriter.getContext(),
2149 value: static_cast<ROCDL::WMMAMatrixScale>(
2150 op.getBFirstScaleLane() / 16)));
2151 attrs.emplace_back(Args: "fmtScaleB", Args: ROCDL::WMMAMatrixScaleFormatAttr::get(
2152 context: rewriter.getContext(), value: *scaleBFmt));
2153
2154 // Reuse flags use default value of false.
2155 attrs.emplace_back(Args: "reuseA", Args: rewriter.getBoolAttr(value: false));
2156 attrs.emplace_back(Args: "reuseB", Args: rewriter.getBoolAttr(value: false));
2157
2158 // Convert typed float vectors to packed format.
2159 Value sourceA =
2160 packSmallFloatVectorOperand(rewriter, loc, input: adaptor.getSourceA());
2161 Value sourceB =
2162 packSmallFloatVectorOperand(rewriter, loc, input: adaptor.getSourceB());
2163
2164 // Pack scale vectors into i32/i64.
2165 Value packedScaleA = castScaleOperand(rewriter, loc, input: adaptor.getScaleA());
2166 Value packedScaleB = castScaleOperand(rewriter, loc, input: adaptor.getScaleB());
2167
2168 // Create the intrinsic call.
2169 OperationState loweredOp(loc, *intrinsicName);
2170 loweredOp.addTypes(newTypes: outType);
2171 loweredOp.addOperands(
2172 newOperands: {sourceA, sourceB, adaptor.getDestC(), packedScaleA, packedScaleB});
2173 loweredOp.addAttributes(newAttributes: attrs);
2174
2175 Operation *lowered = rewriter.create(state: loweredOp);
2176 rewriter.replaceOp(op, newValues: lowered->getResults());
2177
2178 return success();
2179 }
2180};
2181
2182struct TransposeLoadOpLowering
2183 : public ConvertOpToLLVMPattern<TransposeLoadOp> {
2184 TransposeLoadOpLowering(const LLVMTypeConverter &converter,
2185 const ROCDL::TargetInfo &target)
2186 : ConvertOpToLLVMPattern<TransposeLoadOp>(converter), target(target) {}
2187
2188 ROCDL::TargetInfo target;
2189
2190 LogicalResult
2191 matchAndRewrite(TransposeLoadOp op, TransposeLoadOpAdaptor adaptor,
2192 ConversionPatternRewriter &rewriter) const override {
2193 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX950_INSTS) &&
2194 !target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
2195 return op.emitOpError(
2196 message: "transpose_load is only supported on gfx950 and gfx1250+");
2197
2198 Location loc = op.getLoc();
2199 auto srcMemRefType = cast<MemRefType>(Val: op.getSrc().getType());
2200
2201 // Elements in subbyte memrefs are stored non-contiguously,
2202 // reject if source is sub-byte memref. Use emulated memrefs instead.
2203 size_t srcElementSize =
2204 srcMemRefType.getElementType().getIntOrFloatBitWidth();
2205 if (srcElementSize < 8)
2206 return op.emitOpError(message: "Expect source memref to have at least 8 bits "
2207 "element size, got ")
2208 << srcElementSize;
2209
2210 auto resultType = cast<VectorType>(Val: op.getResult().getType());
2211 Value srcPtr =
2212 getStridedElementPtr(rewriter, loc, type: srcMemRefType, memRefDesc: adaptor.getSrc(),
2213 indices: (adaptor.getSrcIndices()));
2214
2215 size_t numElements = resultType.getNumElements();
2216 size_t elementTypeSize =
2217 resultType.getElementType().getIntOrFloatBitWidth();
2218
2219 Type llvmResultType = typeConverter->convertType(t: resultType);
2220 // ROCDL transpose load intrinsics return vectors of 32-bit integers for
2221 // sub-16-bit element types, and otherwise return the converted result type.
2222 Type rocdlResultType =
2223 elementTypeSize < 16
2224 ? VectorType::get(shape: (numElements * elementTypeSize) / 32,
2225 elementType: rewriter.getIntegerType(width: 32))
2226 : llvmResultType;
2227
2228 auto emitNumElementsError = [&](size_t expected, StringRef chipsetName) {
2229 return op.emitOpError()
2230 << elementTypeSize << "-bit transpose_load requires " << expected
2231 << " elements on " << chipsetName;
2232 };
2233
2234 Value intrinsic;
2235 if (target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS)) {
2236 switch (elementTypeSize) {
2237 case 4: {
2238 if (numElements != 16)
2239 return emitNumElementsError(16, "gfx1250+");
2240 intrinsic =
2241 ROCDL::DsLoadTr4_B64::create(builder&: rewriter, location: loc, res: rocdlResultType, ptr: srcPtr,
2242 /*alias_scopes=*/{},
2243 /*noalias_scopes=*/{}, /*tbaa=*/{})
2244 .getResult();
2245 break;
2246 }
2247 case 6: {
2248 if (numElements != 16)
2249 return emitNumElementsError(16, "gfx1250+");
2250 intrinsic =
2251 ROCDL::DsLoadTr6_B96::create(builder&: rewriter, location: loc, res: rocdlResultType, ptr: srcPtr,
2252 /*alias_scopes=*/{},
2253 /*noalias_scopes=*/{}, /*tbaa=*/{})
2254 .getResult();
2255 break;
2256 }
2257 case 8: {
2258 if (numElements != 8)
2259 return emitNumElementsError(8, "gfx1250+");
2260 intrinsic =
2261 ROCDL::DsLoadTr8_B64::create(builder&: rewriter, location: loc, res: rocdlResultType, ptr: srcPtr,
2262 /*alias_scopes=*/{},
2263 /*noalias_scopes=*/{}, /*tbaa=*/{})
2264 .getResult();
2265 break;
2266 }
2267 case 16: {
2268 if (numElements != 8)
2269 return emitNumElementsError(8, "gfx1250+");
2270 intrinsic = ROCDL::DsLoadTr16_B128::create(
2271 builder&: rewriter, location: loc, res: rocdlResultType, ptr: srcPtr,
2272 /*alias_scopes=*/{}, /*noalias_scopes=*/{}, /*tbaa=*/{})
2273 .getResult();
2274 break;
2275 }
2276 default:
2277 return op.emitOpError(message: "Unsupported element size for transpose load");
2278 }
2279 } else {
2280 switch (elementTypeSize) {
2281 case 4: {
2282 if (numElements != 16)
2283 return emitNumElementsError(16, "gfx950");
2284 intrinsic = ROCDL::ds_read_tr4_b64::create(
2285 builder&: rewriter, location: loc, res: rocdlResultType, ptr: srcPtr,
2286 /*alias_scopes=*/{}, /*noalias_scopes=*/{}, /*tbaa=*/{})
2287 .getResult();
2288 break;
2289 }
2290 case 6: {
2291 if (numElements != 16)
2292 return emitNumElementsError(16, "gfx950");
2293 intrinsic = ROCDL::ds_read_tr6_b96::create(
2294 builder&: rewriter, location: loc, res: rocdlResultType, ptr: srcPtr,
2295 /*alias_scopes=*/{}, /*noalias_scopes=*/{}, /*tbaa=*/{})
2296 .getResult();
2297 break;
2298 }
2299 case 8: {
2300 if (numElements != 8)
2301 return emitNumElementsError(8, "gfx950");
2302 intrinsic = ROCDL::ds_read_tr8_b64::create(
2303 builder&: rewriter, location: loc, res: rocdlResultType, ptr: srcPtr,
2304 /*alias_scopes=*/{}, /*noalias_scopes=*/{}, /*tbaa=*/{})
2305 .getResult();
2306 break;
2307 }
2308 case 16: {
2309 if (numElements != 4)
2310 return emitNumElementsError(4, "gfx950");
2311 intrinsic = ROCDL::ds_read_tr16_b64::create(
2312 builder&: rewriter, location: loc, res: rocdlResultType, ptr: srcPtr,
2313 /*alias_scopes=*/{}, /*noalias_scopes=*/{}, /*tbaa=*/{})
2314 .getResult();
2315 break;
2316 }
2317 default:
2318 return op.emitOpError(message: "Unsupported element size for transpose load");
2319 }
2320 }
2321
2322 assert(intrinsic && "expected ROCDL transpose load intrinsic");
2323 if (intrinsic.getType() == llvmResultType) {
2324 rewriter.replaceOp(op, newValues: intrinsic);
2325 return success();
2326 }
2327 rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, args&: llvmResultType, args&: intrinsic);
2328 return success();
2329 }
2330};
2331
2332struct GlobalTransposeLoadOpLowering
2333 : public ConvertOpToLLVMPattern<GlobalTransposeLoadOp> {
2334 GlobalTransposeLoadOpLowering(const LLVMTypeConverter &converter,
2335 const ROCDL::TargetInfo &target)
2336 : ConvertOpToLLVMPattern<GlobalTransposeLoadOp>(converter),
2337 target(target) {}
2338
2339 ROCDL::TargetInfo target;
2340
2341 LogicalResult
2342 matchAndRewrite(GlobalTransposeLoadOp op,
2343 GlobalTransposeLoadOpAdaptor adaptor,
2344 ConversionPatternRewriter &rewriter) const override {
2345 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX12_INSTS))
2346 return op.emitOpError(
2347 message: "global_transpose_load is only supported on gfx1200+");
2348
2349 Location loc = op.getLoc();
2350 auto srcMemRefType = cast<MemRefType>(Val: op.getSrc().getType());
2351 auto resultType = cast<VectorType>(Val: op.getResult().getType());
2352
2353 Value srcPtr = getStridedElementPtr(
2354 rewriter, loc, type: srcMemRefType, memRefDesc: adaptor.getSrc(), indices: adaptor.getSrcIndices(),
2355 noWrapFlags: LLVM::GEPNoWrapFlags::inbounds | LLVM::GEPNoWrapFlags::nuw);
2356
2357 size_t numElements = resultType.getNumElements();
2358 size_t elementTypeSize =
2359 resultType.getElementType().getIntOrFloatBitWidth();
2360
2361 // ROCDL global transpose load intrinsics return vectors of i32 for
2362 // sub-16-bit elements, matching the LDS lowering convention.
2363 Type rocdlResultType =
2364 elementTypeSize < 16
2365 ? VectorType::get(shape: (numElements * elementTypeSize) / 32,
2366 elementType: rewriter.getIntegerType(width: 32))
2367 : typeConverter->convertType(t: resultType);
2368 Type llvmResultType = typeConverter->convertType(t: resultType);
2369
2370 switch (elementTypeSize) {
2371 case 4: {
2372 assert(numElements == 16);
2373 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
2374 return op.emitOpError(message: "4-bit global_transpose_load requires gfx1250+");
2375 auto rocdlOp = ROCDL::GlobalLoadTr4_B64::create(
2376 builder&: rewriter, location: loc, res: rocdlResultType, ptr: srcPtr, alias_scopes: ArrayAttr{}, noalias_scopes: ArrayAttr{},
2377 tbaa: ArrayAttr{});
2378 rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, args&: llvmResultType, args&: rocdlOp);
2379 break;
2380 }
2381 case 6: {
2382 assert(numElements == 16);
2383 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
2384 return op.emitOpError(message: "6-bit global_transpose_load requires gfx1250+");
2385 auto rocdlOp = ROCDL::GlobalLoadTr6_B96::create(
2386 builder&: rewriter, location: loc, res: rocdlResultType, ptr: srcPtr, alias_scopes: ArrayAttr{}, noalias_scopes: ArrayAttr{},
2387 tbaa: ArrayAttr{});
2388 rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, args&: llvmResultType, args&: rocdlOp);
2389 break;
2390 }
2391 case 8: {
2392 assert(numElements == 8);
2393 auto rocdlOp = ROCDL::GlobalLoadTr8_B64::create(
2394 builder&: rewriter, location: loc, res: rocdlResultType, ptr: srcPtr, alias_scopes: ArrayAttr{}, noalias_scopes: ArrayAttr{},
2395 tbaa: ArrayAttr{});
2396 rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, args&: llvmResultType, args&: rocdlOp);
2397 break;
2398 }
2399 case 16: {
2400 assert(numElements == 8);
2401 rewriter.replaceOpWithNewOp<ROCDL::GlobalLoadTr8_B128>(
2402 op, args&: llvmResultType, args&: srcPtr, args: ArrayAttr{}, args: ArrayAttr{}, args: ArrayAttr{});
2403 break;
2404 }
2405 default:
2406 return op.emitOpError(
2407 message: "unsupported element size for global transpose load");
2408 }
2409 return success();
2410 }
2411};
2412
2413struct GatherToLDSOpLowering : public ConvertOpToLLVMPattern<GatherToLDSOp> {
2414 GatherToLDSOpLowering(const LLVMTypeConverter &converter,
2415 const ROCDL::TargetInfo &target)
2416 : ConvertOpToLLVMPattern<GatherToLDSOp>(converter), target(target) {}
2417
2418 ROCDL::TargetInfo target;
2419
2420 LogicalResult
2421 matchAndRewrite(GatherToLDSOp op, GatherToLDSOpAdaptor adaptor,
2422 ConversionPatternRewriter &rewriter) const override {
2423 if (!target.has(feature: llvm::AMDGPU::FEAT_VMEM_TO_LDS_LOAD_INSTS))
2424 return op.emitOpError(message: "pre-gfx9 and post-gfx10 not supported");
2425
2426 Location loc = op.getLoc();
2427
2428 auto srcMemRefType = cast<MemRefType>(Val: op.getSrc().getType());
2429 auto dstMemRefType = cast<MemRefType>(Val: op.getDst().getType());
2430
2431 // TODO: instead of only transfering one element per thread, we could
2432 // augment it to transfer multiple elements per thread by issuing multiple
2433 // `global_load_lds` instructions.
2434 Type transferType = op.getTransferType();
2435 int loadWidth = [&]() -> int {
2436 if (auto transferVectorType = dyn_cast<VectorType>(Val&: transferType)) {
2437 return (transferVectorType.getNumElements() *
2438 transferVectorType.getElementTypeBitWidth()) /
2439 8;
2440 }
2441 return transferType.getIntOrFloatBitWidth() / 8;
2442 }();
2443
2444 // Currently only 1, 2, 4, 12 and 16 byte loads are supported.
2445 if (!llvm::is_contained(Set: {1, 2, 4, 12, 16}, Element: loadWidth))
2446 return op.emitOpError(message: "chipset unsupported element size");
2447
2448 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX950_INSTS) &&
2449 llvm::is_contained(Set: {12, 16}, Element: loadWidth))
2450 return op.emitOpError(message: "Gather to LDS instructions with 12-byte and "
2451 "16-byte load widths are only supported on gfx950");
2452
2453 Value srcPtr =
2454 getStridedElementPtr(rewriter, loc, type: srcMemRefType, memRefDesc: adaptor.getSrc(),
2455 indices: (adaptor.getSrcIndices()));
2456 Value dstPtr =
2457 getStridedElementPtr(rewriter, loc, type: dstMemRefType, memRefDesc: adaptor.getDst(),
2458 indices: (adaptor.getDstIndices()));
2459
2460 if (op.getAsync()) {
2461 rewriter.replaceOpWithNewOp<ROCDL::LoadAsyncToLDSOp>(
2462 op, args&: srcPtr, args&: dstPtr, args: rewriter.getI32IntegerAttr(value: loadWidth),
2463 /*offset=*/args: rewriter.getI32IntegerAttr(value: 0),
2464 /*aux=*/args: rewriter.getI32IntegerAttr(value: 0), args: ArrayAttr{}, args: ArrayAttr{},
2465 args: ArrayAttr{});
2466 } else {
2467 rewriter.replaceOpWithNewOp<ROCDL::LoadToLDSOp>(
2468 op, args&: srcPtr, args&: dstPtr, args: rewriter.getI32IntegerAttr(value: loadWidth),
2469 /*offset=*/args: rewriter.getI32IntegerAttr(value: 0),
2470 /*aux=*/args: rewriter.getI32IntegerAttr(value: 0), args: ArrayAttr{}, args: ArrayAttr{},
2471 args: ArrayAttr{});
2472 }
2473
2474 return success();
2475 }
2476};
2477
2478struct GlobalLoadAsyncToLDSOpLowering
2479 : public ConvertOpToLLVMPattern<GlobalLoadAsyncToLDSOp> {
2480 GlobalLoadAsyncToLDSOpLowering(const LLVMTypeConverter &converter,
2481 const ROCDL::TargetInfo &target)
2482 : ConvertOpToLLVMPattern<GlobalLoadAsyncToLDSOp>(converter),
2483 target(target) {}
2484
2485 ROCDL::TargetInfo target;
2486
2487 LogicalResult
2488 matchAndRewrite(GlobalLoadAsyncToLDSOp op,
2489 GlobalLoadAsyncToLDSOpAdaptor adaptor,
2490 ConversionPatternRewriter &rewriter) const override {
2491 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
2492 return op.emitOpError(
2493 message: "global_load_async_to_lds is only supported on gfx1250+");
2494
2495 Location loc = op.getLoc();
2496 auto srcMemRefType = cast<MemRefType>(Val: op.getSrc().getType());
2497 auto dstMemRefType = cast<MemRefType>(Val: op.getDst().getType());
2498
2499 Type transferType = op.getTransferType();
2500 int transferBits =
2501 isa<VectorType>(Val: transferType)
2502 ? cast<VectorType>(Val&: transferType).getNumElements() *
2503 cast<VectorType>(Val&: transferType).getElementTypeBitWidth()
2504 : transferType.getIntOrFloatBitWidth();
2505
2506 Value srcPtr =
2507 getStridedElementPtr(rewriter, loc, type: srcMemRefType, memRefDesc: adaptor.getSrc(),
2508 indices: adaptor.getSrcIndices());
2509 Value dstPtr =
2510 getStridedElementPtr(rewriter, loc, type: dstMemRefType, memRefDesc: adaptor.getDst(),
2511 indices: adaptor.getDstIndices());
2512
2513 if (op.getMask()) {
2514 Value mask = adaptor.getMask();
2515 int64_t nullptrVal =
2516 llvm::AMDGPU::getNullPointerValue(AS: llvm::AMDGPUAS::LOCAL_ADDRESS);
2517 Value nullInt =
2518 createI32Constant(rewriter, loc, value: static_cast<int32_t>(nullptrVal));
2519 Value nullPtr =
2520 LLVM::IntToPtrOp::create(builder&: rewriter, location: loc, resultType: dstPtr.getType(), operands: nullInt);
2521 dstPtr = LLVM::SelectOp::create(builder&: rewriter, location: loc, condition: mask, trueValue: dstPtr, falseValue: nullPtr);
2522 }
2523
2524 auto offset = rewriter.getI32IntegerAttr(value: 0);
2525 Attribute aux = rewriter.getI32IntegerAttr(value: 0);
2526
2527 switch (transferBits) {
2528 case 8:
2529 rewriter.replaceOpWithNewOp<ROCDL::GlobalLoadAsyncToLDSB8Op>(
2530 op, args&: srcPtr, args&: dstPtr, args&: offset, args&: aux, args: ArrayAttr{}, args: ArrayAttr{},
2531 args: ArrayAttr{});
2532 break;
2533 case 32:
2534 rewriter.replaceOpWithNewOp<ROCDL::GlobalLoadAsyncToLDSB32Op>(
2535 op, args&: srcPtr, args&: dstPtr, args&: offset, args&: aux, args: ArrayAttr{}, args: ArrayAttr{},
2536 args: ArrayAttr{});
2537 break;
2538 case 64:
2539 rewriter.replaceOpWithNewOp<ROCDL::GlobalLoadAsyncToLDSB64Op>(
2540 op, args&: srcPtr, args&: dstPtr, args&: offset, args&: aux, args: ArrayAttr{}, args: ArrayAttr{},
2541 args: ArrayAttr{});
2542 break;
2543 case 128:
2544 rewriter.replaceOpWithNewOp<ROCDL::GlobalLoadAsyncToLDSB128Op>(
2545 op, args&: srcPtr, args&: dstPtr, args&: offset, args&: aux, args: ArrayAttr{}, args: ArrayAttr{},
2546 args: ArrayAttr{});
2547 break;
2548 default:
2549 return op.emitOpError(message: "unsupported transfer width");
2550 }
2551 return success();
2552 }
2553};
2554
2555namespace {
2556struct ExtPackedFp8OpLowering final
2557 : public ConvertOpToLLVMPattern<ExtPackedFp8Op> {
2558 ExtPackedFp8OpLowering(const LLVMTypeConverter &converter,
2559 const ROCDL::TargetInfo &target)
2560 : ConvertOpToLLVMPattern<amdgpu::ExtPackedFp8Op>(converter),
2561 target(target) {}
2562 ROCDL::TargetInfo target;
2563
2564 LogicalResult
2565 matchAndRewrite(ExtPackedFp8Op op, ExtPackedFp8OpAdaptor adaptor,
2566 ConversionPatternRewriter &rewriter) const override;
2567};
2568
2569struct ScaledExtPackedMatrixOpLowering final
2570 : public ConvertOpToLLVMPattern<ScaledExtPackedMatrixOp> {
2571 ScaledExtPackedMatrixOpLowering(const LLVMTypeConverter &converter,
2572 const ROCDL::TargetInfo &target)
2573 : ConvertOpToLLVMPattern<amdgpu::ScaledExtPackedMatrixOp>(converter),
2574 target(target) {}
2575 ROCDL::TargetInfo target;
2576
2577 LogicalResult
2578 matchAndRewrite(ScaledExtPackedMatrixOp op,
2579 ScaledExtPackedMatrixOpAdaptor adaptor,
2580 ConversionPatternRewriter &rewriter) const override;
2581};
2582
2583struct PackedTrunc2xFp8OpLowering final
2584 : public ConvertOpToLLVMPattern<PackedTrunc2xFp8Op> {
2585 PackedTrunc2xFp8OpLowering(const LLVMTypeConverter &converter,
2586 const ROCDL::TargetInfo &target)
2587 : ConvertOpToLLVMPattern<amdgpu::PackedTrunc2xFp8Op>(converter),
2588 target(target) {}
2589 ROCDL::TargetInfo target;
2590
2591 LogicalResult
2592 matchAndRewrite(PackedTrunc2xFp8Op op, PackedTrunc2xFp8OpAdaptor adaptor,
2593 ConversionPatternRewriter &rewriter) const override;
2594};
2595
2596struct PackedStochRoundFp8OpLowering final
2597 : public ConvertOpToLLVMPattern<PackedStochRoundFp8Op> {
2598 PackedStochRoundFp8OpLowering(const LLVMTypeConverter &converter,
2599 const ROCDL::TargetInfo &target)
2600 : ConvertOpToLLVMPattern<amdgpu::PackedStochRoundFp8Op>(converter),
2601 target(target) {}
2602 ROCDL::TargetInfo target;
2603
2604 LogicalResult
2605 matchAndRewrite(PackedStochRoundFp8Op op,
2606 PackedStochRoundFp8OpAdaptor adaptor,
2607 ConversionPatternRewriter &rewriter) const override;
2608};
2609
2610struct ScaledExtPackedOpLowering final
2611 : public ConvertOpToLLVMPattern<ScaledExtPackedOp> {
2612 ScaledExtPackedOpLowering(const LLVMTypeConverter &converter,
2613 const ROCDL::TargetInfo &target)
2614 : ConvertOpToLLVMPattern<amdgpu::ScaledExtPackedOp>(converter),
2615 target(target) {}
2616 ROCDL::TargetInfo target;
2617
2618 LogicalResult
2619 matchAndRewrite(ScaledExtPackedOp op, ScaledExtPackedOpAdaptor adaptor,
2620 ConversionPatternRewriter &rewriter) const override;
2621};
2622
2623struct PackedScaledTruncOpLowering final
2624 : public ConvertOpToLLVMPattern<PackedScaledTruncOp> {
2625 PackedScaledTruncOpLowering(const LLVMTypeConverter &converter,
2626 const ROCDL::TargetInfo &target)
2627 : ConvertOpToLLVMPattern<amdgpu::PackedScaledTruncOp>(converter),
2628 target(target) {}
2629 ROCDL::TargetInfo target;
2630
2631 LogicalResult
2632 matchAndRewrite(PackedScaledTruncOp op, PackedScaledTruncOpAdaptor adaptor,
2633 ConversionPatternRewriter &rewriter) const override;
2634};
2635
2636} // end namespace
2637
2638LogicalResult ExtPackedFp8OpLowering::matchAndRewrite(
2639 ExtPackedFp8Op op, ExtPackedFp8OpAdaptor adaptor,
2640 ConversionPatternRewriter &rewriter) const {
2641 Location loc = op.getLoc();
2642 if (!target.has(feature: llvm::AMDGPU::FEAT_FP8_CONVERSION_INSTS))
2643 return rewriter.notifyMatchFailure(
2644 arg&: loc, msg: "Fp8 conversion instructions are not available on target "
2645 "architecture and their emulation is not implemented");
2646 Type v4i8 =
2647 getTypeConverter()->convertType(t: VectorType::get(shape: 4, elementType: rewriter.getI8Type()));
2648 Type i32 = getTypeConverter()->convertType(t: rewriter.getI32Type());
2649 Type f32 = getTypeConverter()->convertType(t: op.getResult().getType());
2650
2651 Value source = adaptor.getSource();
2652 auto sourceVecType = dyn_cast<VectorType>(Val: op.getSource().getType());
2653 auto resultVecType = dyn_cast<VectorType>(Val: op.getResult().getType());
2654 Type sourceElemType = getElementTypeOrSelf(val: op.getSource());
2655 // Extend to a v4i8
2656 if (!sourceVecType || sourceVecType.getNumElements() < 4) {
2657 Value longVec = LLVM::UndefOp::create(builder&: rewriter, location: loc, res: v4i8);
2658 if (!sourceVecType) {
2659 longVec = LLVM::InsertElementOp::create(
2660 builder&: rewriter, location: loc, vector: longVec, value: source, position: createI32Constant(rewriter, loc, value: 0));
2661 } else {
2662 for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
2663 Value idx = createI32Constant(rewriter, loc, value: i);
2664 Value elem = LLVM::ExtractElementOp::create(builder&: rewriter, location: loc, vector: source, position: idx);
2665 longVec =
2666 LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: longVec, value: elem, position: idx);
2667 }
2668 }
2669 source = longVec;
2670 }
2671 Value i32Source = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: i32, arg: source);
2672 if (resultVecType) {
2673 if (typeIsExpectedBf8ForTarget(target, type: sourceElemType)) {
2674 rewriter.replaceOpWithNewOp<ROCDL::CvtPkF32Bf8Op>(op, args&: f32, args&: i32Source,
2675 args: op.getIndex());
2676 } else if (typeIsExpectedFp8ForTarget(target, type: sourceElemType)) {
2677 rewriter.replaceOpWithNewOp<ROCDL::CvtPkF32Fp8Op>(op, args&: f32, args&: i32Source,
2678 args: op.getIndex());
2679 }
2680 } else {
2681 if (typeIsExpectedBf8ForTarget(target, type: sourceElemType)) {
2682 rewriter.replaceOpWithNewOp<ROCDL::CvtF32Bf8Op>(op, args&: f32, args&: i32Source,
2683 args: op.getIndex());
2684 } else if (typeIsExpectedFp8ForTarget(target, type: sourceElemType)) {
2685 rewriter.replaceOpWithNewOp<ROCDL::CvtF32Fp8Op>(op, args&: f32, args&: i32Source,
2686 args: op.getIndex());
2687 }
2688 }
2689 return success();
2690}
2691
2692int32_t getScaleSel(int32_t blockSize, unsigned bitWidth, int32_t scaleWaveHalf,
2693 int32_t firstScaleByte) {
2694 // When lowering amdgpu.scaled_ext_packed_matrix to rocdl.cvt.scale.pk*.f*.f*
2695 // operations, the attributes blockSize, sourceType, scaleWaveHalf, and
2696 // firstScaleByte are merged into a single attribute scaleSel. This is how
2697 // those values are merged together. (Note: scaleWaveHalf isn't a high-level
2698 // attribute but is derifed from firstScaleLane).
2699 assert(llvm::is_contained({16, 32}, blockSize));
2700 assert(llvm::is_contained({4u, 6u, 8u}, bitWidth));
2701
2702 const bool isFp8 = bitWidth == 8;
2703 const bool isBlock16 = blockSize == 16;
2704
2705 if (!isFp8) {
2706 int32_t bit0 = isBlock16;
2707 assert(llvm::is_contained({0, 1, 2}, firstScaleByte));
2708 int32_t bit1 = (firstScaleByte == 2) << 1;
2709 assert(llvm::is_contained({0, 1}, scaleWaveHalf));
2710 int32_t bit2 = scaleWaveHalf << 2;
2711 return bit2 | bit1 | bit0;
2712 }
2713
2714 int32_t bit0 = isBlock16;
2715 // firstScaleByte is guaranteed to be defined by two bits.
2716 assert(llvm::is_contained({0, 1, 2, 3}, firstScaleByte));
2717 int32_t bits2and1 = firstScaleByte << 1;
2718 assert(llvm::is_contained({0, 1}, scaleWaveHalf));
2719 int32_t bit3 = scaleWaveHalf << 3;
2720 int32_t bits = bit3 | bits2and1 | bit0;
2721 // These are invalid cases.
2722 assert(!llvm::is_contained(
2723 {0b0011, 0b0101, 0b0111, 0b1000, 0b1001, 0b1011, 0b1111}, bits));
2724 return bits;
2725}
2726
2727static std::optional<StringRef>
2728scaledExtPacked816ToIntrinsic(Type srcElemType, Type destElemType) {
2729 using fp4 = Float4E2M1FNType;
2730 using fp8 = Float8E4M3FNType;
2731 using bf8 = Float8E5M2Type;
2732 using fp6 = Float6E2M3FNType;
2733 using bf6 = Float6E3M2FNType;
2734 if (isa<fp4>(Val: srcElemType)) {
2735 if (destElemType.isF16())
2736 return ROCDL::CvtPkScalePk8F16Fp4Op::getOperationName();
2737 if (destElemType.isBF16())
2738 return ROCDL::CvtPkScalePk8Bf16Fp4Op::getOperationName();
2739 if (destElemType.isF32())
2740 return ROCDL::CvtPkScalePk8F32Fp4Op::getOperationName();
2741 return std::nullopt;
2742 }
2743 if (isa<fp8>(Val: srcElemType)) {
2744 if (destElemType.isF16())
2745 return ROCDL::CvtPkScalePk8F16Fp8Op::getOperationName();
2746 if (destElemType.isBF16())
2747 return ROCDL::CvtPkScalePk8Bf16Fp8Op::getOperationName();
2748 if (destElemType.isF32())
2749 return ROCDL::CvtPkScalePk8F32Fp8Op::getOperationName();
2750 return std::nullopt;
2751 }
2752 if (isa<bf8>(Val: srcElemType)) {
2753 if (destElemType.isF16())
2754 return ROCDL::CvtPkScalePk8F16Bf8Op::getOperationName();
2755 if (destElemType.isBF16())
2756 return ROCDL::CvtPkScalePk8Bf16Bf8Op::getOperationName();
2757 if (destElemType.isF32())
2758 return ROCDL::CvtPkScalePk8F32Bf8Op::getOperationName();
2759 return std::nullopt;
2760 }
2761 if (isa<fp6>(Val: srcElemType)) {
2762 if (destElemType.isF16())
2763 return ROCDL::CvtPkScalePk16F16Fp6Op::getOperationName();
2764 if (destElemType.isBF16())
2765 return ROCDL::CvtPkScalePk16Bf16Fp6Op::getOperationName();
2766 if (destElemType.isF32())
2767 return ROCDL::CvtPkScalePk16F32Fp6Op::getOperationName();
2768 return std::nullopt;
2769 }
2770 if (isa<bf6>(Val: srcElemType)) {
2771 if (destElemType.isF16())
2772 return ROCDL::CvtPkScalePk16F16Bf6Op::getOperationName();
2773 if (destElemType.isBF16())
2774 return ROCDL::CvtPkScalePk16Bf16Bf6Op::getOperationName();
2775 if (destElemType.isF32())
2776 return ROCDL::CvtPkScalePk16F32Bf6Op::getOperationName();
2777 return std::nullopt;
2778 }
2779 llvm_unreachable("invalid combination of element types for packed conversion "
2780 "instructions");
2781}
2782
2783LogicalResult ScaledExtPackedMatrixOpLowering::matchAndRewrite(
2784 ScaledExtPackedMatrixOp op, ScaledExtPackedMatrixOpAdaptor adaptor,
2785 ConversionPatternRewriter &rewriter) const {
2786 using fp4 = Float4E2M1FNType;
2787 using fp8 = Float8E4M3FNType;
2788 using bf8 = Float8E5M2Type;
2789 using fp6 = Float6E2M3FNType;
2790 using bf6 = Float6E3M2FNType;
2791 Location loc = op.getLoc();
2792 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS)) {
2793 return rewriter.notifyMatchFailure(
2794 arg&: loc,
2795 msg: "Scaled fp packed conversion instructions are not available on target "
2796 "architecture and their emulation is not implemented");
2797 }
2798 // Convert user-facing firstScaleLane (0 or 16) to the half of the wave that
2799 // is being selected.
2800 int32_t scaleWaveHalf = op.getFirstScaleLane() / 16;
2801 int32_t firstScaleByte = op.getFirstScaleByte();
2802 int32_t blockSize = op.getBlockSize();
2803 auto sourceType = cast<VectorType>(Val: op.getSource().getType());
2804 auto srcElemType = cast<FloatType>(Val: sourceType.getElementType());
2805 unsigned bitWidth = srcElemType.getWidth();
2806
2807 auto targetType = cast<VectorType>(Val: op.getResult().getType());
2808 auto destElemType = cast<FloatType>(Val: targetType.getElementType());
2809
2810 IntegerType i32 = rewriter.getI32Type();
2811 Value source = adaptor.getSource();
2812 Type llvmResultType = typeConverter->convertType(t: op.getResult().getType());
2813 Type packedType = nullptr;
2814 if (isa<fp4>(Val: srcElemType)) {
2815 packedType = i32;
2816 packedType = getTypeConverter()->convertType(t: packedType);
2817 } else if (isa<fp8, bf8>(Val: srcElemType)) {
2818 packedType = VectorType::get(shape: 2, elementType: i32);
2819 packedType = getTypeConverter()->convertType(t: packedType);
2820 } else if (isa<fp6, bf6>(Val: srcElemType)) {
2821 packedType = VectorType::get(shape: 3, elementType: i32);
2822 packedType = getTypeConverter()->convertType(t: packedType);
2823 } else {
2824 llvm_unreachable("invalid element type for packed scaled ext");
2825 }
2826
2827 if (!packedType || !llvmResultType) {
2828 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
2829 }
2830
2831 std::optional<StringRef> maybeIntrinsic =
2832 scaledExtPacked816ToIntrinsic(srcElemType, destElemType);
2833 if (!maybeIntrinsic.has_value())
2834 return op.emitOpError(
2835 message: "no intrinsic matching packed scaled conversion on the given chipset");
2836
2837 int32_t scaleSel =
2838 getScaleSel(blockSize, bitWidth, scaleWaveHalf, firstScaleByte);
2839 Value castedScale =
2840 LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: i32, arg: adaptor.getScale());
2841 Value castedSource =
2842 LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: packedType, arg: source);
2843
2844 OperationState loweredOp(loc, *maybeIntrinsic);
2845 loweredOp.addTypes(newTypes: {llvmResultType});
2846 loweredOp.addOperands(newOperands: {castedSource, castedScale});
2847
2848 SmallVector<NamedAttribute, 1> attrs;
2849 attrs.push_back(
2850 Elt: NamedAttribute("scaleSel", rewriter.getI32IntegerAttr(value: scaleSel)));
2851
2852 loweredOp.addAttributes(newAttributes: attrs);
2853 Operation *lowered = rewriter.create(state: loweredOp);
2854 rewriter.replaceOp(op, newOp: lowered);
2855
2856 return success();
2857}
2858
2859LogicalResult ScaledExtPackedOpLowering::matchAndRewrite(
2860 ScaledExtPackedOp op, ScaledExtPackedOpAdaptor adaptor,
2861 ConversionPatternRewriter &rewriter) const {
2862 Location loc = op.getLoc();
2863 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX950_INSTS))
2864 return rewriter.notifyMatchFailure(
2865 arg&: loc, msg: "Scaled fp conversion instructions are not available on target "
2866 "architecture and their emulation is not implemented");
2867 Type i32 = getTypeConverter()->convertType(t: rewriter.getI32Type());
2868
2869 Value source = adaptor.getSource();
2870 Value scale = adaptor.getScale();
2871
2872 VectorType sourceVecType = cast<VectorType>(Val: op.getSource().getType());
2873 Type sourceElemType = sourceVecType.getElementType();
2874 VectorType destVecType = cast<VectorType>(Val: op.getResult().getType());
2875 Type destElemType = destVecType.getElementType();
2876
2877 VectorType packedVecType;
2878 if (isa<Float8E5M2Type, Float8E4M3FNType>(Val: sourceElemType)) {
2879 VectorType v4i8 = VectorType::get(shape: 4, elementType: rewriter.getI8Type());
2880 packedVecType = cast<VectorType>(Val: getTypeConverter()->convertType(t: v4i8));
2881 } else if (isa<Float4E2M1FNType>(Val: sourceElemType)) {
2882 VectorType v8i4 = VectorType::get(shape: 8, elementType: rewriter.getI4Type());
2883 packedVecType = cast<VectorType>(Val: getTypeConverter()->convertType(t: v8i4));
2884 } else {
2885 llvm_unreachable("invalid element type for scaled ext");
2886 }
2887
2888 // Extend to a packedVectorType
2889 if (sourceVecType.getNumElements() < packedVecType.getNumElements()) {
2890 Value longVec = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: packedVecType);
2891 if (!sourceVecType) {
2892 longVec = LLVM::InsertElementOp::create(
2893 builder&: rewriter, location: loc, vector: longVec, value: source, position: createI32Constant(rewriter, loc, value: 0));
2894 } else {
2895 for (int32_t i = 0, e = sourceVecType.getNumElements(); i < e; ++i) {
2896 Value idx = createI32Constant(rewriter, loc, value: i);
2897 Value elem = LLVM::ExtractElementOp::create(builder&: rewriter, location: loc, vector: source, position: idx);
2898 longVec =
2899 LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: longVec, value: elem, position: idx);
2900 }
2901 }
2902 source = longVec;
2903 }
2904 Value i32Source = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: i32, arg: source);
2905
2906 if (isa<Float8E5M2Type>(Val: sourceElemType) && destElemType.isF32())
2907 rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF32Bf8Op>(
2908 op, args&: destVecType, args&: i32Source, args&: scale, args: op.getIndex());
2909 else if (isa<Float8E5M2Type>(Val: sourceElemType) && destElemType.isF16())
2910 rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF16Bf8Op>(
2911 op, args&: destVecType, args&: i32Source, args&: scale, args: op.getIndex());
2912 else if (isa<Float8E5M2Type>(Val: sourceElemType) && destElemType.isBF16())
2913 rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkBf16Bf8Op>(
2914 op, args&: destVecType, args&: i32Source, args&: scale, args: op.getIndex());
2915 else if (isa<Float8E4M3FNType>(Val: sourceElemType) && destElemType.isF32())
2916 rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF32Fp8Op>(
2917 op, args&: destVecType, args&: i32Source, args&: scale, args: op.getIndex());
2918 else if (isa<Float8E4M3FNType>(Val: sourceElemType) && destElemType.isF16())
2919 rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF16Fp8Op>(
2920 op, args&: destVecType, args&: i32Source, args&: scale, args: op.getIndex());
2921 else if (isa<Float8E4M3FNType>(Val: sourceElemType) && destElemType.isBF16())
2922 rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkBf16Fp8Op>(
2923 op, args&: destVecType, args&: i32Source, args&: scale, args: op.getIndex());
2924 else if (isa<Float4E2M1FNType>(Val: sourceElemType) && destElemType.isF32())
2925 rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF32Fp4Op>(
2926 op, args&: destVecType, args&: i32Source, args&: scale, args: op.getIndex());
2927 else if (isa<Float4E2M1FNType>(Val: sourceElemType) && destElemType.isF16())
2928 rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkF16Fp4Op>(
2929 op, args&: destVecType, args&: i32Source, args&: scale, args: op.getIndex());
2930 else if (isa<Float4E2M1FNType>(Val: sourceElemType) && destElemType.isBF16())
2931 rewriter.replaceOpWithNewOp<ROCDL::CvtScaleF32PkBf16Fp4Op>(
2932 op, args&: destVecType, args&: i32Source, args&: scale, args: op.getIndex());
2933 else
2934 return failure();
2935
2936 return success();
2937}
2938
2939LogicalResult PackedScaledTruncOpLowering::matchAndRewrite(
2940 PackedScaledTruncOp op, PackedScaledTruncOpAdaptor adaptor,
2941 ConversionPatternRewriter &rewriter) const {
2942 Location loc = op.getLoc();
2943 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX950_INSTS))
2944 return rewriter.notifyMatchFailure(
2945 arg&: loc, msg: "Scaled fp conversion instructions are not available on target "
2946 "architecture and their emulation is not implemented");
2947 Type v2i16 = getTypeConverter()->convertType(
2948 t: VectorType::get(shape: 2, elementType: rewriter.getI16Type()));
2949 Type i32 = getTypeConverter()->convertType(t: rewriter.getI32Type());
2950
2951 Type resultType = op.getResult().getType();
2952 Type resultElemType = getElementTypeOrSelf(type: resultType);
2953 VectorType sourceVecType = cast<VectorType>(Val: op.getSource().getType());
2954 Type sourceElemType = sourceVecType.getElementType();
2955
2956 Type intResultType = isa<Float4E2M1FNType>(Val: resultElemType) ? i32 : v2i16;
2957
2958 Value source = adaptor.getSource();
2959 Value scale = adaptor.getScale();
2960 Value existing = adaptor.getExisting();
2961 if (existing)
2962 existing = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: intResultType, arg: existing);
2963 else
2964 existing = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: intResultType);
2965
2966 if (sourceVecType.getNumElements() < 2) {
2967 Value c0 = createI32Constant(rewriter, loc, value: 0);
2968 Value elem0 = LLVM::ExtractElementOp::create(builder&: rewriter, location: loc, vector: source, position: c0);
2969 VectorType v2 = VectorType::get(shape: 2, elementType: sourceElemType);
2970 source = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: v2);
2971 source = LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: source, value: elem0, position: c0);
2972 }
2973
2974 Value sourceA, sourceB;
2975 if (sourceElemType.isF32()) {
2976 Value c0 = createI32Constant(rewriter, loc, value: 0);
2977 Value c1 = createI32Constant(rewriter, loc, value: 1);
2978 sourceA = LLVM::ExtractElementOp::create(builder&: rewriter, location: loc, vector: source, position: c0);
2979 sourceB = LLVM::ExtractElementOp::create(builder&: rewriter, location: loc, vector: source, position: c1);
2980 }
2981
2982 Value result;
2983 if (sourceElemType.isF32() && isa<Float8E5M2Type>(Val: resultElemType))
2984 result = ROCDL::CvtScaleF32PkBf8F32Op::create(builder&: rewriter, location: loc, res: intResultType,
2985 oldVdst: existing, src0: sourceA, src1: sourceB,
2986 scale, dstLoHiSel: op.getIndex());
2987 else if (sourceElemType.isF16() && isa<Float8E5M2Type>(Val: resultElemType))
2988 result = ROCDL::CvtScaleF32PkBf8F16Op::create(
2989 builder&: rewriter, location: loc, res: intResultType, oldVdst: existing, src0: source, scale, dstLoHiSel: op.getIndex());
2990 else if (sourceElemType.isBF16() && isa<Float8E5M2Type>(Val: resultElemType))
2991 result = ROCDL::CvtScaleF32PkBf8Bf16Op::create(
2992 builder&: rewriter, location: loc, res: intResultType, oldVdst: existing, src0: source, scale, dstLoHiSel: op.getIndex());
2993 else if (sourceElemType.isF32() && isa<Float8E4M3FNType>(Val: resultElemType))
2994 result = ROCDL::CvtScaleF32PkFp8F32Op::create(builder&: rewriter, location: loc, res: intResultType,
2995 oldVdst: existing, src0: sourceA, src1: sourceB,
2996 scale, dstLoHiSel: op.getIndex());
2997 else if (sourceElemType.isF16() && isa<Float8E4M3FNType>(Val: resultElemType))
2998 result = ROCDL::CvtScaleF32PkFp8F16Op::create(
2999 builder&: rewriter, location: loc, res: intResultType, oldVdst: existing, src0: source, scale, dstLoHiSel: op.getIndex());
3000 else if (sourceElemType.isBF16() && isa<Float8E4M3FNType>(Val: resultElemType))
3001 result = ROCDL::CvtScaleF32PkFp8Bf16Op::create(
3002 builder&: rewriter, location: loc, res: intResultType, oldVdst: existing, src0: source, scale, dstLoHiSel: op.getIndex());
3003 else if (sourceElemType.isF32() && isa<Float4E2M1FNType>(Val: resultElemType))
3004 result = ROCDL::CvtScaleF32PkFp4F32Op::create(builder&: rewriter, location: loc, res: intResultType,
3005 oldVdst: existing, src0: sourceA, src1: sourceB,
3006 scale, dstSelIndex: op.getIndex());
3007 else if (sourceElemType.isF16() && isa<Float4E2M1FNType>(Val: resultElemType))
3008 result = ROCDL::CvtScaleF32PkFp4F16Op::create(
3009 builder&: rewriter, location: loc, res: intResultType, oldVdst: existing, src: source, scale, dstSelIndex: op.getIndex());
3010 else if (sourceElemType.isBF16() && isa<Float4E2M1FNType>(Val: resultElemType))
3011 result = ROCDL::CvtScaleF32PkFp4Bf16Op::create(
3012 builder&: rewriter, location: loc, res: intResultType, oldVdst: existing, src: source, scale, dstSelIndex: op.getIndex());
3013 else
3014 return failure();
3015
3016 result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
3017 op, args: getTypeConverter()->convertType(t: resultType), args&: result);
3018 return success();
3019}
3020
3021LogicalResult PackedTrunc2xFp8OpLowering::matchAndRewrite(
3022 PackedTrunc2xFp8Op op, PackedTrunc2xFp8OpAdaptor adaptor,
3023 ConversionPatternRewriter &rewriter) const {
3024 Location loc = op.getLoc();
3025 if (!target.has(feature: llvm::AMDGPU::FEAT_FP8_CONVERSION_INSTS))
3026 return rewriter.notifyMatchFailure(
3027 arg&: loc, msg: "Fp8 conversion instructions are not available on target "
3028 "architecture and their emulation is not implemented");
3029 Type i32 = getTypeConverter()->convertType(t: rewriter.getI32Type());
3030
3031 Type resultType = op.getResult().getType();
3032 Type resultElemType = getElementTypeOrSelf(type: resultType);
3033
3034 Value sourceA = adaptor.getSourceA();
3035 Value sourceB = adaptor.getSourceB();
3036 if (!sourceB)
3037 sourceB = LLVM::UndefOp::create(builder&: rewriter, location: loc, res: sourceA.getType());
3038 Value existing = adaptor.getExisting();
3039 if (existing)
3040 existing = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: i32, arg: existing);
3041 else
3042 existing = LLVM::UndefOp::create(builder&: rewriter, location: loc, res: i32);
3043
3044 Value result;
3045 if (typeIsExpectedBf8ForTarget(target, type: resultElemType))
3046 result = ROCDL::CvtPkBf8F32Op::create(builder&: rewriter, location: loc, res: i32, srcA: sourceA, srcB: sourceB,
3047 old: existing, wordSel: op.getWordIndex());
3048 else if (typeIsExpectedFp8ForTarget(target, type: resultElemType))
3049 result = ROCDL::CvtPkFp8F32Op::create(builder&: rewriter, location: loc, res: i32, srcA: sourceA, srcB: sourceB,
3050 old: existing, wordSel: op.getWordIndex());
3051 else
3052 return op.emitOpError(
3053 message: "no truncation to result type available on given chipset");
3054
3055 result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
3056 op, args: getTypeConverter()->convertType(t: resultType), args&: result);
3057 return success();
3058}
3059
3060LogicalResult PackedStochRoundFp8OpLowering::matchAndRewrite(
3061 PackedStochRoundFp8Op op, PackedStochRoundFp8OpAdaptor adaptor,
3062 ConversionPatternRewriter &rewriter) const {
3063 Location loc = op.getLoc();
3064 if (!target.has(feature: llvm::AMDGPU::FEAT_FP8_CONVERSION_INSTS))
3065 return rewriter.notifyMatchFailure(
3066 arg&: loc, msg: "Fp8 conversion instructions are not available on target "
3067 "architecture and their emulation is not implemented");
3068 Type i32 = getTypeConverter()->convertType(t: rewriter.getI32Type());
3069
3070 Type resultType = op.getResult().getType();
3071 Type resultElemType = getElementTypeOrSelf(type: resultType);
3072
3073 Value source = adaptor.getSource();
3074 Value stoch = adaptor.getStochiasticParam();
3075 Value existing = adaptor.getExisting();
3076 if (existing)
3077 existing = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: i32, arg: existing);
3078 else
3079 existing = LLVM::UndefOp::create(builder&: rewriter, location: loc, res: i32);
3080
3081 Value result;
3082 if (typeIsExpectedBf8ForTarget(target, type: resultElemType))
3083 result = ROCDL::CvtSrBf8F32Op::create(builder&: rewriter, location: loc, res: i32, srcA: source, srcB: stoch,
3084 old: existing, byteSel: op.getStoreIndex());
3085 else if (typeIsExpectedFp8ForTarget(target, type: resultElemType))
3086 result = ROCDL::CvtSrFp8F32Op::create(builder&: rewriter, location: loc, res: i32, srcA: source, srcB: stoch,
3087 old: existing, byteSel: op.getStoreIndex());
3088 else
3089 return op.emitOpError(
3090 message: "no stochastic rounding to result type available on given chipset");
3091
3092 result = rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(
3093 op, args: getTypeConverter()->convertType(t: resultType), args&: result);
3094 return success();
3095}
3096
3097// Implement the AMDGPU_DPPLowering class that will convert the amdgpu.dpp
3098// operation into the corresponding ROCDL instructions.
3099struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
3100 AMDGPUDPPLowering(const LLVMTypeConverter &converter,
3101 const ROCDL::TargetInfo &target)
3102 : ConvertOpToLLVMPattern<DPPOp>(converter), target(target) {}
3103 ROCDL::TargetInfo target;
3104
3105 LogicalResult
3106 matchAndRewrite(DPPOp DppOp, DPPOp::Adaptor adaptor,
3107 ConversionPatternRewriter &rewriter) const override {
3108
3109 // Convert the source operand to the corresponding LLVM type
3110 Location loc = DppOp.getLoc();
3111 Value src = adaptor.getSrc();
3112 Value old = adaptor.getOld();
3113 Type srcType = src.getType();
3114 Type oldType = old.getType();
3115 Type llvmType = nullptr;
3116 if (srcType.getIntOrFloatBitWidth() < 32) {
3117 llvmType = rewriter.getI32Type();
3118 } else if (isa<FloatType>(Val: srcType)) {
3119 llvmType = (srcType.getIntOrFloatBitWidth() == 32)
3120 ? rewriter.getF32Type()
3121 : rewriter.getF64Type();
3122 } else if (isa<IntegerType>(Val: srcType)) {
3123 llvmType = (srcType.getIntOrFloatBitWidth() == 32)
3124 ? rewriter.getI32Type()
3125 : rewriter.getI64Type();
3126 }
3127 auto llvmSrcIntType = typeConverter->convertType(
3128 t: rewriter.getIntegerType(width: srcType.getIntOrFloatBitWidth()));
3129
3130 // If the source type is less of 32, use bitcast to convert it to i32.
3131 auto convertOperand = [&](Value operand, Type operandType) {
3132 if (operandType.getIntOrFloatBitWidth() <= 16) {
3133 if (llvm::isa<FloatType>(Val: operandType)) {
3134 operand =
3135 LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: llvmSrcIntType, arg: operand);
3136 }
3137 auto llvmVecType = typeConverter->convertType(t: mlir::VectorType::get(
3138 shape: 32 / operandType.getIntOrFloatBitWidth(), elementType: llvmSrcIntType));
3139 Value undefVec = LLVM::UndefOp::create(builder&: rewriter, location: loc, res: llvmVecType);
3140 operand =
3141 LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: undefVec, value: operand,
3142 position: createI32Constant(rewriter, loc, value: 0));
3143 operand = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: llvmType, arg: operand);
3144 }
3145 return operand;
3146 };
3147
3148 src = convertOperand(src, srcType);
3149 old = convertOperand(old, oldType);
3150
3151 // This is taken from the following file llvm/lib/Target/AMDGPU/SIDefines.h
3152 enum DppCtrl : unsigned {
3153 ROW_SHL0 = 0x100,
3154 ROW_SHR0 = 0x110,
3155 ROW_ROR0 = 0x120,
3156 WAVE_SHL1 = 0x130,
3157 WAVE_ROL1 = 0x134,
3158 WAVE_SHR1 = 0x138,
3159 WAVE_ROR1 = 0x13C,
3160 ROW_MIRROR = 0x140,
3161 ROW_HALF_MIRROR = 0x141,
3162 BCAST15 = 0x142,
3163 BCAST31 = 0x143,
3164 };
3165
3166 auto kind = DppOp.getKind();
3167 auto permArgument = DppOp.getPermArgument();
3168 uint32_t DppCtrl = 0;
3169
3170 switch (kind) {
3171
3172 case DPPPerm::quad_perm: {
3173 auto quadPermAttr = cast<ArrayAttr>(Val&: *permArgument);
3174 int32_t i = 0;
3175 for (auto elem : quadPermAttr.getAsRange<IntegerAttr>()) {
3176 uint32_t num = elem.getInt();
3177 DppCtrl |= num << (i * 2);
3178 i++;
3179 }
3180 break;
3181 }
3182 case DPPPerm::row_shl: {
3183 auto intAttr = cast<IntegerAttr>(Val&: *permArgument);
3184 DppCtrl = intAttr.getInt() + DppCtrl::ROW_SHL0;
3185 break;
3186 }
3187 case DPPPerm::row_shr: {
3188 auto intAttr = cast<IntegerAttr>(Val&: *permArgument);
3189 DppCtrl = intAttr.getInt() + DppCtrl::ROW_SHR0;
3190 break;
3191 }
3192 case DPPPerm::row_ror: {
3193 auto intAttr = cast<IntegerAttr>(Val&: *permArgument);
3194 DppCtrl = intAttr.getInt() + DppCtrl::ROW_ROR0;
3195 break;
3196 }
3197 case DPPPerm::wave_shl:
3198 DppCtrl = DppCtrl::WAVE_SHL1;
3199 break;
3200 case DPPPerm::wave_shr:
3201 DppCtrl = DppCtrl::WAVE_SHR1;
3202 break;
3203 case DPPPerm::wave_rol:
3204 DppCtrl = DppCtrl::WAVE_ROL1;
3205 break;
3206 case DPPPerm::wave_ror:
3207 DppCtrl = DppCtrl::WAVE_ROR1;
3208 break;
3209 case DPPPerm::row_mirror:
3210 DppCtrl = DppCtrl::ROW_MIRROR;
3211 break;
3212 case DPPPerm::row_half_mirror:
3213 DppCtrl = DppCtrl::ROW_HALF_MIRROR;
3214 break;
3215 case DPPPerm::row_bcast_15:
3216 DppCtrl = DppCtrl::BCAST15;
3217 break;
3218 case DPPPerm::row_bcast_31:
3219 DppCtrl = DppCtrl::BCAST31;
3220 break;
3221 }
3222
3223 // Check for row_mask, bank_mask, bound_ctrl if they exist and create
3224 // constants
3225 auto rowMask = DppOp.getRowMask();
3226 auto bankMask = DppOp.getBankMask();
3227 bool boundCtrl = DppOp.getBoundCtrl();
3228
3229 // create a ROCDL_DPPMovOp instruction with the appropriate attributes
3230 auto dppMovOp =
3231 ROCDL::DPPUpdateOp::create(builder&: rewriter, location: loc, res: llvmType, old, src, dppCtrl: DppCtrl,
3232 rowMask, bankMask, boundCtrl);
3233
3234 Value result = dppMovOp.getRes();
3235 if (srcType.getIntOrFloatBitWidth() < 32) {
3236 result = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: llvmSrcIntType, arg: result);
3237 if (!llvm::isa<IntegerType>(Val: srcType)) {
3238 result = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: srcType, arg: result);
3239 }
3240 }
3241
3242 // We are replacing the AMDGPU_DPPOp instruction with the new
3243 // ROCDL_DPPMovOp instruction
3244 rewriter.replaceOp(op: DppOp, newValues: ValueRange(result));
3245 return success();
3246 }
3247};
3248
3249struct AMDGPUSwizzleBitModeLowering
3250 : public ConvertOpToLLVMPattern<SwizzleBitModeOp> {
3251 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
3252
3253 LogicalResult
3254 matchAndRewrite(SwizzleBitModeOp op, OpAdaptor adaptor,
3255 ConversionPatternRewriter &rewriter) const override {
3256 Location loc = op.getLoc();
3257 Type i32 = rewriter.getI32Type();
3258 Value src = adaptor.getSrc();
3259 SmallVector<Value> decomposed;
3260 if (failed(Result: LLVM::decomposeValue(builder&: rewriter, loc, src, dstType: i32, result&: decomposed)))
3261 return rewriter.notifyMatchFailure(arg&: op,
3262 msg: "failed to decompose value to i32");
3263 unsigned andMask = op.getAndMask();
3264 unsigned orMask = op.getOrMask();
3265 unsigned xorMask = op.getXorMask();
3266
3267 // bit 15 is 0 for the BitMode swizzle.
3268 // https://gpuopen.com/learn/amd-gcn-assembly-cross-lane-operations/
3269 unsigned mask = andMask | (orMask << 5) | (xorMask << 10);
3270 Value maskValue = createI32Constant(rewriter, loc, value: mask);
3271 SmallVector<Value> swizzled;
3272 for (Value v : decomposed) {
3273 Value res =
3274 ROCDL::DsSwizzleOp::create(builder&: rewriter, location: loc, res: v.getType(), src: v, offset: maskValue);
3275 swizzled.emplace_back(Args&: res);
3276 }
3277
3278 Value result = LLVM::composeValue(builder&: rewriter, loc, src: swizzled, dstType: src.getType());
3279 rewriter.replaceOp(op, newValues: result);
3280 return success();
3281 }
3282};
3283
3284struct AMDGPUPermlaneLowering : public ConvertOpToLLVMPattern<PermlaneSwapOp> {
3285 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
3286
3287 AMDGPUPermlaneLowering(const LLVMTypeConverter &converter,
3288 const ROCDL::TargetInfo &target)
3289 : ConvertOpToLLVMPattern<PermlaneSwapOp>(converter), target(target) {}
3290 ROCDL::TargetInfo target;
3291
3292 LogicalResult
3293 matchAndRewrite(PermlaneSwapOp op, OpAdaptor adaptor,
3294 ConversionPatternRewriter &rewriter) const override {
3295 unsigned rowLength = op.getRowLength();
3296 bool supported = rowLength == 16
3297 ? target.has(feature: llvm::AMDGPU::FEAT_PERMLANE16_SWAP)
3298 : target.has(feature: llvm::AMDGPU::FEAT_PERMLANE32_SWAP);
3299 if (!supported)
3300 return op->emitOpError(message: "permlane_swap of row length ")
3301 << rowLength << " is not supported on " << target.getArchName();
3302
3303 Location loc = op.getLoc();
3304 Type i32 = rewriter.getI32Type();
3305 Value src = adaptor.getSrc();
3306 bool fi = op.getFetchInactive();
3307 bool boundctrl = op.getBoundCtrl();
3308
3309 SmallVector<Value> decomposed;
3310 if (failed(Result: LLVM::decomposeValue(builder&: rewriter, loc, src, dstType: i32, result&: decomposed)))
3311 return rewriter.notifyMatchFailure(arg&: op,
3312 msg: "failed to decompose value to i32");
3313
3314 SmallVector<Value> permuted;
3315 for (Value v : decomposed) {
3316 Value res;
3317 Type i32pair = LLVM::LLVMStructType::getLiteral(
3318 context: rewriter.getContext(), types: {v.getType(), v.getType()});
3319
3320 if (rowLength == 16)
3321 res = ROCDL::Permlane16SwapOp::create(builder&: rewriter, location: loc, res: i32pair, old: v, src: v, fi,
3322 boundControl: boundctrl);
3323 else if (rowLength == 32)
3324 res = ROCDL::Permlane32SwapOp::create(builder&: rewriter, location: loc, res: i32pair, old: v, src: v, fi,
3325 boundControl: boundctrl);
3326 else
3327 llvm_unreachable("unsupported row length");
3328
3329 Value vdst0 = LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: res, position: {0});
3330 Value vdst1 = LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: res, position: {1});
3331
3332 Value isEqual = LLVM::ICmpOp::create(builder&: rewriter, location: loc,
3333 predicate: LLVM::ICmpPredicate::eq, lhs: vdst0, rhs: v);
3334
3335 // Per `permlane(16|32)` semantics: if the first extracted element equals
3336 // 'v', the result is the second element; otherwise it is the first.
3337 Value vdstNew =
3338 LLVM::SelectOp::create(builder&: rewriter, location: loc, condition: isEqual, trueValue: vdst1, falseValue: vdst0);
3339 permuted.emplace_back(Args&: vdstNew);
3340 }
3341
3342 Value result = LLVM::composeValue(builder&: rewriter, loc, src: permuted, dstType: src.getType());
3343 rewriter.replaceOp(op, newValues: result);
3344 return success();
3345 }
3346};
3347
3348struct AMDGPUPermlaneVarLowering
3349 : public ConvertOpToLLVMPattern<PermlaneVarOp> {
3350 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
3351
3352 AMDGPUPermlaneVarLowering(const LLVMTypeConverter &converter,
3353 const ROCDL::TargetInfo &target)
3354 : ConvertOpToLLVMPattern<PermlaneVarOp>(converter), target(target) {}
3355 ROCDL::TargetInfo target;
3356
3357 LogicalResult
3358 matchAndRewrite(PermlaneVarOp op, OpAdaptor adaptor,
3359 ConversionPatternRewriter &rewriter) const override {
3360 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX12_INSTS))
3361 return op->emitOpError(message: "permlane_var is only supported on GFX12+");
3362
3363 Location loc = op.getLoc();
3364 Type i32 = rewriter.getI32Type();
3365 Value src = adaptor.getSrc();
3366 Value selector = adaptor.getSelector();
3367 bool cross = op.getCross();
3368 bool fi = op.getFetchInactive();
3369 bool boundCtrl = op.getBoundCtrl();
3370
3371 SmallVector<Value> decomposed;
3372 if (failed(Result: LLVM::decomposeValue(builder&: rewriter, loc, src, dstType: i32, result&: decomposed)))
3373 return rewriter.notifyMatchFailure(arg&: op,
3374 msg: "failed to decompose value to i32");
3375
3376 SmallVector<Value> permuted;
3377 for (Value v : decomposed) {
3378 Value res;
3379 if (cross)
3380 res = ROCDL::PermlaneX16VarOp::create(builder&: rewriter, location: loc, res: i32, old: v, src0: v,
3381 src1: selector, fi, boundControl: boundCtrl);
3382 else
3383 res = ROCDL::Permlane16VarOp::create(builder&: rewriter, location: loc, res: i32, old: v, src0: v, src1: selector,
3384 fi, boundControl: boundCtrl);
3385 permuted.emplace_back(Args&: res);
3386 }
3387
3388 Value result = LLVM::composeValue(builder&: rewriter, loc, src: permuted, dstType: src.getType());
3389 rewriter.replaceOp(op, newValues: result);
3390 return success();
3391 }
3392};
3393
3394//===----------------------------------------------------------------------===//
3395// In-LDS Barrier Operations
3396//===----------------------------------------------------------------------===//
3397
3398// Bit layout of ds_barrier_state (as i64):
3399// [63:32] init count (32 bits)
3400// [31:29] phase (3 bits)
3401// [28:0] pending count (29 bits)
3402constexpr int32_t kDsBarrierPendingCountBitWidth = 29;
3403constexpr int32_t kDsBarrierPhasePos = kDsBarrierPendingCountBitWidth;
3404constexpr int32_t kDsBarrierInitCountPos = 32;
3405constexpr int32_t kDsBarrierPendingCountMask =
3406 (1 << kDsBarrierPendingCountBitWidth) - 1;
3407
3408struct DsBarrierInitOpLowering
3409 : public ConvertOpToLLVMPattern<DsBarrierInitOp> {
3410 ROCDL::TargetInfo target;
3411
3412 DsBarrierInitOpLowering(const LLVMTypeConverter &converter,
3413 const ROCDL::TargetInfo &target)
3414 : ConvertOpToLLVMPattern<DsBarrierInitOp>(converter), target(target) {}
3415
3416 LogicalResult
3417 matchAndRewrite(DsBarrierInitOp op, OpAdaptor adaptor,
3418 ConversionPatternRewriter &rewriter) const override {
3419 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
3420 return op->emitOpError(message: "only supported on gfx1250+");
3421
3422 Location loc = op.getLoc();
3423 Type i64 = rewriter.getI64Type();
3424
3425 MemRefType memrefType = cast<MemRefType>(Val: op.getBase().getType());
3426 Value ptr = getStridedElementPtr(rewriter, loc, type: memrefType,
3427 memRefDesc: adaptor.getBase(), indices: adaptor.getIndices());
3428
3429 // Note: We give participants as the number of arrivals that have to occur
3430 // before the phase changes. Hardware changes the phase when updating the
3431 // pending count would underflow, so we subtract 1 to get the behavior we're
3432 // looking for.
3433 Value initCount =
3434 LLVM::SubOp::create(builder&: rewriter, location: loc, lhs: adaptor.getParticipants(),
3435 rhs: createI32Constant(rewriter, loc, value: 1));
3436
3437 // Just a bit of paranoia, but this also allows for configurable width if
3438 // that becomes a thing.
3439 Value countMask =
3440 createI32Constant(rewriter, loc, value: kDsBarrierPendingCountMask);
3441 Value maskedCount32 =
3442 LLVM::AndOp::create(builder&: rewriter, location: loc, lhs: initCount, rhs: countMask);
3443 Value maskedCount = LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: i64, arg: maskedCount32);
3444
3445 Value initCountShifted = LLVM::ShlOp::create(
3446 builder&: rewriter, location: loc, lhs: maskedCount,
3447 rhs: createI64Constant(rewriter, loc, value: kDsBarrierInitCountPos));
3448 Value barrierState =
3449 LLVM::OrOp::create(builder&: rewriter, location: loc, lhs: initCountShifted, rhs: maskedCount);
3450
3451 LLVM::StoreOp::create(
3452 builder&: rewriter, location: loc, value: barrierState, addr: ptr, /*alignment=*/8, /*isVolatile=*/false,
3453 /*isNonTemporal=*/false,
3454 /*isInvariantGroup=*/false, ordering: LLVM::AtomicOrdering::release,
3455 /*syncscope=*/"workgroup");
3456
3457 rewriter.eraseOp(op);
3458 return success();
3459 }
3460};
3461
3462struct DsBarrierPollStateOpLowering
3463 : public ConvertOpToLLVMPattern<DsBarrierPollStateOp> {
3464 ROCDL::TargetInfo target;
3465
3466 DsBarrierPollStateOpLowering(const LLVMTypeConverter &converter,
3467 const ROCDL::TargetInfo &target)
3468 : ConvertOpToLLVMPattern<DsBarrierPollStateOp>(converter),
3469 target(target) {}
3470
3471 LogicalResult
3472 matchAndRewrite(DsBarrierPollStateOp op, OpAdaptor adaptor,
3473 ConversionPatternRewriter &rewriter) const override {
3474 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
3475 return op->emitOpError(message: "only supported on gfx1250+");
3476
3477 Location loc = op.getLoc();
3478 Type i64 = rewriter.getI64Type();
3479
3480 MemRefType memrefType = cast<MemRefType>(Val: op.getBase().getType());
3481 Value ptr = getStridedElementPtr(rewriter, loc, type: memrefType,
3482 memRefDesc: adaptor.getBase(), indices: adaptor.getIndices());
3483
3484 // Atomic load with workgroup scope and acquire ordering should be what
3485 // we're looking for.
3486 rewriter.replaceOpWithNewOp<LLVM::LoadOp>(
3487 op, args&: i64, args&: ptr, /*alignment=*/args: 8, /*volatile_=*/args: false,
3488 /*nontemporal=*/args: false, /*invariant=*/args: false,
3489 /*invariantGroup=*/args: false, /*noundef=*/args: false,
3490 args: LLVM::AtomicOrdering::acquire,
3491 /*syncscope=*/args: "workgroup");
3492 return success();
3493 }
3494};
3495
3496struct DsAsyncBarrierArriveOpLowering
3497 : public ConvertOpToLLVMPattern<DsAsyncBarrierArriveOp> {
3498 ROCDL::TargetInfo target;
3499
3500 DsAsyncBarrierArriveOpLowering(const LLVMTypeConverter &converter,
3501 const ROCDL::TargetInfo &target)
3502 : ConvertOpToLLVMPattern<DsAsyncBarrierArriveOp>(converter),
3503 target(target) {}
3504
3505 LogicalResult
3506 matchAndRewrite(DsAsyncBarrierArriveOp op, OpAdaptor adaptor,
3507 ConversionPatternRewriter &rewriter) const override {
3508 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
3509 return op->emitOpError(message: "only supported on gfx1250+");
3510
3511 Location loc = op.getLoc();
3512
3513 MemRefType memrefType = cast<MemRefType>(Val: op.getBase().getType());
3514 Value ptr = getStridedElementPtr(rewriter, loc, type: memrefType,
3515 memRefDesc: adaptor.getBase(), indices: adaptor.getIndices());
3516
3517 rewriter.replaceOpWithNewOp<ROCDL::DsAtomicAsyncBarrierArriveOp>(
3518 op, args&: ptr, /*alias_scopes=*/args: nullptr, /*noalias_scopes=*/args: nullptr,
3519 /*tbaa=*/args: nullptr);
3520 return success();
3521 }
3522};
3523
3524struct DsBarrierArriveOpLowering
3525 : public ConvertOpToLLVMPattern<DsBarrierArriveOp> {
3526 ROCDL::TargetInfo target;
3527
3528 DsBarrierArriveOpLowering(const LLVMTypeConverter &converter,
3529 const ROCDL::TargetInfo &target)
3530 : ConvertOpToLLVMPattern<DsBarrierArriveOp>(converter), target(target) {}
3531
3532 LogicalResult
3533 matchAndRewrite(DsBarrierArriveOp op, OpAdaptor adaptor,
3534 ConversionPatternRewriter &rewriter) const override {
3535 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
3536 return op->emitOpError(message: "only supported on gfx1250+");
3537
3538 Location loc = op.getLoc();
3539 Type i64 = rewriter.getI64Type();
3540
3541 MemRefType memrefType = cast<MemRefType>(Val: op.getBase().getType());
3542 Value ptr = getStridedElementPtr(rewriter, loc, type: memrefType,
3543 memRefDesc: adaptor.getBase(), indices: adaptor.getIndices());
3544
3545 rewriter.replaceOpWithNewOp<ROCDL::DsAtomicBarrierArriveRtnOp>(
3546 op, args&: i64, args&: ptr, args: adaptor.getCount(), /*alias_scopes=*/args: nullptr,
3547 /*noalias_scopes=*/args: nullptr, /*tbaa=*/args: nullptr);
3548 return success();
3549 }
3550};
3551
3552struct DsBarrierStatePhaseOpLowering
3553 : public ConvertOpToLLVMPattern<DsBarrierStatePhaseOp> {
3554 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
3555
3556 LogicalResult
3557 matchAndRewrite(DsBarrierStatePhaseOp op, OpAdaptor adaptor,
3558 ConversionPatternRewriter &rewriter) const override {
3559 Location loc = op.getLoc();
3560 Type i32 = rewriter.getI32Type();
3561
3562 Value state = adaptor.getState();
3563
3564 Value noInitCount = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i32, arg: state);
3565 Value phase = LLVM::LShrOp::create(
3566 builder&: rewriter, location: loc, lhs: noInitCount,
3567 rhs: createI32Constant(rewriter, loc, value: kDsBarrierPhasePos));
3568
3569 rewriter.replaceOp(op, newValues: phase);
3570 return success();
3571 }
3572};
3573
3574struct DsBarrierStatePendingCountOpLowering
3575 : public ConvertOpToLLVMPattern<DsBarrierStatePendingCountOp> {
3576 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
3577
3578 LogicalResult
3579 matchAndRewrite(DsBarrierStatePendingCountOp op, OpAdaptor adaptor,
3580 ConversionPatternRewriter &rewriter) const override {
3581 Location loc = op.getLoc();
3582 Type i32 = rewriter.getI32Type();
3583
3584 Value state = adaptor.getState();
3585
3586 Value noInitCount = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i32, arg: state);
3587 Value pendingCount = LLVM::AndOp::create(
3588 builder&: rewriter, location: loc, lhs: noInitCount,
3589 rhs: createI32Constant(rewriter, loc,
3590 value: static_cast<uint32_t>(kDsBarrierPendingCountMask)));
3591
3592 rewriter.replaceOp(op, newValues: pendingCount);
3593 return success();
3594 }
3595};
3596
3597struct DsBarrierStateInitCountOpLowering
3598 : public ConvertOpToLLVMPattern<DsBarrierStateInitCountOp> {
3599 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
3600
3601 LogicalResult
3602 matchAndRewrite(DsBarrierStateInitCountOp op, OpAdaptor adaptor,
3603 ConversionPatternRewriter &rewriter) const override {
3604 Location loc = op.getLoc();
3605 Type i32 = rewriter.getI32Type();
3606
3607 Value state = adaptor.getState();
3608
3609 Value initCountI64 = LLVM::LShrOp::create(
3610 builder&: rewriter, location: loc, lhs: state,
3611 rhs: createI64Constant(rewriter, loc, value: kDsBarrierInitCountPos));
3612 Value initCount = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i32, arg: initCountI64);
3613
3614 rewriter.replaceOp(op, newValues: initCount);
3615 return success();
3616 }
3617};
3618
3619struct DsBarrierStatePhaseParityLowering
3620 : public ConvertOpToLLVMPattern<DsBarrierStatePhaseParity> {
3621 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
3622
3623 LogicalResult
3624 matchAndRewrite(DsBarrierStatePhaseParity op, OpAdaptor adaptor,
3625 ConversionPatternRewriter &rewriter) const override {
3626 Location loc = op.getLoc();
3627 Type i1 = rewriter.getI1Type();
3628
3629 Value state = adaptor.getState();
3630
3631 Value noInitCount =
3632 LLVM::TruncOp::create(builder&: rewriter, location: loc, res: rewriter.getI32Type(), arg: state);
3633 Value phase = LLVM::LShrOp::create(
3634 builder&: rewriter, location: loc, lhs: noInitCount,
3635 rhs: createI32Constant(rewriter, loc, value: kDsBarrierPhasePos));
3636 Value parity = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i1, arg: phase);
3637
3638 rewriter.replaceOp(op, newValues: parity);
3639 return success();
3640 }
3641};
3642
3643//===----------------------------------------------------------------------===//
3644// Tensor Data Mover (TDM)
3645//===----------------------------------------------------------------------===//
3646
3647static Value setValueAtOffset(ConversionPatternRewriter &rewriter, Location loc,
3648 Value accumulator, Value value, int64_t shift) {
3649 shift = shift % 32;
3650 Value shiftAmount;
3651 if (shift != 0) {
3652 shiftAmount = createI32Constant(rewriter, loc, value: shift % 32);
3653 value = LLVM::ShlOp::create(builder&: rewriter, location: loc, lhs: value, rhs: shiftAmount);
3654 }
3655
3656 if (matchPattern(value: accumulator, pattern: mlir::m_Zero()))
3657 return value;
3658
3659 constexpr bool isDisjoint = true;
3660 return LLVM::OrOp::create(builder&: rewriter, location: loc, lhs: accumulator, rhs: value, isDisjoint);
3661}
3662
3663template <typename BaseOp>
3664struct AMDGPUMakeDmaBaseLowering : public ConvertOpToLLVMPattern<BaseOp> {
3665 using ConvertOpToLLVMPattern<BaseOp>::ConvertOpToLLVMPattern;
3666 using Adaptor = typename ConvertOpToLLVMPattern<BaseOp>::OpAdaptor;
3667
3668 AMDGPUMakeDmaBaseLowering(const LLVMTypeConverter &converter,
3669 const ROCDL::TargetInfo &target)
3670 : ConvertOpToLLVMPattern<BaseOp>(converter), target(target) {}
3671 ROCDL::TargetInfo target;
3672
3673 LogicalResult
3674 matchAndRewrite(BaseOp op, Adaptor adaptor,
3675 ConversionPatternRewriter &rewriter) const override {
3676 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
3677 return op->emitOpError("make_dma_base is only supported on gfx1250");
3678
3679 Location loc = op.getLoc();
3680
3681 constexpr int32_t constlen = 4;
3682 Value consts[constlen];
3683 for (int64_t i = 0; i < constlen; ++i)
3684 consts[i] = createI32Constant(rewriter, loc, value: i);
3685
3686 constexpr int32_t sgprslen = constlen;
3687 Value sgprs[sgprslen];
3688 for (int64_t i = 0; i < sgprslen; ++i) {
3689 sgprs[i] = consts[0];
3690 }
3691
3692 sgprs[0] = consts[1];
3693
3694 if constexpr (BaseOp::isGather()) {
3695 sgprs[0] = setValueAtOffset(rewriter, loc, accumulator: sgprs[0], value: consts[1], shift: 30);
3696
3697 auto type = cast<TDMGatherBaseType>(op.getResult().getType());
3698 Type indexType = type.getIndexType();
3699 unsigned indexSize = indexType.getIntOrFloatBitWidth();
3700 assert(llvm::is_contained({16u, 32u}, indexSize) &&
3701 "expected index_size to be 16 or 32");
3702 unsigned idx = (indexSize / 16) - 1;
3703
3704 if (idx)
3705 sgprs[0] = setValueAtOffset(rewriter, loc, accumulator: sgprs[0], value: consts[1], shift: 31);
3706 }
3707
3708 ValueRange ldsIndices = adaptor.getLdsIndices();
3709 Value lds = adaptor.getLds();
3710 auto ldsMemRefType = cast<MemRefType>(op.getLds().getType());
3711
3712 Value ldsPtr = ConvertToLLVMPattern::getStridedElementPtr(
3713 rewriter, loc, type: ldsMemRefType, memRefDesc: lds, indices: ldsIndices);
3714
3715 ValueRange globalIndices = adaptor.getGlobalIndices();
3716 Value global = adaptor.getGlobal();
3717 auto globalMemRefType = cast<MemRefType>(op.getGlobal().getType());
3718
3719 Value globalPtr = ConvertToLLVMPattern::getStridedElementPtr(
3720 rewriter, loc, type: globalMemRefType, memRefDesc: global, indices: globalIndices);
3721
3722 Type i32 = rewriter.getI32Type();
3723 Type i64 = rewriter.getI64Type();
3724
3725 sgprs[1] = LLVM::PtrToIntOp::create(builder&: rewriter, location: loc, res: i32, arg: ldsPtr);
3726 Value castForGlobalAddr =
3727 LLVM::PtrToIntOp::create(builder&: rewriter, location: loc, res: i64, arg: globalPtr);
3728
3729 sgprs[2] = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i32, arg: castForGlobalAddr);
3730
3731 Value shift = LLVM::LShrOp::create(builder&: rewriter, location: loc, lhs: castForGlobalAddr,
3732 rhs: createI64Constant(rewriter, loc, value: 32));
3733
3734 Value highHalf = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i32, arg: shift);
3735
3736 Value mask = createI32Constant(rewriter, loc, value: (1ull << 25) - 1);
3737 highHalf = LLVM::AndOp::create(builder&: rewriter, location: loc, lhs: highHalf, rhs: mask);
3738
3739 sgprs[3] = setValueAtOffset(rewriter, loc, accumulator: highHalf, value: consts[2], shift: 30);
3740
3741 Type v4i32 = this->typeConverter->convertType(VectorType::get(shape: 4, elementType: i32));
3742 assert(v4i32 && "expected type conversion to succeed");
3743 Value result = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: v4i32);
3744
3745 for (auto [sgpr, constant] : llvm::zip_equal(t&: sgprs, u&: consts))
3746 result =
3747 LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: result, value: sgpr, position: constant);
3748
3749 rewriter.replaceOp(op, result);
3750 return success();
3751 }
3752};
3753
3754template <typename DescriptorOp>
3755struct AMDGPULowerDescriptor : public ConvertOpToLLVMPattern<DescriptorOp> {
3756 using ConvertOpToLLVMPattern<DescriptorOp>::ConvertOpToLLVMPattern;
3757 using OpAdaptor = typename ConvertOpToLLVMPattern<DescriptorOp>::OpAdaptor;
3758
3759 AMDGPULowerDescriptor(const LLVMTypeConverter &converter,
3760 const ROCDL::TargetInfo &target)
3761 : ConvertOpToLLVMPattern<DescriptorOp>(converter), target(target) {}
3762 ROCDL::TargetInfo target;
3763
3764 Value getDGroup0(OpAdaptor &adaptor) const { return adaptor.getBase(); }
3765
3766 Value setWorkgroupMask(DescriptorOp op, OpAdaptor &adaptor,
3767 ConversionPatternRewriter &rewriter, Location loc,
3768 Value sgpr0) const {
3769 Value mask = op.getWorkgroupMask();
3770 if (!mask)
3771 return sgpr0;
3772
3773 Type i16 = rewriter.getI16Type();
3774 mask = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: i16, arg: mask);
3775 Type i32 = rewriter.getI32Type();
3776 Value extendedMask = LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: i32, arg: mask);
3777 return setValueAtOffset(rewriter, loc, accumulator: sgpr0, value: extendedMask, shift: 0);
3778 }
3779
3780 Value setDataSize(DescriptorOp op, OpAdaptor &adaptor,
3781 ConversionPatternRewriter &rewriter, Location loc,
3782 Value sgpr0, ArrayRef<Value> consts) const {
3783 unsigned elementTypeWidthInBits = op.getElementTypeWidth();
3784 assert(llvm::is_contained({8u, 16u, 32u, 64u}, elementTypeWidthInBits) &&
3785 "expected type width to be 8, 16, 32, or 64.");
3786 int64_t idx = llvm::Log2_32(Value: elementTypeWidthInBits / 8);
3787 Value size = consts[idx];
3788 return setValueAtOffset(rewriter, loc, accumulator: sgpr0, value: size, shift: 16);
3789 }
3790
3791 Value setAtomicBarrier(DescriptorOp op, OpAdaptor &adaptor,
3792 ConversionPatternRewriter &rewriter, Location loc,
3793 Value sgpr0, ArrayRef<Value> consts) const {
3794 if (!adaptor.getAtomicBarrierAddress())
3795 return sgpr0;
3796
3797 return setValueAtOffset(rewriter, loc, accumulator: sgpr0, value: consts[1], shift: 18);
3798 }
3799
3800 Value setIterateEnable(DescriptorOp op, OpAdaptor &adaptor,
3801 ConversionPatternRewriter &rewriter, Location loc,
3802 Value sgpr0, ArrayRef<Value> consts) const {
3803 if (!adaptor.getGlobalIncrement())
3804 return sgpr0;
3805
3806 // Value is ignored when in gather mode.
3807 // TODO: emit error earlier?
3808 return setValueAtOffset(rewriter, loc, accumulator: sgpr0, value: consts[1], shift: 19);
3809 }
3810
3811 Value setPadEnable(DescriptorOp op, OpAdaptor &adaptor,
3812 ConversionPatternRewriter &rewriter, Location loc,
3813 Value sgpr0, ArrayRef<Value> consts) const {
3814 if (!op.getPadAmount())
3815 return sgpr0;
3816
3817 return setValueAtOffset(rewriter, loc, accumulator: sgpr0, value: consts[1], shift: 20);
3818 }
3819
3820 Value setEarlyTimeout(DescriptorOp op, OpAdaptor &adaptor,
3821 ConversionPatternRewriter &rewriter, Location loc,
3822 Value sgpr0, ArrayRef<Value> consts) const {
3823 if (!op.getWorkgroupMask())
3824 return sgpr0;
3825
3826 return setValueAtOffset(rewriter, loc, accumulator: sgpr0, value: consts[1], shift: 21);
3827 }
3828
3829 Value setPadInterval(DescriptorOp op, OpAdaptor &adaptor,
3830 ConversionPatternRewriter &rewriter, Location loc,
3831 Value sgpr0, ArrayRef<Value> consts) const {
3832 if (!op.getPadAmount())
3833 return sgpr0;
3834
3835 // pre-condition: padInterval can be a power of two between 2 and 256.
3836 // TODO: Validation if the value breaks the pre-condition.
3837 // If the pre-condition fails, there is a possibility of
3838 // affecting the higher bits. In a following PR implement
3839 // RuntimeVerifiableOpInterface that instruments conditions that need to be
3840 // checked at runtime.
3841 IntegerType i32 = rewriter.getI32Type();
3842 Value padInterval = adaptor.getPadInterval();
3843 padInterval = LLVM::CountTrailingZerosOp::create(builder&: rewriter, location: loc, res: i32,
3844 in: padInterval, is_zero_poison: false);
3845 padInterval = LLVM::SubOp::create(builder&: rewriter, location: loc, lhs: padInterval, rhs: consts[1]);
3846 // post-condition: padInterval can be a value between 0 and 7.
3847 return setValueAtOffset(rewriter, loc, accumulator: sgpr0, value: padInterval, shift: 22);
3848 }
3849
3850 Value setPadAmount(DescriptorOp op, OpAdaptor &adaptor,
3851 ConversionPatternRewriter &rewriter, Location loc,
3852 Value sgpr0, ArrayRef<Value> consts) const {
3853 if (!op.getPadAmount())
3854 return sgpr0;
3855
3856 // pre-condition: padAmount is a value between 1-128.
3857 // TODO: Validation if the value breaks the pre-condition.
3858 // If the pre-condition fails, there is a possibility of
3859 // affecting the higher bits. In a following PR implement
3860 // RuntimeVerifiableOpInterface that instruments conditions that need to be
3861 // checked at runtime.
3862 Value padAmount = adaptor.getPadAmount();
3863 padAmount = LLVM::SubOp::create(builder&: rewriter, location: loc, lhs: padAmount, rhs: consts[1]);
3864 // post-condition: padAmount is a value between 0-127.
3865 return setValueAtOffset(rewriter, loc, accumulator: sgpr0, value: padAmount, shift: 25);
3866 }
3867
3868 Value setAtomicBarrierAddress(DescriptorOp op, OpAdaptor &adaptor,
3869 ConversionPatternRewriter &rewriter,
3870 Location loc, Value sgpr1,
3871 ArrayRef<Value> consts) const {
3872 if (!adaptor.getAtomicBarrierAddress())
3873 return sgpr1;
3874
3875 Value atomicBarrierAddress = adaptor.getAtomicBarrierAddress();
3876 auto barrierAddressTy =
3877 cast<MemRefType>(op.getAtomicBarrierAddress().getType());
3878 ValueRange atomicBarrierIndices = adaptor.getAtomicBarrierIndices();
3879 atomicBarrierAddress = ConvertToLLVMPattern::getStridedElementPtr(
3880 rewriter, loc, type: barrierAddressTy, memRefDesc: atomicBarrierAddress,
3881 indices: atomicBarrierIndices);
3882 IntegerType i32 = rewriter.getI32Type();
3883 // pre-condition: atomicBarrierAddress is aligned to 8 bytes which implies
3884 // that the 3 LSBs are zero.
3885 // TODO: Validation if the value breaks the pre-condition.
3886 // In a following PR implement RuntimeVerifiableOpInterface
3887 // that instruments conditions that need to be checked at runtime.
3888 atomicBarrierAddress =
3889 LLVM::PtrToIntOp::create(builder&: rewriter, location: loc, res: i32, arg: atomicBarrierAddress);
3890 atomicBarrierAddress =
3891 LLVM::LShrOp::create(builder&: rewriter, location: loc, lhs: atomicBarrierAddress, rhs: consts[3]);
3892 Value mask = createI32Constant(rewriter, loc, value: 0xFFFF);
3893 atomicBarrierAddress =
3894 LLVM::AndOp::create(builder&: rewriter, location: loc, lhs: atomicBarrierAddress, rhs: mask);
3895 return setValueAtOffset(rewriter, loc, accumulator: sgpr1, value: atomicBarrierAddress, shift: 32);
3896 }
3897
3898 std::pair<Value, Value> setTensorDimX(DescriptorOp op, OpAdaptor &adaptor,
3899 ConversionPatternRewriter &rewriter,
3900 Location loc, Value sgpr1, Value sgpr2,
3901 ArrayRef<Value> consts, uint64_t dimX,
3902 uint32_t offset) const {
3903 ArrayRef<int64_t> globalStaticSizes = adaptor.getGlobalStaticSizes();
3904 ValueRange globalDynamicSizes = adaptor.getGlobalDynamicSizes();
3905 SmallVector<OpFoldResult> mixedGlobalSizes =
3906 getMixedValues(staticValues: globalStaticSizes, dynamicValues: globalDynamicSizes, b&: rewriter);
3907 if (mixedGlobalSizes.size() <= dimX)
3908 return {sgpr1, sgpr2};
3909
3910 OpFoldResult tensorDimXOpFoldResult = *(mixedGlobalSizes.rbegin() + dimX);
3911 // pre-condition: tensorDimX is less than 2^32-1
3912 // TODO: Validation if the value breaks the pre-condition.
3913 // In a following PR implement RuntimeVerifiableOpInterface that instruments
3914 // conditions that need to be checked at runtime. This could also be fixed
3915 // by saying that mixedGlobalSizes is a DynamicI32List.
3916 Value tensorDimX;
3917 if (auto attr = dyn_cast<Attribute>(Val&: tensorDimXOpFoldResult)) {
3918 tensorDimX =
3919 createI32Constant(rewriter, loc, value: cast<IntegerAttr>(Val&: attr).getInt());
3920 } else {
3921 IntegerType i32 = rewriter.getI32Type();
3922 tensorDimX = cast<Value>(Val&: tensorDimXOpFoldResult);
3923 tensorDimX = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i32, arg: tensorDimX);
3924 }
3925
3926 sgpr1 = setValueAtOffset(rewriter, loc, accumulator: sgpr1, value: tensorDimX, shift: offset);
3927
3928 Value c16 = createI32Constant(rewriter, loc, value: 16);
3929 Value tensorDimXHigh = LLVM::LShrOp::create(builder&: rewriter, location: loc, lhs: tensorDimX, rhs: c16);
3930 sgpr2 = setValueAtOffset(rewriter, loc, accumulator: sgpr2, value: tensorDimXHigh, shift: offset + 16);
3931 return {sgpr1, sgpr2};
3932 }
3933
3934 std::pair<Value, Value> setTensorDim0(DescriptorOp op, OpAdaptor &adaptor,
3935 ConversionPatternRewriter &rewriter,
3936 Location loc, Value sgpr1, Value sgpr2,
3937 ArrayRef<Value> consts) const {
3938 return setTensorDimX(op, adaptor, rewriter, loc, sgpr1, sgpr2, consts, 0,
3939 48);
3940 }
3941
3942 std::pair<Value, Value> setTensorDim1(DescriptorOp op, OpAdaptor &adaptor,
3943 ConversionPatternRewriter &rewriter,
3944 Location loc, Value sgpr2, Value sgpr3,
3945 ArrayRef<Value> consts) const {
3946 return setTensorDimX(op, adaptor, rewriter, loc, sgpr2, sgpr3, consts, 1,
3947 80);
3948 }
3949
3950 Value setTileDimX(DescriptorOp op, OpAdaptor &adaptor,
3951 ConversionPatternRewriter &rewriter, Location loc,
3952 Value sgpr, ArrayRef<Value> consts, size_t dimX,
3953 int64_t offset) const {
3954 ArrayRef<int64_t> sharedStaticSizes = adaptor.getSharedStaticSizes();
3955 ValueRange sharedDynamicSizes = adaptor.getSharedDynamicSizes();
3956 SmallVector<OpFoldResult> mixedSharedSizes =
3957 getMixedValues(staticValues: sharedStaticSizes, dynamicValues: sharedDynamicSizes, b&: rewriter);
3958 if (mixedSharedSizes.size() <= dimX)
3959 return sgpr;
3960
3961 OpFoldResult tileDimXOpFoldResult = *(mixedSharedSizes.rbegin() + dimX);
3962 // pre-condition: tileDimX is less than 2^16-1
3963 // TODO: Validation if the value breaks the pre-condition.
3964 // If the pre-condition fails, there is a possibility of
3965 // affecting the higher bits. In a following PR implement
3966 // RuntimeVerifiableOpInterface that instruments conditions that need to be
3967 // checked at runtime. This could also be fixed by saying that
3968 // mixedSharedSizes is a DynamicI16List.
3969 Value tileDimX;
3970 if (auto attr = dyn_cast<Attribute>(Val&: tileDimXOpFoldResult)) {
3971 tileDimX =
3972 createI32Constant(rewriter, loc, value: cast<IntegerAttr>(Val&: attr).getInt());
3973 } else {
3974 IntegerType i32 = rewriter.getI32Type();
3975 tileDimX = cast<Value>(Val&: tileDimXOpFoldResult);
3976 tileDimX = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i32, arg: tileDimX);
3977 }
3978
3979 return setValueAtOffset(rewriter, loc, accumulator: sgpr, value: tileDimX, shift: offset);
3980 }
3981
3982 Value setTileDim0(DescriptorOp op, OpAdaptor &adaptor,
3983 ConversionPatternRewriter &rewriter, Location loc,
3984 Value sgpr3, ArrayRef<Value> consts) const {
3985 return setTileDimX(op, adaptor, rewriter, loc, sgpr: sgpr3, consts, dimX: 0, offset: 112);
3986 }
3987
3988 Value setTileDim1(DescriptorOp op, OpAdaptor &adaptor,
3989 ConversionPatternRewriter &rewriter, Location loc,
3990 Value sgpr4, ArrayRef<Value> consts) const {
3991 return setTileDimX(op, adaptor, rewriter, loc, sgpr: sgpr4, consts, dimX: 1, offset: 128);
3992 }
3993
3994 Value setValidIndices(DescriptorOp op, OpAdaptor &adaptor,
3995 ConversionPatternRewriter &rewriter, Location loc,
3996 Value sgpr4, ArrayRef<Value> consts) const {
3997 auto type = cast<VectorType>(op.getIndices().getType());
3998 ArrayRef<int64_t> shape = type.getShape();
3999 assert(shape.size() == 1 && "expected shape to be of rank 1.");
4000 unsigned length = shape.back();
4001 assert(0 < length && length <= 16 && "expected length to be at most 16.");
4002 Value value = createI32Constant(rewriter, loc, value: length);
4003 return setValueAtOffset(rewriter, loc, accumulator: sgpr4, value, shift: 128);
4004 }
4005
4006 Value setTileDim1OrValidIndices(DescriptorOp op, OpAdaptor &adaptor,
4007 ConversionPatternRewriter &rewriter,
4008 Location loc, Value sgpr4,
4009 ArrayRef<Value> consts) const {
4010 if constexpr (DescriptorOp::isGather())
4011 return setValidIndices(op, adaptor, rewriter, loc, sgpr4, consts);
4012 return setTileDim1(op, adaptor, rewriter, loc, sgpr4, consts);
4013 }
4014
4015 Value setTileDim2(DescriptorOp op, OpAdaptor &adaptor,
4016 ConversionPatternRewriter &rewriter, Location loc,
4017 Value sgpr4, ArrayRef<Value> consts) const {
4018 // Value is ignored when in gather mode.
4019 if constexpr (DescriptorOp::isGather())
4020 return sgpr4;
4021 return setTileDimX(op, adaptor, rewriter, loc, sgpr: sgpr4, consts, dimX: 2, offset: 144);
4022 }
4023
4024 std::pair<Value, Value>
4025 setTensorDimXStride(DescriptorOp op, OpAdaptor &adaptor,
4026 ConversionPatternRewriter &rewriter, Location loc,
4027 Value sgprY, Value sgprZ, ArrayRef<Value> consts,
4028 size_t dimX, int64_t offset) const {
4029 ArrayRef<int64_t> globalStaticStrides = adaptor.getGlobalStaticStrides();
4030 ValueRange globalDynamicStrides = adaptor.getGlobalDynamicStrides();
4031 SmallVector<OpFoldResult> mixedGlobalStrides =
4032 getMixedValues(staticValues: globalStaticStrides, dynamicValues: globalDynamicStrides, b&: rewriter);
4033
4034 if (mixedGlobalStrides.size() <= (dimX + 1))
4035 return {sgprY, sgprZ};
4036
4037 OpFoldResult tensorDimXStrideOpFoldResult =
4038 *(mixedGlobalStrides.rbegin() + dimX + 1);
4039 // pre-condition: tensorDimXStride is less than 2^48-1
4040 // TODO: Validation if the value breaks the pre-condition.
4041 // In a following PR implement RuntimeVerifiableOpInterface that instruments
4042 // conditions that need to be checked at runtime.
4043 Value tensorDimXStride;
4044 if (auto attr = dyn_cast<Attribute>(Val&: tensorDimXStrideOpFoldResult))
4045 tensorDimXStride =
4046 createI64Constant(rewriter, loc, value: cast<IntegerAttr>(Val&: attr).getInt());
4047 else
4048 tensorDimXStride = cast<Value>(Val&: tensorDimXStrideOpFoldResult);
4049
4050 constexpr int64_t first48bits = (1ll << 48) - 1;
4051 Value mask = createI64Constant(rewriter, loc, value: first48bits);
4052 tensorDimXStride =
4053 LLVM::AndOp::create(builder&: rewriter, location: loc, lhs: mask, rhs: tensorDimXStride);
4054 IntegerType i32 = rewriter.getI32Type();
4055 Value tensorDimXStrideLow =
4056 LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i32, arg: tensorDimXStride);
4057 sgprY = setValueAtOffset(rewriter, loc, accumulator: sgprY, value: tensorDimXStrideLow, shift: offset);
4058
4059 int64_t shift = (offset % 32) == 0 ? 32 : offset % 32;
4060 Value shiftVal = createI64Constant(rewriter, loc, value: shift);
4061 Value tensorDimXStrideHigh =
4062 LLVM::LShrOp::create(builder&: rewriter, location: loc, lhs: tensorDimXStride, rhs: shiftVal);
4063 tensorDimXStrideHigh =
4064 LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i32, arg: tensorDimXStrideHigh);
4065 sgprZ = setValueAtOffset(rewriter, loc, accumulator: sgprZ, value: tensorDimXStrideHigh,
4066 shift: offset + shift);
4067 return {sgprY, sgprZ};
4068 }
4069
4070 std::pair<Value, Value>
4071 setTensorDim0Stride(DescriptorOp op, OpAdaptor &adaptor,
4072 ConversionPatternRewriter &rewriter, Location loc,
4073 Value sgpr5, Value sgpr6, ArrayRef<Value> consts) const {
4074 return setTensorDimXStride(op, adaptor, rewriter, loc, sgprY: sgpr5, sgprZ: sgpr6, consts,
4075 dimX: 0, offset: 160);
4076 }
4077
4078 std::pair<Value, Value>
4079 setTensorDim1Stride(DescriptorOp op, OpAdaptor &adaptor,
4080 ConversionPatternRewriter &rewriter, Location loc,
4081 Value sgpr5, Value sgpr6, ArrayRef<Value> consts) const {
4082 // Value is ignored when in gather mode.
4083 if constexpr (DescriptorOp::isGather())
4084 return {sgpr5, sgpr6};
4085 return setTensorDimXStride(op, adaptor, rewriter, loc, sgprY: sgpr5, sgprZ: sgpr6, consts,
4086 dimX: 1, offset: 208);
4087 }
4088
4089 Value getDGroup1(DescriptorOp op, OpAdaptor &adaptor,
4090 ConversionPatternRewriter &rewriter, Location loc,
4091 ArrayRef<Value> consts) const {
4092 Value sgprs[8];
4093 for (int64_t i = 0; i < 8; ++i) {
4094 sgprs[i] = consts[0];
4095 }
4096
4097 sgprs[0] = setWorkgroupMask(op, adaptor, rewriter, loc, sgpr0: sgprs[0]);
4098 sgprs[0] = setDataSize(op, adaptor, rewriter, loc, sgpr0: sgprs[0], consts);
4099 sgprs[0] = setAtomicBarrier(op, adaptor, rewriter, loc, sgpr0: sgprs[0], consts);
4100 sgprs[0] = setIterateEnable(op, adaptor, rewriter, loc, sgpr0: sgprs[0], consts);
4101 sgprs[0] = setPadEnable(op, adaptor, rewriter, loc, sgpr0: sgprs[0], consts);
4102 sgprs[0] = setEarlyTimeout(op, adaptor, rewriter, loc, sgpr0: sgprs[0], consts);
4103 sgprs[0] = setPadInterval(op, adaptor, rewriter, loc, sgpr0: sgprs[0], consts);
4104 sgprs[0] = setPadAmount(op, adaptor, rewriter, loc, sgpr0: sgprs[0], consts);
4105
4106 sgprs[1] =
4107 setAtomicBarrierAddress(op, adaptor, rewriter, loc, sgpr1: sgprs[1], consts);
4108 std::tie(args&: sgprs[1], args&: sgprs[2]) =
4109 setTensorDim0(op, adaptor, rewriter, loc, sgpr1: sgprs[1], sgpr2: sgprs[2], consts);
4110 std::tie(args&: sgprs[2], args&: sgprs[3]) =
4111 setTensorDim1(op, adaptor, rewriter, loc, sgpr2: sgprs[2], sgpr3: sgprs[3], consts);
4112
4113 sgprs[3] = setTileDim0(op, adaptor, rewriter, loc, sgpr3: sgprs[3], consts);
4114 sgprs[4] =
4115 setTileDim1OrValidIndices(op, adaptor, rewriter, loc, sgpr4: sgprs[4], consts);
4116 sgprs[4] = setTileDim2(op, adaptor, rewriter, loc, sgpr4: sgprs[4], consts);
4117 std::tie(args&: sgprs[5], args&: sgprs[6]) = setTensorDim0Stride(
4118 op, adaptor, rewriter, loc, sgpr5: sgprs[5], sgpr6: sgprs[6], consts);
4119 std::tie(args&: sgprs[6], args&: sgprs[7]) = setTensorDim1Stride(
4120 op, adaptor, rewriter, loc, sgpr5: sgprs[6], sgpr6: sgprs[7], consts);
4121
4122 IntegerType i32 = rewriter.getI32Type();
4123 Type v8i32 = this->typeConverter->convertType(VectorType::get(shape: 8, elementType: i32));
4124 assert(v8i32 && "expected type conversion to succeed");
4125 Value dgroup1 = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: v8i32);
4126
4127 for (auto [sgpr, constant] : llvm::zip_equal(t&: sgprs, u&: consts)) {
4128 dgroup1 =
4129 LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: dgroup1, value: sgpr, position: constant);
4130 }
4131
4132 return dgroup1;
4133 }
4134
4135 Value setTensorDimX(DescriptorOp op, OpAdaptor &adaptor,
4136 ConversionPatternRewriter &rewriter, Location loc,
4137 Value sgpr0, ArrayRef<Value> consts, int64_t dimX,
4138 int64_t offset) const {
4139 ArrayRef<int64_t> globalStaticSizes = adaptor.getGlobalStaticSizes();
4140 ValueRange globalDynamicSizes = adaptor.getGlobalDynamicSizes();
4141 SmallVector<OpFoldResult> mixedGlobalSizes =
4142 getMixedValues(staticValues: globalStaticSizes, dynamicValues: globalDynamicSizes, b&: rewriter);
4143 if (mixedGlobalSizes.size() <= static_cast<unsigned long>(dimX))
4144 return sgpr0;
4145
4146 OpFoldResult tensorDimXOpFoldResult = *(mixedGlobalSizes.rbegin() + dimX);
4147 Value tensorDimX;
4148 if (auto attr = dyn_cast<Attribute>(Val&: tensorDimXOpFoldResult)) {
4149 tensorDimX =
4150 createI32Constant(rewriter, loc, value: cast<IntegerAttr>(Val&: attr).getInt());
4151 } else {
4152 IntegerType i32 = rewriter.getI32Type();
4153 tensorDimX = cast<Value>(Val&: tensorDimXOpFoldResult);
4154 tensorDimX = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i32, arg: tensorDimX);
4155 }
4156
4157 return setValueAtOffset(rewriter, loc, accumulator: sgpr0, value: tensorDimX, shift: offset);
4158 }
4159
4160 Value setTensorDim2(DescriptorOp op, OpAdaptor &adaptor,
4161 ConversionPatternRewriter &rewriter, Location loc,
4162 Value sgpr0, ArrayRef<Value> consts) const {
4163 return setTensorDimX(op, adaptor, rewriter, loc, sgpr0, consts, 2, 0);
4164 }
4165
4166 Value truncateAndSetValueAtOffset(ConversionPatternRewriter &rewriter,
4167 Location loc, Value accumulator,
4168 Value value, int64_t shift) const {
4169
4170 IntegerType i32 = rewriter.getI32Type();
4171 value = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i32, arg: value);
4172 return setValueAtOffset(rewriter, loc, accumulator, value, shift);
4173 }
4174
4175 Value setLDSAddrIncrement(DescriptorOp op, OpAdaptor &adaptor,
4176 ConversionPatternRewriter &rewriter, Location loc,
4177 Value sgpr1, ArrayRef<Value> consts,
4178 int64_t offset) const {
4179 Value ldsAddrIncrement = adaptor.getLdsIncrement();
4180 return setValueAtOffset(rewriter, loc, accumulator: sgpr1, value: ldsAddrIncrement, shift: offset);
4181 }
4182
4183 std::pair<Value, Value>
4184 setGlobalAddrIncrement(DescriptorOp op, OpAdaptor &adaptor,
4185 ConversionPatternRewriter &rewriter, Location loc,
4186 Value sgpr2, Value sgpr3, ArrayRef<Value> consts,
4187 int64_t offset) const {
4188 Value globalAddrIncrement = adaptor.getGlobalIncrement();
4189 sgpr2 = truncateAndSetValueAtOffset(rewriter, loc, accumulator: sgpr2,
4190 value: globalAddrIncrement, shift: offset);
4191 Value shift = createI64Constant(rewriter, loc, value: 32);
4192 globalAddrIncrement =
4193 LLVM::LShrOp::create(builder&: rewriter, location: loc, lhs: globalAddrIncrement, rhs: shift);
4194 constexpr int64_t first16BitsHigh = (1ll << 16) - 1;
4195 sgpr3 = truncateAndSetValueAtOffset(rewriter, loc, accumulator: sgpr3,
4196 value: globalAddrIncrement, shift: offset + 32);
4197 Value mask = createI32Constant(rewriter, loc, value: first16BitsHigh);
4198 sgpr3 = LLVM::AndOp::create(builder&: rewriter, location: loc, lhs: sgpr3, rhs: mask);
4199 return {sgpr2, sgpr3};
4200 }
4201
4202 Value setTensorDim3OrLDSAddrIncrement(DescriptorOp op, OpAdaptor &adaptor,
4203 ConversionPatternRewriter &rewriter,
4204 Location loc, Value sgpr1,
4205 ArrayRef<Value> consts) const {
4206 Value ldsIncrement = op.getLdsIncrement();
4207 constexpr int64_t dim = 3;
4208 constexpr int64_t offset = 32;
4209 if (!ldsIncrement)
4210 return setTensorDimX(op, adaptor, rewriter, loc, sgpr1, consts, dim,
4211 offset);
4212 return setLDSAddrIncrement(op, adaptor, rewriter, loc, sgpr1, consts,
4213 offset);
4214 }
4215
4216 std::pair<Value, Value> setTensorDim2StrideOrGlobalAddrIncrement(
4217 DescriptorOp op, OpAdaptor &adaptor, ConversionPatternRewriter &rewriter,
4218 Location loc, Value sgpr2, Value sgpr3, ArrayRef<Value> consts) const {
4219 Value globalIncrement = op.getGlobalIncrement();
4220 constexpr int32_t dim = 2;
4221 constexpr int32_t offset = 64;
4222 if (!globalIncrement)
4223 return setTensorDimXStride(op, adaptor, rewriter, loc, sgprY: sgpr2, sgprZ: sgpr3,
4224 consts, dimX: dim, offset);
4225 return setGlobalAddrIncrement(op, adaptor, rewriter, loc, sgpr2, sgpr3,
4226 consts, offset);
4227 }
4228
4229 Value setIterateCount(DescriptorOp op, OpAdaptor &adaptor,
4230 ConversionPatternRewriter &rewriter, Location loc,
4231 Value sgpr3, ArrayRef<Value> consts,
4232 int32_t offset) const {
4233 Value iterationCount = adaptor.getIterationCount();
4234 IntegerType i32 = rewriter.getI32Type();
4235 // pre-condition: iterationCount is in the inclusive interval [1, 256].
4236 // TODO: validation if the value breaks the pre-condition.
4237 // If the pre-condition fails, there is a possibility of
4238 // affecting the higher bits. In a following PR implement
4239 // RuntimeVerifiableOpInterface that instruments conditions that need to be
4240 // checked at runtime.
4241 iterationCount = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: i32, arg: iterationCount);
4242 iterationCount =
4243 LLVM::SubOp::create(builder&: rewriter, location: loc, lhs: iterationCount, rhs: consts[1]);
4244 return setValueAtOffset(rewriter, loc, accumulator: sgpr3, value: iterationCount, shift: offset);
4245 }
4246
4247 Value setTileDim3OrIterateCount(DescriptorOp op, OpAdaptor &adaptor,
4248 ConversionPatternRewriter &rewriter,
4249 Location loc, Value sgpr3,
4250 ArrayRef<Value> consts) const {
4251 Value iterateCount = op.getIterationCount();
4252 constexpr int32_t dim = 2;
4253 constexpr int32_t offset = 112;
4254 if (!iterateCount)
4255 return setTileDimX(op, adaptor, rewriter, loc, sgpr: sgpr3, consts, dimX: dim,
4256 offset);
4257
4258 return setIterateCount(op, adaptor, rewriter, loc, sgpr3, consts, offset);
4259 }
4260
4261 Value getDGroup2(DescriptorOp op, OpAdaptor &adaptor,
4262 ConversionPatternRewriter &rewriter, Location loc,
4263 ArrayRef<Value> consts) const {
4264 if constexpr (DescriptorOp::isGather())
4265 return getDGroup2Gather(op, adaptor, rewriter, loc, consts);
4266 return getDGroup2NonGather(op, adaptor, rewriter, loc, consts);
4267 }
4268
4269 Value getDGroup2NonGather(DescriptorOp op, OpAdaptor &adaptor,
4270 ConversionPatternRewriter &rewriter, Location loc,
4271 ArrayRef<Value> consts) const {
4272 IntegerType i32 = rewriter.getI32Type();
4273 Type v4i32 = this->typeConverter->convertType(VectorType::get(shape: 4, elementType: i32));
4274 assert(v4i32 && "expected type conversion to succeed.");
4275
4276 bool onlyNeedsTwoDescriptors = !op.getLdsIncrement() && op.getRank() <= 2;
4277 if (onlyNeedsTwoDescriptors)
4278 return LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: v4i32);
4279
4280 constexpr int64_t sgprlen = 4;
4281 Value sgprs[sgprlen];
4282 for (int i = 0; i < sgprlen; ++i)
4283 sgprs[i] = consts[0];
4284
4285 sgprs[0] = setTensorDim2(op, adaptor, rewriter, loc, sgpr0: sgprs[0], consts);
4286 sgprs[1] = setTensorDim3OrLDSAddrIncrement(op, adaptor, rewriter, loc,
4287 sgpr1: sgprs[1], consts);
4288 std::tie(args&: sgprs[2], args&: sgprs[3]) = setTensorDim2StrideOrGlobalAddrIncrement(
4289 op, adaptor, rewriter, loc, sgpr2: sgprs[2], sgpr3: sgprs[3], consts);
4290 sgprs[3] =
4291 setTileDim3OrIterateCount(op, adaptor, rewriter, loc, sgpr3: sgprs[3], consts);
4292
4293 Value dgroup2 = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: v4i32);
4294 for (auto [sgpr, constant] : llvm::zip(t&: sgprs, u&: consts))
4295 dgroup2 =
4296 LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: dgroup2, value: sgpr, position: constant);
4297
4298 return dgroup2;
4299 }
4300
4301 Value getGatherIndices(DescriptorOp op, OpAdaptor &adaptor,
4302 ConversionPatternRewriter &rewriter, Location loc,
4303 ArrayRef<Value> consts, bool firstHalf) const {
4304 IntegerType i32 = rewriter.getI32Type();
4305 Type v4i32 = this->typeConverter->convertType(VectorType::get(shape: 4, elementType: i32));
4306 assert(v4i32 && "expected type conversion to succeed.");
4307
4308 Value indices = adaptor.getIndices();
4309 auto vectorType = cast<VectorType>(Val: indices.getType());
4310 unsigned length = vectorType.getShape().back();
4311 Type elementType = vectorType.getElementType();
4312 unsigned maxLength = elementType == i32 ? 4 : 8;
4313 int32_t offset = firstHalf ? 0 : maxLength;
4314 unsigned discountedLength =
4315 std::max(a: static_cast<int32_t>(length - offset), b: 0);
4316
4317 unsigned targetSize = std::min(a: maxLength, b: discountedLength);
4318
4319 SmallVector<Value> indicesVector;
4320 for (unsigned i = offset; i < targetSize + offset; ++i) {
4321 Value idx;
4322 if (i < consts.size())
4323 idx = consts[i];
4324 else
4325 idx = createI32Constant(rewriter, loc, value: i);
4326 Value elem = LLVM::ExtractElementOp::create(builder&: rewriter, location: loc, vector: indices, position: idx);
4327 indicesVector.push_back(Elt: elem);
4328 }
4329
4330 SmallVector<Value> indicesI32Vector;
4331 if (elementType == i32) {
4332 indicesI32Vector = std::move(indicesVector);
4333 } else {
4334 for (unsigned i = 0; i < targetSize; ++i) {
4335 Value index = indicesVector[i];
4336 indicesI32Vector.push_back(
4337 Elt: LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: i32, arg: index));
4338 }
4339 if ((targetSize % 2) != 0)
4340 // Add padding when not divisible by two.
4341 indicesI32Vector.push_back(Elt: consts[0]);
4342 }
4343
4344 SmallVector<Value> indicesToInsert;
4345 if (elementType == i32) {
4346 indicesToInsert = std::move(indicesI32Vector);
4347 } else {
4348 unsigned size = indicesI32Vector.size() / 2;
4349 for (unsigned i = 0; i < size; ++i) {
4350 Value first = indicesI32Vector[2 * i];
4351 Value second = indicesI32Vector[2 * i + 1];
4352 Value joined = setValueAtOffset(rewriter, loc, accumulator: first, value: second, shift: 16);
4353 indicesToInsert.push_back(Elt: joined);
4354 }
4355 }
4356
4357 Value dgroup = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: v4i32);
4358 for (auto [sgpr, constant] : llvm::zip_first(t&: indicesToInsert, u&: consts))
4359 dgroup =
4360 LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: dgroup, value: sgpr, position: constant);
4361
4362 return dgroup;
4363 }
4364
4365 Value getDGroup2Gather(DescriptorOp op, OpAdaptor &adaptor,
4366 ConversionPatternRewriter &rewriter, Location loc,
4367 ArrayRef<Value> consts) const {
4368 return getGatherIndices(op, adaptor, rewriter, loc, consts, firstHalf: true);
4369 }
4370
4371 std::pair<Value, Value>
4372 setTensorDim3Stride(DescriptorOp op, OpAdaptor &adaptor,
4373 ConversionPatternRewriter &rewriter, Location loc,
4374 Value sgpr0, Value sgpr1, ArrayRef<Value> consts) const {
4375 constexpr int32_t dim = 3;
4376 constexpr int32_t offset = 0;
4377 return setTensorDimXStride(op, adaptor, rewriter, loc, sgprY: sgpr0, sgprZ: sgpr1, consts,
4378 dimX: dim, offset);
4379 }
4380
4381 std::pair<Value, Value> setTensorDim4(DescriptorOp op, OpAdaptor &adaptor,
4382 ConversionPatternRewriter &rewriter,
4383 Location loc, Value sgpr1, Value sgpr2,
4384 ArrayRef<Value> consts) const {
4385 constexpr int32_t dim = 4;
4386 constexpr int32_t offset = 48;
4387 return setTensorDimX(op, adaptor, rewriter, loc, sgpr1, sgpr2, consts, dim,
4388 offset);
4389 }
4390
4391 Value setTileDim4(DescriptorOp op, OpAdaptor &adaptor,
4392 ConversionPatternRewriter &rewriter, Location loc,
4393 Value sgpr2, ArrayRef<Value> consts) const {
4394 constexpr int32_t dim = 4;
4395 constexpr int32_t offset = 80;
4396 return setTileDimX(op, adaptor, rewriter, loc, sgpr: sgpr2, consts, dimX: dim, offset);
4397 }
4398
4399 Value getDGroup3(DescriptorOp op, OpAdaptor &adaptor,
4400 ConversionPatternRewriter &rewriter, Location loc,
4401 ArrayRef<Value> consts) const {
4402 if constexpr (DescriptorOp::isGather())
4403 return getDGroup3Gather(op, adaptor, rewriter, loc, consts);
4404 return getDGroup3NonGather(op, adaptor, rewriter, loc, consts);
4405 }
4406
4407 Value getDGroup3NonGather(DescriptorOp op, OpAdaptor &adaptor,
4408 ConversionPatternRewriter &rewriter, Location loc,
4409 ArrayRef<Value> consts) const {
4410 IntegerType i32 = rewriter.getI32Type();
4411 Type v4i32 = this->typeConverter->convertType(VectorType::get(shape: 4, elementType: i32));
4412 assert(v4i32 && "expected type conversion to succeed.");
4413 bool onlyNeedsTwoDescriptors = !op.getLdsIncrement() && op.getRank() <= 2;
4414 if (onlyNeedsTwoDescriptors)
4415 return LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: v4i32);
4416
4417 constexpr int32_t sgprlen = 4;
4418 Value sgprs[sgprlen];
4419 for (int i = 0; i < sgprlen; ++i)
4420 sgprs[i] = consts[0];
4421
4422 std::tie(args&: sgprs[0], args&: sgprs[1]) = setTensorDim3Stride(
4423 op, adaptor, rewriter, loc, sgpr0: sgprs[0], sgpr1: sgprs[1], consts);
4424 std::tie(args&: sgprs[1], args&: sgprs[2]) =
4425 setTensorDim4(op, adaptor, rewriter, loc, sgpr1: sgprs[1], sgpr2: sgprs[2], consts);
4426 sgprs[2] = setTileDim4(op, adaptor, rewriter, loc, sgpr2: sgprs[2], consts);
4427
4428 Value dgroup3 = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: v4i32);
4429 for (auto [sgpr, constant] : llvm::zip(t&: sgprs, u&: consts))
4430 dgroup3 =
4431 LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: dgroup3, value: sgpr, position: constant);
4432
4433 return dgroup3;
4434 }
4435
4436 Value getDGroup3Gather(DescriptorOp op, OpAdaptor &adaptor,
4437 ConversionPatternRewriter &rewriter, Location loc,
4438 ArrayRef<Value> consts) const {
4439 return getGatherIndices(op, adaptor, rewriter, loc, consts, firstHalf: false);
4440 }
4441
4442 LogicalResult
4443 matchAndRewrite(DescriptorOp op, OpAdaptor adaptor,
4444 ConversionPatternRewriter &rewriter) const override {
4445 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
4446 return op->emitOpError(
4447 "make_dma_descriptor is only supported on gfx1250");
4448
4449 Location loc = op.getLoc();
4450
4451 SmallVector<Value> consts;
4452 for (int64_t i = 0; i < 8; ++i)
4453 consts.push_back(Elt: createI32Constant(rewriter, loc, value: i));
4454
4455 Value dgroup0 = this->getDGroup0(adaptor);
4456 Value dgroup1 = this->getDGroup1(op, adaptor, rewriter, loc, consts);
4457 Value dgroup2 = this->getDGroup2(op, adaptor, rewriter, loc, consts);
4458 Value dgroup3 = this->getDGroup3(op, adaptor, rewriter, loc, consts);
4459 SmallVector<Value> results = {dgroup0, dgroup1, dgroup2, dgroup3};
4460 rewriter.replaceOpWithMultiple(op, {results});
4461 return success();
4462 }
4463};
4464
4465template <typename SourceOp, typename TargetOp>
4466struct AMDGPUTensorLoadStoreOpLowering
4467 : public ConvertOpToLLVMPattern<SourceOp> {
4468 using ConvertOpToLLVMPattern<SourceOp>::ConvertOpToLLVMPattern;
4469 using Adaptor = typename ConvertOpToLLVMPattern<SourceOp>::OneToNOpAdaptor;
4470 AMDGPUTensorLoadStoreOpLowering(const LLVMTypeConverter &converter,
4471 const ROCDL::TargetInfo &target)
4472 : ConvertOpToLLVMPattern<SourceOp>(converter), target(target) {}
4473 ROCDL::TargetInfo target;
4474
4475 LogicalResult
4476 matchAndRewrite(SourceOp op, Adaptor adaptor,
4477 ConversionPatternRewriter &rewriter) const override {
4478 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
4479 return op->emitOpError("is only supported on gfx1250");
4480
4481 ValueRange desc = adaptor.getDesc();
4482 // Create a <v8 x i32> 0 as the fifth argument to match llvm intrinsic. It
4483 // will move into the TDM descriptor once it becomes relevant for future use
4484 auto v8i32 = VectorType::get(shape: 8, elementType: rewriter.getI32Type());
4485 Value dgroup4 = LLVM::ZeroOp::create(rewriter, op.getLoc(), v8i32);
4486 Attribute cachePolicy = rewriter.getI32IntegerAttr(value: 0);
4487 rewriter.replaceOpWithNewOp<TargetOp>(op, desc[0], desc[1], desc[2],
4488 desc[3], dgroup4, cachePolicy,
4489 /*alias_scopes=*/nullptr,
4490 /*noalias_scopes=*/nullptr,
4491 /*tbaa=*/nullptr);
4492 return success();
4493 }
4494};
4495
4496struct GlobalPrefetchOpLowering
4497 : public ConvertOpToLLVMPattern<GlobalPrefetchOp> {
4498 GlobalPrefetchOpLowering(const LLVMTypeConverter &converter,
4499 const ROCDL::TargetInfo &target)
4500 : ConvertOpToLLVMPattern<GlobalPrefetchOp>(converter), target(target) {}
4501
4502 LogicalResult
4503 matchAndRewrite(GlobalPrefetchOp op, GlobalPrefetchOpAdaptor adaptor,
4504 ConversionPatternRewriter &rewriter) const override {
4505 if (!target.has(feature: llvm::AMDGPU::FEAT_GFX1250_INSTS))
4506 return op->emitOpError(message: "is only supported on gfx1250+");
4507
4508 const bool isSpeculative = op.getSpeculative();
4509 const int32_t immArgValue = getGlobalPrefetchLLVMEncoding(
4510 hint: op.getTemporalHint(), scope: op.getCacheScope(), isSpeculative);
4511 // amdgpu.global_prefetch is gfx1250+, so its policy bits use gfx12
4512 // encoding.
4513 Attribute cachePolicy = ROCDL::Gfx12CachePolicyAttr::get(
4514 context: rewriter.getContext(),
4515 value: static_cast<ROCDL::Gfx12CachePolicy>(immArgValue));
4516
4517 ValueRange indices = adaptor.getIndices();
4518 Value memRef = adaptor.getSrc();
4519 MemRefDescriptor descriptor(memRef);
4520 MemRefType memRefType = op.getSrc().getType();
4521 Location loc = op->getLoc();
4522 auto inboundsFlags = isSpeculative ? LLVM::GEPNoWrapFlags::none
4523 : LLVM::GEPNoWrapFlags::inbounds |
4524 LLVM::GEPNoWrapFlags::nuw;
4525 Value prefetchPtr = getStridedElementPtr(
4526 rewriter, loc, type: memRefType, memRefDesc: descriptor, indices, noWrapFlags: inboundsFlags);
4527
4528 rewriter.replaceOpWithNewOp<ROCDL::GlobalPrefetchOp>(
4529 op, args&: prefetchPtr, args&: cachePolicy, args: mlir::ArrayAttr{}, args: mlir::ArrayAttr{},
4530 args: mlir::ArrayAttr{});
4531 return success();
4532 }
4533
4534private:
4535 ROCDL::TargetInfo target;
4536};
4537
4538struct ConvertAMDGPUToROCDLPass
4539 : public impl::ConvertAMDGPUToROCDLPassBase<ConvertAMDGPUToROCDLPass> {
4540 using Base::Base;
4541
4542 void runOnOperation() override {
4543 MLIRContext *ctx = &getContext();
4544 FailureOr<ROCDL::TargetInfo> targetInfo = ROCDL::TargetInfo::get(
4545 arch: ROCDL::resolveArchOption(arch, deprecatedAlias: chipset),
4546 /*waveSize=*/0, emitError: [&] { return emitError(loc: UnknownLoc::get(context: ctx)); });
4547 if (failed(Result: targetInfo))
4548 return signalPassFailure();
4549
4550 RewritePatternSet patterns(ctx);
4551 LLVMTypeConverter converter(ctx);
4552
4553 populateAMDGPUToROCDLConversionPatterns(converter, patterns, target: *targetInfo);
4554 amdgpu::populateCommonGPUTypeAndAttributeConversions(typeConverter&: converter);
4555 LLVMConversionTarget target(getContext());
4556 target.addIllegalDialect<::mlir::amdgpu::AMDGPUDialect>();
4557 target.addLegalDialect<::mlir::LLVM::LLVMDialect>();
4558 target.addLegalDialect<::mlir::ROCDL::ROCDLDialect>();
4559 if (failed(Result: applyPartialConversion(op: getOperation(), target,
4560 patterns: std::move(patterns))))
4561 signalPassFailure();
4562 }
4563};
4564} // namespace
4565
4566void mlir::amdgpu::populateCommonGPUTypeAndAttributeConversions(
4567 TypeConverter &typeConverter) {
4568 populateGpuMemorySpaceAttributeConversions(
4569 typeConverter, mapping: [](gpu::AddressSpace space) {
4570 switch (space) {
4571 case gpu::AddressSpace::Global:
4572 return ROCDL::ROCDLDialect::kGlobalMemoryAddressSpace;
4573 case gpu::AddressSpace::Workgroup:
4574 return ROCDL::ROCDLDialect::kSharedMemoryAddressSpace;
4575 case gpu::AddressSpace::Private:
4576 return ROCDL::ROCDLDialect::kPrivateMemoryAddressSpace;
4577 case gpu::AddressSpace::Constant:
4578 return ROCDL::ROCDLDialect::kConstantMemoryAddressSpace;
4579 }
4580 llvm_unreachable("unknown address space enum value");
4581 });
4582 typeConverter.addConversion(callback: [](gpu::NamedBarrierType type) {
4583 return LLVM::LLVMPointerType::get(
4584 context: type.getContext(), addressSpace: ROCDL::ROCDLDialect::kBarrierAddressSpace);
4585 });
4586}
4587
4588void mlir::populateAMDGPUTypeAndAttributeConversions(
4589 TypeConverter &typeConverter) {
4590 typeConverter.addTypeAttributeConversion(
4591 callback: [](BaseMemRefType type, amdgpu::AddressSpaceAttr as)
4592 -> TypeConverter::AttributeConversionResult {
4593 MLIRContext *ctx = as.getContext();
4594 Type i64 = IntegerType::get(context: ctx, width: 64);
4595 switch (as.getValue()) {
4596 case amdgpu::AddressSpace::FatRawBuffer:
4597 return IntegerAttr::get(type: i64, value: 7);
4598 case amdgpu::AddressSpace::BufferRsrc:
4599 return IntegerAttr::get(type: i64, value: 8);
4600 case amdgpu::AddressSpace::FatStructuredBuffer:
4601 return IntegerAttr::get(type: i64, value: 9);
4602 }
4603 return TypeConverter::AttributeConversionResult::abort();
4604 });
4605 typeConverter.addConversion(callback: [&](DsBarrierStateType type) -> Type {
4606 return IntegerType::get(context: type.getContext(), width: 64);
4607 });
4608 typeConverter.addConversion(callback: [&](TDMBaseType type) -> Type {
4609 Type i32 = IntegerType::get(context: type.getContext(), width: 32);
4610 return typeConverter.convertType(t: VectorType::get(shape: 4, elementType: i32));
4611 });
4612 typeConverter.addConversion(callback: [&](TDMGatherBaseType type) -> Type {
4613 Type i32 = IntegerType::get(context: type.getContext(), width: 32);
4614 return typeConverter.convertType(t: VectorType::get(shape: 4, elementType: i32));
4615 });
4616 typeConverter.addConversion(
4617 callback: [&](TDMDescriptorType type,
4618 SmallVectorImpl<Type> &result) -> std::optional<LogicalResult> {
4619 Type i32 = IntegerType::get(context: type.getContext(), width: 32);
4620 Type v4i32 = typeConverter.convertType(t: VectorType::get(shape: 4, elementType: i32));
4621 Type v8i32 = typeConverter.convertType(t: VectorType::get(shape: 8, elementType: i32));
4622 llvm::append_values(C&: result, Values&: v4i32, Values&: v8i32, Values&: v4i32, Values&: v4i32);
4623 return success();
4624 });
4625
4626 auto addUnrealizedCast = [](OpBuilder &builder, TypeRange types,
4627 ValueRange inputs,
4628 Location loc) -> SmallVector<Value> {
4629 // Only create unrealized_conversion_cast for TDMDescriptorType.
4630 // All other types which are not expected, should be
4631 // materialized by other target materialization functions.
4632 if (inputs.size() != 1)
4633 return {};
4634
4635 if (!isa<TDMDescriptorType>(Val: inputs[0].getType()))
4636 return {};
4637
4638 auto cast = UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: types, operands: inputs);
4639 return cast.getResults();
4640 };
4641
4642 typeConverter.addTargetMaterialization(callback&: addUnrealizedCast);
4643}
4644
4645void mlir::populateAMDGPUToROCDLConversionPatterns(
4646 LLVMTypeConverter &converter, RewritePatternSet &patterns,
4647 const ROCDL::TargetInfo &target) {
4648 populateAMDGPUTypeAndAttributeConversions(typeConverter&: converter);
4649 patterns
4650 .add<FatRawBufferCastLowering,
4651 RawBufferOpLowering<RawBufferLoadOp, ROCDL::RawPtrBufferLoadOp>,
4652 RawBufferOpLowering<RawBufferStoreOp, ROCDL::RawPtrBufferStoreOp>,
4653 RawBufferOpLowering<RawBufferAtomicFaddOp,
4654 ROCDL::RawPtrBufferAtomicFaddOp>,
4655 RawBufferOpLowering<RawBufferAtomicFmaxOp,
4656 ROCDL::RawPtrBufferAtomicFmaxOp>,
4657 RawBufferOpLowering<RawBufferAtomicSmaxOp,
4658 ROCDL::RawPtrBufferAtomicSmaxOp>,
4659 RawBufferOpLowering<RawBufferAtomicUminOp,
4660 ROCDL::RawPtrBufferAtomicUminOp>,
4661 RawBufferOpLowering<RawBufferAtomicCmpswapOp,
4662 ROCDL::RawPtrBufferAtomicCmpSwap>,
4663 AMDGPUDPPLowering, MemoryCounterWaitOpLowering, LDSBarrierOpLowering,
4664 SchedBarrierOpLowering, MFMAOpLowering, ScaledMFMAOpLowering,
4665 SparseMFMAOpLowering, WMMAOpLowering, ScaledWMMAOpLowering,
4666 SparseWMMAOpLowering, DotOpLowering, ExtPackedFp8OpLowering,
4667 ScaledExtPackedMatrixOpLowering, ScaledExtPackedOpLowering,
4668 PackedScaledTruncOpLowering, PackedTrunc2xFp8OpLowering,
4669 PackedStochRoundFp8OpLowering, GatherToLDSOpLowering,
4670 GlobalLoadAsyncToLDSOpLowering, TransposeLoadOpLowering,
4671 GlobalTransposeLoadOpLowering, AMDGPUPermlaneLowering,
4672 AMDGPUPermlaneVarLowering, AMDGPUMakeDmaBaseLowering<MakeDmaBaseOp>,
4673 AMDGPUMakeDmaBaseLowering<MakeGatherDmaBaseOp>,
4674 AMDGPULowerDescriptor<MakeDmaDescriptorOp>,
4675 AMDGPULowerDescriptor<MakeGatherDmaDescriptorOp>,
4676 AMDGPUTensorLoadStoreOpLowering<TensorLoadToLDSOp,
4677 ROCDL::TensorLoadToLDSOp>,
4678 AMDGPUTensorLoadStoreOpLowering<TensorStoreFromLDSOp,
4679 ROCDL::TensorStoreFromLDSOp>,
4680 DsBarrierInitOpLowering, DsBarrierPollStateOpLowering,
4681 DsAsyncBarrierArriveOpLowering, DsBarrierArriveOpLowering,
4682 GlobalPrefetchOpLowering>(arg&: converter, args: target);
4683 patterns.add<AMDGPUSwizzleBitModeLowering, DsBarrierStatePhaseOpLowering,
4684 DsBarrierStatePendingCountOpLowering,
4685 DsBarrierStateInitCountOpLowering,
4686 DsBarrierStatePhaseParityLowering>(arg&: converter);
4687}
4688