1//===- ArithToAMDGPU.cpp - Arith to AMDGPU dialect conversion ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/ArithToAMDGPU/ArithToAMDGPU.h"
10
11#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
12#include "mlir/Dialect/Arith/IR/Arith.h"
13#include "mlir/Dialect/Arith/Utils/Utils.h"
14#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
15#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
16#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
17#include "mlir/Dialect/Utils/IndexingUtils.h"
18#include "mlir/Dialect/Vector/IR/VectorOps.h"
19#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
20#include "mlir/IR/BuiltinTypes.h"
21#include "mlir/IR/PatternMatch.h"
22#include "mlir/IR/TypeUtilities.h"
23#include "mlir/Pass/Pass.h"
24#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
25
26namespace mlir {
27#define GEN_PASS_DEF_ARITHTOAMDGPUCONVERSIONPASS
28#include "mlir/Conversion/Passes.h.inc"
29} // namespace mlir
30
31using namespace mlir;
32using namespace mlir::amdgpu;
33
34namespace {
35struct ArithToAMDGPUConversionPass final
36 : impl::ArithToAMDGPUConversionPassBase<ArithToAMDGPUConversionPass> {
37 using impl::ArithToAMDGPUConversionPassBase<
38 ArithToAMDGPUConversionPass>::ArithToAMDGPUConversionPassBase;
39
40 void runOnOperation() override;
41};
42
43struct ExtFOnFloat8RewritePattern final : OpRewritePattern<arith::ExtFOp> {
44 using Base::Base;
45
46 ROCDL::TargetInfo target;
47 ExtFOnFloat8RewritePattern(MLIRContext *ctx, const ROCDL::TargetInfo &target,
48 PatternBenefit benefit)
49 : OpRewritePattern::OpRewritePattern(ctx, benefit), target(target) {}
50
51 LogicalResult matchAndRewrite(arith::ExtFOp op,
52 PatternRewriter &rewriter) const override;
53};
54
55struct TruncFToFloat8RewritePattern final : OpRewritePattern<arith::TruncFOp> {
56 bool saturateFP8 = false;
57 TruncFToFloat8RewritePattern(MLIRContext *ctx, bool saturateFP8,
58 const ROCDL::TargetInfo &target,
59 PatternBenefit benefit)
60 : OpRewritePattern::OpRewritePattern(ctx, benefit),
61 saturateFP8(saturateFP8), target(target) {}
62 ROCDL::TargetInfo target;
63
64 LogicalResult matchAndRewrite(arith::TruncFOp op,
65 PatternRewriter &rewriter) const override;
66};
67
68struct TruncfToFloat16RewritePattern final
69 : public OpRewritePattern<arith::TruncFOp> {
70
71 using Base::Base;
72
73 LogicalResult matchAndRewrite(arith::TruncFOp op,
74 PatternRewriter &rewriter) const override;
75};
76
77struct ScalingExtFRewritePattern final
78 : OpRewritePattern<arith::ScalingExtFOp> {
79 using Base::Base;
80
81 LogicalResult matchAndRewrite(arith::ScalingExtFOp op,
82 PatternRewriter &rewriter) const override;
83};
84
85struct ScalingTruncFRewritePattern final
86 : OpRewritePattern<arith::ScalingTruncFOp> {
87 using Base::Base;
88
89 LogicalResult matchAndRewrite(arith::ScalingTruncFOp op,
90 PatternRewriter &rewriter) const override;
91};
92} // end namespace
93
94static bool isSupportedF8(Type elementType, const ROCDL::TargetInfo &target) {
95 if (target.hasFnuzFp8())
96 return isa<Float8E4M3FNUZType, Float8E5M2FNUZType>(Val: elementType);
97 if (target.hasOcpFp8())
98 return isa<Float8E4M3FNType, Float8E5M2Type>(Val: elementType);
99 return false;
100}
101
102static Value castF32To(Type desType, Value f32, Location loc,
103 PatternRewriter &rewriter) {
104 Type elementType = getElementTypeOrSelf(type: desType);
105 if (elementType.isF32())
106 return f32;
107 if (elementType.getIntOrFloatBitWidth() < 32)
108 return arith::TruncFOp::create(builder&: rewriter, location: loc, out: desType, in: f32);
109 if (elementType.getIntOrFloatBitWidth() > 32)
110 return arith::ExtFOp::create(builder&: rewriter, location: loc, resultTypes: TypeRange{desType},
111 operands: ValueRange{f32}, properties: arith::ExtFOp::Properties{});
112 llvm_unreachable("The only 32-bit float type is f32");
113}
114
115LogicalResult
116ExtFOnFloat8RewritePattern::matchAndRewrite(arith::ExtFOp op,
117 PatternRewriter &rewriter) const {
118 Type inType = op.getIn().getType();
119 auto inVecType = dyn_cast<VectorType>(Val&: inType);
120 if (inVecType) {
121 if (inVecType.isScalable())
122 return failure();
123 inType = inVecType.getElementType();
124 }
125 if (!isSupportedF8(elementType: inType, target))
126 return failure();
127
128 Location loc = op.getLoc();
129 Value in = op.getIn();
130 Type outElemType = getElementTypeOrSelf(type: op.getOut().getType());
131 VectorType extResType = VectorType::get(shape: 2, elementType: rewriter.getF32Type());
132 if (!inVecType) {
133 Value asFloat = amdgpu::ExtPackedFp8Op::create(
134 builder&: rewriter, location: loc, res: rewriter.getF32Type(), source: in, index: 0);
135 Value result = castF32To(desType: outElemType, f32: asFloat, loc, rewriter);
136 rewriter.replaceOp(op, newValues: result);
137 return success();
138 }
139 int64_t numElements = inVecType.getNumElements();
140
141 Value zero = arith::ConstantOp::create(
142 builder&: rewriter, location: loc, result: outElemType, value: rewriter.getFloatAttr(type: outElemType, value: 0.0));
143 VectorType outType = cast<VectorType>(Val: op.getOut().getType());
144
145 if (inVecType.getShape().empty()) {
146 Value zerodSplat =
147 rewriter.createOrFold<vector::BroadcastOp>(location: loc, args&: outType, args&: zero);
148 Value scalarIn =
149 vector::ExtractOp::create(builder&: rewriter, location: loc, source: in, position: ArrayRef<int64_t>{});
150 Value scalarExt = arith::ExtFOp::create(
151 builder&: rewriter, location: loc, resultTypes: TypeRange{outElemType}, operands: ValueRange{scalarIn},
152 properties: arith::ExtFOp::Properties{});
153 Value result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: scalarExt,
154 dest: zerodSplat, position: ArrayRef<int64_t>{});
155 rewriter.replaceOp(op, newValues: result);
156 return success();
157 }
158
159 VectorType flatTy = VectorType::get(shape: SmallVector<int64_t>{numElements},
160 elementType: outType.getElementType());
161 Value result = rewriter.createOrFold<vector::BroadcastOp>(location: loc, args&: flatTy, args&: zero);
162
163 if (inVecType.getRank() > 1) {
164 inVecType = VectorType::get(shape: SmallVector<int64_t>{numElements},
165 elementType: inVecType.getElementType());
166 in = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: inVecType, source: in);
167 }
168
169 for (int64_t i = 0; i < numElements; i += 4) {
170 int64_t elemsThisOp = std::min(a: numElements, b: i + 4) - i;
171 Value inSlice = vector::ExtractStridedSliceOp::create(builder&: rewriter, location: loc, source: in, offsets: i,
172 sizes: elemsThisOp, strides: 1);
173 for (int64_t j = 0; j < elemsThisOp; j += 2) {
174 if (i + j + 1 < numElements) { // Convert two 8-bit elements
175 Value asFloats = amdgpu::ExtPackedFp8Op::create(
176 builder&: rewriter, location: loc, res: extResType, source: inSlice, index: j / 2);
177 Type desType = VectorType::get(shape: 2, elementType: outElemType);
178 Value asType = castF32To(desType, f32: asFloats, loc, rewriter);
179 result = vector::InsertStridedSliceOp::create(builder&: rewriter, location: loc, valueToStore: asType,
180 dest: result, offsets: i + j, strides: 1);
181 } else { // Convert a 8-bit element
182 Value asFloat = amdgpu::ExtPackedFp8Op::create(
183 builder&: rewriter, location: loc, res: rewriter.getF32Type(), source: inSlice, index: j / 2 * 2);
184 Value asType = castF32To(desType: outElemType, f32: asFloat, loc, rewriter);
185 result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: asType, dest: result, position: i + j);
186 }
187 }
188 }
189
190 if (inVecType.getRank() != outType.getRank()) {
191 result = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: outType, source: result);
192 }
193
194 rewriter.replaceOp(op, newValues: result);
195 return success();
196}
197
198static Value castToF32(Value value, Location loc, PatternRewriter &rewriter) {
199 Type type = value.getType();
200 if (type.isF32())
201 return value;
202 if (type.getIntOrFloatBitWidth() < 32)
203 return arith::ExtFOp::create(
204 builder&: rewriter, location: loc, resultTypes: TypeRange{rewriter.getF32Type()}, operands: ValueRange{value},
205 properties: arith::ExtFOp::Properties{});
206 if (type.getIntOrFloatBitWidth() > 32)
207 return arith::TruncFOp::create(builder&: rewriter, location: loc, out: rewriter.getF32Type(), in: value);
208 llvm_unreachable("The only 32-bit float type is f32");
209}
210
211// If `in` is a finite value, clamp it between the maximum and minimum values
212// of `outElemType` so that subsequent conversion instructions don't
213// overflow those out-of-range values to NaN. These semantics are commonly
214// used in machine-learning contexts where failure to clamp would lead to
215// excessive NaN production.
216static Value clampInput(PatternRewriter &rewriter, Location loc,
217 Type outElemType, Value source) {
218 Type sourceType = source.getType();
219 const llvm::fltSemantics &sourceSem =
220 cast<FloatType>(Val: getElementTypeOrSelf(type: sourceType)).getFloatSemantics();
221 const llvm::fltSemantics &targetSem =
222 cast<FloatType>(Val&: outElemType).getFloatSemantics();
223
224 APFloat min = APFloat::getLargest(Sem: targetSem, /*Negative=*/true);
225 APFloat max = APFloat::getLargest(Sem: targetSem, /*Negative=*/false);
226 bool ignoredLosesInfo = false;
227 // We can ignore conversion failures here because this conversion promotes
228 // from a smaller type to a larger one - ex. there can be no loss of precision
229 // when casting fp8 to f16.
230 (void)min.convert(ToSemantics: sourceSem, RM: APFloat::rmNearestTiesToEven, losesInfo: &ignoredLosesInfo);
231 (void)max.convert(ToSemantics: sourceSem, RM: APFloat::rmNearestTiesToEven, losesInfo: &ignoredLosesInfo);
232
233 Value minCst = createScalarOrSplatConstant(builder&: rewriter, loc, type: sourceType, value: min);
234 Value maxCst = createScalarOrSplatConstant(builder&: rewriter, loc, type: sourceType, value: max);
235
236 Value inf = createScalarOrSplatConstant(
237 builder&: rewriter, loc, type: sourceType,
238 value: APFloat::getInf(Sem: sourceSem, /*Negative=*/false));
239 Value negInf = createScalarOrSplatConstant(
240 builder&: rewriter, loc, type: sourceType, value: APFloat::getInf(Sem: sourceSem, /*Negative=*/true));
241 Value isInf = rewriter.createOrFold<arith::CmpFOp>(
242 location: loc, args: arith::CmpFPredicate::OEQ, args&: source, args&: inf);
243 Value isNegInf = rewriter.createOrFold<arith::CmpFOp>(
244 location: loc, args: arith::CmpFPredicate::OEQ, args&: source, args&: negInf);
245 Value isNan = rewriter.createOrFold<arith::CmpFOp>(
246 location: loc, args: arith::CmpFPredicate::UNO, args&: source, args&: source);
247 Value isNonFinite = arith::OrIOp::create(
248 builder&: rewriter, location: loc, lhs: arith::OrIOp::create(builder&: rewriter, location: loc, lhs: isInf, rhs: isNegInf),
249 rhs: isNan);
250
251 Value clampedBelow = arith::MaximumFOp::create(builder&: rewriter, location: loc, lhs: source, rhs: minCst);
252 Value clamped =
253 arith::MinimumFOp::create(builder&: rewriter, location: loc, lhs: clampedBelow, rhs: maxCst);
254 Value res =
255 arith::SelectOp::create(builder&: rewriter, location: loc, condition: isNonFinite, true_value: source, false_value: clamped);
256 return res;
257}
258
259LogicalResult
260TruncFToFloat8RewritePattern::matchAndRewrite(arith::TruncFOp op,
261 PatternRewriter &rewriter) const {
262 // Only supporting default rounding mode as of now.
263 if (op.getRoundingmodeAttr())
264 return failure();
265 Type outType = op.getOut().getType();
266 auto outVecType = dyn_cast<VectorType>(Val&: outType);
267 if (outVecType) {
268 if (outVecType.isScalable())
269 return failure();
270 outType = outVecType.getElementType();
271 }
272 auto inType = dyn_cast<FloatType>(Val: getElementTypeOrSelf(type: op.getIn().getType()));
273 if (inType && inType.getWidth() <= 8 && saturateFP8)
274 // Conversion between 8-bit floats is not supported with truncation enabled.
275 return failure();
276
277 if (!isSupportedF8(elementType: outType, target))
278 return failure();
279
280 Location loc = op.getLoc();
281 Value in = op.getIn();
282 Type outElemType = getElementTypeOrSelf(type: op.getOut().getType());
283 if (saturateFP8)
284 in = clampInput(rewriter, loc, outElemType, source: in);
285 auto inVectorTy = dyn_cast<VectorType>(Val: in.getType());
286 VectorType truncResType = VectorType::get(shape: 4, elementType: outElemType);
287 if (!inVectorTy) {
288 Value asFloat = castToF32(value: in, loc, rewriter);
289 Value asF8s = amdgpu::PackedTrunc2xFp8Op::create(
290 builder&: rewriter, location: loc, res: truncResType, sourceA: asFloat, /*sourceB=*/nullptr, wordIndex: 0,
291 /*existing=*/nullptr);
292 Value result = vector::ExtractOp::create(builder&: rewriter, location: loc, source: asF8s, position: 0);
293 rewriter.replaceOp(op, newValues: result);
294 return success();
295 }
296
297 int64_t numElements = outVecType.getNumElements();
298 Value zero = arith::ConstantOp::create(
299 builder&: rewriter, location: loc, result: outElemType, value: rewriter.getFloatAttr(type: outElemType, value: 0.0));
300 if (outVecType.getShape().empty()) {
301 Value scalarIn =
302 vector::ExtractOp::create(builder&: rewriter, location: loc, source: in, position: ArrayRef<int64_t>{});
303 // Recurse to send the 0-D vector case to the 1-D vector case
304 Value scalarTrunc =
305 arith::TruncFOp::create(builder&: rewriter, location: loc, out: outElemType, in: scalarIn);
306 Value result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: scalarTrunc, dest: zero,
307 position: ArrayRef<int64_t>{});
308 rewriter.replaceOp(op, newValues: result);
309 return success();
310 }
311
312 VectorType flatTy = VectorType::get(shape: SmallVector<int64_t>{numElements},
313 elementType: outVecType.getElementType());
314 Value result = rewriter.createOrFold<vector::BroadcastOp>(location: loc, args&: flatTy, args&: zero);
315
316 if (inVectorTy.getRank() > 1) {
317 inVectorTy = VectorType::get(shape: SmallVector<int64_t>{numElements},
318 elementType: inVectorTy.getElementType());
319 in = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: inVectorTy, source: in);
320 }
321
322 for (int64_t i = 0; i < numElements; i += 4) {
323 int64_t elemsThisOp = std::min(a: numElements, b: i + 4) - i;
324 Value thisResult = nullptr;
325 for (int64_t j = 0; j < elemsThisOp; j += 2) {
326 Value elemA = vector::ExtractOp::create(builder&: rewriter, location: loc, source: in, position: i + j);
327 Value asFloatA = castToF32(value: elemA, loc, rewriter);
328 Value asFloatB = nullptr;
329 if (j + 1 < elemsThisOp) {
330 Value elemB = vector::ExtractOp::create(builder&: rewriter, location: loc, source: in, position: i + j + 1);
331 asFloatB = castToF32(value: elemB, loc, rewriter);
332 }
333 thisResult = amdgpu::PackedTrunc2xFp8Op::create(
334 builder&: rewriter, location: loc, res: truncResType, sourceA: asFloatA, sourceB: asFloatB, wordIndex: j / 2, existing: thisResult);
335 }
336 if (elemsThisOp < 4)
337 thisResult = vector::ExtractStridedSliceOp::create(
338 builder&: rewriter, location: loc, source: thisResult, offsets: 0, sizes: elemsThisOp, strides: 1);
339 result = vector::InsertStridedSliceOp::create(builder&: rewriter, location: loc, valueToStore: thisResult,
340 dest: result, offsets: i, strides: 1);
341 }
342
343 if (inVectorTy.getRank() != outVecType.getRank()) {
344 result = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: outVecType, source: result);
345 }
346
347 rewriter.replaceOp(op, newValues: result);
348 return success();
349}
350
351LogicalResult TruncfToFloat16RewritePattern::matchAndRewrite(
352 arith::TruncFOp op, PatternRewriter &rewriter) const {
353 Type outType = op.getOut().getType();
354 Type inputType = getElementTypeOrSelf(val: op.getIn());
355 auto outVecType = dyn_cast<VectorType>(Val&: outType);
356 if (outVecType) {
357 if (outVecType.isScalable())
358 return failure();
359 outType = outVecType.getElementType();
360 }
361 if (!(outType.isF16() && inputType.isF32()))
362 return failure();
363
364 Location loc = op.getLoc();
365 Value in = op.getIn();
366 Type outElemType = getElementTypeOrSelf(type: op.getOut().getType());
367 VectorType truncResType = VectorType::get(shape: 2, elementType: outElemType);
368 auto inVectorTy = dyn_cast<VectorType>(Val: in.getType());
369
370 // Handle the case where input type is not a vector type
371 if (!inVectorTy) {
372 auto sourceB = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: rewriter.getF32Type());
373 Value asF16s =
374 ROCDL::CvtPkRtz::create(builder&: rewriter, location: loc, res: truncResType, srcA: in, srcB: sourceB);
375 Value result = vector::ExtractOp::create(builder&: rewriter, location: loc, source: asF16s, position: 0);
376 rewriter.replaceOp(op, newValues: result);
377 return success();
378 }
379 int64_t numElements = outVecType.getNumElements();
380 Value zero = rewriter.createOrFold<arith::ConstantOp>(
381 location: loc, args&: outElemType, args: rewriter.getFloatAttr(type: outElemType, value: 0.0));
382 Value result =
383 rewriter.createOrFold<vector::BroadcastOp>(location: loc, args&: outVecType, args&: zero);
384
385 if (inVectorTy.getRank() > 1) {
386 inVectorTy = VectorType::get(shape: SmallVector<int64_t>{numElements},
387 elementType: inVectorTy.getElementType());
388 in = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: inVectorTy, source: in);
389 }
390
391 // Handle the vector case. We also handle the (uncommon) case where the vector
392 // length is odd
393 for (int64_t i = 0; i < numElements; i += 2) {
394 int64_t elemsThisOp = std::min(a: numElements, b: i + 2) - i;
395 Value thisResult = nullptr;
396 Value elemA = vector::ExtractOp::create(builder&: rewriter, location: loc, source: in, position: i);
397 Value elemB = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: rewriter.getF32Type());
398
399 if (elemsThisOp == 2) {
400 elemB = vector::ExtractOp::create(builder&: rewriter, location: loc, source: in, position: i + 1);
401 }
402
403 thisResult =
404 ROCDL::CvtPkRtz::create(builder&: rewriter, location: loc, res: truncResType, srcA: elemA, srcB: elemB);
405 // Place back the truncated result into the possibly larger vector. If we
406 // are operating on a size 2 vector, these operations should be folded away
407 thisResult = vector::ExtractStridedSliceOp::create(
408 builder&: rewriter, location: loc, source: thisResult, offsets: 0, sizes: elemsThisOp, strides: 1);
409 result = vector::InsertStridedSliceOp::create(builder&: rewriter, location: loc, valueToStore: thisResult,
410 dest: result, offsets: i, strides: 1);
411 }
412
413 if (inVectorTy.getRank() != outVecType.getRank()) {
414 result = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: outVecType, source: result);
415 }
416
417 rewriter.replaceOp(op, newValues: result);
418 return success();
419}
420
421/// Get the broadcasted / splatted value for a chain of ops.
422static Value getOriginalVectorValue(Value value) {
423 Value current = value;
424 while (Operation *definingOp = current.getDefiningOp()) {
425 bool skipOp = llvm::TypeSwitch<Operation *, bool>(definingOp)
426 .Case(caseFn: [&current](vector::ShapeCastOp op) {
427 current = op.getSource();
428 return true;
429 })
430 .Case(caseFn: [&current](vector::BroadcastOp op) {
431 current = op.getSource();
432 return false;
433 })
434 .Default(defaultResult: false);
435
436 if (!skipOp) {
437 break;
438 }
439 }
440 return current;
441}
442
443LogicalResult
444ScalingExtFRewritePattern::matchAndRewrite(arith::ScalingExtFOp op,
445 PatternRewriter &rewriter) const {
446 Location loc = op.getLoc();
447 constexpr int64_t opOutWidth = 2;
448
449 Value in = op.getIn();
450 Value scale = op.getScale();
451 Value out = op.getOut();
452
453 Type f32 = rewriter.getF32Type();
454 Type inType = getElementTypeOrSelf(val: in);
455 Type scaleType = getElementTypeOrSelf(val: scale);
456 Type outType = getElementTypeOrSelf(val: out);
457
458 int64_t opInWidth = 32 / inType.getIntOrFloatBitWidth();
459
460 VectorType outVecType = dyn_cast<VectorType>(Val: out.getType());
461 VectorType scaleVecType = dyn_cast<VectorType>(Val: scale.getType());
462
463 if (outVecType && outVecType.isScalable())
464 return failure();
465
466 if (isa<RankedTensorType>(Val: out.getType()) ||
467 isa<RankedTensorType>(Val: in.getType()) ||
468 isa<RankedTensorType>(Val: scale.getType()))
469 return failure();
470
471 Type scaleF32Type =
472 scaleVecType ? VectorType::get(shape: scaleVecType.getShape(), elementType: f32) : f32;
473 if (scaleType.getIntOrFloatBitWidth() < 32)
474 scale =
475 arith::ExtFOp::create(builder&: rewriter, location: loc, resultTypes: TypeRange{scaleF32Type},
476 operands: ValueRange{scale}, properties: arith::ExtFOp::Properties{});
477 else if (scaleType.getIntOrFloatBitWidth() > 32)
478 scale = arith::TruncFOp::create(builder&: rewriter, location: loc, out: scaleF32Type, in: scale);
479
480 VectorType extScaleResultType = VectorType::get(shape: opOutWidth, elementType: outType);
481
482 if (!outVecType) {
483 Value inCast = vector::BroadcastOp::create(builder&: rewriter, location: loc,
484 result: VectorType::get(shape: 1, elementType: inType), source: in);
485 // TODO: replace this with non-packed ScaledExtOp
486 Value scaleExt = amdgpu::ScaledExtPackedOp::create(
487 builder&: rewriter, location: loc, res: extScaleResultType, source: inCast, scale, index: 0);
488 scaleExt = rewriter.replaceOpWithNewOp<vector::ExtractOp>(op, args&: scaleExt, args: 0);
489 return success();
490 }
491
492 VectorType inVecType = cast<VectorType>(Val: in.getType());
493 Value origScale = getOriginalVectorValue(value: op.getScale());
494 VectorType origScaleVecType = dyn_cast<VectorType>(Val: origScale.getType());
495
496 ArrayRef<int64_t> inShape = inVecType.getShape();
497 SmallVector<int64_t> originalScaleShape;
498 if (origScaleVecType)
499 llvm::append_range(C&: originalScaleShape, R: origScaleVecType.getShape());
500
501 originalScaleShape.insert(I: originalScaleShape.end(),
502 NumToInsert: inShape.size() - originalScaleShape.size(), Elt: 1);
503
504 auto maybeRatio = computeShapeRatio(shape: inShape, subShape: originalScaleShape);
505 assert(maybeRatio &&
506 "failed to derive block size from broadcast or splat operation");
507
508 SmallVector<int64_t> ratio =
509 maybeRatio.value_or(u: SmallVector<int64_t>(inShape.size(), 1));
510
511 int64_t blockSize = computeProduct(basis: ratio);
512
513 Value zero = arith::ConstantOp::create(builder&: rewriter, location: loc, result: outType,
514 value: rewriter.getFloatAttr(type: outType, value: 0.0));
515 Value result =
516 rewriter.createOrFold<vector::BroadcastOp>(location: loc, args&: outVecType, args&: zero);
517
518 for (SmallVector<int64_t> offsets : StaticTileOffsetRange(inShape, ratio)) {
519 SmallVector<int64_t> strides(offsets.size(), 1);
520 Value block = vector::ExtractStridedSliceOp::create(
521 builder&: rewriter, location: loc, source: in, offsets, sizes: ratio, strides);
522 VectorType block1DType = VectorType::get(shape: blockSize, elementType: inType);
523 Value block1D =
524 vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: block1DType, source: block);
525 Value uniformScale =
526 vector::ExtractOp::create(builder&: rewriter, location: loc, source: scale, position: offsets);
527
528 VectorType blockResultType = VectorType::get(shape: blockSize, elementType: outType);
529 Value blockResult =
530 rewriter.createOrFold<vector::BroadcastOp>(location: loc, args&: blockResultType, args&: zero);
531
532 for (int64_t i = 0, inSliceWidth = std::min(a: opInWidth, b: blockSize - i);
533 i < blockSize;
534 i += inSliceWidth, inSliceWidth = std::min(a: opInWidth, b: blockSize - i)) {
535 Value inSlice = vector::ExtractStridedSliceOp::create(
536 builder&: rewriter, location: loc, source: block1D, offsets: i, sizes: inSliceWidth, strides: 1);
537 for (int64_t j = 0,
538 outSliceWidth = std::min(a: opOutWidth, b: inSliceWidth - j);
539 j < inSliceWidth; j += outSliceWidth,
540 outSliceWidth = std::min(a: opOutWidth, b: inSliceWidth - j)) {
541 // TODO: replace this with non-packed ScaledExtOp for sliceWidth == 1
542 Value scaleExt = amdgpu::ScaledExtPackedOp::create(
543 builder&: rewriter, location: loc, res: extScaleResultType, source: inSlice, scale: uniformScale,
544 index: j / opOutWidth);
545 if (outSliceWidth < opOutWidth) {
546 scaleExt = vector::ExtractStridedSliceOp::create(
547 builder&: rewriter, location: loc, source: scaleExt, offsets: 0, sizes: outSliceWidth, strides: 1);
548 }
549 blockResult = vector::InsertStridedSliceOp::create(
550 builder&: rewriter, location: loc, valueToStore: scaleExt, dest: blockResult, offsets: i + j, strides: 1);
551 }
552 }
553
554 VectorType resultType = VectorType::get(shape: ratio, elementType: outType);
555 Value cast =
556 vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: resultType, source: blockResult);
557 result = vector::InsertStridedSliceOp::create(builder&: rewriter, location: loc, valueToStore: cast, dest: result,
558 offsets, strides);
559 }
560
561 rewriter.replaceOp(op, newValues: result);
562
563 return success();
564}
565
566LogicalResult
567ScalingTruncFRewritePattern::matchAndRewrite(arith::ScalingTruncFOp op,
568 PatternRewriter &rewriter) const {
569 Location loc = op.getLoc();
570 constexpr int64_t opInWidth = 2;
571
572 Value in = op.getIn();
573 Value scale = op.getScale();
574 Value out = op.getOut();
575
576 Type f32 = rewriter.getF32Type();
577 Type inType = getElementTypeOrSelf(val: in);
578 Type scaleType = getElementTypeOrSelf(val: scale);
579 Type outType = getElementTypeOrSelf(val: out);
580
581 VectorType outVecType = dyn_cast<VectorType>(Val: out.getType());
582 VectorType scaleVecType = dyn_cast<VectorType>(Val: scale.getType());
583 if (outVecType && outVecType.isScalable())
584 return failure();
585
586 if (isa<RankedTensorType>(Val: out.getType()) ||
587 isa<RankedTensorType>(Val: in.getType()) ||
588 isa<RankedTensorType>(Val: scale.getType()))
589 return failure();
590
591 Type scaleF32Type =
592 scaleVecType ? VectorType::get(shape: scaleVecType.getShape(), elementType: f32) : f32;
593 if (scaleType.getIntOrFloatBitWidth() < 32)
594 scale =
595 arith::ExtFOp::create(builder&: rewriter, location: loc, resultTypes: TypeRange{scaleF32Type},
596 operands: ValueRange{scale}, properties: arith::ExtFOp::Properties{});
597 else if (scaleType.getIntOrFloatBitWidth() > 32)
598 scale = arith::TruncFOp::create(builder&: rewriter, location: loc, out: scaleF32Type, in: scale);
599
600 Value zero = arith::ConstantOp::create(builder&: rewriter, location: loc, result: outType,
601 value: rewriter.getFloatAttr(type: outType, value: 0.0));
602 int64_t opOutWidth = 32 / outType.getIntOrFloatBitWidth();
603 VectorType truncScaleResultType = VectorType::get(shape: opOutWidth, elementType: outType);
604
605 if (!outVecType) {
606 Type inVecType = VectorType::get(shape: 1, elementType: inType);
607 Value inCast = vector::BroadcastOp::create(builder&: rewriter, location: loc, result: inVecType, source: in);
608 // TODO: replace this with non-packed ScaledTruncOp
609 Value scaleTrunc = amdgpu::PackedScaledTruncOp::create(
610 builder&: rewriter, location: loc, res: truncScaleResultType, source: inCast, scale, index: 0,
611 /*existing=*/nullptr);
612 scaleTrunc =
613 rewriter.replaceOpWithNewOp<vector::ExtractOp>(op, args&: scaleTrunc, args: 0);
614 return success();
615 }
616
617 VectorType inVecType = cast<VectorType>(Val: in.getType());
618 Value origScale = getOriginalVectorValue(value: op.getScale());
619 VectorType origScaleVecType = dyn_cast<VectorType>(Val: origScale.getType());
620
621 ArrayRef<int64_t> inShape = inVecType.getShape();
622 SmallVector<int64_t> scaleShape;
623 if (origScaleVecType)
624 llvm::append_range(C&: scaleShape, R: origScaleVecType.getShape());
625
626 scaleShape.insert(I: scaleShape.end(), NumToInsert: inShape.size() - scaleShape.size(), Elt: 1);
627
628 auto maybeRatio = computeShapeRatio(shape: inShape, subShape: scaleShape);
629 assert(maybeRatio &&
630 "failed to derive block size from broadcast or splat operation");
631
632 SmallVector<int64_t> ratio =
633 maybeRatio.value_or(u: SmallVector<int64_t>(inShape.size(), 1));
634
635 int64_t blockSize = computeProduct(basis: ratio);
636
637 Value result =
638 rewriter.createOrFold<vector::BroadcastOp>(location: loc, args&: outVecType, args&: zero);
639
640 for (SmallVector<int64_t> offsets : StaticTileOffsetRange(inShape, ratio)) {
641 SmallVector<int64_t> strides(offsets.size(), 1);
642 Value block = vector::ExtractStridedSliceOp::create(
643 builder&: rewriter, location: loc, source: in, offsets, sizes: ratio, strides);
644 VectorType block1DType = VectorType::get(shape: blockSize, elementType: inType);
645 Value block1D =
646 vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: block1DType, source: block);
647 Value uniformScale =
648 vector::ExtractOp::create(builder&: rewriter, location: loc, source: scale, position: offsets);
649
650 VectorType blockResultType = VectorType::get(shape: blockSize, elementType: outType);
651 Value blockResult =
652 rewriter.createOrFold<vector::BroadcastOp>(location: loc, args&: blockResultType, args&: zero);
653
654 for (int64_t i = 0, outSliceWidth = std::min(a: opOutWidth, b: blockSize - i);
655 i < blockSize; i += outSliceWidth,
656 outSliceWidth = std::min(a: opOutWidth, b: blockSize - i)) {
657 Value scaleTrunc;
658 // Case where <= 2 elements are being truncated.
659 if (outSliceWidth <= opInWidth) {
660 Value slice = vector::ExtractStridedSliceOp::create(
661 builder&: rewriter, location: loc, source: block1D, offsets: i, sizes: outSliceWidth, strides: 1);
662 // TODO: replace this with non-packed ScaledTruncOp for sliceWidth == 1
663 scaleTrunc = amdgpu::PackedScaledTruncOp::create(
664 builder&: rewriter, location: loc, res: truncScaleResultType, source: slice, scale: uniformScale, index: 0,
665 /*existing=*/nullptr);
666 } else {
667 scaleTrunc = vector::BroadcastOp::create(builder&: rewriter, location: loc,
668 result: truncScaleResultType, source: zero);
669 for (int64_t j = 0,
670 inSliceWidth = std::min(a: opInWidth, b: outSliceWidth - j);
671 j < outSliceWidth; j += opInWidth,
672 inSliceWidth = std::min(a: opInWidth, b: outSliceWidth - j)) {
673 Value slice = vector::ExtractStridedSliceOp::create(
674 builder&: rewriter, location: loc, source: block1D, offsets: i + j, sizes: inSliceWidth, strides: 1);
675 scaleTrunc = amdgpu::PackedScaledTruncOp::create(
676 builder&: rewriter, location: loc, res: truncScaleResultType, source: slice, scale: uniformScale,
677 index: j / opInWidth, existing: scaleTrunc);
678 }
679 }
680 if (outSliceWidth != opOutWidth) {
681 scaleTrunc = vector::ExtractStridedSliceOp::create(
682 builder&: rewriter, location: loc, source: scaleTrunc, offsets: 0, sizes: outSliceWidth, strides: 1);
683 }
684 blockResult = vector::InsertStridedSliceOp::create(
685 builder&: rewriter, location: loc, valueToStore: scaleTrunc, dest: blockResult, offsets: i, strides: 1);
686 }
687
688 VectorType resultType = VectorType::get(shape: ratio, elementType: outType);
689 Value cast =
690 vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: resultType, source: blockResult);
691 result = vector::InsertStridedSliceOp::create(builder&: rewriter, location: loc, valueToStore: cast, dest: result,
692 offsets, strides);
693 }
694
695 rewriter.replaceOp(op, newValues: result);
696
697 return success();
698}
699
700void mlir::arith::populateArithToAMDGPUConversionPatterns(
701 RewritePatternSet &patterns, bool convertFP8Arithmetic,
702 bool saturateFP8Truncf, bool allowPackedF16Rtz, bool supportsScaledExtTrunc,
703 const ROCDL::TargetInfo &target, PatternBenefit benefit) {
704
705 if (convertFP8Arithmetic) {
706 patterns.add<ExtFOnFloat8RewritePattern>(arg: patterns.getContext(), args: target,
707 args&: benefit);
708 patterns.add<TruncFToFloat8RewritePattern>(
709 arg: patterns.getContext(), args&: saturateFP8Truncf, args: target, args&: benefit);
710 }
711 if (allowPackedF16Rtz)
712 patterns.add<TruncfToFloat16RewritePattern>(arg: patterns.getContext(), args&: benefit);
713
714 if (supportsScaledExtTrunc) {
715 patterns.add<ScalingExtFRewritePattern>(arg: patterns.getContext(), args&: benefit);
716 patterns.add<ScalingTruncFRewritePattern>(arg: patterns.getContext(), args&: benefit);
717 }
718}
719
720void ArithToAMDGPUConversionPass::runOnOperation() {
721 Operation *op = getOperation();
722 MLIRContext *ctx = &getContext();
723 RewritePatternSet patterns(op->getContext());
724 FailureOr<ROCDL::TargetInfo> targetInfo = ROCDL::TargetInfo::get(
725 arch: ROCDL::resolveArchOption(arch, deprecatedAlias: chipset), /*waveSize=*/0,
726 emitError: [&] { return emitError(loc: UnknownLoc::get(context: ctx)); });
727 if (failed(Result: targetInfo))
728 return signalPassFailure();
729
730 bool convertFP8Arithmetic =
731 targetInfo->has(feature: llvm::AMDGPU::FEAT_FP8_CONVERSION_INSTS);
732 bool supportsScaledExtTrunc =
733 targetInfo->has(feature: llvm::AMDGPU::FEAT_GFX950_INSTS);
734 arith::populateArithToAMDGPUConversionPatterns(
735 patterns, convertFP8Arithmetic, saturateFP8Truncf, allowPackedF16Rtz,
736 supportsScaledExtTrunc, target: *targetInfo);
737 if (failed(Result: applyPatternsGreedily(op, patterns: std::move(patterns))))
738 return signalPassFailure();
739}
740