1//===- EmulateAtomics.cpp - Emulate unsupported AMDGPU atomics ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/AMDGPU/Transforms/Passes.h"
10
11#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
12#include "mlir/Dialect/Arith/IR/Arith.h"
13#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
14#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
15#include "mlir/Dialect/Vector/IR/VectorOps.h"
16#include "mlir/IR/BuiltinAttributes.h"
17#include "mlir/IR/TypeUtilities.h"
18#include "mlir/Transforms/DialectConversion.h"
19
20namespace mlir::amdgpu {
21#define GEN_PASS_DEF_AMDGPUEMULATEATOMICSPASS
22#include "mlir/Dialect/AMDGPU/Transforms/Passes.h.inc"
23} // namespace mlir::amdgpu
24
25using namespace mlir;
26using namespace mlir::amdgpu;
27
28namespace {
29struct AmdgpuEmulateAtomicsPass
30 : public amdgpu::impl::AmdgpuEmulateAtomicsPassBase<
31 AmdgpuEmulateAtomicsPass> {
32 using AmdgpuEmulateAtomicsPassBase<
33 AmdgpuEmulateAtomicsPass>::AmdgpuEmulateAtomicsPassBase;
34 void runOnOperation() override;
35};
36
37template <typename AtomicOp, typename ArithOp>
38struct RawBufferAtomicByCasPattern : public OpConversionPattern<AtomicOp> {
39 using OpConversionPattern<AtomicOp>::OpConversionPattern;
40 using Adaptor = typename AtomicOp::Adaptor;
41
42 LogicalResult
43 matchAndRewrite(AtomicOp atomicOp, Adaptor adaptor,
44 ConversionPatternRewriter &rewriter) const override;
45};
46} // namespace
47
48namespace {
49enum class DataArgAction : unsigned char {
50 Duplicate,
51 Drop,
52};
53} // namespace
54
55// Fix up the fact that, when we're migrating from a general bugffer atomic
56// to a load or to a CAS, the number of openrands, and thus the number of
57// entries needed in operandSegmentSizes, needs to change. We use this method
58// because we'd like to preserve unknown attributes on the atomic instead of
59// discarding them.
60static void patchOperandSegmentSizes(ArrayRef<NamedAttribute> attrs,
61 SmallVectorImpl<NamedAttribute> &newAttrs,
62 DataArgAction action) {
63 newAttrs.reserve(N: attrs.size());
64 for (NamedAttribute attr : attrs) {
65 if (attr.getName().getValue() != "operandSegmentSizes") {
66 newAttrs.push_back(Elt: attr);
67 continue;
68 }
69 auto segmentAttr = cast<DenseI32ArrayAttr>(Val: attr.getValue());
70 MLIRContext *context = segmentAttr.getContext();
71 DenseI32ArrayAttr newSegments;
72 switch (action) {
73 case DataArgAction::Drop:
74 newSegments = DenseI32ArrayAttr::get(
75 context, content: segmentAttr.asArrayRef().drop_front());
76 break;
77 case DataArgAction::Duplicate: {
78 SmallVector<int32_t> newVals;
79 ArrayRef<int32_t> oldVals = segmentAttr.asArrayRef();
80 newVals.push_back(Elt: oldVals[0]);
81 newVals.append(in_start: oldVals.begin(), in_end: oldVals.end());
82 newSegments = DenseI32ArrayAttr::get(context, content: newVals);
83 break;
84 }
85 }
86 newAttrs.push_back(Elt: NamedAttribute(attr.getName(), newSegments));
87 }
88}
89
90template <typename OpTy>
91static typename OpTy::Properties
92getPropertiesFromAttrs(OpBuilder &builder, Location loc,
93 ArrayRef<NamedAttribute> attrs) {
94 typename OpTy::Properties properties{};
95 OpTy::populateDefaultProperties(
96 OperationName(OpTy::getOperationName(), builder.getContext()),
97 properties);
98 LogicalResult result =
99 OpTy::setPropertiesFromAttr(properties, builder.getDictionaryAttr(value: attrs),
100 [&]() { return emitError(loc); });
101 assert(succeeded(result) && "failed to convert operation properties");
102 (void)result;
103 return properties;
104}
105
106// A helper function to flatten a vector value to a scalar containing its bits,
107// returning the value itself if othetwise.
108static Value flattenVecToBits(ConversionPatternRewriter &rewriter, Location loc,
109 Value val) {
110 auto vectorType = dyn_cast<VectorType>(Val: val.getType());
111 if (!vectorType)
112 return val;
113
114 int64_t bitwidth =
115 vectorType.getElementTypeBitWidth() * vectorType.getNumElements();
116 Type allBitsType = rewriter.getIntegerType(width: bitwidth);
117 auto allBitsVecType = VectorType::get(shape: {1}, elementType: allBitsType);
118 Value bitcast = vector::BitCastOp::create(builder&: rewriter, location: loc, result: allBitsVecType, source: val);
119 Value scalar = vector::ExtractOp::create(builder&: rewriter, location: loc, source: bitcast, position: 0);
120 return scalar;
121}
122
123template <typename AtomicOp, typename ArithOp>
124LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
125 AtomicOp atomicOp, Adaptor adaptor,
126 ConversionPatternRewriter &rewriter) const {
127 Location loc = atomicOp.getLoc();
128
129 NamedAttrList origProperties;
130 atomicOp->getName().walkInherentAttrs(atomicOp,
131 [&](StringRef name, Attribute &attr) {
132 origProperties.append(name, attr);
133 });
134 ArrayRef<NamedAttribute> discardableAttrs =
135 atomicOp->getDiscardableAttrDictionary().getValue();
136 ValueRange operands = adaptor.getOperands();
137 Value data = operands.take_front()[0];
138 ValueRange invariantArgs = operands.drop_front();
139 Type dataType = data.getType();
140
141 SmallVector<NamedAttribute> loadAttrs;
142 patchOperandSegmentSizes(attrs: origProperties, newAttrs&: loadAttrs, action: DataArgAction::Drop);
143 auto loadProperties =
144 getPropertiesFromAttrs<RawBufferLoadOp>(builder&: rewriter, loc, attrs: loadAttrs);
145 Value initialLoad =
146 RawBufferLoadOp::create(builder&: rewriter, location: loc, resultTypes: TypeRange{dataType}, operands: invariantArgs,
147 properties: loadProperties, discardableAttributes: discardableAttrs);
148 Block *currentBlock = rewriter.getInsertionBlock();
149 Block *afterAtomic =
150 rewriter.splitBlock(block: currentBlock, before: rewriter.getInsertionPoint());
151 afterAtomic->addArgument(type: dataType, loc);
152 Block *loopBlock = rewriter.createBlock(insertBefore: afterAtomic, argTypes: {dataType}, locs: {loc});
153
154 rewriter.setInsertionPointToEnd(currentBlock);
155 cf::BranchOp::create(builder&: rewriter, location: loc, dest: loopBlock, destOperands: initialLoad);
156
157 rewriter.setInsertionPointToEnd(loopBlock);
158 Value prevLoad = loopBlock->getArgument(i: 0);
159 Value operated = ArithOp::create(rewriter, loc, data, prevLoad);
160 dataType = operated.getType();
161
162 SmallVector<NamedAttribute> cmpswapAttrs;
163 patchOperandSegmentSizes(attrs: origProperties, newAttrs&: cmpswapAttrs,
164 action: DataArgAction::Duplicate);
165 SmallVector<Value> cmpswapArgs = {operated, prevLoad};
166 cmpswapArgs.append(in_start: invariantArgs.begin(), in_end: invariantArgs.end());
167 auto cmpswapProperties = getPropertiesFromAttrs<RawBufferAtomicCmpswapOp>(
168 builder&: rewriter, loc, attrs: cmpswapAttrs);
169 Value atomicRes = RawBufferAtomicCmpswapOp::create(
170 builder&: rewriter, location: loc, resultTypes: TypeRange{dataType}, operands: cmpswapArgs, properties: cmpswapProperties,
171 discardableAttributes: discardableAttrs);
172
173 // We care about exact bitwise equality here, so do some bitcasts.
174 // These will fold away during lowering to the ROCDL dialect, where
175 // an int->float bitcast is introduced to account for the fact that cmpswap
176 // only takes integer arguments.
177
178 Value prevLoadForCompare = flattenVecToBits(rewriter, loc, val: prevLoad);
179 Value atomicResForCompare = flattenVecToBits(rewriter, loc, val: atomicRes);
180 if (auto floatDataTy = dyn_cast<FloatType>(Val&: dataType)) {
181 Type equivInt = rewriter.getIntegerType(width: floatDataTy.getWidth());
182 prevLoadForCompare =
183 arith::BitcastOp::create(builder&: rewriter, location: loc, out: equivInt, in: prevLoad);
184 atomicResForCompare =
185 arith::BitcastOp::create(builder&: rewriter, location: loc, out: equivInt, in: atomicRes);
186 }
187 Value canLeave =
188 arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq,
189 lhs: atomicResForCompare, rhs: prevLoadForCompare);
190 cf::CondBranchOp::create(builder&: rewriter, location: loc, condition: canLeave, trueDest: afterAtomic,
191 trueOperands: ValueRange{prevLoad}, falseDest: loopBlock, falseOperands: atomicRes);
192 rewriter.replaceOp(atomicOp, ValueRange{afterAtomic->getArgument(i: 0)});
193 return success();
194}
195
196/// Returns whether \p op can be lowered to a native buffer atomic fadd on
197/// \p target.
198static bool isFaddNativelySupported(const ROCDL::TargetInfo &target,
199 RawBufferAtomicFaddOp op) {
200 namespace AMDGPU = ::llvm::AMDGPU;
201
202 // A target with only the no-return form can still perform the atomic, it
203 // just cannot report the old value, so it suffices when the result is dead.
204 bool hasFadd = target.has(feature: AMDGPU::FEAT_ATOMIC_FADD_RTN_INSTS) ||
205 (target.has(feature: AMDGPU::FEAT_ATOMIC_FADD_NO_RTN_INSTS) &&
206 op.getOldValue().use_empty());
207 if (!hasFadd)
208 return false;
209
210 // The packed 16-bit forms are separate instructions with their own features.
211 Type elemType = getElementTypeOrSelf(type: op.getValue().getType());
212 if (isa<Float16Type>(Val: elemType))
213 return target.has(feature: AMDGPU::FEAT_ATOMIC_BUFFER_GLOBAL_PK_ADD_F16_INSTS);
214 if (isa<BFloat16Type>(Val: elemType))
215 return target.has(feature: AMDGPU::FEAT_ATOMIC_BUFFER_PK_ADD_BF16_INST);
216 return true;
217}
218
219void mlir::amdgpu::populateAmdgpuEmulateAtomicsPatterns(
220 ConversionTarget &target, RewritePatternSet &patterns,
221 const ROCDL::TargetInfo &targetInfo, PatternBenefit benefit) {
222 namespace AMDGPU = ::llvm::AMDGPU;
223
224 target.addDynamicallyLegalOp<RawBufferAtomicFaddOp>(
225 callback: [targetInfo](RawBufferAtomicFaddOp op) -> bool {
226 return isFaddNativelySupported(target: targetInfo, op);
227 });
228
229 // Floating-point min and max are only emulated on gfx9; later generations
230 // have them for every type this op accepts. f64 is the one gfx9 case that
231 // may be native.
232 if (targetInfo.isGeneration(major: 9)) {
233 target.addDynamicallyLegalOp<RawBufferAtomicFmaxOp>(
234 callback: [targetInfo](RawBufferAtomicFmaxOp op) -> bool {
235 return op.getValue().getType().isF64() &&
236 targetInfo.has(feature: AMDGPU::FEAT_ATOMIC_FMIN_FMAX_GLOBAL_F64);
237 });
238 }
239 patterns.add<
240 RawBufferAtomicByCasPattern<RawBufferAtomicFaddOp, arith::AddFOp>,
241 RawBufferAtomicByCasPattern<RawBufferAtomicFmaxOp, arith::MaximumFOp>,
242 RawBufferAtomicByCasPattern<RawBufferAtomicSmaxOp, arith::MaxSIOp>,
243 RawBufferAtomicByCasPattern<RawBufferAtomicUminOp, arith::MinUIOp>>(
244 arg: patterns.getContext(), args&: benefit);
245}
246
247void AmdgpuEmulateAtomicsPass::runOnOperation() {
248 Operation *op = getOperation();
249 FailureOr<ROCDL::TargetInfo> targetInfo =
250 ROCDL::TargetInfo::get(arch: ROCDL::resolveArchOption(arch, deprecatedAlias: chipset),
251 /*waveSize=*/0, emitError: [&] { return op->emitError(); });
252 if (failed(Result: targetInfo))
253 return signalPassFailure();
254
255 MLIRContext &ctx = getContext();
256 ConversionTarget target(ctx);
257 RewritePatternSet patterns(&ctx);
258 target.markUnknownOpDynamicallyLegal(
259 fn: [](Operation *op) -> bool { return true; });
260
261 populateAmdgpuEmulateAtomicsPatterns(target, patterns, targetInfo: *targetInfo);
262 if (failed(Result: applyPartialConversion(op, target, patterns: std::move(patterns))))
263 return signalPassFailure();
264}
265